1; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=CHECK 2; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=4 -S < %s | FileCheck %s --check-prefix=CHECK 3; RUN: opt -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=4 -S < %s | FileCheck %s --check-prefix=CHECK 4 5define i64 @smax_idx(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) { 6; CHECK-LABEL: @smax_idx( 7; CHECK-NOT: vector.body: 8; 9entry: 10 br label %for.body 11 12for.body: 13 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 14 %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ] 15 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ] 16 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv 17 %0 = load i64, ptr %arrayidx 18 %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0) 19 %cmp1 = icmp slt i64 %max.09, %0 20 %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011 21 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 22 %exitcond.not = icmp eq i64 %indvars.iv.next, %n 23 br i1 %exitcond.not, label %exit, label %for.body 24 25exit: 26 store i64 %1, ptr %res_max 27 ret i64 %spec.select7 28} 29 30; 31; Check the different order of reduction phis. 32; 33define i64 @smax_idx_inverted_phi(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) { 34; CHECK-LABEL: @smax_idx_inverted_phi( 35; CHECK-NOT: vector.body: 36; 37entry: 38 br label %for.body 39 40for.body: 41 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 42 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ] 43 %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ] 44 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv 45 %0 = load i64, ptr %arrayidx 46 %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0) 47 %cmp1 = icmp slt i64 %max.09, %0 48 %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011 49 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 50 %exitcond.not = icmp eq i64 %indvars.iv.next, %n 51 br i1 %exitcond.not, label %exit, label %for.body 52 53exit: 54 store i64 %1, ptr %res_max 55 ret i64 %spec.select7 56} 57 58; Check if it is a min/max with index (MMI) pattern when the 59; min/max value is not used outside the loop. 60; 61; Currently, the vectorizer checks if smax value is used outside 62; the loop. However, even if only the index part has external users, 63; and smax itself does not have external users, it can still form a 64; MMI pattern. 65; 66define i64 @smax_idx_max_no_exit_user(ptr nocapture readonly %a, i64 %mm, i64 %ii, i64 %n) { 67; CHECK-LABEL: @smax_idx_max_no_exit_user( 68; CHECK-NOT: vector.body: 69; 70entry: 71 br label %for.body 72 73for.body: 74 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 75 %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ] 76 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ] 77 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv 78 %0 = load i64, ptr %arrayidx 79 %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0) 80 %cmp1 = icmp slt i64 %max.09, %0 81 %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011 82 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 83 %exitcond.not = icmp eq i64 %indvars.iv.next, %n 84 br i1 %exitcond.not, label %exit, label %for.body 85 86exit: 87 ; %1 has no external users 88 ret i64 %spec.select7 89} 90 91; Check smax implemented by select(cmp()). 92; 93; Currently, MMI pattern does not support icmp with multiple users. 94; TODO: It may be possible to reuse some of the methods in instcombine pass to 95; check whether icmp can be duplicated. 96; 97define i64 @smax_idx_select_cmp(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) { 98; CHECK-LABEL: @smax_idx_select_cmp( 99; CHECK-NOT: vector.body: 100; 101entry: 102 br label %for.body 103 104for.body: 105 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 106 %max.09 = phi i64 [ %mm, %entry ], [ %spec.select, %for.body ] 107 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ] 108 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv 109 %0 = load i64, ptr %arrayidx 110 %cmp1 = icmp slt i64 %max.09, %0 111 %spec.select = select i1 %cmp1, i64 %0, i64 %max.09 112 %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011 113 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 114 %exitcond.not = icmp eq i64 %indvars.iv.next, %n 115 br i1 %exitcond.not, label %exit, label %for.body 116 117exit: 118 store i64 %spec.select, ptr %res_max 119 ret i64 %spec.select7 120} 121 122; 123; Check sge case. 124; 125define i64 @smax_idx_inverted_pred(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) { 126; CHECK-LABEL: @smax_idx_inverted_pred( 127; CHECK-NOT: vector.body: 128; 129entry: 130 br label %for.body 131 132for.body: 133 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 134 %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ] 135 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ] 136 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv 137 %0 = load i64, ptr %arrayidx 138 %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0) 139 %cmp1 = icmp sge i64 %0, %max.09 140 %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011 141 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 142 %exitcond.not = icmp eq i64 %indvars.iv.next, %n 143 br i1 %exitcond.not, label %exit, label %for.body 144 145exit: 146 store i64 %1, ptr %res_max 147 ret i64 %spec.select7 148} 149 150; 151; In such cases, the last index should be extracted. 152; 153define i64 @smax_idx_extract_last(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) { 154; CHECK-LABEL: @smax_idx_extract_last( 155; CHECK-NOT: vector.body: 156; 157entry: 158 br label %for.body 159 160for.body: 161 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 162 %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ] 163 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ] 164 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv 165 %0 = load i64, ptr %arrayidx 166 %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0) 167 %cmp1.not = icmp sgt i64 %max.09, %0 168 %spec.select7 = select i1 %cmp1.not, i64 %idx.011, i64 %indvars.iv 169 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 170 %exitcond.not = icmp eq i64 %indvars.iv.next, %n 171 br i1 %exitcond.not, label %exit, label %for.body 172 173exit: 174 store i64 %1, ptr %res_max 175 ret i64 %spec.select7 176} 177 178; 179; The operands of smax intrinsic and icmp are not the same to be recognized as MMI. 180; 181define i64 @smax_idx_not_vec_1(ptr nocapture readonly %a, ptr nocapture readonly %b, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) { 182; CHECK-LABEL: @smax_idx_not_vec_1( 183; CHECK-NOT: vector.body: 184; 185 entry: 186 br label %for.body 187 188for.body: 189 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 190 %max.09 = phi i64 [ %mm, %entry ], [ %2, %for.body ] 191 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ] 192 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv 193 %0 = load i64, ptr %arrayidx 194 %arrayidx.01 = getelementptr inbounds i64, ptr %b, i64 %indvars.iv 195 %1 = load i64, ptr %arrayidx 196 %2 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0) 197 %cmp1 = icmp slt i64 %max.09, %1 198 %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011 199 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 200 %exitcond.not = icmp eq i64 %indvars.iv.next, %n 201 br i1 %exitcond.not, label %exit, label %for.body 202 203exit: 204 store i64 %2, ptr %res_max 205 ret i64 %spec.select7 206} 207 208; 209; It cannot be recognized as MMI when the operand of index select is not an induction variable. 210; 211define i64 @smax_idx_not_vec_2(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) { 212; CHECK-LABEL: @smax_idx_not_vec_2( 213; CHECK-NOT: vector.body: 214; 215entry: 216 br label %for.body 217 218for.body: 219 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 220 %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ] 221 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ] 222 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv 223 %0 = load i64, ptr %arrayidx 224 %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0) 225 %cmp1 = icmp slt i64 %max.09, %0 226 %spec.select7 = select i1 %cmp1, i64 123, i64 %idx.011 227 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 228 %exitcond.not = icmp eq i64 %indvars.iv.next, %n 229 br i1 %exitcond.not, label %exit, label %for.body 230 231exit: 232 store i64 %1, ptr %res_max 233 ret i64 %spec.select7 234} 235 236declare i64 @llvm.smax.i64(i64, i64) 237