1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 2; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -S | FileCheck %s 3 4; int int_inc; 5; 6;void induction_with_global(int init, int *restrict A, int N) { 7; int x = init; 8; for (int i=0;i<N;i++){ 9; A[i] = x; 10; x += int_inc; 11; } 12;} 13 14 15target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" 16 17 18@int_inc = common global i32 0, align 4 19 20define void @induction_with_global(i32 %init, ptr noalias nocapture %A, i32 %N) { 21; CHECK-LABEL: define void @induction_with_global( 22; CHECK-SAME: i32 [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) { 23; CHECK-NEXT: [[ENTRY:.*]]: 24; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr @int_inc, align 4 25; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[N]], -1 26; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP1]] to i64 27; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP5]], 1 28; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 8 29; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] 30; CHECK: [[VECTOR_PH]]: 31; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 8 32; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 33; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32 34; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[DOTCAST]], [[TMP0]] 35; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INIT]], [[TMP3]] 36; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[INIT]], i64 0 37; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer 38; CHECK-NEXT: [[DOTSPLATINSERT2:%.*]] = insertelement <8 x i32> poison, i32 [[TMP0]], i64 0 39; CHECK-NEXT: [[DOTSPLAT3:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT2]], <8 x i32> poison, <8 x i32> zeroinitializer 40; CHECK-NEXT: [[TMP6:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[DOTSPLAT3]] 41; CHECK-NEXT: [[INDUCTION4:%.*]] = add <8 x i32> [[DOTSPLAT]], [[TMP6]] 42; CHECK-NEXT: [[TMP7:%.*]] = mul i32 [[TMP0]], 8 43; CHECK-NEXT: [[DOTSPLATINSERT5:%.*]] = insertelement <8 x i32> poison, i32 [[TMP7]], i64 0 44; CHECK-NEXT: [[DOTSPLAT6:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT5]], <8 x i32> poison, <8 x i32> zeroinitializer 45; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] 46; CHECK: [[VECTOR_BODY]]: 47; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] 48; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ [[INDUCTION4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] 49; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 0 50; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]] 51; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i32 0 52; CHECK-NEXT: store <8 x i32> [[VEC_IND]], ptr [[TMP10]], align 4 53; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 54; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], [[DOTSPLAT6]] 55; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 56; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 57; CHECK: [[MIDDLE_BLOCK]]: 58; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 59; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] 60; CHECK: [[SCALAR_PH]]: 61; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] 62; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ] 63; CHECK-NEXT: br label %[[FOR_BODY:.*]] 64; CHECK: [[FOR_BODY]]: 65; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ] 66; CHECK-NEXT: [[X_05:%.*]] = phi i32 [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ] 67; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]] 68; CHECK-NEXT: store i32 [[X_05]], ptr [[ARRAYIDX]], align 4 69; CHECK-NEXT: [[ADD]] = add nsw i32 [[TMP0]], [[X_05]] 70; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 71; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 72; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 73; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] 74; CHECK: [[EXIT]]: 75; CHECK-NEXT: ret void 76; 77entry: 78 %0 = load i32, ptr @int_inc, align 4 79 br label %for.body 80 81for.body: ; preds = %for.body, %entry 82 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 83 %x.05 = phi i32 [ %init, %entry ], [ %add, %for.body ] 84 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %iv 85 store i32 %x.05, ptr %arrayidx, align 4 86 %add = add nsw i32 %0, %x.05 87 %iv.next = add nuw nsw i64 %iv, 1 88 %iv.next.trunc = trunc i64 %iv.next to i32 89 %exitcond = icmp eq i32 %iv.next.trunc, %N 90 br i1 %exitcond, label %exit, label %for.body 91 92exit: 93 ret void 94} 95 96;int induction_with_loop_inv(int init, int *restrict A, int N, int M) { 97; int x = init; 98; for (int j = 0; j < M; j++) { 99; for (int i=0; i<N; i++){ 100; A[i] = x; 101; x += j; // induction step is a loop invariant variable 102; } 103; } 104; return x; 105;} 106 107define i32 @induction_with_loop_inv(i32 %init, ptr noalias nocapture %A, i32 %N, i32 %M) { 108; CHECK-LABEL: define i32 @induction_with_loop_inv( 109; CHECK-SAME: i32 [[INIT:%.*]], ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]], i32 [[M:%.*]]) { 110; CHECK-NEXT: [[ENTRY:.*]]: 111; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[N]], -1 112; CHECK-NEXT: [[TMP11:%.*]] = zext i32 [[TMP3]] to i64 113; CHECK-NEXT: [[TMP0:%.*]] = add nuw nsw i64 [[TMP11]], 1 114; CHECK-NEXT: br label %[[OUTER_HEADER:.*]] 115; CHECK: [[OUTER_HEADER]]: 116; CHECK-NEXT: [[INDVARS_IV15:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT16:%.*]], %[[OUTER_LATCH:.*]] ] 117; CHECK-NEXT: [[J_012:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC5:%.*]], %[[OUTER_LATCH]] ] 118; CHECK-NEXT: [[X_011:%.*]] = phi i32 [ [[INIT]], %[[ENTRY]] ], [ [[X_0_LCSSA:%.*]], %[[OUTER_LATCH]] ] 119; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8 120; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] 121; CHECK: [[VECTOR_PH]]: 122; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8 123; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]] 124; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32 125; CHECK-NEXT: [[TMP1:%.*]] = mul i32 [[DOTCAST]], [[J_012]] 126; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[X_011]], [[TMP1]] 127; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[X_011]], i64 0 128; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer 129; CHECK-NEXT: [[DOTSPLATINSERT2:%.*]] = insertelement <8 x i32> poison, i32 [[J_012]], i64 0 130; CHECK-NEXT: [[DOTSPLAT3:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT2]], <8 x i32> poison, <8 x i32> zeroinitializer 131; CHECK-NEXT: [[TMP4:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[DOTSPLAT3]] 132; CHECK-NEXT: [[INDUCTION4:%.*]] = add <8 x i32> [[DOTSPLAT]], [[TMP4]] 133; CHECK-NEXT: [[TMP5:%.*]] = mul i32 [[J_012]], 8 134; CHECK-NEXT: [[DOTSPLATINSERT5:%.*]] = insertelement <8 x i32> poison, i32 [[TMP5]], i64 0 135; CHECK-NEXT: [[DOTSPLAT6:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT5]], <8 x i32> poison, <8 x i32> zeroinitializer 136; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] 137; CHECK: [[VECTOR_BODY]]: 138; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] 139; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ [[INDUCTION4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] 140; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 141; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]] 142; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0 143; CHECK-NEXT: store <8 x i32> [[VEC_IND]], ptr [[TMP8]], align 4 144; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 145; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], [[DOTSPLAT6]] 146; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 147; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] 148; CHECK: [[MIDDLE_BLOCK]]: 149; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] 150; CHECK-NEXT: br i1 [[CMP_N]], label %[[INNER_EXIT:.*]], label %[[SCALAR_PH]] 151; CHECK: [[SCALAR_PH]]: 152; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_HEADER]] ] 153; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[X_011]], %[[OUTER_HEADER]] ] 154; CHECK-NEXT: br label %[[INNER:.*]] 155; CHECK: [[INNER]]: 156; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[INNER]] ] 157; CHECK-NEXT: [[X_18:%.*]] = phi i32 [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[INNER]] ] 158; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]] 159; CHECK-NEXT: store i32 [[X_18]], ptr [[ARRAYIDX]], align 4 160; CHECK-NEXT: [[ADD]] = add nsw i32 [[X_18]], [[J_012]] 161; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 162; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 163; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] 164; CHECK-NEXT: br i1 [[EXITCOND]], label %[[INNER_EXIT]], label %[[INNER]], !llvm.loop [[LOOP5:![0-9]+]] 165; CHECK: [[INNER_EXIT]]: 166; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[X_011]], [[INDVARS_IV15]] 167; CHECK-NEXT: br label %[[OUTER_LATCH]] 168; CHECK: [[OUTER_LATCH]]: 169; CHECK-NEXT: [[X_0_LCSSA]] = phi i32 [ [[TMP9]], %[[INNER_EXIT]] ] 170; CHECK-NEXT: [[INC5]] = add nuw nsw i32 [[J_012]], 1 171; CHECK-NEXT: [[INDVARS_IV_NEXT16]] = add i32 [[INDVARS_IV15]], [[N]] 172; CHECK-NEXT: [[EXITCOND17:%.*]] = icmp eq i32 [[INC5]], [[M]] 173; CHECK-NEXT: br i1 [[EXITCOND17]], label %[[EXIT:.*]], label %[[OUTER_HEADER]] 174; CHECK: [[EXIT]]: 175; CHECK-NEXT: ret i32 [[X_0_LCSSA]] 176; 177entry: 178 br label %outer.header 179 180outer.header: 181 %outer.iv = phi i32 [ 0, %entry ], [ %outer.iv.next, %outer.latch ] 182 %j.012 = phi i32 [ 0, %entry ], [ %inc5, %outer.latch ] 183 %x.011 = phi i32 [ %init, %entry ], [ %x.1.lcssa, %outer.latch ] 184 br label %inner 185 186inner: 187 %iv = phi i64 [ 0, %outer.header ], [ %iv.next, %inner ] 188 %x.18 = phi i32 [ %x.011, %outer.header ], [ %add, %inner ] 189 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %iv 190 store i32 %x.18, ptr %arrayidx, align 4 191 %add = add nsw i32 %x.18, %j.012 192 %iv.next = add nuw nsw i64 %iv, 1 193 %iv.next.trunc = trunc i64 %iv.next to i32 194 %inner.ec = icmp eq i32 %iv.next.trunc, %N 195 br i1 %inner.ec, label %inner.exit, label %inner 196 197inner.exit: 198 %add.ivs = add i32 %x.011, %outer.iv 199 br label %outer.latch 200 201outer.latch: 202 %x.1.lcssa = phi i32 [ %add.ivs, %inner.exit ] 203 %inc5 = add nuw nsw i32 %j.012, 1 204 %outer.iv.next = add i32 %outer.iv, %N 205 %outer.ec = icmp eq i32 %inc5, %M 206 br i1 %outer.ec, label %exit, label %outer.header 207 208exit: 209 ret i32 %x.1.lcssa 210} 211 212define void @non_primary_iv_loop_inv_trunc(ptr %a, i64 %n, i64 %step) { 213; CHECK-LABEL: define void @non_primary_iv_loop_inv_trunc( 214; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i64 [[STEP:%.*]]) { 215; CHECK-NEXT: [[ENTRY:.*]]: 216; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) 217; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 8 218; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] 219; CHECK: [[VECTOR_PH]]: 220; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 8 221; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]] 222; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[N_VEC]], [[STEP]] 223; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[STEP]] to i32 224; CHECK-NEXT: [[DOTSPLATINSERT5:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0 225; CHECK-NEXT: [[DOTSPLAT6:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT5]], <8 x i32> poison, <8 x i32> zeroinitializer 226; CHECK-NEXT: [[TMP4:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[DOTSPLAT6]] 227; CHECK-NEXT: [[INDUCTION7:%.*]] = add <8 x i32> zeroinitializer, [[TMP4]] 228; CHECK-NEXT: [[TMP5:%.*]] = mul i32 [[TMP3]], 8 229; CHECK-NEXT: [[DOTSPLATINSERT8:%.*]] = insertelement <8 x i32> poison, i32 [[TMP5]], i64 0 230; CHECK-NEXT: [[DOTSPLAT9:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT8]], <8 x i32> poison, <8 x i32> zeroinitializer 231; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] 232; CHECK: [[VECTOR_BODY]]: 233; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] 234; CHECK-NEXT: [[VEC_IND10:%.*]] = phi <8 x i32> [ [[INDUCTION7]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT11:%.*]], %[[VECTOR_BODY]] ] 235; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 236; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]] 237; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0 238; CHECK-NEXT: store <8 x i32> [[VEC_IND10]], ptr [[TMP8]], align 4 239; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 240; CHECK-NEXT: [[VEC_IND_NEXT11]] = add <8 x i32> [[VEC_IND10]], [[DOTSPLAT9]] 241; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 242; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] 243; CHECK: [[MIDDLE_BLOCK]]: 244; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 245; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]] 246; CHECK: [[SCALAR_PH]]: 247; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] 248; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] 249; CHECK-NEXT: br label %[[FOR_BODY:.*]] 250; CHECK: [[FOR_BODY]]: 251; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ] 252; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[J_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ] 253; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]] 254; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[J]] to i32 255; CHECK-NEXT: store i32 [[TMP3]], ptr [[TMP0]], align 4 256; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 257; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], [[STEP]] 258; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 259; CHECK-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP7:![0-9]+]] 260; CHECK: [[FOR_END]]: 261; CHECK-NEXT: ret void 262; 263entry: 264 br label %for.body 265 266for.body: 267 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 268 %j = phi i64 [ %j.next, %for.body ], [ 0, %entry ] 269 %tmp0 = getelementptr inbounds i32, ptr %a, i64 %i 270 %tmp1 = trunc i64 %j to i32 271 store i32 %tmp1, ptr %tmp0, align 4 272 %i.next = add nuw nsw i64 %i, 1 273 %j.next = add nuw nsw i64 %j, %step 274 %cond = icmp slt i64 %i.next, %n 275 br i1 %cond, label %for.body, label %for.end 276 277for.end: 278 ret void 279} 280 281 282define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) { 283; CHECK-LABEL: define void @iv_no_binary_op_in_descriptor( 284; CHECK-SAME: i1 [[C:%.*]], ptr [[DST:%.*]]) { 285; CHECK-NEXT: [[ENTRY:.*]]: 286; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] 287; CHECK: [[VECTOR_PH]]: 288; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] 289; CHECK: [[VECTOR_BODY]]: 290; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] 291; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] 292; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 293; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]] 294; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 0 295; CHECK-NEXT: store <8 x i64> [[VEC_IND]], ptr [[TMP2]], align 8 296; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 297; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8) 298; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 299; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 300; CHECK: [[MIDDLE_BLOCK]]: 301; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] 302; CHECK: [[SCALAR_PH]]: 303; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] 304; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] 305; CHECK: [[LOOP_HEADER]]: 306; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT_P:%.*]], %[[LOOP_LATCH:.*]] ] 307; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[IV]] 308; CHECK-NEXT: store i64 [[IV]], ptr [[GEP]], align 8 309; CHECK-NEXT: [[IV_NEXT:%.*]] = add i64 [[IV]], 1 310; CHECK-NEXT: br label %[[LOOP_LATCH]] 311; CHECK: [[LOOP_LATCH]]: 312; CHECK-NEXT: [[IV_NEXT_P]] = phi i64 [ [[IV_NEXT]], %[[LOOP_HEADER]] ] 313; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT_P]], 1000 314; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]] 315; CHECK: [[EXIT]]: 316; CHECK-NEXT: ret void 317; 318entry: 319 br label %loop.header 320 321loop.header: 322 %iv = phi i64 [ 0, %entry ], [ %iv.next.p, %loop.latch ] 323 %gep = getelementptr inbounds i64, ptr %dst, i64 %iv 324 store i64 %iv, ptr %gep, align 8 325 %iv.next = add i64 %iv, 1 326 br label %loop.latch 327 328loop.latch: 329 %iv.next.p = phi i64 [ %iv.next, %loop.header ] 330 %exitcond.not = icmp eq i64 %iv.next.p, 1000 331 br i1 %exitcond.not, label %exit, label %loop.header 332 333exit: 334 ret void 335} 336 337define void @wide_add_induction_step_live_in(ptr %dst, i64 %N, i16 %off) { 338; CHECK-LABEL: define void @wide_add_induction_step_live_in( 339; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]], i16 [[OFF:%.*]]) { 340; CHECK-NEXT: [[ENTRY:.*]]: 341; CHECK-NEXT: [[O_1:%.*]] = add i16 [[OFF]], 2 342; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 343; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] 344; CHECK: [[VECTOR_PH]]: 345; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8 346; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 347; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i16 348; CHECK-NEXT: [[TMP0:%.*]] = mul i16 [[DOTCAST]], [[O_1]] 349; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <8 x i16> poison, i16 [[O_1]], i64 0 350; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <8 x i16> [[DOTSPLATINSERT]], <8 x i16> poison, <8 x i32> zeroinitializer 351; CHECK-NEXT: [[TMP1:%.*]] = mul <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, [[DOTSPLAT]] 352; CHECK-NEXT: [[INDUCTION:%.*]] = add <8 x i16> zeroinitializer, [[TMP1]] 353; CHECK-NEXT: [[TMP2:%.*]] = mul i16 [[O_1]], 8 354; CHECK-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement <8 x i16> poison, i16 [[TMP2]], i64 0 355; CHECK-NEXT: [[DOTSPLAT2:%.*]] = shufflevector <8 x i16> [[DOTSPLATINSERT1]], <8 x i16> poison, <8 x i32> zeroinitializer 356; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i16> poison, i16 [[O_1]], i64 0 357; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT]], <8 x i16> poison, <8 x i32> zeroinitializer 358; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] 359; CHECK: [[VECTOR_BODY]]: 360; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] 361; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i16> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] 362; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 363; CHECK-NEXT: [[TMP4:%.*]] = add <8 x i16> [[VEC_IND]], [[BROADCAST_SPLAT]] 364; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[TMP3]] 365; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[TMP5]], i32 0 366; CHECK-NEXT: store <8 x i16> [[TMP4]], ptr [[TMP6]], align 2 367; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 368; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i16> [[VEC_IND]], [[DOTSPLAT2]] 369; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 370; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] 371; CHECK: [[MIDDLE_BLOCK]]: 372; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 373; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] 374; CHECK: [[SCALAR_PH]]: 375; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] 376; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i16 [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] 377; CHECK-NEXT: br label %[[LOOP:.*]] 378; CHECK: [[LOOP]]: 379; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] 380; CHECK-NEXT: [[IV_2:%.*]] = phi i16 [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ] 381; CHECK-NEXT: [[ADD]] = add i16 [[IV_2]], [[O_1]] 382; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[IV]] 383; CHECK-NEXT: store i16 [[ADD]], ptr [[GEP_DST]], align 2 384; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 385; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] 386; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]] 387; CHECK: [[EXIT]]: 388; CHECK-NEXT: ret void 389; 390entry: 391 %o.1 = add i16 %off, 2 392 br label %loop 393 394loop: 395 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] 396 %iv.2 = phi i16 [ 0, %entry ], [ %add, %loop ] 397 %add = add i16 %iv.2, %o.1 398 %gep.dst = getelementptr inbounds i16, ptr %dst, i64 %iv 399 store i16 %add, ptr %gep.dst, align 2 400 %iv.next = add nuw nsw i64 %iv, 1 401 %ec = icmp eq i64 %iv.next, %N 402 br i1 %ec , label %exit, label %loop 403 404exit: 405 ret void 406} 407 408define void @wide_sub_induction_step_live_in(ptr %dst, i64 %N, i16 %off) { 409; CHECK-LABEL: define void @wide_sub_induction_step_live_in( 410; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]], i16 [[OFF:%.*]]) { 411; CHECK-NEXT: [[ENTRY:.*]]: 412; CHECK-NEXT: [[O_1:%.*]] = add i16 [[OFF]], 2 413; CHECK-NEXT: [[TMP0:%.*]] = sub i16 -2, [[OFF]] 414; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 415; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] 416; CHECK: [[VECTOR_PH]]: 417; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8 418; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 419; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i16 420; CHECK-NEXT: [[TMP1:%.*]] = mul i16 [[DOTCAST]], [[TMP0]] 421; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <8 x i16> poison, i16 [[TMP0]], i64 0 422; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <8 x i16> [[DOTSPLATINSERT]], <8 x i16> poison, <8 x i32> zeroinitializer 423; CHECK-NEXT: [[TMP2:%.*]] = mul <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, [[DOTSPLAT]] 424; CHECK-NEXT: [[INDUCTION:%.*]] = add <8 x i16> zeroinitializer, [[TMP2]] 425; CHECK-NEXT: [[TMP3:%.*]] = mul i16 [[TMP0]], 8 426; CHECK-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement <8 x i16> poison, i16 [[TMP3]], i64 0 427; CHECK-NEXT: [[DOTSPLAT2:%.*]] = shufflevector <8 x i16> [[DOTSPLATINSERT1]], <8 x i16> poison, <8 x i32> zeroinitializer 428; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i16> poison, i16 [[O_1]], i64 0 429; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT]], <8 x i16> poison, <8 x i32> zeroinitializer 430; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] 431; CHECK: [[VECTOR_BODY]]: 432; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] 433; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i16> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] 434; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 435; CHECK-NEXT: [[TMP5:%.*]] = sub <8 x i16> [[VEC_IND]], [[BROADCAST_SPLAT]] 436; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[TMP4]] 437; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[TMP6]], i32 0 438; CHECK-NEXT: store <8 x i16> [[TMP5]], ptr [[TMP7]], align 2 439; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 440; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i16> [[VEC_IND]], [[DOTSPLAT2]] 441; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 442; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] 443; CHECK: [[MIDDLE_BLOCK]]: 444; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 445; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] 446; CHECK: [[SCALAR_PH]]: 447; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] 448; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i16 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] 449; CHECK-NEXT: br label %[[LOOP:.*]] 450; CHECK: [[LOOP]]: 451; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] 452; CHECK-NEXT: [[IV_2:%.*]] = phi i16 [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ] 453; CHECK-NEXT: [[SUB]] = sub i16 [[IV_2]], [[O_1]] 454; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[IV]] 455; CHECK-NEXT: store i16 [[SUB]], ptr [[GEP_DST]], align 2 456; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 457; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] 458; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]] 459; CHECK: [[EXIT]]: 460; CHECK-NEXT: ret void 461; 462entry: 463 %o.1 = add i16 %off, 2 464 br label %loop 465 466loop: 467 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] 468 %iv.2 = phi i16 [ 0, %entry ], [ %sub, %loop ] 469 %sub = sub i16 %iv.2, %o.1 470 %gep.dst = getelementptr inbounds i16, ptr %dst, i64 %iv 471 store i16 %sub, ptr %gep.dst, align 2 472 %iv.next = add nuw nsw i64 %iv, 1 473 %ec = icmp eq i64 %iv.next, %N 474 br i1 %ec , label %exit, label %loop 475 476exit: 477 ret void 478} 479;. 480; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} 481; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} 482; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} 483; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]} 484; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]} 485; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]} 486; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]} 487; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]} 488; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]} 489; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]} 490; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]} 491; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]} 492; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]} 493; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]} 494;. 495