1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S | FileCheck %s 3; RUN: opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=1 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S | FileCheck %s --check-prefix=IND 4; RUN: opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=2 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S | FileCheck %s --check-prefix=UNROLL 5; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -S | FileCheck %s --check-prefix=UNROLL-NO-IC 6; RUN: opt < %s -passes=loop-vectorize,instcombine -force-vector-interleave=2 -force-vector-width=4 -force-widen-divrem-via-safe-divisor=0 -enable-interleaved-mem-accesses -S | FileCheck %s --check-prefix=INTERLEAVE 7 8target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" 9 10; Make sure that we can handle multiple integer induction variables. 11; 12define void @multi_int_induction(ptr %A, i32 %N) { 13; CHECK-LABEL: @multi_int_induction( 14; CHECK-NEXT: for.body.lr.ph: 15; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 16; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 17; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 18; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2 19; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 20; CHECK: vector.ph: 21; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2 22; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 23; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32 24; CHECK-NEXT: [[IND_END:%.*]] = add i32 190, [[DOTCAST]] 25; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 26; CHECK: vector.body: 27; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 28; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 190, i32 191>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 29; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 30; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP3]] 31; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i32 0 32; CHECK-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP5]], align 4 33; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 34; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 35; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 36; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 37; CHECK: middle.block: 38; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 39; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 40; CHECK: scalar.ph: 41; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH:%.*]] ] 42; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 190, [[FOR_BODY_LR_PH]] ] 43; CHECK-NEXT: br label [[FOR_BODY:%.*]] 44; CHECK: for.body: 45; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 46; CHECK-NEXT: [[COUNT_09:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 47; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]] 48; CHECK-NEXT: store i32 [[COUNT_09]], ptr [[ARRAYIDX2]], align 4 49; CHECK-NEXT: [[INC]] = add nsw i32 [[COUNT_09]], 1 50; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1 51; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 52; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i32 [[LFTR_WIDEIV]], [[N]] 53; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP3:![0-9]+]] 54; CHECK: for.end: 55; CHECK-NEXT: ret void 56; 57; IND-LABEL: @multi_int_induction( 58; IND-NEXT: for.body.lr.ph: 59; IND-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 60; IND-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 61; IND-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 62; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[TMP0]], 0 63; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 64; IND: vector.ph: 65; IND-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934590 66; IND-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32 67; IND-NEXT: [[IND_END:%.*]] = add i32 [[DOTCAST]], 190 68; IND-NEXT: br label [[VECTOR_BODY:%.*]] 69; IND: vector.body: 70; IND-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 71; IND-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 190, i32 191>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 72; IND-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] 73; IND-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP3]], align 4 74; IND-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 75; IND-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 76; IND-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 77; IND-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 78; IND: middle.block: 79; IND-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 80; IND-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 81; IND: scalar.ph: 82; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH:%.*]] ] 83; IND-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 190, [[FOR_BODY_LR_PH]] ] 84; IND-NEXT: br label [[FOR_BODY:%.*]] 85; IND: for.body: 86; IND-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 87; IND-NEXT: [[COUNT_09:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 88; IND-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]] 89; IND-NEXT: store i32 [[COUNT_09]], ptr [[ARRAYIDX2]], align 4 90; IND-NEXT: [[INC]] = add nsw i32 [[COUNT_09]], 1 91; IND-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1 92; IND-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 93; IND-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]] 94; IND-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] 95; IND: for.end: 96; IND-NEXT: ret void 97; 98; UNROLL-LABEL: @multi_int_induction( 99; UNROLL-NEXT: for.body.lr.ph: 100; UNROLL-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 101; UNROLL-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 102; UNROLL-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 103; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 3 104; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 105; UNROLL: vector.ph: 106; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934588 107; UNROLL-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32 108; UNROLL-NEXT: [[IND_END:%.*]] = add i32 [[DOTCAST]], 190 109; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 110; UNROLL: vector.body: 111; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 112; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 190, i32 191>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 113; UNROLL-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 114; UNROLL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] 115; UNROLL-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i64 8 116; UNROLL-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP3]], align 4 117; UNROLL-NEXT: store <2 x i32> [[STEP_ADD]], ptr [[TMP4]], align 4 118; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 119; UNROLL-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 4) 120; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 121; UNROLL-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 122; UNROLL: middle.block: 123; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 124; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 125; UNROLL: scalar.ph: 126; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH:%.*]] ] 127; UNROLL-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 190, [[FOR_BODY_LR_PH]] ] 128; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 129; UNROLL: for.body: 130; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 131; UNROLL-NEXT: [[COUNT_09:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 132; UNROLL-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]] 133; UNROLL-NEXT: store i32 [[COUNT_09]], ptr [[ARRAYIDX2]], align 4 134; UNROLL-NEXT: [[INC]] = add nsw i32 [[COUNT_09]], 1 135; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1 136; UNROLL-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 137; UNROLL-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]] 138; UNROLL-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] 139; UNROLL: for.end: 140; UNROLL-NEXT: ret void 141; 142; UNROLL-NO-IC-LABEL: @multi_int_induction( 143; UNROLL-NO-IC-NEXT: for.body.lr.ph: 144; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 145; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 146; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 147; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 148; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 149; UNROLL-NO-IC: vector.ph: 150; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 151; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 152; UNROLL-NO-IC-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32 153; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = add i32 190, [[DOTCAST]] 154; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 155; UNROLL-NO-IC: vector.body: 156; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 157; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 190, i32 191>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 158; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 159; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 160; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP3]] 161; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i32 0 162; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i32 2 163; UNROLL-NO-IC-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP5]], align 4 164; UNROLL-NO-IC-NEXT: store <2 x i32> [[STEP_ADD]], ptr [[TMP6]], align 4 165; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 166; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 2) 167; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 168; UNROLL-NO-IC-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 169; UNROLL-NO-IC: middle.block: 170; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 171; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 172; UNROLL-NO-IC: scalar.ph: 173; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH:%.*]] ] 174; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 190, [[FOR_BODY_LR_PH]] ] 175; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 176; UNROLL-NO-IC: for.body: 177; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 178; UNROLL-NO-IC-NEXT: [[COUNT_09:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 179; UNROLL-NO-IC-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]] 180; UNROLL-NO-IC-NEXT: store i32 [[COUNT_09]], ptr [[ARRAYIDX2]], align 4 181; UNROLL-NO-IC-NEXT: [[INC]] = add nsw i32 [[COUNT_09]], 1 182; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1 183; UNROLL-NO-IC-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 184; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp ne i32 [[LFTR_WIDEIV]], [[N]] 185; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP3:![0-9]+]] 186; UNROLL-NO-IC: for.end: 187; UNROLL-NO-IC-NEXT: ret void 188; 189; INTERLEAVE-LABEL: @multi_int_induction( 190; INTERLEAVE-NEXT: for.body.lr.ph: 191; INTERLEAVE-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 192; INTERLEAVE-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 193; INTERLEAVE-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 194; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 7 195; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 196; INTERLEAVE: vector.ph: 197; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934584 198; INTERLEAVE-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32 199; INTERLEAVE-NEXT: [[IND_END:%.*]] = add i32 [[DOTCAST]], 190 200; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 201; INTERLEAVE: vector.body: 202; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 203; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 190, i32 191, i32 192, i32 193>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 204; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4) 205; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] 206; INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i64 16 207; INTERLEAVE-NEXT: store <4 x i32> [[VEC_IND]], ptr [[TMP3]], align 4 208; INTERLEAVE-NEXT: store <4 x i32> [[STEP_ADD]], ptr [[TMP4]], align 4 209; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 210; INTERLEAVE-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 8) 211; INTERLEAVE-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 212; INTERLEAVE-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 213; INTERLEAVE: middle.block: 214; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 215; INTERLEAVE-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 216; INTERLEAVE: scalar.ph: 217; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_LR_PH:%.*]] ] 218; INTERLEAVE-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 190, [[FOR_BODY_LR_PH]] ] 219; INTERLEAVE-NEXT: br label [[FOR_BODY:%.*]] 220; INTERLEAVE: for.body: 221; INTERLEAVE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 222; INTERLEAVE-NEXT: [[COUNT_09:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ] 223; INTERLEAVE-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]] 224; INTERLEAVE-NEXT: store i32 [[COUNT_09]], ptr [[ARRAYIDX2]], align 4 225; INTERLEAVE-NEXT: [[INC]] = add nsw i32 [[COUNT_09]], 1 226; INTERLEAVE-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1 227; INTERLEAVE-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 228; INTERLEAVE-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]] 229; INTERLEAVE-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] 230; INTERLEAVE: for.end: 231; INTERLEAVE-NEXT: ret void 232; 233for.body.lr.ph: 234 br label %for.body 235 236for.body: 237 %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ] 238 %count.09 = phi i32 [ 190, %for.body.lr.ph ], [ %inc, %for.body ] 239 %arrayidx2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv 240 store i32 %count.09, ptr %arrayidx2, align 4 241 %inc = add nsw i32 %count.09, 1 242 %indvars.iv.next = add i64 %indvars.iv, 1 243 %lftr.wideiv = trunc i64 %indvars.iv.next to i32 244 %exitcond = icmp ne i32 %lftr.wideiv, %N 245 br i1 %exitcond, label %for.body, label %for.end 246 247for.end: 248 ret void 249} 250 251; Make sure we remove unneeded vectorization of induction variables. 252; In order for instcombine to cleanup the vectorized induction variables that we 253; create in the loop vectorizer we need to perform some form of redundancy 254; elimination to get rid of multiple uses. 255 256 257; Vectorized induction variable. 258 259define void @scalar_use(ptr %a, float %b, i64 %offset, i64 %offset2, i64 %n) { 260; CHECK-LABEL: @scalar_use( 261; CHECK-NEXT: entry: 262; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2 263; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 264; CHECK: vector.memcheck: 265; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[OFFSET:%.*]], 2 266; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP0]] 267; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[N]], 2 268; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], [[TMP0]] 269; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]] 270; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[OFFSET2:%.*]], 2 271; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]] 272; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], [[TMP3]] 273; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]] 274; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]] 275; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]] 276; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 277; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 278; CHECK: vector.ph: 279; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2 280; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 281; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[B:%.*]], i64 0 282; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer 283; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 284; CHECK: vector.body: 285; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 286; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 287; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], [[OFFSET]] 288; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP6]] 289; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 0 290; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP8]], align 4, !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]] 291; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP5]], [[OFFSET2]] 292; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]] 293; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP10]], i32 0 294; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <2 x float>, ptr [[TMP11]], align 4, !alias.scope [[META7]] 295; CHECK-NEXT: [[TMP12:%.*]] = fmul fast <2 x float> [[BROADCAST_SPLAT]], [[WIDE_LOAD4]] 296; CHECK-NEXT: [[TMP13:%.*]] = fadd fast <2 x float> [[WIDE_LOAD]], [[TMP12]] 297; CHECK-NEXT: store <2 x float> [[TMP13]], ptr [[TMP8]], align 4, !alias.scope [[META4]], !noalias [[META7]] 298; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 299; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 300; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 301; CHECK: middle.block: 302; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 303; CHECK-NEXT: br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]] 304; CHECK: scalar.ph: 305; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ] 306; CHECK-NEXT: br label [[FOR_BODY:%.*]] 307; CHECK: for.body: 308; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 309; CHECK-NEXT: [[IND_SUM:%.*]] = add i64 [[IV]], [[OFFSET]] 310; CHECK-NEXT: [[ARR_IDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IND_SUM]] 311; CHECK-NEXT: [[L1:%.*]] = load float, ptr [[ARR_IDX]], align 4 312; CHECK-NEXT: [[IND_SUM2:%.*]] = add i64 [[IV]], [[OFFSET2]] 313; CHECK-NEXT: [[ARR_IDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IND_SUM2]] 314; CHECK-NEXT: [[L2:%.*]] = load float, ptr [[ARR_IDX2]], align 4 315; CHECK-NEXT: [[M:%.*]] = fmul fast float [[B]], [[L2]] 316; CHECK-NEXT: [[AD:%.*]] = fadd fast float [[L1]], [[M]] 317; CHECK-NEXT: store float [[AD]], ptr [[ARR_IDX]], align 4 318; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 319; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] 320; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] 321; CHECK: loopexit: 322; CHECK-NEXT: ret void 323; 324; IND-LABEL: @scalar_use( 325; IND-NEXT: entry: 326; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2 327; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 328; IND: vector.memcheck: 329; IND-NEXT: [[TMP0:%.*]] = shl i64 [[OFFSET:%.*]], 2 330; IND-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP0]] 331; IND-NEXT: [[TMP1:%.*]] = shl i64 [[N]], 2 332; IND-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]] 333; IND-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TMP2]], i64 [[TMP0]] 334; IND-NEXT: [[TMP3:%.*]] = shl i64 [[OFFSET2:%.*]], 2 335; IND-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]] 336; IND-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]] 337; IND-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[TMP4]], i64 [[TMP3]] 338; IND-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]] 339; IND-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]] 340; IND-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 341; IND-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 342; IND: vector.ph: 343; IND-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -2 344; IND-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[B:%.*]], i64 0 345; IND-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer 346; IND-NEXT: br label [[VECTOR_BODY:%.*]] 347; IND: vector.body: 348; IND-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 349; IND-NEXT: [[TMP5:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]] 350; IND-NEXT: [[TMP6:%.*]] = getelementptr float, ptr [[TMP5]], i64 [[OFFSET]] 351; IND-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]] 352; IND-NEXT: [[TMP7:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]] 353; IND-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[TMP7]], i64 [[OFFSET2]] 354; IND-NEXT: [[WIDE_LOAD4:%.*]] = load <2 x float>, ptr [[TMP8]], align 4, !alias.scope [[META7]] 355; IND-NEXT: [[TMP9:%.*]] = fmul fast <2 x float> [[BROADCAST_SPLAT]], [[WIDE_LOAD4]] 356; IND-NEXT: [[TMP10:%.*]] = fadd fast <2 x float> [[WIDE_LOAD]], [[TMP9]] 357; IND-NEXT: store <2 x float> [[TMP10]], ptr [[TMP6]], align 4, !alias.scope [[META4]], !noalias [[META7]] 358; IND-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 359; IND-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 360; IND-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 361; IND: middle.block: 362; IND-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 363; IND-NEXT: br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]] 364; IND: scalar.ph: 365; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ] 366; IND-NEXT: br label [[FOR_BODY:%.*]] 367; IND: for.body: 368; IND-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 369; IND-NEXT: [[TMP12:%.*]] = getelementptr float, ptr [[A]], i64 [[IV]] 370; IND-NEXT: [[ARR_IDX:%.*]] = getelementptr float, ptr [[TMP12]], i64 [[OFFSET]] 371; IND-NEXT: [[L1:%.*]] = load float, ptr [[ARR_IDX]], align 4 372; IND-NEXT: [[TMP13:%.*]] = getelementptr float, ptr [[A]], i64 [[IV]] 373; IND-NEXT: [[ARR_IDX2:%.*]] = getelementptr float, ptr [[TMP13]], i64 [[OFFSET2]] 374; IND-NEXT: [[L2:%.*]] = load float, ptr [[ARR_IDX2]], align 4 375; IND-NEXT: [[M:%.*]] = fmul fast float [[B]], [[L2]] 376; IND-NEXT: [[AD:%.*]] = fadd fast float [[L1]], [[M]] 377; IND-NEXT: store float [[AD]], ptr [[ARR_IDX]], align 4 378; IND-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 379; IND-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] 380; IND-NEXT: br i1 [[EXITCOND]], label [[LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] 381; IND: loopexit: 382; IND-NEXT: ret void 383; 384; UNROLL-LABEL: @scalar_use( 385; UNROLL-NEXT: entry: 386; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 387; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 388; UNROLL: vector.memcheck: 389; UNROLL-NEXT: [[TMP0:%.*]] = shl i64 [[OFFSET:%.*]], 2 390; UNROLL-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP0]] 391; UNROLL-NEXT: [[TMP1:%.*]] = shl i64 [[N]], 2 392; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]] 393; UNROLL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TMP2]], i64 [[TMP0]] 394; UNROLL-NEXT: [[TMP3:%.*]] = shl i64 [[OFFSET2:%.*]], 2 395; UNROLL-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]] 396; UNROLL-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]] 397; UNROLL-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[TMP4]], i64 [[TMP3]] 398; UNROLL-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]] 399; UNROLL-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]] 400; UNROLL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 401; UNROLL-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 402; UNROLL: vector.ph: 403; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 404; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[B:%.*]], i64 0 405; UNROLL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer 406; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 407; UNROLL: vector.body: 408; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 409; UNROLL-NEXT: [[TMP5:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]] 410; UNROLL-NEXT: [[TMP6:%.*]] = getelementptr float, ptr [[TMP5]], i64 [[OFFSET]] 411; UNROLL-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP6]], i64 8 412; UNROLL-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]] 413; UNROLL-NEXT: [[WIDE_LOAD4:%.*]] = load <2 x float>, ptr [[TMP7]], align 4, !alias.scope [[META4]], !noalias [[META7]] 414; UNROLL-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]] 415; UNROLL-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[TMP8]], i64 [[OFFSET2]] 416; UNROLL-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 8 417; UNROLL-NEXT: [[WIDE_LOAD5:%.*]] = load <2 x float>, ptr [[TMP9]], align 4, !alias.scope [[META7]] 418; UNROLL-NEXT: [[WIDE_LOAD6:%.*]] = load <2 x float>, ptr [[TMP10]], align 4, !alias.scope [[META7]] 419; UNROLL-NEXT: [[TMP11:%.*]] = fmul fast <2 x float> [[BROADCAST_SPLAT]], [[WIDE_LOAD5]] 420; UNROLL-NEXT: [[TMP12:%.*]] = fmul fast <2 x float> [[BROADCAST_SPLAT]], [[WIDE_LOAD6]] 421; UNROLL-NEXT: [[TMP13:%.*]] = fadd fast <2 x float> [[WIDE_LOAD]], [[TMP11]] 422; UNROLL-NEXT: [[TMP14:%.*]] = fadd fast <2 x float> [[WIDE_LOAD4]], [[TMP12]] 423; UNROLL-NEXT: store <2 x float> [[TMP13]], ptr [[TMP6]], align 4, !alias.scope [[META4]], !noalias [[META7]] 424; UNROLL-NEXT: store <2 x float> [[TMP14]], ptr [[TMP7]], align 4, !alias.scope [[META4]], !noalias [[META7]] 425; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 426; UNROLL-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 427; UNROLL-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 428; UNROLL: middle.block: 429; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 430; UNROLL-NEXT: br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]] 431; UNROLL: scalar.ph: 432; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ] 433; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 434; UNROLL: for.body: 435; UNROLL-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 436; UNROLL-NEXT: [[TMP16:%.*]] = getelementptr float, ptr [[A]], i64 [[IV]] 437; UNROLL-NEXT: [[ARR_IDX:%.*]] = getelementptr float, ptr [[TMP16]], i64 [[OFFSET]] 438; UNROLL-NEXT: [[L1:%.*]] = load float, ptr [[ARR_IDX]], align 4 439; UNROLL-NEXT: [[TMP17:%.*]] = getelementptr float, ptr [[A]], i64 [[IV]] 440; UNROLL-NEXT: [[ARR_IDX2:%.*]] = getelementptr float, ptr [[TMP17]], i64 [[OFFSET2]] 441; UNROLL-NEXT: [[L2:%.*]] = load float, ptr [[ARR_IDX2]], align 4 442; UNROLL-NEXT: [[M:%.*]] = fmul fast float [[B]], [[L2]] 443; UNROLL-NEXT: [[AD:%.*]] = fadd fast float [[L1]], [[M]] 444; UNROLL-NEXT: store float [[AD]], ptr [[ARR_IDX]], align 4 445; UNROLL-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 446; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] 447; UNROLL-NEXT: br i1 [[EXITCOND]], label [[LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] 448; UNROLL: loopexit: 449; UNROLL-NEXT: ret void 450; 451; UNROLL-NO-IC-LABEL: @scalar_use( 452; UNROLL-NO-IC-NEXT: entry: 453; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 454; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 455; UNROLL-NO-IC: vector.memcheck: 456; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = shl i64 [[OFFSET:%.*]], 2 457; UNROLL-NO-IC-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP0]] 458; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = shl i64 [[N]], 2 459; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], [[TMP0]] 460; UNROLL-NO-IC-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]] 461; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = shl i64 [[OFFSET2:%.*]], 2 462; UNROLL-NO-IC-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]] 463; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i64 [[TMP1]], [[TMP3]] 464; UNROLL-NO-IC-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP4]] 465; UNROLL-NO-IC-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]] 466; UNROLL-NO-IC-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]] 467; UNROLL-NO-IC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 468; UNROLL-NO-IC-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 469; UNROLL-NO-IC: vector.ph: 470; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 471; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 472; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[B:%.*]], i64 0 473; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer 474; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 475; UNROLL-NO-IC: vector.body: 476; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 477; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 478; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], [[OFFSET]] 479; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP6]] 480; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 0 481; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 2 482; UNROLL-NO-IC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP8]], align 4, !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]] 483; UNROLL-NO-IC-NEXT: [[WIDE_LOAD4:%.*]] = load <2 x float>, ptr [[TMP9]], align 4, !alias.scope [[META4]], !noalias [[META7]] 484; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = add i64 [[TMP5]], [[OFFSET2]] 485; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP10]] 486; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i32 0 487; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i32 2 488; UNROLL-NO-IC-NEXT: [[WIDE_LOAD5:%.*]] = load <2 x float>, ptr [[TMP12]], align 4, !alias.scope [[META7]] 489; UNROLL-NO-IC-NEXT: [[WIDE_LOAD6:%.*]] = load <2 x float>, ptr [[TMP13]], align 4, !alias.scope [[META7]] 490; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = fmul fast <2 x float> [[BROADCAST_SPLAT]], [[WIDE_LOAD5]] 491; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = fmul fast <2 x float> [[BROADCAST_SPLAT]], [[WIDE_LOAD6]] 492; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = fadd fast <2 x float> [[WIDE_LOAD]], [[TMP14]] 493; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = fadd fast <2 x float> [[WIDE_LOAD4]], [[TMP15]] 494; UNROLL-NO-IC-NEXT: store <2 x float> [[TMP16]], ptr [[TMP8]], align 4, !alias.scope [[META4]], !noalias [[META7]] 495; UNROLL-NO-IC-NEXT: store <2 x float> [[TMP17]], ptr [[TMP9]], align 4, !alias.scope [[META4]], !noalias [[META7]] 496; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 497; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 498; UNROLL-NO-IC-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 499; UNROLL-NO-IC: middle.block: 500; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 501; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]] 502; UNROLL-NO-IC: scalar.ph: 503; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ] 504; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 505; UNROLL-NO-IC: for.body: 506; UNROLL-NO-IC-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 507; UNROLL-NO-IC-NEXT: [[IND_SUM:%.*]] = add i64 [[IV]], [[OFFSET]] 508; UNROLL-NO-IC-NEXT: [[ARR_IDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IND_SUM]] 509; UNROLL-NO-IC-NEXT: [[L1:%.*]] = load float, ptr [[ARR_IDX]], align 4 510; UNROLL-NO-IC-NEXT: [[IND_SUM2:%.*]] = add i64 [[IV]], [[OFFSET2]] 511; UNROLL-NO-IC-NEXT: [[ARR_IDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IND_SUM2]] 512; UNROLL-NO-IC-NEXT: [[L2:%.*]] = load float, ptr [[ARR_IDX2]], align 4 513; UNROLL-NO-IC-NEXT: [[M:%.*]] = fmul fast float [[B]], [[L2]] 514; UNROLL-NO-IC-NEXT: [[AD:%.*]] = fadd fast float [[L1]], [[M]] 515; UNROLL-NO-IC-NEXT: store float [[AD]], ptr [[ARR_IDX]], align 4 516; UNROLL-NO-IC-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 517; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] 518; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] 519; UNROLL-NO-IC: loopexit: 520; UNROLL-NO-IC-NEXT: ret void 521; 522; INTERLEAVE-LABEL: @scalar_use( 523; INTERLEAVE-NEXT: entry: 524; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8 525; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 526; INTERLEAVE: vector.memcheck: 527; INTERLEAVE-NEXT: [[TMP0:%.*]] = shl i64 [[OFFSET:%.*]], 2 528; INTERLEAVE-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP0]] 529; INTERLEAVE-NEXT: [[TMP1:%.*]] = shl i64 [[N]], 2 530; INTERLEAVE-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]] 531; INTERLEAVE-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TMP2]], i64 [[TMP0]] 532; INTERLEAVE-NEXT: [[TMP3:%.*]] = shl i64 [[OFFSET2:%.*]], 2 533; INTERLEAVE-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]] 534; INTERLEAVE-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]] 535; INTERLEAVE-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[TMP4]], i64 [[TMP3]] 536; INTERLEAVE-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]] 537; INTERLEAVE-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP1]] 538; INTERLEAVE-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 539; INTERLEAVE-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 540; INTERLEAVE: vector.ph: 541; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -8 542; INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[B:%.*]], i64 0 543; INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer 544; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 545; INTERLEAVE: vector.body: 546; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 547; INTERLEAVE-NEXT: [[TMP5:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]] 548; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr float, ptr [[TMP5]], i64 [[OFFSET]] 549; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP6]], i64 16 550; INTERLEAVE-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP6]], align 4, !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]] 551; INTERLEAVE-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP7]], align 4, !alias.scope [[META4]], !noalias [[META7]] 552; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]] 553; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[TMP8]], i64 [[OFFSET2]] 554; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 16 555; INTERLEAVE-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP9]], align 4, !alias.scope [[META7]] 556; INTERLEAVE-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP10]], align 4, !alias.scope [[META7]] 557; INTERLEAVE-NEXT: [[TMP11:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], [[WIDE_LOAD5]] 558; INTERLEAVE-NEXT: [[TMP12:%.*]] = fmul fast <4 x float> [[BROADCAST_SPLAT]], [[WIDE_LOAD6]] 559; INTERLEAVE-NEXT: [[TMP13:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[TMP11]] 560; INTERLEAVE-NEXT: [[TMP14:%.*]] = fadd fast <4 x float> [[WIDE_LOAD4]], [[TMP12]] 561; INTERLEAVE-NEXT: store <4 x float> [[TMP13]], ptr [[TMP6]], align 4, !alias.scope [[META4]], !noalias [[META7]] 562; INTERLEAVE-NEXT: store <4 x float> [[TMP14]], ptr [[TMP7]], align 4, !alias.scope [[META4]], !noalias [[META7]] 563; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 564; INTERLEAVE-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 565; INTERLEAVE-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] 566; INTERLEAVE: middle.block: 567; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 568; INTERLEAVE-NEXT: br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]] 569; INTERLEAVE: scalar.ph: 570; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ] 571; INTERLEAVE-NEXT: br label [[FOR_BODY:%.*]] 572; INTERLEAVE: for.body: 573; INTERLEAVE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] 574; INTERLEAVE-NEXT: [[TMP16:%.*]] = getelementptr float, ptr [[A]], i64 [[IV]] 575; INTERLEAVE-NEXT: [[ARR_IDX:%.*]] = getelementptr float, ptr [[TMP16]], i64 [[OFFSET]] 576; INTERLEAVE-NEXT: [[L1:%.*]] = load float, ptr [[ARR_IDX]], align 4 577; INTERLEAVE-NEXT: [[TMP17:%.*]] = getelementptr float, ptr [[A]], i64 [[IV]] 578; INTERLEAVE-NEXT: [[ARR_IDX2:%.*]] = getelementptr float, ptr [[TMP17]], i64 [[OFFSET2]] 579; INTERLEAVE-NEXT: [[L2:%.*]] = load float, ptr [[ARR_IDX2]], align 4 580; INTERLEAVE-NEXT: [[M:%.*]] = fmul fast float [[B]], [[L2]] 581; INTERLEAVE-NEXT: [[AD:%.*]] = fadd fast float [[L1]], [[M]] 582; INTERLEAVE-NEXT: store float [[AD]], ptr [[ARR_IDX]], align 4 583; INTERLEAVE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 584; INTERLEAVE-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] 585; INTERLEAVE-NEXT: br i1 [[EXITCOND]], label [[LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] 586; INTERLEAVE: loopexit: 587; INTERLEAVE-NEXT: ret void 588; 589entry: 590 br label %for.body 591 592for.body: 593 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] 594 %ind.sum = add i64 %iv, %offset 595 %arr.idx = getelementptr inbounds float, ptr %a, i64 %ind.sum 596 %l1 = load float, ptr %arr.idx, align 4 597 %ind.sum2 = add i64 %iv, %offset2 598 %arr.idx2 = getelementptr inbounds float, ptr %a, i64 %ind.sum2 599 %l2 = load float, ptr %arr.idx2, align 4 600 %m = fmul fast float %b, %l2 601 %ad = fadd fast float %l1, %m 602 store float %ad, ptr %arr.idx, align 4 603 %iv.next = add nuw nsw i64 %iv, 1 604 %exitcond = icmp eq i64 %iv.next, %n 605 br i1 %exitcond, label %loopexit, label %for.body 606 607loopexit: 608 ret void 609} 610 611; Make sure we don't create a vector induction phi node that is unused. 612; Scalarize the step vectors instead. 613; 614; for (int i = 0; i < n; ++i) 615; sum += a[i]; 616; 617; 618; 619; 620 621define i64 @scalarize_induction_variable_01(ptr %a, i64 %n) { 622; CHECK-LABEL: @scalarize_induction_variable_01( 623; CHECK-NEXT: entry: 624; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) 625; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 2 626; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 627; CHECK: vector.ph: 628; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 2 629; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]] 630; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 631; CHECK: vector.body: 632; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 633; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ] 634; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 635; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]] 636; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 0 637; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8 638; CHECK-NEXT: [[TMP3]] = add <2 x i64> [[WIDE_LOAD]], [[VEC_PHI]] 639; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 640; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 641; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] 642; CHECK: middle.block: 643; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[TMP3]]) 644; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 645; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 646; CHECK: scalar.ph: 647; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 648; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 649; CHECK-NEXT: br label [[FOR_BODY:%.*]] 650; CHECK: for.body: 651; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 652; CHECK-NEXT: [[SUM:%.*]] = phi i64 [ [[TMP8:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 653; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]] 654; CHECK-NEXT: [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8 655; CHECK-NEXT: [[TMP8]] = add i64 [[TMP7]], [[SUM]] 656; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 657; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 658; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP12:![0-9]+]] 659; CHECK: for.end: 660; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ [[TMP8]], [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ] 661; CHECK-NEXT: ret i64 [[TMP9]] 662; 663; IND-LABEL: @scalarize_induction_variable_01( 664; IND-NEXT: entry: 665; IND-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) 666; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 2 667; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 668; IND: vector.ph: 669; IND-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775806 670; IND-NEXT: br label [[VECTOR_BODY:%.*]] 671; IND: vector.body: 672; IND-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 673; IND-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 674; IND-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] 675; IND-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP0]], align 8 676; IND-NEXT: [[TMP1]] = add <2 x i64> [[WIDE_LOAD]], [[VEC_PHI]] 677; IND-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 678; IND-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 679; IND-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] 680; IND: middle.block: 681; IND-NEXT: [[TMP3:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[TMP1]]) 682; IND-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 683; IND-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 684; IND: scalar.ph: 685; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 686; IND-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 687; IND-NEXT: br label [[FOR_BODY:%.*]] 688; IND: for.body: 689; IND-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 690; IND-NEXT: [[SUM:%.*]] = phi i64 [ [[TMP6:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 691; IND-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[I]] 692; IND-NEXT: [[TMP5:%.*]] = load i64, ptr [[TMP4]], align 8 693; IND-NEXT: [[TMP6]] = add i64 [[TMP5]], [[SUM]] 694; IND-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 695; IND-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 696; IND-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP12:![0-9]+]] 697; IND: for.end: 698; IND-NEXT: [[TMP7:%.*]] = phi i64 [ [[TMP6]], [[FOR_BODY]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ] 699; IND-NEXT: ret i64 [[TMP7]] 700; 701; UNROLL-LABEL: @scalarize_induction_variable_01( 702; UNROLL-NEXT: entry: 703; UNROLL-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) 704; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 705; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 706; UNROLL: vector.ph: 707; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804 708; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 709; UNROLL: vector.body: 710; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 711; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ] 712; UNROLL-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ] 713; UNROLL-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] 714; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16 715; UNROLL-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP0]], align 8 716; UNROLL-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8 717; UNROLL-NEXT: [[TMP2]] = add <2 x i64> [[WIDE_LOAD]], [[VEC_PHI]] 718; UNROLL-NEXT: [[TMP3]] = add <2 x i64> [[WIDE_LOAD2]], [[VEC_PHI1]] 719; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 720; UNROLL-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 721; UNROLL-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] 722; UNROLL: middle.block: 723; UNROLL-NEXT: [[BIN_RDX:%.*]] = add <2 x i64> [[TMP3]], [[TMP2]] 724; UNROLL-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX]]) 725; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 726; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 727; UNROLL: scalar.ph: 728; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 729; UNROLL-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 730; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 731; UNROLL: for.body: 732; UNROLL-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 733; UNROLL-NEXT: [[SUM:%.*]] = phi i64 [ [[TMP8:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 734; UNROLL-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[I]] 735; UNROLL-NEXT: [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8 736; UNROLL-NEXT: [[TMP8]] = add i64 [[TMP7]], [[SUM]] 737; UNROLL-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 738; UNROLL-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 739; UNROLL-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP12:![0-9]+]] 740; UNROLL: for.end: 741; UNROLL-NEXT: [[TMP9:%.*]] = phi i64 [ [[TMP8]], [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ] 742; UNROLL-NEXT: ret i64 [[TMP9]] 743; 744; UNROLL-NO-IC-LABEL: @scalarize_induction_variable_01( 745; UNROLL-NO-IC-NEXT: entry: 746; UNROLL-NO-IC-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) 747; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4 748; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 749; UNROLL-NO-IC: vector.ph: 750; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4 751; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]] 752; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 753; UNROLL-NO-IC: vector.body: 754; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 755; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ] 756; UNROLL-NO-IC-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ] 757; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 758; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP0]] 759; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 0 760; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 2 761; UNROLL-NO-IC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8 762; UNROLL-NO-IC-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP3]], align 8 763; UNROLL-NO-IC-NEXT: [[TMP4]] = add <2 x i64> [[WIDE_LOAD]], [[VEC_PHI]] 764; UNROLL-NO-IC-NEXT: [[TMP5]] = add <2 x i64> [[WIDE_LOAD2]], [[VEC_PHI1]] 765; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 766; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 767; UNROLL-NO-IC-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] 768; UNROLL-NO-IC: middle.block: 769; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = add <2 x i64> [[TMP5]], [[TMP4]] 770; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX]]) 771; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 772; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 773; UNROLL-NO-IC: scalar.ph: 774; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 775; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 776; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 777; UNROLL-NO-IC: for.body: 778; UNROLL-NO-IC-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 779; UNROLL-NO-IC-NEXT: [[SUM:%.*]] = phi i64 [ [[TMP10:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 780; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[I]] 781; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8 782; UNROLL-NO-IC-NEXT: [[TMP10]] = add i64 [[TMP9]], [[SUM]] 783; UNROLL-NO-IC-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 784; UNROLL-NO-IC-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 785; UNROLL-NO-IC-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP12:![0-9]+]] 786; UNROLL-NO-IC: for.end: 787; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = phi i64 [ [[TMP10]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] 788; UNROLL-NO-IC-NEXT: ret i64 [[TMP11]] 789; 790; INTERLEAVE-LABEL: @scalarize_induction_variable_01( 791; INTERLEAVE-NEXT: entry: 792; INTERLEAVE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) 793; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 8 794; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 795; INTERLEAVE: vector.ph: 796; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775800 797; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 798; INTERLEAVE: vector.body: 799; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 800; INTERLEAVE-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ] 801; INTERLEAVE-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ] 802; INTERLEAVE-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] 803; INTERLEAVE-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32 804; INTERLEAVE-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8 805; INTERLEAVE-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8 806; INTERLEAVE-NEXT: [[TMP2]] = add <4 x i64> [[WIDE_LOAD]], [[VEC_PHI]] 807; INTERLEAVE-NEXT: [[TMP3]] = add <4 x i64> [[WIDE_LOAD2]], [[VEC_PHI1]] 808; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 809; INTERLEAVE-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 810; INTERLEAVE-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] 811; INTERLEAVE: middle.block: 812; INTERLEAVE-NEXT: [[BIN_RDX:%.*]] = add <4 x i64> [[TMP3]], [[TMP2]] 813; INTERLEAVE-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[BIN_RDX]]) 814; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 815; INTERLEAVE-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 816; INTERLEAVE: scalar.ph: 817; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 818; INTERLEAVE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 819; INTERLEAVE-NEXT: br label [[FOR_BODY:%.*]] 820; INTERLEAVE: for.body: 821; INTERLEAVE-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 822; INTERLEAVE-NEXT: [[SUM:%.*]] = phi i64 [ [[TMP8:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 823; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[I]] 824; INTERLEAVE-NEXT: [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8 825; INTERLEAVE-NEXT: [[TMP8]] = add i64 [[TMP7]], [[SUM]] 826; INTERLEAVE-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 827; INTERLEAVE-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 828; INTERLEAVE-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP12:![0-9]+]] 829; INTERLEAVE: for.end: 830; INTERLEAVE-NEXT: [[TMP9:%.*]] = phi i64 [ [[TMP8]], [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ] 831; INTERLEAVE-NEXT: ret i64 [[TMP9]] 832; 833entry: 834 br label %for.body 835 836for.body: 837 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 838 %sum = phi i64 [ %2, %for.body ], [ 0, %entry ] 839 %0 = getelementptr inbounds i64, ptr %a, i64 %i 840 %1 = load i64, ptr %0, align 8 841 %2 = add i64 %1, %sum 842 %i.next = add nuw nsw i64 %i, 1 843 %cond = icmp slt i64 %i.next, %n 844 br i1 %cond, label %for.body, label %for.end 845 846for.end: 847 %3 = phi i64 [ %2, %for.body ] 848 ret i64 %3 849} 850 851; Make sure we scalarize the step vectors used for the pointer arithmetic. We 852; can't easily simplify vectorized step vectors. 853; 854; float s = 0; 855; for (int i ; 0; i < n; i += 8) 856; s += (a[i] + b[i] + 1.0f); 857; 858; 859; 860; 861 862define float @scalarize_induction_variable_02(ptr %a, ptr %b, i64 %n) { 863; CHECK-LABEL: @scalarize_induction_variable_02( 864; CHECK-NEXT: entry: 865; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 8) 866; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 867; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 3 868; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 869; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2 870; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 871; CHECK: vector.ph: 872; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2 873; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 874; CHECK-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 8 875; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 876; CHECK: vector.body: 877; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 878; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ] 879; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8 880; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 0 881; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 8 882; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[TMP3]] 883; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP4]] 884; CHECK-NEXT: [[TMP7:%.*]] = load float, ptr [[TMP5]], align 4 885; CHECK-NEXT: [[TMP8:%.*]] = load float, ptr [[TMP6]], align 4 886; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP7]], i32 0 887; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x float> [[TMP9]], float [[TMP8]], i32 1 888; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[TMP3]] 889; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP4]] 890; CHECK-NEXT: [[TMP13:%.*]] = load float, ptr [[TMP11]], align 4 891; CHECK-NEXT: [[TMP14:%.*]] = load float, ptr [[TMP12]], align 4 892; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x float> poison, float [[TMP13]], i32 0 893; CHECK-NEXT: [[TMP16:%.*]] = insertelement <2 x float> [[TMP15]], float [[TMP14]], i32 1 894; CHECK-NEXT: [[TMP17:%.*]] = fadd fast <2 x float> [[VEC_PHI]], splat (float 1.000000e+00) 895; CHECK-NEXT: [[TMP18:%.*]] = fadd fast <2 x float> [[TMP17]], [[TMP10]] 896; CHECK-NEXT: [[TMP19]] = fadd fast <2 x float> [[TMP18]], [[TMP16]] 897; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 898; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 899; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] 900; CHECK: middle.block: 901; CHECK-NEXT: [[TMP21:%.*]] = call fast float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> [[TMP19]]) 902; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 903; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 904; CHECK: scalar.ph: 905; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 906; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ] 907; CHECK-NEXT: br label [[FOR_BODY:%.*]] 908; CHECK: for.body: 909; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_BODY]] ] 910; CHECK-NEXT: [[S:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP28:%.*]], [[FOR_BODY]] ] 911; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[I]] 912; CHECK-NEXT: [[TMP23:%.*]] = load float, ptr [[TMP22]], align 4 913; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[I]] 914; CHECK-NEXT: [[TMP25:%.*]] = load float, ptr [[TMP24]], align 4 915; CHECK-NEXT: [[TMP26:%.*]] = fadd fast float [[S]], 1.000000e+00 916; CHECK-NEXT: [[TMP27:%.*]] = fadd fast float [[TMP26]], [[TMP23]] 917; CHECK-NEXT: [[TMP28]] = fadd fast float [[TMP27]], [[TMP25]] 918; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 8 919; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 920; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]] 921; CHECK: for.end: 922; CHECK-NEXT: [[S_LCSSA:%.*]] = phi float [ [[TMP28]], [[FOR_BODY]] ], [ [[TMP21]], [[MIDDLE_BLOCK]] ] 923; CHECK-NEXT: ret float [[S_LCSSA]] 924; 925; IND-LABEL: @scalarize_induction_variable_02( 926; IND-NEXT: entry: 927; IND-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 8) 928; IND-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 929; IND-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 3 930; IND-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 931; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 9 932; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 933; IND: vector.ph: 934; IND-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 4611686018427387902 935; IND-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 3 936; IND-NEXT: br label [[VECTOR_BODY:%.*]] 937; IND: vector.body: 938; IND-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 939; IND-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ] 940; IND-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 3 941; IND-NEXT: [[TMP3:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8 942; IND-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[OFFSET_IDX]] 943; IND-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP3]] 944; IND-NEXT: [[TMP6:%.*]] = load float, ptr [[TMP4]], align 4 945; IND-NEXT: [[TMP7:%.*]] = load float, ptr [[TMP5]], align 4 946; IND-NEXT: [[TMP8:%.*]] = insertelement <2 x float> poison, float [[TMP6]], i64 0 947; IND-NEXT: [[TMP9:%.*]] = insertelement <2 x float> [[TMP8]], float [[TMP7]], i64 1 948; IND-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[OFFSET_IDX]] 949; IND-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP3]] 950; IND-NEXT: [[TMP12:%.*]] = load float, ptr [[TMP10]], align 4 951; IND-NEXT: [[TMP13:%.*]] = load float, ptr [[TMP11]], align 4 952; IND-NEXT: [[TMP14:%.*]] = insertelement <2 x float> poison, float [[TMP12]], i64 0 953; IND-NEXT: [[TMP15:%.*]] = insertelement <2 x float> [[TMP14]], float [[TMP13]], i64 1 954; IND-NEXT: [[TMP16:%.*]] = fadd fast <2 x float> [[VEC_PHI]], splat (float 1.000000e+00) 955; IND-NEXT: [[TMP17:%.*]] = fadd fast <2 x float> [[TMP16]], [[TMP9]] 956; IND-NEXT: [[TMP18]] = fadd fast <2 x float> [[TMP17]], [[TMP15]] 957; IND-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 958; IND-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 959; IND-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] 960; IND: middle.block: 961; IND-NEXT: [[TMP20:%.*]] = call fast float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> [[TMP18]]) 962; IND-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 963; IND-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 964; IND: scalar.ph: 965; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 966; IND-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP20]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ] 967; IND-NEXT: br label [[FOR_BODY:%.*]] 968; IND: for.body: 969; IND-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_BODY]] ] 970; IND-NEXT: [[S:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP27:%.*]], [[FOR_BODY]] ] 971; IND-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[I]] 972; IND-NEXT: [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4 973; IND-NEXT: [[TMP23:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[I]] 974; IND-NEXT: [[TMP24:%.*]] = load float, ptr [[TMP23]], align 4 975; IND-NEXT: [[TMP25:%.*]] = fadd fast float [[S]], 1.000000e+00 976; IND-NEXT: [[TMP26:%.*]] = fadd fast float [[TMP25]], [[TMP22]] 977; IND-NEXT: [[TMP27]] = fadd fast float [[TMP26]], [[TMP24]] 978; IND-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 8 979; IND-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 980; IND-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]] 981; IND: for.end: 982; IND-NEXT: [[S_LCSSA:%.*]] = phi float [ [[TMP27]], [[FOR_BODY]] ], [ [[TMP20]], [[MIDDLE_BLOCK]] ] 983; IND-NEXT: ret float [[S_LCSSA]] 984; 985; UNROLL-LABEL: @scalarize_induction_variable_02( 986; UNROLL-NEXT: entry: 987; UNROLL-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 8) 988; UNROLL-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 989; UNROLL-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 3 990; UNROLL-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 991; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 25 992; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 993; UNROLL: vector.ph: 994; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 4611686018427387900 995; UNROLL-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 3 996; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 997; UNROLL: vector.body: 998; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 999; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP34:%.*]], [[VECTOR_BODY]] ] 1000; UNROLL-NEXT: [[VEC_PHI1:%.*]] = phi <2 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP35:%.*]], [[VECTOR_BODY]] ] 1001; UNROLL-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 3 1002; UNROLL-NEXT: [[TMP3:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8 1003; UNROLL-NEXT: [[TMP4:%.*]] = or disjoint i64 [[OFFSET_IDX]], 16 1004; UNROLL-NEXT: [[TMP5:%.*]] = or disjoint i64 [[OFFSET_IDX]], 24 1005; UNROLL-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[OFFSET_IDX]] 1006; UNROLL-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP3]] 1007; UNROLL-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP4]] 1008; UNROLL-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP5]] 1009; UNROLL-NEXT: [[TMP10:%.*]] = load float, ptr [[TMP6]], align 4 1010; UNROLL-NEXT: [[TMP11:%.*]] = load float, ptr [[TMP7]], align 4 1011; UNROLL-NEXT: [[TMP12:%.*]] = insertelement <2 x float> poison, float [[TMP10]], i64 0 1012; UNROLL-NEXT: [[TMP13:%.*]] = insertelement <2 x float> [[TMP12]], float [[TMP11]], i64 1 1013; UNROLL-NEXT: [[TMP14:%.*]] = load float, ptr [[TMP8]], align 4 1014; UNROLL-NEXT: [[TMP15:%.*]] = load float, ptr [[TMP9]], align 4 1015; UNROLL-NEXT: [[TMP16:%.*]] = insertelement <2 x float> poison, float [[TMP14]], i64 0 1016; UNROLL-NEXT: [[TMP17:%.*]] = insertelement <2 x float> [[TMP16]], float [[TMP15]], i64 1 1017; UNROLL-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[OFFSET_IDX]] 1018; UNROLL-NEXT: [[TMP19:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP3]] 1019; UNROLL-NEXT: [[TMP20:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP4]] 1020; UNROLL-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP5]] 1021; UNROLL-NEXT: [[TMP22:%.*]] = load float, ptr [[TMP18]], align 4 1022; UNROLL-NEXT: [[TMP23:%.*]] = load float, ptr [[TMP19]], align 4 1023; UNROLL-NEXT: [[TMP24:%.*]] = insertelement <2 x float> poison, float [[TMP22]], i64 0 1024; UNROLL-NEXT: [[TMP25:%.*]] = insertelement <2 x float> [[TMP24]], float [[TMP23]], i64 1 1025; UNROLL-NEXT: [[TMP26:%.*]] = load float, ptr [[TMP20]], align 4 1026; UNROLL-NEXT: [[TMP27:%.*]] = load float, ptr [[TMP21]], align 4 1027; UNROLL-NEXT: [[TMP28:%.*]] = insertelement <2 x float> poison, float [[TMP26]], i64 0 1028; UNROLL-NEXT: [[TMP29:%.*]] = insertelement <2 x float> [[TMP28]], float [[TMP27]], i64 1 1029; UNROLL-NEXT: [[TMP30:%.*]] = fadd fast <2 x float> [[VEC_PHI]], splat (float 1.000000e+00) 1030; UNROLL-NEXT: [[TMP31:%.*]] = fadd fast <2 x float> [[VEC_PHI1]], splat (float 1.000000e+00) 1031; UNROLL-NEXT: [[TMP32:%.*]] = fadd fast <2 x float> [[TMP30]], [[TMP13]] 1032; UNROLL-NEXT: [[TMP33:%.*]] = fadd fast <2 x float> [[TMP31]], [[TMP17]] 1033; UNROLL-NEXT: [[TMP34]] = fadd fast <2 x float> [[TMP32]], [[TMP25]] 1034; UNROLL-NEXT: [[TMP35]] = fadd fast <2 x float> [[TMP33]], [[TMP29]] 1035; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 1036; UNROLL-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1037; UNROLL-NEXT: br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] 1038; UNROLL: middle.block: 1039; UNROLL-NEXT: [[BIN_RDX:%.*]] = fadd fast <2 x float> [[TMP35]], [[TMP34]] 1040; UNROLL-NEXT: [[TMP37:%.*]] = call fast float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> [[BIN_RDX]]) 1041; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 1042; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1043; UNROLL: scalar.ph: 1044; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1045; UNROLL-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP37]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ] 1046; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 1047; UNROLL: for.body: 1048; UNROLL-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_BODY]] ] 1049; UNROLL-NEXT: [[S:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP44:%.*]], [[FOR_BODY]] ] 1050; UNROLL-NEXT: [[TMP38:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[I]] 1051; UNROLL-NEXT: [[TMP39:%.*]] = load float, ptr [[TMP38]], align 4 1052; UNROLL-NEXT: [[TMP40:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[I]] 1053; UNROLL-NEXT: [[TMP41:%.*]] = load float, ptr [[TMP40]], align 4 1054; UNROLL-NEXT: [[TMP42:%.*]] = fadd fast float [[S]], 1.000000e+00 1055; UNROLL-NEXT: [[TMP43:%.*]] = fadd fast float [[TMP42]], [[TMP39]] 1056; UNROLL-NEXT: [[TMP44]] = fadd fast float [[TMP43]], [[TMP41]] 1057; UNROLL-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 8 1058; UNROLL-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 1059; UNROLL-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]] 1060; UNROLL: for.end: 1061; UNROLL-NEXT: [[S_LCSSA:%.*]] = phi float [ [[TMP44]], [[FOR_BODY]] ], [ [[TMP37]], [[MIDDLE_BLOCK]] ] 1062; UNROLL-NEXT: ret float [[S_LCSSA]] 1063; 1064; UNROLL-NO-IC-LABEL: @scalarize_induction_variable_02( 1065; UNROLL-NO-IC-NEXT: entry: 1066; UNROLL-NO-IC-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 8) 1067; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 1068; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 3 1069; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 1070; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 1071; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1072; UNROLL-NO-IC: vector.ph: 1073; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 1074; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 1075; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 8 1076; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 1077; UNROLL-NO-IC: vector.body: 1078; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1079; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <2 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP35:%.*]], [[VECTOR_BODY]] ] 1080; UNROLL-NO-IC-NEXT: [[VEC_PHI1:%.*]] = phi <2 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP36:%.*]], [[VECTOR_BODY]] ] 1081; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8 1082; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 0 1083; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 8 1084; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 16 1085; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 24 1086; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[TMP3]] 1087; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP4]] 1088; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP5]] 1089; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP6]] 1090; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = load float, ptr [[TMP7]], align 4 1091; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = load float, ptr [[TMP8]], align 4 1092; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = insertelement <2 x float> poison, float [[TMP11]], i32 0 1093; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = insertelement <2 x float> [[TMP13]], float [[TMP12]], i32 1 1094; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = load float, ptr [[TMP9]], align 4 1095; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = load float, ptr [[TMP10]], align 4 1096; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = insertelement <2 x float> poison, float [[TMP15]], i32 0 1097; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = insertelement <2 x float> [[TMP17]], float [[TMP16]], i32 1 1098; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[TMP3]] 1099; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP4]] 1100; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP5]] 1101; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP6]] 1102; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = load float, ptr [[TMP19]], align 4 1103; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = load float, ptr [[TMP20]], align 4 1104; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = insertelement <2 x float> poison, float [[TMP23]], i32 0 1105; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = insertelement <2 x float> [[TMP25]], float [[TMP24]], i32 1 1106; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = load float, ptr [[TMP21]], align 4 1107; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = load float, ptr [[TMP22]], align 4 1108; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = insertelement <2 x float> poison, float [[TMP27]], i32 0 1109; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = insertelement <2 x float> [[TMP29]], float [[TMP28]], i32 1 1110; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = fadd fast <2 x float> [[VEC_PHI]], splat (float 1.000000e+00) 1111; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = fadd fast <2 x float> [[VEC_PHI1]], splat (float 1.000000e+00) 1112; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = fadd fast <2 x float> [[TMP31]], [[TMP14]] 1113; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = fadd fast <2 x float> [[TMP32]], [[TMP18]] 1114; UNROLL-NO-IC-NEXT: [[TMP35]] = fadd fast <2 x float> [[TMP33]], [[TMP26]] 1115; UNROLL-NO-IC-NEXT: [[TMP36]] = fadd fast <2 x float> [[TMP34]], [[TMP30]] 1116; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 1117; UNROLL-NO-IC-NEXT: [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1118; UNROLL-NO-IC-NEXT: br i1 [[TMP37]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] 1119; UNROLL-NO-IC: middle.block: 1120; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = fadd fast <2 x float> [[TMP36]], [[TMP35]] 1121; UNROLL-NO-IC-NEXT: [[TMP38:%.*]] = call fast float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> [[BIN_RDX]]) 1122; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 1123; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1124; UNROLL-NO-IC: scalar.ph: 1125; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1126; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP38]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ] 1127; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 1128; UNROLL-NO-IC: for.body: 1129; UNROLL-NO-IC-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_BODY]] ] 1130; UNROLL-NO-IC-NEXT: [[S:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP45:%.*]], [[FOR_BODY]] ] 1131; UNROLL-NO-IC-NEXT: [[TMP39:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[I]] 1132; UNROLL-NO-IC-NEXT: [[TMP40:%.*]] = load float, ptr [[TMP39]], align 4 1133; UNROLL-NO-IC-NEXT: [[TMP41:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[I]] 1134; UNROLL-NO-IC-NEXT: [[TMP42:%.*]] = load float, ptr [[TMP41]], align 4 1135; UNROLL-NO-IC-NEXT: [[TMP43:%.*]] = fadd fast float [[S]], 1.000000e+00 1136; UNROLL-NO-IC-NEXT: [[TMP44:%.*]] = fadd fast float [[TMP43]], [[TMP40]] 1137; UNROLL-NO-IC-NEXT: [[TMP45]] = fadd fast float [[TMP44]], [[TMP42]] 1138; UNROLL-NO-IC-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 8 1139; UNROLL-NO-IC-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 1140; UNROLL-NO-IC-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]] 1141; UNROLL-NO-IC: for.end: 1142; UNROLL-NO-IC-NEXT: [[S_LCSSA:%.*]] = phi float [ [[TMP45]], [[FOR_BODY]] ], [ [[TMP38]], [[MIDDLE_BLOCK]] ] 1143; UNROLL-NO-IC-NEXT: ret float [[S_LCSSA]] 1144; 1145; INTERLEAVE-LABEL: @scalarize_induction_variable_02( 1146; INTERLEAVE-NEXT: entry: 1147; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 65 1148; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1149; INTERLEAVE: vector.ph: 1150; INTERLEAVE-NEXT: [[TMP0:%.*]] = add nsw i64 [[N]], -1 1151; INTERLEAVE-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 3 1152; INTERLEAVE-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 1153; INTERLEAVE-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP2]], 7 1154; INTERLEAVE-NEXT: [[TMP3:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 1155; INTERLEAVE-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i64 8, i64 [[N_MOD_VF]] 1156; INTERLEAVE-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[TMP2]], [[TMP4]] 1157; INTERLEAVE-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 3 1158; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 1159; INTERLEAVE: vector.body: 1160; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1161; INTERLEAVE-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ] 1162; INTERLEAVE-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ] 1163; INTERLEAVE-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 3 1164; INTERLEAVE-NEXT: [[TMP5:%.*]] = or disjoint i64 [[OFFSET_IDX]], 32 1165; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[OFFSET_IDX]] 1166; INTERLEAVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP5]] 1167; INTERLEAVE-NEXT: [[WIDE_VEC:%.*]] = load <32 x float>, ptr [[TMP6]], align 4 1168; INTERLEAVE-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <32 x float> [[WIDE_VEC]], <32 x float> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24> 1169; INTERLEAVE-NEXT: [[WIDE_VEC2:%.*]] = load <32 x float>, ptr [[TMP7]], align 4 1170; INTERLEAVE-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <32 x float> [[WIDE_VEC2]], <32 x float> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24> 1171; INTERLEAVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[OFFSET_IDX]] 1172; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP5]] 1173; INTERLEAVE-NEXT: [[WIDE_VEC4:%.*]] = load <32 x float>, ptr [[TMP8]], align 4 1174; INTERLEAVE-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <32 x float> [[WIDE_VEC4]], <32 x float> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24> 1175; INTERLEAVE-NEXT: [[WIDE_VEC6:%.*]] = load <32 x float>, ptr [[TMP9]], align 4 1176; INTERLEAVE-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <32 x float> [[WIDE_VEC6]], <32 x float> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24> 1177; INTERLEAVE-NEXT: [[TMP10:%.*]] = fadd fast <4 x float> [[VEC_PHI]], splat (float 1.000000e+00) 1178; INTERLEAVE-NEXT: [[TMP11:%.*]] = fadd fast <4 x float> [[VEC_PHI1]], splat (float 1.000000e+00) 1179; INTERLEAVE-NEXT: [[TMP12:%.*]] = fadd fast <4 x float> [[TMP10]], [[STRIDED_VEC]] 1180; INTERLEAVE-NEXT: [[TMP13:%.*]] = fadd fast <4 x float> [[TMP11]], [[STRIDED_VEC3]] 1181; INTERLEAVE-NEXT: [[TMP14]] = fadd fast <4 x float> [[TMP12]], [[STRIDED_VEC5]] 1182; INTERLEAVE-NEXT: [[TMP15]] = fadd fast <4 x float> [[TMP13]], [[STRIDED_VEC7]] 1183; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 1184; INTERLEAVE-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1185; INTERLEAVE-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] 1186; INTERLEAVE: middle.block: 1187; INTERLEAVE-NEXT: [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP15]], [[TMP14]] 1188; INTERLEAVE-NEXT: [[TMP17:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[BIN_RDX]]) 1189; INTERLEAVE-NEXT: br label [[SCALAR_PH]] 1190; INTERLEAVE: scalar.ph: 1191; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1192; INTERLEAVE-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP17]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ] 1193; INTERLEAVE-NEXT: br label [[FOR_BODY:%.*]] 1194; INTERLEAVE: for.body: 1195; INTERLEAVE-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_BODY]] ] 1196; INTERLEAVE-NEXT: [[S:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP24:%.*]], [[FOR_BODY]] ] 1197; INTERLEAVE-NEXT: [[TMP18:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[I]] 1198; INTERLEAVE-NEXT: [[TMP19:%.*]] = load float, ptr [[TMP18]], align 4 1199; INTERLEAVE-NEXT: [[TMP20:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[I]] 1200; INTERLEAVE-NEXT: [[TMP21:%.*]] = load float, ptr [[TMP20]], align 4 1201; INTERLEAVE-NEXT: [[TMP22:%.*]] = fadd fast float [[S]], 1.000000e+00 1202; INTERLEAVE-NEXT: [[TMP23:%.*]] = fadd fast float [[TMP22]], [[TMP19]] 1203; INTERLEAVE-NEXT: [[TMP24]] = fadd fast float [[TMP23]], [[TMP21]] 1204; INTERLEAVE-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 8 1205; INTERLEAVE-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 1206; INTERLEAVE-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP14:![0-9]+]] 1207; INTERLEAVE: for.end: 1208; INTERLEAVE-NEXT: ret float [[TMP24]] 1209; 1210entry: 1211 br label %for.body 1212 1213for.body: 1214 %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ] 1215 %s = phi float [ 0.0, %entry ], [ %6, %for.body ] 1216 %0 = getelementptr inbounds float, ptr %a, i64 %i 1217 %1 = load float, ptr %0, align 4 1218 %2 = getelementptr inbounds float, ptr %b, i64 %i 1219 %3 = load float, ptr %2, align 4 1220 %4 = fadd fast float %s, 1.0 1221 %5 = fadd fast float %4, %1 1222 %6 = fadd fast float %5, %3 1223 %i.next = add nuw nsw i64 %i, 8 1224 %cond = icmp slt i64 %i.next, %n 1225 br i1 %cond, label %for.body, label %for.end 1226 1227for.end: 1228 %s.lcssa = phi float [ %6, %for.body ] 1229 ret float %s.lcssa 1230} 1231 1232; Make sure we scalarize the step vectors used for the pointer arithmetic. We 1233; can't easily simplify vectorized step vectors. (Interleaved accesses.) 1234; 1235; for (int i = 0; i < n; ++i) 1236; a[i].f ^= y; 1237; 1238 1239%pair.i32 = type { i32, i32 } 1240define void @scalarize_induction_variable_03(ptr %p, i32 %y, i64 %n) { 1241; CHECK-LABEL: @scalarize_induction_variable_03( 1242; CHECK-NEXT: entry: 1243; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) 1244; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 2 1245; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1246; CHECK: vector.ph: 1247; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 2 1248; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]] 1249; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i64 0 1250; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 1251; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 1252; CHECK: vector.body: 1253; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1254; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 1255; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1 1256; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], i64 [[TMP0]], i32 1 1257; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP1]], i32 1 1258; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 8 1259; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 8 1260; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i32 0 1261; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP5]], i32 1 1262; CHECK-NEXT: [[TMP8:%.*]] = xor <2 x i32> [[TMP7]], [[BROADCAST_SPLAT]] 1263; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP8]], i32 0 1264; CHECK-NEXT: store i32 [[TMP9]], ptr [[TMP2]], align 8 1265; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[TMP8]], i32 1 1266; CHECK-NEXT: store i32 [[TMP10]], ptr [[TMP3]], align 8 1267; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 1268; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1269; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] 1270; CHECK: middle.block: 1271; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 1272; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1273; CHECK: scalar.ph: 1274; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1275; CHECK-NEXT: br label [[FOR_BODY:%.*]] 1276; CHECK: for.body: 1277; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1278; CHECK-NEXT: [[F:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1 1279; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[F]], align 8 1280; CHECK-NEXT: [[TMP13:%.*]] = xor i32 [[TMP12]], [[Y]] 1281; CHECK-NEXT: store i32 [[TMP13]], ptr [[F]], align 8 1282; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1283; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 1284; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP16:![0-9]+]] 1285; CHECK: for.end: 1286; CHECK-NEXT: ret void 1287; 1288; IND-LABEL: @scalarize_induction_variable_03( 1289; IND-NEXT: entry: 1290; IND-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) 1291; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 2 1292; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1293; IND: vector.ph: 1294; IND-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775806 1295; IND-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i64 0 1296; IND-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 1297; IND-NEXT: br label [[VECTOR_BODY:%.*]] 1298; IND: vector.body: 1299; IND-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1300; IND-NEXT: [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 1 1301; IND-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 1 1302; IND-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP0]], i32 1 1303; IND-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP1]], align 8 1304; IND-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 8 1305; IND-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i64 0 1306; IND-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP4]], i64 1 1307; IND-NEXT: [[TMP7:%.*]] = xor <2 x i32> [[TMP6]], [[BROADCAST_SPLAT]] 1308; IND-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[TMP7]], i64 0 1309; IND-NEXT: store i32 [[TMP8]], ptr [[TMP1]], align 8 1310; IND-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP7]], i64 1 1311; IND-NEXT: store i32 [[TMP9]], ptr [[TMP2]], align 8 1312; IND-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 1313; IND-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1314; IND-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] 1315; IND: middle.block: 1316; IND-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 1317; IND-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1318; IND: scalar.ph: 1319; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1320; IND-NEXT: br label [[FOR_BODY:%.*]] 1321; IND: for.body: 1322; IND-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1323; IND-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1 1324; IND-NEXT: [[TMP11:%.*]] = load i32, ptr [[F]], align 8 1325; IND-NEXT: [[TMP12:%.*]] = xor i32 [[TMP11]], [[Y]] 1326; IND-NEXT: store i32 [[TMP12]], ptr [[F]], align 8 1327; IND-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1328; IND-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 1329; IND-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP16:![0-9]+]] 1330; IND: for.end: 1331; IND-NEXT: ret void 1332; 1333; UNROLL-LABEL: @scalarize_induction_variable_03( 1334; UNROLL-NEXT: entry: 1335; UNROLL-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) 1336; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 1337; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1338; UNROLL: vector.ph: 1339; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804 1340; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i64 0 1341; UNROLL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 1342; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 1343; UNROLL: vector.body: 1344; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1345; UNROLL-NEXT: [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 1 1346; UNROLL-NEXT: [[TMP1:%.*]] = or disjoint i64 [[INDEX]], 2 1347; UNROLL-NEXT: [[TMP2:%.*]] = or disjoint i64 [[INDEX]], 3 1348; UNROLL-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 1 1349; UNROLL-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP0]], i32 1 1350; UNROLL-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP1]], i32 1 1351; UNROLL-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP2]], i32 1 1352; UNROLL-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP3]], align 8 1353; UNROLL-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 8 1354; UNROLL-NEXT: [[TMP9:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0 1355; UNROLL-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP8]], i64 1 1356; UNROLL-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP5]], align 8 1357; UNROLL-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP6]], align 8 1358; UNROLL-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> poison, i32 [[TMP11]], i64 0 1359; UNROLL-NEXT: [[TMP14:%.*]] = insertelement <2 x i32> [[TMP13]], i32 [[TMP12]], i64 1 1360; UNROLL-NEXT: [[TMP15:%.*]] = xor <2 x i32> [[TMP10]], [[BROADCAST_SPLAT]] 1361; UNROLL-NEXT: [[TMP16:%.*]] = xor <2 x i32> [[TMP14]], [[BROADCAST_SPLAT]] 1362; UNROLL-NEXT: [[TMP17:%.*]] = extractelement <2 x i32> [[TMP15]], i64 0 1363; UNROLL-NEXT: store i32 [[TMP17]], ptr [[TMP3]], align 8 1364; UNROLL-NEXT: [[TMP18:%.*]] = extractelement <2 x i32> [[TMP15]], i64 1 1365; UNROLL-NEXT: store i32 [[TMP18]], ptr [[TMP4]], align 8 1366; UNROLL-NEXT: [[TMP19:%.*]] = extractelement <2 x i32> [[TMP16]], i64 0 1367; UNROLL-NEXT: store i32 [[TMP19]], ptr [[TMP5]], align 8 1368; UNROLL-NEXT: [[TMP20:%.*]] = extractelement <2 x i32> [[TMP16]], i64 1 1369; UNROLL-NEXT: store i32 [[TMP20]], ptr [[TMP6]], align 8 1370; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 1371; UNROLL-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1372; UNROLL-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] 1373; UNROLL: middle.block: 1374; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 1375; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1376; UNROLL: scalar.ph: 1377; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1378; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 1379; UNROLL: for.body: 1380; UNROLL-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1381; UNROLL-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1 1382; UNROLL-NEXT: [[TMP22:%.*]] = load i32, ptr [[F]], align 8 1383; UNROLL-NEXT: [[TMP23:%.*]] = xor i32 [[TMP22]], [[Y]] 1384; UNROLL-NEXT: store i32 [[TMP23]], ptr [[F]], align 8 1385; UNROLL-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1386; UNROLL-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 1387; UNROLL-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP16:![0-9]+]] 1388; UNROLL: for.end: 1389; UNROLL-NEXT: ret void 1390; 1391; UNROLL-NO-IC-LABEL: @scalarize_induction_variable_03( 1392; UNROLL-NO-IC-NEXT: entry: 1393; UNROLL-NO-IC-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) 1394; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4 1395; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1396; UNROLL-NO-IC: vector.ph: 1397; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4 1398; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]] 1399; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i64 0 1400; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 1401; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 1402; UNROLL-NO-IC: vector.body: 1403; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1404; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 1405; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1 1406; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2 1407; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3 1408; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], i64 [[TMP0]], i32 1 1409; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP1]], i32 1 1410; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP2]], i32 1 1411; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP3]], i32 1 1412; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 8 1413; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 8 1414; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> poison, i32 [[TMP8]], i32 0 1415; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP10]], i32 [[TMP9]], i32 1 1416; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP6]], align 8 1417; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP7]], align 8 1418; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = insertelement <2 x i32> poison, i32 [[TMP12]], i32 0 1419; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = insertelement <2 x i32> [[TMP14]], i32 [[TMP13]], i32 1 1420; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = xor <2 x i32> [[TMP11]], [[BROADCAST_SPLAT]] 1421; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = xor <2 x i32> [[TMP15]], [[BROADCAST_SPLAT]] 1422; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = extractelement <2 x i32> [[TMP16]], i32 0 1423; UNROLL-NO-IC-NEXT: store i32 [[TMP18]], ptr [[TMP4]], align 8 1424; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = extractelement <2 x i32> [[TMP16]], i32 1 1425; UNROLL-NO-IC-NEXT: store i32 [[TMP19]], ptr [[TMP5]], align 8 1426; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = extractelement <2 x i32> [[TMP17]], i32 0 1427; UNROLL-NO-IC-NEXT: store i32 [[TMP20]], ptr [[TMP6]], align 8 1428; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = extractelement <2 x i32> [[TMP17]], i32 1 1429; UNROLL-NO-IC-NEXT: store i32 [[TMP21]], ptr [[TMP7]], align 8 1430; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 1431; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1432; UNROLL-NO-IC-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] 1433; UNROLL-NO-IC: middle.block: 1434; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 1435; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1436; UNROLL-NO-IC: scalar.ph: 1437; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1438; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 1439; UNROLL-NO-IC: for.body: 1440; UNROLL-NO-IC-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1441; UNROLL-NO-IC-NEXT: [[F:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1 1442; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = load i32, ptr [[F]], align 8 1443; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = xor i32 [[TMP23]], [[Y]] 1444; UNROLL-NO-IC-NEXT: store i32 [[TMP24]], ptr [[F]], align 8 1445; UNROLL-NO-IC-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1446; UNROLL-NO-IC-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 1447; UNROLL-NO-IC-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP16:![0-9]+]] 1448; UNROLL-NO-IC: for.end: 1449; UNROLL-NO-IC-NEXT: ret void 1450; 1451; INTERLEAVE-LABEL: @scalarize_induction_variable_03( 1452; INTERLEAVE-NEXT: entry: 1453; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 9 1454; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1455; INTERLEAVE: vector.ph: 1456; INTERLEAVE-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 7 1457; INTERLEAVE-NEXT: [[TMP0:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 1458; INTERLEAVE-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i64 8, i64 [[N_MOD_VF]] 1459; INTERLEAVE-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[TMP1]] 1460; INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[Y:%.*]], i64 0 1461; INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 1462; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 1463; INTERLEAVE: vector.body: 1464; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1465; INTERLEAVE-NEXT: [[TMP2:%.*]] = or disjoint i64 [[INDEX]], 1 1466; INTERLEAVE-NEXT: [[TMP3:%.*]] = or disjoint i64 [[INDEX]], 2 1467; INTERLEAVE-NEXT: [[TMP4:%.*]] = or disjoint i64 [[INDEX]], 3 1468; INTERLEAVE-NEXT: [[TMP5:%.*]] = or disjoint i64 [[INDEX]], 4 1469; INTERLEAVE-NEXT: [[TMP6:%.*]] = or disjoint i64 [[INDEX]], 5 1470; INTERLEAVE-NEXT: [[TMP7:%.*]] = or disjoint i64 [[INDEX]], 6 1471; INTERLEAVE-NEXT: [[TMP8:%.*]] = or disjoint i64 [[INDEX]], 7 1472; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 1 1473; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP2]], i32 1 1474; INTERLEAVE-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP3]], i32 1 1475; INTERLEAVE-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP4]], i32 1 1476; INTERLEAVE-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP5]], i32 1 1477; INTERLEAVE-NEXT: [[TMP14:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP6]], i32 1 1478; INTERLEAVE-NEXT: [[TMP15:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP7]], i32 1 1479; INTERLEAVE-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP8]], i32 1 1480; INTERLEAVE-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP9]], align 8 1481; INTERLEAVE-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 1482; INTERLEAVE-NEXT: [[WIDE_VEC1:%.*]] = load <8 x i32>, ptr [[TMP13]], align 8 1483; INTERLEAVE-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> 1484; INTERLEAVE-NEXT: [[TMP17:%.*]] = xor <4 x i32> [[STRIDED_VEC]], [[BROADCAST_SPLAT]] 1485; INTERLEAVE-NEXT: [[TMP18:%.*]] = xor <4 x i32> [[STRIDED_VEC2]], [[BROADCAST_SPLAT]] 1486; INTERLEAVE-NEXT: [[TMP19:%.*]] = extractelement <4 x i32> [[TMP17]], i64 0 1487; INTERLEAVE-NEXT: store i32 [[TMP19]], ptr [[TMP9]], align 8 1488; INTERLEAVE-NEXT: [[TMP20:%.*]] = extractelement <4 x i32> [[TMP17]], i64 1 1489; INTERLEAVE-NEXT: store i32 [[TMP20]], ptr [[TMP10]], align 8 1490; INTERLEAVE-NEXT: [[TMP21:%.*]] = extractelement <4 x i32> [[TMP17]], i64 2 1491; INTERLEAVE-NEXT: store i32 [[TMP21]], ptr [[TMP11]], align 8 1492; INTERLEAVE-NEXT: [[TMP22:%.*]] = extractelement <4 x i32> [[TMP17]], i64 3 1493; INTERLEAVE-NEXT: store i32 [[TMP22]], ptr [[TMP12]], align 8 1494; INTERLEAVE-NEXT: [[TMP23:%.*]] = extractelement <4 x i32> [[TMP18]], i64 0 1495; INTERLEAVE-NEXT: store i32 [[TMP23]], ptr [[TMP13]], align 8 1496; INTERLEAVE-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[TMP18]], i64 1 1497; INTERLEAVE-NEXT: store i32 [[TMP24]], ptr [[TMP14]], align 8 1498; INTERLEAVE-NEXT: [[TMP25:%.*]] = extractelement <4 x i32> [[TMP18]], i64 2 1499; INTERLEAVE-NEXT: store i32 [[TMP25]], ptr [[TMP15]], align 8 1500; INTERLEAVE-NEXT: [[TMP26:%.*]] = extractelement <4 x i32> [[TMP18]], i64 3 1501; INTERLEAVE-NEXT: store i32 [[TMP26]], ptr [[TMP16]], align 8 1502; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 1503; INTERLEAVE-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1504; INTERLEAVE-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] 1505; INTERLEAVE: middle.block: 1506; INTERLEAVE-NEXT: br label [[SCALAR_PH]] 1507; INTERLEAVE: scalar.ph: 1508; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 1509; INTERLEAVE-NEXT: br label [[FOR_BODY:%.*]] 1510; INTERLEAVE: for.body: 1511; INTERLEAVE-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1512; INTERLEAVE-NEXT: [[F:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1 1513; INTERLEAVE-NEXT: [[TMP28:%.*]] = load i32, ptr [[F]], align 8 1514; INTERLEAVE-NEXT: [[TMP29:%.*]] = xor i32 [[TMP28]], [[Y]] 1515; INTERLEAVE-NEXT: store i32 [[TMP29]], ptr [[F]], align 8 1516; INTERLEAVE-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1517; INTERLEAVE-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 1518; INTERLEAVE-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP16:![0-9]+]] 1519; INTERLEAVE: for.end: 1520; INTERLEAVE-NEXT: ret void 1521; 1522entry: 1523 br label %for.body 1524 1525for.body: 1526 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 1527 %f = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 1 1528 %0 = load i32, ptr %f, align 8 1529 %1 = xor i32 %0, %y 1530 store i32 %1, ptr %f, align 8 1531 %i.next = add nuw nsw i64 %i, 1 1532 %cond = icmp slt i64 %i.next, %n 1533 br i1 %cond, label %for.body, label %for.end 1534 1535for.end: 1536 ret void 1537} 1538 1539; Make sure we scalarize the step vectors used for the pointer arithmetic. We 1540; can't easily simplify vectorized step vectors. (Interleaved accesses.) 1541; 1542; for (int i = 0; i < n; ++i) 1543; p[i].f = a[i * 4] 1544; 1545 1546define void @scalarize_induction_variable_04(ptr %a, ptr %p, i32 %n) { 1547; CHECK-LABEL: @scalarize_induction_variable_04( 1548; CHECK-NEXT: entry: 1549; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 1550; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 1551; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 1552; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2 1553; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 1554; CHECK: vector.memcheck: 1555; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P:%.*]], i64 4 1556; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[N]], -1 1557; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 1558; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3 1559; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 8 1560; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP6]] 1561; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP4]], 4 1562; CHECK-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[TMP7]], 4 1563; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP8]] 1564; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]] 1565; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]] 1566; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 1567; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 1568; CHECK: vector.ph: 1569; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2 1570; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 1571; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 1572; CHECK: vector.body: 1573; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1574; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 1575; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 0 1576; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 1 1577; CHECK-NEXT: [[TMP11:%.*]] = shl nsw <2 x i64> [[VEC_IND]], splat (i64 2) 1578; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i64> [[TMP11]], i32 0 1579; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP12]] 1580; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i64> [[TMP11]], i32 1 1581; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP14]] 1582; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP13]], align 1, !alias.scope [[META17:![0-9]+]] 1583; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP15]], align 1, !alias.scope [[META17]] 1584; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P]], i64 [[TMP9]], i32 1 1585; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP10]], i32 1 1586; CHECK-NEXT: store i32 [[TMP16]], ptr [[TMP18]], align 1, !alias.scope [[META20:![0-9]+]], !noalias [[META17]] 1587; CHECK-NEXT: store i32 [[TMP17]], ptr [[TMP19]], align 1, !alias.scope [[META20]], !noalias [[META17]] 1588; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 1589; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2) 1590; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1591; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 1592; CHECK: middle.block: 1593; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 1594; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1595; CHECK: scalar.ph: 1596; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ] 1597; CHECK-NEXT: br label [[FOR_BODY:%.*]] 1598; CHECK: for.body: 1599; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1600; CHECK-NEXT: [[TMP21:%.*]] = shl nsw i64 [[I]], 2 1601; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP21]] 1602; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 1 1603; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1 1604; CHECK-NEXT: store i32 [[TMP23]], ptr [[TMP24]], align 1 1605; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1606; CHECK-NEXT: [[TMP25:%.*]] = trunc i64 [[I_NEXT]] to i32 1607; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[TMP25]], [[N]] 1608; CHECK-NEXT: br i1 [[COND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 1609; CHECK: for.end: 1610; CHECK-NEXT: ret void 1611; 1612; IND-LABEL: @scalarize_induction_variable_04( 1613; IND-NEXT: entry: 1614; IND-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 1615; IND-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 1616; IND-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 1617; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[TMP0]], 0 1618; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 1619; IND: vector.memcheck: 1620; IND-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P:%.*]], i64 4 1621; IND-NEXT: [[TMP3:%.*]] = add i32 [[N]], -1 1622; IND-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 1623; IND-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3 1624; IND-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]] 1625; IND-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8 1626; IND-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP4]], 4 1627; IND-NEXT: [[TMP8:%.*]] = or disjoint i64 [[TMP7]], 4 1628; IND-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP8]] 1629; IND-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]] 1630; IND-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]] 1631; IND-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 1632; IND-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 1633; IND: vector.ph: 1634; IND-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934590 1635; IND-NEXT: br label [[VECTOR_BODY:%.*]] 1636; IND: vector.body: 1637; IND-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1638; IND-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 1639; IND-NEXT: [[TMP9:%.*]] = or disjoint i64 [[INDEX]], 1 1640; IND-NEXT: [[TMP10:%.*]] = shl nsw <2 x i64> [[VEC_IND]], splat (i64 2) 1641; IND-NEXT: [[TMP11:%.*]] = extractelement <2 x i64> [[TMP10]], i64 0 1642; IND-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]] 1643; IND-NEXT: [[TMP13:%.*]] = extractelement <2 x i64> [[TMP10]], i64 1 1644; IND-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP13]] 1645; IND-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP12]], align 1, !alias.scope [[META17:![0-9]+]] 1646; IND-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP14]], align 1, !alias.scope [[META17]] 1647; IND-NEXT: [[TMP17:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P]], i64 [[INDEX]], i32 1 1648; IND-NEXT: [[TMP18:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP9]], i32 1 1649; IND-NEXT: store i32 [[TMP15]], ptr [[TMP17]], align 1, !alias.scope [[META20:![0-9]+]], !noalias [[META17]] 1650; IND-NEXT: store i32 [[TMP16]], ptr [[TMP18]], align 1, !alias.scope [[META20]], !noalias [[META17]] 1651; IND-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 1652; IND-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2) 1653; IND-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1654; IND-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 1655; IND: middle.block: 1656; IND-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 1657; IND-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1658; IND: scalar.ph: 1659; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ] 1660; IND-NEXT: br label [[FOR_BODY:%.*]] 1661; IND: for.body: 1662; IND-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1663; IND-NEXT: [[DOTIDX:%.*]] = shl nsw i64 [[I]], 4 1664; IND-NEXT: [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[DOTIDX]] 1665; IND-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 1 1666; IND-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1 1667; IND-NEXT: store i32 [[TMP21]], ptr [[TMP22]], align 1 1668; IND-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1669; IND-NEXT: [[TMP23:%.*]] = trunc i64 [[I_NEXT]] to i32 1670; IND-NEXT: [[COND:%.*]] = icmp eq i32 [[N]], [[TMP23]] 1671; IND-NEXT: br i1 [[COND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 1672; IND: for.end: 1673; IND-NEXT: ret void 1674; 1675; UNROLL-LABEL: @scalarize_induction_variable_04( 1676; UNROLL-NEXT: entry: 1677; UNROLL-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 1678; UNROLL-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 1679; UNROLL-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 1680; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 3 1681; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 1682; UNROLL: vector.memcheck: 1683; UNROLL-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P:%.*]], i64 4 1684; UNROLL-NEXT: [[TMP3:%.*]] = add i32 [[N]], -1 1685; UNROLL-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 1686; UNROLL-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3 1687; UNROLL-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]] 1688; UNROLL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8 1689; UNROLL-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP4]], 4 1690; UNROLL-NEXT: [[TMP8:%.*]] = or disjoint i64 [[TMP7]], 4 1691; UNROLL-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP8]] 1692; UNROLL-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]] 1693; UNROLL-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]] 1694; UNROLL-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 1695; UNROLL-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 1696; UNROLL: vector.ph: 1697; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934588 1698; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 1699; UNROLL: vector.body: 1700; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1701; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 1702; UNROLL-NEXT: [[TMP9:%.*]] = or disjoint i64 [[INDEX]], 1 1703; UNROLL-NEXT: [[TMP10:%.*]] = or disjoint i64 [[INDEX]], 2 1704; UNROLL-NEXT: [[TMP11:%.*]] = or disjoint i64 [[INDEX]], 3 1705; UNROLL-NEXT: [[TMP12:%.*]] = shl nsw <2 x i64> [[VEC_IND]], splat (i64 2) 1706; UNROLL-NEXT: [[STEP_ADD:%.*]] = shl <2 x i64> [[VEC_IND]], splat (i64 2) 1707; UNROLL-NEXT: [[TMP13:%.*]] = add <2 x i64> [[STEP_ADD]], splat (i64 8) 1708; UNROLL-NEXT: [[TMP14:%.*]] = extractelement <2 x i64> [[TMP12]], i64 0 1709; UNROLL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP14]] 1710; UNROLL-NEXT: [[TMP16:%.*]] = extractelement <2 x i64> [[TMP12]], i64 1 1711; UNROLL-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP16]] 1712; UNROLL-NEXT: [[TMP18:%.*]] = extractelement <2 x i64> [[TMP13]], i64 0 1713; UNROLL-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP18]] 1714; UNROLL-NEXT: [[TMP20:%.*]] = extractelement <2 x i64> [[TMP13]], i64 1 1715; UNROLL-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP20]] 1716; UNROLL-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP15]], align 1, !alias.scope [[META17:![0-9]+]] 1717; UNROLL-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP17]], align 1, !alias.scope [[META17]] 1718; UNROLL-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP19]], align 1, !alias.scope [[META17]] 1719; UNROLL-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP21]], align 1, !alias.scope [[META17]] 1720; UNROLL-NEXT: [[TMP26:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P]], i64 [[INDEX]], i32 1 1721; UNROLL-NEXT: [[TMP27:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP9]], i32 1 1722; UNROLL-NEXT: [[TMP28:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP10]], i32 1 1723; UNROLL-NEXT: [[TMP29:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP11]], i32 1 1724; UNROLL-NEXT: store i32 [[TMP22]], ptr [[TMP26]], align 1, !alias.scope [[META20:![0-9]+]], !noalias [[META17]] 1725; UNROLL-NEXT: store i32 [[TMP23]], ptr [[TMP27]], align 1, !alias.scope [[META20]], !noalias [[META17]] 1726; UNROLL-NEXT: store i32 [[TMP24]], ptr [[TMP28]], align 1, !alias.scope [[META20]], !noalias [[META17]] 1727; UNROLL-NEXT: store i32 [[TMP25]], ptr [[TMP29]], align 1, !alias.scope [[META20]], !noalias [[META17]] 1728; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 1729; UNROLL-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 4) 1730; UNROLL-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1731; UNROLL-NEXT: br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 1732; UNROLL: middle.block: 1733; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 1734; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1735; UNROLL: scalar.ph: 1736; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ] 1737; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 1738; UNROLL: for.body: 1739; UNROLL-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1740; UNROLL-NEXT: [[DOTIDX:%.*]] = shl nsw i64 [[I]], 4 1741; UNROLL-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[DOTIDX]] 1742; UNROLL-NEXT: [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 1 1743; UNROLL-NEXT: [[TMP33:%.*]] = getelementptr inbounds nuw [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1 1744; UNROLL-NEXT: store i32 [[TMP32]], ptr [[TMP33]], align 1 1745; UNROLL-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1746; UNROLL-NEXT: [[TMP34:%.*]] = trunc i64 [[I_NEXT]] to i32 1747; UNROLL-NEXT: [[COND:%.*]] = icmp eq i32 [[N]], [[TMP34]] 1748; UNROLL-NEXT: br i1 [[COND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 1749; UNROLL: for.end: 1750; UNROLL-NEXT: ret void 1751; 1752; UNROLL-NO-IC-LABEL: @scalarize_induction_variable_04( 1753; UNROLL-NO-IC-NEXT: entry: 1754; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 1755; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 1756; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 1757; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 1758; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 1759; UNROLL-NO-IC: vector.memcheck: 1760; UNROLL-NO-IC-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P:%.*]], i64 4 1761; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i32 [[N]], -1 1762; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 1763; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3 1764; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 8 1765; UNROLL-NO-IC-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP6]] 1766; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP4]], 4 1767; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[TMP7]], 4 1768; UNROLL-NO-IC-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP8]] 1769; UNROLL-NO-IC-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]] 1770; UNROLL-NO-IC-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]] 1771; UNROLL-NO-IC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 1772; UNROLL-NO-IC-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 1773; UNROLL-NO-IC: vector.ph: 1774; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 1775; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 1776; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 1777; UNROLL-NO-IC: vector.body: 1778; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1779; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 1780; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 2) 1781; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 0 1782; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 1 1783; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 2 1784; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 3 1785; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = shl nsw <2 x i64> [[VEC_IND]], splat (i64 2) 1786; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = shl nsw <2 x i64> [[STEP_ADD]], splat (i64 2) 1787; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = extractelement <2 x i64> [[TMP13]], i32 0 1788; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP15]] 1789; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = extractelement <2 x i64> [[TMP13]], i32 1 1790; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP17]] 1791; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = extractelement <2 x i64> [[TMP14]], i32 0 1792; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP19]] 1793; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = extractelement <2 x i64> [[TMP14]], i32 1 1794; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP21]] 1795; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP16]], align 1, !alias.scope [[META17:![0-9]+]] 1796; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP18]], align 1, !alias.scope [[META17]] 1797; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP20]], align 1, !alias.scope [[META17]] 1798; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP22]], align 1, !alias.scope [[META17]] 1799; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P]], i64 [[TMP9]], i32 1 1800; UNROLL-NO-IC-NEXT: [[TMP28:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP10]], i32 1 1801; UNROLL-NO-IC-NEXT: [[TMP29:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP11]], i32 1 1802; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP12]], i32 1 1803; UNROLL-NO-IC-NEXT: store i32 [[TMP23]], ptr [[TMP27]], align 1, !alias.scope [[META20:![0-9]+]], !noalias [[META17]] 1804; UNROLL-NO-IC-NEXT: store i32 [[TMP24]], ptr [[TMP28]], align 1, !alias.scope [[META20]], !noalias [[META17]] 1805; UNROLL-NO-IC-NEXT: store i32 [[TMP25]], ptr [[TMP29]], align 1, !alias.scope [[META20]], !noalias [[META17]] 1806; UNROLL-NO-IC-NEXT: store i32 [[TMP26]], ptr [[TMP30]], align 1, !alias.scope [[META20]], !noalias [[META17]] 1807; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 1808; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[STEP_ADD]], splat (i64 2) 1809; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1810; UNROLL-NO-IC-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 1811; UNROLL-NO-IC: middle.block: 1812; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 1813; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 1814; UNROLL-NO-IC: scalar.ph: 1815; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ] 1816; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 1817; UNROLL-NO-IC: for.body: 1818; UNROLL-NO-IC-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1819; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = shl nsw i64 [[I]], 2 1820; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP32]] 1821; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = load i32, ptr [[TMP33]], align 1 1822; UNROLL-NO-IC-NEXT: [[TMP35:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1 1823; UNROLL-NO-IC-NEXT: store i32 [[TMP34]], ptr [[TMP35]], align 1 1824; UNROLL-NO-IC-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1825; UNROLL-NO-IC-NEXT: [[TMP36:%.*]] = trunc i64 [[I_NEXT]] to i32 1826; UNROLL-NO-IC-NEXT: [[COND:%.*]] = icmp eq i32 [[TMP36]], [[N]] 1827; UNROLL-NO-IC-NEXT: br i1 [[COND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 1828; UNROLL-NO-IC: for.end: 1829; UNROLL-NO-IC-NEXT: ret void 1830; 1831; INTERLEAVE-LABEL: @scalarize_induction_variable_04( 1832; INTERLEAVE-NEXT: entry: 1833; INTERLEAVE-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 1834; INTERLEAVE-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 1835; INTERLEAVE-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 1836; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 8 1837; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] 1838; INTERLEAVE: vector.memcheck: 1839; INTERLEAVE-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P:%.*]], i64 4 1840; INTERLEAVE-NEXT: [[TMP3:%.*]] = add i32 [[N]], -1 1841; INTERLEAVE-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 1842; INTERLEAVE-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3 1843; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]] 1844; INTERLEAVE-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8 1845; INTERLEAVE-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP4]], 4 1846; INTERLEAVE-NEXT: [[TMP8:%.*]] = or disjoint i64 [[TMP7]], 4 1847; INTERLEAVE-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP8]] 1848; INTERLEAVE-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]] 1849; INTERLEAVE-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]] 1850; INTERLEAVE-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] 1851; INTERLEAVE-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 1852; INTERLEAVE: vector.ph: 1853; INTERLEAVE-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP2]], 7 1854; INTERLEAVE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 1855; INTERLEAVE-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i64 8, i64 [[N_MOD_VF]] 1856; INTERLEAVE-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[TMP2]], [[TMP10]] 1857; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 1858; INTERLEAVE: vector.body: 1859; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 1860; INTERLEAVE-NEXT: [[TMP11:%.*]] = or disjoint i64 [[INDEX]], 1 1861; INTERLEAVE-NEXT: [[TMP12:%.*]] = or disjoint i64 [[INDEX]], 2 1862; INTERLEAVE-NEXT: [[TMP13:%.*]] = or disjoint i64 [[INDEX]], 3 1863; INTERLEAVE-NEXT: [[TMP14:%.*]] = or disjoint i64 [[INDEX]], 4 1864; INTERLEAVE-NEXT: [[TMP15:%.*]] = or disjoint i64 [[INDEX]], 5 1865; INTERLEAVE-NEXT: [[TMP16:%.*]] = or disjoint i64 [[INDEX]], 6 1866; INTERLEAVE-NEXT: [[TMP17:%.*]] = or disjoint i64 [[INDEX]], 7 1867; INTERLEAVE-NEXT: [[DOTIDX:%.*]] = shl nsw i64 [[INDEX]], 4 1868; INTERLEAVE-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[DOTIDX]] 1869; INTERLEAVE-NEXT: [[DOTIDX5:%.*]] = shl nsw i64 [[TMP14]], 4 1870; INTERLEAVE-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[DOTIDX5]] 1871; INTERLEAVE-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP18]], align 1 1872; INTERLEAVE-NEXT: [[WIDE_VEC3:%.*]] = load <16 x i32>, ptr [[TMP19]], align 1 1873; INTERLEAVE-NEXT: [[TMP20:%.*]] = getelementptr inbounds [[PAIR_I32:%.*]], ptr [[P]], i64 [[INDEX]], i32 1 1874; INTERLEAVE-NEXT: [[TMP21:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP11]], i32 1 1875; INTERLEAVE-NEXT: [[TMP22:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP12]], i32 1 1876; INTERLEAVE-NEXT: [[TMP23:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP13]], i32 1 1877; INTERLEAVE-NEXT: [[TMP24:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP14]], i32 1 1878; INTERLEAVE-NEXT: [[TMP25:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP15]], i32 1 1879; INTERLEAVE-NEXT: [[TMP26:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP16]], i32 1 1880; INTERLEAVE-NEXT: [[TMP27:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[TMP17]], i32 1 1881; INTERLEAVE-NEXT: [[TMP28:%.*]] = extractelement <16 x i32> [[WIDE_VEC]], i64 0 1882; INTERLEAVE-NEXT: store i32 [[TMP28]], ptr [[TMP20]], align 1, !alias.scope [[META17:![0-9]+]], !noalias [[META20:![0-9]+]] 1883; INTERLEAVE-NEXT: [[TMP29:%.*]] = extractelement <16 x i32> [[WIDE_VEC]], i64 4 1884; INTERLEAVE-NEXT: store i32 [[TMP29]], ptr [[TMP21]], align 1, !alias.scope [[META17]], !noalias [[META20]] 1885; INTERLEAVE-NEXT: [[TMP30:%.*]] = extractelement <16 x i32> [[WIDE_VEC]], i64 8 1886; INTERLEAVE-NEXT: store i32 [[TMP30]], ptr [[TMP22]], align 1, !alias.scope [[META17]], !noalias [[META20]] 1887; INTERLEAVE-NEXT: [[TMP31:%.*]] = extractelement <16 x i32> [[WIDE_VEC]], i64 12 1888; INTERLEAVE-NEXT: store i32 [[TMP31]], ptr [[TMP23]], align 1, !alias.scope [[META17]], !noalias [[META20]] 1889; INTERLEAVE-NEXT: [[TMP32:%.*]] = extractelement <16 x i32> [[WIDE_VEC3]], i64 0 1890; INTERLEAVE-NEXT: store i32 [[TMP32]], ptr [[TMP24]], align 1, !alias.scope [[META17]], !noalias [[META20]] 1891; INTERLEAVE-NEXT: [[TMP33:%.*]] = extractelement <16 x i32> [[WIDE_VEC3]], i64 4 1892; INTERLEAVE-NEXT: store i32 [[TMP33]], ptr [[TMP25]], align 1, !alias.scope [[META17]], !noalias [[META20]] 1893; INTERLEAVE-NEXT: [[TMP34:%.*]] = extractelement <16 x i32> [[WIDE_VEC3]], i64 8 1894; INTERLEAVE-NEXT: store i32 [[TMP34]], ptr [[TMP26]], align 1, !alias.scope [[META17]], !noalias [[META20]] 1895; INTERLEAVE-NEXT: [[TMP35:%.*]] = extractelement <16 x i32> [[WIDE_VEC3]], i64 12 1896; INTERLEAVE-NEXT: store i32 [[TMP35]], ptr [[TMP27]], align 1, !alias.scope [[META17]], !noalias [[META20]] 1897; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 1898; INTERLEAVE-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 1899; INTERLEAVE-NEXT: br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] 1900; INTERLEAVE: middle.block: 1901; INTERLEAVE-NEXT: br label [[SCALAR_PH]] 1902; INTERLEAVE: scalar.ph: 1903; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ENTRY:%.*]] ] 1904; INTERLEAVE-NEXT: br label [[FOR_BODY:%.*]] 1905; INTERLEAVE: for.body: 1906; INTERLEAVE-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 1907; INTERLEAVE-NEXT: [[DOTIDX6:%.*]] = shl nsw i64 [[I]], 4 1908; INTERLEAVE-NEXT: [[TMP37:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[DOTIDX6]] 1909; INTERLEAVE-NEXT: [[TMP38:%.*]] = load i32, ptr [[TMP37]], align 1 1910; INTERLEAVE-NEXT: [[TMP39:%.*]] = getelementptr inbounds [[PAIR_I32]], ptr [[P]], i64 [[I]], i32 1 1911; INTERLEAVE-NEXT: store i32 [[TMP38]], ptr [[TMP39]], align 1 1912; INTERLEAVE-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 1913; INTERLEAVE-NEXT: [[TMP40:%.*]] = trunc i64 [[I_NEXT]] to i32 1914; INTERLEAVE-NEXT: [[COND:%.*]] = icmp eq i32 [[N]], [[TMP40]] 1915; INTERLEAVE-NEXT: br i1 [[COND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] 1916; INTERLEAVE: for.end: 1917; INTERLEAVE-NEXT: ret void 1918; 1919entry: 1920 br label %for.body 1921 1922for.body: 1923 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry] 1924 %0 = shl nsw i64 %i, 2 1925 %1 = getelementptr inbounds i32, ptr %a, i64 %0 1926 %2 = load i32, ptr %1, align 1 1927 %3 = getelementptr inbounds %pair.i32, ptr %p, i64 %i, i32 1 1928 store i32 %2, ptr %3, align 1 1929 %i.next = add nuw nsw i64 %i, 1 1930 %4 = trunc i64 %i.next to i32 1931 %cond = icmp eq i32 %4, %n 1932 br i1 %cond, label %for.end, label %for.body 1933 1934for.end: 1935 ret void 1936} 1937 1938; PR30542. Ensure we generate all the scalar steps for the induction variable. 1939; The scalar induction variable is used by a getelementptr instruction 1940; (uniform), and a udiv (non-uniform). 1941; 1942; int sum = 0; 1943; for (int i = 0; i < n; ++i) { 1944; int x = a[i]; 1945; if (c) 1946; x /= i; 1947; sum += x; 1948; } 1949; 1950; 1951; 1952; 1953 1954define i32 @scalarize_induction_variable_05(ptr %a, i32 %x, i1 %c, i32 %n) { 1955; CHECK-LABEL: @scalarize_induction_variable_05( 1956; CHECK-NEXT: entry: 1957; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 1) 1958; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[SMAX]], 2 1959; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 1960; CHECK: vector.ph: 1961; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[SMAX]], 2 1962; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[SMAX]], [[N_MOD_VF]] 1963; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C:%.*]], i64 0 1964; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer 1965; CHECK-NEXT: [[TMP14:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true) 1966; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 1967; CHECK: vector.body: 1968; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE2:%.*]] ] 1969; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_UDIV_CONTINUE2]] ] 1970; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 1971; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP0]] 1972; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0 1973; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4 1974; CHECK-NEXT: br i1 [[C]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 1975; CHECK: pred.udiv.if: 1976; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0 1977; CHECK-NEXT: [[TMP5:%.*]] = udiv i32 [[TMP4]], [[TMP0]] 1978; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i32 0 1979; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]] 1980; CHECK: pred.udiv.continue: 1981; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UDIV_IF]] ] 1982; CHECK-NEXT: br i1 [[C]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]] 1983; CHECK: pred.udiv.if1: 1984; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[INDEX]], 1 1985; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1 1986; CHECK-NEXT: [[TMP11:%.*]] = udiv i32 [[TMP10]], [[TMP9]] 1987; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP11]], i32 1 1988; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE2]] 1989; CHECK: pred.udiv.continue2: 1990; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i32> [ [[TMP7]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP12]], [[PRED_UDIV_IF1]] ] 1991; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP14]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP13]] 1992; CHECK-NEXT: [[TMP15]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]] 1993; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 1994; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 1995; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 1996; CHECK: middle.block: 1997; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP15]]) 1998; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[SMAX]], [[N_VEC]] 1999; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2000; CHECK: scalar.ph: 2001; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 2002; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2003; CHECK-NEXT: br label [[FOR_BODY:%.*]] 2004; CHECK: for.body: 2005; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ] 2006; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[VAR4:%.*]], [[IF_END]] ] 2007; CHECK-NEXT: [[VAR0:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[I]] 2008; CHECK-NEXT: [[VAR1:%.*]] = load i32, ptr [[VAR0]], align 4 2009; CHECK-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]] 2010; CHECK: if.then: 2011; CHECK-NEXT: [[VAR2:%.*]] = udiv i32 [[VAR1]], [[I]] 2012; CHECK-NEXT: br label [[IF_END]] 2013; CHECK: if.end: 2014; CHECK-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR2]], [[IF_THEN]] ], [ [[VAR1]], [[FOR_BODY]] ] 2015; CHECK-NEXT: [[VAR4]] = add i32 [[VAR3]], [[SUM]] 2016; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i32 [[I]], 1 2017; CHECK-NEXT: [[COND:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] 2018; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP25:![0-9]+]] 2019; CHECK: for.end: 2020; CHECK-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR4]], [[IF_END]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ] 2021; CHECK-NEXT: ret i32 [[VAR5]] 2022; 2023; IND-LABEL: @scalarize_induction_variable_05( 2024; IND-NEXT: entry: 2025; IND-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 1) 2026; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i32 [[N]], 2 2027; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2028; IND: vector.ph: 2029; IND-NEXT: [[N_VEC:%.*]] = and i32 [[SMAX]], 2147483646 2030; IND-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C:%.*]], i64 0 2031; IND-NEXT: [[TMP11:%.*]] = xor <2 x i1> [[BROADCAST_SPLATINSERT]], <i1 true, i1 poison> 2032; IND-NEXT: [[TMP12:%.*]] = shufflevector <2 x i1> [[TMP11]], <2 x i1> poison, <2 x i32> zeroinitializer 2033; IND-NEXT: br label [[VECTOR_BODY:%.*]] 2034; IND: vector.body: 2035; IND-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE2:%.*]] ] 2036; IND-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[PRED_UDIV_CONTINUE2]] ] 2037; IND-NEXT: [[TMP0:%.*]] = sext i32 [[INDEX]] to i64 2038; IND-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]] 2039; IND-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4 2040; IND-NEXT: br i1 [[C]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 2041; IND: pred.udiv.if: 2042; IND-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0 2043; IND-NEXT: [[TMP3:%.*]] = udiv i32 [[TMP2]], [[INDEX]] 2044; IND-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i64 0 2045; IND-NEXT: br label [[PRED_UDIV_CONTINUE]] 2046; IND: pred.udiv.continue: 2047; IND-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ] 2048; IND-NEXT: br i1 [[C]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2]] 2049; IND: pred.udiv.if1: 2050; IND-NEXT: [[TMP6:%.*]] = or disjoint i32 [[INDEX]], 1 2051; IND-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1 2052; IND-NEXT: [[TMP8:%.*]] = udiv i32 [[TMP7]], [[TMP6]] 2053; IND-NEXT: [[TMP9:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP8]], i64 1 2054; IND-NEXT: br label [[PRED_UDIV_CONTINUE2]] 2055; IND: pred.udiv.continue2: 2056; IND-NEXT: [[TMP10:%.*]] = phi <2 x i32> [ [[TMP5]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP9]], [[PRED_UDIV_IF1]] ] 2057; IND-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP12]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP10]] 2058; IND-NEXT: [[TMP13]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]] 2059; IND-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 2060; IND-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 2061; IND-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 2062; IND: middle.block: 2063; IND-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP13]]) 2064; IND-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[SMAX]], [[N_VEC]] 2065; IND-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2066; IND: scalar.ph: 2067; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 2068; IND-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2069; IND-NEXT: br label [[FOR_BODY:%.*]] 2070; IND: for.body: 2071; IND-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ] 2072; IND-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[VAR4:%.*]], [[IF_END]] ] 2073; IND-NEXT: [[TMP16:%.*]] = zext nneg i32 [[I]] to i64 2074; IND-NEXT: [[VAR0:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP16]] 2075; IND-NEXT: [[VAR1:%.*]] = load i32, ptr [[VAR0]], align 4 2076; IND-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]] 2077; IND: if.then: 2078; IND-NEXT: [[VAR2:%.*]] = udiv i32 [[VAR1]], [[I]] 2079; IND-NEXT: br label [[IF_END]] 2080; IND: if.end: 2081; IND-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR2]], [[IF_THEN]] ], [ [[VAR1]], [[FOR_BODY]] ] 2082; IND-NEXT: [[VAR4]] = add i32 [[VAR3]], [[SUM]] 2083; IND-NEXT: [[I_NEXT]] = add nuw nsw i32 [[I]], 1 2084; IND-NEXT: [[COND:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] 2085; IND-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP25:![0-9]+]] 2086; IND: for.end: 2087; IND-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR4]], [[IF_END]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ] 2088; IND-NEXT: ret i32 [[VAR5]] 2089; 2090; UNROLL-LABEL: @scalarize_induction_variable_05( 2091; UNROLL-NEXT: entry: 2092; UNROLL-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 1) 2093; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i32 [[N]], 4 2094; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2095; UNROLL: vector.ph: 2096; UNROLL-NEXT: [[N_VEC:%.*]] = and i32 [[SMAX]], 2147483644 2097; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C:%.*]], i64 0 2098; UNROLL-NEXT: [[TMP27:%.*]] = xor <2 x i1> [[BROADCAST_SPLATINSERT]], <i1 true, i1 poison> 2099; UNROLL-NEXT: [[TMP28:%.*]] = shufflevector <2 x i1> [[TMP27]], <2 x i1> poison, <2 x i32> zeroinitializer 2100; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 2101; UNROLL: vector.body: 2102; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE8:%.*]] ] 2103; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[PRED_UDIV_CONTINUE8]] ] 2104; UNROLL-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[PRED_UDIV_CONTINUE8]] ] 2105; UNROLL-NEXT: [[TMP0:%.*]] = sext i32 [[INDEX]] to i64 2106; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]] 2107; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 8 2108; UNROLL-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4 2109; UNROLL-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4 2110; UNROLL-NEXT: br i1 [[C]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 2111; UNROLL: pred.udiv.if: 2112; UNROLL-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0 2113; UNROLL-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP3]], [[INDEX]] 2114; UNROLL-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0 2115; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE]] 2116; UNROLL: pred.udiv.continue: 2117; UNROLL-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ] 2118; UNROLL-NEXT: br i1 [[C]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]] 2119; UNROLL: pred.udiv.if3: 2120; UNROLL-NEXT: [[TMP7:%.*]] = or disjoint i32 [[INDEX]], 1 2121; UNROLL-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1 2122; UNROLL-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP8]], [[TMP7]] 2123; UNROLL-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP9]], i64 1 2124; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE4]] 2125; UNROLL: pred.udiv.continue4: 2126; UNROLL-NEXT: [[TMP11:%.*]] = phi <2 x i32> [ [[TMP6]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF3]] ] 2127; UNROLL-NEXT: br i1 [[C]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]] 2128; UNROLL: pred.udiv.if5: 2129; UNROLL-NEXT: [[TMP12:%.*]] = or disjoint i32 [[INDEX]], 2 2130; UNROLL-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 0 2131; UNROLL-NEXT: [[TMP14:%.*]] = udiv i32 [[TMP13]], [[TMP12]] 2132; UNROLL-NEXT: [[TMP15:%.*]] = insertelement <2 x i32> poison, i32 [[TMP14]], i64 0 2133; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE6]] 2134; UNROLL: pred.udiv.continue6: 2135; UNROLL-NEXT: [[TMP16:%.*]] = phi <2 x i32> [ poison, [[PRED_UDIV_CONTINUE4]] ], [ [[TMP15]], [[PRED_UDIV_IF5]] ] 2136; UNROLL-NEXT: br i1 [[C]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]] 2137; UNROLL: pred.udiv.if7: 2138; UNROLL-NEXT: [[TMP17:%.*]] = or disjoint i32 [[INDEX]], 3 2139; UNROLL-NEXT: [[TMP18:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i64 1 2140; UNROLL-NEXT: [[TMP19:%.*]] = udiv i32 [[TMP18]], [[TMP17]] 2141; UNROLL-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> [[TMP16]], i32 [[TMP19]], i64 1 2142; UNROLL-NEXT: br label [[PRED_UDIV_CONTINUE8]] 2143; UNROLL: pred.udiv.continue8: 2144; UNROLL-NEXT: [[TMP21:%.*]] = phi <2 x i32> [ [[TMP16]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP20]], [[PRED_UDIV_IF7]] ] 2145; UNROLL-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP28]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP11]] 2146; UNROLL-NEXT: [[PREDPHI9:%.*]] = select <2 x i1> [[TMP28]], <2 x i32> [[WIDE_LOAD2]], <2 x i32> [[TMP21]] 2147; UNROLL-NEXT: [[TMP22]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]] 2148; UNROLL-NEXT: [[TMP23]] = add <2 x i32> [[PREDPHI9]], [[VEC_PHI1]] 2149; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 2150; UNROLL-NEXT: [[TMP24:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 2151; UNROLL-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 2152; UNROLL: middle.block: 2153; UNROLL-NEXT: [[BIN_RDX:%.*]] = add <2 x i32> [[TMP23]], [[TMP22]] 2154; UNROLL-NEXT: [[TMP25:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[BIN_RDX]]) 2155; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[SMAX]], [[N_VEC]] 2156; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2157; UNROLL: scalar.ph: 2158; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 2159; UNROLL-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP25]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2160; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 2161; UNROLL: for.body: 2162; UNROLL-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ] 2163; UNROLL-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[VAR4:%.*]], [[IF_END]] ] 2164; UNROLL-NEXT: [[TMP26:%.*]] = zext nneg i32 [[I]] to i64 2165; UNROLL-NEXT: [[VAR0:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP26]] 2166; UNROLL-NEXT: [[VAR1:%.*]] = load i32, ptr [[VAR0]], align 4 2167; UNROLL-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]] 2168; UNROLL: if.then: 2169; UNROLL-NEXT: [[VAR2:%.*]] = udiv i32 [[VAR1]], [[I]] 2170; UNROLL-NEXT: br label [[IF_END]] 2171; UNROLL: if.end: 2172; UNROLL-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR2]], [[IF_THEN]] ], [ [[VAR1]], [[FOR_BODY]] ] 2173; UNROLL-NEXT: [[VAR4]] = add i32 [[VAR3]], [[SUM]] 2174; UNROLL-NEXT: [[I_NEXT]] = add nuw nsw i32 [[I]], 1 2175; UNROLL-NEXT: [[COND:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] 2176; UNROLL-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP25:![0-9]+]] 2177; UNROLL: for.end: 2178; UNROLL-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR4]], [[IF_END]] ], [ [[TMP25]], [[MIDDLE_BLOCK]] ] 2179; UNROLL-NEXT: ret i32 [[VAR5]] 2180; 2181; UNROLL-NO-IC-LABEL: @scalarize_induction_variable_05( 2182; UNROLL-NO-IC-NEXT: entry: 2183; UNROLL-NO-IC-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 1) 2184; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[SMAX]], 4 2185; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2186; UNROLL-NO-IC: vector.ph: 2187; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[SMAX]], 4 2188; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i32 [[SMAX]], [[N_MOD_VF]] 2189; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C:%.*]], i64 0 2190; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer 2191; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true) 2192; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 2193; UNROLL-NO-IC: vector.body: 2194; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE8:%.*]] ] 2195; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP29:%.*]], [[PRED_UDIV_CONTINUE8]] ] 2196; UNROLL-NO-IC-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP30:%.*]], [[PRED_UDIV_CONTINUE8]] ] 2197; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 2198; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP0]] 2199; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0 2200; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 2 2201; UNROLL-NO-IC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4 2202; UNROLL-NO-IC-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4 2203; UNROLL-NO-IC-NEXT: br i1 [[C]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 2204; UNROLL-NO-IC: pred.udiv.if: 2205; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0 2206; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = udiv i32 [[TMP5]], [[TMP0]] 2207; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP6]], i32 0 2208; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE]] 2209; UNROLL-NO-IC: pred.udiv.continue: 2210; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_UDIV_IF]] ] 2211; UNROLL-NO-IC-NEXT: br i1 [[C]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]] 2212; UNROLL-NO-IC: pred.udiv.if3: 2213; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = add i32 [[INDEX]], 1 2214; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1 2215; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = udiv i32 [[TMP11]], [[TMP10]] 2216; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP12]], i32 1 2217; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE4]] 2218; UNROLL-NO-IC: pred.udiv.continue4: 2219; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = phi <2 x i32> [ [[TMP8]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP13]], [[PRED_UDIV_IF3]] ] 2220; UNROLL-NO-IC-NEXT: br i1 [[C]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]] 2221; UNROLL-NO-IC: pred.udiv.if5: 2222; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = add i32 [[INDEX]], 2 2223; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i32 0 2224; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = udiv i32 [[TMP17]], [[TMP16]] 2225; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = insertelement <2 x i32> poison, i32 [[TMP18]], i32 0 2226; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE6]] 2227; UNROLL-NO-IC: pred.udiv.continue6: 2228; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = phi <2 x i32> [ poison, [[PRED_UDIV_CONTINUE4]] ], [ [[TMP19]], [[PRED_UDIV_IF5]] ] 2229; UNROLL-NO-IC-NEXT: br i1 [[C]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8]] 2230; UNROLL-NO-IC: pred.udiv.if7: 2231; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = add i32 [[INDEX]], 3 2232; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = extractelement <2 x i32> [[WIDE_LOAD2]], i32 1 2233; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = udiv i32 [[TMP23]], [[TMP22]] 2234; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[TMP24]], i32 1 2235; UNROLL-NO-IC-NEXT: br label [[PRED_UDIV_CONTINUE8]] 2236; UNROLL-NO-IC: pred.udiv.continue8: 2237; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = phi <2 x i32> [ [[TMP20]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP25]], [[PRED_UDIV_IF7]] ] 2238; UNROLL-NO-IC-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP27]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP14]] 2239; UNROLL-NO-IC-NEXT: [[PREDPHI9:%.*]] = select <2 x i1> [[TMP27]], <2 x i32> [[WIDE_LOAD2]], <2 x i32> [[TMP26]] 2240; UNROLL-NO-IC-NEXT: [[TMP29]] = add <2 x i32> [[PREDPHI]], [[VEC_PHI]] 2241; UNROLL-NO-IC-NEXT: [[TMP30]] = add <2 x i32> [[PREDPHI9]], [[VEC_PHI1]] 2242; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 2243; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 2244; UNROLL-NO-IC-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 2245; UNROLL-NO-IC: middle.block: 2246; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = add <2 x i32> [[TMP30]], [[TMP29]] 2247; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[BIN_RDX]]) 2248; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[SMAX]], [[N_VEC]] 2249; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2250; UNROLL-NO-IC: scalar.ph: 2251; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 2252; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP32]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2253; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 2254; UNROLL-NO-IC: for.body: 2255; UNROLL-NO-IC-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ] 2256; UNROLL-NO-IC-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[VAR4:%.*]], [[IF_END]] ] 2257; UNROLL-NO-IC-NEXT: [[VAR0:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[I]] 2258; UNROLL-NO-IC-NEXT: [[VAR1:%.*]] = load i32, ptr [[VAR0]], align 4 2259; UNROLL-NO-IC-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]] 2260; UNROLL-NO-IC: if.then: 2261; UNROLL-NO-IC-NEXT: [[VAR2:%.*]] = udiv i32 [[VAR1]], [[I]] 2262; UNROLL-NO-IC-NEXT: br label [[IF_END]] 2263; UNROLL-NO-IC: if.end: 2264; UNROLL-NO-IC-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR2]], [[IF_THEN]] ], [ [[VAR1]], [[FOR_BODY]] ] 2265; UNROLL-NO-IC-NEXT: [[VAR4]] = add i32 [[VAR3]], [[SUM]] 2266; UNROLL-NO-IC-NEXT: [[I_NEXT]] = add nuw nsw i32 [[I]], 1 2267; UNROLL-NO-IC-NEXT: [[COND:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] 2268; UNROLL-NO-IC-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP25:![0-9]+]] 2269; UNROLL-NO-IC: for.end: 2270; UNROLL-NO-IC-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR4]], [[IF_END]] ], [ [[TMP32]], [[MIDDLE_BLOCK]] ] 2271; UNROLL-NO-IC-NEXT: ret i32 [[VAR5]] 2272; 2273; INTERLEAVE-LABEL: @scalarize_induction_variable_05( 2274; INTERLEAVE-NEXT: entry: 2275; INTERLEAVE-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 1) 2276; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i32 [[N]], 8 2277; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2278; INTERLEAVE: vector.ph: 2279; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i32 [[SMAX]], 2147483640 2280; INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0 2281; INTERLEAVE-NEXT: [[TMP47:%.*]] = xor <4 x i1> [[BROADCAST_SPLATINSERT]], <i1 true, i1 poison, i1 poison, i1 poison> 2282; INTERLEAVE-NEXT: [[TMP48:%.*]] = shufflevector <4 x i1> [[TMP47]], <4 x i1> poison, <4 x i32> zeroinitializer 2283; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 2284; INTERLEAVE: vector.body: 2285; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE16:%.*]] ] 2286; INTERLEAVE-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP42:%.*]], [[PRED_UDIV_CONTINUE16]] ] 2287; INTERLEAVE-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP43:%.*]], [[PRED_UDIV_CONTINUE16]] ] 2288; INTERLEAVE-NEXT: [[TMP0:%.*]] = sext i32 [[INDEX]] to i64 2289; INTERLEAVE-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]] 2290; INTERLEAVE-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16 2291; INTERLEAVE-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4 2292; INTERLEAVE-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4 2293; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]] 2294; INTERLEAVE: pred.udiv.if: 2295; INTERLEAVE-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 0 2296; INTERLEAVE-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP3]], [[INDEX]] 2297; INTERLEAVE-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 2298; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE]] 2299; INTERLEAVE: pred.udiv.continue: 2300; INTERLEAVE-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UDIV_IF]] ] 2301; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]] 2302; INTERLEAVE: pred.udiv.if3: 2303; INTERLEAVE-NEXT: [[TMP7:%.*]] = or disjoint i32 [[INDEX]], 1 2304; INTERLEAVE-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 1 2305; INTERLEAVE-NEXT: [[TMP9:%.*]] = udiv i32 [[TMP8]], [[TMP7]] 2306; INTERLEAVE-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP9]], i64 1 2307; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE4]] 2308; INTERLEAVE: pred.udiv.continue4: 2309; INTERLEAVE-NEXT: [[TMP11:%.*]] = phi <4 x i32> [ [[TMP6]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_UDIV_IF3]] ] 2310; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]] 2311; INTERLEAVE: pred.udiv.if5: 2312; INTERLEAVE-NEXT: [[TMP12:%.*]] = or disjoint i32 [[INDEX]], 2 2313; INTERLEAVE-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 2 2314; INTERLEAVE-NEXT: [[TMP14:%.*]] = udiv i32 [[TMP13]], [[TMP12]] 2315; INTERLEAVE-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP14]], i64 2 2316; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE6]] 2317; INTERLEAVE: pred.udiv.continue6: 2318; INTERLEAVE-NEXT: [[TMP16:%.*]] = phi <4 x i32> [ [[TMP11]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP15]], [[PRED_UDIV_IF5]] ] 2319; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]] 2320; INTERLEAVE: pred.udiv.if7: 2321; INTERLEAVE-NEXT: [[TMP17:%.*]] = or disjoint i32 [[INDEX]], 3 2322; INTERLEAVE-NEXT: [[TMP18:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 3 2323; INTERLEAVE-NEXT: [[TMP19:%.*]] = udiv i32 [[TMP18]], [[TMP17]] 2324; INTERLEAVE-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP19]], i64 3 2325; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE8]] 2326; INTERLEAVE: pred.udiv.continue8: 2327; INTERLEAVE-NEXT: [[TMP21:%.*]] = phi <4 x i32> [ [[TMP16]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP20]], [[PRED_UDIV_IF7]] ] 2328; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]] 2329; INTERLEAVE: pred.udiv.if9: 2330; INTERLEAVE-NEXT: [[TMP22:%.*]] = or disjoint i32 [[INDEX]], 4 2331; INTERLEAVE-NEXT: [[TMP23:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 0 2332; INTERLEAVE-NEXT: [[TMP24:%.*]] = udiv i32 [[TMP23]], [[TMP22]] 2333; INTERLEAVE-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> poison, i32 [[TMP24]], i64 0 2334; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE10]] 2335; INTERLEAVE: pred.udiv.continue10: 2336; INTERLEAVE-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ poison, [[PRED_UDIV_CONTINUE8]] ], [ [[TMP25]], [[PRED_UDIV_IF9]] ] 2337; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF11:%.*]], label [[PRED_UDIV_CONTINUE12:%.*]] 2338; INTERLEAVE: pred.udiv.if11: 2339; INTERLEAVE-NEXT: [[TMP27:%.*]] = or disjoint i32 [[INDEX]], 5 2340; INTERLEAVE-NEXT: [[TMP28:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 1 2341; INTERLEAVE-NEXT: [[TMP29:%.*]] = udiv i32 [[TMP28]], [[TMP27]] 2342; INTERLEAVE-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> [[TMP26]], i32 [[TMP29]], i64 1 2343; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE12]] 2344; INTERLEAVE: pred.udiv.continue12: 2345; INTERLEAVE-NEXT: [[TMP31:%.*]] = phi <4 x i32> [ [[TMP26]], [[PRED_UDIV_CONTINUE10]] ], [ [[TMP30]], [[PRED_UDIV_IF11]] ] 2346; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF13:%.*]], label [[PRED_UDIV_CONTINUE14:%.*]] 2347; INTERLEAVE: pred.udiv.if13: 2348; INTERLEAVE-NEXT: [[TMP32:%.*]] = or disjoint i32 [[INDEX]], 6 2349; INTERLEAVE-NEXT: [[TMP33:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 2 2350; INTERLEAVE-NEXT: [[TMP34:%.*]] = udiv i32 [[TMP33]], [[TMP32]] 2351; INTERLEAVE-NEXT: [[TMP35:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP34]], i64 2 2352; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE14]] 2353; INTERLEAVE: pred.udiv.continue14: 2354; INTERLEAVE-NEXT: [[TMP36:%.*]] = phi <4 x i32> [ [[TMP31]], [[PRED_UDIV_CONTINUE12]] ], [ [[TMP35]], [[PRED_UDIV_IF13]] ] 2355; INTERLEAVE-NEXT: br i1 [[C]], label [[PRED_UDIV_IF15:%.*]], label [[PRED_UDIV_CONTINUE16]] 2356; INTERLEAVE: pred.udiv.if15: 2357; INTERLEAVE-NEXT: [[TMP37:%.*]] = or disjoint i32 [[INDEX]], 7 2358; INTERLEAVE-NEXT: [[TMP38:%.*]] = extractelement <4 x i32> [[WIDE_LOAD2]], i64 3 2359; INTERLEAVE-NEXT: [[TMP39:%.*]] = udiv i32 [[TMP38]], [[TMP37]] 2360; INTERLEAVE-NEXT: [[TMP40:%.*]] = insertelement <4 x i32> [[TMP36]], i32 [[TMP39]], i64 3 2361; INTERLEAVE-NEXT: br label [[PRED_UDIV_CONTINUE16]] 2362; INTERLEAVE: pred.udiv.continue16: 2363; INTERLEAVE-NEXT: [[TMP41:%.*]] = phi <4 x i32> [ [[TMP36]], [[PRED_UDIV_CONTINUE14]] ], [ [[TMP40]], [[PRED_UDIV_IF15]] ] 2364; INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP48]], <4 x i32> [[WIDE_LOAD]], <4 x i32> [[TMP21]] 2365; INTERLEAVE-NEXT: [[PREDPHI17:%.*]] = select <4 x i1> [[TMP48]], <4 x i32> [[WIDE_LOAD2]], <4 x i32> [[TMP41]] 2366; INTERLEAVE-NEXT: [[TMP42]] = add <4 x i32> [[PREDPHI]], [[VEC_PHI]] 2367; INTERLEAVE-NEXT: [[TMP43]] = add <4 x i32> [[PREDPHI17]], [[VEC_PHI1]] 2368; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 2369; INTERLEAVE-NEXT: [[TMP44:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 2370; INTERLEAVE-NEXT: br i1 [[TMP44]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] 2371; INTERLEAVE: middle.block: 2372; INTERLEAVE-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP43]], [[TMP42]] 2373; INTERLEAVE-NEXT: [[TMP45:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) 2374; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[SMAX]], [[N_VEC]] 2375; INTERLEAVE-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2376; INTERLEAVE: scalar.ph: 2377; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 2378; INTERLEAVE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP45]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2379; INTERLEAVE-NEXT: br label [[FOR_BODY:%.*]] 2380; INTERLEAVE: for.body: 2381; INTERLEAVE-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ] 2382; INTERLEAVE-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[VAR4:%.*]], [[IF_END]] ] 2383; INTERLEAVE-NEXT: [[TMP46:%.*]] = zext nneg i32 [[I]] to i64 2384; INTERLEAVE-NEXT: [[VAR0:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP46]] 2385; INTERLEAVE-NEXT: [[VAR1:%.*]] = load i32, ptr [[VAR0]], align 4 2386; INTERLEAVE-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]] 2387; INTERLEAVE: if.then: 2388; INTERLEAVE-NEXT: [[VAR2:%.*]] = udiv i32 [[VAR1]], [[I]] 2389; INTERLEAVE-NEXT: br label [[IF_END]] 2390; INTERLEAVE: if.end: 2391; INTERLEAVE-NEXT: [[VAR3:%.*]] = phi i32 [ [[VAR2]], [[IF_THEN]] ], [ [[VAR1]], [[FOR_BODY]] ] 2392; INTERLEAVE-NEXT: [[VAR4]] = add i32 [[VAR3]], [[SUM]] 2393; INTERLEAVE-NEXT: [[I_NEXT]] = add nuw nsw i32 [[I]], 1 2394; INTERLEAVE-NEXT: [[COND:%.*]] = icmp slt i32 [[I_NEXT]], [[N]] 2395; INTERLEAVE-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP25:![0-9]+]] 2396; INTERLEAVE: for.end: 2397; INTERLEAVE-NEXT: [[VAR5:%.*]] = phi i32 [ [[VAR4]], [[IF_END]] ], [ [[TMP45]], [[MIDDLE_BLOCK]] ] 2398; INTERLEAVE-NEXT: ret i32 [[VAR5]] 2399; 2400entry: 2401 br label %for.body 2402 2403for.body: 2404 %i = phi i32 [ 0, %entry ], [ %i.next, %if.end ] 2405 %sum = phi i32 [ 0, %entry ], [ %var4, %if.end ] 2406 %var0 = getelementptr inbounds i32, ptr %a, i32 %i 2407 %var1 = load i32, ptr %var0, align 4 2408 br i1 %c, label %if.then, label %if.end 2409 2410if.then: 2411 %var2 = udiv i32 %var1, %i 2412 br label %if.end 2413 2414if.end: 2415 %var3 = phi i32 [ %var2, %if.then ], [ %var1, %for.body ] 2416 %var4 = add i32 %var3, %sum 2417 %i.next = add nuw nsw i32 %i, 1 2418 %cond = icmp slt i32 %i.next, %n 2419 br i1 %cond, label %for.body, label %for.end 2420 2421for.end: 2422 %var5 = phi i32 [ %var4, %if.end ] 2423 ret i32 %var5 2424} 2425 2426; Ensure we generate both a vector and a scalar induction variable. In this 2427; test, the induction variable is used by an instruction that will be 2428; vectorized (trunc) as well as an instruction that will remain in scalar form 2429; (gepelementptr). 2430; 2431; 2432; 2433 2434%pair.i16 = type { i16, i16 } 2435define void @iv_vector_and_scalar_users(ptr %p, i32 %a, i32 %n) { 2436; CHECK-LABEL: @iv_vector_and_scalar_users( 2437; CHECK-NEXT: entry: 2438; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 2439; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 2440; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 2441; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2 2442; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2443; CHECK: vector.ph: 2444; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2 2445; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 2446; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[A:%.*]], i64 0 2447; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 2448; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 2449; CHECK: vector.body: 2450; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2451; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 2452; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 2453; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 1 2454; CHECK-NEXT: [[TMP5:%.*]] = add <2 x i32> [[BROADCAST_SPLAT]], [[VEC_IND]] 2455; CHECK-NEXT: [[TMP6:%.*]] = trunc <2 x i32> [[TMP5]] to <2 x i16> 2456; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[PAIR_I16:%.*]], ptr [[P:%.*]], i64 [[TMP3]], i32 1 2457; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP4]], i32 1 2458; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i16> [[TMP6]], i32 0 2459; CHECK-NEXT: store i16 [[TMP9]], ptr [[TMP7]], align 2 2460; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i16> [[TMP6]], i32 1 2461; CHECK-NEXT: store i16 [[TMP10]], ptr [[TMP8]], align 2 2462; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 2463; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 2464; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 2465; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]] 2466; CHECK: middle.block: 2467; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 2468; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2469; CHECK: scalar.ph: 2470; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 2471; CHECK-NEXT: br label [[FOR_BODY:%.*]] 2472; CHECK: for.body: 2473; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 2474; CHECK-NEXT: [[TMP12:%.*]] = trunc i64 [[I]] to i32 2475; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[A]], [[TMP12]] 2476; CHECK-NEXT: [[TMP14:%.*]] = trunc i32 [[TMP13]] to i16 2477; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[I]], i32 1 2478; CHECK-NEXT: store i16 [[TMP14]], ptr [[TMP15]], align 2 2479; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 2480; CHECK-NEXT: [[TMP16:%.*]] = trunc i64 [[I_NEXT]] to i32 2481; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[TMP16]], [[N]] 2482; CHECK-NEXT: br i1 [[COND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]] 2483; CHECK: for.end: 2484; CHECK-NEXT: ret void 2485; 2486; IND-LABEL: @iv_vector_and_scalar_users( 2487; IND-NEXT: entry: 2488; IND-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 2489; IND-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 2490; IND-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 2491; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i32 [[TMP0]], 0 2492; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2493; IND: vector.ph: 2494; IND-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934590 2495; IND-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[A:%.*]], i64 0 2496; IND-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 2497; IND-NEXT: br label [[VECTOR_BODY:%.*]] 2498; IND: vector.body: 2499; IND-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2500; IND-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 2501; IND-NEXT: [[TMP3:%.*]] = or disjoint i64 [[INDEX]], 1 2502; IND-NEXT: [[TMP4:%.*]] = add <2 x i32> [[BROADCAST_SPLAT]], [[VEC_IND]] 2503; IND-NEXT: [[TMP5:%.*]] = trunc <2 x i32> [[TMP4]] to <2 x i16> 2504; IND-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR_I16:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 1 2505; IND-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP3]], i32 1 2506; IND-NEXT: [[TMP8:%.*]] = extractelement <2 x i16> [[TMP5]], i64 0 2507; IND-NEXT: store i16 [[TMP8]], ptr [[TMP6]], align 2 2508; IND-NEXT: [[TMP9:%.*]] = extractelement <2 x i16> [[TMP5]], i64 1 2509; IND-NEXT: store i16 [[TMP9]], ptr [[TMP7]], align 2 2510; IND-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 2511; IND-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 2512; IND-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 2513; IND-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]] 2514; IND: middle.block: 2515; IND-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 2516; IND-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2517; IND: scalar.ph: 2518; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 2519; IND-NEXT: br label [[FOR_BODY:%.*]] 2520; IND: for.body: 2521; IND-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 2522; IND-NEXT: [[TMP11:%.*]] = trunc i64 [[I]] to i32 2523; IND-NEXT: [[TMP12:%.*]] = add i32 [[A]], [[TMP11]] 2524; IND-NEXT: [[TMP13:%.*]] = trunc i32 [[TMP12]] to i16 2525; IND-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw [[PAIR_I16]], ptr [[P]], i64 [[I]], i32 1 2526; IND-NEXT: store i16 [[TMP13]], ptr [[TMP14]], align 2 2527; IND-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 2528; IND-NEXT: [[TMP15:%.*]] = trunc i64 [[I_NEXT]] to i32 2529; IND-NEXT: [[COND:%.*]] = icmp eq i32 [[N]], [[TMP15]] 2530; IND-NEXT: br i1 [[COND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]] 2531; IND: for.end: 2532; IND-NEXT: ret void 2533; 2534; UNROLL-LABEL: @iv_vector_and_scalar_users( 2535; UNROLL-NEXT: entry: 2536; UNROLL-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 2537; UNROLL-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 2538; UNROLL-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 2539; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 3 2540; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2541; UNROLL: vector.ph: 2542; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934588 2543; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[A:%.*]], i64 0 2544; UNROLL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 2545; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 2546; UNROLL: vector.body: 2547; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2548; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 2549; UNROLL-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 2550; UNROLL-NEXT: [[TMP3:%.*]] = or disjoint i64 [[INDEX]], 1 2551; UNROLL-NEXT: [[TMP4:%.*]] = or disjoint i64 [[INDEX]], 2 2552; UNROLL-NEXT: [[TMP5:%.*]] = or disjoint i64 [[INDEX]], 3 2553; UNROLL-NEXT: [[TMP6:%.*]] = add <2 x i32> [[BROADCAST_SPLAT]], [[VEC_IND]] 2554; UNROLL-NEXT: [[TMP7:%.*]] = add <2 x i32> [[BROADCAST_SPLAT]], [[STEP_ADD]] 2555; UNROLL-NEXT: [[TMP8:%.*]] = trunc <2 x i32> [[TMP6]] to <2 x i16> 2556; UNROLL-NEXT: [[TMP9:%.*]] = trunc <2 x i32> [[TMP7]] to <2 x i16> 2557; UNROLL-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[PAIR_I16:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 1 2558; UNROLL-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP3]], i32 1 2559; UNROLL-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP4]], i32 1 2560; UNROLL-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP5]], i32 1 2561; UNROLL-NEXT: [[TMP14:%.*]] = extractelement <2 x i16> [[TMP8]], i64 0 2562; UNROLL-NEXT: store i16 [[TMP14]], ptr [[TMP10]], align 2 2563; UNROLL-NEXT: [[TMP15:%.*]] = extractelement <2 x i16> [[TMP8]], i64 1 2564; UNROLL-NEXT: store i16 [[TMP15]], ptr [[TMP11]], align 2 2565; UNROLL-NEXT: [[TMP16:%.*]] = extractelement <2 x i16> [[TMP9]], i64 0 2566; UNROLL-NEXT: store i16 [[TMP16]], ptr [[TMP12]], align 2 2567; UNROLL-NEXT: [[TMP17:%.*]] = extractelement <2 x i16> [[TMP9]], i64 1 2568; UNROLL-NEXT: store i16 [[TMP17]], ptr [[TMP13]], align 2 2569; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 2570; UNROLL-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 4) 2571; UNROLL-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 2572; UNROLL-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]] 2573; UNROLL: middle.block: 2574; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 2575; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2576; UNROLL: scalar.ph: 2577; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 2578; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 2579; UNROLL: for.body: 2580; UNROLL-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 2581; UNROLL-NEXT: [[TMP19:%.*]] = trunc i64 [[I]] to i32 2582; UNROLL-NEXT: [[TMP20:%.*]] = add i32 [[A]], [[TMP19]] 2583; UNROLL-NEXT: [[TMP21:%.*]] = trunc i32 [[TMP20]] to i16 2584; UNROLL-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw [[PAIR_I16]], ptr [[P]], i64 [[I]], i32 1 2585; UNROLL-NEXT: store i16 [[TMP21]], ptr [[TMP22]], align 2 2586; UNROLL-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 2587; UNROLL-NEXT: [[TMP23:%.*]] = trunc i64 [[I_NEXT]] to i32 2588; UNROLL-NEXT: [[COND:%.*]] = icmp eq i32 [[N]], [[TMP23]] 2589; UNROLL-NEXT: br i1 [[COND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]] 2590; UNROLL: for.end: 2591; UNROLL-NEXT: ret void 2592; 2593; UNROLL-NO-IC-LABEL: @iv_vector_and_scalar_users( 2594; UNROLL-NO-IC-NEXT: entry: 2595; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 2596; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 2597; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 2598; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 2599; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2600; UNROLL-NO-IC: vector.ph: 2601; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 2602; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] 2603; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[A:%.*]], i64 0 2604; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 2605; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 2606; UNROLL-NO-IC: vector.body: 2607; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2608; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 2609; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 2610; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 2611; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 1 2612; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 2 2613; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 3 2614; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add <2 x i32> [[BROADCAST_SPLAT]], [[VEC_IND]] 2615; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add <2 x i32> [[BROADCAST_SPLAT]], [[STEP_ADD]] 2616; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = trunc <2 x i32> [[TMP7]] to <2 x i16> 2617; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = trunc <2 x i32> [[TMP8]] to <2 x i16> 2618; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[PAIR_I16:%.*]], ptr [[P:%.*]], i64 [[TMP3]], i32 1 2619; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP4]], i32 1 2620; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP5]], i32 1 2621; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP6]], i32 1 2622; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = extractelement <2 x i16> [[TMP9]], i32 0 2623; UNROLL-NO-IC-NEXT: store i16 [[TMP15]], ptr [[TMP11]], align 2 2624; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = extractelement <2 x i16> [[TMP9]], i32 1 2625; UNROLL-NO-IC-NEXT: store i16 [[TMP16]], ptr [[TMP12]], align 2 2626; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = extractelement <2 x i16> [[TMP10]], i32 0 2627; UNROLL-NO-IC-NEXT: store i16 [[TMP17]], ptr [[TMP13]], align 2 2628; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = extractelement <2 x i16> [[TMP10]], i32 1 2629; UNROLL-NO-IC-NEXT: store i16 [[TMP18]], ptr [[TMP14]], align 2 2630; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 2631; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 2) 2632; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 2633; UNROLL-NO-IC-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]] 2634; UNROLL-NO-IC: middle.block: 2635; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 2636; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2637; UNROLL-NO-IC: scalar.ph: 2638; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 2639; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 2640; UNROLL-NO-IC: for.body: 2641; UNROLL-NO-IC-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 2642; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = trunc i64 [[I]] to i32 2643; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = add i32 [[A]], [[TMP20]] 2644; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = trunc i32 [[TMP21]] to i16 2645; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[I]], i32 1 2646; UNROLL-NO-IC-NEXT: store i16 [[TMP22]], ptr [[TMP23]], align 2 2647; UNROLL-NO-IC-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 2648; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = trunc i64 [[I_NEXT]] to i32 2649; UNROLL-NO-IC-NEXT: [[COND:%.*]] = icmp eq i32 [[TMP24]], [[N]] 2650; UNROLL-NO-IC-NEXT: br i1 [[COND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]] 2651; UNROLL-NO-IC: for.end: 2652; UNROLL-NO-IC-NEXT: ret void 2653; 2654; INTERLEAVE-LABEL: @iv_vector_and_scalar_users( 2655; INTERLEAVE-NEXT: entry: 2656; INTERLEAVE-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 2657; INTERLEAVE-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 2658; INTERLEAVE-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 2659; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 7 2660; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2661; INTERLEAVE: vector.ph: 2662; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934584 2663; INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[A:%.*]], i64 0 2664; INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 2665; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 2666; INTERLEAVE: vector.body: 2667; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2668; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 2669; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4) 2670; INTERLEAVE-NEXT: [[TMP3:%.*]] = or disjoint i64 [[INDEX]], 1 2671; INTERLEAVE-NEXT: [[TMP4:%.*]] = or disjoint i64 [[INDEX]], 2 2672; INTERLEAVE-NEXT: [[TMP5:%.*]] = or disjoint i64 [[INDEX]], 3 2673; INTERLEAVE-NEXT: [[TMP6:%.*]] = or disjoint i64 [[INDEX]], 4 2674; INTERLEAVE-NEXT: [[TMP7:%.*]] = or disjoint i64 [[INDEX]], 5 2675; INTERLEAVE-NEXT: [[TMP8:%.*]] = or disjoint i64 [[INDEX]], 6 2676; INTERLEAVE-NEXT: [[TMP9:%.*]] = or disjoint i64 [[INDEX]], 7 2677; INTERLEAVE-NEXT: [[TMP10:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], [[VEC_IND]] 2678; INTERLEAVE-NEXT: [[TMP11:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], [[STEP_ADD]] 2679; INTERLEAVE-NEXT: [[TMP12:%.*]] = trunc <4 x i32> [[TMP10]] to <4 x i16> 2680; INTERLEAVE-NEXT: [[TMP13:%.*]] = trunc <4 x i32> [[TMP11]] to <4 x i16> 2681; INTERLEAVE-NEXT: [[TMP14:%.*]] = getelementptr inbounds [[PAIR_I16:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 1 2682; INTERLEAVE-NEXT: [[TMP15:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP3]], i32 1 2683; INTERLEAVE-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP4]], i32 1 2684; INTERLEAVE-NEXT: [[TMP17:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP5]], i32 1 2685; INTERLEAVE-NEXT: [[TMP18:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP6]], i32 1 2686; INTERLEAVE-NEXT: [[TMP19:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP7]], i32 1 2687; INTERLEAVE-NEXT: [[TMP20:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP8]], i32 1 2688; INTERLEAVE-NEXT: [[TMP21:%.*]] = getelementptr inbounds [[PAIR_I16]], ptr [[P]], i64 [[TMP9]], i32 1 2689; INTERLEAVE-NEXT: [[TMP22:%.*]] = extractelement <4 x i16> [[TMP12]], i64 0 2690; INTERLEAVE-NEXT: store i16 [[TMP22]], ptr [[TMP14]], align 2 2691; INTERLEAVE-NEXT: [[TMP23:%.*]] = extractelement <4 x i16> [[TMP12]], i64 1 2692; INTERLEAVE-NEXT: store i16 [[TMP23]], ptr [[TMP15]], align 2 2693; INTERLEAVE-NEXT: [[TMP24:%.*]] = extractelement <4 x i16> [[TMP12]], i64 2 2694; INTERLEAVE-NEXT: store i16 [[TMP24]], ptr [[TMP16]], align 2 2695; INTERLEAVE-NEXT: [[TMP25:%.*]] = extractelement <4 x i16> [[TMP12]], i64 3 2696; INTERLEAVE-NEXT: store i16 [[TMP25]], ptr [[TMP17]], align 2 2697; INTERLEAVE-NEXT: [[TMP26:%.*]] = extractelement <4 x i16> [[TMP13]], i64 0 2698; INTERLEAVE-NEXT: store i16 [[TMP26]], ptr [[TMP18]], align 2 2699; INTERLEAVE-NEXT: [[TMP27:%.*]] = extractelement <4 x i16> [[TMP13]], i64 1 2700; INTERLEAVE-NEXT: store i16 [[TMP27]], ptr [[TMP19]], align 2 2701; INTERLEAVE-NEXT: [[TMP28:%.*]] = extractelement <4 x i16> [[TMP13]], i64 2 2702; INTERLEAVE-NEXT: store i16 [[TMP28]], ptr [[TMP20]], align 2 2703; INTERLEAVE-NEXT: [[TMP29:%.*]] = extractelement <4 x i16> [[TMP13]], i64 3 2704; INTERLEAVE-NEXT: store i16 [[TMP29]], ptr [[TMP21]], align 2 2705; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 2706; INTERLEAVE-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 8) 2707; INTERLEAVE-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 2708; INTERLEAVE-NEXT: br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]] 2709; INTERLEAVE: middle.block: 2710; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] 2711; INTERLEAVE-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 2712; INTERLEAVE: scalar.ph: 2713; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 2714; INTERLEAVE-NEXT: br label [[FOR_BODY:%.*]] 2715; INTERLEAVE: for.body: 2716; INTERLEAVE-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 2717; INTERLEAVE-NEXT: [[TMP31:%.*]] = trunc i64 [[I]] to i32 2718; INTERLEAVE-NEXT: [[TMP32:%.*]] = add i32 [[A]], [[TMP31]] 2719; INTERLEAVE-NEXT: [[TMP33:%.*]] = trunc i32 [[TMP32]] to i16 2720; INTERLEAVE-NEXT: [[TMP34:%.*]] = getelementptr inbounds nuw [[PAIR_I16]], ptr [[P]], i64 [[I]], i32 1 2721; INTERLEAVE-NEXT: store i16 [[TMP33]], ptr [[TMP34]], align 2 2722; INTERLEAVE-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 2723; INTERLEAVE-NEXT: [[TMP35:%.*]] = trunc i64 [[I_NEXT]] to i32 2724; INTERLEAVE-NEXT: [[COND:%.*]] = icmp eq i32 [[N]], [[TMP35]] 2725; INTERLEAVE-NEXT: br i1 [[COND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]] 2726; INTERLEAVE: for.end: 2727; INTERLEAVE-NEXT: ret void 2728; 2729entry: 2730 br label %for.body 2731 2732for.body: 2733 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 2734 %0 = trunc i64 %i to i32 2735 %1 = add i32 %a, %0 2736 %2 = trunc i32 %1 to i16 2737 %3 = getelementptr inbounds %pair.i16, ptr %p, i64 %i, i32 1 2738 store i16 %2, ptr %3, align 2 2739 %i.next = add nuw nsw i64 %i, 1 2740 %4 = trunc i64 %i.next to i32 2741 %cond = icmp eq i32 %4, %n 2742 br i1 %cond, label %for.end, label %for.body 2743 2744for.end: 2745 ret void 2746} 2747 2748; Make sure that the loop exit count computation does not overflow for i8 and 2749; i16. The exit count of these loops is i8/i16 max + 1. If we don't cast the 2750; induction variable to a bigger type the exit count computation will overflow 2751; to 0. 2752; PR17532 2753 2754define i32 @i8_loop() nounwind readnone ssp uwtable { 2755; CHECK-LABEL: @i8_loop( 2756; CHECK-NEXT: entry: 2757; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2758; CHECK: vector.ph: 2759; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 2760; CHECK: vector.body: 2761; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2762; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ <i32 1, i32 -1>, [[VECTOR_PH]] ], [ [[TMP0:%.*]], [[VECTOR_BODY]] ] 2763; CHECK-NEXT: [[TMP0]] = and <2 x i32> [[VEC_PHI]], splat (i32 4) 2764; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 2765; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256 2766; CHECK-NEXT: br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]] 2767; CHECK: middle.block: 2768; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> [[TMP0]]) 2769; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 2770; CHECK: scalar.ph: 2771; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP2]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ] 2772; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2773; CHECK-NEXT: br label [[LOOP:%.*]] 2774; CHECK: loop: 2775; CHECK-NEXT: [[A_0:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_0_AND:%.*]], [[LOOP]] ] 2776; CHECK-NEXT: [[B_0:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[B_NEXT:%.*]], [[LOOP]] ] 2777; CHECK-NEXT: [[A_0_AND]] = and i32 [[A_0]], 4 2778; CHECK-NEXT: [[B_NEXT]] = add i8 [[B_0]], -1 2779; CHECK-NEXT: [[EC:%.*]] = icmp eq i8 [[B_NEXT]], 0 2780; CHECK-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP29:![0-9]+]] 2781; CHECK: exit: 2782; CHECK-NEXT: [[A_0_AND_LCSSA:%.*]] = phi i32 [ [[A_0_AND]], [[LOOP]] ], [ [[TMP2]], [[MIDDLE_BLOCK]] ] 2783; CHECK-NEXT: ret i32 [[A_0_AND_LCSSA]] 2784; 2785; IND-LABEL: @i8_loop( 2786; IND-NEXT: entry: 2787; IND-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2788; IND: vector.ph: 2789; IND-NEXT: br label [[VECTOR_BODY:%.*]] 2790; IND: vector.body: 2791; IND-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2792; IND-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 2793; IND-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256 2794; IND-NEXT: br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]] 2795; IND: middle.block: 2796; IND-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 2797; IND: scalar.ph: 2798; IND-NEXT: br label [[LOOP:%.*]] 2799; IND: loop: 2800; IND-NEXT: br i1 poison, label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP29:![0-9]+]] 2801; IND: exit: 2802; IND-NEXT: ret i32 0 2803; 2804; UNROLL-LABEL: @i8_loop( 2805; UNROLL-NEXT: entry: 2806; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2807; UNROLL: vector.ph: 2808; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 2809; UNROLL: vector.body: 2810; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2811; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 2812; UNROLL-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256 2813; UNROLL-NEXT: br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]] 2814; UNROLL: middle.block: 2815; UNROLL-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 2816; UNROLL: scalar.ph: 2817; UNROLL-NEXT: br label [[LOOP:%.*]] 2818; UNROLL: loop: 2819; UNROLL-NEXT: br i1 poison, label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP29:![0-9]+]] 2820; UNROLL: exit: 2821; UNROLL-NEXT: ret i32 0 2822; 2823; UNROLL-NO-IC-LABEL: @i8_loop( 2824; UNROLL-NO-IC-NEXT: entry: 2825; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2826; UNROLL-NO-IC: vector.ph: 2827; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 2828; UNROLL-NO-IC: vector.body: 2829; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2830; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ <i32 1, i32 -1>, [[VECTOR_PH]] ], [ [[TMP0:%.*]], [[VECTOR_BODY]] ] 2831; UNROLL-NO-IC-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ splat (i32 -1), [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 2832; UNROLL-NO-IC-NEXT: [[TMP0]] = and <2 x i32> [[VEC_PHI]], splat (i32 4) 2833; UNROLL-NO-IC-NEXT: [[TMP1]] = and <2 x i32> [[VEC_PHI1]], splat (i32 4) 2834; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 2835; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256 2836; UNROLL-NO-IC-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]] 2837; UNROLL-NO-IC: middle.block: 2838; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = and <2 x i32> [[TMP1]], [[TMP0]] 2839; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> [[BIN_RDX]]) 2840; UNROLL-NO-IC-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 2841; UNROLL-NO-IC: scalar.ph: 2842; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ] 2843; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2844; UNROLL-NO-IC-NEXT: br label [[LOOP:%.*]] 2845; UNROLL-NO-IC: loop: 2846; UNROLL-NO-IC-NEXT: [[A_0:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_0_AND:%.*]], [[LOOP]] ] 2847; UNROLL-NO-IC-NEXT: [[B_0:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[B_NEXT:%.*]], [[LOOP]] ] 2848; UNROLL-NO-IC-NEXT: [[A_0_AND]] = and i32 [[A_0]], 4 2849; UNROLL-NO-IC-NEXT: [[B_NEXT]] = add i8 [[B_0]], -1 2850; UNROLL-NO-IC-NEXT: [[EC:%.*]] = icmp eq i8 [[B_NEXT]], 0 2851; UNROLL-NO-IC-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP29:![0-9]+]] 2852; UNROLL-NO-IC: exit: 2853; UNROLL-NO-IC-NEXT: [[A_0_AND_LCSSA:%.*]] = phi i32 [ [[A_0_AND]], [[LOOP]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ] 2854; UNROLL-NO-IC-NEXT: ret i32 [[A_0_AND_LCSSA]] 2855; 2856; INTERLEAVE-LABEL: @i8_loop( 2857; INTERLEAVE-NEXT: entry: 2858; INTERLEAVE-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2859; INTERLEAVE: vector.ph: 2860; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 2861; INTERLEAVE: vector.body: 2862; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2863; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 2864; INTERLEAVE-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256 2865; INTERLEAVE-NEXT: br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]] 2866; INTERLEAVE: middle.block: 2867; INTERLEAVE-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 2868; INTERLEAVE: scalar.ph: 2869; INTERLEAVE-NEXT: br label [[LOOP:%.*]] 2870; INTERLEAVE: loop: 2871; INTERLEAVE-NEXT: br i1 poison, label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP29:![0-9]+]] 2872; INTERLEAVE: exit: 2873; INTERLEAVE-NEXT: ret i32 0 2874; 2875entry: 2876 br label %loop 2877 2878loop: 2879 %a.0 = phi i32 [ 1, %entry ], [ %a.0.and, %loop ] 2880 %b.0 = phi i8 [ 0, %entry ], [ %b.next, %loop ] 2881 %a.0.and = and i32 %a.0, 4 2882 %b.next = add i8 %b.0, -1 2883 %ec = icmp eq i8 %b.next, 0 2884 br i1 %ec, label %exit, label %loop 2885 2886exit: 2887 ret i32 %a.0.and 2888} 2889 2890 2891define i32 @i16_loop() nounwind readnone ssp uwtable { 2892; CHECK-LABEL: @i16_loop( 2893; CHECK-NEXT: entry: 2894; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2895; CHECK: vector.ph: 2896; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 2897; CHECK: vector.body: 2898; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2899; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ <i32 1, i32 -1>, [[VECTOR_PH]] ], [ [[TMP0:%.*]], [[VECTOR_BODY]] ] 2900; CHECK-NEXT: [[TMP0]] = and <2 x i32> [[VEC_PHI]], splat (i32 4) 2901; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 2902; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 65536 2903; CHECK-NEXT: br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]] 2904; CHECK: middle.block: 2905; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> [[TMP0]]) 2906; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 2907; CHECK: scalar.ph: 2908; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP2]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ] 2909; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2910; CHECK-NEXT: br label [[LOOP:%.*]] 2911; CHECK: loop: 2912; CHECK-NEXT: [[A_0:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_0_AND:%.*]], [[LOOP]] ] 2913; CHECK-NEXT: [[B_0:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[B_0_NEXT:%.*]], [[LOOP]] ] 2914; CHECK-NEXT: [[A_0_AND]] = and i32 [[A_0]], 4 2915; CHECK-NEXT: [[B_0_NEXT]] = add i16 [[B_0]], -1 2916; CHECK-NEXT: [[EC:%.*]] = icmp eq i16 [[B_0_NEXT]], 0 2917; CHECK-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP31:![0-9]+]] 2918; CHECK: exit: 2919; CHECK-NEXT: [[A_0_AND_LCSSA:%.*]] = phi i32 [ [[A_0_AND]], [[LOOP]] ], [ [[TMP2]], [[MIDDLE_BLOCK]] ] 2920; CHECK-NEXT: ret i32 [[A_0_AND_LCSSA]] 2921; 2922; IND-LABEL: @i16_loop( 2923; IND-NEXT: entry: 2924; IND-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2925; IND: vector.ph: 2926; IND-NEXT: br label [[VECTOR_BODY:%.*]] 2927; IND: vector.body: 2928; IND-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2929; IND-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 2930; IND-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 65536 2931; IND-NEXT: br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]] 2932; IND: middle.block: 2933; IND-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 2934; IND: scalar.ph: 2935; IND-NEXT: br label [[LOOP:%.*]] 2936; IND: loop: 2937; IND-NEXT: br i1 poison, label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP31:![0-9]+]] 2938; IND: exit: 2939; IND-NEXT: ret i32 0 2940; 2941; UNROLL-LABEL: @i16_loop( 2942; UNROLL-NEXT: entry: 2943; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2944; UNROLL: vector.ph: 2945; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 2946; UNROLL: vector.body: 2947; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2948; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 2949; UNROLL-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 65536 2950; UNROLL-NEXT: br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]] 2951; UNROLL: middle.block: 2952; UNROLL-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 2953; UNROLL: scalar.ph: 2954; UNROLL-NEXT: br label [[LOOP:%.*]] 2955; UNROLL: loop: 2956; UNROLL-NEXT: br i1 poison, label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP31:![0-9]+]] 2957; UNROLL: exit: 2958; UNROLL-NEXT: ret i32 0 2959; 2960; UNROLL-NO-IC-LABEL: @i16_loop( 2961; UNROLL-NO-IC-NEXT: entry: 2962; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2963; UNROLL-NO-IC: vector.ph: 2964; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 2965; UNROLL-NO-IC: vector.body: 2966; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 2967; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ <i32 1, i32 -1>, [[VECTOR_PH]] ], [ [[TMP0:%.*]], [[VECTOR_BODY]] ] 2968; UNROLL-NO-IC-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ splat (i32 -1), [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ] 2969; UNROLL-NO-IC-NEXT: [[TMP0]] = and <2 x i32> [[VEC_PHI]], splat (i32 4) 2970; UNROLL-NO-IC-NEXT: [[TMP1]] = and <2 x i32> [[VEC_PHI1]], splat (i32 4) 2971; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 2972; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 65536 2973; UNROLL-NO-IC-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]] 2974; UNROLL-NO-IC: middle.block: 2975; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = and <2 x i32> [[TMP1]], [[TMP0]] 2976; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> [[BIN_RDX]]) 2977; UNROLL-NO-IC-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 2978; UNROLL-NO-IC: scalar.ph: 2979; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ] 2980; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 2981; UNROLL-NO-IC-NEXT: br label [[LOOP:%.*]] 2982; UNROLL-NO-IC: loop: 2983; UNROLL-NO-IC-NEXT: [[A_0:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_0_AND:%.*]], [[LOOP]] ] 2984; UNROLL-NO-IC-NEXT: [[B_0:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[B_0_NEXT:%.*]], [[LOOP]] ] 2985; UNROLL-NO-IC-NEXT: [[A_0_AND]] = and i32 [[A_0]], 4 2986; UNROLL-NO-IC-NEXT: [[B_0_NEXT]] = add i16 [[B_0]], -1 2987; UNROLL-NO-IC-NEXT: [[EC:%.*]] = icmp eq i16 [[B_0_NEXT]], 0 2988; UNROLL-NO-IC-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP31:![0-9]+]] 2989; UNROLL-NO-IC: exit: 2990; UNROLL-NO-IC-NEXT: [[A_0_AND_LCSSA:%.*]] = phi i32 [ [[A_0_AND]], [[LOOP]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ] 2991; UNROLL-NO-IC-NEXT: ret i32 [[A_0_AND_LCSSA]] 2992; 2993; INTERLEAVE-LABEL: @i16_loop( 2994; INTERLEAVE-NEXT: entry: 2995; INTERLEAVE-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 2996; INTERLEAVE: vector.ph: 2997; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 2998; INTERLEAVE: vector.body: 2999; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3000; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 3001; INTERLEAVE-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 65536 3002; INTERLEAVE-NEXT: br i1 [[TMP0]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]] 3003; INTERLEAVE: middle.block: 3004; INTERLEAVE-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 3005; INTERLEAVE: scalar.ph: 3006; INTERLEAVE-NEXT: br label [[LOOP:%.*]] 3007; INTERLEAVE: loop: 3008; INTERLEAVE-NEXT: br i1 poison, label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP31:![0-9]+]] 3009; INTERLEAVE: exit: 3010; INTERLEAVE-NEXT: ret i32 0 3011; 3012entry: 3013 br label %loop 3014 3015loop: 3016 %a.0 = phi i32 [ 1, %entry ], [ %a.0.and, %loop ] 3017 %b.0 = phi i16 [ 0, %entry ], [ %b.0.next, %loop ] 3018 %a.0.and = and i32 %a.0, 4 3019 %b.0.next = add i16 %b.0, -1 3020 %ec = icmp eq i16 %b.0.next, 0 3021 br i1 %ec, label %exit, label %loop 3022 3023exit: 3024 ret i32 %a.0.and 3025} 3026 3027; This loop has a backedge taken count of i32_max. We need to check for this 3028; condition and can skip vectorizing. 3029 3030define i32 @max_i32_backedgetaken() { 3031; CHECK-LABEL: @max_i32_backedgetaken( 3032; CHECK-NEXT: entry: 3033; CHECK-NEXT: br label [[LOOP:%.*]] 3034; CHECK: loop: 3035; CHECK-NEXT: [[A_0:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[A_0_AND:%.*]], [[LOOP]] ] 3036; CHECK-NEXT: [[B_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[B_NEXT:%.*]], [[LOOP]] ] 3037; CHECK-NEXT: [[A_0_AND]] = and i32 [[A_0]], 4 3038; CHECK-NEXT: [[B_NEXT]] = add i32 [[B_0]], -1 3039; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[B_NEXT]], 0 3040; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] 3041; CHECK: exit: 3042; CHECK-NEXT: [[A_0_AND_LCSSA:%.*]] = phi i32 [ [[A_0_AND]], [[LOOP]] ] 3043; CHECK-NEXT: ret i32 [[A_0_AND_LCSSA]] 3044; 3045; IND-LABEL: @max_i32_backedgetaken( 3046; IND-NEXT: entry: 3047; IND-NEXT: br label [[LOOP:%.*]] 3048; IND: loop: 3049; IND-NEXT: [[B_0:%.*]] = phi i32 [ 0, [[SCALAR_PH:%.*]] ], [ [[B_NEXT:%.*]], [[LOOP]] ] 3050; IND-NEXT: [[B_NEXT]] = add i32 [[B_0]], -1 3051; IND-NEXT: [[EC:%.*]] = icmp eq i32 [[B_NEXT]], 0 3052; IND-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] 3053; IND: exit: 3054; IND-NEXT: ret i32 0 3055; 3056; UNROLL-LABEL: @max_i32_backedgetaken( 3057; UNROLL-NEXT: entry: 3058; UNROLL-NEXT: br label [[LOOP:%.*]] 3059; UNROLL: loop: 3060; UNROLL-NEXT: [[B_0:%.*]] = phi i32 [ 0, [[SCALAR_PH:%.*]] ], [ [[B_NEXT:%.*]], [[LOOP]] ] 3061; UNROLL-NEXT: [[B_NEXT]] = add i32 [[B_0]], -1 3062; UNROLL-NEXT: [[EC:%.*]] = icmp eq i32 [[B_NEXT]], 0 3063; UNROLL-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] 3064; UNROLL: exit: 3065; UNROLL-NEXT: ret i32 0 3066; 3067; UNROLL-NO-IC-LABEL: @max_i32_backedgetaken( 3068; UNROLL-NO-IC-NEXT: entry: 3069; UNROLL-NO-IC-NEXT: br label [[LOOP:%.*]] 3070; UNROLL-NO-IC: loop: 3071; UNROLL-NO-IC-NEXT: [[A_0:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[A_0_AND:%.*]], [[LOOP]] ] 3072; UNROLL-NO-IC-NEXT: [[B_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[B_NEXT:%.*]], [[LOOP]] ] 3073; UNROLL-NO-IC-NEXT: [[A_0_AND]] = and i32 [[A_0]], 4 3074; UNROLL-NO-IC-NEXT: [[B_NEXT]] = add i32 [[B_0]], -1 3075; UNROLL-NO-IC-NEXT: [[EC:%.*]] = icmp eq i32 [[B_NEXT]], 0 3076; UNROLL-NO-IC-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] 3077; UNROLL-NO-IC: exit: 3078; UNROLL-NO-IC-NEXT: [[A_0_AND_LCSSA:%.*]] = phi i32 [ [[A_0_AND]], [[LOOP]] ] 3079; UNROLL-NO-IC-NEXT: ret i32 [[A_0_AND_LCSSA]] 3080; 3081; INTERLEAVE-LABEL: @max_i32_backedgetaken( 3082; INTERLEAVE-NEXT: entry: 3083; INTERLEAVE-NEXT: br label [[LOOP:%.*]] 3084; INTERLEAVE: loop: 3085; INTERLEAVE-NEXT: [[B_0:%.*]] = phi i32 [ 0, [[SCALAR_PH:%.*]] ], [ [[B_NEXT:%.*]], [[LOOP]] ] 3086; INTERLEAVE-NEXT: [[B_NEXT]] = add i32 [[B_0]], -1 3087; INTERLEAVE-NEXT: [[EC:%.*]] = icmp eq i32 [[B_NEXT]], 0 3088; INTERLEAVE-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]] 3089; INTERLEAVE: exit: 3090; INTERLEAVE-NEXT: ret i32 0 3091; 3092entry: 3093 br label %loop 3094 3095loop: 3096 %a.0 = phi i32 [ 1, %entry ], [ %a.0.and, %loop ] 3097 %b.0 = phi i32 [ 0, %entry ], [ %b.next, %loop ] 3098 %a.0.and = and i32 %a.0, 4 3099 %b.next = add i32 %b.0, -1 3100 %ec = icmp eq i32 %b.next, 0 3101 br i1 %ec, label %exit, label %loop 3102 3103exit: 3104 ret i32 %a.0.and 3105} 3106 3107; When generating the overflow check we must sure that the induction start value 3108; is defined before the branch to the scalar preheader. 3109 3110 3111 3112@e = global i8 1, align 1 3113@d = common global i32 0, align 4 3114@c = common global i32 0, align 4 3115define i32 @testoverflowcheck() { 3116; CHECK-LABEL: @testoverflowcheck( 3117; CHECK-NEXT: entry: 3118; CHECK-NEXT: [[DOTPR_I:%.*]] = load i8, ptr @e, align 1 3119; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr @d, align 4 3120; CHECK-NEXT: [[C_PROMOTED_I:%.*]] = load i32, ptr @c, align 4 3121; CHECK-NEXT: [[TMP1:%.*]] = sub i8 -1, [[DOTPR_I]] 3122; CHECK-NEXT: [[TMP2:%.*]] = zext i8 [[TMP1]] to i32 3123; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], 1 3124; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP3]], 2 3125; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 3126; CHECK: vector.ph: 3127; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP3]], 2 3128; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP3]], [[N_MOD_VF]] 3129; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 3130; CHECK-NEXT: [[IND_END:%.*]] = add i8 [[DOTPR_I]], [[DOTCAST]] 3131; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> splat (i32 -1), i32 [[C_PROMOTED_I]], i32 0 3132; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0 3133; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 3134; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 3135; CHECK: vector.body: 3136; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3137; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ] 3138; CHECK-NEXT: [[TMP5]] = and <2 x i32> [[BROADCAST_SPLAT]], [[VEC_PHI]] 3139; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 3140; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 3141; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]] 3142; CHECK: middle.block: 3143; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> [[TMP5]]) 3144; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP3]], [[N_VEC]] 3145; CHECK-NEXT: br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]] 3146; CHECK: scalar.ph: 3147; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[DOTPR_I]], [[ENTRY:%.*]] ] 3148; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[C_PROMOTED_I]], [[ENTRY]] ] 3149; CHECK-NEXT: br label [[COND_END_I:%.*]] 3150; CHECK: cond.end.i: 3151; CHECK-NEXT: [[INC4_I:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC_I:%.*]], [[COND_END_I]] ] 3152; CHECK-NEXT: [[AND3_I:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[AND_I:%.*]], [[COND_END_I]] ] 3153; CHECK-NEXT: [[AND_I]] = and i32 [[TMP0]], [[AND3_I]] 3154; CHECK-NEXT: [[INC_I]] = add i8 [[INC4_I]], 1 3155; CHECK-NEXT: [[TOBOOL_I:%.*]] = icmp eq i8 [[INC_I]], 0 3156; CHECK-NEXT: br i1 [[TOBOOL_I]], label [[LOOPEXIT]], label [[COND_END_I]], !llvm.loop [[LOOP35:![0-9]+]] 3157; CHECK: loopexit: 3158; CHECK-NEXT: [[AND_I_LCSSA:%.*]] = phi i32 [ [[AND_I]], [[COND_END_I]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] 3159; CHECK-NEXT: ret i32 [[AND_I_LCSSA]] 3160; 3161; IND-LABEL: @testoverflowcheck( 3162; IND-NEXT: entry: 3163; IND-NEXT: [[DOTPR_I:%.*]] = load i8, ptr @e, align 1 3164; IND-NEXT: [[TMP0:%.*]] = load i32, ptr @d, align 4 3165; IND-NEXT: [[C_PROMOTED_I:%.*]] = load i32, ptr @c, align 4 3166; IND-NEXT: [[TMP1:%.*]] = xor i8 [[DOTPR_I]], -1 3167; IND-NEXT: [[TMP2:%.*]] = zext i8 [[TMP1]] to i32 3168; IND-NEXT: [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], 1 3169; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp eq i8 [[DOTPR_I]], -1 3170; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 3171; IND: vector.ph: 3172; IND-NEXT: [[N_VEC:%.*]] = and i32 [[TMP3]], 510 3173; IND-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 3174; IND-NEXT: [[IND_END:%.*]] = add i8 [[DOTPR_I]], [[DOTCAST]] 3175; IND-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> <i32 poison, i32 -1>, i32 [[C_PROMOTED_I]], i64 0 3176; IND-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0 3177; IND-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 3178; IND-NEXT: br label [[VECTOR_BODY:%.*]] 3179; IND: vector.body: 3180; IND-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3181; IND-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 3182; IND-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 3183; IND-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]] 3184; IND: middle.block: 3185; IND-NEXT: [[TMP6:%.*]] = and <2 x i32> [[TMP4]], [[BROADCAST_SPLAT]] 3186; IND-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> [[TMP6]]) 3187; IND-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP3]], [[N_VEC]] 3188; IND-NEXT: br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]] 3189; IND: scalar.ph: 3190; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[DOTPR_I]], [[ENTRY:%.*]] ] 3191; IND-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[C_PROMOTED_I]], [[ENTRY]] ] 3192; IND-NEXT: br label [[COND_END_I:%.*]] 3193; IND: cond.end.i: 3194; IND-NEXT: [[INC4_I:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC_I:%.*]], [[COND_END_I]] ] 3195; IND-NEXT: [[AND_I:%.*]] = and i32 [[BC_MERGE_RDX]], [[TMP0]] 3196; IND-NEXT: [[INC_I]] = add i8 [[INC4_I]], 1 3197; IND-NEXT: [[TOBOOL_I:%.*]] = icmp eq i8 [[INC_I]], 0 3198; IND-NEXT: br i1 [[TOBOOL_I]], label [[LOOPEXIT]], label [[COND_END_I]], !llvm.loop [[LOOP35:![0-9]+]] 3199; IND: loopexit: 3200; IND-NEXT: [[AND_I_LCSSA:%.*]] = phi i32 [ [[AND_I]], [[COND_END_I]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] 3201; IND-NEXT: ret i32 [[AND_I_LCSSA]] 3202; 3203; UNROLL-LABEL: @testoverflowcheck( 3204; UNROLL-NEXT: entry: 3205; UNROLL-NEXT: [[DOTPR_I:%.*]] = load i8, ptr @e, align 1 3206; UNROLL-NEXT: [[TMP0:%.*]] = load i32, ptr @d, align 4 3207; UNROLL-NEXT: [[C_PROMOTED_I:%.*]] = load i32, ptr @c, align 4 3208; UNROLL-NEXT: [[TMP1:%.*]] = xor i8 [[DOTPR_I]], -1 3209; UNROLL-NEXT: [[TMP2:%.*]] = zext i8 [[TMP1]] to i32 3210; UNROLL-NEXT: [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], 1 3211; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i8 [[DOTPR_I]], -4 3212; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 3213; UNROLL: vector.ph: 3214; UNROLL-NEXT: [[N_VEC:%.*]] = and i32 [[TMP3]], 508 3215; UNROLL-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 3216; UNROLL-NEXT: [[IND_END:%.*]] = add i8 [[DOTPR_I]], [[DOTCAST]] 3217; UNROLL-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> <i32 poison, i32 -1>, i32 [[C_PROMOTED_I]], i64 0 3218; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0 3219; UNROLL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 3220; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 3221; UNROLL: vector.body: 3222; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3223; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 3224; UNROLL-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 3225; UNROLL-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]] 3226; UNROLL: middle.block: 3227; UNROLL-NEXT: [[TMP6:%.*]] = and <2 x i32> [[TMP4]], [[BROADCAST_SPLAT]] 3228; UNROLL-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> [[TMP6]]) 3229; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP3]], [[N_VEC]] 3230; UNROLL-NEXT: br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]] 3231; UNROLL: scalar.ph: 3232; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[DOTPR_I]], [[ENTRY:%.*]] ] 3233; UNROLL-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[C_PROMOTED_I]], [[ENTRY]] ] 3234; UNROLL-NEXT: br label [[COND_END_I:%.*]] 3235; UNROLL: cond.end.i: 3236; UNROLL-NEXT: [[INC4_I:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC_I:%.*]], [[COND_END_I]] ] 3237; UNROLL-NEXT: [[AND_I:%.*]] = and i32 [[BC_MERGE_RDX]], [[TMP0]] 3238; UNROLL-NEXT: [[INC_I]] = add i8 [[INC4_I]], 1 3239; UNROLL-NEXT: [[TOBOOL_I:%.*]] = icmp eq i8 [[INC_I]], 0 3240; UNROLL-NEXT: br i1 [[TOBOOL_I]], label [[LOOPEXIT]], label [[COND_END_I]], !llvm.loop [[LOOP35:![0-9]+]] 3241; UNROLL: loopexit: 3242; UNROLL-NEXT: [[AND_I_LCSSA:%.*]] = phi i32 [ [[AND_I]], [[COND_END_I]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] 3243; UNROLL-NEXT: ret i32 [[AND_I_LCSSA]] 3244; 3245; UNROLL-NO-IC-LABEL: @testoverflowcheck( 3246; UNROLL-NO-IC-NEXT: entry: 3247; UNROLL-NO-IC-NEXT: [[DOTPR_I:%.*]] = load i8, ptr @e, align 1 3248; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = load i32, ptr @d, align 4 3249; UNROLL-NO-IC-NEXT: [[C_PROMOTED_I:%.*]] = load i32, ptr @c, align 4 3250; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = sub i8 -1, [[DOTPR_I]] 3251; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = zext i8 [[TMP1]] to i32 3252; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], 1 3253; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP3]], 4 3254; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 3255; UNROLL-NO-IC: vector.ph: 3256; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP3]], 4 3257; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP3]], [[N_MOD_VF]] 3258; UNROLL-NO-IC-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 3259; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = add i8 [[DOTPR_I]], [[DOTCAST]] 3260; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> splat (i32 -1), i32 [[C_PROMOTED_I]], i32 0 3261; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0 3262; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 3263; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 3264; UNROLL-NO-IC: vector.body: 3265; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3266; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ] 3267; UNROLL-NO-IC-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ splat (i32 -1), [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ] 3268; UNROLL-NO-IC-NEXT: [[TMP5]] = and <2 x i32> [[BROADCAST_SPLAT]], [[VEC_PHI]] 3269; UNROLL-NO-IC-NEXT: [[TMP6]] = and <2 x i32> [[BROADCAST_SPLAT]], [[VEC_PHI1]] 3270; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 3271; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 3272; UNROLL-NO-IC-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]] 3273; UNROLL-NO-IC: middle.block: 3274; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = and <2 x i32> [[TMP6]], [[TMP5]] 3275; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> [[BIN_RDX]]) 3276; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP3]], [[N_VEC]] 3277; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]] 3278; UNROLL-NO-IC: scalar.ph: 3279; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[DOTPR_I]], [[ENTRY:%.*]] ] 3280; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP8]], [[MIDDLE_BLOCK]] ], [ [[C_PROMOTED_I]], [[ENTRY]] ] 3281; UNROLL-NO-IC-NEXT: br label [[COND_END_I:%.*]] 3282; UNROLL-NO-IC: cond.end.i: 3283; UNROLL-NO-IC-NEXT: [[INC4_I:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC_I:%.*]], [[COND_END_I]] ] 3284; UNROLL-NO-IC-NEXT: [[AND3_I:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[AND_I:%.*]], [[COND_END_I]] ] 3285; UNROLL-NO-IC-NEXT: [[AND_I]] = and i32 [[TMP0]], [[AND3_I]] 3286; UNROLL-NO-IC-NEXT: [[INC_I]] = add i8 [[INC4_I]], 1 3287; UNROLL-NO-IC-NEXT: [[TOBOOL_I:%.*]] = icmp eq i8 [[INC_I]], 0 3288; UNROLL-NO-IC-NEXT: br i1 [[TOBOOL_I]], label [[LOOPEXIT]], label [[COND_END_I]], !llvm.loop [[LOOP35:![0-9]+]] 3289; UNROLL-NO-IC: loopexit: 3290; UNROLL-NO-IC-NEXT: [[AND_I_LCSSA:%.*]] = phi i32 [ [[AND_I]], [[COND_END_I]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ] 3291; UNROLL-NO-IC-NEXT: ret i32 [[AND_I_LCSSA]] 3292; 3293; INTERLEAVE-LABEL: @testoverflowcheck( 3294; INTERLEAVE-NEXT: entry: 3295; INTERLEAVE-NEXT: [[DOTPR_I:%.*]] = load i8, ptr @e, align 1 3296; INTERLEAVE-NEXT: [[TMP0:%.*]] = load i32, ptr @d, align 4 3297; INTERLEAVE-NEXT: [[C_PROMOTED_I:%.*]] = load i32, ptr @c, align 4 3298; INTERLEAVE-NEXT: [[TMP1:%.*]] = xor i8 [[DOTPR_I]], -1 3299; INTERLEAVE-NEXT: [[TMP2:%.*]] = zext i8 [[TMP1]] to i32 3300; INTERLEAVE-NEXT: [[TMP3:%.*]] = add nuw nsw i32 [[TMP2]], 1 3301; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i8 [[DOTPR_I]], -8 3302; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 3303; INTERLEAVE: vector.ph: 3304; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i32 [[TMP3]], 504 3305; INTERLEAVE-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 3306; INTERLEAVE-NEXT: [[IND_END:%.*]] = add i8 [[DOTPR_I]], [[DOTCAST]] 3307; INTERLEAVE-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> <i32 poison, i32 -1, i32 -1, i32 -1>, i32 [[C_PROMOTED_I]], i64 0 3308; INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 0 3309; INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 3310; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 3311; INTERLEAVE: vector.body: 3312; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3313; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 3314; INTERLEAVE-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 3315; INTERLEAVE-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]] 3316; INTERLEAVE: middle.block: 3317; INTERLEAVE-NEXT: [[TMP6:%.*]] = and <4 x i32> [[TMP4]], [[BROADCAST_SPLAT]] 3318; INTERLEAVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP6]]) 3319; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP3]], [[N_VEC]] 3320; INTERLEAVE-NEXT: br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]] 3321; INTERLEAVE: scalar.ph: 3322; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[DOTPR_I]], [[ENTRY:%.*]] ] 3323; INTERLEAVE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[C_PROMOTED_I]], [[ENTRY]] ] 3324; INTERLEAVE-NEXT: br label [[COND_END_I:%.*]] 3325; INTERLEAVE: cond.end.i: 3326; INTERLEAVE-NEXT: [[INC4_I:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC_I:%.*]], [[COND_END_I]] ] 3327; INTERLEAVE-NEXT: [[AND_I:%.*]] = and i32 [[BC_MERGE_RDX]], [[TMP0]] 3328; INTERLEAVE-NEXT: [[INC_I]] = add i8 [[INC4_I]], 1 3329; INTERLEAVE-NEXT: [[TOBOOL_I:%.*]] = icmp eq i8 [[INC_I]], 0 3330; INTERLEAVE-NEXT: br i1 [[TOBOOL_I]], label [[LOOPEXIT]], label [[COND_END_I]], !llvm.loop [[LOOP35:![0-9]+]] 3331; INTERLEAVE: loopexit: 3332; INTERLEAVE-NEXT: [[AND_I_LCSSA:%.*]] = phi i32 [ [[AND_I]], [[COND_END_I]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] 3333; INTERLEAVE-NEXT: ret i32 [[AND_I_LCSSA]] 3334; 3335entry: 3336 %.pr.i = load i8, ptr @e, align 1 3337 %0 = load i32, ptr @d, align 4 3338 %c.promoted.i = load i32, ptr @c, align 4 3339 br label %cond.end.i 3340 3341cond.end.i: 3342 %inc4.i = phi i8 [ %.pr.i, %entry ], [ %inc.i, %cond.end.i ] 3343 %and3.i = phi i32 [ %c.promoted.i, %entry ], [ %and.i, %cond.end.i ] 3344 %and.i = and i32 %0, %and3.i 3345 %inc.i = add i8 %inc4.i, 1 3346 %tobool.i = icmp eq i8 %inc.i, 0 3347 br i1 %tobool.i, label %loopexit, label %cond.end.i 3348 3349loopexit: 3350 ret i32 %and.i 3351} 3352 3353; The SCEV expression of %sphi is (zext i8 {%t,+,1}<%loop> to i32) 3354; In order to recognize %sphi as an induction PHI and vectorize this loop, 3355; we need to convert the SCEV expression into an AddRecExpr. 3356; The expression gets converted to {zext i8 %t to i32,+,1}. 3357 3358define void @wrappingindvars1(i8 %t, i32 %len, ptr %A) { 3359; CHECK-LABEL: @wrappingindvars1( 3360; CHECK-NEXT: entry: 3361; CHECK-NEXT: [[ST:%.*]] = zext i8 [[T:%.*]] to i16 3362; CHECK-NEXT: [[EXT:%.*]] = zext i8 [[T]] to i32 3363; CHECK-NEXT: [[ECMP:%.*]] = icmp ult i16 [[ST]], 42 3364; CHECK-NEXT: br i1 [[ECMP]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]] 3365; CHECK: loop.preheader: 3366; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[LEN:%.*]], 1 3367; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 2 3368; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 3369; CHECK: vector.scevcheck: 3370; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8 3371; CHECK-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]] 3372; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[T]] 3373; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255 3374; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] 3375; CHECK-NEXT: [[TMP6:%.*]] = trunc i32 [[LEN]] to i8 3376; CHECK-NEXT: [[TMP7:%.*]] = add i8 [[T]], [[TMP6]] 3377; CHECK-NEXT: [[TMP8:%.*]] = icmp slt i8 [[TMP7]], [[T]] 3378; CHECK-NEXT: [[TMP9:%.*]] = icmp ugt i32 [[LEN]], 255 3379; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]] 3380; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP5]], [[TMP10]] 3381; CHECK-NEXT: br i1 [[TMP11]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3382; CHECK: vector.ph: 3383; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2 3384; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]] 3385; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 3386; CHECK-NEXT: [[IND_END:%.*]] = add i8 [[T]], [[DOTCAST]] 3387; CHECK-NEXT: [[IND_END2:%.*]] = add i32 [[EXT]], [[N_VEC]] 3388; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[EXT]], i64 0 3389; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 3390; CHECK-NEXT: [[INDUCTION:%.*]] = add <2 x i32> [[DOTSPLAT]], <i32 0, i32 1> 3391; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 3392; CHECK: vector.body: 3393; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3394; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 3395; CHECK-NEXT: [[DOTCAST4:%.*]] = trunc i32 [[INDEX]] to i8 3396; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i8 [[T]], [[DOTCAST4]] 3397; CHECK-NEXT: [[TMP12:%.*]] = add i8 [[OFFSET_IDX]], 0 3398; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i8 [[TMP12]] 3399; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i32 0 3400; CHECK-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP14]], align 4 3401; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 3402; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 3403; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 3404; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]] 3405; CHECK: middle.block: 3406; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 3407; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] 3408; CHECK: scalar.ph: 3409; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ] 3410; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ] 3411; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ] 3412; CHECK-NEXT: br label [[LOOP:%.*]] 3413; CHECK: loop: 3414; CHECK-NEXT: [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 3415; CHECK-NEXT: [[IDX_B:%.*]] = phi i32 [ [[IDX_B_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 3416; CHECK-NEXT: [[SPHI:%.*]] = phi i32 [ [[IDX_INC_EXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ] 3417; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i8 [[IDX]] 3418; CHECK-NEXT: store i32 [[SPHI]], ptr [[PTR]], align 4 3419; CHECK-NEXT: [[IDX_INC]] = add i8 [[IDX]], 1 3420; CHECK-NEXT: [[IDX_INC_EXT]] = zext i8 [[IDX_INC]] to i32 3421; CHECK-NEXT: [[IDX_B_INC]] = add nuw nsw i32 [[IDX_B]], 1 3422; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[IDX_B]], [[LEN]] 3423; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP37:![0-9]+]] 3424; CHECK: exit.loopexit: 3425; CHECK-NEXT: br label [[EXIT]] 3426; CHECK: exit: 3427; CHECK-NEXT: ret void 3428; 3429; IND-LABEL: @wrappingindvars1( 3430; IND-NEXT: entry: 3431; IND-NEXT: [[EXT:%.*]] = zext i8 [[T:%.*]] to i32 3432; IND-NEXT: [[ECMP:%.*]] = icmp ult i8 [[T]], 42 3433; IND-NEXT: br i1 [[ECMP]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]] 3434; IND: loop.preheader: 3435; IND-NEXT: [[TMP0:%.*]] = add i32 [[LEN:%.*]], 1 3436; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 2 3437; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 3438; IND: vector.scevcheck: 3439; IND-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8 3440; IND-NEXT: [[TMP2:%.*]] = xor i8 [[T]], -1 3441; IND-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[TMP1]] 3442; IND-NEXT: [[TMP4:%.*]] = trunc i32 [[LEN]] to i8 3443; IND-NEXT: [[TMP5:%.*]] = add i8 [[T]], [[TMP4]] 3444; IND-NEXT: [[TMP6:%.*]] = icmp slt i8 [[TMP5]], [[T]] 3445; IND-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[LEN]], 255 3446; IND-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] 3447; IND-NEXT: [[TMP9:%.*]] = or i1 [[TMP3]], [[TMP8]] 3448; IND-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3449; IND: vector.ph: 3450; IND-NEXT: [[N_VEC:%.*]] = and i32 [[TMP0]], 510 3451; IND-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 3452; IND-NEXT: [[IND_END:%.*]] = add i8 [[T]], [[DOTCAST]] 3453; IND-NEXT: [[IND_END2:%.*]] = add nuw nsw i32 [[N_VEC]], [[EXT]] 3454; IND-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[EXT]], i64 0 3455; IND-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 3456; IND-NEXT: [[INDUCTION:%.*]] = add nuw nsw <2 x i32> [[DOTSPLAT]], <i32 0, i32 1> 3457; IND-NEXT: br label [[VECTOR_BODY:%.*]] 3458; IND: vector.body: 3459; IND-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3460; IND-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 3461; IND-NEXT: [[DOTCAST4:%.*]] = trunc i32 [[INDEX]] to i8 3462; IND-NEXT: [[OFFSET_IDX:%.*]] = add i8 [[T]], [[DOTCAST4]] 3463; IND-NEXT: [[TMP10:%.*]] = sext i8 [[OFFSET_IDX]] to i64 3464; IND-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP10]] 3465; IND-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP11]], align 4 3466; IND-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 3467; IND-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 3468; IND-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 3469; IND-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]] 3470; IND: middle.block: 3471; IND-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 3472; IND-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] 3473; IND: scalar.ph: 3474; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ] 3475; IND-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ] 3476; IND-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ] 3477; IND-NEXT: br label [[LOOP:%.*]] 3478; IND: loop: 3479; IND-NEXT: [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 3480; IND-NEXT: [[IDX_B:%.*]] = phi i32 [ [[IDX_B_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 3481; IND-NEXT: [[SPHI:%.*]] = phi i32 [ [[IDX_INC_EXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ] 3482; IND-NEXT: [[TMP13:%.*]] = sext i8 [[IDX]] to i64 3483; IND-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP13]] 3484; IND-NEXT: store i32 [[SPHI]], ptr [[PTR]], align 4 3485; IND-NEXT: [[IDX_INC]] = add i8 [[IDX]], 1 3486; IND-NEXT: [[IDX_INC_EXT]] = zext i8 [[IDX_INC]] to i32 3487; IND-NEXT: [[IDX_B_INC]] = add nuw nsw i32 [[IDX_B]], 1 3488; IND-NEXT: [[C:%.*]] = icmp ult i32 [[IDX_B]], [[LEN]] 3489; IND-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP37:![0-9]+]] 3490; IND: exit.loopexit: 3491; IND-NEXT: br label [[EXIT]] 3492; IND: exit: 3493; IND-NEXT: ret void 3494; 3495; UNROLL-LABEL: @wrappingindvars1( 3496; UNROLL-NEXT: entry: 3497; UNROLL-NEXT: [[EXT:%.*]] = zext i8 [[T:%.*]] to i32 3498; UNROLL-NEXT: [[ECMP:%.*]] = icmp ult i8 [[T]], 42 3499; UNROLL-NEXT: br i1 [[ECMP]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]] 3500; UNROLL: loop.preheader: 3501; UNROLL-NEXT: [[TMP0:%.*]] = add i32 [[LEN:%.*]], 1 3502; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 4 3503; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 3504; UNROLL: vector.scevcheck: 3505; UNROLL-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8 3506; UNROLL-NEXT: [[TMP2:%.*]] = xor i8 [[T]], -1 3507; UNROLL-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[TMP1]] 3508; UNROLL-NEXT: [[TMP4:%.*]] = trunc i32 [[LEN]] to i8 3509; UNROLL-NEXT: [[TMP5:%.*]] = add i8 [[T]], [[TMP4]] 3510; UNROLL-NEXT: [[TMP6:%.*]] = icmp slt i8 [[TMP5]], [[T]] 3511; UNROLL-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[LEN]], 255 3512; UNROLL-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] 3513; UNROLL-NEXT: [[TMP9:%.*]] = or i1 [[TMP3]], [[TMP8]] 3514; UNROLL-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3515; UNROLL: vector.ph: 3516; UNROLL-NEXT: [[N_VEC:%.*]] = and i32 [[TMP0]], 508 3517; UNROLL-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 3518; UNROLL-NEXT: [[IND_END:%.*]] = add i8 [[T]], [[DOTCAST]] 3519; UNROLL-NEXT: [[IND_END2:%.*]] = add nuw nsw i32 [[N_VEC]], [[EXT]] 3520; UNROLL-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[EXT]], i64 0 3521; UNROLL-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 3522; UNROLL-NEXT: [[INDUCTION:%.*]] = add nuw nsw <2 x i32> [[DOTSPLAT]], <i32 0, i32 1> 3523; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 3524; UNROLL: vector.body: 3525; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3526; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 3527; UNROLL-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 3528; UNROLL-NEXT: [[DOTCAST4:%.*]] = trunc i32 [[INDEX]] to i8 3529; UNROLL-NEXT: [[OFFSET_IDX:%.*]] = add i8 [[T]], [[DOTCAST4]] 3530; UNROLL-NEXT: [[TMP10:%.*]] = sext i8 [[OFFSET_IDX]] to i64 3531; UNROLL-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP10]] 3532; UNROLL-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP11]], i64 8 3533; UNROLL-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP11]], align 4 3534; UNROLL-NEXT: store <2 x i32> [[STEP_ADD]], ptr [[TMP12]], align 4 3535; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 3536; UNROLL-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 4) 3537; UNROLL-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 3538; UNROLL-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]] 3539; UNROLL: middle.block: 3540; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 3541; UNROLL-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] 3542; UNROLL: scalar.ph: 3543; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ] 3544; UNROLL-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ] 3545; UNROLL-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ] 3546; UNROLL-NEXT: br label [[LOOP:%.*]] 3547; UNROLL: loop: 3548; UNROLL-NEXT: [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 3549; UNROLL-NEXT: [[IDX_B:%.*]] = phi i32 [ [[IDX_B_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 3550; UNROLL-NEXT: [[SPHI:%.*]] = phi i32 [ [[IDX_INC_EXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ] 3551; UNROLL-NEXT: [[TMP14:%.*]] = sext i8 [[IDX]] to i64 3552; UNROLL-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP14]] 3553; UNROLL-NEXT: store i32 [[SPHI]], ptr [[PTR]], align 4 3554; UNROLL-NEXT: [[IDX_INC]] = add i8 [[IDX]], 1 3555; UNROLL-NEXT: [[IDX_INC_EXT]] = zext i8 [[IDX_INC]] to i32 3556; UNROLL-NEXT: [[IDX_B_INC]] = add nuw nsw i32 [[IDX_B]], 1 3557; UNROLL-NEXT: [[C:%.*]] = icmp ult i32 [[IDX_B]], [[LEN]] 3558; UNROLL-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP37:![0-9]+]] 3559; UNROLL: exit.loopexit: 3560; UNROLL-NEXT: br label [[EXIT]] 3561; UNROLL: exit: 3562; UNROLL-NEXT: ret void 3563; 3564; UNROLL-NO-IC-LABEL: @wrappingindvars1( 3565; UNROLL-NO-IC-NEXT: entry: 3566; UNROLL-NO-IC-NEXT: [[ST:%.*]] = zext i8 [[T:%.*]] to i16 3567; UNROLL-NO-IC-NEXT: [[EXT:%.*]] = zext i8 [[T]] to i32 3568; UNROLL-NO-IC-NEXT: [[ECMP:%.*]] = icmp ult i16 [[ST]], 42 3569; UNROLL-NO-IC-NEXT: br i1 [[ECMP]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]] 3570; UNROLL-NO-IC: loop.preheader: 3571; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[LEN:%.*]], 1 3572; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 4 3573; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 3574; UNROLL-NO-IC: vector.scevcheck: 3575; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8 3576; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]] 3577; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[T]] 3578; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255 3579; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] 3580; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = trunc i32 [[LEN]] to i8 3581; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i8 [[T]], [[TMP6]] 3582; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = icmp slt i8 [[TMP7]], [[T]] 3583; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = icmp ugt i32 [[LEN]], 255 3584; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]] 3585; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = or i1 [[TMP5]], [[TMP10]] 3586; UNROLL-NO-IC-NEXT: br i1 [[TMP11]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3587; UNROLL-NO-IC: vector.ph: 3588; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 4 3589; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]] 3590; UNROLL-NO-IC-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 3591; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = add i8 [[T]], [[DOTCAST]] 3592; UNROLL-NO-IC-NEXT: [[IND_END2:%.*]] = add i32 [[EXT]], [[N_VEC]] 3593; UNROLL-NO-IC-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[EXT]], i64 0 3594; UNROLL-NO-IC-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 3595; UNROLL-NO-IC-NEXT: [[INDUCTION:%.*]] = add <2 x i32> [[DOTSPLAT]], <i32 0, i32 1> 3596; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 3597; UNROLL-NO-IC: vector.body: 3598; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3599; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 3600; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 3601; UNROLL-NO-IC-NEXT: [[DOTCAST4:%.*]] = trunc i32 [[INDEX]] to i8 3602; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = add i8 [[T]], [[DOTCAST4]] 3603; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = add i8 [[OFFSET_IDX]], 0 3604; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i8 [[TMP12]] 3605; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i32 0 3606; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i32 2 3607; UNROLL-NO-IC-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP14]], align 4 3608; UNROLL-NO-IC-NEXT: store <2 x i32> [[STEP_ADD]], ptr [[TMP15]], align 4 3609; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 3610; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 2) 3611; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 3612; UNROLL-NO-IC-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]] 3613; UNROLL-NO-IC: middle.block: 3614; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 3615; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] 3616; UNROLL-NO-IC: scalar.ph: 3617; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ] 3618; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ] 3619; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ] 3620; UNROLL-NO-IC-NEXT: br label [[LOOP:%.*]] 3621; UNROLL-NO-IC: loop: 3622; UNROLL-NO-IC-NEXT: [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 3623; UNROLL-NO-IC-NEXT: [[IDX_B:%.*]] = phi i32 [ [[IDX_B_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 3624; UNROLL-NO-IC-NEXT: [[SPHI:%.*]] = phi i32 [ [[IDX_INC_EXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ] 3625; UNROLL-NO-IC-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i8 [[IDX]] 3626; UNROLL-NO-IC-NEXT: store i32 [[SPHI]], ptr [[PTR]], align 4 3627; UNROLL-NO-IC-NEXT: [[IDX_INC]] = add i8 [[IDX]], 1 3628; UNROLL-NO-IC-NEXT: [[IDX_INC_EXT]] = zext i8 [[IDX_INC]] to i32 3629; UNROLL-NO-IC-NEXT: [[IDX_B_INC]] = add nuw nsw i32 [[IDX_B]], 1 3630; UNROLL-NO-IC-NEXT: [[C:%.*]] = icmp ult i32 [[IDX_B]], [[LEN]] 3631; UNROLL-NO-IC-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP37:![0-9]+]] 3632; UNROLL-NO-IC: exit.loopexit: 3633; UNROLL-NO-IC-NEXT: br label [[EXIT]] 3634; UNROLL-NO-IC: exit: 3635; UNROLL-NO-IC-NEXT: ret void 3636; 3637; INTERLEAVE-LABEL: @wrappingindvars1( 3638; INTERLEAVE-NEXT: entry: 3639; INTERLEAVE-NEXT: [[EXT:%.*]] = zext i8 [[T:%.*]] to i32 3640; INTERLEAVE-NEXT: [[ECMP:%.*]] = icmp ult i8 [[T]], 42 3641; INTERLEAVE-NEXT: br i1 [[ECMP]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]] 3642; INTERLEAVE: loop.preheader: 3643; INTERLEAVE-NEXT: [[TMP0:%.*]] = add i32 [[LEN:%.*]], 1 3644; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 8 3645; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 3646; INTERLEAVE: vector.scevcheck: 3647; INTERLEAVE-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8 3648; INTERLEAVE-NEXT: [[TMP2:%.*]] = xor i8 [[T]], -1 3649; INTERLEAVE-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[TMP1]] 3650; INTERLEAVE-NEXT: [[TMP4:%.*]] = trunc i32 [[LEN]] to i8 3651; INTERLEAVE-NEXT: [[TMP5:%.*]] = add i8 [[T]], [[TMP4]] 3652; INTERLEAVE-NEXT: [[TMP6:%.*]] = icmp slt i8 [[TMP5]], [[T]] 3653; INTERLEAVE-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[LEN]], 255 3654; INTERLEAVE-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] 3655; INTERLEAVE-NEXT: [[TMP9:%.*]] = or i1 [[TMP3]], [[TMP8]] 3656; INTERLEAVE-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3657; INTERLEAVE: vector.ph: 3658; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i32 [[TMP0]], 504 3659; INTERLEAVE-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 3660; INTERLEAVE-NEXT: [[IND_END:%.*]] = add i8 [[T]], [[DOTCAST]] 3661; INTERLEAVE-NEXT: [[IND_END2:%.*]] = add nuw nsw i32 [[N_VEC]], [[EXT]] 3662; INTERLEAVE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[EXT]], i64 0 3663; INTERLEAVE-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 3664; INTERLEAVE-NEXT: [[INDUCTION:%.*]] = add nuw nsw <4 x i32> [[DOTSPLAT]], <i32 0, i32 1, i32 2, i32 3> 3665; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 3666; INTERLEAVE: vector.body: 3667; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3668; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 3669; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4) 3670; INTERLEAVE-NEXT: [[DOTCAST4:%.*]] = trunc i32 [[INDEX]] to i8 3671; INTERLEAVE-NEXT: [[OFFSET_IDX:%.*]] = add i8 [[T]], [[DOTCAST4]] 3672; INTERLEAVE-NEXT: [[TMP10:%.*]] = sext i8 [[OFFSET_IDX]] to i64 3673; INTERLEAVE-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP10]] 3674; INTERLEAVE-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP11]], i64 16 3675; INTERLEAVE-NEXT: store <4 x i32> [[VEC_IND]], ptr [[TMP11]], align 4 3676; INTERLEAVE-NEXT: store <4 x i32> [[STEP_ADD]], ptr [[TMP12]], align 4 3677; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 3678; INTERLEAVE-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 8) 3679; INTERLEAVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 3680; INTERLEAVE-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]] 3681; INTERLEAVE: middle.block: 3682; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 3683; INTERLEAVE-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] 3684; INTERLEAVE: scalar.ph: 3685; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ] 3686; INTERLEAVE-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ] 3687; INTERLEAVE-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[EXT]], [[VECTOR_SCEVCHECK]] ], [ [[EXT]], [[LOOP_PREHEADER]] ] 3688; INTERLEAVE-NEXT: br label [[LOOP:%.*]] 3689; INTERLEAVE: loop: 3690; INTERLEAVE-NEXT: [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 3691; INTERLEAVE-NEXT: [[IDX_B:%.*]] = phi i32 [ [[IDX_B_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 3692; INTERLEAVE-NEXT: [[SPHI:%.*]] = phi i32 [ [[IDX_INC_EXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ] 3693; INTERLEAVE-NEXT: [[TMP14:%.*]] = sext i8 [[IDX]] to i64 3694; INTERLEAVE-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP14]] 3695; INTERLEAVE-NEXT: store i32 [[SPHI]], ptr [[PTR]], align 4 3696; INTERLEAVE-NEXT: [[IDX_INC]] = add i8 [[IDX]], 1 3697; INTERLEAVE-NEXT: [[IDX_INC_EXT]] = zext i8 [[IDX_INC]] to i32 3698; INTERLEAVE-NEXT: [[IDX_B_INC]] = add nuw nsw i32 [[IDX_B]], 1 3699; INTERLEAVE-NEXT: [[C:%.*]] = icmp ult i32 [[IDX_B]], [[LEN]] 3700; INTERLEAVE-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP37:![0-9]+]] 3701; INTERLEAVE: exit.loopexit: 3702; INTERLEAVE-NEXT: br label [[EXIT]] 3703; INTERLEAVE: exit: 3704; INTERLEAVE-NEXT: ret void 3705; 3706 entry: 3707 %st = zext i8 %t to i16 3708 %ext = zext i8 %t to i32 3709 %ecmp = icmp ult i16 %st, 42 3710 br i1 %ecmp, label %loop, label %exit 3711 3712 loop: 3713 3714 %idx = phi i8 [ %t, %entry ], [ %idx.inc, %loop ] 3715 %idx.b = phi i32 [ 0, %entry ], [ %idx.b.inc, %loop ] 3716 %sphi = phi i32 [ %ext, %entry ], [%idx.inc.ext, %loop] 3717 3718 %ptr = getelementptr inbounds i32, ptr %A, i8 %idx 3719 store i32 %sphi, ptr %ptr 3720 3721 %idx.inc = add i8 %idx, 1 3722 %idx.inc.ext = zext i8 %idx.inc to i32 3723 %idx.b.inc = add nuw nsw i32 %idx.b, 1 3724 3725 %c = icmp ult i32 %idx.b, %len 3726 br i1 %c, label %loop, label %exit 3727 3728 exit: 3729 ret void 3730} 3731 3732; The SCEV expression of %sphi is (4 * (zext i8 {%t,+,1}<%loop> to i32)) 3733; In order to recognize %sphi as an induction PHI and vectorize this loop, 3734; we need to convert the SCEV expression into an AddRecExpr. 3735; The expression gets converted to ({4 * (zext %t to i32),+,4}). 3736define void @wrappingindvars2(i8 %t, i32 %len, ptr %A) { 3737; CHECK-LABEL: @wrappingindvars2( 3738; CHECK-NEXT: entry: 3739; CHECK-NEXT: [[ST:%.*]] = zext i8 [[T:%.*]] to i16 3740; CHECK-NEXT: [[EXT:%.*]] = zext i8 [[T]] to i32 3741; CHECK-NEXT: [[EXT_MUL:%.*]] = mul i32 [[EXT]], 4 3742; CHECK-NEXT: [[ECMP:%.*]] = icmp ult i16 [[ST]], 42 3743; CHECK-NEXT: br i1 [[ECMP]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]] 3744; CHECK: loop.preheader: 3745; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[LEN:%.*]], 1 3746; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 2 3747; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 3748; CHECK: vector.scevcheck: 3749; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8 3750; CHECK-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]] 3751; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[T]] 3752; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255 3753; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] 3754; CHECK-NEXT: [[TMP6:%.*]] = trunc i32 [[LEN]] to i8 3755; CHECK-NEXT: [[TMP7:%.*]] = add i8 [[T]], [[TMP6]] 3756; CHECK-NEXT: [[TMP8:%.*]] = icmp slt i8 [[TMP7]], [[T]] 3757; CHECK-NEXT: [[TMP9:%.*]] = icmp ugt i32 [[LEN]], 255 3758; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]] 3759; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP5]], [[TMP10]] 3760; CHECK-NEXT: br i1 [[TMP11]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3761; CHECK: vector.ph: 3762; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2 3763; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]] 3764; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 3765; CHECK-NEXT: [[IND_END:%.*]] = add i8 [[T]], [[DOTCAST]] 3766; CHECK-NEXT: [[TMP12:%.*]] = mul i32 [[N_VEC]], 4 3767; CHECK-NEXT: [[IND_END1:%.*]] = add i32 [[EXT_MUL]], [[TMP12]] 3768; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[EXT_MUL]], i64 0 3769; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 3770; CHECK-NEXT: [[INDUCTION:%.*]] = add <2 x i32> [[DOTSPLAT]], <i32 0, i32 4> 3771; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 3772; CHECK: vector.body: 3773; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3774; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 3775; CHECK-NEXT: [[DOTCAST4:%.*]] = trunc i32 [[INDEX]] to i8 3776; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i8 [[T]], [[DOTCAST4]] 3777; CHECK-NEXT: [[TMP13:%.*]] = add i8 [[OFFSET_IDX]], 0 3778; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i8 [[TMP13]] 3779; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0 3780; CHECK-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP15]], align 4 3781; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 3782; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 8) 3783; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 3784; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]] 3785; CHECK: middle.block: 3786; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 3787; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] 3788; CHECK: scalar.ph: 3789; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ] 3790; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ] 3791; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ] 3792; CHECK-NEXT: br label [[LOOP:%.*]] 3793; CHECK: loop: 3794; CHECK-NEXT: [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 3795; CHECK-NEXT: [[SPHI:%.*]] = phi i32 [ [[MUL:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ] 3796; CHECK-NEXT: [[IDX_B:%.*]] = phi i32 [ [[IDX_B_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ] 3797; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i8 [[IDX]] 3798; CHECK-NEXT: store i32 [[SPHI]], ptr [[PTR]], align 4 3799; CHECK-NEXT: [[IDX_INC]] = add i8 [[IDX]], 1 3800; CHECK-NEXT: [[IDX_INC_EXT:%.*]] = zext i8 [[IDX_INC]] to i32 3801; CHECK-NEXT: [[MUL]] = mul i32 [[IDX_INC_EXT]], 4 3802; CHECK-NEXT: [[IDX_B_INC]] = add nuw nsw i32 [[IDX_B]], 1 3803; CHECK-NEXT: [[C:%.*]] = icmp ult i32 [[IDX_B]], [[LEN]] 3804; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP39:![0-9]+]] 3805; CHECK: exit.loopexit: 3806; CHECK-NEXT: br label [[EXIT]] 3807; CHECK: exit: 3808; CHECK-NEXT: ret void 3809; 3810; IND-LABEL: @wrappingindvars2( 3811; IND-NEXT: entry: 3812; IND-NEXT: [[EXT:%.*]] = zext i8 [[T:%.*]] to i32 3813; IND-NEXT: [[EXT_MUL:%.*]] = shl nuw nsw i32 [[EXT]], 2 3814; IND-NEXT: [[ECMP:%.*]] = icmp ult i8 [[T]], 42 3815; IND-NEXT: br i1 [[ECMP]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]] 3816; IND: loop.preheader: 3817; IND-NEXT: [[TMP0:%.*]] = add i32 [[LEN:%.*]], 1 3818; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 2 3819; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 3820; IND: vector.scevcheck: 3821; IND-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8 3822; IND-NEXT: [[TMP2:%.*]] = xor i8 [[T]], -1 3823; IND-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[TMP1]] 3824; IND-NEXT: [[TMP4:%.*]] = trunc i32 [[LEN]] to i8 3825; IND-NEXT: [[TMP5:%.*]] = add i8 [[T]], [[TMP4]] 3826; IND-NEXT: [[TMP6:%.*]] = icmp slt i8 [[TMP5]], [[T]] 3827; IND-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[LEN]], 255 3828; IND-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] 3829; IND-NEXT: [[TMP9:%.*]] = or i1 [[TMP3]], [[TMP8]] 3830; IND-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3831; IND: vector.ph: 3832; IND-NEXT: [[N_VEC:%.*]] = and i32 [[TMP0]], 510 3833; IND-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 3834; IND-NEXT: [[IND_END:%.*]] = add i8 [[T]], [[DOTCAST]] 3835; IND-NEXT: [[EXT_MUL5:%.*]] = add nuw nsw i32 [[N_VEC]], [[EXT]] 3836; IND-NEXT: [[IND_END1:%.*]] = shl nuw nsw i32 [[EXT_MUL5]], 2 3837; IND-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[EXT_MUL]], i64 0 3838; IND-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 3839; IND-NEXT: [[INDUCTION:%.*]] = add nuw nsw <2 x i32> [[DOTSPLAT]], <i32 0, i32 4> 3840; IND-NEXT: br label [[VECTOR_BODY:%.*]] 3841; IND: vector.body: 3842; IND-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3843; IND-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 3844; IND-NEXT: [[DOTCAST4:%.*]] = trunc i32 [[INDEX]] to i8 3845; IND-NEXT: [[OFFSET_IDX:%.*]] = add i8 [[T]], [[DOTCAST4]] 3846; IND-NEXT: [[TMP10:%.*]] = sext i8 [[OFFSET_IDX]] to i64 3847; IND-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP10]] 3848; IND-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP11]], align 4 3849; IND-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 3850; IND-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 8) 3851; IND-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 3852; IND-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]] 3853; IND: middle.block: 3854; IND-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 3855; IND-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] 3856; IND: scalar.ph: 3857; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ] 3858; IND-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ] 3859; IND-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ] 3860; IND-NEXT: br label [[LOOP:%.*]] 3861; IND: loop: 3862; IND-NEXT: [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 3863; IND-NEXT: [[SPHI:%.*]] = phi i32 [ [[MUL:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ] 3864; IND-NEXT: [[IDX_B:%.*]] = phi i32 [ [[IDX_B_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ] 3865; IND-NEXT: [[TMP13:%.*]] = sext i8 [[IDX]] to i64 3866; IND-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP13]] 3867; IND-NEXT: store i32 [[SPHI]], ptr [[PTR]], align 4 3868; IND-NEXT: [[IDX_INC]] = add i8 [[IDX]], 1 3869; IND-NEXT: [[IDX_INC_EXT:%.*]] = zext i8 [[IDX_INC]] to i32 3870; IND-NEXT: [[MUL]] = shl nuw nsw i32 [[IDX_INC_EXT]], 2 3871; IND-NEXT: [[IDX_B_INC]] = add nuw nsw i32 [[IDX_B]], 1 3872; IND-NEXT: [[C:%.*]] = icmp ult i32 [[IDX_B]], [[LEN]] 3873; IND-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP39:![0-9]+]] 3874; IND: exit.loopexit: 3875; IND-NEXT: br label [[EXIT]] 3876; IND: exit: 3877; IND-NEXT: ret void 3878; 3879; UNROLL-LABEL: @wrappingindvars2( 3880; UNROLL-NEXT: entry: 3881; UNROLL-NEXT: [[EXT:%.*]] = zext i8 [[T:%.*]] to i32 3882; UNROLL-NEXT: [[EXT_MUL:%.*]] = shl nuw nsw i32 [[EXT]], 2 3883; UNROLL-NEXT: [[ECMP:%.*]] = icmp ult i8 [[T]], 42 3884; UNROLL-NEXT: br i1 [[ECMP]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]] 3885; UNROLL: loop.preheader: 3886; UNROLL-NEXT: [[TMP0:%.*]] = add i32 [[LEN:%.*]], 1 3887; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 4 3888; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 3889; UNROLL: vector.scevcheck: 3890; UNROLL-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8 3891; UNROLL-NEXT: [[TMP2:%.*]] = xor i8 [[T]], -1 3892; UNROLL-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[TMP1]] 3893; UNROLL-NEXT: [[TMP4:%.*]] = trunc i32 [[LEN]] to i8 3894; UNROLL-NEXT: [[TMP5:%.*]] = add i8 [[T]], [[TMP4]] 3895; UNROLL-NEXT: [[TMP6:%.*]] = icmp slt i8 [[TMP5]], [[T]] 3896; UNROLL-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[LEN]], 255 3897; UNROLL-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] 3898; UNROLL-NEXT: [[TMP9:%.*]] = or i1 [[TMP3]], [[TMP8]] 3899; UNROLL-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3900; UNROLL: vector.ph: 3901; UNROLL-NEXT: [[N_VEC:%.*]] = and i32 [[TMP0]], 508 3902; UNROLL-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 3903; UNROLL-NEXT: [[IND_END:%.*]] = add i8 [[T]], [[DOTCAST]] 3904; UNROLL-NEXT: [[EXT_MUL5:%.*]] = add nuw nsw i32 [[N_VEC]], [[EXT]] 3905; UNROLL-NEXT: [[IND_END1:%.*]] = shl nuw nsw i32 [[EXT_MUL5]], 2 3906; UNROLL-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[EXT_MUL]], i64 0 3907; UNROLL-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 3908; UNROLL-NEXT: [[INDUCTION:%.*]] = add nuw nsw <2 x i32> [[DOTSPLAT]], <i32 0, i32 4> 3909; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 3910; UNROLL: vector.body: 3911; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3912; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 3913; UNROLL-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 8) 3914; UNROLL-NEXT: [[DOTCAST4:%.*]] = trunc i32 [[INDEX]] to i8 3915; UNROLL-NEXT: [[OFFSET_IDX:%.*]] = add i8 [[T]], [[DOTCAST4]] 3916; UNROLL-NEXT: [[TMP10:%.*]] = sext i8 [[OFFSET_IDX]] to i64 3917; UNROLL-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP10]] 3918; UNROLL-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP11]], i64 8 3919; UNROLL-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP11]], align 4 3920; UNROLL-NEXT: store <2 x i32> [[STEP_ADD]], ptr [[TMP12]], align 4 3921; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 3922; UNROLL-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 16) 3923; UNROLL-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 3924; UNROLL-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]] 3925; UNROLL: middle.block: 3926; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 3927; UNROLL-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] 3928; UNROLL: scalar.ph: 3929; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ] 3930; UNROLL-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ] 3931; UNROLL-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ] 3932; UNROLL-NEXT: br label [[LOOP:%.*]] 3933; UNROLL: loop: 3934; UNROLL-NEXT: [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 3935; UNROLL-NEXT: [[SPHI:%.*]] = phi i32 [ [[MUL:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ] 3936; UNROLL-NEXT: [[IDX_B:%.*]] = phi i32 [ [[IDX_B_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ] 3937; UNROLL-NEXT: [[TMP14:%.*]] = sext i8 [[IDX]] to i64 3938; UNROLL-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP14]] 3939; UNROLL-NEXT: store i32 [[SPHI]], ptr [[PTR]], align 4 3940; UNROLL-NEXT: [[IDX_INC]] = add i8 [[IDX]], 1 3941; UNROLL-NEXT: [[IDX_INC_EXT:%.*]] = zext i8 [[IDX_INC]] to i32 3942; UNROLL-NEXT: [[MUL]] = shl nuw nsw i32 [[IDX_INC_EXT]], 2 3943; UNROLL-NEXT: [[IDX_B_INC]] = add nuw nsw i32 [[IDX_B]], 1 3944; UNROLL-NEXT: [[C:%.*]] = icmp ult i32 [[IDX_B]], [[LEN]] 3945; UNROLL-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP39:![0-9]+]] 3946; UNROLL: exit.loopexit: 3947; UNROLL-NEXT: br label [[EXIT]] 3948; UNROLL: exit: 3949; UNROLL-NEXT: ret void 3950; 3951; UNROLL-NO-IC-LABEL: @wrappingindvars2( 3952; UNROLL-NO-IC-NEXT: entry: 3953; UNROLL-NO-IC-NEXT: [[ST:%.*]] = zext i8 [[T:%.*]] to i16 3954; UNROLL-NO-IC-NEXT: [[EXT:%.*]] = zext i8 [[T]] to i32 3955; UNROLL-NO-IC-NEXT: [[EXT_MUL:%.*]] = mul i32 [[EXT]], 4 3956; UNROLL-NO-IC-NEXT: [[ECMP:%.*]] = icmp ult i16 [[ST]], 42 3957; UNROLL-NO-IC-NEXT: br i1 [[ECMP]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]] 3958; UNROLL-NO-IC: loop.preheader: 3959; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[LEN:%.*]], 1 3960; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 4 3961; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 3962; UNROLL-NO-IC: vector.scevcheck: 3963; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8 3964; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = add i8 [[T]], [[TMP1]] 3965; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[T]] 3966; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[LEN]], 255 3967; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] 3968; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = trunc i32 [[LEN]] to i8 3969; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i8 [[T]], [[TMP6]] 3970; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = icmp slt i8 [[TMP7]], [[T]] 3971; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = icmp ugt i32 [[LEN]], 255 3972; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]] 3973; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = or i1 [[TMP5]], [[TMP10]] 3974; UNROLL-NO-IC-NEXT: br i1 [[TMP11]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 3975; UNROLL-NO-IC: vector.ph: 3976; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 4 3977; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]] 3978; UNROLL-NO-IC-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 3979; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = add i8 [[T]], [[DOTCAST]] 3980; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = mul i32 [[N_VEC]], 4 3981; UNROLL-NO-IC-NEXT: [[IND_END1:%.*]] = add i32 [[EXT_MUL]], [[TMP12]] 3982; UNROLL-NO-IC-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[EXT_MUL]], i64 0 3983; UNROLL-NO-IC-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 3984; UNROLL-NO-IC-NEXT: [[INDUCTION:%.*]] = add <2 x i32> [[DOTSPLAT]], <i32 0, i32 4> 3985; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 3986; UNROLL-NO-IC: vector.body: 3987; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 3988; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 3989; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 8) 3990; UNROLL-NO-IC-NEXT: [[DOTCAST4:%.*]] = trunc i32 [[INDEX]] to i8 3991; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = add i8 [[T]], [[DOTCAST4]] 3992; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = add i8 [[OFFSET_IDX]], 0 3993; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i8 [[TMP13]] 3994; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0 3995; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 2 3996; UNROLL-NO-IC-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP15]], align 4 3997; UNROLL-NO-IC-NEXT: store <2 x i32> [[STEP_ADD]], ptr [[TMP16]], align 4 3998; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 3999; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 8) 4000; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4001; UNROLL-NO-IC-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]] 4002; UNROLL-NO-IC: middle.block: 4003; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 4004; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] 4005; UNROLL-NO-IC: scalar.ph: 4006; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ] 4007; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ] 4008; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ] 4009; UNROLL-NO-IC-NEXT: br label [[LOOP:%.*]] 4010; UNROLL-NO-IC: loop: 4011; UNROLL-NO-IC-NEXT: [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4012; UNROLL-NO-IC-NEXT: [[SPHI:%.*]] = phi i32 [ [[MUL:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ] 4013; UNROLL-NO-IC-NEXT: [[IDX_B:%.*]] = phi i32 [ [[IDX_B_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ] 4014; UNROLL-NO-IC-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i8 [[IDX]] 4015; UNROLL-NO-IC-NEXT: store i32 [[SPHI]], ptr [[PTR]], align 4 4016; UNROLL-NO-IC-NEXT: [[IDX_INC]] = add i8 [[IDX]], 1 4017; UNROLL-NO-IC-NEXT: [[IDX_INC_EXT:%.*]] = zext i8 [[IDX_INC]] to i32 4018; UNROLL-NO-IC-NEXT: [[MUL]] = mul i32 [[IDX_INC_EXT]], 4 4019; UNROLL-NO-IC-NEXT: [[IDX_B_INC]] = add nuw nsw i32 [[IDX_B]], 1 4020; UNROLL-NO-IC-NEXT: [[C:%.*]] = icmp ult i32 [[IDX_B]], [[LEN]] 4021; UNROLL-NO-IC-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP39:![0-9]+]] 4022; UNROLL-NO-IC: exit.loopexit: 4023; UNROLL-NO-IC-NEXT: br label [[EXIT]] 4024; UNROLL-NO-IC: exit: 4025; UNROLL-NO-IC-NEXT: ret void 4026; 4027; INTERLEAVE-LABEL: @wrappingindvars2( 4028; INTERLEAVE-NEXT: entry: 4029; INTERLEAVE-NEXT: [[EXT:%.*]] = zext i8 [[T:%.*]] to i32 4030; INTERLEAVE-NEXT: [[EXT_MUL:%.*]] = shl nuw nsw i32 [[EXT]], 2 4031; INTERLEAVE-NEXT: [[ECMP:%.*]] = icmp ult i8 [[T]], 42 4032; INTERLEAVE-NEXT: br i1 [[ECMP]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]] 4033; INTERLEAVE: loop.preheader: 4034; INTERLEAVE-NEXT: [[TMP0:%.*]] = add i32 [[LEN:%.*]], 1 4035; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 8 4036; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 4037; INTERLEAVE: vector.scevcheck: 4038; INTERLEAVE-NEXT: [[TMP1:%.*]] = trunc i32 [[LEN]] to i8 4039; INTERLEAVE-NEXT: [[TMP2:%.*]] = xor i8 [[T]], -1 4040; INTERLEAVE-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[TMP1]] 4041; INTERLEAVE-NEXT: [[TMP4:%.*]] = trunc i32 [[LEN]] to i8 4042; INTERLEAVE-NEXT: [[TMP5:%.*]] = add i8 [[T]], [[TMP4]] 4043; INTERLEAVE-NEXT: [[TMP6:%.*]] = icmp slt i8 [[TMP5]], [[T]] 4044; INTERLEAVE-NEXT: [[TMP7:%.*]] = icmp ugt i32 [[LEN]], 255 4045; INTERLEAVE-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] 4046; INTERLEAVE-NEXT: [[TMP9:%.*]] = or i1 [[TMP3]], [[TMP8]] 4047; INTERLEAVE-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 4048; INTERLEAVE: vector.ph: 4049; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i32 [[TMP0]], 504 4050; INTERLEAVE-NEXT: [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i8 4051; INTERLEAVE-NEXT: [[IND_END:%.*]] = add i8 [[T]], [[DOTCAST]] 4052; INTERLEAVE-NEXT: [[EXT_MUL5:%.*]] = add nuw nsw i32 [[N_VEC]], [[EXT]] 4053; INTERLEAVE-NEXT: [[IND_END1:%.*]] = shl nuw nsw i32 [[EXT_MUL5]], 2 4054; INTERLEAVE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[EXT_MUL]], i64 0 4055; INTERLEAVE-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 4056; INTERLEAVE-NEXT: [[INDUCTION:%.*]] = add nuw nsw <4 x i32> [[DOTSPLAT]], <i32 0, i32 4, i32 8, i32 12> 4057; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 4058; INTERLEAVE: vector.body: 4059; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4060; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4061; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 16) 4062; INTERLEAVE-NEXT: [[DOTCAST4:%.*]] = trunc i32 [[INDEX]] to i8 4063; INTERLEAVE-NEXT: [[OFFSET_IDX:%.*]] = add i8 [[T]], [[DOTCAST4]] 4064; INTERLEAVE-NEXT: [[TMP10:%.*]] = sext i8 [[OFFSET_IDX]] to i64 4065; INTERLEAVE-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP10]] 4066; INTERLEAVE-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP11]], i64 16 4067; INTERLEAVE-NEXT: store <4 x i32> [[VEC_IND]], ptr [[TMP11]], align 4 4068; INTERLEAVE-NEXT: store <4 x i32> [[STEP_ADD]], ptr [[TMP12]], align 4 4069; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 4070; INTERLEAVE-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 32) 4071; INTERLEAVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4072; INTERLEAVE-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]] 4073; INTERLEAVE: middle.block: 4074; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 4075; INTERLEAVE-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] 4076; INTERLEAVE: scalar.ph: 4077; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[T]], [[VECTOR_SCEVCHECK]] ], [ [[T]], [[LOOP_PREHEADER]] ] 4078; INTERLEAVE-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END1]], [[MIDDLE_BLOCK]] ], [ [[EXT_MUL]], [[VECTOR_SCEVCHECK]] ], [ [[EXT_MUL]], [[LOOP_PREHEADER]] ] 4079; INTERLEAVE-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[LOOP_PREHEADER]] ] 4080; INTERLEAVE-NEXT: br label [[LOOP:%.*]] 4081; INTERLEAVE: loop: 4082; INTERLEAVE-NEXT: [[IDX:%.*]] = phi i8 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4083; INTERLEAVE-NEXT: [[SPHI:%.*]] = phi i32 [ [[MUL:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ] 4084; INTERLEAVE-NEXT: [[IDX_B:%.*]] = phi i32 [ [[IDX_B_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ] 4085; INTERLEAVE-NEXT: [[TMP14:%.*]] = sext i8 [[IDX]] to i64 4086; INTERLEAVE-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP14]] 4087; INTERLEAVE-NEXT: store i32 [[SPHI]], ptr [[PTR]], align 4 4088; INTERLEAVE-NEXT: [[IDX_INC]] = add i8 [[IDX]], 1 4089; INTERLEAVE-NEXT: [[IDX_INC_EXT:%.*]] = zext i8 [[IDX_INC]] to i32 4090; INTERLEAVE-NEXT: [[MUL]] = shl nuw nsw i32 [[IDX_INC_EXT]], 2 4091; INTERLEAVE-NEXT: [[IDX_B_INC]] = add nuw nsw i32 [[IDX_B]], 1 4092; INTERLEAVE-NEXT: [[C:%.*]] = icmp ult i32 [[IDX_B]], [[LEN]] 4093; INTERLEAVE-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT_LOOPEXIT]], !llvm.loop [[LOOP39:![0-9]+]] 4094; INTERLEAVE: exit.loopexit: 4095; INTERLEAVE-NEXT: br label [[EXIT]] 4096; INTERLEAVE: exit: 4097; INTERLEAVE-NEXT: ret void 4098; 4099entry: 4100 %st = zext i8 %t to i16 4101 %ext = zext i8 %t to i32 4102 %ext.mul = mul i32 %ext, 4 4103 4104 %ecmp = icmp ult i16 %st, 42 4105 br i1 %ecmp, label %loop, label %exit 4106 4107 loop: 4108 4109 %idx = phi i8 [ %t, %entry ], [ %idx.inc, %loop ] 4110 %sphi = phi i32 [ %ext.mul, %entry ], [%mul, %loop] 4111 %idx.b = phi i32 [ 0, %entry ], [ %idx.b.inc, %loop ] 4112 4113 %ptr = getelementptr inbounds i32, ptr %A, i8 %idx 4114 store i32 %sphi, ptr %ptr 4115 4116 %idx.inc = add i8 %idx, 1 4117 %idx.inc.ext = zext i8 %idx.inc to i32 4118 %mul = mul i32 %idx.inc.ext, 4 4119 %idx.b.inc = add nuw nsw i32 %idx.b, 1 4120 4121 %c = icmp ult i32 %idx.b, %len 4122 br i1 %c, label %loop, label %exit 4123 4124 exit: 4125 ret void 4126} 4127 4128; Check that we generate vectorized IVs in the pre-header 4129; instead of widening the scalar IV inside the loop, when 4130; we know how to do that. 4131define void @veciv(ptr nocapture %a, i32 %start, i32 %k) { 4132; CHECK-LABEL: @veciv( 4133; CHECK-NEXT: for.body.preheader: 4134; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K:%.*]], 2 4135; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4136; CHECK: vector.ph: 4137; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 2 4138; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]] 4139; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 4140; CHECK: vector.body: 4141; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4142; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4143; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 4144; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP0]] 4145; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0 4146; CHECK-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP2]], align 4 4147; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 4148; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 4149; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4150; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]] 4151; CHECK: middle.block: 4152; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]] 4153; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4154; CHECK: scalar.ph: 4155; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ] 4156; CHECK-NEXT: br label [[FOR_BODY:%.*]] 4157; CHECK: for.body: 4158; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4159; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDVARS_IV]] 4160; CHECK-NEXT: store i32 [[INDVARS_IV]], ptr [[ARRAYIDX]], align 4 4161; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4162; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[K]] 4163; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 4164; CHECK: exit: 4165; CHECK-NEXT: ret void 4166; 4167; IND-LABEL: @veciv( 4168; IND-NEXT: for.body.preheader: 4169; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K:%.*]], 2 4170; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4171; IND: vector.ph: 4172; IND-NEXT: [[N_VEC:%.*]] = and i32 [[K]], -2 4173; IND-NEXT: br label [[VECTOR_BODY:%.*]] 4174; IND: vector.body: 4175; IND-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4176; IND-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4177; IND-NEXT: [[TMP0:%.*]] = sext i32 [[INDEX]] to i64 4178; IND-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]] 4179; IND-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP1]], align 4 4180; IND-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 4181; IND-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 4182; IND-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4183; IND-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]] 4184; IND: middle.block: 4185; IND-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]] 4186; IND-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4187; IND: scalar.ph: 4188; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ] 4189; IND-NEXT: br label [[FOR_BODY:%.*]] 4190; IND: for.body: 4191; IND-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4192; IND-NEXT: [[TMP3:%.*]] = sext i32 [[INDVARS_IV]] to i64 4193; IND-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP3]] 4194; IND-NEXT: store i32 [[INDVARS_IV]], ptr [[ARRAYIDX]], align 4 4195; IND-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4196; IND-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[K]] 4197; IND-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 4198; IND: exit: 4199; IND-NEXT: ret void 4200; 4201; UNROLL-LABEL: @veciv( 4202; UNROLL-NEXT: for.body.preheader: 4203; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K:%.*]], 4 4204; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4205; UNROLL: vector.ph: 4206; UNROLL-NEXT: [[N_VEC:%.*]] = and i32 [[K]], -4 4207; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 4208; UNROLL: vector.body: 4209; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4210; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4211; UNROLL-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 4212; UNROLL-NEXT: [[TMP0:%.*]] = sext i32 [[INDEX]] to i64 4213; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]] 4214; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 8 4215; UNROLL-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP1]], align 4 4216; UNROLL-NEXT: store <2 x i32> [[STEP_ADD]], ptr [[TMP2]], align 4 4217; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 4218; UNROLL-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 4) 4219; UNROLL-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4220; UNROLL-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]] 4221; UNROLL: middle.block: 4222; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]] 4223; UNROLL-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4224; UNROLL: scalar.ph: 4225; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ] 4226; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 4227; UNROLL: for.body: 4228; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4229; UNROLL-NEXT: [[TMP4:%.*]] = sext i32 [[INDVARS_IV]] to i64 4230; UNROLL-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]] 4231; UNROLL-NEXT: store i32 [[INDVARS_IV]], ptr [[ARRAYIDX]], align 4 4232; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4233; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[K]] 4234; UNROLL-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 4235; UNROLL: exit: 4236; UNROLL-NEXT: ret void 4237; 4238; UNROLL-NO-IC-LABEL: @veciv( 4239; UNROLL-NO-IC-NEXT: for.body.preheader: 4240; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K:%.*]], 4 4241; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4242; UNROLL-NO-IC: vector.ph: 4243; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 4 4244; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]] 4245; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 4246; UNROLL-NO-IC: vector.body: 4247; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4248; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4249; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 4250; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 4251; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP0]] 4252; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0 4253; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 2 4254; UNROLL-NO-IC-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP2]], align 4 4255; UNROLL-NO-IC-NEXT: store <2 x i32> [[STEP_ADD]], ptr [[TMP3]], align 4 4256; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 4257; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 2) 4258; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4259; UNROLL-NO-IC-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]] 4260; UNROLL-NO-IC: middle.block: 4261; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]] 4262; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4263; UNROLL-NO-IC: scalar.ph: 4264; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ] 4265; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 4266; UNROLL-NO-IC: for.body: 4267; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4268; UNROLL-NO-IC-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDVARS_IV]] 4269; UNROLL-NO-IC-NEXT: store i32 [[INDVARS_IV]], ptr [[ARRAYIDX]], align 4 4270; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4271; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[K]] 4272; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 4273; UNROLL-NO-IC: exit: 4274; UNROLL-NO-IC-NEXT: ret void 4275; 4276; INTERLEAVE-LABEL: @veciv( 4277; INTERLEAVE-NEXT: for.body.preheader: 4278; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K:%.*]], 8 4279; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4280; INTERLEAVE: vector.ph: 4281; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i32 [[K]], -8 4282; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 4283; INTERLEAVE: vector.body: 4284; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4285; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4286; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4) 4287; INTERLEAVE-NEXT: [[TMP0:%.*]] = sext i32 [[INDEX]] to i64 4288; INTERLEAVE-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]] 4289; INTERLEAVE-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16 4290; INTERLEAVE-NEXT: store <4 x i32> [[VEC_IND]], ptr [[TMP1]], align 4 4291; INTERLEAVE-NEXT: store <4 x i32> [[STEP_ADD]], ptr [[TMP2]], align 4 4292; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 4293; INTERLEAVE-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 8) 4294; INTERLEAVE-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4295; INTERLEAVE-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]] 4296; INTERLEAVE: middle.block: 4297; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]] 4298; INTERLEAVE-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4299; INTERLEAVE: scalar.ph: 4300; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ] 4301; INTERLEAVE-NEXT: br label [[FOR_BODY:%.*]] 4302; INTERLEAVE: for.body: 4303; INTERLEAVE-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4304; INTERLEAVE-NEXT: [[TMP4:%.*]] = sext i32 [[INDVARS_IV]] to i64 4305; INTERLEAVE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]] 4306; INTERLEAVE-NEXT: store i32 [[INDVARS_IV]], ptr [[ARRAYIDX]], align 4 4307; INTERLEAVE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4308; INTERLEAVE-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[K]] 4309; INTERLEAVE-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] 4310; INTERLEAVE: exit: 4311; INTERLEAVE-NEXT: ret void 4312; 4313for.body.preheader: 4314 br label %for.body 4315 4316for.body: 4317 %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ] 4318 %arrayidx = getelementptr inbounds i32, ptr %a, i32 %indvars.iv 4319 store i32 %indvars.iv, ptr %arrayidx, align 4 4320 %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1 4321 %exitcond = icmp eq i32 %indvars.iv.next, %k 4322 br i1 %exitcond, label %exit, label %for.body 4323 4324exit: 4325 ret void 4326} 4327 4328define void @trunciv(ptr nocapture %a, i32 %start, i64 %k) { 4329; CHECK-LABEL: @trunciv( 4330; CHECK-NEXT: for.body.preheader: 4331; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[K:%.*]], 2 4332; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 4333; CHECK: vector.scevcheck: 4334; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[K]], -1 4335; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32 4336; CHECK-NEXT: [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0 4337; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt i64 [[TMP0]], 4294967295 4338; CHECK-NEXT: [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]] 4339; CHECK-NEXT: br i1 [[TMP4]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 4340; CHECK: vector.ph: 4341; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[K]], 2 4342; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[K]], [[N_MOD_VF]] 4343; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 4344; CHECK: vector.body: 4345; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4346; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4347; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[INDEX]] to i32 4348; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[TMP5]], 0 4349; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP6]] 4350; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0 4351; CHECK-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP8]], align 4 4352; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 4353; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 4354; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4355; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] 4356; CHECK: middle.block: 4357; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[K]], [[N_VEC]] 4358; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4359; CHECK: scalar.ph: 4360; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ] 4361; CHECK-NEXT: br label [[FOR_BODY:%.*]] 4362; CHECK: for.body: 4363; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4364; CHECK-NEXT: [[TRUNC_IV:%.*]] = trunc i64 [[INDVARS_IV]] to i32 4365; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TRUNC_IV]] 4366; CHECK-NEXT: store i32 [[TRUNC_IV]], ptr [[ARRAYIDX]], align 4 4367; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 4368; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[K]] 4369; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]] 4370; CHECK: exit: 4371; CHECK-NEXT: ret void 4372; 4373; IND-LABEL: @trunciv( 4374; IND-NEXT: for.body.preheader: 4375; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[K:%.*]], 2 4376; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 4377; IND: vector.scevcheck: 4378; IND-NEXT: [[DOTNOT:%.*]] = icmp ult i64 [[K]], 2147483649 4379; IND-NEXT: br i1 [[DOTNOT]], label [[VECTOR_PH:%.*]], label [[SCALAR_PH]] 4380; IND: vector.ph: 4381; IND-NEXT: [[N_VEC:%.*]] = and i64 [[K]], 4294967294 4382; IND-NEXT: br label [[VECTOR_BODY:%.*]] 4383; IND: vector.body: 4384; IND-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4385; IND-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4386; IND-NEXT: [[SEXT:%.*]] = shl i64 [[INDEX]], 32 4387; IND-NEXT: [[TMP0:%.*]] = ashr exact i64 [[SEXT]], 30 4388; IND-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP0]] 4389; IND-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP1]], align 4 4390; IND-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 4391; IND-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 4392; IND-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4393; IND-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] 4394; IND: middle.block: 4395; IND-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[K]], [[N_VEC]] 4396; IND-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4397; IND: scalar.ph: 4398; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ] 4399; IND-NEXT: br label [[FOR_BODY:%.*]] 4400; IND: for.body: 4401; IND-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4402; IND-NEXT: [[TRUNC_IV:%.*]] = trunc i64 [[INDVARS_IV]] to i32 4403; IND-NEXT: [[SEXT1:%.*]] = shl i64 [[INDVARS_IV]], 32 4404; IND-NEXT: [[TMP3:%.*]] = ashr exact i64 [[SEXT1]], 30 4405; IND-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP3]] 4406; IND-NEXT: store i32 [[TRUNC_IV]], ptr [[ARRAYIDX]], align 4 4407; IND-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 4408; IND-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[K]] 4409; IND-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]] 4410; IND: exit: 4411; IND-NEXT: ret void 4412; 4413; UNROLL-LABEL: @trunciv( 4414; UNROLL-NEXT: for.body.preheader: 4415; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[K:%.*]], 4 4416; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 4417; UNROLL: vector.scevcheck: 4418; UNROLL-NEXT: [[DOTNOT:%.*]] = icmp ult i64 [[K]], 2147483649 4419; UNROLL-NEXT: br i1 [[DOTNOT]], label [[VECTOR_PH:%.*]], label [[SCALAR_PH]] 4420; UNROLL: vector.ph: 4421; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[K]], 4294967292 4422; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 4423; UNROLL: vector.body: 4424; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4425; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4426; UNROLL-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 4427; UNROLL-NEXT: [[SEXT:%.*]] = shl i64 [[INDEX]], 32 4428; UNROLL-NEXT: [[TMP0:%.*]] = ashr exact i64 [[SEXT]], 30 4429; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP0]] 4430; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 8 4431; UNROLL-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP1]], align 4 4432; UNROLL-NEXT: store <2 x i32> [[STEP_ADD]], ptr [[TMP2]], align 4 4433; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 4434; UNROLL-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 4) 4435; UNROLL-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4436; UNROLL-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] 4437; UNROLL: middle.block: 4438; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[K]], [[N_VEC]] 4439; UNROLL-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4440; UNROLL: scalar.ph: 4441; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ] 4442; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 4443; UNROLL: for.body: 4444; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4445; UNROLL-NEXT: [[TRUNC_IV:%.*]] = trunc i64 [[INDVARS_IV]] to i32 4446; UNROLL-NEXT: [[SEXT1:%.*]] = shl i64 [[INDVARS_IV]], 32 4447; UNROLL-NEXT: [[TMP4:%.*]] = ashr exact i64 [[SEXT1]], 30 4448; UNROLL-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP4]] 4449; UNROLL-NEXT: store i32 [[TRUNC_IV]], ptr [[ARRAYIDX]], align 4 4450; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 4451; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[K]] 4452; UNROLL-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]] 4453; UNROLL: exit: 4454; UNROLL-NEXT: ret void 4455; 4456; UNROLL-NO-IC-LABEL: @trunciv( 4457; UNROLL-NO-IC-NEXT: for.body.preheader: 4458; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[K:%.*]], 4 4459; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 4460; UNROLL-NO-IC: vector.scevcheck: 4461; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[K]], -1 4462; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32 4463; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0 4464; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp ugt i64 [[TMP0]], 4294967295 4465; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]] 4466; UNROLL-NO-IC-NEXT: br i1 [[TMP4]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 4467; UNROLL-NO-IC: vector.ph: 4468; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[K]], 4 4469; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[K]], [[N_MOD_VF]] 4470; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 4471; UNROLL-NO-IC: vector.body: 4472; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4473; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4474; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 4475; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = trunc i64 [[INDEX]] to i32 4476; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add i32 [[TMP5]], 0 4477; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP6]] 4478; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0 4479; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 2 4480; UNROLL-NO-IC-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP8]], align 4 4481; UNROLL-NO-IC-NEXT: store <2 x i32> [[STEP_ADD]], ptr [[TMP9]], align 4 4482; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 4483; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 2) 4484; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4485; UNROLL-NO-IC-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] 4486; UNROLL-NO-IC: middle.block: 4487; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[K]], [[N_VEC]] 4488; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4489; UNROLL-NO-IC: scalar.ph: 4490; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ] 4491; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 4492; UNROLL-NO-IC: for.body: 4493; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4494; UNROLL-NO-IC-NEXT: [[TRUNC_IV:%.*]] = trunc i64 [[INDVARS_IV]] to i32 4495; UNROLL-NO-IC-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TRUNC_IV]] 4496; UNROLL-NO-IC-NEXT: store i32 [[TRUNC_IV]], ptr [[ARRAYIDX]], align 4 4497; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 4498; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[K]] 4499; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]] 4500; UNROLL-NO-IC: exit: 4501; UNROLL-NO-IC-NEXT: ret void 4502; 4503; INTERLEAVE-LABEL: @trunciv( 4504; INTERLEAVE-NEXT: for.body.preheader: 4505; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[K:%.*]], 8 4506; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 4507; INTERLEAVE: vector.scevcheck: 4508; INTERLEAVE-NEXT: [[DOTNOT:%.*]] = icmp ult i64 [[K]], 2147483649 4509; INTERLEAVE-NEXT: br i1 [[DOTNOT]], label [[VECTOR_PH:%.*]], label [[SCALAR_PH]] 4510; INTERLEAVE: vector.ph: 4511; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i64 [[K]], 4294967288 4512; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 4513; INTERLEAVE: vector.body: 4514; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4515; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4516; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4) 4517; INTERLEAVE-NEXT: [[SEXT:%.*]] = shl i64 [[INDEX]], 32 4518; INTERLEAVE-NEXT: [[TMP0:%.*]] = ashr exact i64 [[SEXT]], 30 4519; INTERLEAVE-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP0]] 4520; INTERLEAVE-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16 4521; INTERLEAVE-NEXT: store <4 x i32> [[VEC_IND]], ptr [[TMP1]], align 4 4522; INTERLEAVE-NEXT: store <4 x i32> [[STEP_ADD]], ptr [[TMP2]], align 4 4523; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 4524; INTERLEAVE-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 8) 4525; INTERLEAVE-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4526; INTERLEAVE-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] 4527; INTERLEAVE: middle.block: 4528; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[K]], [[N_VEC]] 4529; INTERLEAVE-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4530; INTERLEAVE: scalar.ph: 4531; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ] 4532; INTERLEAVE-NEXT: br label [[FOR_BODY:%.*]] 4533; INTERLEAVE: for.body: 4534; INTERLEAVE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4535; INTERLEAVE-NEXT: [[TRUNC_IV:%.*]] = trunc i64 [[INDVARS_IV]] to i32 4536; INTERLEAVE-NEXT: [[SEXT1:%.*]] = shl i64 [[INDVARS_IV]], 32 4537; INTERLEAVE-NEXT: [[TMP4:%.*]] = ashr exact i64 [[SEXT1]], 30 4538; INTERLEAVE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP4]] 4539; INTERLEAVE-NEXT: store i32 [[TRUNC_IV]], ptr [[ARRAYIDX]], align 4 4540; INTERLEAVE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 4541; INTERLEAVE-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[K]] 4542; INTERLEAVE-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]] 4543; INTERLEAVE: exit: 4544; INTERLEAVE-NEXT: ret void 4545; 4546for.body.preheader: 4547 br label %for.body 4548 4549for.body: 4550 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ] 4551 %trunc.iv = trunc i64 %indvars.iv to i32 4552 %arrayidx = getelementptr inbounds i32, ptr %a, i32 %trunc.iv 4553 store i32 %trunc.iv, ptr %arrayidx, align 4 4554 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 4555 %exitcond = icmp eq i64 %indvars.iv.next, %k 4556 br i1 %exitcond, label %exit, label %for.body 4557 4558exit: 4559 ret void 4560} 4561 4562; 4563; 4564define void @nonprimary(ptr nocapture %a, i32 %start, i32 %i, i32 %k) { 4565; CHECK-LABEL: @nonprimary( 4566; CHECK-NEXT: for.body.preheader: 4567; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[K:%.*]], [[I:%.*]] 4568; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 2 4569; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4570; CHECK: vector.ph: 4571; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2 4572; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]] 4573; CHECK-NEXT: [[IND_END:%.*]] = add i32 [[I]], [[N_VEC]] 4574; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[I]], i64 0 4575; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 4576; CHECK-NEXT: [[INDUCTION:%.*]] = add <2 x i32> [[DOTSPLAT]], <i32 0, i32 1> 4577; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 4578; CHECK: vector.body: 4579; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4580; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4581; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i32 [[I]], [[INDEX]] 4582; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[OFFSET_IDX]], 0 4583; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP1]] 4584; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0 4585; CHECK-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP3]], align 4 4586; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 4587; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 4588; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4589; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]] 4590; CHECK: middle.block: 4591; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 4592; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4593; CHECK: scalar.ph: 4594; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[I]], [[FOR_BODY_PREHEADER:%.*]] ] 4595; CHECK-NEXT: br label [[FOR_BODY:%.*]] 4596; CHECK: for.body: 4597; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4598; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDVARS_IV]] 4599; CHECK-NEXT: store i32 [[INDVARS_IV]], ptr [[ARRAYIDX]], align 4 4600; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4601; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[K]] 4602; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]] 4603; CHECK: exit: 4604; CHECK-NEXT: ret void 4605; 4606; IND-LABEL: @nonprimary( 4607; IND-NEXT: for.body.preheader: 4608; IND-NEXT: [[TMP0:%.*]] = sub i32 [[K:%.*]], [[I:%.*]] 4609; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 2 4610; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4611; IND: vector.ph: 4612; IND-NEXT: [[N_VEC:%.*]] = and i32 [[TMP0]], -2 4613; IND-NEXT: [[IND_END:%.*]] = add i32 [[I]], [[N_VEC]] 4614; IND-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[I]], i64 0 4615; IND-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 4616; IND-NEXT: [[INDUCTION:%.*]] = add <2 x i32> [[DOTSPLAT]], <i32 0, i32 1> 4617; IND-NEXT: br label [[VECTOR_BODY:%.*]] 4618; IND: vector.body: 4619; IND-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4620; IND-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4621; IND-NEXT: [[OFFSET_IDX:%.*]] = add i32 [[I]], [[INDEX]] 4622; IND-NEXT: [[TMP1:%.*]] = sext i32 [[OFFSET_IDX]] to i64 4623; IND-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP1]] 4624; IND-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP2]], align 4 4625; IND-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 4626; IND-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 4627; IND-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4628; IND-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]] 4629; IND: middle.block: 4630; IND-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 4631; IND-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4632; IND: scalar.ph: 4633; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[I]], [[FOR_BODY_PREHEADER:%.*]] ] 4634; IND-NEXT: br label [[FOR_BODY:%.*]] 4635; IND: for.body: 4636; IND-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4637; IND-NEXT: [[TMP4:%.*]] = sext i32 [[INDVARS_IV]] to i64 4638; IND-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]] 4639; IND-NEXT: store i32 [[INDVARS_IV]], ptr [[ARRAYIDX]], align 4 4640; IND-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4641; IND-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[K]] 4642; IND-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]] 4643; IND: exit: 4644; IND-NEXT: ret void 4645; 4646; UNROLL-LABEL: @nonprimary( 4647; UNROLL-NEXT: for.body.preheader: 4648; UNROLL-NEXT: [[TMP0:%.*]] = sub i32 [[K:%.*]], [[I:%.*]] 4649; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 4 4650; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4651; UNROLL: vector.ph: 4652; UNROLL-NEXT: [[N_VEC:%.*]] = and i32 [[TMP0]], -4 4653; UNROLL-NEXT: [[IND_END:%.*]] = add i32 [[I]], [[N_VEC]] 4654; UNROLL-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[I]], i64 0 4655; UNROLL-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 4656; UNROLL-NEXT: [[INDUCTION:%.*]] = add <2 x i32> [[DOTSPLAT]], <i32 0, i32 1> 4657; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 4658; UNROLL: vector.body: 4659; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4660; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4661; UNROLL-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 4662; UNROLL-NEXT: [[OFFSET_IDX:%.*]] = add i32 [[I]], [[INDEX]] 4663; UNROLL-NEXT: [[TMP1:%.*]] = sext i32 [[OFFSET_IDX]] to i64 4664; UNROLL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP1]] 4665; UNROLL-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 8 4666; UNROLL-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP2]], align 4 4667; UNROLL-NEXT: store <2 x i32> [[STEP_ADD]], ptr [[TMP3]], align 4 4668; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 4669; UNROLL-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 4) 4670; UNROLL-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4671; UNROLL-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]] 4672; UNROLL: middle.block: 4673; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 4674; UNROLL-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4675; UNROLL: scalar.ph: 4676; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[I]], [[FOR_BODY_PREHEADER:%.*]] ] 4677; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 4678; UNROLL: for.body: 4679; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4680; UNROLL-NEXT: [[TMP5:%.*]] = sext i32 [[INDVARS_IV]] to i64 4681; UNROLL-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP5]] 4682; UNROLL-NEXT: store i32 [[INDVARS_IV]], ptr [[ARRAYIDX]], align 4 4683; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4684; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[K]] 4685; UNROLL-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]] 4686; UNROLL: exit: 4687; UNROLL-NEXT: ret void 4688; 4689; UNROLL-NO-IC-LABEL: @nonprimary( 4690; UNROLL-NO-IC-NEXT: for.body.preheader: 4691; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = sub i32 [[K:%.*]], [[I:%.*]] 4692; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 4 4693; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4694; UNROLL-NO-IC: vector.ph: 4695; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 4 4696; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]] 4697; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = add i32 [[I]], [[N_VEC]] 4698; UNROLL-NO-IC-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[I]], i64 0 4699; UNROLL-NO-IC-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 4700; UNROLL-NO-IC-NEXT: [[INDUCTION:%.*]] = add <2 x i32> [[DOTSPLAT]], <i32 0, i32 1> 4701; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 4702; UNROLL-NO-IC: vector.body: 4703; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4704; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4705; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 4706; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = add i32 [[I]], [[INDEX]] 4707; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i32 [[OFFSET_IDX]], 0 4708; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[TMP1]] 4709; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0 4710; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 2 4711; UNROLL-NO-IC-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP3]], align 4 4712; UNROLL-NO-IC-NEXT: store <2 x i32> [[STEP_ADD]], ptr [[TMP4]], align 4 4713; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 4714; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 2) 4715; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4716; UNROLL-NO-IC-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]] 4717; UNROLL-NO-IC: middle.block: 4718; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 4719; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4720; UNROLL-NO-IC: scalar.ph: 4721; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[I]], [[FOR_BODY_PREHEADER:%.*]] ] 4722; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 4723; UNROLL-NO-IC: for.body: 4724; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4725; UNROLL-NO-IC-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDVARS_IV]] 4726; UNROLL-NO-IC-NEXT: store i32 [[INDVARS_IV]], ptr [[ARRAYIDX]], align 4 4727; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4728; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[K]] 4729; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]] 4730; UNROLL-NO-IC: exit: 4731; UNROLL-NO-IC-NEXT: ret void 4732; 4733; INTERLEAVE-LABEL: @nonprimary( 4734; INTERLEAVE-NEXT: for.body.preheader: 4735; INTERLEAVE-NEXT: [[TMP0:%.*]] = sub i32 [[K:%.*]], [[I:%.*]] 4736; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 8 4737; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4738; INTERLEAVE: vector.ph: 4739; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i32 [[TMP0]], -8 4740; INTERLEAVE-NEXT: [[IND_END:%.*]] = add i32 [[I]], [[N_VEC]] 4741; INTERLEAVE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[I]], i64 0 4742; INTERLEAVE-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 4743; INTERLEAVE-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[DOTSPLAT]], <i32 0, i32 1, i32 2, i32 3> 4744; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 4745; INTERLEAVE: vector.body: 4746; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4747; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4748; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4) 4749; INTERLEAVE-NEXT: [[OFFSET_IDX:%.*]] = add i32 [[I]], [[INDEX]] 4750; INTERLEAVE-NEXT: [[TMP1:%.*]] = sext i32 [[OFFSET_IDX]] to i64 4751; INTERLEAVE-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP1]] 4752; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16 4753; INTERLEAVE-NEXT: store <4 x i32> [[VEC_IND]], ptr [[TMP2]], align 4 4754; INTERLEAVE-NEXT: store <4 x i32> [[STEP_ADD]], ptr [[TMP3]], align 4 4755; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 4756; INTERLEAVE-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 8) 4757; INTERLEAVE-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 4758; INTERLEAVE-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]] 4759; INTERLEAVE: middle.block: 4760; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] 4761; INTERLEAVE-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 4762; INTERLEAVE: scalar.ph: 4763; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[I]], [[FOR_BODY_PREHEADER:%.*]] ] 4764; INTERLEAVE-NEXT: br label [[FOR_BODY:%.*]] 4765; INTERLEAVE: for.body: 4766; INTERLEAVE-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4767; INTERLEAVE-NEXT: [[TMP5:%.*]] = sext i32 [[INDVARS_IV]] to i64 4768; INTERLEAVE-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP5]] 4769; INTERLEAVE-NEXT: store i32 [[INDVARS_IV]], ptr [[ARRAYIDX]], align 4 4770; INTERLEAVE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1 4771; INTERLEAVE-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[K]] 4772; INTERLEAVE-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]] 4773; INTERLEAVE: exit: 4774; INTERLEAVE-NEXT: ret void 4775; 4776for.body.preheader: 4777 br label %for.body 4778 4779for.body: 4780 %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ %i, %for.body.preheader ] 4781 %arrayidx = getelementptr inbounds i32, ptr %a, i32 %indvars.iv 4782 store i32 %indvars.iv, ptr %arrayidx, align 4 4783 %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1 4784 %exitcond = icmp eq i32 %indvars.iv.next, %k 4785 br i1 %exitcond, label %exit, label %for.body 4786 4787exit: 4788 ret void 4789} 4790 4791define void @non_primary_iv_trunc(ptr %a, i64 %n) { 4792; CHECK-LABEL: @non_primary_iv_trunc( 4793; CHECK-NEXT: entry: 4794; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) 4795; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 2 4796; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4797; CHECK: vector.ph: 4798; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 2 4799; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]] 4800; CHECK-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 4801; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 4802; CHECK: vector.body: 4803; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4804; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4805; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 4806; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]] 4807; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0 4808; CHECK-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP2]], align 4 4809; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 4810; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 4) 4811; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4812; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]] 4813; CHECK: middle.block: 4814; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 4815; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4816; CHECK: scalar.ph: 4817; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 4818; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 4819; CHECK-NEXT: br label [[FOR_BODY:%.*]] 4820; CHECK: for.body: 4821; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4822; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[J_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 4823; CHECK-NEXT: [[VAR0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]] 4824; CHECK-NEXT: [[VAR1:%.*]] = trunc i64 [[J]] to i32 4825; CHECK-NEXT: store i32 [[VAR1]], ptr [[VAR0]], align 4 4826; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 4827; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 2 4828; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 4829; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP47:![0-9]+]] 4830; CHECK: for.end: 4831; CHECK-NEXT: ret void 4832; 4833; IND-LABEL: @non_primary_iv_trunc( 4834; IND-NEXT: entry: 4835; IND-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) 4836; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 2 4837; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4838; IND: vector.ph: 4839; IND-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775806 4840; IND-NEXT: [[IND_END:%.*]] = shl nuw i64 [[N_VEC]], 1 4841; IND-NEXT: br label [[VECTOR_BODY:%.*]] 4842; IND: vector.body: 4843; IND-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4844; IND-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4845; IND-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] 4846; IND-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP0]], align 4 4847; IND-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 4848; IND-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 4) 4849; IND-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4850; IND-NEXT: br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]] 4851; IND: middle.block: 4852; IND-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 4853; IND-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4854; IND: scalar.ph: 4855; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 4856; IND-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 4857; IND-NEXT: br label [[FOR_BODY:%.*]] 4858; IND: for.body: 4859; IND-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4860; IND-NEXT: [[J:%.*]] = phi i64 [ [[J_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 4861; IND-NEXT: [[VAR0:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[I]] 4862; IND-NEXT: [[VAR1:%.*]] = trunc i64 [[J]] to i32 4863; IND-NEXT: store i32 [[VAR1]], ptr [[VAR0]], align 4 4864; IND-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 4865; IND-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 2 4866; IND-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 4867; IND-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP47:![0-9]+]] 4868; IND: for.end: 4869; IND-NEXT: ret void 4870; 4871; UNROLL-LABEL: @non_primary_iv_trunc( 4872; UNROLL-NEXT: entry: 4873; UNROLL-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) 4874; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 4 4875; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4876; UNROLL: vector.ph: 4877; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804 4878; UNROLL-NEXT: [[IND_END:%.*]] = shl nuw i64 [[N_VEC]], 1 4879; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 4880; UNROLL: vector.body: 4881; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4882; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4883; UNROLL-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 4) 4884; UNROLL-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] 4885; UNROLL-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8 4886; UNROLL-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP0]], align 4 4887; UNROLL-NEXT: store <2 x i32> [[STEP_ADD]], ptr [[TMP1]], align 4 4888; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 4889; UNROLL-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 8) 4890; UNROLL-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4891; UNROLL-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]] 4892; UNROLL: middle.block: 4893; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 4894; UNROLL-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4895; UNROLL: scalar.ph: 4896; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 4897; UNROLL-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 4898; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 4899; UNROLL: for.body: 4900; UNROLL-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4901; UNROLL-NEXT: [[J:%.*]] = phi i64 [ [[J_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 4902; UNROLL-NEXT: [[VAR0:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[I]] 4903; UNROLL-NEXT: [[VAR1:%.*]] = trunc i64 [[J]] to i32 4904; UNROLL-NEXT: store i32 [[VAR1]], ptr [[VAR0]], align 4 4905; UNROLL-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 4906; UNROLL-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 2 4907; UNROLL-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 4908; UNROLL-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP47:![0-9]+]] 4909; UNROLL: for.end: 4910; UNROLL-NEXT: ret void 4911; 4912; UNROLL-NO-IC-LABEL: @non_primary_iv_trunc( 4913; UNROLL-NO-IC-NEXT: entry: 4914; UNROLL-NO-IC-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) 4915; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4 4916; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4917; UNROLL-NO-IC: vector.ph: 4918; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4 4919; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]] 4920; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 4921; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 4922; UNROLL-NO-IC: vector.body: 4923; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4924; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4925; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 4) 4926; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 4927; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP0]] 4928; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0 4929; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 2 4930; UNROLL-NO-IC-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP2]], align 4 4931; UNROLL-NO-IC-NEXT: store <2 x i32> [[STEP_ADD]], ptr [[TMP3]], align 4 4932; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 4933; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 4) 4934; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4935; UNROLL-NO-IC-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]] 4936; UNROLL-NO-IC: middle.block: 4937; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 4938; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4939; UNROLL-NO-IC: scalar.ph: 4940; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 4941; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 4942; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 4943; UNROLL-NO-IC: for.body: 4944; UNROLL-NO-IC-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4945; UNROLL-NO-IC-NEXT: [[J:%.*]] = phi i64 [ [[J_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 4946; UNROLL-NO-IC-NEXT: [[VAR0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]] 4947; UNROLL-NO-IC-NEXT: [[VAR1:%.*]] = trunc i64 [[J]] to i32 4948; UNROLL-NO-IC-NEXT: store i32 [[VAR1]], ptr [[VAR0]], align 4 4949; UNROLL-NO-IC-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 4950; UNROLL-NO-IC-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 2 4951; UNROLL-NO-IC-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 4952; UNROLL-NO-IC-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP47:![0-9]+]] 4953; UNROLL-NO-IC: for.end: 4954; UNROLL-NO-IC-NEXT: ret void 4955; 4956; INTERLEAVE-LABEL: @non_primary_iv_trunc( 4957; INTERLEAVE-NEXT: entry: 4958; INTERLEAVE-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1) 4959; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N]], 8 4960; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 4961; INTERLEAVE: vector.ph: 4962; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775800 4963; INTERLEAVE-NEXT: [[IND_END:%.*]] = shl nuw i64 [[N_VEC]], 1 4964; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 4965; INTERLEAVE: vector.body: 4966; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 4967; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 4968; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 8) 4969; INTERLEAVE-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] 4970; INTERLEAVE-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16 4971; INTERLEAVE-NEXT: store <4 x i32> [[VEC_IND]], ptr [[TMP0]], align 4 4972; INTERLEAVE-NEXT: store <4 x i32> [[STEP_ADD]], ptr [[TMP1]], align 4 4973; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 4974; INTERLEAVE-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 16) 4975; INTERLEAVE-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 4976; INTERLEAVE-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]] 4977; INTERLEAVE: middle.block: 4978; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] 4979; INTERLEAVE-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] 4980; INTERLEAVE: scalar.ph: 4981; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 4982; INTERLEAVE-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 4983; INTERLEAVE-NEXT: br label [[FOR_BODY:%.*]] 4984; INTERLEAVE: for.body: 4985; INTERLEAVE-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 4986; INTERLEAVE-NEXT: [[J:%.*]] = phi i64 [ [[J_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 4987; INTERLEAVE-NEXT: [[VAR0:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[I]] 4988; INTERLEAVE-NEXT: [[VAR1:%.*]] = trunc i64 [[J]] to i32 4989; INTERLEAVE-NEXT: store i32 [[VAR1]], ptr [[VAR0]], align 4 4990; INTERLEAVE-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 4991; INTERLEAVE-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 2 4992; INTERLEAVE-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 4993; INTERLEAVE-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP47:![0-9]+]] 4994; INTERLEAVE: for.end: 4995; INTERLEAVE-NEXT: ret void 4996; 4997entry: 4998 br label %for.body 4999 5000for.body: 5001 %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ] 5002 %j = phi i64 [ %j.next, %for.body ], [ 0, %entry ] 5003 %var0 = getelementptr inbounds i32, ptr %a, i64 %i 5004 %var1 = trunc i64 %j to i32 5005 store i32 %var1, ptr %var0, align 4 5006 %i.next = add nuw nsw i64 %i, 1 5007 %j.next = add nuw nsw i64 %j, 2 5008 %cond = icmp slt i64 %i.next, %n 5009 br i1 %cond, label %for.body, label %for.end 5010 5011for.end: 5012 ret void 5013} 5014 5015; PR32419. Ensure we transform truncated non-primary induction variables. In 5016; the test case below we replace %var1 with a new induction variable. Because 5017; the truncated value is non-primary, we must compute an offset from the 5018; primary induction variable. 5019; 5020; 5021define i32 @PR32419(i32 %a, i16 %b) { 5022; CHECK-LABEL: @PR32419( 5023; CHECK-NEXT: entry: 5024; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5025; CHECK: vector.ph: 5026; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[A:%.*]], i32 0 5027; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 5028; CHECK: vector.body: 5029; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UREM_CONTINUE2:%.*]] ] 5030; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_UREM_CONTINUE2]] ] 5031; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 -20, i16 -19>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE2]] ] 5032; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i32 -20, [[INDEX]] 5033; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[OFFSET_IDX]] to i16 5034; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <2 x i16> [[VEC_IND]], zeroinitializer 5035; CHECK-NEXT: [[TMP3:%.*]] = xor <2 x i1> [[TMP2]], splat (i1 true) 5036; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i32 0 5037; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]] 5038; CHECK: pred.urem.if: 5039; CHECK-NEXT: [[TMP5:%.*]] = add i16 [[TMP1]], 0 5040; CHECK-NEXT: [[TMP6:%.*]] = urem i16 [[B:%.*]], [[TMP5]] 5041; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP6]], i32 0 5042; CHECK-NEXT: br label [[PRED_UREM_CONTINUE]] 5043; CHECK: pred.urem.continue: 5044; CHECK-NEXT: [[TMP8:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_UREM_IF]] ] 5045; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1 5046; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_UREM_IF1:%.*]], label [[PRED_UREM_CONTINUE2]] 5047; CHECK: pred.urem.if1: 5048; CHECK-NEXT: [[TMP10:%.*]] = add i16 [[TMP1]], 1 5049; CHECK-NEXT: [[TMP11:%.*]] = urem i16 [[B]], [[TMP10]] 5050; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i16> [[TMP8]], i16 [[TMP11]], i32 1 5051; CHECK-NEXT: br label [[PRED_UREM_CONTINUE2]] 5052; CHECK: pred.urem.continue2: 5053; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i16> [ [[TMP8]], [[PRED_UREM_CONTINUE]] ], [ [[TMP12]], [[PRED_UREM_IF1]] ] 5054; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i16> zeroinitializer, <2 x i16> [[TMP13]] 5055; CHECK-NEXT: [[TMP14:%.*]] = sext <2 x i16> [[PREDPHI]] to <2 x i32> 5056; CHECK-NEXT: [[TMP15]] = or <2 x i32> [[VEC_PHI]], [[TMP14]] 5057; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 5058; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2) 5059; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], 20 5060; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 5061; CHECK: middle.block: 5062; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP15]]) 5063; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 5064; CHECK: scalar.ph: 5065; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[MIDDLE_BLOCK]] ], [ -20, [[ENTRY:%.*]] ] 5066; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], [[MIDDLE_BLOCK]] ], [ [[A]], [[ENTRY]] ] 5067; CHECK-NEXT: br label [[FOR_BODY:%.*]] 5068; CHECK: for.body: 5069; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ] 5070; CHECK-NEXT: [[VAR0:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[VAR6:%.*]], [[FOR_INC]] ] 5071; CHECK-NEXT: [[VAR1:%.*]] = trunc i32 [[I]] to i16 5072; CHECK-NEXT: [[VAR2:%.*]] = icmp eq i16 [[VAR1]], 0 5073; CHECK-NEXT: br i1 [[VAR2]], label [[FOR_INC]], label [[FOR_COND:%.*]] 5074; CHECK: for.cond: 5075; CHECK-NEXT: [[VAR3:%.*]] = urem i16 [[B]], [[VAR1]] 5076; CHECK-NEXT: br label [[FOR_INC]] 5077; CHECK: for.inc: 5078; CHECK-NEXT: [[VAR4:%.*]] = phi i16 [ [[VAR3]], [[FOR_COND]] ], [ 0, [[FOR_BODY]] ] 5079; CHECK-NEXT: [[VAR5:%.*]] = sext i16 [[VAR4]] to i32 5080; CHECK-NEXT: [[VAR6]] = or i32 [[VAR0]], [[VAR5]] 5081; CHECK-NEXT: [[I_NEXT]] = add nsw i32 [[I]], 1 5082; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[I_NEXT]], 0 5083; CHECK-NEXT: br i1 [[COND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 5084; CHECK: for.end: 5085; CHECK-NEXT: [[VAR7:%.*]] = phi i32 [ [[VAR6]], [[FOR_INC]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ] 5086; CHECK-NEXT: ret i32 [[VAR7]] 5087; 5088; IND-LABEL: @PR32419( 5089; IND-NEXT: entry: 5090; IND-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5091; IND: vector.ph: 5092; IND-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[A:%.*]], i64 0 5093; IND-NEXT: br label [[VECTOR_BODY:%.*]] 5094; IND: vector.body: 5095; IND-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UREM_CONTINUE2:%.*]] ] 5096; IND-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[PRED_UREM_CONTINUE2]] ] 5097; IND-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 -20, i16 -19>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE2]] ] 5098; IND-NEXT: [[TMP1:%.*]] = trunc i32 [[INDEX]] to i16 5099; IND-NEXT: [[TMP2:%.*]] = icmp ne <2 x i16> [[VEC_IND]], zeroinitializer 5100; IND-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0 5101; IND-NEXT: br i1 [[TMP3]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]] 5102; IND: pred.urem.if: 5103; IND-NEXT: [[TMP4:%.*]] = add i16 [[TMP1]], -20 5104; IND-NEXT: [[TMP5:%.*]] = urem i16 [[B:%.*]], [[TMP4]] 5105; IND-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i64 0 5106; IND-NEXT: br label [[PRED_UREM_CONTINUE]] 5107; IND: pred.urem.continue: 5108; IND-NEXT: [[TMP7:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_UREM_IF]] ] 5109; IND-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP2]], i64 1 5110; IND-NEXT: br i1 [[TMP8]], label [[PRED_UREM_IF1:%.*]], label [[PRED_UREM_CONTINUE2]] 5111; IND: pred.urem.if1: 5112; IND-NEXT: [[TMP9:%.*]] = add i16 [[TMP1]], -19 5113; IND-NEXT: [[TMP10:%.*]] = urem i16 [[B]], [[TMP9]] 5114; IND-NEXT: [[TMP11:%.*]] = insertelement <2 x i16> [[TMP7]], i16 [[TMP10]], i64 1 5115; IND-NEXT: br label [[PRED_UREM_CONTINUE2]] 5116; IND: pred.urem.continue2: 5117; IND-NEXT: [[TMP12:%.*]] = phi <2 x i16> [ [[TMP7]], [[PRED_UREM_CONTINUE]] ], [ [[TMP11]], [[PRED_UREM_IF1]] ] 5118; IND-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i16> [[TMP12]], <2 x i16> zeroinitializer 5119; IND-NEXT: [[TMP13:%.*]] = sext <2 x i16> [[PREDPHI]] to <2 x i32> 5120; IND-NEXT: [[TMP14]] = or <2 x i32> [[VEC_PHI]], [[TMP13]] 5121; IND-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 5122; IND-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2) 5123; IND-NEXT: [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], 20 5124; IND-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 5125; IND: middle.block: 5126; IND-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP14]]) 5127; IND-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 5128; IND: scalar.ph: 5129; IND-NEXT: br label [[FOR_BODY:%.*]] 5130; IND: for.body: 5131; IND-NEXT: br i1 poison, label [[FOR_INC:%.*]], label [[FOR_COND:%.*]] 5132; IND: for.cond: 5133; IND-NEXT: br label [[FOR_INC]] 5134; IND: for.inc: 5135; IND-NEXT: br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 5136; IND: for.end: 5137; IND-NEXT: [[VAR7:%.*]] = phi i32 [ poison, [[FOR_INC]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ] 5138; IND-NEXT: ret i32 [[VAR7]] 5139; 5140; UNROLL-LABEL: @PR32419( 5141; UNROLL-NEXT: entry: 5142; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5143; UNROLL: vector.ph: 5144; UNROLL-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[A:%.*]], i64 0 5145; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 5146; UNROLL: vector.body: 5147; UNROLL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UREM_CONTINUE7:%.*]] ] 5148; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP26:%.*]], [[PRED_UREM_CONTINUE7]] ] 5149; UNROLL-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[PRED_UREM_CONTINUE7]] ] 5150; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 -20, i16 -19>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE7]] ] 5151; UNROLL-NEXT: [[TMP1:%.*]] = trunc i32 [[INDEX]] to i16 5152; UNROLL-NEXT: [[TMP2:%.*]] = icmp ne <2 x i16> [[VEC_IND]], zeroinitializer 5153; UNROLL-NEXT: [[TMP3:%.*]] = icmp ne <2 x i16> [[VEC_IND]], splat (i16 -2) 5154; UNROLL-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0 5155; UNROLL-NEXT: br i1 [[TMP4]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]] 5156; UNROLL: pred.urem.if: 5157; UNROLL-NEXT: [[TMP5:%.*]] = add i16 [[TMP1]], -20 5158; UNROLL-NEXT: [[TMP6:%.*]] = urem i16 [[B:%.*]], [[TMP5]] 5159; UNROLL-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP6]], i64 0 5160; UNROLL-NEXT: br label [[PRED_UREM_CONTINUE]] 5161; UNROLL: pred.urem.continue: 5162; UNROLL-NEXT: [[TMP8:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_UREM_IF]] ] 5163; UNROLL-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP2]], i64 1 5164; UNROLL-NEXT: br i1 [[TMP9]], label [[PRED_UREM_IF2:%.*]], label [[PRED_UREM_CONTINUE3:%.*]] 5165; UNROLL: pred.urem.if2: 5166; UNROLL-NEXT: [[TMP10:%.*]] = add i16 [[TMP1]], -19 5167; UNROLL-NEXT: [[TMP11:%.*]] = urem i16 [[B]], [[TMP10]] 5168; UNROLL-NEXT: [[TMP12:%.*]] = insertelement <2 x i16> [[TMP8]], i16 [[TMP11]], i64 1 5169; UNROLL-NEXT: br label [[PRED_UREM_CONTINUE3]] 5170; UNROLL: pred.urem.continue3: 5171; UNROLL-NEXT: [[TMP13:%.*]] = phi <2 x i16> [ [[TMP8]], [[PRED_UREM_CONTINUE]] ], [ [[TMP12]], [[PRED_UREM_IF2]] ] 5172; UNROLL-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0 5173; UNROLL-NEXT: br i1 [[TMP14]], label [[PRED_UREM_IF4:%.*]], label [[PRED_UREM_CONTINUE5:%.*]] 5174; UNROLL: pred.urem.if4: 5175; UNROLL-NEXT: [[TMP15:%.*]] = add i16 [[TMP1]], -18 5176; UNROLL-NEXT: [[TMP16:%.*]] = urem i16 [[B]], [[TMP15]] 5177; UNROLL-NEXT: [[TMP17:%.*]] = insertelement <2 x i16> poison, i16 [[TMP16]], i64 0 5178; UNROLL-NEXT: br label [[PRED_UREM_CONTINUE5]] 5179; UNROLL: pred.urem.continue5: 5180; UNROLL-NEXT: [[TMP18:%.*]] = phi <2 x i16> [ poison, [[PRED_UREM_CONTINUE3]] ], [ [[TMP17]], [[PRED_UREM_IF4]] ] 5181; UNROLL-NEXT: [[TMP19:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1 5182; UNROLL-NEXT: br i1 [[TMP19]], label [[PRED_UREM_IF6:%.*]], label [[PRED_UREM_CONTINUE7]] 5183; UNROLL: pred.urem.if6: 5184; UNROLL-NEXT: [[TMP20:%.*]] = add i16 [[TMP1]], -17 5185; UNROLL-NEXT: [[TMP21:%.*]] = urem i16 [[B]], [[TMP20]] 5186; UNROLL-NEXT: [[TMP22:%.*]] = insertelement <2 x i16> [[TMP18]], i16 [[TMP21]], i64 1 5187; UNROLL-NEXT: br label [[PRED_UREM_CONTINUE7]] 5188; UNROLL: pred.urem.continue7: 5189; UNROLL-NEXT: [[TMP23:%.*]] = phi <2 x i16> [ [[TMP18]], [[PRED_UREM_CONTINUE5]] ], [ [[TMP22]], [[PRED_UREM_IF6]] ] 5190; UNROLL-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i16> [[TMP13]], <2 x i16> zeroinitializer 5191; UNROLL-NEXT: [[PREDPHI8:%.*]] = select <2 x i1> [[TMP3]], <2 x i16> [[TMP23]], <2 x i16> zeroinitializer 5192; UNROLL-NEXT: [[TMP24:%.*]] = sext <2 x i16> [[PREDPHI]] to <2 x i32> 5193; UNROLL-NEXT: [[TMP25:%.*]] = sext <2 x i16> [[PREDPHI8]] to <2 x i32> 5194; UNROLL-NEXT: [[TMP26]] = or <2 x i32> [[VEC_PHI]], [[TMP24]] 5195; UNROLL-NEXT: [[TMP27]] = or <2 x i32> [[VEC_PHI1]], [[TMP25]] 5196; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 5197; UNROLL-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 4) 5198; UNROLL-NEXT: [[TMP28:%.*]] = icmp eq i32 [[INDEX_NEXT]], 20 5199; UNROLL-NEXT: br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 5200; UNROLL: middle.block: 5201; UNROLL-NEXT: [[BIN_RDX:%.*]] = or <2 x i32> [[TMP27]], [[TMP26]] 5202; UNROLL-NEXT: [[TMP29:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[BIN_RDX]]) 5203; UNROLL-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 5204; UNROLL: scalar.ph: 5205; UNROLL-NEXT: br label [[FOR_BODY:%.*]] 5206; UNROLL: for.body: 5207; UNROLL-NEXT: br i1 poison, label [[FOR_INC:%.*]], label [[FOR_COND:%.*]] 5208; UNROLL: for.cond: 5209; UNROLL-NEXT: br label [[FOR_INC]] 5210; UNROLL: for.inc: 5211; UNROLL-NEXT: br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 5212; UNROLL: for.end: 5213; UNROLL-NEXT: [[VAR7:%.*]] = phi i32 [ poison, [[FOR_INC]] ], [ [[TMP29]], [[MIDDLE_BLOCK]] ] 5214; UNROLL-NEXT: ret i32 [[VAR7]] 5215; 5216; UNROLL-NO-IC-LABEL: @PR32419( 5217; UNROLL-NO-IC-NEXT: entry: 5218; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5219; UNROLL-NO-IC: vector.ph: 5220; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[A:%.*]], i32 0 5221; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 5222; UNROLL-NO-IC: vector.body: 5223; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UREM_CONTINUE7:%.*]] ] 5224; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[PRED_UREM_CONTINUE7]] ] 5225; UNROLL-NO-IC-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP29:%.*]], [[PRED_UREM_CONTINUE7]] ] 5226; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 -20, i16 -19>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE7]] ] 5227; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <2 x i16> [[VEC_IND]], splat (i16 2) 5228; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = add i32 -20, [[INDEX]] 5229; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = trunc i32 [[OFFSET_IDX]] to i16 5230; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = icmp eq <2 x i16> [[VEC_IND]], zeroinitializer 5231; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp eq <2 x i16> [[STEP_ADD]], zeroinitializer 5232; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = xor <2 x i1> [[TMP2]], splat (i1 true) 5233; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true) 5234; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0 5235; UNROLL-NO-IC-NEXT: br i1 [[TMP6]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]] 5236; UNROLL-NO-IC: pred.urem.if: 5237; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i16 [[TMP1]], 0 5238; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = urem i16 [[B:%.*]], [[TMP7]] 5239; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = insertelement <2 x i16> poison, i16 [[TMP8]], i32 0 5240; UNROLL-NO-IC-NEXT: br label [[PRED_UREM_CONTINUE]] 5241; UNROLL-NO-IC: pred.urem.continue: 5242; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP9]], [[PRED_UREM_IF]] ] 5243; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1 5244; UNROLL-NO-IC-NEXT: br i1 [[TMP11]], label [[PRED_UREM_IF2:%.*]], label [[PRED_UREM_CONTINUE3:%.*]] 5245; UNROLL-NO-IC: pred.urem.if2: 5246; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = add i16 [[TMP1]], 1 5247; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = urem i16 [[B]], [[TMP12]] 5248; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = insertelement <2 x i16> [[TMP10]], i16 [[TMP13]], i32 1 5249; UNROLL-NO-IC-NEXT: br label [[PRED_UREM_CONTINUE3]] 5250; UNROLL-NO-IC: pred.urem.continue3: 5251; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = phi <2 x i16> [ [[TMP10]], [[PRED_UREM_CONTINUE]] ], [ [[TMP14]], [[PRED_UREM_IF2]] ] 5252; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0 5253; UNROLL-NO-IC-NEXT: br i1 [[TMP16]], label [[PRED_UREM_IF4:%.*]], label [[PRED_UREM_CONTINUE5:%.*]] 5254; UNROLL-NO-IC: pred.urem.if4: 5255; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = add i16 [[TMP1]], 2 5256; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = urem i16 [[B]], [[TMP17]] 5257; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> poison, i16 [[TMP18]], i32 0 5258; UNROLL-NO-IC-NEXT: br label [[PRED_UREM_CONTINUE5]] 5259; UNROLL-NO-IC: pred.urem.continue5: 5260; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = phi <2 x i16> [ poison, [[PRED_UREM_CONTINUE3]] ], [ [[TMP19]], [[PRED_UREM_IF4]] ] 5261; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = extractelement <2 x i1> [[TMP5]], i32 1 5262; UNROLL-NO-IC-NEXT: br i1 [[TMP21]], label [[PRED_UREM_IF6:%.*]], label [[PRED_UREM_CONTINUE7]] 5263; UNROLL-NO-IC: pred.urem.if6: 5264; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = add i16 [[TMP1]], 3 5265; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = urem i16 [[B]], [[TMP22]] 5266; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = insertelement <2 x i16> [[TMP20]], i16 [[TMP23]], i32 1 5267; UNROLL-NO-IC-NEXT: br label [[PRED_UREM_CONTINUE7]] 5268; UNROLL-NO-IC: pred.urem.continue7: 5269; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = phi <2 x i16> [ [[TMP20]], [[PRED_UREM_CONTINUE5]] ], [ [[TMP24]], [[PRED_UREM_IF6]] ] 5270; UNROLL-NO-IC-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i16> zeroinitializer, <2 x i16> [[TMP15]] 5271; UNROLL-NO-IC-NEXT: [[PREDPHI8:%.*]] = select <2 x i1> [[TMP3]], <2 x i16> zeroinitializer, <2 x i16> [[TMP25]] 5272; UNROLL-NO-IC-NEXT: [[TMP26:%.*]] = sext <2 x i16> [[PREDPHI]] to <2 x i32> 5273; UNROLL-NO-IC-NEXT: [[TMP27:%.*]] = sext <2 x i16> [[PREDPHI8]] to <2 x i32> 5274; UNROLL-NO-IC-NEXT: [[TMP28]] = or <2 x i32> [[VEC_PHI]], [[TMP26]] 5275; UNROLL-NO-IC-NEXT: [[TMP29]] = or <2 x i32> [[VEC_PHI1]], [[TMP27]] 5276; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 5277; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[STEP_ADD]], splat (i16 2) 5278; UNROLL-NO-IC-NEXT: [[TMP30:%.*]] = icmp eq i32 [[INDEX_NEXT]], 20 5279; UNROLL-NO-IC-NEXT: br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 5280; UNROLL-NO-IC: middle.block: 5281; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = or <2 x i32> [[TMP29]], [[TMP28]] 5282; UNROLL-NO-IC-NEXT: [[TMP31:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[BIN_RDX]]) 5283; UNROLL-NO-IC-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] 5284; UNROLL-NO-IC: scalar.ph: 5285; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[MIDDLE_BLOCK]] ], [ -20, [[ENTRY:%.*]] ] 5286; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP31]], [[MIDDLE_BLOCK]] ], [ [[A]], [[ENTRY]] ] 5287; UNROLL-NO-IC-NEXT: br label [[FOR_BODY:%.*]] 5288; UNROLL-NO-IC: for.body: 5289; UNROLL-NO-IC-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ] 5290; UNROLL-NO-IC-NEXT: [[VAR0:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[VAR6:%.*]], [[FOR_INC]] ] 5291; UNROLL-NO-IC-NEXT: [[VAR1:%.*]] = trunc i32 [[I]] to i16 5292; UNROLL-NO-IC-NEXT: [[VAR2:%.*]] = icmp eq i16 [[VAR1]], 0 5293; UNROLL-NO-IC-NEXT: br i1 [[VAR2]], label [[FOR_INC]], label [[FOR_COND:%.*]] 5294; UNROLL-NO-IC: for.cond: 5295; UNROLL-NO-IC-NEXT: [[VAR3:%.*]] = urem i16 [[B]], [[VAR1]] 5296; UNROLL-NO-IC-NEXT: br label [[FOR_INC]] 5297; UNROLL-NO-IC: for.inc: 5298; UNROLL-NO-IC-NEXT: [[VAR4:%.*]] = phi i16 [ [[VAR3]], [[FOR_COND]] ], [ 0, [[FOR_BODY]] ] 5299; UNROLL-NO-IC-NEXT: [[VAR5:%.*]] = sext i16 [[VAR4]] to i32 5300; UNROLL-NO-IC-NEXT: [[VAR6]] = or i32 [[VAR0]], [[VAR5]] 5301; UNROLL-NO-IC-NEXT: [[I_NEXT]] = add nsw i32 [[I]], 1 5302; UNROLL-NO-IC-NEXT: [[COND:%.*]] = icmp eq i32 [[I_NEXT]], 0 5303; UNROLL-NO-IC-NEXT: br i1 [[COND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 5304; UNROLL-NO-IC: for.end: 5305; UNROLL-NO-IC-NEXT: [[VAR7:%.*]] = phi i32 [ [[VAR6]], [[FOR_INC]] ], [ [[TMP31]], [[MIDDLE_BLOCK]] ] 5306; UNROLL-NO-IC-NEXT: ret i32 [[VAR7]] 5307; 5308; INTERLEAVE-LABEL: @PR32419( 5309; INTERLEAVE-NEXT: entry: 5310; INTERLEAVE-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5311; INTERLEAVE: vector.ph: 5312; INTERLEAVE-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> <i32 poison, i32 0, i32 0, i32 0>, i32 [[A:%.*]], i64 0 5313; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 5314; INTERLEAVE: vector.body: 5315; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UREM_CONTINUE15:%.*]] ] 5316; INTERLEAVE-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[PRED_UREM_CONTINUE15]] ] 5317; INTERLEAVE-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[PRED_UREM_CONTINUE15]] ] 5318; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 -20, i16 -19, i16 -18, i16 -17>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE15]] ] 5319; INTERLEAVE-NEXT: [[TMP1:%.*]] = trunc i32 [[INDEX]] to i16 5320; INTERLEAVE-NEXT: [[TMP2:%.*]] = icmp ne <4 x i16> [[VEC_IND]], zeroinitializer 5321; INTERLEAVE-NEXT: [[TMP3:%.*]] = icmp ne <4 x i16> [[VEC_IND]], splat (i16 -4) 5322; INTERLEAVE-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0 5323; INTERLEAVE-NEXT: br i1 [[TMP4]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]] 5324; INTERLEAVE: pred.urem.if: 5325; INTERLEAVE-NEXT: [[TMP5:%.*]] = add i16 [[TMP1]], -20 5326; INTERLEAVE-NEXT: [[TMP6:%.*]] = urem i16 [[B:%.*]], [[TMP5]] 5327; INTERLEAVE-NEXT: [[TMP7:%.*]] = insertelement <4 x i16> poison, i16 [[TMP6]], i64 0 5328; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE]] 5329; INTERLEAVE: pred.urem.continue: 5330; INTERLEAVE-NEXT: [[TMP8:%.*]] = phi <4 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_UREM_IF]] ] 5331; INTERLEAVE-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1 5332; INTERLEAVE-NEXT: br i1 [[TMP9]], label [[PRED_UREM_IF2:%.*]], label [[PRED_UREM_CONTINUE3:%.*]] 5333; INTERLEAVE: pred.urem.if2: 5334; INTERLEAVE-NEXT: [[TMP10:%.*]] = add i16 [[TMP1]], -19 5335; INTERLEAVE-NEXT: [[TMP11:%.*]] = urem i16 [[B]], [[TMP10]] 5336; INTERLEAVE-NEXT: [[TMP12:%.*]] = insertelement <4 x i16> [[TMP8]], i16 [[TMP11]], i64 1 5337; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE3]] 5338; INTERLEAVE: pred.urem.continue3: 5339; INTERLEAVE-NEXT: [[TMP13:%.*]] = phi <4 x i16> [ [[TMP8]], [[PRED_UREM_CONTINUE]] ], [ [[TMP12]], [[PRED_UREM_IF2]] ] 5340; INTERLEAVE-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2 5341; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[PRED_UREM_IF4:%.*]], label [[PRED_UREM_CONTINUE5:%.*]] 5342; INTERLEAVE: pred.urem.if4: 5343; INTERLEAVE-NEXT: [[TMP15:%.*]] = add i16 [[TMP1]], -18 5344; INTERLEAVE-NEXT: [[TMP16:%.*]] = urem i16 [[B]], [[TMP15]] 5345; INTERLEAVE-NEXT: [[TMP17:%.*]] = insertelement <4 x i16> [[TMP13]], i16 [[TMP16]], i64 2 5346; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE5]] 5347; INTERLEAVE: pred.urem.continue5: 5348; INTERLEAVE-NEXT: [[TMP18:%.*]] = phi <4 x i16> [ [[TMP13]], [[PRED_UREM_CONTINUE3]] ], [ [[TMP17]], [[PRED_UREM_IF4]] ] 5349; INTERLEAVE-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3 5350; INTERLEAVE-NEXT: br i1 [[TMP19]], label [[PRED_UREM_IF6:%.*]], label [[PRED_UREM_CONTINUE7:%.*]] 5351; INTERLEAVE: pred.urem.if6: 5352; INTERLEAVE-NEXT: [[TMP20:%.*]] = add i16 [[TMP1]], -17 5353; INTERLEAVE-NEXT: [[TMP21:%.*]] = urem i16 [[B]], [[TMP20]] 5354; INTERLEAVE-NEXT: [[TMP22:%.*]] = insertelement <4 x i16> [[TMP18]], i16 [[TMP21]], i64 3 5355; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE7]] 5356; INTERLEAVE: pred.urem.continue7: 5357; INTERLEAVE-NEXT: [[TMP23:%.*]] = phi <4 x i16> [ [[TMP18]], [[PRED_UREM_CONTINUE5]] ], [ [[TMP22]], [[PRED_UREM_IF6]] ] 5358; INTERLEAVE-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0 5359; INTERLEAVE-NEXT: br i1 [[TMP24]], label [[PRED_UREM_IF8:%.*]], label [[PRED_UREM_CONTINUE9:%.*]] 5360; INTERLEAVE: pred.urem.if8: 5361; INTERLEAVE-NEXT: [[TMP25:%.*]] = add i16 [[TMP1]], -16 5362; INTERLEAVE-NEXT: [[TMP26:%.*]] = urem i16 [[B]], [[TMP25]] 5363; INTERLEAVE-NEXT: [[TMP27:%.*]] = insertelement <4 x i16> poison, i16 [[TMP26]], i64 0 5364; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE9]] 5365; INTERLEAVE: pred.urem.continue9: 5366; INTERLEAVE-NEXT: [[TMP28:%.*]] = phi <4 x i16> [ poison, [[PRED_UREM_CONTINUE7]] ], [ [[TMP27]], [[PRED_UREM_IF8]] ] 5367; INTERLEAVE-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1 5368; INTERLEAVE-NEXT: br i1 [[TMP29]], label [[PRED_UREM_IF10:%.*]], label [[PRED_UREM_CONTINUE11:%.*]] 5369; INTERLEAVE: pred.urem.if10: 5370; INTERLEAVE-NEXT: [[TMP30:%.*]] = add i16 [[TMP1]], -15 5371; INTERLEAVE-NEXT: [[TMP31:%.*]] = urem i16 [[B]], [[TMP30]] 5372; INTERLEAVE-NEXT: [[TMP32:%.*]] = insertelement <4 x i16> [[TMP28]], i16 [[TMP31]], i64 1 5373; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE11]] 5374; INTERLEAVE: pred.urem.continue11: 5375; INTERLEAVE-NEXT: [[TMP33:%.*]] = phi <4 x i16> [ [[TMP28]], [[PRED_UREM_CONTINUE9]] ], [ [[TMP32]], [[PRED_UREM_IF10]] ] 5376; INTERLEAVE-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2 5377; INTERLEAVE-NEXT: br i1 [[TMP34]], label [[PRED_UREM_IF12:%.*]], label [[PRED_UREM_CONTINUE13:%.*]] 5378; INTERLEAVE: pred.urem.if12: 5379; INTERLEAVE-NEXT: [[TMP35:%.*]] = add i16 [[TMP1]], -14 5380; INTERLEAVE-NEXT: [[TMP36:%.*]] = urem i16 [[B]], [[TMP35]] 5381; INTERLEAVE-NEXT: [[TMP37:%.*]] = insertelement <4 x i16> [[TMP33]], i16 [[TMP36]], i64 2 5382; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE13]] 5383; INTERLEAVE: pred.urem.continue13: 5384; INTERLEAVE-NEXT: [[TMP38:%.*]] = phi <4 x i16> [ [[TMP33]], [[PRED_UREM_CONTINUE11]] ], [ [[TMP37]], [[PRED_UREM_IF12]] ] 5385; INTERLEAVE-NEXT: [[TMP39:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3 5386; INTERLEAVE-NEXT: br i1 [[TMP39]], label [[PRED_UREM_IF14:%.*]], label [[PRED_UREM_CONTINUE15]] 5387; INTERLEAVE: pred.urem.if14: 5388; INTERLEAVE-NEXT: [[TMP40:%.*]] = add i16 [[TMP1]], -13 5389; INTERLEAVE-NEXT: [[TMP41:%.*]] = urem i16 [[B]], [[TMP40]] 5390; INTERLEAVE-NEXT: [[TMP42:%.*]] = insertelement <4 x i16> [[TMP38]], i16 [[TMP41]], i64 3 5391; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE15]] 5392; INTERLEAVE: pred.urem.continue15: 5393; INTERLEAVE-NEXT: [[TMP43:%.*]] = phi <4 x i16> [ [[TMP38]], [[PRED_UREM_CONTINUE13]] ], [ [[TMP42]], [[PRED_UREM_IF14]] ] 5394; INTERLEAVE-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i16> [[TMP23]], <4 x i16> zeroinitializer 5395; INTERLEAVE-NEXT: [[PREDPHI16:%.*]] = select <4 x i1> [[TMP3]], <4 x i16> [[TMP43]], <4 x i16> zeroinitializer 5396; INTERLEAVE-NEXT: [[TMP44:%.*]] = sext <4 x i16> [[PREDPHI]] to <4 x i32> 5397; INTERLEAVE-NEXT: [[TMP45:%.*]] = sext <4 x i16> [[PREDPHI16]] to <4 x i32> 5398; INTERLEAVE-NEXT: [[TMP46]] = or <4 x i32> [[VEC_PHI]], [[TMP44]] 5399; INTERLEAVE-NEXT: [[TMP47]] = or <4 x i32> [[VEC_PHI1]], [[TMP45]] 5400; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 5401; INTERLEAVE-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 8) 5402; INTERLEAVE-NEXT: [[TMP48:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 5403; INTERLEAVE-NEXT: br i1 [[TMP48]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] 5404; INTERLEAVE: middle.block: 5405; INTERLEAVE-NEXT: [[BIN_RDX:%.*]] = or <4 x i32> [[TMP47]], [[TMP46]] 5406; INTERLEAVE-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[BIN_RDX]]) 5407; INTERLEAVE-NEXT: br i1 false, label [[FOR_END:%.*]], label [[SCALAR_PH]] 5408; INTERLEAVE: scalar.ph: 5409; INTERLEAVE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP49]], [[MIDDLE_BLOCK]] ], [ poison, [[ENTRY:%.*]] ] 5410; INTERLEAVE-NEXT: br label [[FOR_BODY:%.*]] 5411; INTERLEAVE: for.body: 5412; INTERLEAVE-NEXT: [[I:%.*]] = phi i32 [ -4, [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ] 5413; INTERLEAVE-NEXT: [[VAR0:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[VAR6:%.*]], [[FOR_INC]] ] 5414; INTERLEAVE-NEXT: [[VAR1:%.*]] = trunc i32 [[I]] to i16 5415; INTERLEAVE-NEXT: [[VAR2:%.*]] = icmp eq i16 [[VAR1]], 0 5416; INTERLEAVE-NEXT: br i1 [[VAR2]], label [[FOR_INC]], label [[FOR_COND:%.*]] 5417; INTERLEAVE: for.cond: 5418; INTERLEAVE-NEXT: [[VAR3:%.*]] = urem i16 [[B]], [[VAR1]] 5419; INTERLEAVE-NEXT: [[TMP50:%.*]] = sext i16 [[VAR3]] to i32 5420; INTERLEAVE-NEXT: br label [[FOR_INC]] 5421; INTERLEAVE: for.inc: 5422; INTERLEAVE-NEXT: [[VAR4:%.*]] = phi i32 [ [[TMP50]], [[FOR_COND]] ], [ 0, [[FOR_BODY]] ] 5423; INTERLEAVE-NEXT: [[VAR6]] = or i32 [[VAR0]], [[VAR4]] 5424; INTERLEAVE-NEXT: [[I_NEXT]] = add nsw i32 [[I]], 1 5425; INTERLEAVE-NEXT: [[COND:%.*]] = icmp eq i32 [[I_NEXT]], 0 5426; INTERLEAVE-NEXT: br i1 [[COND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]] 5427; INTERLEAVE: for.end: 5428; INTERLEAVE-NEXT: [[VAR7:%.*]] = phi i32 [ [[VAR6]], [[FOR_INC]] ], [ poison, [[MIDDLE_BLOCK]] ] 5429; INTERLEAVE-NEXT: ret i32 [[VAR7]] 5430; 5431entry: 5432 br label %for.body 5433 5434for.body: 5435 %i = phi i32 [ -20, %entry ], [ %i.next, %for.inc ] 5436 %var0 = phi i32 [ %a, %entry ], [ %var6, %for.inc ] 5437 %var1 = trunc i32 %i to i16 5438 %var2 = icmp eq i16 %var1, 0 5439 br i1 %var2, label %for.inc, label %for.cond 5440 5441for.cond: 5442 %var3 = urem i16 %b, %var1 5443 br label %for.inc 5444 5445for.inc: 5446 %var4 = phi i16 [ %var3, %for.cond ], [ 0, %for.body ] 5447 %var5 = sext i16 %var4 to i32 5448 %var6 = or i32 %var0, %var5 5449 %i.next = add nsw i32 %i, 1 5450 %cond = icmp eq i32 %i.next, 0 5451 br i1 %cond, label %for.end, label %for.body 5452 5453for.end: 5454 %var7 = phi i32 [ %var6, %for.inc ] 5455 ret i32 %var7 5456} 5457 5458; Ensure that the shuffle vector for first order recurrence is inserted 5459; correctly after all the phis. These new phis correspond to new IVs 5460; that are generated by optimizing non-free truncs of IVs to IVs themselves. 5461; This also ensures the first-order recurrence splice recipe is placed 5462; correctly if it is fed by an induction. 5463define i64 @trunc_with_first_order_recurrence() { 5464; CHECK-LABEL: @trunc_with_first_order_recurrence( 5465; CHECK-NEXT: entry: 5466; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5467; CHECK: vector.ph: 5468; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 5469; CHECK: vector.body: 5470; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5471; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ] 5472; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 5473; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 42>, [[VECTOR_PH]] ], [ [[VEC_IND2:%.*]], [[VECTOR_BODY]] ] 5474; CHECK-NEXT: [[VEC_IND2]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ] 5475; CHECK-NEXT: [[VEC_IND4:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ] 5476; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND2]], <2 x i32> <i32 1, i32 2> 5477; CHECK-NEXT: [[TMP1:%.*]] = mul <2 x i32> [[VEC_IND]], [[VEC_IND2]] 5478; CHECK-NEXT: [[TMP2:%.*]] = add <2 x i32> [[TMP1]], splat (i32 42) 5479; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i32> [[TMP0]], [[VEC_IND2]] 5480; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP3]], [[TMP2]] 5481; CHECK-NEXT: [[TMP5:%.*]] = sext <2 x i32> [[TMP4]] to <2 x i64> 5482; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i64> [[VEC_PHI]], [[TMP5]] 5483; CHECK-NEXT: [[TMP7:%.*]] = shl <2 x i32> [[VEC_IND4]], splat (i32 1) 5484; CHECK-NEXT: [[TMP8:%.*]] = add <2 x i32> [[TMP2]], [[TMP7]] 5485; CHECK-NEXT: [[TMP9:%.*]] = sext <2 x i32> [[TMP8]] to <2 x i64> 5486; CHECK-NEXT: [[TMP10]] = add <2 x i64> [[TMP6]], [[TMP9]] 5487; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 5488; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 5489; CHECK-NEXT: [[VEC_IND_NEXT3]] = add <2 x i32> [[VEC_IND2]], splat (i32 2) 5490; CHECK-NEXT: [[VEC_IND_NEXT5]] = add <2 x i32> [[VEC_IND4]], splat (i32 2) 5491; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 112 5492; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] 5493; CHECK: middle.block: 5494; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[TMP10]]) 5495; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[VEC_IND2]], i32 1 5496; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]] 5497; CHECK: scalar.ph: 5498; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 5499; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 113, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY]] ] 5500; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 113, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY]] ] 5501; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 42, [[ENTRY]] ] 5502; CHECK-NEXT: br label [[LOOP:%.*]] 5503; CHECK: exit: 5504; CHECK-NEXT: [[DOTLCSSA:%.*]] = phi i64 [ [[C23:%.*]], [[LOOP]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ] 5505; CHECK-NEXT: ret i64 [[DOTLCSSA]] 5506; CHECK: loop: 5507; CHECK-NEXT: [[C5:%.*]] = phi i64 [ [[C23]], [[LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 5508; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 5509; CHECK-NEXT: [[X:%.*]] = phi i32 [ [[C24:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 5510; CHECK-NEXT: [[Y:%.*]] = phi i32 [ [[C6:%.*]], [[LOOP]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 5511; CHECK-NEXT: [[C6]] = trunc i64 [[INDVARS_IV]] to i32 5512; CHECK-NEXT: [[C8:%.*]] = mul i32 [[X]], [[C6]] 5513; CHECK-NEXT: [[C9:%.*]] = add i32 [[C8]], 42 5514; CHECK-NEXT: [[C10:%.*]] = add i32 [[Y]], [[C6]] 5515; CHECK-NEXT: [[C11:%.*]] = add i32 [[C10]], [[C9]] 5516; CHECK-NEXT: [[C12:%.*]] = sext i32 [[C11]] to i64 5517; CHECK-NEXT: [[C13:%.*]] = add i64 [[C5]], [[C12]] 5518; CHECK-NEXT: [[INDVARS_IV_TR:%.*]] = trunc i64 [[INDVARS_IV]] to i32 5519; CHECK-NEXT: [[C14:%.*]] = shl i32 [[INDVARS_IV_TR]], 1 5520; CHECK-NEXT: [[C15:%.*]] = add i32 [[C9]], [[C14]] 5521; CHECK-NEXT: [[C16:%.*]] = sext i32 [[C15]] to i64 5522; CHECK-NEXT: [[C23]] = add i64 [[C13]], [[C16]] 5523; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 5524; CHECK-NEXT: [[C24]] = add nuw nsw i32 [[X]], 1 5525; CHECK-NEXT: [[EXITCOND_I:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 114 5526; CHECK-NEXT: br i1 [[EXITCOND_I]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP51:![0-9]+]] 5527; 5528; IND-LABEL: @trunc_with_first_order_recurrence( 5529; IND-NEXT: entry: 5530; IND-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5531; IND: vector.ph: 5532; IND-NEXT: br label [[VECTOR_BODY:%.*]] 5533; IND: vector.body: 5534; IND-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5535; IND-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ] 5536; IND-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 5537; IND-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 42>, [[VECTOR_PH]] ], [ [[VEC_IND2:%.*]], [[VECTOR_BODY]] ] 5538; IND-NEXT: [[VEC_IND2]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ] 5539; IND-NEXT: [[VEC_IND4:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ] 5540; IND-NEXT: [[TMP0:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND2]], <2 x i32> <i32 1, i32 2> 5541; IND-NEXT: [[TMP1:%.*]] = mul <2 x i32> [[VEC_IND]], [[VEC_IND2]] 5542; IND-NEXT: [[TMP2:%.*]] = add <2 x i32> [[TMP1]], splat (i32 42) 5543; IND-NEXT: [[TMP3:%.*]] = add <2 x i32> [[TMP0]], [[VEC_IND2]] 5544; IND-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP3]], [[TMP2]] 5545; IND-NEXT: [[TMP5:%.*]] = sext <2 x i32> [[TMP4]] to <2 x i64> 5546; IND-NEXT: [[TMP6:%.*]] = add <2 x i64> [[VEC_PHI]], [[TMP5]] 5547; IND-NEXT: [[TMP7:%.*]] = shl <2 x i32> [[VEC_IND4]], splat (i32 1) 5548; IND-NEXT: [[TMP8:%.*]] = add <2 x i32> [[TMP2]], [[TMP7]] 5549; IND-NEXT: [[TMP9:%.*]] = sext <2 x i32> [[TMP8]] to <2 x i64> 5550; IND-NEXT: [[TMP10]] = add <2 x i64> [[TMP6]], [[TMP9]] 5551; IND-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 5552; IND-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 5553; IND-NEXT: [[VEC_IND_NEXT3]] = add <2 x i32> [[VEC_IND2]], splat (i32 2) 5554; IND-NEXT: [[VEC_IND_NEXT5]] = add <2 x i32> [[VEC_IND4]], splat (i32 2) 5555; IND-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 112 5556; IND-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] 5557; IND: middle.block: 5558; IND-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[TMP10]]) 5559; IND-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[VEC_IND2]], i64 1 5560; IND-NEXT: br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]] 5561; IND: scalar.ph: 5562; IND-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ poison, [[ENTRY:%.*]] ] 5563; IND-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ poison, [[ENTRY]] ] 5564; IND-NEXT: br label [[LOOP:%.*]] 5565; IND: exit: 5566; IND-NEXT: [[DOTLCSSA:%.*]] = phi i64 [ [[C23:%.*]], [[LOOP]] ], [ poison, [[MIDDLE_BLOCK]] ] 5567; IND-NEXT: ret i64 [[DOTLCSSA]] 5568; IND: loop: 5569; IND-NEXT: [[C5:%.*]] = phi i64 [ [[C23]], [[LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 5570; IND-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ], [ 113, [[SCALAR_PH]] ] 5571; IND-NEXT: [[X:%.*]] = phi i32 [ [[C24:%.*]], [[LOOP]] ], [ 113, [[SCALAR_PH]] ] 5572; IND-NEXT: [[Y:%.*]] = phi i32 [ [[C6:%.*]], [[LOOP]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 5573; IND-NEXT: [[C6]] = trunc i64 [[INDVARS_IV]] to i32 5574; IND-NEXT: [[C8:%.*]] = mul i32 [[X]], [[C6]] 5575; IND-NEXT: [[C9:%.*]] = add i32 [[C8]], 42 5576; IND-NEXT: [[C10:%.*]] = add i32 [[Y]], [[C6]] 5577; IND-NEXT: [[C11:%.*]] = add i32 [[C10]], [[C9]] 5578; IND-NEXT: [[C12:%.*]] = sext i32 [[C11]] to i64 5579; IND-NEXT: [[C13:%.*]] = add i64 [[C5]], [[C12]] 5580; IND-NEXT: [[INDVARS_IV_TR:%.*]] = trunc i64 [[INDVARS_IV]] to i32 5581; IND-NEXT: [[C14:%.*]] = shl i32 [[INDVARS_IV_TR]], 1 5582; IND-NEXT: [[C15:%.*]] = add i32 [[C9]], [[C14]] 5583; IND-NEXT: [[C16:%.*]] = sext i32 [[C15]] to i64 5584; IND-NEXT: [[C23]] = add i64 [[C13]], [[C16]] 5585; IND-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 5586; IND-NEXT: [[C24]] = add nuw nsw i32 [[X]], 1 5587; IND-NEXT: [[EXITCOND_I:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 114 5588; IND-NEXT: br i1 [[EXITCOND_I]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP51:![0-9]+]] 5589; 5590; UNROLL-LABEL: @trunc_with_first_order_recurrence( 5591; UNROLL-NEXT: entry: 5592; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5593; UNROLL: vector.ph: 5594; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 5595; UNROLL: vector.body: 5596; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5597; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ] 5598; UNROLL-NEXT: [[VEC_PHI2:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ] 5599; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 5600; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 42>, [[VECTOR_PH]] ], [ [[STEP_ADD7:%.*]], [[VECTOR_BODY]] ] 5601; UNROLL-NEXT: [[VEC_IND3:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], [[VECTOR_BODY]] ] 5602; UNROLL-NEXT: [[VEC_IND5:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT6:%.*]], [[VECTOR_BODY]] ] 5603; UNROLL-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 5604; UNROLL-NEXT: [[STEP_ADD7]] = add <2 x i32> [[VEC_IND3]], splat (i32 2) 5605; UNROLL-NEXT: [[TMP0:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND3]], <2 x i32> <i32 1, i32 2> 5606; UNROLL-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[VEC_IND3]], <2 x i32> [[STEP_ADD7]], <2 x i32> <i32 1, i32 2> 5607; UNROLL-NEXT: [[TMP2:%.*]] = mul <2 x i32> [[VEC_IND]], [[VEC_IND3]] 5608; UNROLL-NEXT: [[TMP3:%.*]] = mul <2 x i32> [[STEP_ADD]], [[STEP_ADD7]] 5609; UNROLL-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP2]], splat (i32 42) 5610; UNROLL-NEXT: [[TMP5:%.*]] = add <2 x i32> [[TMP3]], splat (i32 42) 5611; UNROLL-NEXT: [[TMP6:%.*]] = add <2 x i32> [[TMP0]], [[VEC_IND3]] 5612; UNROLL-NEXT: [[TMP7:%.*]] = add <2 x i32> [[TMP1]], [[STEP_ADD7]] 5613; UNROLL-NEXT: [[TMP8:%.*]] = add <2 x i32> [[TMP6]], [[TMP4]] 5614; UNROLL-NEXT: [[TMP9:%.*]] = add <2 x i32> [[TMP7]], [[TMP5]] 5615; UNROLL-NEXT: [[TMP10:%.*]] = sext <2 x i32> [[TMP8]] to <2 x i64> 5616; UNROLL-NEXT: [[TMP11:%.*]] = sext <2 x i32> [[TMP9]] to <2 x i64> 5617; UNROLL-NEXT: [[TMP12:%.*]] = add <2 x i64> [[VEC_PHI]], [[TMP10]] 5618; UNROLL-NEXT: [[TMP13:%.*]] = add <2 x i64> [[VEC_PHI2]], [[TMP11]] 5619; UNROLL-NEXT: [[TMP14:%.*]] = shl <2 x i32> [[VEC_IND5]], splat (i32 1) 5620; UNROLL-NEXT: [[STEP_ADD8:%.*]] = shl <2 x i32> [[VEC_IND5]], splat (i32 1) 5621; UNROLL-NEXT: [[TMP15:%.*]] = add <2 x i32> [[STEP_ADD8]], splat (i32 4) 5622; UNROLL-NEXT: [[TMP16:%.*]] = add <2 x i32> [[TMP4]], [[TMP14]] 5623; UNROLL-NEXT: [[TMP17:%.*]] = add <2 x i32> [[TMP5]], [[TMP15]] 5624; UNROLL-NEXT: [[TMP18:%.*]] = sext <2 x i32> [[TMP16]] to <2 x i64> 5625; UNROLL-NEXT: [[TMP19:%.*]] = sext <2 x i32> [[TMP17]] to <2 x i64> 5626; UNROLL-NEXT: [[TMP20]] = add <2 x i64> [[TMP12]], [[TMP18]] 5627; UNROLL-NEXT: [[TMP21]] = add <2 x i64> [[TMP13]], [[TMP19]] 5628; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 5629; UNROLL-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 4) 5630; UNROLL-NEXT: [[VEC_IND_NEXT4]] = add <2 x i32> [[VEC_IND3]], splat (i32 4) 5631; UNROLL-NEXT: [[VEC_IND_NEXT6]] = add <2 x i32> [[VEC_IND5]], splat (i32 4) 5632; UNROLL-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 112 5633; UNROLL-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] 5634; UNROLL: middle.block: 5635; UNROLL-NEXT: [[BIN_RDX:%.*]] = add <2 x i64> [[TMP21]], [[TMP20]] 5636; UNROLL-NEXT: [[TMP23:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX]]) 5637; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[STEP_ADD7]], i64 1 5638; UNROLL-NEXT: br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]] 5639; UNROLL: scalar.ph: 5640; UNROLL-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP23]], [[MIDDLE_BLOCK]] ], [ poison, [[ENTRY:%.*]] ] 5641; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ poison, [[ENTRY]] ] 5642; UNROLL-NEXT: br label [[LOOP:%.*]] 5643; UNROLL: exit: 5644; UNROLL-NEXT: [[DOTLCSSA:%.*]] = phi i64 [ [[C23:%.*]], [[LOOP]] ], [ poison, [[MIDDLE_BLOCK]] ] 5645; UNROLL-NEXT: ret i64 [[DOTLCSSA]] 5646; UNROLL: loop: 5647; UNROLL-NEXT: [[C5:%.*]] = phi i64 [ [[C23]], [[LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 5648; UNROLL-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ], [ 113, [[SCALAR_PH]] ] 5649; UNROLL-NEXT: [[X:%.*]] = phi i32 [ [[C24:%.*]], [[LOOP]] ], [ 113, [[SCALAR_PH]] ] 5650; UNROLL-NEXT: [[Y:%.*]] = phi i32 [ [[C6:%.*]], [[LOOP]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 5651; UNROLL-NEXT: [[C6]] = trunc i64 [[INDVARS_IV]] to i32 5652; UNROLL-NEXT: [[C8:%.*]] = mul i32 [[X]], [[C6]] 5653; UNROLL-NEXT: [[C9:%.*]] = add i32 [[C8]], 42 5654; UNROLL-NEXT: [[C10:%.*]] = add i32 [[Y]], [[C6]] 5655; UNROLL-NEXT: [[C11:%.*]] = add i32 [[C10]], [[C9]] 5656; UNROLL-NEXT: [[C12:%.*]] = sext i32 [[C11]] to i64 5657; UNROLL-NEXT: [[C13:%.*]] = add i64 [[C5]], [[C12]] 5658; UNROLL-NEXT: [[INDVARS_IV_TR:%.*]] = trunc i64 [[INDVARS_IV]] to i32 5659; UNROLL-NEXT: [[C14:%.*]] = shl i32 [[INDVARS_IV_TR]], 1 5660; UNROLL-NEXT: [[C15:%.*]] = add i32 [[C9]], [[C14]] 5661; UNROLL-NEXT: [[C16:%.*]] = sext i32 [[C15]] to i64 5662; UNROLL-NEXT: [[C23]] = add i64 [[C13]], [[C16]] 5663; UNROLL-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 5664; UNROLL-NEXT: [[C24]] = add nuw nsw i32 [[X]], 1 5665; UNROLL-NEXT: [[EXITCOND_I:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 114 5666; UNROLL-NEXT: br i1 [[EXITCOND_I]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP51:![0-9]+]] 5667; 5668; UNROLL-NO-IC-LABEL: @trunc_with_first_order_recurrence( 5669; UNROLL-NO-IC-NEXT: entry: 5670; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5671; UNROLL-NO-IC: vector.ph: 5672; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 5673; UNROLL-NO-IC: vector.body: 5674; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5675; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ] 5676; UNROLL-NO-IC-NEXT: [[VEC_PHI2:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ] 5677; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 5678; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 42>, [[VECTOR_PH]] ], [ [[STEP_ADD7:%.*]], [[VECTOR_BODY]] ] 5679; UNROLL-NO-IC-NEXT: [[VEC_IND3:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], [[VECTOR_BODY]] ] 5680; UNROLL-NO-IC-NEXT: [[VEC_IND5:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT6:%.*]], [[VECTOR_BODY]] ] 5681; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 5682; UNROLL-NO-IC-NEXT: [[STEP_ADD7]] = add <2 x i32> [[VEC_IND3]], splat (i32 2) 5683; UNROLL-NO-IC-NEXT: [[STEP_ADD8:%.*]] = add <2 x i32> [[VEC_IND5]], splat (i32 2) 5684; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND3]], <2 x i32> <i32 1, i32 2> 5685; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[VEC_IND3]], <2 x i32> [[STEP_ADD7]], <2 x i32> <i32 1, i32 2> 5686; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = mul <2 x i32> [[VEC_IND]], [[VEC_IND3]] 5687; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = mul <2 x i32> [[STEP_ADD]], [[STEP_ADD7]] 5688; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP2]], splat (i32 42) 5689; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = add <2 x i32> [[TMP3]], splat (i32 42) 5690; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = add <2 x i32> [[TMP0]], [[VEC_IND3]] 5691; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add <2 x i32> [[TMP1]], [[STEP_ADD7]] 5692; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add <2 x i32> [[TMP6]], [[TMP4]] 5693; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = add <2 x i32> [[TMP7]], [[TMP5]] 5694; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = sext <2 x i32> [[TMP8]] to <2 x i64> 5695; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = sext <2 x i32> [[TMP9]] to <2 x i64> 5696; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = add <2 x i64> [[VEC_PHI]], [[TMP10]] 5697; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = add <2 x i64> [[VEC_PHI2]], [[TMP11]] 5698; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = shl <2 x i32> [[VEC_IND5]], splat (i32 1) 5699; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = shl <2 x i32> [[STEP_ADD8]], splat (i32 1) 5700; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = add <2 x i32> [[TMP4]], [[TMP14]] 5701; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = add <2 x i32> [[TMP5]], [[TMP15]] 5702; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = sext <2 x i32> [[TMP16]] to <2 x i64> 5703; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = sext <2 x i32> [[TMP17]] to <2 x i64> 5704; UNROLL-NO-IC-NEXT: [[TMP20]] = add <2 x i64> [[TMP12]], [[TMP18]] 5705; UNROLL-NO-IC-NEXT: [[TMP21]] = add <2 x i64> [[TMP13]], [[TMP19]] 5706; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 5707; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 2) 5708; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT4]] = add <2 x i32> [[STEP_ADD7]], splat (i32 2) 5709; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT6]] = add <2 x i32> [[STEP_ADD8]], splat (i32 2) 5710; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 112 5711; UNROLL-NO-IC-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] 5712; UNROLL-NO-IC: middle.block: 5713; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = add <2 x i64> [[TMP21]], [[TMP20]] 5714; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX]]) 5715; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[STEP_ADD7]], i32 1 5716; UNROLL-NO-IC-NEXT: br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]] 5717; UNROLL-NO-IC: scalar.ph: 5718; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP23]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 5719; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 113, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY]] ] 5720; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 113, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY]] ] 5721; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 42, [[ENTRY]] ] 5722; UNROLL-NO-IC-NEXT: br label [[LOOP:%.*]] 5723; UNROLL-NO-IC: exit: 5724; UNROLL-NO-IC-NEXT: [[DOTLCSSA:%.*]] = phi i64 [ [[C23:%.*]], [[LOOP]] ], [ [[TMP23]], [[MIDDLE_BLOCK]] ] 5725; UNROLL-NO-IC-NEXT: ret i64 [[DOTLCSSA]] 5726; UNROLL-NO-IC: loop: 5727; UNROLL-NO-IC-NEXT: [[C5:%.*]] = phi i64 [ [[C23]], [[LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 5728; UNROLL-NO-IC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 5729; UNROLL-NO-IC-NEXT: [[X:%.*]] = phi i32 [ [[C24:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ] 5730; UNROLL-NO-IC-NEXT: [[Y:%.*]] = phi i32 [ [[C6:%.*]], [[LOOP]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 5731; UNROLL-NO-IC-NEXT: [[C6]] = trunc i64 [[INDVARS_IV]] to i32 5732; UNROLL-NO-IC-NEXT: [[C8:%.*]] = mul i32 [[X]], [[C6]] 5733; UNROLL-NO-IC-NEXT: [[C9:%.*]] = add i32 [[C8]], 42 5734; UNROLL-NO-IC-NEXT: [[C10:%.*]] = add i32 [[Y]], [[C6]] 5735; UNROLL-NO-IC-NEXT: [[C11:%.*]] = add i32 [[C10]], [[C9]] 5736; UNROLL-NO-IC-NEXT: [[C12:%.*]] = sext i32 [[C11]] to i64 5737; UNROLL-NO-IC-NEXT: [[C13:%.*]] = add i64 [[C5]], [[C12]] 5738; UNROLL-NO-IC-NEXT: [[INDVARS_IV_TR:%.*]] = trunc i64 [[INDVARS_IV]] to i32 5739; UNROLL-NO-IC-NEXT: [[C14:%.*]] = shl i32 [[INDVARS_IV_TR]], 1 5740; UNROLL-NO-IC-NEXT: [[C15:%.*]] = add i32 [[C9]], [[C14]] 5741; UNROLL-NO-IC-NEXT: [[C16:%.*]] = sext i32 [[C15]] to i64 5742; UNROLL-NO-IC-NEXT: [[C23]] = add i64 [[C13]], [[C16]] 5743; UNROLL-NO-IC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 5744; UNROLL-NO-IC-NEXT: [[C24]] = add nuw nsw i32 [[X]], 1 5745; UNROLL-NO-IC-NEXT: [[EXITCOND_I:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 114 5746; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND_I]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP51:![0-9]+]] 5747; 5748; INTERLEAVE-LABEL: @trunc_with_first_order_recurrence( 5749; INTERLEAVE-NEXT: entry: 5750; INTERLEAVE-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5751; INTERLEAVE: vector.ph: 5752; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 5753; INTERLEAVE: vector.body: 5754; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5755; INTERLEAVE-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ] 5756; INTERLEAVE-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ] 5757; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 5758; INTERLEAVE-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 42>, [[VECTOR_PH]] ], [ [[STEP_ADD7:%.*]], [[VECTOR_BODY]] ] 5759; INTERLEAVE-NEXT: [[VEC_IND3:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], [[VECTOR_BODY]] ] 5760; INTERLEAVE-NEXT: [[VEC_IND5:%.*]] = phi <4 x i32> [ <i32 1, i32 2, i32 3, i32 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT6:%.*]], [[VECTOR_BODY]] ] 5761; INTERLEAVE-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4) 5762; INTERLEAVE-NEXT: [[STEP_ADD7]] = add <4 x i32> [[VEC_IND3]], splat (i32 4) 5763; INTERLEAVE-NEXT: [[TMP0:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5764; INTERLEAVE-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[VEC_IND3]], <4 x i32> [[STEP_ADD7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 5765; INTERLEAVE-NEXT: [[TMP2:%.*]] = mul <4 x i32> [[VEC_IND]], [[VEC_IND3]] 5766; INTERLEAVE-NEXT: [[TMP3:%.*]] = mul <4 x i32> [[STEP_ADD]], [[STEP_ADD7]] 5767; INTERLEAVE-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP2]], splat (i32 42) 5768; INTERLEAVE-NEXT: [[TMP5:%.*]] = add <4 x i32> [[TMP3]], splat (i32 42) 5769; INTERLEAVE-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP0]], [[VEC_IND3]] 5770; INTERLEAVE-NEXT: [[TMP7:%.*]] = add <4 x i32> [[TMP1]], [[STEP_ADD7]] 5771; INTERLEAVE-NEXT: [[TMP8:%.*]] = add <4 x i32> [[TMP6]], [[TMP4]] 5772; INTERLEAVE-NEXT: [[TMP9:%.*]] = add <4 x i32> [[TMP7]], [[TMP5]] 5773; INTERLEAVE-NEXT: [[TMP10:%.*]] = sext <4 x i32> [[TMP8]] to <4 x i64> 5774; INTERLEAVE-NEXT: [[TMP11:%.*]] = sext <4 x i32> [[TMP9]] to <4 x i64> 5775; INTERLEAVE-NEXT: [[TMP12:%.*]] = add <4 x i64> [[VEC_PHI]], [[TMP10]] 5776; INTERLEAVE-NEXT: [[TMP13:%.*]] = add <4 x i64> [[VEC_PHI2]], [[TMP11]] 5777; INTERLEAVE-NEXT: [[TMP14:%.*]] = shl <4 x i32> [[VEC_IND5]], splat (i32 1) 5778; INTERLEAVE-NEXT: [[STEP_ADD8:%.*]] = shl <4 x i32> [[VEC_IND5]], splat (i32 1) 5779; INTERLEAVE-NEXT: [[TMP15:%.*]] = add <4 x i32> [[STEP_ADD8]], splat (i32 8) 5780; INTERLEAVE-NEXT: [[TMP16:%.*]] = add <4 x i32> [[TMP4]], [[TMP14]] 5781; INTERLEAVE-NEXT: [[TMP17:%.*]] = add <4 x i32> [[TMP5]], [[TMP15]] 5782; INTERLEAVE-NEXT: [[TMP18:%.*]] = sext <4 x i32> [[TMP16]] to <4 x i64> 5783; INTERLEAVE-NEXT: [[TMP19:%.*]] = sext <4 x i32> [[TMP17]] to <4 x i64> 5784; INTERLEAVE-NEXT: [[TMP20]] = add <4 x i64> [[TMP12]], [[TMP18]] 5785; INTERLEAVE-NEXT: [[TMP21]] = add <4 x i64> [[TMP13]], [[TMP19]] 5786; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 5787; INTERLEAVE-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 8) 5788; INTERLEAVE-NEXT: [[VEC_IND_NEXT4]] = add <4 x i32> [[VEC_IND3]], splat (i32 8) 5789; INTERLEAVE-NEXT: [[VEC_IND_NEXT6]] = add <4 x i32> [[VEC_IND5]], splat (i32 8) 5790; INTERLEAVE-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 112 5791; INTERLEAVE-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] 5792; INTERLEAVE: middle.block: 5793; INTERLEAVE-NEXT: [[BIN_RDX:%.*]] = add <4 x i64> [[TMP21]], [[TMP20]] 5794; INTERLEAVE-NEXT: [[TMP23:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[BIN_RDX]]) 5795; INTERLEAVE-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[STEP_ADD7]], i64 3 5796; INTERLEAVE-NEXT: br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]] 5797; INTERLEAVE: scalar.ph: 5798; INTERLEAVE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP23]], [[MIDDLE_BLOCK]] ], [ poison, [[ENTRY:%.*]] ] 5799; INTERLEAVE-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ poison, [[ENTRY]] ] 5800; INTERLEAVE-NEXT: br label [[LOOP:%.*]] 5801; INTERLEAVE: exit: 5802; INTERLEAVE-NEXT: [[DOTLCSSA:%.*]] = phi i64 [ [[C23:%.*]], [[LOOP]] ], [ poison, [[MIDDLE_BLOCK]] ] 5803; INTERLEAVE-NEXT: ret i64 [[DOTLCSSA]] 5804; INTERLEAVE: loop: 5805; INTERLEAVE-NEXT: [[C5:%.*]] = phi i64 [ [[C23]], [[LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] 5806; INTERLEAVE-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ], [ 113, [[SCALAR_PH]] ] 5807; INTERLEAVE-NEXT: [[X:%.*]] = phi i32 [ [[C24:%.*]], [[LOOP]] ], [ 113, [[SCALAR_PH]] ] 5808; INTERLEAVE-NEXT: [[Y:%.*]] = phi i32 [ [[C6:%.*]], [[LOOP]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ] 5809; INTERLEAVE-NEXT: [[C6]] = trunc i64 [[INDVARS_IV]] to i32 5810; INTERLEAVE-NEXT: [[C8:%.*]] = mul i32 [[X]], [[C6]] 5811; INTERLEAVE-NEXT: [[C9:%.*]] = add i32 [[C8]], 42 5812; INTERLEAVE-NEXT: [[C10:%.*]] = add i32 [[Y]], [[C6]] 5813; INTERLEAVE-NEXT: [[C11:%.*]] = add i32 [[C10]], [[C9]] 5814; INTERLEAVE-NEXT: [[C12:%.*]] = sext i32 [[C11]] to i64 5815; INTERLEAVE-NEXT: [[C13:%.*]] = add i64 [[C5]], [[C12]] 5816; INTERLEAVE-NEXT: [[INDVARS_IV_TR:%.*]] = trunc i64 [[INDVARS_IV]] to i32 5817; INTERLEAVE-NEXT: [[C14:%.*]] = shl i32 [[INDVARS_IV_TR]], 1 5818; INTERLEAVE-NEXT: [[C15:%.*]] = add i32 [[C9]], [[C14]] 5819; INTERLEAVE-NEXT: [[C16:%.*]] = sext i32 [[C15]] to i64 5820; INTERLEAVE-NEXT: [[C23]] = add i64 [[C13]], [[C16]] 5821; INTERLEAVE-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 5822; INTERLEAVE-NEXT: [[C24]] = add nuw nsw i32 [[X]], 1 5823; INTERLEAVE-NEXT: [[EXITCOND_I:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 114 5824; INTERLEAVE-NEXT: br i1 [[EXITCOND_I]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP51:![0-9]+]] 5825; 5826entry: 5827 br label %loop 5828 5829exit: ; preds = %loop 5830 %.lcssa = phi i64 [ %c23, %loop ] 5831 ret i64 %.lcssa 5832 5833loop: ; preds = %loop, %entry 5834 %c5 = phi i64 [ %c23, %loop ], [ 0, %entry ] 5835 %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ 1, %entry ] 5836 %x = phi i32 [ %c24, %loop ], [ 1, %entry ] 5837 %y = phi i32 [ %c6, %loop ], [ 42, %entry ] 5838 %c6 = trunc i64 %indvars.iv to i32 5839 %c8 = mul i32 %x, %c6 5840 %c9 = add i32 %c8, 42 5841 %c10 = add i32 %y, %c6 5842 %c11 = add i32 %c10, %c9 5843 %c12 = sext i32 %c11 to i64 5844 %c13 = add i64 %c5, %c12 5845 %indvars.iv.tr = trunc i64 %indvars.iv to i32 5846 %c14 = shl i32 %indvars.iv.tr, 1 5847 %c15 = add i32 %c9, %c14 5848 %c16 = sext i32 %c15 to i64 5849 %c23 = add i64 %c13, %c16 5850 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 5851 %c24 = add nuw nsw i32 %x, 1 5852 %exitcond.i = icmp eq i64 %indvars.iv.next, 114 5853 br i1 %exitcond.i, label %exit, label %loop 5854 5855} 5856 5857; Test case for PR52460. 5858define void @pr52460_first_order_recurrence_truncated_iv(ptr noalias %src, ptr %dst) { 5859; 5860; CHECK-LABEL: @pr52460_first_order_recurrence_truncated_iv( 5861; CHECK-NEXT: entry: 5862; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5863; CHECK: vector.ph: 5864; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 5865; CHECK: vector.body: 5866; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5867; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[VEC_IND:%.*]], [[VECTOR_BODY]] ] 5868; CHECK-NEXT: [[VEC_IND]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 5869; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[INDEX]] to i32 5870; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 0 5871; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2> 5872; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[SRC:%.*]], align 4 5873; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i64 0 5874; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 5875; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <2 x i32> [[BROADCAST_SPLAT]], [[TMP2]] 5876; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[DST:%.*]], i32 [[TMP1]] 5877; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i32> [[VEC_IND]], [[TMP4]] 5878; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP5]], i32 0 5879; CHECK-NEXT: store <2 x i32> [[TMP6]], ptr [[TMP7]], align 4 5880; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 5881; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 5882; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100 5883; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]] 5884; CHECK: middle.block: 5885; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[VEC_IND]], i32 1 5886; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 5887; CHECK: scalar.ph: 5888; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 5889; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 100, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 5890; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 5891; CHECK-NEXT: br label [[LOOP:%.*]] 5892; CHECK: loop: 5893; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] 5894; CHECK-NEXT: [[TRUNC_IV:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[TRUNC_IV_NEXT:%.*]], [[LOOP]] ] 5895; CHECK-NEXT: [[RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_TRUNC:%.*]], [[LOOP]] ] 5896; CHECK-NEXT: [[LV:%.*]] = load i32, ptr [[SRC]], align 4 5897; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[LV]], [[RECUR]] 5898; CHECK-NEXT: [[TRUNC_IV_NEXT]] = add i32 [[TRUNC_IV]], 1 5899; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 5900; CHECK-NEXT: [[IV_TRUNC]] = trunc i64 [[IV]] to i32 5901; CHECK-NEXT: [[DST_GEP:%.*]] = getelementptr i32, ptr [[DST]], i32 [[IV_TRUNC]] 5902; CHECK-NEXT: [[ADD:%.*]] = add i32 [[IV_TRUNC]], [[MUL]] 5903; CHECK-NEXT: store i32 [[ADD]], ptr [[DST_GEP]], align 4 5904; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[TRUNC_IV_NEXT]], 100 5905; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP53:![0-9]+]] 5906; CHECK: exit: 5907; CHECK-NEXT: ret void 5908; 5909; IND-LABEL: @pr52460_first_order_recurrence_truncated_iv( 5910; IND-NEXT: entry: 5911; IND-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5912; IND: vector.ph: 5913; IND-NEXT: br label [[VECTOR_BODY:%.*]] 5914; IND: vector.body: 5915; IND-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5916; IND-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[VEC_IND:%.*]], [[VECTOR_BODY]] ] 5917; IND-NEXT: [[VEC_IND]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 5918; IND-NEXT: [[TMP0:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2> 5919; IND-NEXT: [[TMP1:%.*]] = load i32, ptr [[SRC:%.*]], align 4 5920; IND-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP1]], i64 0 5921; IND-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 5922; IND-NEXT: [[TMP2:%.*]] = mul nsw <2 x i32> [[BROADCAST_SPLAT]], [[TMP0]] 5923; IND-NEXT: [[SEXT:%.*]] = shl i64 [[INDEX]], 32 5924; IND-NEXT: [[TMP3:%.*]] = ashr exact i64 [[SEXT]], 30 5925; IND-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[TMP3]] 5926; IND-NEXT: [[TMP5:%.*]] = add <2 x i32> [[VEC_IND]], [[TMP2]] 5927; IND-NEXT: store <2 x i32> [[TMP5]], ptr [[TMP4]], align 4 5928; IND-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 5929; IND-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 5930; IND-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100 5931; IND-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]] 5932; IND: middle.block: 5933; IND-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 5934; IND: scalar.ph: 5935; IND-NEXT: br label [[LOOP:%.*]] 5936; IND: loop: 5937; IND-NEXT: br i1 poison, label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP53:![0-9]+]] 5938; IND: exit: 5939; IND-NEXT: ret void 5940; 5941; UNROLL-LABEL: @pr52460_first_order_recurrence_truncated_iv( 5942; UNROLL-NEXT: entry: 5943; UNROLL-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5944; UNROLL: vector.ph: 5945; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 5946; UNROLL: vector.body: 5947; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5948; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], [[VECTOR_BODY]] ] 5949; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 5950; UNROLL-NEXT: [[STEP_ADD]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 5951; UNROLL-NEXT: [[TMP0:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2> 5952; UNROLL-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[VEC_IND]], <2 x i32> [[STEP_ADD]], <2 x i32> <i32 1, i32 2> 5953; UNROLL-NEXT: [[TMP2:%.*]] = load i32, ptr [[SRC:%.*]], align 4 5954; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP2]], i64 0 5955; UNROLL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 5956; UNROLL-NEXT: [[TMP3:%.*]] = mul nsw <2 x i32> [[BROADCAST_SPLAT]], [[TMP0]] 5957; UNROLL-NEXT: [[TMP4:%.*]] = mul nsw <2 x i32> [[BROADCAST_SPLAT]], [[TMP1]] 5958; UNROLL-NEXT: [[SEXT:%.*]] = shl i64 [[INDEX]], 32 5959; UNROLL-NEXT: [[TMP5:%.*]] = ashr exact i64 [[SEXT]], 30 5960; UNROLL-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[TMP5]] 5961; UNROLL-NEXT: [[TMP7:%.*]] = add <2 x i32> [[VEC_IND]], [[TMP3]] 5962; UNROLL-NEXT: [[TMP8:%.*]] = add <2 x i32> [[STEP_ADD]], [[TMP4]] 5963; UNROLL-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8 5964; UNROLL-NEXT: store <2 x i32> [[TMP7]], ptr [[TMP6]], align 4 5965; UNROLL-NEXT: store <2 x i32> [[TMP8]], ptr [[TMP9]], align 4 5966; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 5967; UNROLL-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 4) 5968; UNROLL-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100 5969; UNROLL-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]] 5970; UNROLL: middle.block: 5971; UNROLL-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 5972; UNROLL: scalar.ph: 5973; UNROLL-NEXT: br label [[LOOP:%.*]] 5974; UNROLL: loop: 5975; UNROLL-NEXT: br i1 poison, label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP53:![0-9]+]] 5976; UNROLL: exit: 5977; UNROLL-NEXT: ret void 5978; 5979; UNROLL-NO-IC-LABEL: @pr52460_first_order_recurrence_truncated_iv( 5980; UNROLL-NO-IC-NEXT: entry: 5981; UNROLL-NO-IC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 5982; UNROLL-NO-IC: vector.ph: 5983; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 5984; UNROLL-NO-IC: vector.body: 5985; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 5986; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], [[VECTOR_BODY]] ] 5987; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 5988; UNROLL-NO-IC-NEXT: [[STEP_ADD]] = add <2 x i32> [[VEC_IND]], splat (i32 2) 5989; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = trunc i64 [[INDEX]] to i32 5990; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 0 5991; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2> 5992; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[VEC_IND]], <2 x i32> [[STEP_ADD]], <2 x i32> <i32 1, i32 2> 5993; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = load i32, ptr [[SRC:%.*]], align 4 5994; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0 5995; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 5996; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = mul nsw <2 x i32> [[BROADCAST_SPLAT]], [[TMP2]] 5997; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = mul nsw <2 x i32> [[BROADCAST_SPLAT]], [[TMP3]] 5998; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[DST:%.*]], i32 [[TMP1]] 5999; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add <2 x i32> [[VEC_IND]], [[TMP5]] 6000; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = add <2 x i32> [[STEP_ADD]], [[TMP6]] 6001; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[TMP7]], i32 0 6002; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[TMP7]], i32 2 6003; UNROLL-NO-IC-NEXT: store <2 x i32> [[TMP8]], ptr [[TMP10]], align 4 6004; UNROLL-NO-IC-NEXT: store <2 x i32> [[TMP9]], ptr [[TMP11]], align 4 6005; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 6006; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 2) 6007; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100 6008; UNROLL-NO-IC-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]] 6009; UNROLL-NO-IC: middle.block: 6010; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[STEP_ADD]], i32 1 6011; UNROLL-NO-IC-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 6012; UNROLL-NO-IC: scalar.ph: 6013; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 6014; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 100, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 6015; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] 6016; UNROLL-NO-IC-NEXT: br label [[LOOP:%.*]] 6017; UNROLL-NO-IC: loop: 6018; UNROLL-NO-IC-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] 6019; UNROLL-NO-IC-NEXT: [[TRUNC_IV:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[TRUNC_IV_NEXT:%.*]], [[LOOP]] ] 6020; UNROLL-NO-IC-NEXT: [[RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_TRUNC:%.*]], [[LOOP]] ] 6021; UNROLL-NO-IC-NEXT: [[LV:%.*]] = load i32, ptr [[SRC]], align 4 6022; UNROLL-NO-IC-NEXT: [[MUL:%.*]] = mul nsw i32 [[LV]], [[RECUR]] 6023; UNROLL-NO-IC-NEXT: [[TRUNC_IV_NEXT]] = add i32 [[TRUNC_IV]], 1 6024; UNROLL-NO-IC-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 6025; UNROLL-NO-IC-NEXT: [[IV_TRUNC]] = trunc i64 [[IV]] to i32 6026; UNROLL-NO-IC-NEXT: [[DST_GEP:%.*]] = getelementptr i32, ptr [[DST]], i32 [[IV_TRUNC]] 6027; UNROLL-NO-IC-NEXT: [[ADD:%.*]] = add i32 [[IV_TRUNC]], [[MUL]] 6028; UNROLL-NO-IC-NEXT: store i32 [[ADD]], ptr [[DST_GEP]], align 4 6029; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[TRUNC_IV_NEXT]], 100 6030; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP53:![0-9]+]] 6031; UNROLL-NO-IC: exit: 6032; UNROLL-NO-IC-NEXT: ret void 6033; 6034; INTERLEAVE-LABEL: @pr52460_first_order_recurrence_truncated_iv( 6035; INTERLEAVE-NEXT: entry: 6036; INTERLEAVE-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 6037; INTERLEAVE: vector.ph: 6038; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 6039; INTERLEAVE: vector.body: 6040; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 6041; INTERLEAVE-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], [[VECTOR_BODY]] ] 6042; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 6043; INTERLEAVE-NEXT: [[STEP_ADD]] = add <4 x i32> [[VEC_IND]], splat (i32 4) 6044; INTERLEAVE-NEXT: [[TMP0:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 6045; INTERLEAVE-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[VEC_IND]], <4 x i32> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 6046; INTERLEAVE-NEXT: [[TMP2:%.*]] = load i32, ptr [[SRC:%.*]], align 4 6047; INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2]], i64 0 6048; INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 6049; INTERLEAVE-NEXT: [[TMP3:%.*]] = mul nsw <4 x i32> [[BROADCAST_SPLAT]], [[TMP0]] 6050; INTERLEAVE-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[BROADCAST_SPLAT]], [[TMP1]] 6051; INTERLEAVE-NEXT: [[SEXT:%.*]] = shl i64 [[INDEX]], 32 6052; INTERLEAVE-NEXT: [[TMP5:%.*]] = ashr exact i64 [[SEXT]], 30 6053; INTERLEAVE-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[TMP5]] 6054; INTERLEAVE-NEXT: [[TMP7:%.*]] = add <4 x i32> [[VEC_IND]], [[TMP3]] 6055; INTERLEAVE-NEXT: [[TMP8:%.*]] = add <4 x i32> [[STEP_ADD]], [[TMP4]] 6056; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP6]], i64 16 6057; INTERLEAVE-NEXT: store <4 x i32> [[TMP7]], ptr [[TMP6]], align 4 6058; INTERLEAVE-NEXT: store <4 x i32> [[TMP8]], ptr [[TMP9]], align 4 6059; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 6060; INTERLEAVE-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 8) 6061; INTERLEAVE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96 6062; INTERLEAVE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]] 6063; INTERLEAVE: middle.block: 6064; INTERLEAVE-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[STEP_ADD]], i64 3 6065; INTERLEAVE-NEXT: br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]] 6066; INTERLEAVE: scalar.ph: 6067; INTERLEAVE-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ poison, [[ENTRY:%.*]] ] 6068; INTERLEAVE-NEXT: br label [[LOOP:%.*]] 6069; INTERLEAVE: loop: 6070; INTERLEAVE-NEXT: [[IV:%.*]] = phi i64 [ 96, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] 6071; INTERLEAVE-NEXT: [[TRUNC_IV:%.*]] = phi i32 [ 96, [[SCALAR_PH]] ], [ [[TRUNC_IV_NEXT:%.*]], [[LOOP]] ] 6072; INTERLEAVE-NEXT: [[RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_TRUNC:%.*]], [[LOOP]] ] 6073; INTERLEAVE-NEXT: [[LV:%.*]] = load i32, ptr [[SRC]], align 4 6074; INTERLEAVE-NEXT: [[MUL:%.*]] = mul nsw i32 [[LV]], [[RECUR]] 6075; INTERLEAVE-NEXT: [[TRUNC_IV_NEXT]] = add i32 [[TRUNC_IV]], 1 6076; INTERLEAVE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 6077; INTERLEAVE-NEXT: [[IV_TRUNC]] = trunc i64 [[IV]] to i32 6078; INTERLEAVE-NEXT: [[SEXT2:%.*]] = shl i64 [[IV]], 32 6079; INTERLEAVE-NEXT: [[TMP11:%.*]] = ashr exact i64 [[SEXT2]], 30 6080; INTERLEAVE-NEXT: [[DST_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP11]] 6081; INTERLEAVE-NEXT: [[ADD:%.*]] = add i32 [[MUL]], [[IV_TRUNC]] 6082; INTERLEAVE-NEXT: store i32 [[ADD]], ptr [[DST_GEP]], align 4 6083; INTERLEAVE-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[TRUNC_IV_NEXT]], 100 6084; INTERLEAVE-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP53:![0-9]+]] 6085; INTERLEAVE: exit: 6086; INTERLEAVE-NEXT: ret void 6087; 6088entry: 6089 br label %loop 6090 6091loop: 6092 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] 6093 %trunc.iv = phi i32 [ 0, %entry ], [ %trunc.iv.next, %loop ] 6094 %recur = phi i32 [ 0, %entry ], [ %iv.trunc, %loop ] 6095 %lv = load i32, ptr %src, align 4 6096 %mul = mul nsw i32 %lv, %recur 6097 %trunc.iv.next = add i32 %trunc.iv, 1 6098 %iv.next = add nuw nsw i64 %iv, 1 6099 %iv.trunc = trunc i64 %iv to i32 6100 %dst.gep = getelementptr i32, ptr %dst, i32 %iv.trunc 6101 %add = add i32 %iv.trunc, %mul 6102 store i32 %add, ptr %dst.gep 6103 %exitcond = icmp eq i32 %trunc.iv.next, 100 6104 br i1 %exitcond, label %exit, label %loop 6105 6106exit: 6107 ret void 6108} 6109 6110; Test case where %iv.2.ext and %iv.2.conv become redundant due to the SCEV 6111; predicates generated for the vector loop. They should be removed in the 6112; vector loop. 6113define void @test_optimized_cast_induction_feeding_first_order_recurrence(i64 %n, i32 %step, ptr %ptr) { 6114; 6115; CHECK-LABEL: @test_optimized_cast_induction_feeding_first_order_recurrence( 6116; CHECK-NEXT: entry: 6117; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2 6118; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 6119; CHECK: vector.scevcheck: 6120; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 6121; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[STEP:%.*]] to i8 6122; CHECK-NEXT: [[TMP2:%.*]] = sub i8 0, [[TMP1]] 6123; CHECK-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP1]], 0 6124; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i8 [[TMP2]], i8 [[TMP1]] 6125; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP0]] to i8 6126; CHECK-NEXT: [[MUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[TMP4]], i8 [[TMP5]]) 6127; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i8, i1 } [[MUL]], 0 6128; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i8, i1 } [[MUL]], 1 6129; CHECK-NEXT: [[TMP6:%.*]] = sub i8 0, [[MUL_RESULT]] 6130; CHECK-NEXT: [[TMP7:%.*]] = icmp slt i8 [[MUL_RESULT]], 0 6131; CHECK-NEXT: [[TMP8:%.*]] = icmp sgt i8 [[TMP6]], 0 6132; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP3]], i1 [[TMP8]], i1 [[TMP7]] 6133; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP9]], [[MUL_OVERFLOW]] 6134; CHECK-NEXT: [[TMP11:%.*]] = icmp ugt i64 [[TMP0]], 255 6135; CHECK-NEXT: [[TMP12:%.*]] = icmp ne i8 [[TMP1]], 0 6136; CHECK-NEXT: [[TMP13:%.*]] = and i1 [[TMP11]], [[TMP12]] 6137; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP10]], [[TMP13]] 6138; CHECK-NEXT: [[TMP15:%.*]] = sext i8 [[TMP1]] to i32 6139; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STEP]], [[TMP15]] 6140; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[TMP14]], [[IDENT_CHECK]] 6141; CHECK-NEXT: br i1 [[TMP16]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 6142; CHECK: vector.ph: 6143; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2 6144; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 6145; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32 6146; CHECK-NEXT: [[IND_END:%.*]] = mul i32 [[DOTCAST]], [[STEP]] 6147; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[STEP]], i64 0 6148; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 6149; CHECK-NEXT: [[TMP17:%.*]] = mul <2 x i32> <i32 0, i32 1>, [[DOTSPLAT]] 6150; CHECK-NEXT: [[INDUCTION:%.*]] = add <2 x i32> zeroinitializer, [[TMP17]] 6151; CHECK-NEXT: [[TMP18:%.*]] = mul i32 [[STEP]], 2 6152; CHECK-NEXT: [[DOTSPLATINSERT2:%.*]] = insertelement <2 x i32> poison, i32 [[TMP18]], i64 0 6153; CHECK-NEXT: [[DOTSPLAT3:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT2]], <2 x i32> poison, <2 x i32> zeroinitializer 6154; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 6155; CHECK: vector.body: 6156; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 6157; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[VEC_IND:%.*]], [[VECTOR_BODY]] ] 6158; CHECK-NEXT: [[VEC_IND]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 6159; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX]], 0 6160; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2> 6161; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 [[TMP19]] 6162; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i32 0 6163; CHECK-NEXT: store <2 x i32> [[TMP20]], ptr [[TMP22]], align 4 6164; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 6165; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], [[DOTSPLAT3]] 6166; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 6167; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]] 6168; CHECK: middle.block: 6169; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[VEC_IND]], i32 1 6170; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 6171; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 6172; CHECK: scalar.ph: 6173; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ] 6174; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ] 6175; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ] 6176; CHECK-NEXT: br label [[LOOP:%.*]] 6177; CHECK: loop: 6178; CHECK-NEXT: [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_2_CONV:%.*]], [[LOOP]] ] 6179; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ] 6180; CHECK-NEXT: [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP]] ] 6181; CHECK-NEXT: [[IV_2_EXT:%.*]] = shl i32 [[IV_2]], 24 6182; CHECK-NEXT: [[IV_2_CONV]] = ashr exact i32 [[IV_2_EXT]], 24 6183; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[IV_1]] 6184; CHECK-NEXT: store i32 [[FOR]], ptr [[GEP]], align 4 6185; CHECK-NEXT: [[IV_2_NEXT]] = add nsw i32 [[IV_2_CONV]], [[STEP]] 6186; CHECK-NEXT: [[IV_1_NEXT]] = add nuw nsw i64 [[IV_1]], 1 6187; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_1_NEXT]], [[N]] 6188; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP55:![0-9]+]] 6189; CHECK: exit: 6190; CHECK-NEXT: ret void 6191; 6192; IND-LABEL: @test_optimized_cast_induction_feeding_first_order_recurrence( 6193; IND-NEXT: entry: 6194; IND-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2 6195; IND-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 6196; IND: vector.scevcheck: 6197; IND-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 6198; IND-NEXT: [[TMP1:%.*]] = trunc i32 [[STEP:%.*]] to i8 6199; IND-NEXT: [[TMP2:%.*]] = icmp slt i8 [[TMP1]], 0 6200; IND-NEXT: [[TMP3:%.*]] = call i8 @llvm.abs.i8(i8 [[TMP1]], i1 false) 6201; IND-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP0]] to i8 6202; IND-NEXT: [[MUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[TMP3]], i8 [[TMP4]]) 6203; IND-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i8, i1 } [[MUL]], 0 6204; IND-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i8, i1 } [[MUL]], 1 6205; IND-NEXT: [[TMP5:%.*]] = icmp slt i8 [[MUL_RESULT]], 0 6206; IND-NEXT: [[TMP6:%.*]] = icmp ugt i8 [[MUL_RESULT]], -128 6207; IND-NEXT: [[TMP7:%.*]] = select i1 [[TMP2]], i1 [[TMP6]], i1 [[TMP5]] 6208; IND-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]] 6209; IND-NEXT: [[TMP9:%.*]] = icmp ugt i64 [[TMP0]], 255 6210; IND-NEXT: [[TMP10:%.*]] = icmp ne i8 [[TMP1]], 0 6211; IND-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] 6212; IND-NEXT: [[TMP12:%.*]] = or i1 [[TMP8]], [[TMP11]] 6213; IND-NEXT: [[TMP13:%.*]] = add i32 [[STEP]], -128 6214; IND-NEXT: [[IDENT_CHECK:%.*]] = icmp ult i32 [[TMP13]], -256 6215; IND-NEXT: [[TMP14:%.*]] = or i1 [[TMP12]], [[IDENT_CHECK]] 6216; IND-NEXT: br i1 [[TMP14]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 6217; IND: vector.ph: 6218; IND-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -2 6219; IND-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32 6220; IND-NEXT: [[IND_END:%.*]] = mul i32 [[STEP]], [[DOTCAST]] 6221; IND-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[STEP]], i64 0 6222; IND-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 6223; IND-NEXT: [[TMP15:%.*]] = mul nuw <2 x i32> [[DOTSPLAT]], <i32 0, i32 1> 6224; IND-NEXT: [[TMP16:%.*]] = shl i32 [[STEP]], 1 6225; IND-NEXT: [[DOTSPLATINSERT2:%.*]] = insertelement <2 x i32> poison, i32 [[TMP16]], i64 0 6226; IND-NEXT: [[DOTSPLAT3:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT2]], <2 x i32> poison, <2 x i32> zeroinitializer 6227; IND-NEXT: br label [[VECTOR_BODY:%.*]] 6228; IND: vector.body: 6229; IND-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 6230; IND-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[VEC_IND:%.*]], [[VECTOR_BODY]] ] 6231; IND-NEXT: [[VEC_IND]] = phi <2 x i32> [ [[TMP15]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 6232; IND-NEXT: [[TMP17:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2> 6233; IND-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 [[INDEX]] 6234; IND-NEXT: store <2 x i32> [[TMP17]], ptr [[TMP18]], align 4 6235; IND-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 6236; IND-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], [[DOTSPLAT3]] 6237; IND-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 6238; IND-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]] 6239; IND: middle.block: 6240; IND-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[VEC_IND]], i64 1 6241; IND-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 6242; IND-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 6243; IND: scalar.ph: 6244; IND-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ] 6245; IND-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ] 6246; IND-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ] 6247; IND-NEXT: br label [[LOOP:%.*]] 6248; IND: loop: 6249; IND-NEXT: [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_2_CONV:%.*]], [[LOOP]] ] 6250; IND-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ] 6251; IND-NEXT: [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP]] ] 6252; IND-NEXT: [[IV_2_EXT:%.*]] = shl i32 [[IV_2]], 24 6253; IND-NEXT: [[IV_2_CONV]] = ashr exact i32 [[IV_2_EXT]], 24 6254; IND-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[IV_1]] 6255; IND-NEXT: store i32 [[FOR]], ptr [[GEP]], align 4 6256; IND-NEXT: [[IV_2_NEXT]] = add nsw i32 [[IV_2_CONV]], [[STEP]] 6257; IND-NEXT: [[IV_1_NEXT]] = add nuw nsw i64 [[IV_1]], 1 6258; IND-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_1_NEXT]], [[N]] 6259; IND-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP55:![0-9]+]] 6260; IND: exit: 6261; IND-NEXT: ret void 6262; 6263; UNROLL-LABEL: @test_optimized_cast_induction_feeding_first_order_recurrence( 6264; UNROLL-NEXT: entry: 6265; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 6266; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 6267; UNROLL: vector.scevcheck: 6268; UNROLL-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 6269; UNROLL-NEXT: [[TMP1:%.*]] = trunc i32 [[STEP:%.*]] to i8 6270; UNROLL-NEXT: [[TMP2:%.*]] = icmp slt i8 [[TMP1]], 0 6271; UNROLL-NEXT: [[TMP3:%.*]] = call i8 @llvm.abs.i8(i8 [[TMP1]], i1 false) 6272; UNROLL-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP0]] to i8 6273; UNROLL-NEXT: [[MUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[TMP3]], i8 [[TMP4]]) 6274; UNROLL-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i8, i1 } [[MUL]], 0 6275; UNROLL-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i8, i1 } [[MUL]], 1 6276; UNROLL-NEXT: [[TMP5:%.*]] = icmp slt i8 [[MUL_RESULT]], 0 6277; UNROLL-NEXT: [[TMP6:%.*]] = icmp ugt i8 [[MUL_RESULT]], -128 6278; UNROLL-NEXT: [[TMP7:%.*]] = select i1 [[TMP2]], i1 [[TMP6]], i1 [[TMP5]] 6279; UNROLL-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]] 6280; UNROLL-NEXT: [[TMP9:%.*]] = icmp ugt i64 [[TMP0]], 255 6281; UNROLL-NEXT: [[TMP10:%.*]] = icmp ne i8 [[TMP1]], 0 6282; UNROLL-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] 6283; UNROLL-NEXT: [[TMP12:%.*]] = or i1 [[TMP8]], [[TMP11]] 6284; UNROLL-NEXT: [[TMP13:%.*]] = add i32 [[STEP]], -128 6285; UNROLL-NEXT: [[IDENT_CHECK:%.*]] = icmp ult i32 [[TMP13]], -256 6286; UNROLL-NEXT: [[TMP14:%.*]] = or i1 [[TMP12]], [[IDENT_CHECK]] 6287; UNROLL-NEXT: br i1 [[TMP14]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 6288; UNROLL: vector.ph: 6289; UNROLL-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 6290; UNROLL-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32 6291; UNROLL-NEXT: [[IND_END:%.*]] = mul i32 [[STEP]], [[DOTCAST]] 6292; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[STEP]], i64 0 6293; UNROLL-NEXT: [[TMP15:%.*]] = shl <2 x i32> [[BROADCAST_SPLATINSERT]], <i32 1, i32 0> 6294; UNROLL-NEXT: [[TMP16:%.*]] = shufflevector <2 x i32> [[TMP15]], <2 x i32> poison, <2 x i32> zeroinitializer 6295; UNROLL-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[STEP]], i64 0 6296; UNROLL-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 6297; UNROLL-NEXT: [[TMP17:%.*]] = mul nuw <2 x i32> [[DOTSPLAT]], <i32 0, i32 1> 6298; UNROLL-NEXT: br label [[VECTOR_BODY:%.*]] 6299; UNROLL: vector.body: 6300; UNROLL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 6301; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], [[VECTOR_BODY]] ] 6302; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[TMP17]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 6303; UNROLL-NEXT: [[STEP_ADD]] = add <2 x i32> [[VEC_IND]], [[TMP16]] 6304; UNROLL-NEXT: [[TMP18:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2> 6305; UNROLL-NEXT: [[TMP19:%.*]] = shufflevector <2 x i32> [[VEC_IND]], <2 x i32> [[STEP_ADD]], <2 x i32> <i32 1, i32 2> 6306; UNROLL-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 [[INDEX]] 6307; UNROLL-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP20]], i64 8 6308; UNROLL-NEXT: store <2 x i32> [[TMP18]], ptr [[TMP20]], align 4 6309; UNROLL-NEXT: store <2 x i32> [[TMP19]], ptr [[TMP21]], align 4 6310; UNROLL-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 6311; UNROLL-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], [[TMP16]] 6312; UNROLL-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 6313; UNROLL-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]] 6314; UNROLL: middle.block: 6315; UNROLL-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[STEP_ADD]], i64 1 6316; UNROLL-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 6317; UNROLL-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 6318; UNROLL: scalar.ph: 6319; UNROLL-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ] 6320; UNROLL-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ] 6321; UNROLL-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ] 6322; UNROLL-NEXT: br label [[LOOP:%.*]] 6323; UNROLL: loop: 6324; UNROLL-NEXT: [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_2_CONV:%.*]], [[LOOP]] ] 6325; UNROLL-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ] 6326; UNROLL-NEXT: [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP]] ] 6327; UNROLL-NEXT: [[IV_2_EXT:%.*]] = shl i32 [[IV_2]], 24 6328; UNROLL-NEXT: [[IV_2_CONV]] = ashr exact i32 [[IV_2_EXT]], 24 6329; UNROLL-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[IV_1]] 6330; UNROLL-NEXT: store i32 [[FOR]], ptr [[GEP]], align 4 6331; UNROLL-NEXT: [[IV_2_NEXT]] = add nsw i32 [[IV_2_CONV]], [[STEP]] 6332; UNROLL-NEXT: [[IV_1_NEXT]] = add nuw nsw i64 [[IV_1]], 1 6333; UNROLL-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_1_NEXT]], [[N]] 6334; UNROLL-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP55:![0-9]+]] 6335; UNROLL: exit: 6336; UNROLL-NEXT: ret void 6337; 6338; UNROLL-NO-IC-LABEL: @test_optimized_cast_induction_feeding_first_order_recurrence( 6339; UNROLL-NO-IC-NEXT: entry: 6340; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 6341; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 6342; UNROLL-NO-IC: vector.scevcheck: 6343; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 6344; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = trunc i32 [[STEP:%.*]] to i8 6345; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = sub i8 0, [[TMP1]] 6346; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp slt i8 [[TMP1]], 0 6347; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i8 [[TMP2]], i8 [[TMP1]] 6348; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP0]] to i8 6349; UNROLL-NO-IC-NEXT: [[MUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[TMP4]], i8 [[TMP5]]) 6350; UNROLL-NO-IC-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i8, i1 } [[MUL]], 0 6351; UNROLL-NO-IC-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i8, i1 } [[MUL]], 1 6352; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = sub i8 0, [[MUL_RESULT]] 6353; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = icmp slt i8 [[MUL_RESULT]], 0 6354; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = icmp sgt i8 [[TMP6]], 0 6355; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = select i1 [[TMP3]], i1 [[TMP8]], i1 [[TMP7]] 6356; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = or i1 [[TMP9]], [[MUL_OVERFLOW]] 6357; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = icmp ugt i64 [[TMP0]], 255 6358; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = icmp ne i8 [[TMP1]], 0 6359; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = and i1 [[TMP11]], [[TMP12]] 6360; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = or i1 [[TMP10]], [[TMP13]] 6361; UNROLL-NO-IC-NEXT: [[TMP15:%.*]] = sext i8 [[TMP1]] to i32 6362; UNROLL-NO-IC-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STEP]], [[TMP15]] 6363; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = or i1 [[TMP14]], [[IDENT_CHECK]] 6364; UNROLL-NO-IC-NEXT: br i1 [[TMP16]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 6365; UNROLL-NO-IC: vector.ph: 6366; UNROLL-NO-IC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 6367; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] 6368; UNROLL-NO-IC-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32 6369; UNROLL-NO-IC-NEXT: [[IND_END:%.*]] = mul i32 [[DOTCAST]], [[STEP]] 6370; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[STEP]], i64 0 6371; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 6372; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = mul <2 x i32> splat (i32 2), [[BROADCAST_SPLAT]] 6373; UNROLL-NO-IC-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[STEP]], i64 0 6374; UNROLL-NO-IC-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32> [[DOTSPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer 6375; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = mul <2 x i32> <i32 0, i32 1>, [[DOTSPLAT]] 6376; UNROLL-NO-IC-NEXT: [[INDUCTION:%.*]] = add <2 x i32> zeroinitializer, [[TMP18]] 6377; UNROLL-NO-IC-NEXT: br label [[VECTOR_BODY:%.*]] 6378; UNROLL-NO-IC: vector.body: 6379; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 6380; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], [[VECTOR_BODY]] ] 6381; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 6382; UNROLL-NO-IC-NEXT: [[STEP_ADD]] = add <2 x i32> [[VEC_IND]], [[TMP17]] 6383; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = add i64 [[INDEX]], 0 6384; UNROLL-NO-IC-NEXT: [[TMP20:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2> 6385; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = shufflevector <2 x i32> [[VEC_IND]], <2 x i32> [[STEP_ADD]], <2 x i32> <i32 1, i32 2> 6386; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 [[TMP19]] 6387; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[TMP22]], i32 0 6388; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP22]], i32 2 6389; UNROLL-NO-IC-NEXT: store <2 x i32> [[TMP20]], ptr [[TMP23]], align 4 6390; UNROLL-NO-IC-NEXT: store <2 x i32> [[TMP21]], ptr [[TMP24]], align 4 6391; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 6392; UNROLL-NO-IC-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], [[TMP17]] 6393; UNROLL-NO-IC-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 6394; UNROLL-NO-IC-NEXT: br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]] 6395; UNROLL-NO-IC: middle.block: 6396; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i32> [[STEP_ADD]], i32 1 6397; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 6398; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 6399; UNROLL-NO-IC: scalar.ph: 6400; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ] 6401; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ] 6402; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ] 6403; UNROLL-NO-IC-NEXT: br label [[LOOP:%.*]] 6404; UNROLL-NO-IC: loop: 6405; UNROLL-NO-IC-NEXT: [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_2_CONV:%.*]], [[LOOP]] ] 6406; UNROLL-NO-IC-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ] 6407; UNROLL-NO-IC-NEXT: [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP]] ] 6408; UNROLL-NO-IC-NEXT: [[IV_2_EXT:%.*]] = shl i32 [[IV_2]], 24 6409; UNROLL-NO-IC-NEXT: [[IV_2_CONV]] = ashr exact i32 [[IV_2_EXT]], 24 6410; UNROLL-NO-IC-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[IV_1]] 6411; UNROLL-NO-IC-NEXT: store i32 [[FOR]], ptr [[GEP]], align 4 6412; UNROLL-NO-IC-NEXT: [[IV_2_NEXT]] = add nsw i32 [[IV_2_CONV]], [[STEP]] 6413; UNROLL-NO-IC-NEXT: [[IV_1_NEXT]] = add nuw nsw i64 [[IV_1]], 1 6414; UNROLL-NO-IC-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_1_NEXT]], [[N]] 6415; UNROLL-NO-IC-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP55:![0-9]+]] 6416; UNROLL-NO-IC: exit: 6417; UNROLL-NO-IC-NEXT: ret void 6418; 6419; INTERLEAVE-LABEL: @test_optimized_cast_induction_feeding_first_order_recurrence( 6420; INTERLEAVE-NEXT: entry: 6421; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8 6422; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] 6423; INTERLEAVE: vector.scevcheck: 6424; INTERLEAVE-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 6425; INTERLEAVE-NEXT: [[TMP1:%.*]] = trunc i32 [[STEP:%.*]] to i8 6426; INTERLEAVE-NEXT: [[TMP2:%.*]] = icmp slt i8 [[TMP1]], 0 6427; INTERLEAVE-NEXT: [[TMP3:%.*]] = call i8 @llvm.abs.i8(i8 [[TMP1]], i1 false) 6428; INTERLEAVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP0]] to i8 6429; INTERLEAVE-NEXT: [[MUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[TMP3]], i8 [[TMP4]]) 6430; INTERLEAVE-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i8, i1 } [[MUL]], 0 6431; INTERLEAVE-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i8, i1 } [[MUL]], 1 6432; INTERLEAVE-NEXT: [[TMP5:%.*]] = icmp slt i8 [[MUL_RESULT]], 0 6433; INTERLEAVE-NEXT: [[TMP6:%.*]] = icmp ugt i8 [[MUL_RESULT]], -128 6434; INTERLEAVE-NEXT: [[TMP7:%.*]] = select i1 [[TMP2]], i1 [[TMP6]], i1 [[TMP5]] 6435; INTERLEAVE-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]] 6436; INTERLEAVE-NEXT: [[TMP9:%.*]] = icmp ugt i64 [[TMP0]], 255 6437; INTERLEAVE-NEXT: [[TMP10:%.*]] = icmp ne i8 [[TMP1]], 0 6438; INTERLEAVE-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] 6439; INTERLEAVE-NEXT: [[TMP12:%.*]] = or i1 [[TMP8]], [[TMP11]] 6440; INTERLEAVE-NEXT: [[TMP13:%.*]] = add i32 [[STEP]], -128 6441; INTERLEAVE-NEXT: [[IDENT_CHECK:%.*]] = icmp ult i32 [[TMP13]], -256 6442; INTERLEAVE-NEXT: [[TMP14:%.*]] = or i1 [[TMP12]], [[IDENT_CHECK]] 6443; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] 6444; INTERLEAVE: vector.ph: 6445; INTERLEAVE-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -8 6446; INTERLEAVE-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32 6447; INTERLEAVE-NEXT: [[IND_END:%.*]] = mul i32 [[STEP]], [[DOTCAST]] 6448; INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[STEP]], i64 0 6449; INTERLEAVE-NEXT: [[TMP15:%.*]] = shl <4 x i32> [[BROADCAST_SPLATINSERT]], <i32 2, i32 0, i32 0, i32 0> 6450; INTERLEAVE-NEXT: [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP15]], <4 x i32> poison, <4 x i32> zeroinitializer 6451; INTERLEAVE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[STEP]], i64 0 6452; INTERLEAVE-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer 6453; INTERLEAVE-NEXT: [[TMP17:%.*]] = mul <4 x i32> [[DOTSPLAT]], <i32 0, i32 1, i32 2, i32 3> 6454; INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]] 6455; INTERLEAVE: vector.body: 6456; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 6457; INTERLEAVE-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], [[VECTOR_BODY]] ] 6458; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[TMP17]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 6459; INTERLEAVE-NEXT: [[STEP_ADD]] = add <4 x i32> [[VEC_IND]], [[TMP16]] 6460; INTERLEAVE-NEXT: [[TMP18:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 6461; INTERLEAVE-NEXT: [[TMP19:%.*]] = shufflevector <4 x i32> [[VEC_IND]], <4 x i32> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> 6462; INTERLEAVE-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 [[INDEX]] 6463; INTERLEAVE-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP20]], i64 16 6464; INTERLEAVE-NEXT: store <4 x i32> [[TMP18]], ptr [[TMP20]], align 4 6465; INTERLEAVE-NEXT: store <4 x i32> [[TMP19]], ptr [[TMP21]], align 4 6466; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 6467; INTERLEAVE-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], [[TMP16]] 6468; INTERLEAVE-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 6469; INTERLEAVE-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]] 6470; INTERLEAVE: middle.block: 6471; INTERLEAVE-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[STEP_ADD]], i64 3 6472; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] 6473; INTERLEAVE-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 6474; INTERLEAVE: scalar.ph: 6475; INTERLEAVE-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY:%.*]] ] 6476; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ] 6477; INTERLEAVE-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[ENTRY]] ] 6478; INTERLEAVE-NEXT: br label [[LOOP:%.*]] 6479; INTERLEAVE: loop: 6480; INTERLEAVE-NEXT: [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_2_CONV:%.*]], [[LOOP]] ] 6481; INTERLEAVE-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ] 6482; INTERLEAVE-NEXT: [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP]] ] 6483; INTERLEAVE-NEXT: [[IV_2_EXT:%.*]] = shl i32 [[IV_2]], 24 6484; INTERLEAVE-NEXT: [[IV_2_CONV]] = ashr exact i32 [[IV_2_EXT]], 24 6485; INTERLEAVE-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[IV_1]] 6486; INTERLEAVE-NEXT: store i32 [[FOR]], ptr [[GEP]], align 4 6487; INTERLEAVE-NEXT: [[IV_2_NEXT]] = add nsw i32 [[IV_2_CONV]], [[STEP]] 6488; INTERLEAVE-NEXT: [[IV_1_NEXT]] = add nuw nsw i64 [[IV_1]], 1 6489; INTERLEAVE-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_1_NEXT]], [[N]] 6490; INTERLEAVE-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP55:![0-9]+]] 6491; INTERLEAVE: exit: 6492; INTERLEAVE-NEXT: ret void 6493; 6494entry: 6495 br label %loop 6496 6497loop: 6498 %for = phi i32 [ 0, %entry ], [ %iv.2.conv, %loop ] 6499 %iv.1 = phi i64 [ 0, %entry ], [ %iv.1.next, %loop ] 6500 %iv.2 = phi i32 [ 0, %entry ], [ %iv.2.next, %loop ] 6501 %iv.2.ext = shl i32 %iv.2, 24 6502 %iv.2.conv = ashr exact i32 %iv.2.ext, 24 6503 %gep = getelementptr inbounds i32, ptr %ptr, i64 %iv.1 6504 store i32 %for, ptr %gep, align 4 6505 %iv.2.next = add nsw i32 %iv.2.conv, %step 6506 %iv.1.next = add nuw nsw i64 %iv.1, 1 6507 %exitcond = icmp eq i64 %iv.1.next, %n 6508 br i1 %exitcond, label %exit, label %loop 6509 6510exit: 6511 ret void 6512} 6513