1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt -passes=loop-vectorize -mcpu=skylake-avx512 -S %s | FileCheck %s 3 4target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128-ni:1-p2:32:8:8:32-ni:2" 5target triple = "x86_64-unknown-linux-gnu" 6 7; Make sure selected VF for the epilog loop doesn't exceed remaining TC. 8define void @test_tc_17_no_epilogue_vectorization(ptr noalias %src, ptr noalias %dst) { 9; CHECK-LABEL: @test_tc_17_no_epilogue_vectorization( 10; CHECK-NEXT: entry: 11; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 12; CHECK: vector.ph: 13; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 14; CHECK: vector.body: 15; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 16; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 17; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP0]] 18; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0 19; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 64 20; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP0]] 21; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0 22; CHECK-NEXT: store <16 x i8> [[WIDE_LOAD]], ptr [[TMP4]], align 64 23; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 24; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 25; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 26; CHECK: middle.block: 27; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]] 28; CHECK: scalar.ph: 29; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 30; CHECK-NEXT: br label [[LOOP:%.*]] 31; CHECK: loop: 32; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[LOOP]] ] 33; CHECK-NEXT: [[LDADDR:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[I]] 34; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[LDADDR]], align 64 35; CHECK-NEXT: [[STADDR:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[I]] 36; CHECK-NEXT: store i8 [[VAL]], ptr [[STADDR]], align 64 37; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1 38; CHECK-NEXT: [[IS_NEXT:%.*]] = icmp ult i64 [[I_NEXT]], 17 39; CHECK-NEXT: br i1 [[IS_NEXT]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP3:![0-9]+]] 40; CHECK: exit: 41; CHECK-NEXT: ret void 42; 43entry: 44 br label %loop 45 46loop: 47 %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] 48 %ldaddr = getelementptr inbounds i8, ptr %src, i64 %i 49 %val = load i8, ptr %ldaddr, align 64 50 %staddr = getelementptr inbounds i8, ptr %dst, i64 %i 51 store i8 %val, ptr %staddr, align 64 52 %i.next = add i64 %i, 1 53 %is.next = icmp ult i64 %i.next, 17 54 br i1 %is.next, label %loop, label %exit 55 56exit: 57 ret void 58} 59 60define void @test_tc_18(ptr noalias %src, ptr noalias %dst) { 61; CHECK-LABEL: @test_tc_18( 62; CHECK-NEXT: iter.check: 63; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]] 64; CHECK: vector.main.loop.iter.check: 65; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]] 66; CHECK: vector.ph: 67; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 68; CHECK: vector.body: 69; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 70; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 71; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP0]] 72; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0 73; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 64 74; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP0]] 75; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0 76; CHECK-NEXT: store <16 x i8> [[WIDE_LOAD]], ptr [[TMP4]], align 64 77; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 78; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 79; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] 80; CHECK: middle.block: 81; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]] 82; CHECK: vec.epilog.iter.check: 83; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]] 84; CHECK: vec.epilog.ph: 85; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ] 86; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]] 87; CHECK: vec.epilog.vector.body: 88; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] 89; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX1]], 0 90; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[TMP6]] 91; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0 92; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i8>, ptr [[TMP8]], align 64 93; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP6]] 94; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i32 0 95; CHECK-NEXT: store <2 x i8> [[WIDE_LOAD2]], ptr [[TMP10]], align 64 96; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 2 97; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 18 98; CHECK-NEXT: br i1 [[TMP11]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] 99; CHECK: vec.epilog.middle.block: 100; CHECK-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]] 101; CHECK: vec.epilog.scalar.ph: 102; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 18, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ] 103; CHECK-NEXT: br label [[LOOP:%.*]] 104; CHECK: loop: 105; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[LOOP]] ] 106; CHECK-NEXT: [[LDADDR:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[I]] 107; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[LDADDR]], align 64 108; CHECK-NEXT: [[STADDR:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[I]] 109; CHECK-NEXT: store i8 [[VAL]], ptr [[STADDR]], align 64 110; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1 111; CHECK-NEXT: [[IS_NEXT:%.*]] = icmp ult i64 [[I_NEXT]], 18 112; CHECK-NEXT: br i1 [[IS_NEXT]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP6:![0-9]+]] 113; CHECK: exit: 114; CHECK-NEXT: ret void 115; 116entry: 117 br label %loop 118 119loop: 120 %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] 121 %ldaddr = getelementptr inbounds i8, ptr %src, i64 %i 122 %val = load i8, ptr %ldaddr, align 64 123 %staddr = getelementptr inbounds i8, ptr %dst, i64 %i 124 store i8 %val, ptr %staddr, align 64 125 %i.next = add i64 %i, 1 126 %is.next = icmp ult i64 %i.next, 18 127 br i1 %is.next, label %loop, label %exit 128 129exit: 130 ret void 131} 132 133define void @test_tc_19(ptr noalias %src, ptr noalias %dst) { 134; CHECK-LABEL: @test_tc_19( 135; CHECK-NEXT: iter.check: 136; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]] 137; CHECK: vector.main.loop.iter.check: 138; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]] 139; CHECK: vector.ph: 140; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 141; CHECK: vector.body: 142; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 143; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 144; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP0]] 145; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0 146; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 64 147; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP0]] 148; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0 149; CHECK-NEXT: store <16 x i8> [[WIDE_LOAD]], ptr [[TMP4]], align 64 150; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 151; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 152; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] 153; CHECK: middle.block: 154; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]] 155; CHECK: vec.epilog.iter.check: 156; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]] 157; CHECK: vec.epilog.ph: 158; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ] 159; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]] 160; CHECK: vec.epilog.vector.body: 161; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] 162; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX1]], 0 163; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[TMP6]] 164; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0 165; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i8>, ptr [[TMP8]], align 64 166; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP6]] 167; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i32 0 168; CHECK-NEXT: store <2 x i8> [[WIDE_LOAD2]], ptr [[TMP10]], align 64 169; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 2 170; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 18 171; CHECK-NEXT: br i1 [[TMP11]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] 172; CHECK: vec.epilog.middle.block: 173; CHECK-NEXT: br i1 false, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]] 174; CHECK: vec.epilog.scalar.ph: 175; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 18, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ] 176; CHECK-NEXT: br label [[LOOP:%.*]] 177; CHECK: loop: 178; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[LOOP]] ] 179; CHECK-NEXT: [[LDADDR:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[I]] 180; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[LDADDR]], align 64 181; CHECK-NEXT: [[STADDR:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[I]] 182; CHECK-NEXT: store i8 [[VAL]], ptr [[STADDR]], align 64 183; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1 184; CHECK-NEXT: [[IS_NEXT:%.*]] = icmp ult i64 [[I_NEXT]], 19 185; CHECK-NEXT: br i1 [[IS_NEXT]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP9:![0-9]+]] 186; CHECK: exit: 187; CHECK-NEXT: ret void 188; 189entry: 190 br label %loop 191 192loop: 193 %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] 194 %ldaddr = getelementptr inbounds i8, ptr %src, i64 %i 195 %val = load i8, ptr %ldaddr, align 64 196 %staddr = getelementptr inbounds i8, ptr %dst, i64 %i 197 store i8 %val, ptr %staddr, align 64 198 %i.next = add i64 %i, 1 199 %is.next = icmp ult i64 %i.next, 19 200 br i1 %is.next, label %loop, label %exit 201 202exit: 203 ret void 204} 205 206define void @test_tc_20(ptr noalias %src, ptr noalias %dst) { 207; CHECK-LABEL: @test_tc_20( 208; CHECK-NEXT: iter.check: 209; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]] 210; CHECK: vector.main.loop.iter.check: 211; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]] 212; CHECK: vector.ph: 213; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 214; CHECK: vector.body: 215; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 216; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 217; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP0]] 218; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0 219; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 4 220; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 8 221; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 12 222; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 64 223; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 64 224; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 64 225; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP5]], align 64 226; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP0]] 227; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 0 228; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 4 229; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 8 230; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 12 231; CHECK-NEXT: store <4 x i8> [[WIDE_LOAD]], ptr [[TMP7]], align 64 232; CHECK-NEXT: store <4 x i8> [[WIDE_LOAD1]], ptr [[TMP8]], align 64 233; CHECK-NEXT: store <4 x i8> [[WIDE_LOAD2]], ptr [[TMP9]], align 64 234; CHECK-NEXT: store <4 x i8> [[WIDE_LOAD3]], ptr [[TMP10]], align 64 235; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 236; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 237; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] 238; CHECK: middle.block: 239; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]] 240; CHECK: vec.epilog.iter.check: 241; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]] 242; CHECK: vec.epilog.ph: 243; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ] 244; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]] 245; CHECK: vec.epilog.vector.body: 246; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] 247; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX4]], 0 248; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[TMP12]] 249; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i32 0 250; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP14]], align 64 251; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP12]] 252; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP15]], i32 0 253; CHECK-NEXT: store <4 x i8> [[WIDE_LOAD5]], ptr [[TMP16]], align 64 254; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 4 255; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT6]], 20 256; CHECK-NEXT: br i1 [[TMP17]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] 257; CHECK: vec.epilog.middle.block: 258; CHECK-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]] 259; CHECK: vec.epilog.scalar.ph: 260; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 20, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[ITER_CHECK:%.*]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ] 261; CHECK-NEXT: br label [[LOOP:%.*]] 262; CHECK: loop: 263; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[LOOP]] ] 264; CHECK-NEXT: [[LDADDR:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[I]] 265; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[LDADDR]], align 64 266; CHECK-NEXT: [[STADDR:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[I]] 267; CHECK-NEXT: store i8 [[VAL]], ptr [[STADDR]], align 64 268; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1 269; CHECK-NEXT: [[IS_NEXT:%.*]] = icmp ult i64 [[I_NEXT]], 20 270; CHECK-NEXT: br i1 [[IS_NEXT]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP12:![0-9]+]] 271; CHECK: exit: 272; CHECK-NEXT: ret void 273; 274entry: 275 br label %loop 276 277loop: 278 %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] 279 %ldaddr = getelementptr inbounds i8, ptr %src, i64 %i 280 %val = load i8, ptr %ldaddr, align 64 281 %staddr = getelementptr inbounds i8, ptr %dst, i64 %i 282 store i8 %val, ptr %staddr, align 64 283 %i.next = add i64 %i, 1 284 %is.next = icmp ult i64 %i.next, 20 285 br i1 %is.next, label %loop, label %exit 286 287exit: 288 ret void 289} 290 291define void @limit_main_loop_vf_to_avoid_dead_main_vector_loop(ptr noalias %src, ptr noalias %dst) { 292; CHECK-LABEL: @limit_main_loop_vf_to_avoid_dead_main_vector_loop( 293; CHECK-NEXT: entry: 294; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 295; CHECK: vector.ph: 296; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 297; CHECK: vector.body: 298; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 299; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 300; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [3 x i8], ptr [[SRC:%.*]], i64 [[TMP0]], i64 0 301; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <24 x i8>, ptr [[TMP1]], align 1 302; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <24 x i8> [[WIDE_VEC]], <24 x i8> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21> 303; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP0]] 304; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0 305; CHECK-NEXT: store <8 x i8> [[STRIDED_VEC]], ptr [[TMP4]], align 1 306; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 307; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24 308; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] 309; CHECK: middle.block: 310; CHECK-NEXT: br label [[SCALAR_PH]] 311; CHECK: scalar.ph: 312; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 24, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 313; CHECK-NEXT: br label [[LOOP:%.*]] 314; CHECK: loop: 315; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] 316; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds [3 x i8], ptr [[SRC]], i64 [[IV]], i64 0 317; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP_SRC]], align 1 318; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]] 319; CHECK-NEXT: store i8 [[L]], ptr [[GEP_DST]], align 1 320; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 321; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 32 322; CHECK-NEXT: br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP14:![0-9]+]] 323; CHECK: exit: 324; CHECK-NEXT: ret void 325; 326entry: 327 br label %loop 328 329loop: 330 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] 331 %gep.src = getelementptr inbounds [3 x i8], ptr %src, i64 %iv, i64 0 332 %l = load i8, ptr %gep.src, align 1 333 %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv 334 store i8 %l, ptr %gep.dst, align 1 335 %iv.next = add nuw nsw i64 %iv, 1 336 %cmp = icmp eq i64 %iv.next, 32 337 br i1 %cmp, label %exit, label %loop 338 339exit: 340 ret void 341} 342