1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 2; RUN: opt -S -passes=loop-vectorize -mcpu=skylake-avx512 -mtriple=x86_64-apple-macosx -S %s | FileCheck %s 3 4target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" 5 6; Test cases based on https://github.com/llvm/llvm-project/issues/91883. 7define void @iv.4_used_as_vector_and_first_lane(ptr %src, ptr noalias %dst) { 8; CHECK-LABEL: define void @iv.4_used_as_vector_and_first_lane( 9; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0:[0-9]+]] { 10; CHECK-NEXT: entry: 11; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 12; CHECK: vector.ph: 13; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 14; CHECK: vector.body: 15; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 16; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] 17; CHECK-NEXT: [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4) 18; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i64> [[STEP_ADD]], splat (i64 4) 19; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add <4 x i64> [[STEP_ADD_2]], splat (i64 4) 20; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 21; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP0]] 22; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0 23; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 4 24; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 8 25; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 12 26; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8 27; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8 28; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i64>, ptr [[TMP10]], align 8 29; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i64>, ptr [[TMP11]], align 8 30; CHECK-NEXT: [[TMP12:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4) 31; CHECK-NEXT: [[TMP13:%.*]] = add <4 x i64> [[STEP_ADD]], splat (i64 4) 32; CHECK-NEXT: [[TMP14:%.*]] = add <4 x i64> [[STEP_ADD_2]], splat (i64 4) 33; CHECK-NEXT: [[TMP15:%.*]] = add <4 x i64> [[STEP_ADD_3]], splat (i64 4) 34; CHECK-NEXT: [[TMP16:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD]], splat (i64 128) 35; CHECK-NEXT: [[TMP17:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD4]], splat (i64 128) 36; CHECK-NEXT: [[TMP18:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD5]], splat (i64 128) 37; CHECK-NEXT: [[TMP19:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD6]], splat (i64 128) 38; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i64> [[TMP12]], i32 0 39; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], 1 40; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP27]] 41; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i64, ptr [[TMP28]], i32 0 42; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i64, ptr [[TMP28]], i32 4 43; CHECK-NEXT: [[TMP34:%.*]] = getelementptr i64, ptr [[TMP28]], i32 8 44; CHECK-NEXT: [[TMP35:%.*]] = getelementptr i64, ptr [[TMP28]], i32 12 45; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP12]], ptr [[TMP32]], i32 4, <4 x i1> [[TMP16]]) 46; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP13]], ptr [[TMP33]], i32 4, <4 x i1> [[TMP17]]) 47; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP14]], ptr [[TMP34]], i32 4, <4 x i1> [[TMP18]]) 48; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP15]], ptr [[TMP35]], i32 4, <4 x i1> [[TMP19]]) 49; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 50; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD_3]], splat (i64 4) 51; CHECK-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32 52; CHECK-NEXT: br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 53; CHECK: middle.block: 54; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 55; CHECK: scalar.ph: 56; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 32, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 57; CHECK-NEXT: br label [[LOOP_HEADER:%.*]] 58; CHECK: loop.header: 59; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ] 60; CHECK-NEXT: [[G_SRC:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]] 61; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[G_SRC]], align 8 62; CHECK-NEXT: [[IV_4:%.*]] = add nuw nsw i64 [[IV]], 4 63; CHECK-NEXT: [[C:%.*]] = icmp ule i64 [[L]], 128 64; CHECK-NEXT: br i1 [[C]], label [[LOOP_THEN:%.*]], label [[LOOP_LATCH]] 65; CHECK: loop.then: 66; CHECK-NEXT: [[OR:%.*]] = or disjoint i64 [[IV_4]], 1 67; CHECK-NEXT: [[G_DST:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[OR]] 68; CHECK-NEXT: store i64 [[IV_4]], ptr [[G_DST]], align 4 69; CHECK-NEXT: br label [[LOOP_LATCH]] 70; CHECK: loop.latch: 71; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 72; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 32 73; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]] 74; CHECK: exit: 75; CHECK-NEXT: ret void 76; 77entry: 78 br label %loop.header 79 80loop.header: 81 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] 82 %g.src = getelementptr inbounds i64, ptr %src, i64 %iv 83 %l = load i64, ptr %g.src 84 %iv.4 = add nuw nsw i64 %iv, 4 85 %c = icmp ule i64 %l, 128 86 br i1 %c, label %loop.then, label %loop.latch 87 88loop.then: 89 %or = or disjoint i64 %iv.4, 1 90 %g.dst = getelementptr inbounds i64, ptr %dst, i64 %or 91 store i64 %iv.4, ptr %g.dst, align 4 92 br label %loop.latch 93 94loop.latch: 95 %iv.next = add nuw nsw i64 %iv, 1 96 %exitcond = icmp eq i64 %iv.next, 32 97 br i1 %exitcond, label %exit, label %loop.header 98 99exit: 100 ret void 101} 102 103define void @iv.4_used_as_first_lane(ptr %src, ptr noalias %dst) { 104; CHECK-LABEL: define void @iv.4_used_as_first_lane( 105; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] { 106; CHECK-NEXT: entry: 107; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 108; CHECK: vector.ph: 109; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 110; CHECK: vector.body: 111; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 112; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 113; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP0]] 114; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0 115; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 4 116; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 8 117; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 12 118; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8 119; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8 120; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP10]], align 8 121; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i64>, ptr [[TMP11]], align 8 122; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[TMP0]], 4 123; CHECK-NEXT: [[TMP16:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD]], splat (i64 128) 124; CHECK-NEXT: [[TMP17:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD1]], splat (i64 128) 125; CHECK-NEXT: [[TMP18:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD2]], splat (i64 128) 126; CHECK-NEXT: [[TMP19:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD3]], splat (i64 128) 127; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP15]], 1 128; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP23]] 129; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i64, ptr [[TMP24]], i32 0 130; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i64, ptr [[TMP24]], i32 4 131; CHECK-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[TMP24]], i32 8 132; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i64, ptr [[TMP24]], i32 12 133; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_LOAD]], ptr [[TMP28]], i32 4, <4 x i1> [[TMP16]]) 134; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_LOAD1]], ptr [[TMP29]], i32 4, <4 x i1> [[TMP17]]) 135; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_LOAD2]], ptr [[TMP30]], i32 4, <4 x i1> [[TMP18]]) 136; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_LOAD3]], ptr [[TMP31]], i32 4, <4 x i1> [[TMP19]]) 137; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 138; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32 139; CHECK-NEXT: br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] 140; CHECK: middle.block: 141; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 142; CHECK: scalar.ph: 143; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 32, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 144; CHECK-NEXT: br label [[LOOP_HEADER:%.*]] 145; CHECK: loop.header: 146; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ] 147; CHECK-NEXT: [[G_SRC:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]] 148; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[G_SRC]], align 8 149; CHECK-NEXT: [[IV_4:%.*]] = add nuw nsw i64 [[IV]], 4 150; CHECK-NEXT: [[C:%.*]] = icmp ule i64 [[L]], 128 151; CHECK-NEXT: br i1 [[C]], label [[LOOP_THEN:%.*]], label [[LOOP_LATCH]] 152; CHECK: loop.then: 153; CHECK-NEXT: [[OR:%.*]] = or disjoint i64 [[IV_4]], 1 154; CHECK-NEXT: [[G_DST:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[OR]] 155; CHECK-NEXT: store i64 [[L]], ptr [[G_DST]], align 4 156; CHECK-NEXT: br label [[LOOP_LATCH]] 157; CHECK: loop.latch: 158; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 159; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 32 160; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]] 161; CHECK: exit: 162; CHECK-NEXT: ret void 163; 164entry: 165 br label %loop.header 166 167loop.header: 168 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] 169 %g.src = getelementptr inbounds i64, ptr %src, i64 %iv 170 %l = load i64, ptr %g.src 171 %iv.4 = add nuw nsw i64 %iv, 4 172 %c = icmp ule i64 %l, 128 173 br i1 %c, label %loop.then, label %loop.latch 174 175loop.then: 176 %or = or disjoint i64 %iv.4, 1 177 %g.dst = getelementptr inbounds i64, ptr %dst, i64 %or 178 store i64 %l, ptr %g.dst, align 4 179 br label %loop.latch 180 181loop.latch: 182 %iv.next = add nuw nsw i64 %iv, 1 183 %exitcond = icmp eq i64 %iv.next, 32 184 br i1 %exitcond, label %exit, label %loop.header 185 186exit: 187 ret void 188} 189;. 190; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} 191; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} 192; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} 193; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]} 194; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]} 195; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]} 196;. 197