1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 2; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s 3 4target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128" 5 6; Test cases for https://github.com/llvm/llvm-project/issues/87410. 7define void @test_not_first_lane_only_constant(ptr %A, ptr noalias %B) { 8; CHECK-LABEL: define void @test_not_first_lane_only_constant( 9; CHECK-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]]) { 10; CHECK-NEXT: entry: 11; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 12; CHECK: vector.ph: 13; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 14; CHECK: vector.body: 15; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 16; CHECK-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16 17; CHECK-NEXT: [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0 18; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[TMP0]] 19; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[B]], align 2 20; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i16> poison, i16 [[TMP13]], i64 0 21; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT5]], <4 x i16> poison, <4 x i32> zeroinitializer 22; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0 23; CHECK-NEXT: store <4 x i16> [[BROADCAST_SPLAT6]], ptr [[TMP2]], align 2 24; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 25; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000 26; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 27; CHECK: middle.block: 28; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 29; CHECK: scalar.ph: 30; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 31; CHECK-NEXT: br label [[LOOP_HEADER:%.*]] 32; CHECK: loop.header: 33; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ] 34; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[IV]] 35; CHECK-NEXT: br i1 false, label [[LOOP_LATCH]], label [[ELSE_1:%.*]] 36; CHECK: else.1: 37; CHECK-NEXT: br i1 false, label [[THEN_2:%.*]], label [[ELSE_2:%.*]] 38; CHECK: then.2: 39; CHECK-NEXT: br label [[ELSE_2]] 40; CHECK: else.2: 41; CHECK-NEXT: br label [[LOOP_LATCH]] 42; CHECK: loop.latch: 43; CHECK-NEXT: [[MERGE:%.*]] = phi ptr [ [[B]], [[ELSE_2]] ], [ poison, [[LOOP_HEADER]] ] 44; CHECK-NEXT: [[L:%.*]] = load i16, ptr [[MERGE]], align 2 45; CHECK-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 46; CHECK-NEXT: store i16 [[L]], ptr [[GEP_A]], align 2 47; CHECK-NEXT: [[C_2:%.*]] = icmp eq i16 [[IV_NEXT]], 1000 48; CHECK-NEXT: br i1 [[C_2]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]] 49; CHECK: exit: 50; CHECK-NEXT: ret void 51; 52entry: 53 br label %loop.header 54 55loop.header: 56 %iv = phi i16 [ 0, %entry ], [ %iv.next, %loop.latch ] 57 %gep.A = getelementptr inbounds i16, ptr %A, i16 %iv 58 br i1 false, label %loop.latch, label %else.1 59 60else.1: 61 br i1 false, label %then.2, label %else.2 62 63then.2: 64 br label %else.2 65 66else.2: 67 br label %loop.latch 68 69loop.latch: 70 %merge = phi ptr [ %B, %else.2 ], [ poison, %loop.header ] 71 %l = load i16, ptr %merge, align 2 72 %iv.next = add i16 %iv, 1 73 store i16 %l, ptr %gep.A 74 %c.2 = icmp eq i16 %iv.next, 1000 75 br i1 %c.2, label %exit, label %loop.header 76 77exit: 78 ret void 79} 80 81define void @test_not_first_lane_only_wide_compare(ptr %A, ptr noalias %B, i16 %x, i16 %y) { 82; CHECK-LABEL: define void @test_not_first_lane_only_wide_compare( 83; CHECK-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], i16 [[X:%.*]], i16 [[Y:%.*]]) { 84; CHECK-NEXT: entry: 85; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 86; CHECK: vector.ph: 87; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 88; CHECK: vector.body: 89; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 90; CHECK-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16 91; CHECK-NEXT: [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0 92; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[TMP0]] 93; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0 94; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP2]], align 2 95; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 0 96; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i16 [[TMP3]], [[X]] 97; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP4]], ptr poison, ptr [[B]] 98; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[TMP12]], align 2 99; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i16> poison, i16 [[TMP13]], i64 0 100; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT5]], <4 x i16> poison, <4 x i32> zeroinitializer 101; CHECK-NEXT: store <4 x i16> [[BROADCAST_SPLAT6]], ptr [[TMP2]], align 2 102; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 103; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000 104; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] 105; CHECK: middle.block: 106; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 107; CHECK: scalar.ph: 108; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 109; CHECK-NEXT: br label [[LOOP_HEADER:%.*]] 110; CHECK: loop.header: 111; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ] 112; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[IV]] 113; CHECK-NEXT: [[L_0:%.*]] = load i16, ptr [[GEP_A]], align 2 114; CHECK-NEXT: [[C_0:%.*]] = icmp ult i16 [[L_0]], [[X]] 115; CHECK-NEXT: br i1 [[C_0]], label [[LOOP_LATCH]], label [[ELSE_1:%.*]] 116; CHECK: else.1: 117; CHECK-NEXT: [[C_1:%.*]] = icmp ult i16 [[L_0]], [[Y]] 118; CHECK-NEXT: br i1 [[C_1]], label [[THEN_2:%.*]], label [[ELSE_2:%.*]] 119; CHECK: then.2: 120; CHECK-NEXT: br label [[ELSE_2]] 121; CHECK: else.2: 122; CHECK-NEXT: br label [[LOOP_LATCH]] 123; CHECK: loop.latch: 124; CHECK-NEXT: [[MERGE:%.*]] = phi ptr [ [[B]], [[ELSE_2]] ], [ poison, [[LOOP_HEADER]] ] 125; CHECK-NEXT: [[L:%.*]] = load i16, ptr [[MERGE]], align 2 126; CHECK-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 127; CHECK-NEXT: store i16 [[L]], ptr [[GEP_A]], align 2 128; CHECK-NEXT: [[C_2:%.*]] = icmp eq i16 [[IV_NEXT]], 1000 129; CHECK-NEXT: br i1 [[C_2]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]] 130; CHECK: exit: 131; CHECK-NEXT: ret void 132; 133entry: 134 br label %loop.header 135 136loop.header: 137 %iv = phi i16 [ 0, %entry ], [ %iv.next, %loop.latch ] 138 %gep.A = getelementptr inbounds i16, ptr %A, i16 %iv 139 %l.0 = load i16, ptr %gep.A 140 %c.0 = icmp ult i16 %l.0, %x 141 br i1 %c.0, label %loop.latch, label %else.1 142 143else.1: 144 %c.1 = icmp ult i16 %l.0, %y 145 br i1 %c.1, label %then.2, label %else.2 146 147then.2: 148 br label %else.2 149 150else.2: 151 br label %loop.latch 152 153loop.latch: 154 %merge = phi ptr [ %B, %else.2 ], [ poison, %loop.header ] 155 %l = load i16, ptr %merge, align 2 156 %iv.next = add i16 %iv, 1 157 store i16 %l, ptr %gep.A 158 %c.2 = icmp eq i16 %iv.next, 1000 159 br i1 %c.2, label %exit, label %loop.header 160 161exit: 162 ret void 163} 164 165define void @test_not_first_lane_only_wide_compare_incoming_order_swapped(ptr %A, ptr noalias %B, i16 %x, i16 %y) { 166; CHECK-LABEL: define void @test_not_first_lane_only_wide_compare_incoming_order_swapped( 167; CHECK-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], i16 [[X:%.*]], i16 [[Y:%.*]]) { 168; CHECK-NEXT: entry: 169; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 170; CHECK: vector.ph: 171; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[X]], i64 0 172; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer 173; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 174; CHECK: vector.body: 175; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 176; CHECK-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16 177; CHECK-NEXT: [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0 178; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[TMP0]] 179; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0 180; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP2]], align 2 181; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <4 x i16> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] 182; CHECK-NEXT: [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true) 183; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0 184; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP9]], ptr [[B]], ptr poison 185; CHECK-NEXT: [[TMP12:%.*]] = load i16, ptr [[PREDPHI]], align 2 186; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i16> poison, i16 [[TMP12]], i64 0 187; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT3]], <4 x i16> poison, <4 x i32> zeroinitializer 188; CHECK-NEXT: store <4 x i16> [[BROADCAST_SPLAT4]], ptr [[TMP2]], align 2 189; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 190; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000 191; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] 192; CHECK: middle.block: 193; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] 194; CHECK: scalar.ph: 195; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 196; CHECK-NEXT: br label [[LOOP_HEADER:%.*]] 197; CHECK: loop.header: 198; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ] 199; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[IV]] 200; CHECK-NEXT: [[L_0:%.*]] = load i16, ptr [[GEP_A]], align 2 201; CHECK-NEXT: [[C_0:%.*]] = icmp ult i16 [[L_0]], [[X]] 202; CHECK-NEXT: br i1 [[C_0]], label [[LOOP_LATCH]], label [[ELSE_1:%.*]] 203; CHECK: else.1: 204; CHECK-NEXT: [[C_1:%.*]] = icmp ult i16 [[L_0]], [[Y]] 205; CHECK-NEXT: br i1 [[C_1]], label [[THEN_2:%.*]], label [[ELSE_2:%.*]] 206; CHECK: then.2: 207; CHECK-NEXT: br label [[ELSE_2]] 208; CHECK: else.2: 209; CHECK-NEXT: br label [[LOOP_LATCH]] 210; CHECK: loop.latch: 211; CHECK-NEXT: [[MERGE:%.*]] = phi ptr [ poison, [[LOOP_HEADER]] ], [ [[B]], [[ELSE_2]] ] 212; CHECK-NEXT: [[L:%.*]] = load i16, ptr [[MERGE]], align 2 213; CHECK-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1 214; CHECK-NEXT: store i16 [[L]], ptr [[GEP_A]], align 2 215; CHECK-NEXT: [[C_2:%.*]] = icmp eq i16 [[IV_NEXT]], 1000 216; CHECK-NEXT: br i1 [[C_2]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]] 217; CHECK: exit: 218; CHECK-NEXT: ret void 219; 220entry: 221 br label %loop.header 222 223loop.header: 224 %iv = phi i16 [ 0, %entry ], [ %iv.next, %loop.latch ] 225 %gep.A = getelementptr inbounds i16, ptr %A, i16 %iv 226 %l.0 = load i16, ptr %gep.A 227 %c.0 = icmp ult i16 %l.0, %x 228 br i1 %c.0, label %loop.latch, label %else.1 229 230else.1: 231 %c.1 = icmp ult i16 %l.0, %y 232 br i1 %c.1, label %then.2, label %else.2 233 234then.2: 235 br label %else.2 236 237else.2: 238 br label %loop.latch 239 240loop.latch: 241 %merge = phi ptr [ poison, %loop.header ], [ %B, %else.2 ] 242 %l = load i16, ptr %merge, align 2 243 %iv.next = add i16 %iv, 1 244 store i16 %l, ptr %gep.A 245 %c.2 = icmp eq i16 %iv.next, 1000 246 br i1 %c.2, label %exit, label %loop.header 247 248exit: 249 ret void 250} 251;. 252; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} 253; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} 254; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} 255; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]} 256; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]} 257; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]} 258; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]} 259; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]} 260;. 261