1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt -S -passes=loop-vectorize,instcombine -force-vector-width=2 -force-vector-interleave=1 -enable-interleaved-mem-accesses < %s | FileCheck %s 3; RUN: opt -S -passes=loop-vectorize,instcombine -force-vector-width=2 -force-vector-interleave=1 -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses < %s | FileCheck %s 4 5target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" 6%pair = type { i64, i64 } 7 8; Ensure that we vectorize the interleaved load group even though the loop 9; contains a conditional store. The store group contains gaps and is not 10; vectorized. 11; 12; 13; 14; 15; 16 17define void @interleaved_with_cond_store_0(ptr %p, i64 %x, i64 %n) { 18; CHECK-LABEL: @interleaved_with_cond_store_0( 19; CHECK-NEXT: entry: 20; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 3 21; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 22; CHECK: vector.ph: 23; CHECK-NEXT: [[DOTNEG:%.*]] = or i64 [[N]], -2 24; CHECK-NEXT: [[N_VEC:%.*]] = add nsw i64 [[DOTNEG]], [[N]] 25; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[X:%.*]], i64 0 26; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer 27; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 28; CHECK: vector.body: 29; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ] 30; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 1 31; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8 32; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2> 33; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <2 x i64> [[STRIDED_VEC]], [[BROADCAST_SPLAT]] 34; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0 35; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 36; CHECK: pred.store.if: 37; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[INDEX]], i32 1 38; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 0 39; CHECK-NEXT: store i64 [[TMP4]], ptr [[TMP3]], align 8 40; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] 41; CHECK: pred.store.continue: 42; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1 43; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]] 44; CHECK: pred.store.if1: 45; CHECK-NEXT: [[TMP6:%.*]] = or disjoint i64 [[INDEX]], 1 46; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP6]], i32 1 47; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 2 48; CHECK-NEXT: store i64 [[TMP8]], ptr [[TMP7]], align 8 49; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]] 50; CHECK: pred.store.continue2: 51; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 52; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 53; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 54; CHECK: middle.block: 55; CHECK-NEXT: br label [[SCALAR_PH]] 56; CHECK: scalar.ph: 57; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 58; CHECK-NEXT: br label [[FOR_BODY:%.*]] 59; CHECK: for.body: 60; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[IF_MERGE:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 61; CHECK-NEXT: [[P_1:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 1 62; CHECK-NEXT: [[TMP10:%.*]] = load i64, ptr [[P_1]], align 8 63; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], [[X]] 64; CHECK-NEXT: br i1 [[TMP11]], label [[IF_THEN:%.*]], label [[IF_MERGE]] 65; CHECK: if.then: 66; CHECK-NEXT: store i64 [[TMP10]], ptr [[P_1]], align 8 67; CHECK-NEXT: br label [[IF_MERGE]] 68; CHECK: if.merge: 69; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 70; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 71; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP3:![0-9]+]] 72; CHECK: for.end: 73; CHECK-NEXT: ret void 74; 75entry: 76 br label %for.body 77 78for.body: 79 %i = phi i64 [ %i.next, %if.merge ], [ 0, %entry ] 80 %p.1 = getelementptr inbounds %pair, ptr %p, i64 %i, i32 1 81 %0 = load i64, ptr %p.1, align 8 82 %1 = icmp eq i64 %0, %x 83 br i1 %1, label %if.then, label %if.merge 84 85if.then: 86 store i64 %0, ptr %p.1, align 8 87 br label %if.merge 88 89if.merge: 90 %i.next = add nuw nsw i64 %i, 1 91 %cond = icmp slt i64 %i.next, %n 92 br i1 %cond, label %for.body, label %for.end 93 94for.end: 95 ret void 96} 97 98; Ensure that we don't form a single interleaved group for the two loads. The 99; conditional store prevents the second load from being hoisted. The two load 100; groups are separately vectorized. The store group contains gaps and is not 101; vectorized. 102; 103; 104; 105; 106; 107; 108 109define void @interleaved_with_cond_store_1(ptr %p, i64 %x, i64 %n) { 110; CHECK-LABEL: @interleaved_with_cond_store_1( 111; CHECK-NEXT: entry: 112; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 3 113; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 114; CHECK: vector.ph: 115; CHECK-NEXT: [[DOTNEG:%.*]] = or i64 [[N]], -2 116; CHECK-NEXT: [[N_VEC:%.*]] = add nsw i64 [[DOTNEG]], [[N]] 117; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[X:%.*]], i64 0 118; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer 119; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 120; CHECK: vector.body: 121; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ] 122; CHECK-NEXT: [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 1 123; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 0 124; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[INDEX]], i32 1 125; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP0]], i32 1 126; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8 127; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2> 128; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[STRIDED_VEC]], [[BROADCAST_SPLAT]] 129; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0 130; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 131; CHECK: pred.store.if: 132; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[INDEX]], i32 0 133; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 0 134; CHECK-NEXT: store i64 [[TMP7]], ptr [[TMP6]], align 8 135; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] 136; CHECK: pred.store.continue: 137; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1 138; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]] 139; CHECK: pred.store.if1: 140; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP0]], i32 0 141; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 2 142; CHECK-NEXT: store i64 [[TMP10]], ptr [[TMP9]], align 8 143; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]] 144; CHECK: pred.store.continue2: 145; CHECK-NEXT: [[WIDE_VEC3:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8 146; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[WIDE_VEC3]], i64 0 147; CHECK-NEXT: store i64 [[TMP11]], ptr [[TMP2]], align 8 148; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[WIDE_VEC3]], i64 2 149; CHECK-NEXT: store i64 [[TMP12]], ptr [[TMP3]], align 8 150; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 151; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 152; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] 153; CHECK: middle.block: 154; CHECK-NEXT: br label [[SCALAR_PH]] 155; CHECK: scalar.ph: 156; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 157; CHECK-NEXT: br label [[FOR_BODY:%.*]] 158; CHECK: for.body: 159; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[IF_MERGE:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 160; CHECK-NEXT: [[P_0:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 0 161; CHECK-NEXT: [[P_1:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 1 162; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr [[P_1]], align 8 163; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], [[X]] 164; CHECK-NEXT: br i1 [[TMP15]], label [[IF_THEN:%.*]], label [[IF_MERGE]] 165; CHECK: if.then: 166; CHECK-NEXT: store i64 [[TMP14]], ptr [[P_0]], align 8 167; CHECK-NEXT: br label [[IF_MERGE]] 168; CHECK: if.merge: 169; CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr [[P_0]], align 8 170; CHECK-NEXT: store i64 [[TMP16]], ptr [[P_1]], align 8 171; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 172; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 173; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP5:![0-9]+]] 174; CHECK: for.end: 175; CHECK-NEXT: ret void 176; 177entry: 178 br label %for.body 179 180for.body: 181 %i = phi i64 [ %i.next, %if.merge ], [ 0, %entry ] 182 %p.0 = getelementptr inbounds %pair, ptr %p, i64 %i, i32 0 183 %p.1 = getelementptr inbounds %pair, ptr %p, i64 %i, i32 1 184 %0 = load i64, ptr %p.1, align 8 185 %1 = icmp eq i64 %0, %x 186 br i1 %1, label %if.then, label %if.merge 187 188if.then: 189 store i64 %0, ptr %p.0, align 8 190 br label %if.merge 191 192if.merge: 193 %2 = load i64, ptr %p.0, align 8 194 store i64 %2, ptr %p.1, align 8 195 %i.next = add nuw nsw i64 %i, 1 196 %cond = icmp slt i64 %i.next, %n 197 br i1 %cond, label %for.body, label %for.end 198 199for.end: 200 ret void 201} 202 203; Ensure that we don't create a single interleaved group for the two stores. 204; The second store is conditional and we can't sink the first store inside the 205; predicated block. The load group is vectorized, and the store groups contain 206; gaps and are not vectorized. 207; 208; 209; 210; 211; 212 213define void @interleaved_with_cond_store_2(ptr %p, i64 %x, i64 %n) { 214; CHECK-LABEL: @interleaved_with_cond_store_2( 215; CHECK-NEXT: entry: 216; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i64 [[N:%.*]], 3 217; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 218; CHECK: vector.ph: 219; CHECK-NEXT: [[DOTNEG:%.*]] = or i64 [[N]], -2 220; CHECK-NEXT: [[N_VEC:%.*]] = add nsw i64 [[DOTNEG]], [[N]] 221; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[X:%.*]], i64 0 222; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer 223; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 224; CHECK: vector.body: 225; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ] 226; CHECK-NEXT: [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 1 227; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[PAIR:%.*]], ptr [[P:%.*]], i64 [[INDEX]], i32 0 228; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP0]], i32 0 229; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[INDEX]], i32 1 230; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8 231; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2> 232; CHECK-NEXT: store i64 [[X]], ptr [[TMP1]], align 8 233; CHECK-NEXT: store i64 [[X]], ptr [[TMP2]], align 8 234; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[STRIDED_VEC]], [[BROADCAST_SPLAT]] 235; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0 236; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]] 237; CHECK: pred.store.if: 238; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[INDEX]], i32 1 239; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 0 240; CHECK-NEXT: store i64 [[TMP7]], ptr [[TMP6]], align 8 241; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]] 242; CHECK: pred.store.continue: 243; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1 244; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]] 245; CHECK: pred.store.if1: 246; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[TMP0]], i32 1 247; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[WIDE_VEC]], i64 2 248; CHECK-NEXT: store i64 [[TMP10]], ptr [[TMP9]], align 8 249; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]] 250; CHECK: pred.store.continue2: 251; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 252; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 253; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] 254; CHECK: middle.block: 255; CHECK-NEXT: br label [[SCALAR_PH]] 256; CHECK: scalar.ph: 257; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 258; CHECK-NEXT: br label [[FOR_BODY:%.*]] 259; CHECK: for.body: 260; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[IF_MERGE:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] 261; CHECK-NEXT: [[P_0:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 0 262; CHECK-NEXT: [[P_1:%.*]] = getelementptr inbounds [[PAIR]], ptr [[P]], i64 [[I]], i32 1 263; CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr [[P_1]], align 8 264; CHECK-NEXT: store i64 [[X]], ptr [[P_0]], align 8 265; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[TMP12]], [[X]] 266; CHECK-NEXT: br i1 [[TMP13]], label [[IF_THEN:%.*]], label [[IF_MERGE]] 267; CHECK: if.then: 268; CHECK-NEXT: store i64 [[TMP12]], ptr [[P_1]], align 8 269; CHECK-NEXT: br label [[IF_MERGE]] 270; CHECK: if.merge: 271; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 272; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]] 273; CHECK-NEXT: br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP7:![0-9]+]] 274; CHECK: for.end: 275; CHECK-NEXT: ret void 276; 277entry: 278 br label %for.body 279 280for.body: 281 %i = phi i64 [ %i.next, %if.merge ], [ 0, %entry ] 282 %p.0 = getelementptr inbounds %pair, ptr %p, i64 %i, i32 0 283 %p.1 = getelementptr inbounds %pair, ptr %p, i64 %i, i32 1 284 %0 = load i64, ptr %p.1, align 8 285 store i64 %x, ptr %p.0, align 8 286 %1 = icmp eq i64 %0, %x 287 br i1 %1, label %if.then, label %if.merge 288 289if.then: 290 store i64 %0, ptr %p.1, align 8 291 br label %if.merge 292 293if.merge: 294 %i.next = add nuw nsw i64 %i, 1 295 %cond = icmp slt i64 %i.next, %n 296 br i1 %cond, label %for.body, label %for.end 297 298for.end: 299 ret void 300} 301