1; RUN: opt -passes=loop-vectorize -force-vector-width=4 -enable-vplan-native-path -S %s | FileCheck %s 2 3; Test that VPlan native path is able to widen select instruction in the 4; innermost loop under different conditions when outer loop is marked to be 5; vectorized. These conditions include following: 6; * Inner and outer loop invariant select condition 7; * Select condition depending on outer loop iteration variable. 8; * Select condidition depending on inner loop iteration variable. 9; * Select conditition depending on both outer and inner loop iteration 10; variables. 11 12define void @loop_invariant_select(ptr noalias nocapture %out, i1 %select, double %a, double %b) { 13; CHECK-LABEL: @loop_invariant_select( 14; CHECK-NEXT: entry: 15; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 16; CHECK: vector.ph: 17; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A:%.*]], i64 0 18; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer 19; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0 20; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer 21; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 22; CHECK: vector.body: 23; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR1_LATCH4:%.*]] ] 24; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR1_LATCH4]] ] 25; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT:%.*]], <4 x i64> [[VEC_IND]] 26; CHECK-NEXT: br label [[FOR2_HEADER1:%.*]] 27; CHECK: for2.header1: 28; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP2:%.*]], [[FOR2_HEADER1]] ] 29; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[SELECT:%.*]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]] 30; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP1]], <4 x ptr> [[TMP0]], i32 8, <4 x i1> splat (i1 true)) 31entry: 32 br label %for1.header 33 34for1.header: 35 %indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ] 36 %ptr = getelementptr inbounds double, ptr %out, i64 %indvar1 37 br label %for2.header 38 39for2.header: 40 %indvar2 = phi i64 [ 0, %for1.header ], [ %indvar21, %for2.header ] 41 ; Select condition is loop invariant for both inner and outer loop. 42 %select.b = select i1 %select, double %a, double %b 43 store double %select.b, ptr %ptr, align 8 44 %indvar21 = add nuw nsw i64 %indvar2, 1 45 %for2.cond = icmp eq i64 %indvar21, 10000 46 br i1 %for2.cond, label %for1.latch, label %for2.header 47 48for1.latch: 49 %indvar11 = add nuw nsw i64 %indvar1, 1 50 %for1.cond = icmp eq i64 %indvar11, 1000 51 br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0 52 53exit: 54 ret void 55} 56 57define void @outer_loop_dependant_select(ptr noalias nocapture %out, double %a, double %b) { 58; CHECK-LABEL: @outer_loop_dependant_select( 59; CHECK-NEXT: entry: 60; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 61; CHECK: vector.ph: 62; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A:%.*]], i64 0 63; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer 64; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0 65; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer 66; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 67; CHECK: vector.body: 68; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR1_LATCH4:%.*]] ] 69; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR1_LATCH4]] ] 70; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT:%.*]], <4 x i64> [[VEC_IND]] 71; CHECK-NEXT: br label [[FOR2_HEADER1:%.*]] 72; CHECK: for2.header1: 73; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP3:%.*]], [[FOR2_HEADER1]] ] 74; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[VEC_IND]] to <4 x i1> 75; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]] 76; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP2]], <4 x ptr> [[TMP0]], i32 8, <4 x i1> splat (i1 true)) 77entry: 78 br label %for1.header 79 80for1.header: 81 %indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ] 82 %ptr = getelementptr inbounds double, ptr %out, i64 %indvar1 83 br label %for2.header 84 85for2.header: 86 %indvar2 = phi i64 [ 0, %for1.header ], [ %indvar21, %for2.header ] 87 %select = trunc i64 %indvar1 to i1 88 ; Select condition only depends on outer loop iteration variable. 89 %select.b = select i1 %select, double %a, double %b 90 store double %select.b, ptr %ptr, align 8 91 %indvar21 = add nuw nsw i64 %indvar2, 1 92 %for2.cond = icmp eq i64 %indvar21, 10000 93 br i1 %for2.cond, label %for1.latch, label %for2.header 94 95for1.latch: 96 %indvar11 = add nuw nsw i64 %indvar1, 1 97 %for1.cond = icmp eq i64 %indvar11, 1000 98 br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0 99 100exit: 101 ret void 102} 103 104define void @inner_loop_dependant_select(ptr noalias nocapture %out, double %a, double %b) { 105; CHECK-LABEL: @inner_loop_dependant_select( 106; CHECK-NEXT: entry: 107; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 108; CHECK: vector.ph: 109; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A:%.*]], i64 0 110; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer 111; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0 112; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer 113; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 114; CHECK: vector.body: 115; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR1_LATCH4:%.*]] ] 116; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR1_LATCH4]] ] 117; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT:%.*]], <4 x i64> [[VEC_IND]] 118; CHECK-NEXT: br label [[FOR2_HEADER1:%.*]] 119; CHECK: for2.header1: 120; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP3:%.*]], [[FOR2_HEADER1]] ] 121; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[VEC_PHI]] to <4 x i1> 122; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]] 123; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP2]], <4 x ptr> [[TMP0]], i32 8, <4 x i1> splat (i1 true)) 124entry: 125 br label %for1.header 126 127for1.header: 128 %indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ] 129 %ptr = getelementptr inbounds double, ptr %out, i64 %indvar1 130 br label %for2.header 131 132for2.header: 133 %indvar2 = phi i64 [ 0, %for1.header ], [ %indvar21, %for2.header ] 134 %select = trunc i64 %indvar2 to i1 135 ; Select condition only depends on inner loop iteration variable. 136 %select.b = select i1 %select, double %a, double %b 137 store double %select.b, ptr %ptr, align 8 138 %indvar21 = add nuw nsw i64 %indvar2, 1 139 %for2.cond = icmp eq i64 %indvar21, 10000 140 br i1 %for2.cond, label %for1.latch, label %for2.header 141 142for1.latch: 143 %indvar11 = add nuw nsw i64 %indvar1, 1 144 %for1.cond = icmp eq i64 %indvar11, 1000 145 br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0 146 147exit: 148 ret void 149} 150 151define void @outer_and_inner_loop_dependant_select(ptr noalias nocapture %out, double %a, double %b) { 152; CHECK-LABEL: @outer_and_inner_loop_dependant_select( 153; CHECK-NEXT: entry: 154; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 155; CHECK: vector.ph: 156; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A:%.*]], i64 0 157; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer 158; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0 159; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer 160; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 161; CHECK: vector.body: 162; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR1_LATCH4:%.*]] ] 163; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR1_LATCH4]] ] 164; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT:%.*]], <4 x i64> [[VEC_IND]] 165; CHECK-NEXT: br label [[FOR2_HEADER1:%.*]] 166; CHECK: for2.header1: 167; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP4:%.*]], [[FOR2_HEADER1]] ] 168; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], [[VEC_PHI]] 169; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i64> [[TMP1]] to <4 x i1> 170; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]] 171; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP3]], <4 x ptr> [[TMP0]], i32 8, <4 x i1> splat (i1 true)) 172entry: 173 br label %for1.header 174 175for1.header: 176 %indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ] 177 %ptr = getelementptr inbounds double, ptr %out, i64 %indvar1 178 br label %for2.header 179 180for2.header: 181 %indvar2 = phi i64 [ 0, %for1.header ], [ %indvar21, %for2.header ] 182 %sum = add nuw nsw i64 %indvar1, %indvar2 183 %select = trunc i64 %sum to i1 184 ; Select condition depends on both inner and outer loop iteration variables. 185 %select.b = select i1 %select, double %a, double %b 186 store double %select.b, ptr %ptr, align 8 187 %indvar21 = add nuw nsw i64 %indvar2, 1 188 %for2.cond = icmp eq i64 %indvar21, 10000 189 br i1 %for2.cond, label %for1.latch, label %for2.header 190 191for1.latch: 192 %indvar11 = add nuw nsw i64 %indvar1, 1 193 %for1.cond = icmp eq i64 %indvar11, 1000 194 br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0 195 196exit: 197 ret void 198} 199!0 = distinct !{!0, !1} 200!1 = !{!"llvm.loop.vectorize.enable", i1 true} 201