1; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=enabled -mattr=+mve %s -S -o - | FileCheck %s 2 3; CHECK-LABEL: reduction_i32 4; CHECK: phi i32 [ 0, %vector.ph ] 5; CHECK: phi <8 x i16> [ zeroinitializer, %vector.ph ] 6; CHECK: phi i32 7; CHECK: [[PHI:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[ELEMS:%[^ ]+]], %vector.body ] 8; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.mve.vctp16(i32 [[PHI]]) 9; CHECK: [[ELEMS]] = sub i32 [[PHI]], 8 10; CHECK: call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %tmp2, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef) 11; CHECK: call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %tmp5, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef) 12define i16 @reduction_i32(ptr nocapture readonly %A, ptr nocapture readonly %B, i32 %N) { 13entry: 14 %cmp8 = icmp eq i32 %N, 0 15 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph 16 17vector.ph: 18 %tmp = add i32 %N, -1 19 %n.rnd.up = add i32 %tmp, 8 20 %n.vec = and i32 %n.rnd.up, -8 21 %0 = add i32 %n.vec, -8 22 %1 = lshr i32 %0, 3 23 %2 = add i32 %1, 1 24 %start = call i32 @llvm.start.loop.iterations.i32(i32 %2) 25 br label %vector.body 26 27vector.body: ; preds = %vector.body, %vector.ph 28 %index = phi i32 [ 0, %vector.ph], [ %index.next, %vector.body ] 29 %vec.phi = phi <8 x i16> [ zeroinitializer, %vector.ph], [ %tmp8, %vector.body ] 30 %3 = phi i32 [ %start, %vector.ph], [ %4, %vector.body ] 31 %tmp2 = getelementptr inbounds i16, ptr %A, i32 %index 32 %tmp3 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %N) 33 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %tmp2, i32 4, <8 x i1> %tmp3, <8 x i16> undef) 34 %tmp5 = getelementptr inbounds i16, ptr %B, i32 %index 35 %wide.masked.load3 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %tmp5, i32 4, <8 x i1> %tmp3, <8 x i16> undef) 36 %tmp7 = add <8 x i16> %wide.masked.load, %vec.phi 37 %tmp8 = add <8 x i16> %tmp7, %wide.masked.load3 38 %index.next = add i32 %index, 8 39 %4 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %3, i32 1) 40 %5 = icmp ne i32 %4, 0 41 br i1 %5, label %vector.body, label %middle.block 42 43middle.block: ; preds = %vector.body 44 %vec.phi.lcssa = phi <8 x i16> [ %vec.phi, %vector.body ] 45 %.lcssa3 = phi <8 x i1> [ %tmp3, %vector.body ] 46 %.lcssa = phi <8 x i16> [ %tmp8, %vector.body ] 47 %tmp10 = select <8 x i1> %.lcssa3, <8 x i16> %.lcssa, <8 x i16> %vec.phi.lcssa 48 %rdx.shuf = shufflevector <8 x i16> %tmp10, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef> 49 %bin.rdx = add <8 x i16> %rdx.shuf, %tmp10 50 %rdx.shuf4 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 51 %bin.rdx5 = add <8 x i16> %rdx.shuf4, %bin.rdx 52 %rdx.shuf6 = shufflevector <8 x i16> %bin.rdx5, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 53 %bin.rdx7 = add <8 x i16> %rdx.shuf6, %bin.rdx5 54 %tmp11 = extractelement <8 x i16> %bin.rdx7, i32 0 55 ret i16 %tmp11 56 57for.cond.cleanup: 58 %res.0 = phi i16 [ 0, %entry ] 59 ret i16 %res.0 60} 61 62; CHECK-LABEL: reduction_i32_with_scalar 63; CHECK: vector.body: 64; CHECK: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 65; CHECK: %vec.phi = phi <8 x i16> [ zeroinitializer, %vector.ph ], [ %{{.*}}, %vector.body ] 66; CHECK: %{{.*}} = phi i32 [ %{{.*}}, %vector.ph ], [ %{{.*}}, %vector.body ] 67; CHECK: [[PHI:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[ELEMS:%[^ ]+]], %vector.body ] 68; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.mve.vctp16(i32 [[PHI]]) 69; CHECK: [[ELEMS]] = sub i32 [[PHI]], 8 70; CHECK: call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %tmp2, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef) 71define i16 @reduction_i32_with_scalar(ptr nocapture readonly %A, i16 %B, i32 %N) local_unnamed_addr { 72entry: 73 %cmp8 = icmp eq i32 %N, 0 74 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph 75 76vector.ph: 77 %tmp = add i32 %N, -1 78 %n.rnd.up = add nuw nsw i32 %tmp, 8 79 %n.vec = and i32 %n.rnd.up, -8 80 %broadcast.splatinsert3 = insertelement <8 x i16> undef, i16 %B, i32 0 81 %broadcast.splat4 = shufflevector <8 x i16> %broadcast.splatinsert3, <8 x i16> undef, <8 x i32> zeroinitializer 82 %0 = add i32 %n.vec, -8 83 %1 = lshr i32 %0, 3 84 %2 = add nuw nsw i32 %1, 1 85 %start = call i32 @llvm.start.loop.iterations.i32(i32 %2) 86 br label %vector.body 87 88vector.body: ; preds = %vector.body, %vector.ph 89 %index = phi i32 [ 0, %vector.ph], [ %index.next, %vector.body ] 90 %vec.phi = phi <8 x i16> [ zeroinitializer, %vector.ph], [ %tmp6, %vector.body ] 91 %3 = phi i32 [ %start, %vector.ph], [ %4, %vector.body ] 92 %tmp2 = getelementptr inbounds i16, ptr %A, i32 %index 93 %tmp3 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %N) 94 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %tmp2, i32 4, <8 x i1> %tmp3, <8 x i16> undef) 95 %tmp5 = add <8 x i16> %vec.phi, %broadcast.splat4 96 %tmp6 = add <8 x i16> %tmp5, %wide.masked.load 97 %index.next = add nuw nsw i32 %index, 8 98 %4 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %3, i32 1) 99 %5 = icmp ne i32 %4, 0 100 br i1 %5, label %vector.body, label %middle.block 101 102middle.block: ; preds = %vector.body 103 %tmp8 = select <8 x i1> %tmp3, <8 x i16> %tmp6, <8 x i16> %vec.phi 104 %rdx.shuf = shufflevector <8 x i16> %tmp8, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef> 105 %bin.rdx = add <8 x i16> %rdx.shuf, %tmp8 106 %rdx.shuf5 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 107 %bin.rdx6 = add <8 x i16> %rdx.shuf5, %bin.rdx 108 %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx6, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 109 %bin.rdx8 = add <8 x i16> %rdx.shuf7, %bin.rdx6 110 %tmp9 = extractelement <8 x i16> %bin.rdx8, i32 0 111 ret i16 %tmp9 112 113for.cond.cleanup: 114 %res.0 = phi i16 [ 0, %entry ] 115 ret i16 %res.0 116} 117 118; The vector loop is not guarded with an entry check (N == 0). Check that 119; despite this we can still calculate a precise enough range so that the 120; the overflow checks for get.active.active.lane.mask don't reject 121; tail-predication. 122; 123; CHECK-LABEL: @reduction_not_guarded 124; 125; CHECK: vector.body: 126; CHECK: @llvm.arm.mve.vctp 127; CHECK-NOT: @llvm.get.active.lane.mask.v8i1.i32 128; CHECK: ret 129; 130define i16 @reduction_not_guarded(ptr nocapture readonly %A, i16 %B, i32 %N) local_unnamed_addr { 131entry: 132 %tmp = add i32 %N, -1 133 %n.rnd.up = add nuw nsw i32 %tmp, 8 134 %n.vec = and i32 %n.rnd.up, -8 135 %broadcast.splatinsert3 = insertelement <8 x i16> undef, i16 %B, i32 0 136 %broadcast.splat4 = shufflevector <8 x i16> %broadcast.splatinsert3, <8 x i16> undef, <8 x i32> zeroinitializer 137 %0 = add i32 %n.vec, -8 138 %1 = lshr i32 %0, 3 139 %2 = add nuw nsw i32 %1, 1 140 %start = call i32 @llvm.start.loop.iterations.i32(i32 %2) 141 br label %vector.body 142 143vector.body: ; preds = %vector.body, %vector.ph 144 %index = phi i32 [ 0, %entry], [ %index.next, %vector.body ] 145 %vec.phi = phi <8 x i16> [ zeroinitializer, %entry], [ %tmp6, %vector.body ] 146 %3 = phi i32 [ %start, %entry ], [ %4, %vector.body ] 147 %tmp2 = getelementptr inbounds i16, ptr %A, i32 %index 148 %tmp3 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %N) 149 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %tmp2, i32 4, <8 x i1> %tmp3, <8 x i16> undef) 150 %tmp5 = add <8 x i16> %vec.phi, %broadcast.splat4 151 %tmp6 = add <8 x i16> %tmp5, %wide.masked.load 152 %index.next = add nuw nsw i32 %index, 8 153 %4 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %3, i32 1) 154 %5 = icmp ne i32 %4, 0 155 br i1 %5, label %vector.body, label %middle.block 156 157middle.block: ; preds = %vector.body 158 %tmp8 = select <8 x i1> %tmp3, <8 x i16> %tmp6, <8 x i16> %vec.phi 159 %rdx.shuf = shufflevector <8 x i16> %tmp8, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef> 160 %bin.rdx = add <8 x i16> %rdx.shuf, %tmp8 161 %rdx.shuf5 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 162 %bin.rdx6 = add <8 x i16> %rdx.shuf5, %bin.rdx 163 %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx6, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> 164 %bin.rdx8 = add <8 x i16> %rdx.shuf7, %bin.rdx6 165 %tmp9 = extractelement <8 x i16> %bin.rdx8, i32 0 166 ret i16 %tmp9 167} 168 169; CHECK-LABEL: @Correlation 170; CHECK: vector.body: 171; CHECK: @llvm.arm.mve.vctp 172; CHECK-NOT: %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask 173; 174define dso_local void @Correlation(ptr nocapture readonly %Input, ptr nocapture %Output, i16 signext %Size, i16 signext %N, i16 signext %Scale) local_unnamed_addr #0 { 175entry: 176 %conv = sext i16 %N to i32 177 %cmp36 = icmp sgt i16 %N, 0 178 br i1 %cmp36, label %for.body.lr.ph, label %for.end17 179 180for.body.lr.ph: 181 %conv2 = sext i16 %Size to i32 182 %conv1032 = zext i16 %Scale to i32 183 %0 = add i32 %conv2, 3 184 br label %for.body 185 186for.body: 187 %lsr.iv51 = phi i32 [ %lsr.iv.next, %for.end ], [ %0, %for.body.lr.ph ] 188 %lsr.iv46 = phi ptr [ %scevgep47, %for.end ], [ %Input, %for.body.lr.ph ] 189 %i.037 = phi i32 [ 0, %for.body.lr.ph ], [ %inc16, %for.end ] 190 %1 = mul nsw i32 %i.037, -1 191 %2 = add i32 %0, %1 192 %3 = lshr i32 %2, 2 193 %4 = shl nuw i32 %3, 2 194 %5 = add i32 %4, -4 195 %6 = lshr i32 %5, 2 196 %7 = add nuw nsw i32 %6, 1 197 %8 = sub i32 %conv2, %i.037 198 %cmp433 = icmp slt i32 %i.037, %conv2 199 br i1 %cmp433, label %vector.ph, label %for.end 200 201vector.ph: ; preds = %for.body 202 %start = call i32 @llvm.start.loop.iterations.i32(i32 %7) 203 br label %vector.body 204 205vector.body: ; preds = %vector.body, %vector.ph 206 %lsr.iv48 = phi ptr [ %scevgep49, %vector.body ], [ %lsr.iv46, %vector.ph ] 207 %lsr.iv = phi ptr [ %scevgep, %vector.body ], [ %Input, %vector.ph ] 208 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 209 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %16, %vector.body ] 210 %9 = phi i32 [ %start, %vector.ph ], [ %17, %vector.body ] 211 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %8) 212 %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr %lsr.iv, i32 2, <4 x i1> %active.lane.mask, <4 x i16> undef) 213 %10 = sext <4 x i16> %wide.masked.load to <4 x i32> 214 %wide.masked.load42 = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr %lsr.iv48, i32 2, <4 x i1> %active.lane.mask, <4 x i16> undef) 215 %11 = sext <4 x i16> %wide.masked.load42 to <4 x i32> 216 %12 = mul nsw <4 x i32> %11, %10 217 %13 = insertelement <4 x i32> undef, i32 %conv1032, i32 0 218 %14 = shufflevector <4 x i32> %13, <4 x i32> undef, <4 x i32> zeroinitializer 219 %15 = ashr <4 x i32> %12, %14 220 %16 = add <4 x i32> %15, %vec.phi 221 %index.next = add i32 %index, 4 222 %scevgep = getelementptr i16, ptr %lsr.iv, i32 4 223 %scevgep49 = getelementptr i16, ptr %lsr.iv48, i32 4 224 %17 = call i32 @llvm.loop.decrement.reg.i32(i32 %9, i32 1) 225 %18 = icmp ne i32 %17, 0 226 br i1 %18, label %vector.body, label %middle.block 227 228middle.block: ; preds = %vector.body 229 %19 = select <4 x i1> %active.lane.mask, <4 x i32> %16, <4 x i32> %vec.phi 230 %20 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %19) 231 br label %for.end 232 233for.end: ; preds = %middle.block, %for.body 234 %Sum.0.lcssa = phi i32 [ 0, %for.body ], [ %20, %middle.block ] 235 %21 = lshr i32 %Sum.0.lcssa, 16 236 %conv13 = trunc i32 %21 to i16 237 %arrayidx14 = getelementptr inbounds i16, ptr %Output, i32 %i.037 238 store i16 %conv13, ptr %arrayidx14, align 2 239 %inc16 = add nuw nsw i32 %i.037, 1 240 %scevgep47 = getelementptr i16, ptr %lsr.iv46, i32 1 241 %lsr.iv.next = add i32 %lsr.iv51, -1 242 %exitcond39 = icmp eq i32 %inc16, %conv 243 br i1 %exitcond39, label %for.end17, label %for.body 244 245for.end17: ; preds = %for.end, %entry 246 ret void 247} 248 249declare <8 x i16> @llvm.masked.load.v8i16.p0(ptr, i32 immarg, <8 x i1>, <8 x i16>) 250declare i32 @llvm.start.loop.iterations.i32(i32) 251declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) 252declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32) 253declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32) 254declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>) 255declare i32 @llvm.loop.decrement.reg.i32(i32, i32) 256declare <4 x i16> @llvm.masked.load.v4i16.p0(ptr, i32 immarg, <4 x i1>, <4 x i16>) 257