1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt < %s -passes="default<O3>" -S | FileCheck %s 3 4; This test after a lot of cleanup should produce pick a tail-predicated 8x 5; vector loop. The 8x will be more profitable, to pick a VQDMULH.s16 instruction. 6; FIXME: Tailpredicate too, but not at the expense of 8x vectorized. 7 8target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64" 9target triple = "thumbv8.1m.main-arm-none-eabi" 10 11define void @arm_mult_q15(ptr %pSrcA, ptr %pSrcB, ptr noalias %pDst, i32 %blockSize) #0 { 12; CHECK-LABEL: @arm_mult_q15( 13; CHECK-NEXT: entry: 14; CHECK-NEXT: [[CMP_NOT2:%.*]] = icmp eq i32 [[BLOCKSIZE:%.*]], 0 15; CHECK-NEXT: br i1 [[CMP_NOT2]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]] 16; CHECK: while.body.preheader: 17; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[BLOCKSIZE]], 8 18; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[WHILE_BODY_PREHEADER15:%.*]], label [[VECTOR_PH:%.*]] 19; CHECK: vector.ph: 20; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[BLOCKSIZE]], -8 21; CHECK-NEXT: [[IND_END:%.*]] = and i32 [[BLOCKSIZE]], 7 22; CHECK-NEXT: [[TMP0:%.*]] = shl i32 [[N_VEC]], 1 23; CHECK-NEXT: [[IND_END7:%.*]] = getelementptr i8, ptr [[PSRCA:%.*]], i32 [[TMP0]] 24; CHECK-NEXT: [[TMP1:%.*]] = shl i32 [[N_VEC]], 1 25; CHECK-NEXT: [[IND_END9:%.*]] = getelementptr i8, ptr [[PDST:%.*]], i32 [[TMP1]] 26; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[N_VEC]], 1 27; CHECK-NEXT: [[IND_END11:%.*]] = getelementptr i8, ptr [[PSRCB:%.*]], i32 [[TMP2]] 28; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 29; CHECK: vector.body: 30; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] 31; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 1 32; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PSRCA]], i32 [[OFFSET_IDX]] 33; CHECK-NEXT: [[OFFSET_IDX13:%.*]] = shl i32 [[INDEX]], 1 34; CHECK-NEXT: [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[OFFSET_IDX13]] 35; CHECK-NEXT: [[OFFSET_IDX15:%.*]] = shl i32 [[INDEX]], 1 36; CHECK-NEXT: [[NEXT_GEP16:%.*]] = getelementptr i8, ptr [[PSRCB]], i32 [[OFFSET_IDX15]] 37; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[NEXT_GEP]], align 2 38; CHECK-NEXT: [[TMP3:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32> 39; CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[NEXT_GEP16]], align 2 40; CHECK-NEXT: [[TMP4:%.*]] = sext <8 x i16> [[WIDE_LOAD17]] to <8 x i32> 41; CHECK-NEXT: [[TMP5:%.*]] = mul nsw <8 x i32> [[TMP4]], [[TMP3]] 42; CHECK-NEXT: [[TMP6:%.*]] = ashr <8 x i32> [[TMP5]], splat (i32 15) 43; CHECK-NEXT: [[TMP7:%.*]] = tail call <8 x i32> @llvm.smin.v8i32(<8 x i32> [[TMP6]], <8 x i32> splat (i32 32767)) 44; CHECK-NEXT: [[TMP8:%.*]] = trunc <8 x i32> [[TMP7]] to <8 x i16> 45; CHECK-NEXT: store <8 x i16> [[TMP8]], ptr [[NEXT_GEP14]], align 2 46; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 47; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] 48; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 49; CHECK: middle.block: 50; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[BLOCKSIZE]], [[N_VEC]] 51; CHECK-NEXT: br i1 [[CMP_N]], label [[WHILE_END]], label [[WHILE_BODY_PREHEADER15]] 52; CHECK: while.body.preheader15: 53; CHECK-NEXT: [[BLKCNT_06_PH:%.*]] = phi i32 [ [[BLOCKSIZE]], [[WHILE_BODY_PREHEADER]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ] 54; CHECK-NEXT: [[PSRCA_ADDR_05_PH:%.*]] = phi ptr [ [[PSRCA]], [[WHILE_BODY_PREHEADER]] ], [ [[IND_END7]], [[MIDDLE_BLOCK]] ] 55; CHECK-NEXT: [[PDST_ADDR_04_PH:%.*]] = phi ptr [ [[PDST]], [[WHILE_BODY_PREHEADER]] ], [ [[IND_END9]], [[MIDDLE_BLOCK]] ] 56; CHECK-NEXT: [[PSRCB_ADDR_03_PH:%.*]] = phi ptr [ [[PSRCB]], [[WHILE_BODY_PREHEADER]] ], [ [[IND_END11]], [[MIDDLE_BLOCK]] ] 57; CHECK-NEXT: br label [[WHILE_BODY:%.*]] 58; CHECK: while.body: 59; CHECK-NEXT: [[BLKCNT_06:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BLKCNT_06_PH]], [[WHILE_BODY_PREHEADER15]] ] 60; CHECK-NEXT: [[PSRCA_ADDR_05:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[PSRCA_ADDR_05_PH]], [[WHILE_BODY_PREHEADER15]] ] 61; CHECK-NEXT: [[PDST_ADDR_04:%.*]] = phi ptr [ [[INCDEC_PTR4:%.*]], [[WHILE_BODY]] ], [ [[PDST_ADDR_04_PH]], [[WHILE_BODY_PREHEADER15]] ] 62; CHECK-NEXT: [[PSRCB_ADDR_03:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], [[WHILE_BODY]] ], [ [[PSRCB_ADDR_03_PH]], [[WHILE_BODY_PREHEADER15]] ] 63; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds nuw i8, ptr [[PSRCA_ADDR_05]], i32 2 64; CHECK-NEXT: [[TMP10:%.*]] = load i16, ptr [[PSRCA_ADDR_05]], align 2 65; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP10]] to i32 66; CHECK-NEXT: [[INCDEC_PTR1]] = getelementptr inbounds nuw i8, ptr [[PSRCB_ADDR_03]], i32 2 67; CHECK-NEXT: [[TMP11:%.*]] = load i16, ptr [[PSRCB_ADDR_03]], align 2 68; CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP11]] to i32 69; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]] 70; CHECK-NEXT: [[SHR:%.*]] = ashr i32 [[MUL]], 15 71; CHECK-NEXT: [[SPEC_SELECT_I:%.*]] = tail call i32 @llvm.smin.i32(i32 [[SHR]], i32 32767) 72; CHECK-NEXT: [[CONV3:%.*]] = trunc nsw i32 [[SPEC_SELECT_I]] to i16 73; CHECK-NEXT: [[INCDEC_PTR4]] = getelementptr inbounds nuw i8, ptr [[PDST_ADDR_04]], i32 2 74; CHECK-NEXT: store i16 [[CONV3]], ptr [[PDST_ADDR_04]], align 2 75; CHECK-NEXT: [[DEC]] = add i32 [[BLKCNT_06]], -1 76; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[DEC]], 0 77; CHECK-NEXT: br i1 [[CMP_NOT]], label [[WHILE_END]], label [[WHILE_BODY]], !llvm.loop [[LOOP3:![0-9]+]] 78; CHECK: while.end: 79; CHECK-NEXT: ret void 80; 81entry: 82 %pSrcA.addr = alloca ptr, align 4 83 %pSrcB.addr = alloca ptr, align 4 84 %pDst.addr = alloca ptr, align 4 85 %blockSize.addr = alloca i32, align 4 86 %blkCnt = alloca i32, align 4 87 store ptr %pSrcA, ptr %pSrcA.addr, align 4 88 store ptr %pSrcB, ptr %pSrcB.addr, align 4 89 store ptr %pDst, ptr %pDst.addr, align 4 90 store i32 %blockSize, ptr %blockSize.addr, align 4 91 call void @llvm.lifetime.start.p0(i64 4, ptr %blkCnt) #3 92 %0 = load i32, ptr %blockSize.addr, align 4 93 store i32 %0, ptr %blkCnt, align 4 94 br label %while.cond 95 96while.cond: ; preds = %while.body, %entry 97 %1 = load i32, ptr %blkCnt, align 4 98 %cmp = icmp ugt i32 %1, 0 99 br i1 %cmp, label %while.body, label %while.end 100 101while.body: ; preds = %while.cond 102 %2 = load ptr, ptr %pSrcA.addr, align 4 103 %incdec.ptr = getelementptr inbounds i16, ptr %2, i32 1 104 store ptr %incdec.ptr, ptr %pSrcA.addr, align 4 105 %3 = load i16, ptr %2, align 2 106 %conv = sext i16 %3 to i32 107 %4 = load ptr, ptr %pSrcB.addr, align 4 108 %incdec.ptr1 = getelementptr inbounds i16, ptr %4, i32 1 109 store ptr %incdec.ptr1, ptr %pSrcB.addr, align 4 110 %5 = load i16, ptr %4, align 2 111 %conv2 = sext i16 %5 to i32 112 %mul = mul nsw i32 %conv, %conv2 113 %shr = ashr i32 %mul, 15 114 %call = call i32 @__SSAT(i32 %shr, i32 16) 115 %conv3 = trunc i32 %call to i16 116 %6 = load ptr, ptr %pDst.addr, align 4 117 %incdec.ptr4 = getelementptr inbounds i16, ptr %6, i32 1 118 store ptr %incdec.ptr4, ptr %pDst.addr, align 4 119 store i16 %conv3, ptr %6, align 2 120 %7 = load i32, ptr %blkCnt, align 4 121 %dec = add i32 %7, -1 122 store i32 %dec, ptr %blkCnt, align 4 123 br label %while.cond 124 125while.end: ; preds = %while.cond 126 call void @llvm.lifetime.end.p0(i64 4, ptr %blkCnt) #3 127 ret void 128} 129 130declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1 131 132define internal i32 @__SSAT(i32 %val, i32 %sat) #2 { 133entry: 134 %retval = alloca i32, align 4 135 %val.addr = alloca i32, align 4 136 %sat.addr = alloca i32, align 4 137 %max = alloca i32, align 4 138 %min = alloca i32, align 4 139 %cleanup.dest.slot = alloca i32, align 4 140 store i32 %val, ptr %val.addr, align 4 141 store i32 %sat, ptr %sat.addr, align 4 142 %0 = load i32, ptr %sat.addr, align 4 143 %cmp = icmp uge i32 %0, 1 144 br i1 %cmp, label %land.lhs.true, label %if.end10 145 146land.lhs.true: ; preds = %entry 147 %1 = load i32, ptr %sat.addr, align 4 148 %cmp1 = icmp ule i32 %1, 32 149 br i1 %cmp1, label %if.then, label %if.end10 150 151if.then: ; preds = %land.lhs.true 152 call void @llvm.lifetime.start.p0(i64 4, ptr %max) #3 153 %2 = load i32, ptr %sat.addr, align 4 154 %sub = sub i32 %2, 1 155 %shl = shl i32 1, %sub 156 %sub2 = sub i32 %shl, 1 157 store i32 %sub2, ptr %max, align 4 158 call void @llvm.lifetime.start.p0(i64 4, ptr %min) #3 159 %3 = load i32, ptr %max, align 4 160 %sub3 = sub nsw i32 -1, %3 161 store i32 %sub3, ptr %min, align 4 162 %4 = load i32, ptr %val.addr, align 4 163 %5 = load i32, ptr %max, align 4 164 %cmp4 = icmp sgt i32 %4, %5 165 br i1 %cmp4, label %if.then5, label %if.else 166 167if.then5: ; preds = %if.then 168 %6 = load i32, ptr %max, align 4 169 store i32 %6, ptr %retval, align 4 170 store i32 1, ptr %cleanup.dest.slot, align 4 171 br label %cleanup 172 173if.else: ; preds = %if.then 174 %7 = load i32, ptr %val.addr, align 4 175 %8 = load i32, ptr %min, align 4 176 %cmp6 = icmp slt i32 %7, %8 177 br i1 %cmp6, label %if.then7, label %if.end 178 179if.then7: ; preds = %if.else 180 %9 = load i32, ptr %min, align 4 181 store i32 %9, ptr %retval, align 4 182 store i32 1, ptr %cleanup.dest.slot, align 4 183 br label %cleanup 184 185if.end: ; preds = %if.else 186 br label %if.end8 187 188if.end8: ; preds = %if.end 189 store i32 0, ptr %cleanup.dest.slot, align 4 190 br label %cleanup 191 192cleanup: ; preds = %if.end8, %if.then7, %if.then5 193 call void @llvm.lifetime.end.p0(i64 4, ptr %min) #3 194 call void @llvm.lifetime.end.p0(i64 4, ptr %max) #3 195 %cleanup.dest = load i32, ptr %cleanup.dest.slot, align 4 196 switch i32 %cleanup.dest, label %unreachable [ 197 i32 0, label %cleanup.cont 198 i32 1, label %return 199 ] 200 201cleanup.cont: ; preds = %cleanup 202 br label %if.end10 203 204if.end10: ; preds = %cleanup.cont, %land.lhs.true, %entry 205 %10 = load i32, ptr %val.addr, align 4 206 store i32 %10, ptr %retval, align 4 207 br label %return 208 209return: ; preds = %if.end10, %cleanup 210 %11 = load i32, ptr %retval, align 4 211 ret i32 %11 212 213unreachable: ; preds = %cleanup 214 unreachable 215} 216 217declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1 218 219attributes #0 = { nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m55" "target-features"="+armv8.1-m.main,+dsp,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fullfp16,+hwdiv,+lob,+mve,+mve.fp,+ras,+strict-align,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp,-aes,-bf16,-cdecp0,-cdecp1,-cdecp2,-cdecp3,-cdecp4,-cdecp5,-cdecp6,-cdecp7,-crc,-crypto,-d32,-dotprod,-fp-armv8,-fp-armv8sp,-fp16fml,-hwdiv-arm,-i8mm,-neon,-sb,-sha2,-vfp3,-vfp3sp,-vfp4,-vfp4sp" "unsafe-fp-math"="true" } 220attributes #1 = { argmemonly nofree nosync nounwind willreturn } 221attributes #2 = { alwaysinline nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m55" "target-features"="+armv8.1-m.main,+dsp,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fullfp16,+hwdiv,+lob,+mve,+mve.fp,+ras,+strict-align,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp,-aes,-bf16,-cdecp0,-cdecp1,-cdecp2,-cdecp3,-cdecp4,-cdecp5,-cdecp6,-cdecp7,-crc,-crypto,-d32,-dotprod,-fp-armv8,-fp-armv8sp,-fp16fml,-hwdiv-arm,-i8mm,-neon,-sb,-sha2,-vfp3,-vfp3sp,-vfp4,-vfp4sp" "unsafe-fp-math"="true" } 222attributes #3 = { nounwind } 223