Lines Matching defs:Builder
123 Value *insertb(IRBuilderBase &Builder, Value *Dest, Value *Src, int Start,
125 Value *vlalignb(IRBuilderBase &Builder, Value *Lo, Value *Hi,
127 Value *vralignb(IRBuilderBase &Builder, Value *Lo, Value *Hi,
129 Value *concat(IRBuilderBase &Builder, ArrayRef<Value *> Vecs) const;
130 Value *vresize(IRBuilderBase &Builder, Value *Val, int NewSize,
132 Value *rescale(IRBuilderBase &Builder, Value *Mask, Type *FromTy,
134 Value *vlsb(IRBuilderBase &Builder, Value *Val) const;
135 Value *vbytes(IRBuilderBase &Builder, Value *Val) const;
136 Value *subvector(IRBuilderBase &Builder, Value *Val, unsigned Start,
138 Value *sublo(IRBuilderBase &Builder, Value *Val) const;
139 Value *subhi(IRBuilderBase &Builder, Value *Val) const;
140 Value *vdeal(IRBuilderBase &Builder, Value *Val0, Value *Val1) const;
141 Value *vshuff(IRBuilderBase &Builder, Value *Val0, Value *Val1) const;
143 Value *createHvxIntrinsic(IRBuilderBase &Builder, Intrinsic::ID IntID,
147 SmallVector<Value *> splitVectorElements(IRBuilderBase &Builder, Value *Vec,
149 Value *joinVectorElements(IRBuilderBase &Builder, ArrayRef<Value *> Values,
179 Value *getElementRange(IRBuilderBase &Builder, Value *Lo, Value *Hi,
312 Value *createAdjustedPointer(IRBuilderBase &Builder, Value *Ptr, Type *ValTy,
315 Value *createAlignedPointer(IRBuilderBase &Builder, Value *Ptr, Type *ValTy,
319 Value *createLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
323 Value *createSimpleLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
327 Value *createStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
330 Value *createSimpleStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
334 Value *createPredicatedLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
338 createPredicatedStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
349 void realignLoadGroup(IRBuilderBase &Builder, const ByteSpan &VSpan,
351 void realignStoreGroup(IRBuilderBase &Builder, const ByteSpan &VSpan,
355 Value *makeTestIfUnaligned(IRBuilderBase &Builder, Value *AlignVal,
457 auto processFxpMulChopped(IRBuilderBase &Builder, Instruction &In,
459 auto createMulQ15(IRBuilderBase &Builder, SValue X, SValue Y,
461 auto createMulQ31(IRBuilderBase &Builder, SValue X, SValue Y,
464 auto createAddCarry(IRBuilderBase &Builder, Value *X, Value *Y,
467 auto createMul16(IRBuilderBase &Builder, SValue X, SValue Y) const -> Value *;
468 auto createMulH16(IRBuilderBase &Builder, SValue X, SValue Y) const
470 auto createMul32(IRBuilderBase &Builder, SValue X, SValue Y) const
472 auto createAddLong(IRBuilderBase &Builder, ArrayRef<Value *> WordX,
474 auto createMulLong(IRBuilderBase &Builder, ArrayRef<Value *> WordX,
683 auto AlignVectors::createAdjustedPointer(IRBuilderBase &Builder, Value *Ptr,
690 return Builder.CreatePtrAdd(Ptr, HVC.getConstInt(Adjust), "gep");
693 auto AlignVectors::createAlignedPointer(IRBuilderBase &Builder, Value *Ptr,
705 Value *AsInt = Builder.CreatePtrToInt(Ptr, HVC.getIntTy(), "pti");
707 Value *And = Builder.CreateAnd(remap(AsInt), Mask, "and");
708 return Builder.CreateIntToPtr(
712 auto AlignVectors::createLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
724 Value *Load = createPredicatedLoad(Builder, ValTy, Ptr, Predicate,
726 return Builder.CreateSelect(Mask, Load, PassThru);
734 return createSimpleLoad(Builder, ValTy, Ptr, Alignment, MDSources);
736 Instruction *Load = Builder.CreateMaskedLoad(ValTy, Ptr, Align(Alignment),
742 auto AlignVectors::createSimpleLoad(IRBuilderBase &Builder, Type *ValTy,
747 Builder.CreateAlignedLoad(ValTy, Ptr, Align(Alignment), "ald");
752 auto AlignVectors::createPredicatedLoad(IRBuilderBase &Builder, Type *ValTy,
765 return createSimpleLoad(Builder, ValTy, Ptr, Alignment, MDSources);
769 return HVC.createHvxIntrinsic(Builder, V6_vL32b_pred_ai, ValTy,
774 auto AlignVectors::createStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
791 return createPredicatedStore(Builder, Val, Ptr, Predicate, Alignment,
795 return createSimpleStore(Builder, Val, Ptr, Alignment, MDSources);
801 Builder.CreateMaskedStore(Val, Ptr, Align(Alignment), Mask);
808 Value *PredLoad = createPredicatedLoad(Builder, Val->getType(), Ptr,
810 Value *Mux = Builder.CreateSelect(Mask, Val, PredLoad);
811 return createPredicatedStore(Builder, Mux, Ptr, Predicate, Alignment,
815 auto AlignVectors::createSimpleStore(IRBuilderBase &Builder, Value *Val,
819 Instruction *Store = Builder.CreateAlignedStore(Val, Ptr, Align(Alignment));
824 auto AlignVectors::createPredicatedStore(IRBuilderBase &Builder, Value *Val,
835 return createSimpleStore(Builder, Val, Ptr, Alignment, MDSources);
840 return HVC.createHvxIntrinsic(Builder, V6_vS32b_pred_ai, nullptr,
1097 auto AlignVectors::realignLoadGroup(IRBuilderBase &Builder,
1106 BasicBlock::iterator BasePos = Builder.GetInsertPoint();
1107 BasicBlock *BaseBlock = Builder.GetInsertBlock();
1190 auto createLoad = [&](IRBuilderBase &Builder, const ByteSpan &VSpan,
1193 createAdjustedPointer(Builder, AlignAddr, SecTy, Index * ScLen);
1195 MakePred ? makeTestIfUnaligned(Builder, AlignVal, ScLen) : nullptr;
1201 return this->createLoad(Builder, SecTy, Ptr, Predicate, ScLen, True, Undef,
1230 Builder.SetInsertPoint(Where);
1232 createLoad(Builder, VSpan, Index, DoAlign && Index == NumSectors);
1252 Builder.SetInsertPoint(Where);
1258 Val = HVC.vralignb(Builder, Val, NextLoad, AlignVal);
1268 Builder.SetInsertPoint(cast<Instruction>(B.Seg.Val));
1287 Builder.SetInsertPoint(&*std::next(SegI->getIterator()));
1288 Value *Pay = HVC.vbytes(Builder, getPayload(S->Seg.Val));
1290 HVC.insertb(Builder, Accum, Pay, S->Seg.Start, S->Seg.Size, S->Pos);
1299 Value *Cast = Builder.CreateBitCast(Accum, ValTy, "cst");
1300 Value *Sel = Builder.CreateSelect(getMask(B.Seg.Val), Cast,
1306 auto AlignVectors::realignStoreGroup(IRBuilderBase &Builder,
1321 auto MakeVec = [](IRBuilderBase &Builder, Value *Val) -> Value * {
1326 return Builder.CreateBitCast(Val, VecTy, "cst");
1342 Value *Mask = HVC.rescale(Builder, MakeVec(Builder, getMask(S.Seg.Val)),
1344 Value *PartM = HVC.insertb(Builder, Zero, HVC.vbytes(Builder, Mask),
1346 AccumM = Builder.CreateOr(AccumM, PartM);
1348 Value *PartV = HVC.insertb(Builder, Undef, HVC.vbytes(Builder, Pay),
1351 AccumV = Builder.CreateSelect(
1352 Builder.CreateICmp(CmpInst::ICMP_NE, PartM, Zero), PartV, AccumV);
1369 ASpanV[Index - 1].Seg.Val = HVC.vlalignb(Builder, PrevV, ThisV, AlignVal);
1370 ASpanM[Index - 1].Seg.Val = HVC.vlalignb(Builder, PrevM, ThisM, AlignVal);
1379 auto createStore = [&](IRBuilderBase &Builder, const ByteSpan &ASpanV,
1386 createAdjustedPointer(Builder, AlignAddr, SecTy, Index * ScLen);
1388 MakePred ? makeTestIfUnaligned(Builder, AlignVal, ScLen) : nullptr;
1394 this->createStore(Builder, Val, Ptr, Predicate, ScLen,
1395 HVC.vlsb(Builder, Mask),
1400 createStore(Builder, ASpanV, ASpanM, Index, DoAlign && Index == NumSectors);
1463 IRBuilder Builder(InsertAt->getParent(), InsertAt->getIterator(),
1478 AlignAddr = createAdjustedPointer(Builder, WithMaxAlign.Addr,
1493 createAlignedPointer(Builder, WithMinOffset.Addr, WithMinOffset.ValTy,
1496 Builder.CreatePtrToInt(WithMinOffset.Addr, HVC.getIntTy(), "pti");
1525 realignLoadGroup(Builder, VSpan, ScLen, AlignVal, AlignAddr);
1527 realignStoreGroup(Builder, VSpan, ScLen, AlignVal, AlignAddr);
1535 auto AlignVectors::makeTestIfUnaligned(IRBuilderBase &Builder, Value *AlignVal,
1538 Value *And = Builder.CreateAnd(
1541 return Builder.CreateICmpNE(And, Zero, "isz");
1744 IRBuilder Builder(In.getParent(), In.getIterator(),
1770 X = Builder.CreateTrunc(X, ResizeTy, "trn");
1771 Y = Builder.CreateTrunc(Y, ResizeTy, "trn");
1773 X = SignX == Signed ? Builder.CreateSExt(X, ResizeTy, "sxt")
1774 : Builder.CreateZExt(X, ResizeTy, "zxt");
1775 Y = SignY == Signed ? Builder.CreateSExt(Y, ResizeTy, "sxt")
1776 : Builder.CreateZExt(Y, ResizeTy, "zxt");
1789 ChopOp.X.Val = HVC.subvector(Builder, X, V * ChopLen, ChopLen);
1790 ChopOp.Y.Val = HVC.subvector(Builder, Y, V * ChopLen, ChopLen);
1791 Results.push_back(processFxpMulChopped(Builder, In, ChopOp));
1799 Value *Cat = HVC.concat(Builder, Results);
1801 ? Builder.CreateSExt(Cat, VecTy, "sxt")
1802 : Builder.CreateZExt(Cat, VecTy, "zxt");
1806 auto HvxIdioms::processFxpMulChopped(IRBuilderBase &Builder, Instruction &In,
1818 QMul = createMulQ15(Builder, Op.X, Op.Y, Rounding);
1820 QMul = createMulQ31(Builder, Op.X, Op.Y, Rounding);
1841 if (Value *MulH = createMulH16(Builder, Op.X, Op.Y))
1845 Value *Prod32 = createMul16(Builder, Op.X, Op.Y);
1848 Prod32 = Builder.CreateAdd(Prod32, RoundVal, "add");
1853 ? Builder.CreateAShr(Prod32, ShiftAmt, "asr")
1854 : Builder.CreateLShr(Prod32, ShiftAmt, "lsr");
1855 return Builder.CreateTrunc(Shifted, InpTy, "trn");
1862 auto WordX = HVC.splitVectorElements(Builder, Op.X.Val, /*ToWidth=*/32);
1863 auto WordY = HVC.splitVectorElements(Builder, Op.Y.Val, /*ToWidth=*/32);
1864 auto WordP = createMulLong(Builder, WordX, Op.X.Sgn, WordY, Op.Y.Sgn);
1874 WordP = createAddLong(Builder, WordP, RoundV);
1888 WordP[Dst] = Builder.CreateIntrinsic(HvxWordTy, Intrinsic::fshr,
1893 WordP[Dst] = Builder.CreateAShr(Lo, ShiftAmt, "asr");
1899 return HVC.joinVectorElements(Builder, WordP, Op.ResTy);
1902 auto HvxIdioms::createMulQ15(IRBuilderBase &Builder, SValue X, SValue Y,
1913 return HVC.createHvxIntrinsic(Builder, V6_vmpyhvsrs, X.Val->getType(),
1917 auto HvxIdioms::createMulQ31(IRBuilderBase &Builder, SValue X, SValue Y,
1932 HVC.createHvxIntrinsic(Builder, V6_vmpyewuh, InpTy, {X.Val, Y.Val});
1933 return HVC.createHvxIntrinsic(Builder, V6_vmpyo_acc, InpTy,
1937 auto HvxIdioms::createAddCarry(IRBuilderBase &Builder, Value *X, Value *Y,
1953 Value *Ret = HVC.createHvxIntrinsic(Builder, AddCarry,
1955 Value *Result = Builder.CreateExtractValue(Ret, {0}, "ext");
1956 Value *CarryOut = Builder.CreateExtractValue(Ret, {1}, "ext");
1973 HVC.createHvxIntrinsic(Builder, V6_vandqrt, /*RetTy=*/nullptr,
1975 Result1 = Builder.CreateAdd(X, ValueIn, "add");
1978 Value *CarryOut1 = Builder.CreateCmp(CmpInst::ICMP_ULT, Result1, X, "cmp");
1979 Value *Result2 = Builder.CreateAdd(Result1, Y, "add");
1980 Value *CarryOut2 = Builder.CreateCmp(CmpInst::ICMP_ULT, Result2, Y, "cmp");
1981 return {Result2, Builder.CreateOr(CarryOut1, CarryOut2, "orb")};
1984 auto HvxIdioms::createMul16(IRBuilderBase &Builder, SValue X, SValue Y) const
2000 HVC.createHvxIntrinsic(Builder, V6_vmpyh, HvxP32Ty, {Y.Val, X.Val});
2002 return HVC.vshuff(Builder, HVC.sublo(Builder, P), HVC.subhi(Builder, P));
2005 auto HvxIdioms::createMulH16(IRBuilderBase &Builder, SValue X, SValue Y) const
2012 return HVC.createHvxIntrinsic(Builder, V6_vmpyuhvs, HvxI16Ty,
2019 Builder.CreateBitCast(createMul16(Builder, X, Y), HvxP16Ty, "cst");
2026 return Builder.CreateShuffleVector(
2027 HVC.sublo(Builder, Pair16), HVC.subhi(Builder, Pair16), PickOdd, "shf");
2030 auto HvxIdioms::createMul32(IRBuilderBase &Builder, SValue X, SValue Y) const
2046 Value *Parts = HVC.createHvxIntrinsic(Builder, V6_vmpy_parts, nullptr,
2048 Value *Hi = Builder.CreateExtractValue(Parts, {0}, "ext");
2049 Value *Lo = Builder.CreateExtractValue(Parts, {1}, "ext");
2053 auto HvxIdioms::createAddLong(IRBuilderBase &Builder, ArrayRef<Value *> WordX,
2073 createAddCarry(Builder, WordX[Idx], WordY[Idx], Carry);
2080 auto HvxIdioms::createMulLong(IRBuilderBase &Builder, ArrayRef<Value *> WordX,
2092 auto [Lo, Hi] = createMul32(Builder, {WordX[i], SX}, {WordY[j], SY});
2113 auto [Sum, CarryOut] = createAddCarry(Builder, pop_back_or_zero(ProdJ),
2310 auto HexagonVectorCombine::insertb(IRBuilderBase &Builder, Value *Dst,
2321 Value *P2Src = vresize(Builder, Src, P2Len, Undef);
2322 Value *P2Dst = vresize(Builder, Dst, P2Len, Undef);
2332 Value *P2Insert = Builder.CreateShuffleVector(P2Dst, P2Src, SMask, "shf");
2333 return vresize(Builder, P2Insert, DstLen, Undef);
2336 auto HexagonVectorCombine::vlalignb(IRBuilderBase &Builder, Value *Lo,
2343 return getElementRange(Builder, Lo, Hi, VecLen - IntAmt->getSExtValue(),
2349 return createHvxIntrinsic(Builder, HST.getIntrinsicId(Hexagon::V6_vlalignb),
2354 Value *Pair = concat(Builder, {Lo, Hi});
2356 Builder.CreateLShr(Builder.CreateShl(Pair, Amt, "shl"), 32, "lsr");
2358 Builder.CreateTrunc(Shift, Type::getInt32Ty(F.getContext()), "trn");
2359 return Builder.CreateBitCast(Trunc, Hi->getType(), "cst");
2362 Value *Sub = Builder.CreateSub(getConstInt(VecLen), Amt, "sub");
2363 return vralignb(Builder, Lo, Hi, Sub);
2368 auto HexagonVectorCombine::vralignb(IRBuilderBase &Builder, Value *Lo,
2375 return getElementRange(Builder, Lo, Hi, IntAmt->getSExtValue(), VecLen);
2380 return createHvxIntrinsic(Builder, HST.getIntrinsicId(Hexagon::V6_valignb),
2385 Value *Pair = concat(Builder, {Lo, Hi});
2386 Value *Shift = Builder.CreateLShr(Pair, Amt, "lsr");
2388 Builder.CreateTrunc(Shift, Type::getInt32Ty(F.getContext()), "trn");
2389 return Builder.CreateBitCast(Trunc, Lo->getType(), "cst");
2393 Value *Lo64 = Builder.CreateBitCast(Lo, Int64Ty, "cst");
2394 Value *Hi64 = Builder.CreateBitCast(Hi, Int64Ty, "cst");
2397 Value *Call = Builder.CreateCall(FI, {Hi64, Lo64, Amt}, "cup");
2398 return Builder.CreateBitCast(Call, Lo->getType(), "cst");
2404 auto HexagonVectorCombine::concat(IRBuilderBase &Builder,
2421 Value *Joined = Builder.CreateShuffleVector(
2434 return Builder.CreateShuffleVector(Total, SMask, "shf");
2437 auto HexagonVectorCombine::vresize(IRBuilderBase &Builder, Value *Val,
2448 return getElementRange(Builder, Val, /*Ignored*/ Val, 0, NewSize);
2453 Value *PadVec = Builder.CreateVectorSplat(CurSize, Pad, "spt");
2454 return Builder.CreateShuffleVector(Val, PadVec, SMask, "shf");
2457 auto HexagonVectorCombine::rescale(IRBuilderBase &Builder, Value *Mask,
2483 Value *Ext = Builder.CreateSExt(
2485 Value *Cast = Builder.CreateBitCast(
2487 return Builder.CreateTrunc(
2492 auto HexagonVectorCombine::vlsb(IRBuilderBase &Builder, Value *Val) const
2498 Value *Bytes = vbytes(Builder, Val);
2500 return Builder.CreateTrunc(Bytes, getBoolTy(getSizeOf(VecTy)), "trn");
2503 return Builder.CreateTrunc(Bytes, getBoolTy(), "trn");
2507 auto HexagonVectorCombine::vbytes(IRBuilderBase &Builder, Value *Val) const
2514 return Builder.CreateBitCast(Val, getByteTy(getSizeOf(Val)), "cst");
2517 return Builder.CreateSExt(Val, VectorType::get(getByteTy(), VecTy), "sxt");
2518 return Builder.CreateSExt(Val, getByteTy(), "sxt");
2521 auto HexagonVectorCombine::subvector(IRBuilderBase &Builder, Value *Val,
2525 return getElementRange(Builder, Val, /*Ignored*/ Val, Start, Length);
2528 auto HexagonVectorCombine::sublo(IRBuilderBase &Builder, Value *Val) const
2532 return subvector(Builder, Val, 0, Len / 2);
2535 auto HexagonVectorCombine::subhi(IRBuilderBase &Builder, Value *Val) const
2539 return subvector(Builder, Val, Len / 2, Len / 2);
2542 auto HexagonVectorCombine::vdeal(IRBuilderBase &Builder, Value *Val0,
2552 return Builder.CreateShuffleVector(Val0, Val1, Mask, "shf");
2555 auto HexagonVectorCombine::vshuff(IRBuilderBase &Builder, Value *Val0,
2565 return Builder.CreateShuffleVector(Val0, Val1, Mask, "shf");
2568 auto HexagonVectorCombine::createHvxIntrinsic(IRBuilderBase &Builder,
2574 auto getCast = [&](IRBuilderBase &Builder, Value *Val,
2586 return Builder.CreateBitCast(Val, DestTy, "cst");
2594 return Builder.CreateCall(FI, {Val}, "cup");
2605 IntrArgs.push_back(getCast(Builder, A, T));
2611 CallInst *Call = Builder.CreateCall(IntrFn, IntrArgs, MaybeName);
2622 return getCast(Builder, Call, RetTy);
2625 auto HexagonVectorCombine::splitVectorElements(IRBuilderBase &Builder,
2665 Value *VVal = Builder.CreateBitCast(Val, VTy, "cst");
2667 Value *Res = vdeal(Builder, sublo(Builder, VVal), subhi(Builder, VVal));
2670 Results[Begin] = sublo(Builder, Res);
2671 Results[Half] = subhi(Builder, Res);
2681 auto HexagonVectorCombine::joinVectorElements(IRBuilderBase &Builder,
2707 Value *Sign = Builder.CreateAShr(
2716 Value *Res = vshuff(Builder, Inputs[i], Inputs[i + 1]);
2717 Inputs[i / 2] = Builder.CreateBitCast(Res, VTy, "cst");
2739 struct Builder : IRBuilder<> {
2740 Builder(BasicBlock *B) : IRBuilder<>(B->getTerminator()) {}
2741 ~Builder() {
2780 Builder B(Gep0->getParent());
2923 auto HexagonVectorCombine::getElementRange(IRBuilderBase &Builder, Value *Lo,
2929 return Builder.CreateShuffleVector(Lo, Hi, SMask, "shf");