Lines Matching defs:IC
867 static std::optional<Instruction *> processPhiNode(InstCombiner &IC,
889 IC.Builder.SetInsertPoint(PN);
890 PHINode *NPN = IC.Builder.CreatePHI(RequiredType, PN->getNumIncomingValues());
900 return IC.replaceInstUsesWith(II, NPN);
913 tryCombineFromSVBoolBinOp(InstCombiner &IC, IntrinsicInst &II) {
947 auto NarrowBinOpOp1 = IC.Builder.CreateIntrinsic(
953 NarrowedBinOpArgs.push_back(IC.Builder.CreateIntrinsic(
957 IC.Builder.CreateIntrinsic(IntrinsicID, {PredOpTy}, NarrowedBinOpArgs);
958 return IC.replaceInstUsesWith(II, NarrowedBinOp);
962 instCombineConvertFromSVBool(InstCombiner &IC, IntrinsicInst &II) {
965 return processPhiNode(IC, II);
967 if (auto BinOpCombine = tryCombineFromSVBoolBinOp(IC, II))
1011 return IC.replaceInstUsesWith(II, EarliestReplacement);
1032 instCombineSVENoActiveUnaryErase(InstCombiner &IC, IntrinsicInst &II,
1035 return IC.eraseInstFromFunction(II);
1043 instCombineSVENoActiveUnaryZero(InstCombiner &IC, IntrinsicInst &II) {
1061 IC.replaceInstUsesWith(II, Node);
1062 return IC.eraseInstFromFunction(II);
1067 static std::optional<Instruction *> instCombineSVESel(InstCombiner &IC,
1072 return IC.replaceInstUsesWith(II, II.getOperand(1));
1075 IC.Builder.CreateSelect(OpPredicate, II.getOperand(1), II.getOperand(2));
1076 return IC.replaceInstUsesWith(II, Select);
1079 static std::optional<Instruction *> instCombineSVEDup(InstCombiner &IC,
1100 return IC.replaceInstUsesWith(II, Insert);
1103 static std::optional<Instruction *> instCombineSVEDupX(InstCombiner &IC,
1107 Value *Splat = IC.Builder.CreateVectorSplat(RetTy->getElementCount(),
1110 return IC.replaceInstUsesWith(II, Splat);
1113 static std::optional<Instruction *> instCombineSVECmpNE(InstCombiner &IC,
1180 return IC.replaceInstUsesWith(II, PFalse);
1200 auto *PTrue = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptrue,
1202 auto *ConvertToSVBool = IC.Builder.CreateIntrinsic(
1205 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool,
1209 return IC.replaceInstUsesWith(II, ConvertFromSVBool);
1212 static std::optional<Instruction *> instCombineSVELast(InstCombiner &IC,
1221 return IC.replaceInstUsesWith(II, SplatVal);
1231 IC.Builder.CreateIntrinsic(IntrinsicID, {Vec->getType()}, {Pg, LHS});
1233 IC.Builder.CreateIntrinsic(IntrinsicID, {Vec->getType()}, {Pg, RHS});
1236 return IC.replaceInstUsesWith(II, NewBinOp);
1247 return IC.replaceInstUsesWith(II, Extract);
1283 return IC.replaceInstUsesWith(II, Extract);
1286 static std::optional<Instruction *> instCombineSVECondLast(InstCombiner &IC,
1308 FPTy = IC.Builder.getHalfTy();
1311 FPTy = IC.Builder.getFloatTy();
1314 FPTy = IC.Builder.getDoubleTy();
1318 Value *FPFallBack = IC.Builder.CreateBitCast(Fallback, FPTy);
1321 Value *FPVec = IC.Builder.CreateBitCast(Vec, FPVTy);
1322 auto *FPII = IC.Builder.CreateIntrinsic(
1324 Value *FPIItoInt = IC.Builder.CreateBitCast(FPII, II.getType());
1325 return IC.replaceInstUsesWith(II, FPIItoInt);
1328 static std::optional<Instruction *> instCombineRDFFR(InstCombiner &IC,
1335 auto *PTrue = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptrue,
1338 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_rdffr_z, {}, {PTrue});
1340 return IC.replaceInstUsesWith(II, RDFFR);
1344 instCombineSVECntElts(InstCombiner &IC, IntrinsicInst &II, unsigned NumElts) {
1349 auto *VScale = IC.Builder.CreateVScale(StepVal);
1351 return IC.replaceInstUsesWith(II, VScale);
1357 ? std::optional<Instruction *>(IC.replaceInstUsesWith(
1362 static std::optional<Instruction *> instCombineSVEPTest(InstCombiner &IC,
1376 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptest_any, Tys, Ops);
1379 return IC.replaceInstUsesWith(II, PTest);
1396 auto *PTest = IC.Builder.CreateIntrinsic(II.getIntrinsicID(), Tys, Ops);
1399 return IC.replaceInstUsesWith(II, PTest);
1421 auto *PTest = IC.Builder.CreateIntrinsic(II.getIntrinsicID(), Tys, Ops);
1424 return IC.replaceInstUsesWith(II, PTest);
1432 instCombineSVEVectorFuseMulAddSub(InstCombiner &IC, IntrinsicInst &II,
1465 Res = IC.Builder.CreateIntrinsic(FuseOpc, {II.getType()},
1468 Res = IC.Builder.CreateIntrinsic(FuseOpc, {II.getType()},
1471 return IC.replaceInstUsesWith(II, Res);
1475 instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
1481 if (auto II_NA = instCombineSVENoActiveUnaryZero(IC, II))
1485 LoadInst *Load = IC.Builder.CreateLoad(VecTy, PtrOp);
1487 return IC.replaceInstUsesWith(II, Load);
1491 IC.Builder.CreateMaskedLoad(VecTy, PtrOp, PtrOp->getPointerAlignment(DL),
1494 return IC.replaceInstUsesWith(II, MaskedLoad);
1498 instCombineSVEST1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
1504 StoreInst *Store = IC.Builder.CreateStore(VecOp, PtrOp);
1506 return IC.eraseInstFromFunction(II);
1509 CallInst *MaskedStore = IC.Builder.CreateMaskedStore(
1512 return IC.eraseInstFromFunction(II);
1529 instCombineSVEVectorBinOp(InstCombiner &IC, IntrinsicInst &II) {
1540 IRBuilderBase::FastMathFlagGuard FMFGuard(IC.Builder);
1541 IC.Builder.setFastMathFlags(II.getFastMathFlags());
1543 IC.Builder.CreateBinOp(BinOpCode, II.getOperand(1), II.getOperand(2));
1544 return IC.replaceInstUsesWith(II, BinOp);
1566 instCombineSVEAllOrNoActive(InstCombiner &IC, IntrinsicInst &II,
1571 return IC.replaceInstUsesWith(II, II.getOperand(1));
1576 static std::optional<Instruction *> instCombineSVEVectorAdd(InstCombiner &IC,
1579 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_add_u))
1583 IC, II, true))
1587 IC, II, false))
1593 instCombineSVEVectorFAdd(InstCombiner &IC, IntrinsicInst &II) {
1595 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fadd_u))
1599 Intrinsic::aarch64_sve_fmla>(IC, II,
1604 Intrinsic::aarch64_sve_fmad>(IC, II,
1609 Intrinsic::aarch64_sve_fmla>(IC, II,
1616 instCombineSVEVectorFAddU(InstCombiner &IC, IntrinsicInst &II) {
1619 Intrinsic::aarch64_sve_fmla>(IC, II,
1624 Intrinsic::aarch64_sve_fmad>(IC, II,
1630 IC, II, true))
1632 return instCombineSVEVectorBinOp(IC, II);
1636 instCombineSVEVectorFSub(InstCombiner &IC, IntrinsicInst &II) {
1638 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fsub_u))
1642 Intrinsic::aarch64_sve_fmls>(IC, II,
1648 IC, II, false))
1652 Intrinsic::aarch64_sve_fmls>(IC, II,
1659 instCombineSVEVectorFSubU(InstCombiner &IC, IntrinsicInst &II) {
1662 Intrinsic::aarch64_sve_fmls>(IC, II,
1668 IC, II, false))
1673 IC, II, true))
1675 return instCombineSVEVectorBinOp(IC, II);
1678 static std::optional<Instruction *> instCombineSVEVectorSub(InstCombiner &IC,
1681 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sub_u))
1685 IC, II, true))
1690 static std::optional<Instruction *> instCombineSVEVectorMul(InstCombiner &IC,
1719 return IC.replaceInstUsesWith(II, OpMultiplicand);
1728 return IC.replaceInstUsesWith(II, OpMultiplicand);
1732 return instCombineSVEVectorBinOp(IC, II);
1735 static std::optional<Instruction *> instCombineSVEUnpack(InstCombiner &IC,
1746 IC.Builder.CreateIntCast(ScalarArg, RetTy->getScalarType(), IsSigned);
1748 IC.Builder.CreateVectorSplat(RetTy->getElementCount(), ScalarArg);
1750 return IC.replaceInstUsesWith(II, NewVal);
1755 static std::optional<Instruction *> instCombineSVETBL(InstCombiner &IC,
1770 auto *Extract = IC.Builder.CreateExtractElement(OpVal, SplatValue);
1772 IC.Builder.CreateVectorSplat(VTy->getElementCount(), Extract);
1775 return IC.replaceInstUsesWith(II, VectorSplat);
1778 static std::optional<Instruction *> instCombineSVEUzp1(InstCombiner &IC,
1796 auto *SubVec = IC.Builder.CreateInsertVector(
1797 RetTy, PoisonValue::get(RetTy), A, IC.Builder.getInt64(0));
1798 auto *ConcatVec = IC.Builder.CreateInsertVector(
1799 RetTy, SubVec, B, IC.Builder.getInt64(TyA->getMinNumElements()));
1801 return IC.replaceInstUsesWith(II, ConcatVec);
1808 static std::optional<Instruction *> instCombineSVEZip(InstCombiner &IC,
1817 return IC.replaceInstUsesWith(
1824 instCombineLD1GatherIndex(InstCombiner &IC, IntrinsicInst &II) {
1832 if (auto II_NA = instCombineSVENoActiveUnaryZero(IC, II))
1845 Value *Ptr = IC.Builder.CreateGEP(cast<VectorType>(Ty)->getElementType(),
1847 Ptr = IC.Builder.CreateBitCast(Ptr, VecPtrTy);
1849 IC.Builder.CreateMaskedLoad(Ty, Ptr, Alignment, Mask, PassThru);
1851 return IC.replaceInstUsesWith(II, MaskedLoad);
1858 instCombineST1ScatterIndex(InstCombiner &IC, IntrinsicInst &II) {
1874 Value *Ptr = IC.Builder.CreateGEP(cast<VectorType>(Ty)->getElementType(),
1877 Ptr = IC.Builder.CreateBitCast(Ptr, VecPtrTy);
1879 (void)IC.Builder.CreateMaskedStore(Val, Ptr, Alignment, Mask);
1881 return IC.eraseInstFromFunction(II);
1887 static std::optional<Instruction *> instCombineSVESDIV(InstCombiner &IC,
1889 Type *Int32Ty = IC.Builder.getInt32Ty();
1902 auto ASRD = IC.Builder.CreateIntrinsic(
1904 return IC.replaceInstUsesWith(II, ASRD);
1909 auto ASRD = IC.Builder.CreateIntrinsic(
1911 auto NEG = IC.Builder.CreateIntrinsic(
1913 return IC.replaceInstUsesWith(II, NEG);
1948 static std::optional<Instruction *> instCombineSVEDupqLane(InstCombiner &IC,
1976 InsertEltChain = IC.Builder.CreateInsertElement(InsertEltChain, Elts[I],
1977 IC.Builder.getInt64(I));
1991 IntegerType *WideTy = IC.Builder.getIntNTy(PatternWidth);
1994 ScalableVectorType::get(IC.Builder.getInt32Ty(), PatternElementCount);
1996 auto ZeroIdx = ConstantInt::get(IC.Builder.getInt64Ty(), APInt(64, 0));
1997 auto InsertSubvector = IC.Builder.CreateInsertVector(
2000 IC.Builder.CreateBitOrPointerCast(InsertSubvector, WideScalableTy);
2002 auto WideShuffle = IC.Builder.CreateShuffleVector(
2005 IC.Builder.CreateBitOrPointerCast(WideShuffle, II.getType());
2007 return IC.replaceInstUsesWith(II, NarrowBitcast);
2010 static std::optional<Instruction *> instCombineMaxMinNM(InstCombiner &IC,
2015 return IC.replaceInstUsesWith(II, A);
2020 static std::optional<Instruction *> instCombineSVESrshl(InstCombiner &IC,
2048 auto LSL = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_lsl,
2051 return IC.replaceInstUsesWith(II, LSL);
2055 AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
2078 return instCombineSVENoActiveUnaryErase(IC, II, 1);
2081 return instCombineSVENoActiveUnaryErase(IC, II, 2);
2084 return instCombineSVENoActiveUnaryErase(IC, II, 3);
2087 return instCombineSVENoActiveUnaryErase(IC, II, 4);
2121 return instCombineSVENoActiveUnaryZero(IC, II);
2124 return instCombineMaxMinNM(IC, II);
2126 return instCombineConvertFromSVBool(IC, II);
2128 return instCombineSVEDup(IC, II);
2130 return instCombineSVEDupX(IC, II);
2133 return instCombineSVECmpNE(IC, II);
2135 return instCombineRDFFR(IC, II);
2138 return instCombineSVELast(IC, II);
2141 return instCombineSVECondLast(IC, II);
2143 return instCombineSVECntElts(IC, II, 2);
2145 return instCombineSVECntElts(IC, II, 4);
2147 return instCombineSVECntElts(IC, II, 8);
2149 return instCombineSVECntElts(IC, II, 16);
2153 return instCombineSVEPTest(IC, II);
2155 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fabd_u);
2157 return instCombineSVEVectorFAdd(IC, II);
2159 return instCombineSVEVectorFAddU(IC, II);
2161 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fdiv_u);
2163 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmax_u);
2165 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmaxnm_u);
2167 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmin_u);
2169 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fminnm_u);
2171 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmla_u);
2173 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmls_u);
2176 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmul_u))
2178 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_fmul_u);
2180 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_fmul_u);
2182 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmulx_u);
2184 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fnmla_u);
2186 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fnmls_u);
2188 return instCombineSVEVectorFSub(IC, II);
2190 return instCombineSVEVectorFSubU(IC, II);
2192 return instCombineSVEVectorAdd(IC, II);
2196 IC, II, true);
2198 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mla_u);
2200 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mls_u);
2203 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mul_u))
2205 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_mul_u);
2207 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_mul_u);
2209 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sabd_u);
2211 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smax_u);
2213 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smin_u);
2215 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smulh_u);
2217 return instCombineSVEVectorSub(IC, II);
2221 IC, II, true);
2223 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_uabd_u);
2225 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umax_u);
2227 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umin_u);
2229 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umulh_u);
2231 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_asr_u);
2233 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_lsl_u);
2235 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_lsr_u);
2237 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_and_u);
2239 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_bic_u);
2241 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_eor_u);
2243 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_orr_u);
2245 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sqsub_u);
2247 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_uqsub_u);
2249 return instCombineSVETBL(IC, II);
2254 return instCombineSVEUnpack(IC, II);
2256 return instCombineSVEUzp1(IC, II);
2259 return instCombineSVEZip(IC, II);
2261 return instCombineLD1GatherIndex(IC, II);
2263 return instCombineST1ScatterIndex(IC, II);
2265 return instCombineSVELD1(IC, II, DL);
2267 return instCombineSVEST1(IC, II, DL);
2269 return instCombineSVESDIV(IC, II);
2271 return instCombineSVESel(IC, II);
2273 return instCombineSVESrshl(IC, II);
2275 return instCombineSVEDupqLane(IC, II);
2282 InstCombiner &IC, IntrinsicInst &II, APInt OrigDemandedElts,