Lines Matching defs:IC
951 static std::optional<Instruction *> processPhiNode(InstCombiner &IC,
973 IC.Builder.SetInsertPoint(PN);
974 PHINode *NPN = IC.Builder.CreatePHI(RequiredType, PN->getNumIncomingValues());
984 return IC.replaceInstUsesWith(II, NPN);
997 tryCombineFromSVBoolBinOp(InstCombiner &IC, IntrinsicInst &II) {
1031 auto NarrowBinOpOp1 = IC.Builder.CreateIntrinsic(
1037 NarrowedBinOpArgs.push_back(IC.Builder.CreateIntrinsic(
1041 IC.Builder.CreateIntrinsic(IntrinsicID, {PredOpTy}, NarrowedBinOpArgs);
1042 return IC.replaceInstUsesWith(II, NarrowedBinOp);
1046 instCombineConvertFromSVBool(InstCombiner &IC, IntrinsicInst &II) {
1049 return processPhiNode(IC, II);
1051 if (auto BinOpCombine = tryCombineFromSVBoolBinOp(IC, II))
1095 return IC.replaceInstUsesWith(II, EarliestReplacement);
1117 instCombineSVENoActiveReplace(InstCombiner &IC, IntrinsicInst &II,
1122 IC.replaceInstUsesWith(II, II.getOperand(ReplaceOperand));
1123 return IC.eraseInstFromFunction(II);
1131 instCombineSVEAllOrNoActiveUnary(InstCombiner &IC, IntrinsicInst &II) {
1136 return IC.replaceOperand(II, 0, Undef);
1138 return instCombineSVENoActiveReplace(IC, II, true);
1143 instCombineSVENoActiveUnaryErase(InstCombiner &IC, IntrinsicInst &II,
1146 return IC.eraseInstFromFunction(II);
1154 instCombineSVENoActiveZero(InstCombiner &IC, IntrinsicInst &II) {
1171 IC.replaceInstUsesWith(II, Node);
1172 return IC.eraseInstFromFunction(II);
1177 static std::optional<Instruction *> instCombineSVESel(InstCombiner &IC,
1182 return IC.replaceInstUsesWith(II, II.getOperand(1));
1185 IC.Builder.CreateSelect(OpPredicate, II.getOperand(1), II.getOperand(2));
1186 return IC.replaceInstUsesWith(II, Select);
1189 static std::optional<Instruction *> instCombineSVEDup(InstCombiner &IC,
1210 return IC.replaceInstUsesWith(II, Insert);
1213 static std::optional<Instruction *> instCombineSVEDupX(InstCombiner &IC,
1217 Value *Splat = IC.Builder.CreateVectorSplat(RetTy->getElementCount(),
1220 return IC.replaceInstUsesWith(II, Splat);
1223 static std::optional<Instruction *> instCombineSVECmpNE(InstCombiner &IC,
1228 if (auto II_NA = instCombineSVENoActiveZero(IC, II))
1295 return IC.replaceInstUsesWith(II, PFalse);
1315 auto *PTrue = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptrue,
1317 auto *ConvertToSVBool = IC.Builder.CreateIntrinsic(
1320 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool,
1324 return IC.replaceInstUsesWith(II, ConvertFromSVBool);
1327 static std::optional<Instruction *> instCombineSVELast(InstCombiner &IC,
1336 return IC.replaceInstUsesWith(II, SplatVal);
1346 IC.Builder.CreateIntrinsic(IntrinsicID, {Vec->getType()}, {Pg, LHS});
1348 IC.Builder.CreateIntrinsic(IntrinsicID, {Vec->getType()}, {Pg, RHS});
1351 return IC.replaceInstUsesWith(II, NewBinOp);
1362 return IC.replaceInstUsesWith(II, Extract);
1398 return IC.replaceInstUsesWith(II, Extract);
1401 static std::optional<Instruction *> instCombineSVECondLast(InstCombiner &IC,
1423 FPTy = IC.Builder.getHalfTy();
1426 FPTy = IC.Builder.getFloatTy();
1429 FPTy = IC.Builder.getDoubleTy();
1433 Value *FPFallBack = IC.Builder.CreateBitCast(Fallback, FPTy);
1436 Value *FPVec = IC.Builder.CreateBitCast(Vec, FPVTy);
1437 auto *FPII = IC.Builder.CreateIntrinsic(
1439 Value *FPIItoInt = IC.Builder.CreateBitCast(FPII, II.getType());
1440 return IC.replaceInstUsesWith(II, FPIItoInt);
1443 static std::optional<Instruction *> instCombineRDFFR(InstCombiner &IC,
1450 auto *PTrue = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptrue,
1453 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_rdffr_z, {}, {PTrue});
1455 return IC.replaceInstUsesWith(II, RDFFR);
1459 instCombineSVECntElts(InstCombiner &IC, IntrinsicInst &II, unsigned NumElts) {
1464 auto *VScale = IC.Builder.CreateVScale(StepVal);
1466 return IC.replaceInstUsesWith(II, VScale);
1472 ? std::optional<Instruction *>(IC.replaceInstUsesWith(
1477 static std::optional<Instruction *> instCombineSVEPTest(InstCombiner &IC,
1491 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptest_any, Tys, Ops);
1494 return IC.replaceInstUsesWith(II, PTest);
1511 auto *PTest = IC.Builder.CreateIntrinsic(II.getIntrinsicID(), Tys, Ops);
1514 return IC.replaceInstUsesWith(II, PTest);
1536 auto *PTest = IC.Builder.CreateIntrinsic(II.getIntrinsicID(), Tys, Ops);
1539 return IC.replaceInstUsesWith(II, PTest);
1547 instCombineSVEVectorFuseMulAddSub(InstCombiner &IC, IntrinsicInst &II,
1580 Res = IC.Builder.CreateIntrinsic(FuseOpc, {II.getType()},
1583 Res = IC.Builder.CreateIntrinsic(FuseOpc, {II.getType()},
1586 return IC.replaceInstUsesWith(II, Res);
1590 instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
1596 if (auto II_NA = instCombineSVENoActiveZero(IC, II))
1600 LoadInst *Load = IC.Builder.CreateLoad(VecTy, PtrOp);
1602 return IC.replaceInstUsesWith(II, Load);
1606 IC.Builder.CreateMaskedLoad(VecTy, PtrOp, PtrOp->getPointerAlignment(DL),
1609 return IC.replaceInstUsesWith(II, MaskedLoad);
1613 instCombineSVEST1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
1619 StoreInst *Store = IC.Builder.CreateStore(VecOp, PtrOp);
1621 return IC.eraseInstFromFunction(II);
1624 CallInst *MaskedStore = IC.Builder.CreateMaskedStore(
1627 return IC.eraseInstFromFunction(II);
1644 instCombineSVEVectorBinOp(InstCombiner &IC, IntrinsicInst &II) {
1655 auto BinOp = IC.Builder.CreateBinOpFMF(
1657 return IC.replaceInstUsesWith(II, BinOp);
1679 instCombineSVEAllOrNoActive(InstCombiner &IC, IntrinsicInst &II,
1684 return IC.replaceInstUsesWith(II, II.getOperand(1));
1689 static std::optional<Instruction *> instCombineSVEVectorAdd(InstCombiner &IC,
1692 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_add_u))
1696 IC, II, true))
1700 IC, II, false))
1706 instCombineSVEVectorFAdd(InstCombiner &IC, IntrinsicInst &II) {
1708 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fadd_u))
1712 Intrinsic::aarch64_sve_fmla>(IC, II,
1717 Intrinsic::aarch64_sve_fmad>(IC, II,
1722 Intrinsic::aarch64_sve_fmla>(IC, II,
1729 instCombineSVEVectorFAddU(InstCombiner &IC, IntrinsicInst &II) {
1732 Intrinsic::aarch64_sve_fmla>(IC, II,
1737 Intrinsic::aarch64_sve_fmad>(IC, II,
1743 IC, II, true))
1745 return instCombineSVEVectorBinOp(IC, II);
1749 instCombineSVEVectorFSub(InstCombiner &IC, IntrinsicInst &II) {
1751 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fsub_u))
1755 Intrinsic::aarch64_sve_fmls>(IC, II,
1761 IC, II, false))
1765 Intrinsic::aarch64_sve_fmls>(IC, II,
1772 instCombineSVEVectorFSubU(InstCombiner &IC, IntrinsicInst &II) {
1775 Intrinsic::aarch64_sve_fmls>(IC, II,
1781 IC, II, false))
1786 IC, II, true))
1788 return instCombineSVEVectorBinOp(IC, II);
1791 static std::optional<Instruction *> instCombineSVEVectorSub(InstCombiner &IC,
1794 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sub_u))
1798 IC, II, true))
1803 static std::optional<Instruction *> instCombineSVEVectorMul(InstCombiner &IC,
1832 return IC.replaceInstUsesWith(II, OpMultiplicand);
1841 return IC.replaceInstUsesWith(II, OpMultiplicand);
1845 return instCombineSVEVectorBinOp(IC, II);
1848 static std::optional<Instruction *> instCombineSVEUnpack(InstCombiner &IC,
1859 IC.Builder.CreateIntCast(ScalarArg, RetTy->getScalarType(), IsSigned);
1861 IC.Builder.CreateVectorSplat(RetTy->getElementCount(), ScalarArg);
1863 return IC.replaceInstUsesWith(II, NewVal);
1868 static std::optional<Instruction *> instCombineSVETBL(InstCombiner &IC,
1883 auto *Extract = IC.Builder.CreateExtractElement(OpVal, SplatValue);
1885 IC.Builder.CreateVectorSplat(VTy->getElementCount(), Extract);
1888 return IC.replaceInstUsesWith(II, VectorSplat);
1891 static std::optional<Instruction *> instCombineSVEUzp1(InstCombiner &IC,
1909 auto *SubVec = IC.Builder.CreateInsertVector(
1910 RetTy, PoisonValue::get(RetTy), A, IC.Builder.getInt64(0));
1911 auto *ConcatVec = IC.Builder.CreateInsertVector(
1912 RetTy, SubVec, B, IC.Builder.getInt64(TyA->getMinNumElements()));
1914 return IC.replaceInstUsesWith(II, ConcatVec);
1921 static std::optional<Instruction *> instCombineSVEZip(InstCombiner &IC,
1930 return IC.replaceInstUsesWith(
1937 instCombineLD1GatherIndex(InstCombiner &IC, IntrinsicInst &II) {
1945 if (auto II_NA = instCombineSVENoActiveZero(IC, II))
1957 Value *Ptr = IC.Builder.CreateGEP(cast<VectorType>(Ty)->getElementType(),
1960 IC.Builder.CreateMaskedLoad(Ty, Ptr, Alignment, Mask, PassThru);
1962 return IC.replaceInstUsesWith(II, MaskedLoad);
1969 instCombineST1ScatterIndex(InstCombiner &IC, IntrinsicInst &II) {
1985 Value *Ptr = IC.Builder.CreateGEP(cast<VectorType>(Ty)->getElementType(),
1987 (void)IC.Builder.CreateMaskedStore(Val, Ptr, Alignment, Mask);
1989 return IC.eraseInstFromFunction(II);
1995 static std::optional<Instruction *> instCombineSVESDIV(InstCombiner &IC,
1997 Type *Int32Ty = IC.Builder.getInt32Ty();
2012 IC.replaceInstUsesWith(II, Vec);
2016 auto ASRD = IC.Builder.CreateIntrinsic(
2018 return IC.replaceInstUsesWith(II, ASRD);
2023 auto ASRD = IC.Builder.CreateIntrinsic(
2025 auto NEG = IC.Builder.CreateIntrinsic(
2027 return IC.replaceInstUsesWith(II, NEG);
2062 static std::optional<Instruction *> instCombineSVEDupqLane(InstCombiner &IC,
2090 InsertEltChain = IC.Builder.CreateInsertElement(InsertEltChain, Elts[I],
2091 IC.Builder.getInt64(I));
2105 IntegerType *WideTy = IC.Builder.getIntNTy(PatternWidth);
2108 ScalableVectorType::get(IC.Builder.getInt32Ty(), PatternElementCount);
2110 auto ZeroIdx = ConstantInt::get(IC.Builder.getInt64Ty(), APInt(64, 0));
2111 auto InsertSubvector = IC.Builder.CreateInsertVector(
2114 IC.Builder.CreateBitOrPointerCast(InsertSubvector, WideScalableTy);
2116 auto WideShuffle = IC.Builder.CreateShuffleVector(
2119 IC.Builder.CreateBitOrPointerCast(WideShuffle, II.getType());
2121 return IC.replaceInstUsesWith(II, NarrowBitcast);
2124 static std::optional<Instruction *> instCombineMaxMinNM(InstCombiner &IC,
2129 return IC.replaceInstUsesWith(II, A);
2134 static std::optional<Instruction *> instCombineSVESrshl(InstCombiner &IC,
2162 auto LSL = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_lsl,
2165 return IC.replaceInstUsesWith(II, LSL);
2168 static std::optional<Instruction *> instCombineSVEInsr(InstCombiner &IC,
2173 return IC.replaceInstUsesWith(II, Vec);
2178 static std::optional<Instruction *> instCombineDMB(InstCombiner &IC,
2198 return IC.eraseInstFromFunction(II);
2204 AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
2211 return instCombineDMB(IC, II);
2242 return instCombineSVEAllOrNoActiveUnary(IC, II);
2247 return instCombineSVENoActiveReplace(IC, II, true);
2264 return instCombineSVENoActiveUnaryErase(IC, II, 1);
2267 return instCombineSVENoActiveUnaryErase(IC, II, 2);
2270 return instCombineSVENoActiveUnaryErase(IC, II, 3);
2273 return instCombineSVENoActiveUnaryErase(IC, II, 4);
2353 return instCombineSVENoActiveZero(IC, II);
2371 return instCombineSVENoActiveUnaryErase(IC, II, 0);
2374 return instCombineMaxMinNM(IC, II);
2376 return instCombineConvertFromSVBool(IC, II);
2378 return instCombineSVEDup(IC, II);
2380 return instCombineSVEDupX(IC, II);
2383 return instCombineSVECmpNE(IC, II);
2385 return instCombineRDFFR(IC, II);
2388 return instCombineSVELast(IC, II);
2391 return instCombineSVECondLast(IC, II);
2393 return instCombineSVECntElts(IC, II, 2);
2395 return instCombineSVECntElts(IC, II, 4);
2397 return instCombineSVECntElts(IC, II, 8);
2399 return instCombineSVECntElts(IC, II, 16);
2403 return instCombineSVEPTest(IC, II);
2405 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fabd_u);
2407 return instCombineSVEVectorFAdd(IC, II);
2409 return instCombineSVEVectorFAddU(IC, II);
2411 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fdiv_u);
2413 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmax_u);
2415 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmaxnm_u);
2417 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmin_u);
2419 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fminnm_u);
2421 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmla_u);
2423 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmls_u);
2426 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmul_u))
2428 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_fmul_u);
2430 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_fmul_u);
2432 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmulx_u);
2434 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fnmla_u);
2436 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fnmls_u);
2438 return instCombineSVEVectorFSub(IC, II);
2440 return instCombineSVEVectorFSubU(IC, II);
2442 return instCombineSVEVectorAdd(IC, II);
2446 IC, II, true);
2448 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mla_u);
2450 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mls_u);
2453 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mul_u))
2455 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_mul_u);
2457 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_mul_u);
2459 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sabd_u);
2461 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smax_u);
2463 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smin_u);
2465 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smulh_u);
2467 return instCombineSVEVectorSub(IC, II);
2471 IC, II, true);
2473 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_uabd_u);
2475 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umax_u);
2477 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umin_u);
2479 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umulh_u);
2481 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_asr_u);
2483 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_lsl_u);
2485 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_lsr_u);
2487 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_and_u);
2489 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_bic_u);
2491 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_eor_u);
2493 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_orr_u);
2495 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sqsub_u);
2497 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_uqsub_u);
2499 return instCombineSVETBL(IC, II);
2504 return instCombineSVEUnpack(IC, II);
2506 return instCombineSVEUzp1(IC, II);
2509 return instCombineSVEZip(IC, II);
2511 return instCombineLD1GatherIndex(IC, II);
2513 return instCombineST1ScatterIndex(IC, II);
2515 return instCombineSVELD1(IC, II, DL);
2517 return instCombineSVEST1(IC, II, DL);
2519 return instCombineSVESDIV(IC, II);
2521 return instCombineSVESel(IC, II);
2523 return instCombineSVESrshl(IC, II);
2525 return instCombineSVEDupqLane(IC, II);
2527 return instCombineSVEInsr(IC, II);
2534 InstCombiner &IC, IntrinsicInst &II, APInt OrigDemandedElts,