Lines Matching defs:Intrinsic
452 AArch64TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
466 if (IID >= Intrinsic::aarch64_addg && IID <= Intrinsic::aarch64_udiv)
472 case Intrinsic::sadd_with_overflow:
473 case Intrinsic::uadd_with_overflow:
474 case Intrinsic::ssub_with_overflow:
475 case Intrinsic::usub_with_overflow:
476 case Intrinsic::smul_with_overflow:
477 case Intrinsic::umul_with_overflow:
486 case Intrinsic::experimental_stackmap:
490 case Intrinsic::experimental_patchpoint_void:
491 case Intrinsic::experimental_patchpoint:
495 case Intrinsic::experimental_gc_statepoint:
554 case Intrinsic::experimental_vector_histogram_add:
558 case Intrinsic::umin:
559 case Intrinsic::umax:
560 case Intrinsic::smin:
561 case Intrinsic::smax: {
574 case Intrinsic::sadd_sat:
575 case Intrinsic::ssub_sat:
576 case Intrinsic::uadd_sat:
577 case Intrinsic::usub_sat: {
590 case Intrinsic::abs: {
599 case Intrinsic::bswap: {
608 case Intrinsic::experimental_stepvector: {
621 case Intrinsic::vector_extract:
622 case Intrinsic::vector_insert: {
635 bool IsExtract = ICA.getID() == Intrinsic::vector_extract;
654 case Intrinsic::bitreverse: {
656 {Intrinsic::bitreverse, MVT::i32, 1},
657 {Intrinsic::bitreverse, MVT::i64, 1},
658 {Intrinsic::bitreverse, MVT::v8i8, 1},
659 {Intrinsic::bitreverse, MVT::v16i8, 1},
660 {Intrinsic::bitreverse, MVT::v4i16, 2},
661 {Intrinsic::bitreverse, MVT::v8i16, 2},
662 {Intrinsic::bitreverse, MVT::v2i32, 2},
663 {Intrinsic::bitreverse, MVT::v4i32, 2},
664 {Intrinsic::bitreverse, MVT::v1i64, 2},
665 {Intrinsic::bitreverse, MVT::v2i64, 2},
681 case Intrinsic::ctpop: {
710 case Intrinsic::sadd_with_overflow:
711 case Intrinsic::uadd_with_overflow:
712 case Intrinsic::ssub_with_overflow:
713 case Intrinsic::usub_with_overflow:
714 case Intrinsic::smul_with_overflow:
715 case Intrinsic::umul_with_overflow: {
717 {Intrinsic::sadd_with_overflow, MVT::i8, 3},
718 {Intrinsic::uadd_with_overflow, MVT::i8, 3},
719 {Intrinsic::sadd_with_overflow, MVT::i16, 3},
720 {Intrinsic::uadd_with_overflow, MVT::i16, 3},
721 {Intrinsic::sadd_with_overflow, MVT::i32, 1},
722 {Intrinsic::uadd_with_overflow, MVT::i32, 1},
723 {Intrinsic::sadd_with_overflow, MVT::i64, 1},
724 {Intrinsic::uadd_with_overflow, MVT::i64, 1},
725 {Intrinsic::ssub_with_overflow, MVT::i8, 3},
726 {Intrinsic::usub_with_overflow, MVT::i8, 3},
727 {Intrinsic::ssub_with_overflow, MVT::i16, 3},
728 {Intrinsic::usub_with_overflow, MVT::i16, 3},
729 {Intrinsic::ssub_with_overflow, MVT::i32, 1},
730 {Intrinsic::usub_with_overflow, MVT::i32, 1},
731 {Intrinsic::ssub_with_overflow, MVT::i64, 1},
732 {Intrinsic::usub_with_overflow, MVT::i64, 1},
733 {Intrinsic::smul_with_overflow, MVT::i8, 5},
734 {Intrinsic::umul_with_overflow, MVT::i8, 4},
735 {Intrinsic::smul_with_overflow, MVT::i16, 5},
736 {Intrinsic::umul_with_overflow, MVT::i16, 4},
737 {Intrinsic::smul_with_overflow, MVT::i32, 2}, // eg umull;tst
738 {Intrinsic::umul_with_overflow, MVT::i32, 2}, // eg umull;cmp sxtw
739 {Intrinsic::smul_with_overflow, MVT::i64, 3}, // eg mul;smulh;cmp
740 {Intrinsic::umul_with_overflow, MVT::i64, 3}, // eg mul;umulh;cmp asr
749 case Intrinsic::fptosi_sat:
750 case Intrinsic::fptoui_sat: {
753 bool IsSigned = ICA.getID() == Intrinsic::fptosi_sat;
782 IntrinsicCostAttributes Attrs1(IsSigned ? Intrinsic::smin : Intrinsic::umin,
785 IntrinsicCostAttributes Attrs2(IsSigned ? Intrinsic::smax : Intrinsic::umax,
792 case Intrinsic::fshl:
793 case Intrinsic::fshr: {
806 {Intrinsic::fshl, MVT::v4i32, 3}, // ushr + shl + orr
807 {Intrinsic::fshl, MVT::v2i64, 3}, {Intrinsic::fshl, MVT::v16i8, 4},
808 {Intrinsic::fshl, MVT::v8i16, 4}, {Intrinsic::fshl, MVT::v2i32, 3},
809 {Intrinsic::fshl, MVT::v8i8, 4}, {Intrinsic::fshl, MVT::v4i16, 4}};
810 // Costs for both fshl & fshr are the same, so just pass Intrinsic::fshl
813 CostTableLookup(FshlTbl, Intrinsic::fshl, LegalisationCost.second);
838 case Intrinsic::get_active_lane_mask: {
883 Intrinsic::aarch64_sve_convert_to_svbool ||
920 case Intrinsic::aarch64_sve_and_z:
921 case Intrinsic::aarch64_sve_bic_z:
922 case Intrinsic::aarch64_sve_eor_z:
923 case Intrinsic::aarch64_sve_nand_z:
924 case Intrinsic::aarch64_sve_nor_z:
925 case Intrinsic::aarch64_sve_orn_z:
926 case Intrinsic::aarch64_sve_orr_z:
938 PredIntr->getIntrinsicID() != Intrinsic::aarch64_sve_convert_to_svbool)
948 Intrinsic::aarch64_sve_convert_from_svbool, {PredOpTy}, {BinOpOp1});
954 Intrinsic::aarch64_sve_convert_from_svbool, {PredOpTy}, {BinOpOp2}));
997 Intrinsic::aarch64_sve_convert_to_svbool ||
999 Intrinsic::aarch64_sve_convert_from_svbool))
1017 if (match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_convert_from_svbool>(
1018 m_Intrinsic<Intrinsic::aarch64_sve_convert_to_svbool>(
1026 return match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>(
1085 if (Pg->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue)
1119 if (!Pg || Pg->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue)
1136 DupQLane->getIntrinsicID() != Intrinsic::aarch64_sve_dupq_lane)
1144 if (!VecIns || VecIns->getIntrinsicID() != Intrinsic::vector_insert)
1200 auto *PTrue = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptrue,
1203 Intrinsic::aarch64_sve_convert_to_svbool, {PredType}, {PTrue});
1205 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool,
1217 bool IsAfter = IntrinsicID == Intrinsic::aarch64_sve_lasta;
1254 if (IntrPG->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue)
1335 auto *PTrue = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptrue,
1338 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_rdffr_z, {}, {PTrue});
1370 (II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_first ||
1371 II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_last)) {
1376 IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptest_any, Tys, Ops);
1388 Intrinsic::ID OpIID = Op->getIntrinsicID();
1390 if (Pg->getIntrinsicID() == Intrinsic::aarch64_sve_convert_to_svbool &&
1391 OpIID == Intrinsic::aarch64_sve_convert_to_svbool &&
1405 if ((Pg == Op) && (II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_any) &&
1406 ((OpIID == Intrinsic::aarch64_sve_brka_z) ||
1407 (OpIID == Intrinsic::aarch64_sve_brkb_z) ||
1408 (OpIID == Intrinsic::aarch64_sve_brkpa_z) ||
1409 (OpIID == Intrinsic::aarch64_sve_brkpb_z) ||
1410 (OpIID == Intrinsic::aarch64_sve_rdffr_z) ||
1411 (OpIID == Intrinsic::aarch64_sve_and_z) ||
1412 (OpIID == Intrinsic::aarch64_sve_bic_z) ||
1413 (OpIID == Intrinsic::aarch64_sve_eor_z) ||
1414 (OpIID == Intrinsic::aarch64_sve_nand_z) ||
1415 (OpIID == Intrinsic::aarch64_sve_nor_z) ||
1416 (OpIID == Intrinsic::aarch64_sve_orn_z) ||
1417 (OpIID == Intrinsic::aarch64_sve_orr_z))) {
1430 template <Intrinsic::ID MulOpc, typename Intrinsic::ID FuseOpc>
1515 static Instruction::BinaryOps intrinsicIDToBinOpCode(unsigned Intrinsic) {
1516 switch (Intrinsic) {
1517 case Intrinsic::aarch64_sve_fmul_u:
1519 case Intrinsic::aarch64_sve_fadd_u:
1521 case Intrinsic::aarch64_sve_fsub_u:
1537 !match(OpPredicate, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>(
1550 Intrinsic::ID IID) {
1552 if (!match(OpPredicate, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>(
1557 auto *NewDecl = Intrinsic::getDeclaration(Mod, IID, {II.getType()});
1567 Intrinsic::ID IID) {
1579 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_add_u))
1581 if (auto MLA = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
1582 Intrinsic::aarch64_sve_mla>(
1585 if (auto MAD = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
1586 Intrinsic::aarch64_sve_mad>(
1595 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fadd_u))
1598 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
1599 Intrinsic::aarch64_sve_fmla>(IC, II,
1603 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
1604 Intrinsic::aarch64_sve_fmad>(IC, II,
1608 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
1609 Intrinsic::aarch64_sve_fmla>(IC, II,
1618 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
1619 Intrinsic::aarch64_sve_fmla>(IC, II,
1623 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
1624 Intrinsic::aarch64_sve_fmad>(IC, II,
1628 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
1629 Intrinsic::aarch64_sve_fmla_u>(
1638 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fsub_u))
1641 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
1642 Intrinsic::aarch64_sve_fmls>(IC, II,
1646 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
1647 Intrinsic::aarch64_sve_fnmsb>(
1651 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
1652 Intrinsic::aarch64_sve_fmls>(IC, II,
1661 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
1662 Intrinsic::aarch64_sve_fmls>(IC, II,
1666 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
1667 Intrinsic::aarch64_sve_fnmsb>(
1671 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
1672 Intrinsic::aarch64_sve_fmls_u>(
1681 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sub_u))
1683 if (auto MLS = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
1684 Intrinsic::aarch64_sve_mls>(
1692 Intrinsic::ID IID) {
1709 if (!IntrI || IntrI->getIntrinsicID() != Intrinsic::aarch64_sve_dup)
1739 bool IsSigned = II.getIntrinsicID() == Intrinsic::aarch64_sve_sunpkhi ||
1740 II.getIntrinsicID() == Intrinsic::aarch64_sve_sunpklo;
1782 constexpr Intrinsic::ID FromSVB = Intrinsic::aarch64_sve_convert_from_svbool;
1783 constexpr Intrinsic::ID ToSVB = Intrinsic::aarch64_sve_convert_to_svbool;
1814 m_Intrinsic<Intrinsic::aarch64_sve_uzp1>(m_Value(A), m_Value(B))) &&
1815 match(II.getArgOperand(1), m_Intrinsic<Intrinsic::aarch64_sve_uzp2>(
1818 II, (II.getIntrinsicID() == Intrinsic::aarch64_sve_zip1 ? A : B));
1839 if (match(Index, m_Intrinsic<Intrinsic::aarch64_sve_index>(
1869 if (match(Index, m_Intrinsic<Intrinsic::aarch64_sve_index>(
1903 Intrinsic::aarch64_sve_asrd, {II.getType()}, {Pred, Vec, DivisorLog2});
1910 Intrinsic::aarch64_sve_asrd, {II.getType()}, {Pred, Vec, DivisorLog2});
1912 Intrinsic::aarch64_sve_neg, {ASRD->getType()}, {ASRD, Pred, ASRD});
1952 m_Intrinsic<Intrinsic::vector_insert>(
2028 if (!match(Vec, m_Intrinsic<Intrinsic::aarch64_sve_sqabs>(
2030 !match(Vec, m_Intrinsic<Intrinsic::aarch64_sve_abs>(
2048 auto LSL = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_lsl,
2057 Intrinsic::ID IID = II.getIntrinsicID();
2062 case Intrinsic::aarch64_sve_st1_scatter:
2063 case Intrinsic::aarch64_sve_st1_scatter_scalar_offset:
2064 case Intrinsic::aarch64_sve_st1_scatter_sxtw:
2065 case Intrinsic::aarch64_sve_st1_scatter_sxtw_index:
2066 case Intrinsic::aarch64_sve_st1_scatter_uxtw:
2067 case Intrinsic::aarch64_sve_st1_scatter_uxtw_index:
2068 case Intrinsic::aarch64_sve_st1dq:
2069 case Intrinsic::aarch64_sve_st1q_scatter_index:
2070 case Intrinsic::aarch64_sve_st1q_scatter_scalar_offset:
2071 case Intrinsic::aarch64_sve_st1q_scatter_vector_offset:
2072 case Intrinsic::aarch64_sve_st1wq:
2073 case Intrinsic::aarch64_sve_stnt1:
2074 case Intrinsic::aarch64_sve_stnt1_scatter:
2075 case Intrinsic::aarch64_sve_stnt1_scatter_index:
2076 case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset:
2077 case Intrinsic::aarch64_sve_stnt1_scatter_uxtw:
2079 case Intrinsic::aarch64_sve_st2:
2080 case Intrinsic::aarch64_sve_st2q:
2082 case Intrinsic::aarch64_sve_st3:
2083 case Intrinsic::aarch64_sve_st3q:
2085 case Intrinsic::aarch64_sve_st4:
2086 case Intrinsic::aarch64_sve_st4q:
2088 case Intrinsic::aarch64_sve_ld1_gather:
2089 case Intrinsic::aarch64_sve_ld1_gather_scalar_offset:
2090 case Intrinsic::aarch64_sve_ld1_gather_sxtw:
2091 case Intrinsic::aarch64_sve_ld1_gather_sxtw_index:
2092 case Intrinsic::aarch64_sve_ld1_gather_uxtw:
2093 case Intrinsic::aarch64_sve_ld1_gather_uxtw_index:
2094 case Intrinsic::aarch64_sve_ld1q_gather_index:
2095 case Intrinsic::aarch64_sve_ld1q_gather_scalar_offset:
2096 case Intrinsic::aarch64_sve_ld1q_gather_vector_offset:
2097 case Intrinsic::aarch64_sve_ld1ro:
2098 case Intrinsic::aarch64_sve_ld1rq:
2099 case Intrinsic::aarch64_sve_ld1udq:
2100 case Intrinsic::aarch64_sve_ld1uwq:
2101 case Intrinsic::aarch64_sve_ld2_sret:
2102 case Intrinsic::aarch64_sve_ld2q_sret:
2103 case Intrinsic::aarch64_sve_ld3_sret:
2104 case Intrinsic::aarch64_sve_ld3q_sret:
2105 case Intrinsic::aarch64_sve_ld4_sret:
2106 case Intrinsic::aarch64_sve_ld4q_sret:
2107 case Intrinsic::aarch64_sve_ldff1:
2108 case Intrinsic::aarch64_sve_ldff1_gather:
2109 case Intrinsic::aarch64_sve_ldff1_gather_index:
2110 case Intrinsic::aarch64_sve_ldff1_gather_scalar_offset:
2111 case Intrinsic::aarch64_sve_ldff1_gather_sxtw:
2112 case Intrinsic::aarch64_sve_ldff1_gather_sxtw_index:
2113 case Intrinsic::aarch64_sve_ldff1_gather_uxtw:
2114 case Intrinsic::aarch64_sve_ldff1_gather_uxtw_index:
2115 case Intrinsic::aarch64_sve_ldnf1:
2116 case Intrinsic::aarch64_sve_ldnt1:
2117 case Intrinsic::aarch64_sve_ldnt1_gather:
2118 case Intrinsic::aarch64_sve_ldnt1_gather_index:
2119 case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
2120 case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
2122 case Intrinsic::aarch64_neon_fmaxnm:
2123 case Intrinsic::aarch64_neon_fminnm:
2125 case Intrinsic::aarch64_sve_convert_from_svbool:
2127 case Intrinsic::aarch64_sve_dup:
2129 case Intrinsic::aarch64_sve_dup_x:
2131 case Intrinsic::aarch64_sve_cmpne:
2132 case Intrinsic::aarch64_sve_cmpne_wide:
2134 case Intrinsic::aarch64_sve_rdffr:
2136 case Intrinsic::aarch64_sve_lasta:
2137 case Intrinsic::aarch64_sve_lastb:
2139 case Intrinsic::aarch64_sve_clasta_n:
2140 case Intrinsic::aarch64_sve_clastb_n:
2142 case Intrinsic::aarch64_sve_cntd:
2144 case Intrinsic::aarch64_sve_cntw:
2146 case Intrinsic::aarch64_sve_cnth:
2148 case Intrinsic::aarch64_sve_cntb:
2150 case Intrinsic::aarch64_sve_ptest_any:
2151 case Intrinsic::aarch64_sve_ptest_first:
2152 case Intrinsic::aarch64_sve_ptest_last:
2154 case Intrinsic::aarch64_sve_fabd:
2155 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fabd_u);
2156 case Intrinsic::aarch64_sve_fadd:
2158 case Intrinsic::aarch64_sve_fadd_u:
2160 case Intrinsic::aarch64_sve_fdiv:
2161 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fdiv_u);
2162 case Intrinsic::aarch64_sve_fmax:
2163 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmax_u);
2164 case Intrinsic::aarch64_sve_fmaxnm:
2165 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmaxnm_u);
2166 case Intrinsic::aarch64_sve_fmin:
2167 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmin_u);
2168 case Intrinsic::aarch64_sve_fminnm:
2169 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fminnm_u);
2170 case Intrinsic::aarch64_sve_fmla:
2171 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmla_u);
2172 case Intrinsic::aarch64_sve_fmls:
2173 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmls_u);
2174 case Intrinsic::aarch64_sve_fmul:
2176 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmul_u))
2178 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_fmul_u);
2179 case Intrinsic::aarch64_sve_fmul_u:
2180 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_fmul_u);
2181 case Intrinsic::aarch64_sve_fmulx:
2182 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmulx_u);
2183 case Intrinsic::aarch64_sve_fnmla:
2184 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fnmla_u);
2185 case Intrinsic::aarch64_sve_fnmls:
2186 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fnmls_u);
2187 case Intrinsic::aarch64_sve_fsub:
2189 case Intrinsic::aarch64_sve_fsub_u:
2191 case Intrinsic::aarch64_sve_add:
2193 case Intrinsic::aarch64_sve_add_u:
2194 return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u,
2195 Intrinsic::aarch64_sve_mla_u>(
2197 case Intrinsic::aarch64_sve_mla:
2198 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mla_u);
2199 case Intrinsic::aarch64_sve_mls:
2200 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mls_u);
2201 case Intrinsic::aarch64_sve_mul:
2203 instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mul_u))
2205 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_mul_u);
2206 case Intrinsic::aarch64_sve_mul_u:
2207 return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_mul_u);
2208 case Intrinsic::aarch64_sve_sabd:
2209 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sabd_u);
2210 case Intrinsic::aarch64_sve_smax:
2211 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smax_u);
2212 case Intrinsic::aarch64_sve_smin:
2213 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smin_u);
2214 case Intrinsic::aarch64_sve_smulh:
2215 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smulh_u);
2216 case Intrinsic::aarch64_sve_sub:
2218 case Intrinsic::aarch64_sve_sub_u:
2219 return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u,
2220 Intrinsic::aarch64_sve_mls_u>(
2222 case Intrinsic::aarch64_sve_uabd:
2223 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_uabd_u);
2224 case Intrinsic::aarch64_sve_umax:
2225 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umax_u);
2226 case Intrinsic::aarch64_sve_umin:
2227 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umin_u);
2228 case Intrinsic::aarch64_sve_umulh:
2229 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umulh_u);
2230 case Intrinsic::aarch64_sve_asr:
2231 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_asr_u);
2232 case Intrinsic::aarch64_sve_lsl:
2233 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_lsl_u);
2234 case Intrinsic::aarch64_sve_lsr:
2235 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_lsr_u);
2236 case Intrinsic::aarch64_sve_and:
2237 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_and_u);
2238 case Intrinsic::aarch64_sve_bic:
2239 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_bic_u);
2240 case Intrinsic::aarch64_sve_eor:
2241 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_eor_u);
2242 case Intrinsic::aarch64_sve_orr:
2243 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_orr_u);
2244 case Intrinsic::aarch64_sve_sqsub:
2245 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sqsub_u);
2246 case Intrinsic::aarch64_sve_uqsub:
2247 return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_uqsub_u);
2248 case Intrinsic::aarch64_sve_tbl:
2250 case Intrinsic::aarch64_sve_uunpkhi:
2251 case Intrinsic::aarch64_sve_uunpklo:
2252 case Intrinsic::aarch64_sve_sunpkhi:
2253 case Intrinsic::aarch64_sve_sunpklo:
2255 case Intrinsic::aarch64_sve_uzp1:
2257 case Intrinsic::aarch64_sve_zip1:
2258 case Intrinsic::aarch64_sve_zip2:
2260 case Intrinsic::aarch64_sve_ld1_gather_index:
2262 case Intrinsic::aarch64_sve_st1_scatter_index:
2264 case Intrinsic::aarch64_sve_ld1:
2266 case Intrinsic::aarch64_sve_st1:
2268 case Intrinsic::aarch64_sve_sdiv:
2270 case Intrinsic::aarch64_sve_sel:
2272 case Intrinsic::aarch64_sve_srshl:
2274 case Intrinsic::aarch64_sve_dupq_lane:
2289 case Intrinsic::aarch64_neon_fcvtxn:
2290 case Intrinsic::aarch64_neon_rshrn:
2291 case Intrinsic::aarch64_neon_sqrshrn:
2292 case Intrinsic::aarch64_neon_sqrshrun:
2293 case Intrinsic::aarch64_neon_sqshrn:
2294 case Intrinsic::aarch64_neon_sqshrun:
2295 case Intrinsic::aarch64_neon_sqxtn:
2296 case Intrinsic::aarch64_neon_sqxtun:
2297 case Intrinsic::aarch64_neon_uqrshrn:
2298 case Intrinsic::aarch64_neon_uqshrn:
2299 case Intrinsic::aarch64_neon_uqxtn:
3680 case Intrinsic::aarch64_neon_st2:
3681 case Intrinsic::aarch64_neon_st3:
3682 case Intrinsic::aarch64_neon_st4: {
3702 case Intrinsic::aarch64_neon_ld2:
3703 case Intrinsic::aarch64_neon_ld3:
3704 case Intrinsic::aarch64_neon_ld4:
3716 case Intrinsic::aarch64_neon_ld2:
3717 case Intrinsic::aarch64_neon_ld3:
3718 case Intrinsic::aarch64_neon_ld4:
3723 case Intrinsic::aarch64_neon_st2:
3724 case Intrinsic::aarch64_neon_st3:
3725 case Intrinsic::aarch64_neon_st4:
3735 case Intrinsic::aarch64_neon_ld2:
3736 case Intrinsic::aarch64_neon_st2:
3739 case Intrinsic::aarch64_neon_ld3:
3740 case Intrinsic::aarch64_neon_st3:
3743 case Intrinsic::aarch64_neon_ld4:
3744 case Intrinsic::aarch64_neon_st4:
3814 AArch64TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,