Lines Matching defs:SDValue
281 static bool isZeroingInactiveLanes(SDValue Op) {
342 static std::tuple<SDValue, SDValue>
343 extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) {
345 SDValue AddrDisc;
346 SDValue ConstDisc;
2167 static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm,
2243 SDValue New;
2254 SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
2255 New = SDValue(
2263 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
2308 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
2314 SDValue SrcOp = Op.getOperand(0);
2435 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
3226 static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
3227 static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
3228 static SDValue convertFixedMaskToScalableVector(SDValue Mask,
3230 static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT);
3231 static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
3412 static bool cannotBeIntMin(SDValue CheckedVal, SelectionDAG &DAG) {
3427 static bool isCMN(SDValue Op, ISD::CondCode CC, SelectionDAG &DAG) {
3434 static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &dl,
3435 SelectionDAG &DAG, SDValue Chain,
3455 static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
3491 const SDValue ANDSNode = DAG.getNode(AArch64ISD::ANDS, dl,
3561 static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
3562 ISD::CondCode CC, SDValue CCOp,
3596 SDValue Condition = DAG.getConstant(Predicate, DL, MVT_CC);
3599 SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
3617 static bool canEmitConjunction(const SDValue Val, bool &CanNegate,
3635 SDValue O0 = Val->getOperand(0);
3636 SDValue O1 = Val->getOperand(1);
3675 /// and sets @p OutCC to the flags that should be tested or returns SDValue() if
3679 static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val,
3680 AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp,
3685 SDValue LHS = Val->getOperand(0);
3686 SDValue RHS = Val->getOperand(1);
3702 SDValue ExtraCmp;
3724 SDValue LHS = Val->getOperand(0);
3731 SDValue RHS = Val->getOperand(1);
3778 SDValue CmpR = emitConjunctionRec(DAG, RHS, RHSCC, NegateR, CCOp, Predicate);
3781 SDValue CmpL = emitConjunctionRec(DAG, LHS, OutCC, NegateL, CmpR, RHSCC);
3791 static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val,
3796 return SDValue();
3798 return emitConjunctionRec(DAG, Val, OutCC, false, SDValue(), AArch64CC::AL);
3805 static unsigned getCmpOperandFoldingProfit(SDValue Op) {
3806 auto isSupportedExtend = [&](SDValue V) {
3839 static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
3840 SDValue &AArch64cc, SelectionDAG &DAG,
3911 SDValue TheLHS = LHSIsCMN ? LHS.getOperand(1) : LHS;
3912 SDValue TheRHS = RHSIsCMN ? RHS.getOperand(1) : RHS;
3921 SDValue Cmp;
3948 SDValue SExt =
3974 static std::pair<SDValue, SDValue>
3975 getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
3978 SDValue Value, Overflow;
3980 SDValue LHS = Op.getOperand(0);
3981 SDValue RHS = Op.getOperand(1);
4012 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
4019 SDValue SExtMul = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Value);
4024 SDValue UpperBits = DAG.getConstant(0xFFFFFFFF00000000, DL, MVT::i64);
4034 SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS);
4035 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value,
4043 SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS);
4064 SDValue AArch64TargetLowering::LowerXOR(SDValue Op, SelectionDAG &DAG) const {
4069 SDValue Sel = Op.getOperand(0);
4070 SDValue Other = Op.getOperand(1);
4083 return SDValue();
4085 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
4086 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
4088 SDValue Value, Overflow;
4090 SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
4108 SDValue LHS = Sel.getOperand(0);
4109 SDValue RHS = Sel.getOperand(1);
4110 SDValue TVal = Sel.getOperand(2);
4111 SDValue FVal = Sel.getOperand(3);
4134 SDValue CCVal;
4135 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
4151 static SDValue valueToCarryFlag(SDValue Value, SelectionDAG &DAG, bool Invert) {
4154 SDValue Op0 = Invert ? DAG.getConstant(0, DL, VT) : Value;
4155 SDValue Op1 = Invert ? Value : DAG.getConstant(1, DL, VT);
4156 SDValue Cmp =
4163 static SDValue carryFlagToValue(SDValue Glue, EVT VT, SelectionDAG &DAG,
4167 SDValue Zero = DAG.getConstant(0, DL, VT);
4168 SDValue One = DAG.getConstant(1, DL, VT);
4170 SDValue CC = DAG.getConstant(Cond, DL, MVT::i32);
4175 static SDValue overflowFlagToValue(SDValue Glue, EVT VT, SelectionDAG &DAG) {
4178 SDValue Zero = DAG.getConstant(0, DL, VT);
4179 SDValue One = DAG.getConstant(1, DL, VT);
4180 SDValue CC = DAG.getConstant(AArch64CC::VS, DL, MVT::i32);
4186 static SDValue lowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG,
4192 return SDValue();
4195 SDValue OpLHS = Op.getOperand(0);
4196 SDValue OpRHS = Op.getOperand(1);
4197 SDValue OpCarryIn = valueToCarryFlag(Op.getOperand(2), DAG, InvertCarry);
4202 SDValue Sum = DAG.getNode(Opcode, DL, DAG.getVTList(VT0, MVT::Glue), OpLHS,
4205 SDValue OutFlag =
4212 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
4215 return SDValue();
4220 SDValue Value, Overflow;
4224 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
4225 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
4230 SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
4243 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
4270 SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
4280 return SDValue();
4283 SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
4290 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
4303 SDValue Narrow = SrcVal;
4304 SDValue NaN;
4319 return SDValue();
4322 SDValue One = DAG.getConstant(1, dl, I32);
4323 SDValue Lsb = DAG.getNode(ISD::SRL, dl, I32, Narrow,
4326 SDValue RoundingBias =
4334 SDValue IsNaN = DAG.getSetCC(
4349 SDValue Result = DAG.getTargetExtractSubreg(AArch64::hsub, dl, VT, Narrow);
4357 return SDValue();
4363 return SDValue();
4366 SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
4394 SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NewVT, MVT::Other},
4410 SDValue Cv = DAG.getNode(Op.getOpcode(), dl, {InVT, MVT::Other},
4412 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, Cv);
4415 SDValue Cv =
4427 SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {ExtVT, MVT::Other},
4432 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0));
4440 SDValue Extract = DAG.getNode(
4454 SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
4457 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
4467 SDValue Ext =
4483 return SDValue();
4486 SDValue
4487 AArch64TargetLowering::LowerVectorFP_TO_INT_SAT(SDValue Op,
4491 SDValue SrcVal = Op.getOperand(0);
4506 return SDValue();
4521 return SDValue();
4543 return SDValue();
4546 SDValue NativeCvt = DAG.getNode(Op.getOpcode(), DL, IntVT, SrcVal,
4548 SDValue Sat;
4550 SDValue MinC = DAG.getConstant(
4552 SDValue Min = DAG.getNode(ISD::SMIN, DL, IntVT, NativeCvt, MinC);
4553 SDValue MaxC = DAG.getConstant(
4557 SDValue MinC = DAG.getConstant(
4565 SDValue AArch64TargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
4569 SDValue SrcVal = Op.getOperand(0);
4587 return SDValue();
4601 return SDValue();
4603 SDValue NativeCvt =
4605 SDValue Sat;
4607 SDValue MinC = DAG.getConstant(
4609 SDValue Min = DAG.getNode(ISD::SMIN, DL, DstVT, NativeCvt, MinC);
4610 SDValue MaxC = DAG.getConstant(
4614 SDValue MinC = DAG.getConstant(
4622 SDValue AArch64TargetLowering::LowerVectorXRINT(SDValue Op,
4625 SDValue Src = Op.getOperand(0);
4635 SDValue FOp = DAG.getNode(ISD::FRINT, DL, CastVT, Src);
4642 SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op,
4650 SDValue In = Op.getOperand(IsStrict ? 1 : 0);
4677 SDValue Val = DAG.getNode(Op.getOpcode(), dl, {F32, MVT::Other},
4718 SDValue Extract = DAG.getNode(
4731 SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
4737 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
4745 SDValue Val = DAG.getNode(Op.getOpcode(), dl, {PromoteVT, MVT::Other},
4801 SDValue SignBit;
4807 SDValue SrcHi = DAG.getNode(ISD::AND, DL, MVT::i64, SrcVal,
4809 SDValue SrcLo = DAG.getNode(ISD::AND, DL, MVT::i64, SrcVal,
4811 SDValue Highest =
4814 SDValue Zero64 = DAG.getConstant(0, DL, MVT::i64);
4815 SDValue ToRound =
4817 SDValue Rounded =
4822 SDValue RoundedBits = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Rounded);
4827 SDValue HasHighest = DAG.getSetCC(
4832 SDValue HasLo = DAG.getSetCC(
4837 SDValue NeedsAdjustment =
4841 SDValue AdjustedBits =
4843 SDValue Adjusted = DAG.getNode(ISD::BITCAST, DL, MVT::f64, AdjustedBits);
4861 return SDValue();
4867 return SDValue();
4870 SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
4875 SDValue Arg = Op.getOperand(0);
4891 SDValue Callee =
4900 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
4906 SDValue AArch64TargetLowering::LowerBITCAST(SDValue Op,
4921 return SDValue();
4926 SDValue ExtResult =
4935 return SDValue();
4966 static SDValue addRequiredExtensionForVectorMULL(SDValue N, SelectionDAG &DAG,
4986 getConstantLaneNumOfExtractHalfOperand(SDValue &Op) {
4999 static bool isExtendedBUILD_VECTOR(SDValue N, SelectionDAG &DAG,
5006 for (const SDValue &Elt : N->op_values()) {
5025 static SDValue skipExtensionForVectorMULL(SDValue N, SelectionDAG &DAG) {
5045 SmallVector<SDValue, 8> Ops;
5055 static bool isSignExtended(SDValue N, SelectionDAG &DAG) {
5061 static bool isZeroExtended(SDValue N, SelectionDAG &DAG) {
5067 static bool isAddSubSExt(SDValue N, SelectionDAG &DAG) {
5070 SDValue N0 = N.getOperand(0);
5071 SDValue N1 = N.getOperand(1);
5078 static bool isAddSubZExt(SDValue N, SelectionDAG &DAG) {
5081 SDValue N0 = N.getOperand(0);
5082 SDValue N1 = N.getOperand(1);
5089 SDValue AArch64TargetLowering::LowerGET_ROUNDING(SDValue Op,
5097 SDValue Chain = Op.getOperand(0);
5098 SDValue FPCR_64 = DAG.getNode(
5102 SDValue FPCR_32 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, FPCR_64);
5103 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPCR_32,
5105 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
5107 SDValue AND = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
5112 SDValue AArch64TargetLowering::LowerSET_ROUNDING(SDValue Op,
5115 SDValue Chain = Op->getOperand(0);
5116 SDValue RMValue = Op->getOperand(1);
5138 SDValue Ops[] = {
5140 SDValue FPCR =
5150 SDValue Ops2[] = {
5156 SDValue AArch64TargetLowering::LowerGET_FPMODE(SDValue Op,
5159 SDValue Chain = Op->getOperand(0);
5162 SDValue Ops[] = {
5164 SDValue FPCR =
5170 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPCR);
5175 SDValue AArch64TargetLowering::LowerSET_FPMODE(SDValue Op,
5178 SDValue Chain = Op->getOperand(0);
5179 SDValue Mode = Op->getOperand(1);
5182 SDValue FPCR = DAG.getZExtOrTrunc(Mode, DL, MVT::i64);
5185 SDValue Ops2[] = {
5190 SDValue AArch64TargetLowering::LowerRESET_FPMODE(SDValue Op,
5193 SDValue Chain = Op->getOperand(0);
5196 SDValue Ops[] = {
5198 SDValue FPCR =
5204 SDValue FPSCRMasked = DAG.getNode(
5209 SDValue Ops2[] = {Chain,
5215 static unsigned selectUmullSmull(SDValue &N0, SDValue &N1, SelectionDAG &DAG,
5232 SDValue ZextOperand;
5238 SDValue NewSext =
5278 SDValue AArch64TargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
5289 SDValue N0 = Op.getOperand(0);
5290 SDValue N1 = Op.getOperand(1);
5306 return SDValue();
5323 return SDValue();
5330 SDValue Op0;
5331 SDValue Op1 = skipExtensionForVectorMULL(N1, DAG);
5344 SDValue N00 = skipExtensionForVectorMULL(N0.getOperand(0), DAG);
5345 SDValue N01 = skipExtensionForVectorMULL(N0.getOperand(1), DAG);
5357 static inline SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT,
5365 static SDValue optimizeIncrementingWhile(SDValue Op, SelectionDAG &DAG,
5369 return SDValue();
5379 return SDValue();
5386 return SDValue();
5398 return SDValue();
5403 static SDValue getSVEPredicateBitCast(EVT VT, SDValue Op, SelectionDAG &DAG) {
5420 SDValue Reinterpret = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op);
5435 SDValue Mask = DAG.getConstant(1, DL, InVT);
5440 SDValue AArch64TargetLowering::getRuntimePStateSM(SelectionDAG &DAG,
5441 SDValue Chain, SDLoc DL,
5443 SDValue Callee = DAG.getExternalSymbol("__arm_sme_state",
5452 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
5453 SDValue Mask = DAG.getConstant(/*PSTATE.SM*/ 1, DL, MVT::i64);
5495 SDValue LowerSMELdrStr(SDValue N, SelectionDAG &DAG, bool IsLoad) {
5498 SDValue TileSlice = N->getOperand(2);
5499 SDValue Base = N->getOperand(3);
5500 SDValue VecNum = N->getOperand(4);
5502 SDValue VarAddend = VecNum;
5511 VarAddend = SDValue();
5516 SDValue CVal = DAG.getTargetConstant(C, DL, MVT::i32);
5528 SDValue Mul = DAG.getNode(
5542 SDValue AArch64TargetLowering::LowerINTRINSIC_VOID(SDValue Op,
5548 return SDValue(); // Don't custom lower most intrinsics.
5550 SDValue Chain = Op.getOperand(0);
5551 SDValue Addr = Op.getOperand(2);
5584 SDValue AArch64TargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
5590 return SDValue(); // Don't custom lower most intrinsics.
5593 SDValue Chain = Node->getChain();
5594 SDValue Dst = Op.getOperand(2);
5595 SDValue Val = Op.getOperand(3);
5597 SDValue Size = Op.getOperand(4);
5604 SDValue MS =
5617 SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
5622 default: return SDValue(); // Don't custom lower most intrinsics.
5630 SDValue Result = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64,
5641 SDValue LHS = Op.getOperand(1);
5642 SDValue RHS = Op.getOperand(2);
5656 auto TryVectorizeOperand = [](SDValue N, std::optional<uint64_t> NLane,
5659 SelectionDAG &DAG) -> SDValue {
5720 return SDValue();
5794 SDValue One = DAG.getConstant(1, dl, MVT::i32);
5795 SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, dl, Op.getValueType(), One);
5799 SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, dl, Op.getValueType(),
5805 SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, dl, Op.getValueType(),
5811 SDValue Data = Op.getOperand(3);
5896 SDValue Scalar = Op.getOperand(2);
5968 SDValue FnOp = Op.getOperand(1);
5969 SDValue IncomingFPOp = Op.getOperand(2);
6029 SDValue ID =
6044 SDValue Mask = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, WhileVT, ID,
6046 SDValue MaskAsInt = DAG.getNode(ISD::SIGN_EXTEND, dl, ContainerVT, Mask);
6056 SDValue UADDLV =
6058 SDValue EXTRACT_VEC_ELT =
6063 return SDValue();
6066 SDValue CttzOp = Op.getOperand(1);
6074 SDValue Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, NewVT, CttzOp);
6078 SDValue NewCttzElts =
6094 bool AArch64TargetLowering::shouldRemoveExtendFromGSIndex(SDValue Extend,
6110 bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
6185 SDValue AArch64TargetLowering::LowerMGATHER(SDValue Op,
6190 SDValue Chain = MGT->getChain();
6191 SDValue PassThru = MGT->getPassThru();
6192 SDValue Mask = MGT->getMask();
6193 SDValue BasePtr = MGT->getBasePtr();
6194 SDValue Index = MGT->getIndex();
6195 SDValue Scale = MGT->getScale();
6204 SDValue Ops[] = {Chain, DAG.getUNDEF(VT), Mask, BasePtr, Index, Scale};
6205 SDValue Load =
6208 SDValue Select = DAG.getSelect(DL, VT, Mask, Load, PassThru);
6225 SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale};
6266 SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale};
6267 SDValue Load =
6272 SDValue Result = convertFromScalableVector(DAG, PromotedVT, Load);
6284 SDValue AArch64TargetLowering::LowerMSCATTER(SDValue Op,
6289 SDValue Chain = MSC->getChain();
6290 SDValue StoreVal = MSC->getValue();
6291 SDValue Mask = MSC->getMask();
6292 SDValue BasePtr = MSC->getBasePtr();
6293 SDValue Index = MSC->getIndex();
6294 SDValue Scale = MSC->getScale();
6313 SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale};
6356 SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale};
6365 SDValue AArch64TargetLowering::LowerMLOAD(SDValue Op, SelectionDAG &DAG) const {
6374 SDValue PassThru = LoadNode->getPassThru();
6375 SDValue Mask = LoadNode->getMask();
6380 SDValue Load = DAG.getMaskedLoad(
6386 SDValue Result = DAG.getSelect(DL, VT, Mask, Load, PassThru);
6392 static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST,
6398 SDValue Value = ST->getValue();
6407 SDValue Undef = DAG.getUNDEF(MVT::i16);
6408 SDValue UndefVec = DAG.getBuildVector(MVT::v4i16, DL,
6411 SDValue TruncExt = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16,
6413 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, TruncExt);
6416 SDValue ExtractTrunc = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32,
6426 SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
6432 SDValue Value = StoreNode->getValue();
6466 SDValue Lo =
6470 SDValue Hi =
6475 SDValue Result = DAG.getMemIntrinsicNode(
6484 SDValue Value = StoreNode->getValue();
6486 SDValue Chain = StoreNode->getChain();
6487 SDValue Base = StoreNode->getBasePtr();
6490 SDValue Part = DAG.getNode(AArch64ISD::LS64_EXTRACT, Dl, MVT::i64,
6492 SDValue Ptr = DAG.getNode(ISD::ADD, Dl, PtrVT, Base,
6500 return SDValue();
6504 SDValue AArch64TargetLowering::LowerStore128(SDValue Op,
6518 SDValue Value = (StoreNode->getOpcode() == ISD::STORE ||
6527 SDValue Result = DAG.getMemIntrinsicNode(
6535 SDValue AArch64TargetLowering::LowerLOAD(SDValue Op,
6542 SmallVector<SDValue, 8> Ops;
6543 SDValue Base = LoadNode->getBasePtr();
6544 SDValue Chain = LoadNode->getChain();
6547 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Base,
6549 SDValue Part = DAG.getLoad(MVT::i64, DL, Chain, Ptr,
6553 Chain = SDValue(Part.getNode(), 1);
6555 SDValue Loaded = DAG.getNode(AArch64ISD::LS64_BUILD, DL, MVT::i64x8, Ops);
6564 return SDValue();
6568 return SDValue();
6577 return SDValue();
6579 SDValue Load = DAG.getLoad(MVT::f32, DL, LoadNode->getChain(),
6581 SDValue Chain = Load.getValue(1);
6582 SDValue Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f32, Load);
6583 SDValue BC = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Vec);
6584 SDValue Ext = DAG.getNode(ExtType, DL, MVT::v8i16, BC);
6593 SDValue AArch64TargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const {
6600 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
6603 SDValue Cmp =
6611 static SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) {
6612 SDValue Chain = Op.getOperand(0);
6613 SDValue Cond = Op.getOperand(1);
6614 SDValue Dest = Op.getOperand(2);
6617 if (SDValue Cmp = emitConjunction(DAG, Cond, CC)) {
6619 SDValue CCVal = DAG.getConstant(CC, dl, MVT::i32);
6624 return SDValue();
6629 static SDValue LowerFunnelShift(SDValue Op, SelectionDAG &DAG) {
6630 SDValue Shifts = Op.getOperand(2);
6648 return SDValue();
6651 static SDValue LowerFLDEXP(SDValue Op, SelectionDAG &DAG) {
6652 SDValue X = Op.getOperand(0);
6654 SDValue Exp = Op.getOperand(1);
6660 return SDValue();
6676 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
6677 SDValue VX =
6679 SDValue VExp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ExpVT,
6681 SDValue VPg = getPTrue(DAG, DL, XVT.changeVectorElementType(MVT::i1),
6683 SDValue FScale =
6687 SDValue Final =
6695 SDValue AArch64TargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
6705 SDValue AArch64TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
6712 SDValue Chain = Op.getOperand(0);
6713 SDValue Trmp = Op.getOperand(1); // trampoline
6714 SDValue FPtr = Op.getOperand(2); // nested function
6715 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
6741 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
6745 SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
6753 return SDValue();
6941 return SDValue();
6982 return SDValue();
7049 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
7060 SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
7070 SDValue Chain = Op.getOperand(0);
7071 SDValue SysRegName = Op.getOperand(1);
7072 std::pair<SDValue, SDValue> Pair =
7076 SDValue Result = DAG.getNode(AArch64ISD::MSRR, DL, MVT::Other, Chain,
7159 bool AArch64TargetLowering::isReassocProfitable(SelectionDAG &DAG, SDValue N0,
7160 SDValue N1) const {
7256 SDValue AArch64TargetLowering::LowerFormalArguments(
7257 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
7259 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
7277 DenseMap<unsigned, SDValue> CopiedRegs;
7319 SDValue Glue = Chain.getValue(1);
7321 SmallVector<SDValue, 16> ArgValues;
7337 SDValue FrameIdxN = DAG.getFrameIndex(FrameIdx, PtrVT);
7346 SDValue ArgValue;
7441 SDValue FIN;
7452 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
7509 SDValue Ptr = ArgValue;
7518 SDValue BytesIncrement;
7561 SDValue PStateSM;
7637 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, Reg, InVals[I]);
7672 SDValue SVL = DAG.getNode(AArch64ISD::RDSVL, DL, MVT::i64,
7675 SDValue Buffer;
7680 SDValue Size = DAG.getNode(ISD::MUL, DL, MVT::i64, SVL, SVL);
7711 SDValue &Chain) const {
7720 SmallVector<SDValue, 8> MemOps;
7742 SDValue FIN;
7749 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
7758 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
7759 SDValue Store =
7782 SDValue FIN = DAG.getFrameIndex(FPRIdx, PtrVT);
7786 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f128);
7788 SDValue Store = DAG.getStore(Val.getValue(1), DL, Val, FIN,
7806 SDValue AArch64TargetLowering::LowerCallResult(
7807 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,
7809 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
7810 SDValue ThisVal, bool RequiresSMChange) const {
7811 DenseMap<unsigned, SDValue> CopiedRegs;
7827 SDValue Val = CopiedRegs.lookup(VA.getLocReg());
7961 SDValue Callee = CLI.Callee;
7964 const SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
8111 SDValue AArch64TargetLowering::addTokenForArgument(SDValue Chain,
8115 SmallVector<SDValue, 8> ArgChains;
8135 ArgChains.push_back(SDValue(L, 1));
8149 static bool checkZExtBool(SDValue Arg, const SelectionDAG &DAG) {
8202 SDValue AArch64TargetLowering::changeStreamingMode(SelectionDAG &DAG, SDLoc DL,
8203 bool Enable, SDValue Chain,
8204 SDValue InGlue,
8206 SDValue PStateSM) const {
8212 SDValue RegMask = DAG.getRegisterMask(TRI->getSMStartStopCallPreservedMask());
8213 SDValue MSROp =
8215 SDValue ConditionOp = DAG.getTargetConstant(Condition, DL, MVT::i64);
8216 SmallVector<SDValue> Ops = {Chain, MSROp, ConditionOp};
8245 SDValue
8247 SmallVectorImpl<SDValue> &InVals) const {
8251 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
8253 SDValue Chain = CLI.Chain;
8254 SDValue Callee = CLI.Callee;
8394 SDValue TPIDR2ObjAddr = DAG.getFrameIndex(
8397 SDValue NumZaSaveSlicesAddr =
8400 SDValue NumZaSaveSlices = DAG.getNode(AArch64ISD::RDSVL, DL, MVT::i64,
8418 SDValue PStateSM;
8438 SDValue ZTFrameIdx;
8471 SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP,
8474 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
8476 SmallVector<SDValue, 8> MemOpChains;
8482 SDValue Val = DAG.getCopyFromReg(Chain, DL, F.VReg, F.VT);
8491 SDValue Arg = OutVals[i];
8565 SDValue Ptr = DAG.getFrameIndex(
8567 SDValue SpillSlot = Ptr;
8572 SDValue Store = DAG.getStore(Chain, DL, OutVals[i], Ptr, MPI);
8577 SDValue BytesIncrement;
8616 SDValue &Bits =
8618 [=](const std::pair<unsigned, SDValue> &Elt) {
8646 SDValue DstAddr;
8667 SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
8682 SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
8689 SDValue SizeNode =
8691 SDValue Cpy = DAG.getMemcpy(
8706 SDValue Store = DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo);
8713 SDValue ParamPtr = StackPtr;
8733 SDValue InGlue;
8741 SDValue NewChain = changeStreamingMode(
8794 std::vector<SDValue> Ops;
8830 SDValue AddrDisc, IntDisc;
8880 SDValue Ret = DAG.getNode(Opc, DL, NodeTys, Ops);
8906 SDValue Result = LowerCallResult(
8908 IsThisReturn ? OutVals[0] : SDValue(), RequiresSMChange);
8942 SDValue RegMask = DAG.getRegisterMask(
8944 SDValue RestoreRoutine = DAG.getTargetExternalSymbol(
8946 SDValue TPIDR2_EL0 = DAG.getNode(
8952 SDValue Glue;
8953 SDValue TPIDR2Block = DAG.getFrameIndex(
8979 SDValue X = DAG.getCopyToReg(Result, DL, Reg, InVals[I]);
9011 SDValue
9012 AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
9015 const SmallVectorImpl<SDValue> &OutVals,
9026 SDValue Glue;
9027 SmallVector<std::pair<unsigned, SDValue>, 4> RetVals;
9033 SDValue Arg = OutVals[realRVLocIdx];
9063 SDValue &Bits =
9064 llvm::find_if(RetVals, [=](const std::pair<unsigned, SDValue> &Elt) {
9082 SDValue PStateSM = DAG.getCopyFromReg(Chain, DL, Reg, MVT::i64);
9084 /*Glue*/ SDValue(),
9088 /*Glue*/ SDValue(), AArch64SME::Always);
9092 SmallVector<SDValue, 4> RetOps(1, Chain);
9109 SDValue Val = DAG.getCopyFromReg(RetOps[0], DL, SRetReg,
9144 SDValue Arm64ECRetDest =
9162 SDValue AArch64TargetLowering::getTargetNode(GlobalAddressSDNode *N, EVT Ty,
9169 SDValue AArch64TargetLowering::getTargetNode(JumpTableSDNode *N, EVT Ty,
9175 SDValue AArch64TargetLowering::getTargetNode(ConstantPoolSDNode *N, EVT Ty,
9182 SDValue AArch64TargetLowering::getTargetNode(BlockAddressSDNode* N, EVT Ty,
9188 SDValue AArch64TargetLowering::getTargetNode(ExternalSymbolSDNode *N, EVT Ty,
9196 SDValue AArch64TargetLowering::getGOT(NodeTy *N, SelectionDAG &DAG,
9201 SDValue GotAddr = getTargetNode(N, Ty, DAG, AArch64II::MO_GOT | Flags);
9209 SDValue AArch64TargetLowering::getAddrLarge(NodeTy *N, SelectionDAG &DAG,
9225 SDValue AArch64TargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
9230 SDValue Hi = getTargetNode(N, Ty, DAG, AArch64II::MO_PAGE | Flags);
9231 SDValue Lo = getTargetNode(N, Ty, DAG,
9233 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, Ty, Hi);
9239 SDValue AArch64TargetLowering::getAddrTiny(NodeTy *N, SelectionDAG &DAG,
9244 SDValue Sym = getTargetNode(N, Ty, DAG, Flags);
9248 SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
9264 SDValue Result;
9283 /// return an SDValue containing the final node.
9309 SDValue
9310 AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op,
9320 SDValue TLVPAddr =
9322 SDValue DescAddr = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TLVPAddr);
9326 SDValue Chain = DAG.getEntryNode();
9327 SDValue FuncTLVGet = DAG.getLoad(
9351 Chain = DAG.getCopyToReg(Chain, DL, AArch64::X0, DescAddr, SDValue());
9354 SmallVector<SDValue, 8> Ops;
9376 SDValue AArch64TargetLowering::LowerELFTLSLocalExec(const GlobalValue *GV,
9377 SDValue ThreadBase,
9381 SDValue TPOff, Addr;
9390 SDValue Var = DAG.getTargetGlobalAddress(
9392 return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
9402 SDValue HiVar = DAG.getTargetGlobalAddress(
9404 SDValue LoVar = DAG.getTargetGlobalAddress(
9407 Addr = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
9411 return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, Addr,
9422 SDValue HiVar = DAG.getTargetGlobalAddress(
9424 SDValue LoVar = DAG.getTargetGlobalAddress(
9427 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
9430 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
9442 SDValue HiVar = DAG.getTargetGlobalAddress(
9444 SDValue MiVar = DAG.getTargetGlobalAddress(
9447 SDValue LoVar = DAG.getTargetGlobalAddress(
9450 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
9453 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, MiVar,
9456 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
9482 SDValue AArch64TargetLowering::LowerELFTLSDescCallSeq(SDValue SymAddr,
9487 SDValue Chain = DAG.getEntryNode();
9492 SDValue Glue = Chain.getValue(1);
9497 SDValue
9498 AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op,
9522 SDValue TPOff;
9527 SDValue ThreadBase = DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT);
9548 SDValue SymAddr = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT,
9557 SDValue HiVar = DAG.getTargetGlobalAddress(
9559 SDValue LoVar = DAG.getTargetGlobalAddress(
9563 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, HiVar,
9566 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, LoVar,
9573 SDValue SymAddr =
9584 SDValue
9585 AArch64TargetLowering::LowerWindowsGlobalTLSAddress(SDValue Op,
9589 SDValue Chain = DAG.getEntryNode();
9593 SDValue TEB = DAG.getRegister(AArch64::X18, MVT::i64);
9597 SDValue TLSArray =
9606 SDValue TLSIndexHi =
9608 SDValue TLSIndexLo = DAG.getTargetExternalSymbol(
9610 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, TLSIndexHi);
9611 SDValue TLSIndex =
9619 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
9621 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
9628 SDValue TGAHi = DAG.getTargetGlobalAddress(
9630 SDValue TGALo = DAG.getTargetGlobalAddress(
9635 SDValue Addr =
9636 SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TLS, TGAHi,
9643 SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
9688 SDValue AArch64TargetLowering::LowerPtrAuthGlobalAddressStatically(
9689 SDValue TGA, SDLoc DL, EVT VT, AArch64PACKey::ID KeyC,
9690 SDValue Discriminator, SDValue AddrDiscriminator, SelectionDAG &DAG) const {
9705 SDValue Key = DAG.getTargetConstant(KeyC, DL, MVT::i32);
9706 return SDValue(DAG.getMachineNode(AArch64::LOADauthptrstatic, DL, MVT::i64,
9711 SDValue
9712 AArch64TargetLowering::LowerPtrAuthGlobalAddress(SDValue Op,
9714 SDValue Ptr = Op.getOperand(0);
9716 SDValue AddrDiscriminator = Op.getOperand(2);
9751 SDValue TPtr = DAG.getTargetGlobalAddress(PtrGV, DL, VT, PtrOffsetC,
9756 SDValue Key = DAG.getTargetConstant(KeyC, DL, MVT::i32);
9757 SDValue Discriminator = DAG.getTargetConstant(DiscriminatorC, DL, MVT::i64);
9758 SDValue TAddrDiscriminator = !isNullConstant(AddrDiscriminator)
9765 return SDValue(
9774 return SDValue(
9788 std::pair<SDValue, uint64_t> lookThroughSignExtension(SDValue Val) {
9801 SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
9802 SDValue Chain = Op.getOperand(0);
9804 SDValue LHS = Op.getOperand(2);
9805 SDValue RHS = Op.getOperand(3);
9806 SDValue Dest = Op.getOperand(4);
9836 return SDValue();
9840 SDValue Value, Overflow;
9845 SDValue CCVal = DAG.getConstant(OFCC, dl, MVT::i32);
9867 SDValue Test = LHS.getOperand(0);
9883 SDValue Test = LHS.getOperand(0);
9912 SDValue CCVal;
9913 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
9923 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
9926 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
9927 SDValue BR1 =
9930 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
9938 SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
9942 return SDValue();
9948 SDValue In1 = Op.getOperand(0);
9949 SDValue In2 = Op.getOperand(1);
9966 SDValue Res = DAG.getNode(ISD::FCOPYSIGN, DL, ContainerVT, In1, In2);
9970 auto BitCast = [this](EVT VT, SDValue Op, SelectionDAG &DAG) {
9977 SDValue VecVal1, VecVal2;
10007 SDValue SignMaskV = DAG.getConstant(~APInt::getSignMask(BitWidth), DL, VecVT);
10019 SDValue BSP =
10031 SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
10035 return SDValue();
10043 return SDValue();
10046 SDValue Val = Op.getOperand(0);
10052 return SDValue();
10067 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val);
10068 SDValue UaddLV = DAG.getNode(AArch64ISD::UADDLV, DL, MVT::v4i32, CtPop);
10082 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v16i8, Val);
10083 SDValue UaddLV = DAG.getNode(AArch64ISD::UADDLV, DL, MVT::v4i32, CtPop);
10107 SDValue Zeros = DAG.getConstant(0, DL, DT);
10108 SDValue Ones = DAG.getConstant(1, DL, VT8Bit);
10137 SDValue AArch64TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
10144 SDValue RBIT = DAG.getNode(ISD::BITREVERSE, DL, VT, Op.getOperand(0));
10148 SDValue AArch64TargetLowering::LowerMinMax(SDValue Op,
10189 SDValue Op0 = Op.getOperand(0);
10190 SDValue Op1 = Op.getOperand(1);
10191 SDValue Cond = DAG.getSetCC(DL, VT, Op0, Op1, CC);
10195 SDValue AArch64TargetLowering::LowerBitreverse(SDValue Op,
10205 SDValue REVB;
10247 isOrXorChain(SDValue N, unsigned &Num,
10248 SmallVector<std::pair<SDValue, SDValue>, 16> &WorkList) {
10274 static SDValue performOrXorChainCombine(SDNode *N, SelectionDAG &DAG) {
10275 SDValue LHS = N->getOperand(0);
10276 SDValue RHS = N->getOperand(1);
10279 SmallVector<std::pair<SDValue, SDValue>, 16> WorkList;
10283 return SDValue();
10292 SDValue XOR0, XOR1;
10295 SDValue Cmp = DAG.getSetCC(DL, VT, XOR0, XOR1, Cond);
10298 SDValue CmpChain = DAG.getSetCC(DL, VT, XOR0, XOR1, Cond);
10306 return SDValue();
10309 SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
10317 SDValue Chain;
10320 SDValue LHS = Op.getOperand(OpNo + 0);
10321 SDValue RHS = Op.getOperand(OpNo + 1);
10327 SDValue TVal = DAG.getConstant(1, dl, VT);
10328 SDValue FVal = DAG.getConstant(0, dl, VT);
10345 SDValue CCVal;
10346 SDValue Cmp = getAArch64Cmp(
10352 SDValue Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp);
10362 SDValue Cmp;
10370 SDValue Res;
10374 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
10387 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
10388 SDValue CS1 =
10391 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
10397 SDValue AArch64TargetLowering::LowerSETCCCARRY(SDValue Op,
10400 SDValue LHS = Op.getOperand(0);
10401 SDValue RHS = Op.getOperand(1);
10404 return SDValue();
10407 SDValue Carry = Op.getOperand(2);
10409 SDValue InvCarry = valueToCarryFlag(Carry, DAG, true);
10410 SDValue Cmp = DAG.getNode(AArch64ISD::SBCS, DL, DAG.getVTList(VT, MVT::Glue),
10414 SDValue TVal = DAG.getConstant(1, DL, OpVT);
10415 SDValue FVal = DAG.getConstant(0, DL, OpVT);
10419 SDValue CCVal =
10427 SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
10428 SDValue RHS, SDValue TVal,
10429 SDValue FVal, const SDLoc &dl,
10466 SDValue Shift =
10480 SDValue Shift =
10602 SDValue CCVal;
10603 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
10613 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
10639 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
10640 SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
10645 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
10653 SDValue AArch64TargetLowering::LowerVECTOR_SPLICE(SDValue Op,
10676 SDValue Pred = getPTrue(DAG, DL, PredVT, *PredPattern);
10689 return SDValue();
10692 SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
10695 SDValue LHS = Op.getOperand(0);
10696 SDValue RHS = Op.getOperand(1);
10697 SDValue TVal = Op.getOperand(2);
10698 SDValue FVal = Op.getOperand(3);
10703 SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
10705 SDValue CCVal = Op->getOperand(0);
10706 SDValue TVal = Op->getOperand(1);
10707 SDValue FVal = Op->getOperand(2);
10714 SDValue Sel =
10721 SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, PredVT, CCVal);
10731 SDValue SplatVal = DAG.getSExtOrTrunc(CCVal, DL, SplatValVT);
10732 SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, PredVT, SplatVal);
10741 return SDValue();
10744 SDValue Value, Overflow;
10746 SDValue CCVal = DAG.getConstant(OFCC, DL, MVT::i32);
10754 SDValue LHS, RHS;
10774 SDValue Res = LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
10783 SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
10798 SDValue AArch64TargetLowering::LowerBR_JT(SDValue Op,
10803 SDValue JT = Op.getOperand(1);
10804 SDValue Entry = Op.getOperand(2);
10826 SDValue X16Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, AArch64::X16,
10827 Entry, SDValue());
10831 return SDValue(B, 0);
10837 SDValue JTInfo = DAG.getJumpTableDebugInfo(JTI, Op.getOperand(0), DL);
10838 return DAG.getNode(ISD::BRIND, DL, MVT::Other, JTInfo, SDValue(Dest, 0));
10841 SDValue AArch64TargetLowering::LowerBRIND(SDValue Op, SelectionDAG &DAG) const {
10842 SDValue Chain = Op.getOperand(0);
10843 SDValue Dest = Op.getOperand(1);
10849 return SDValue();
10855 return SDValue();
10859 SDValue Disc = DAG.getTargetConstant(*BADisc, DL, MVT::i64);
10860 SDValue Key = DAG.getTargetConstant(AArch64PACKey::IA, DL, MVT::i32);
10861 SDValue AddrDisc = DAG.getRegister(AArch64::XZR, MVT::i64);
10865 return SDValue(BrA, 0);
10868 SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
10885 SDValue AArch64TargetLowering::LowerBlockAddress(SDValue Op,
10896 SDValue TargetBA = DAG.getTargetBlockAddress(BA, BAN->getValueType(0));
10898 SDValue Disc = DAG.getTargetConstant(*BADisc, DL, MVT::i64);
10900 SDValue Key = DAG.getTargetConstant(AArch64PACKey::IA, DL, MVT::i32);
10901 SDValue AddrDisc = DAG.getRegister(AArch64::XZR, MVT::i64);
10906 return DAG.getCopyFromReg(SDValue(MOV, 0), DL, AArch64::X16, MVT::i64,
10907 SDValue(MOV, 1));
10920 SDValue AArch64TargetLowering::LowerDarwin_VASTART(SDValue Op,
10926 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(),
10934 SDValue AArch64TargetLowering::LowerWin64_VASTART(SDValue Op,
10940 SDValue FR;
10946 SDValue Val = DAG.getCopyFromReg(DAG.getEntryNode(), DL, VReg, MVT::i64);
10965 SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
10976 SDValue Chain = Op.getOperand(0);
10977 SDValue VAList = Op.getOperand(1);
10979 SmallVector<SDValue, 4> MemOps;
10983 SDValue Stack = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), PtrVT);
10992 SDValue GRTop, GRTopAddr;
11011 SDValue VRTop, VRTopAddr;
11027 SDValue GROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
11035 SDValue VROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
11044 SDValue AArch64TargetLowering::LowerVASTART(SDValue Op,
11057 SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
11077 SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
11084 SDValue Chain = Op.getOperand(0);
11085 SDValue Addr = Op.getOperand(1);
11090 SDValue VAList =
11122 SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
11127 SDValue APStore =
11133 SDValue WideFP =
11136 SDValue NarrowFP =
11139 SDValue Ops[] = { NarrowFP, WideFP.getValue(1) };
11147 SDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op,
11155 SDValue FrameAddr =
11168 SDValue AArch64TargetLowering::LowerSPONENTRY(SDValue Op,
11199 SDValue AArch64TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
11206 SDValue FrameAddr =
11208 SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
11213 SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op,
11222 SDValue ReturnAddress;
11224 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
11225 SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
11245 SDValue Chain =
11249 return SDValue(St, 0);
11254 SDValue AArch64TargetLowering::LowerShiftParts(SDValue Op,
11256 SDValue Lo, Hi;
11315 static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode,
11316 SDValue Operand, SelectionDAG &DAG,
11342 return SDValue();
11345 SDValue
11346 AArch64TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
11351 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
11355 SDValue
11356 AArch64TargetLowering::getSqrtResultForDenormInput(SDValue Op,
11361 SDValue AArch64TargetLowering::getSqrtEstimate(SDValue Operand,
11368 if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRSQRTE, Operand,
11379 SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate,
11391 return SDValue();
11394 SDValue AArch64TargetLowering::getRecipEstimate(SDValue Operand,
11398 if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRECPE, Operand,
11409 SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand,
11418 return SDValue();
11553 static SDValue getSETCC(AArch64CC::CondCode CC, SDValue NZCV, const SDLoc &DL,
11562 SDValue AArch64TargetLowering::LowerAsmOutputForConstraint(
11563 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
11567 return SDValue();
11580 SDValue CC = getSETCC(Cond, Glue, DL, DAG);
11582 SDValue Result;
11786 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
11788 SDValue Result;
11932 static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG) {
11945 static unsigned getExtFactor(SDValue &V) {
11953 SDValue ReconstructShuffleWithRuntimeMask(SDValue Op, SelectionDAG &DAG) {
11963 return SDValue();
11969 SDValue SourceVec;
11970 SDValue MaskSourceVec;
11971 SmallVector<SDValue, 16> AndMaskConstants;
11974 SDValue V = Op.getOperand(i);
11976 return SDValue();
11978 SDValue OperandSourceVec = V.getOperand(0);
11982 return SDValue();
11987 SDValue MaskSource = V.getOperand(1);
11990 return SDValue();
11996 return SDValue();
12005 return SDValue();
12007 SDValue MaskIdx = MaskSource.getOperand(1);
12010 return SDValue();
12017 return SDValue();
12019 return SDValue();
12044 SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
12055 SDValue Vec;
12062 SDValue ShuffleVec;
12069 ShuffleSourceInfo(SDValue Vec)
12073 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
12080 SDValue V = Op.getOperand(i);
12091 return SDValue();
12095 SDValue SourceVec = V.getOperand(0);
12114 SDValue V = Op.getOperand(I);
12137 SmallVector<SDValue, 16> TBLOperands;
12143 SDValue Src = Sources[i].Vec;
12154 SmallVector<SDValue, 16> TBLMask;
12162 SDValue Shuffle =
12172 return SDValue();
12219 return SDValue();
12225 return SDValue();
12241 SDValue VEXTSrc1 =
12244 SDValue VEXTSrc2 =
12253 return SDValue();
12291 SDValue Entry = Op.getOperand(i);
12319 return SDValue();
12322 SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
12326 SDValue Shuffle = DAG.getVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
12328 SDValue V;
12375 static SDValue ReconstructTruncateFromBuildVector(SDValue V, SelectionDAG &DAG) {
12377 return SDValue();
12383 SDValue BaseExt = V.getOperand(X * 4);
12389 return SDValue();
12390 SDValue Base = BaseExt.getOperand(0);
12393 SDValue Ext = V.getOperand(X * 4 + Y);
12398 return SDValue();
12407 SDValue Trunc[4] = {
12410 for (SDValue &V : Trunc)
12413 SDValue Concat0 =
12415 SDValue Concat1 =
12417 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, Concat0);
12418 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, Concat1);
12651 static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG) {
12654 SDValue V0 = Op.getOperand(0);
12655 SDValue V1 = Op.getOperand(1);
12660 return SDValue();
12665 return SDValue();
12684 static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1,
12685 SDValue V2, unsigned PFEntry, SDValue LHS,
12686 SDValue RHS, SelectionDAG &DAG,
12733 SDValue OpLHS = GeneratePerfectShuffle(
12738 SDValue Input;
12769 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
12772 SDValue Ins =
12778 SDValue OpLHS, OpRHS;
12820 SDValue Lane = DAG.getConstant(OpNum - OP_VDUP0, dl, MVT::i64);
12845 static SDValue GenerateTBL(SDValue Op, ArrayRef<int> ShuffleMask,
12848 SDValue V1 = Op.getOperand(0);
12849 SDValue V2 = Op.getOperand(1);
12872 SmallVector<SDValue, 8> TBLMask;
12884 SDValue V1Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V1);
12885 SDValue V2Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V2);
12887 SDValue Shuffle;
12932 static SDValue constructDup(SDValue V, int Lane, SDLoc dl, EVT VT,
12935 auto getScaledOffsetDup = [](SDValue BitCast, int &LaneC, MVT &CastVT) {
12943 SDValue Extract = BitCast.getOperand(0);
13039 static SDValue tryWidenMaskForShuffle(SDValue Op, SelectionDAG &DAG) {
13044 SDValue V0 = Op.getOperand(0);
13045 SDValue V1 = Op.getOperand(1);
13052 return SDValue();
13068 return SDValue();
13072 static SDValue tryToConvertShuffleOfTbl2ToTbl4(SDValue Op,
13075 SDValue Tbl1 = Op->getOperand(0);
13076 SDValue Tbl2 = Op->getOperand(1);
13078 SDValue Tbl2ID =
13086 return SDValue();
13090 return SDValue();
13092 SDValue Mask1 = Tbl1->getOperand(3);
13093 SDValue Mask2 = Tbl2->getOperand(3);
13094 SmallVector<SDValue, 16> TBLMaskParts(16, SDValue());
13102 return SDValue();
13107 SDValue TBLMask = DAG.getBuildVector(VT, dl, TBLMaskParts);
13108 SDValue ID =
13119 SDValue
13120 AArch64TargetLowering::LowerZERO_EXTEND_VECTOR_INREG(SDValue Op,
13124 SDValue SrcOp = Op.getOperand(0);
13131 return SDValue();
13132 SDValue Zeros = DAG.getConstant(0, dl, SrcVT);
13137 SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
13153 SDValue V1 = Op.getOperand(0);
13154 SDValue V2 = Op.getOperand(1);
13160 if (SDValue Res = tryToConvertShuffleOfTbl2ToTbl4(Op, ShuffleMask, DAG))
13213 SDValue Rev = DAG.getNode(AArch64ISD::REV64, dl, VT, V1);
13259 if (SDValue Concat = tryFormConcatFromShuffle(Op, DAG))
13266 SDValue DstVec = DstIsLeft ? V1 : V2;
13267 SDValue DstLaneV = DAG.getConstant(Anomaly, dl, MVT::i64);
13269 SDValue SrcVec = V1;
13275 SDValue SrcLaneV = DAG.getConstant(SrcLane, dl, MVT::i64);
13288 if (SDValue NewSD = tryWidenMaskForShuffle(Op, DAG))
13313 SDValue AArch64TargetLowering::LowerSPLAT_VECTOR(SDValue Op,
13330 SDValue SplatVal = DAG.getAnyExtOrTrunc(Op.getOperand(0), DL, MVT::i64);
13333 SDValue ID =
13335 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
13344 SDValue AArch64TargetLowering::LowerDUPQLane(SDValue Op,
13350 return SDValue();
13354 return SDValue();
13357 SDValue Idx128 = Op.getOperand(2);
13362 SDValue CI = DAG.getTargetConstant(CIdx->getZExtValue(), DL, MVT::i64);
13366 SDValue V = DAG.getNode(ISD::BITCAST, DL, MVT::nxv2i64, Op.getOperand(1));
13372 SDValue One = DAG.getConstant(1, DL, MVT::i64);
13373 SDValue SplatOne = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, One);
13376 SDValue SV = DAG.getStepVector(DL, MVT::nxv2i64);
13380 SDValue Idx64 = DAG.getNode(ISD::ADD, DL, MVT::i64, Idx128, Idx128);
13381 SDValue SplatIdx64 = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Idx64);
13382 SDValue ShuffleMask = DAG.getNode(ISD::ADD, DL, MVT::nxv2i64, SV, SplatIdx64);
13385 SDValue TBL = DAG.getNode(AArch64ISD::TBL, DL, MVT::nxv2i64, V, ShuffleMask);
13413 static SDValue tryAdvSIMDModImm64(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
13424 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
13430 return SDValue();
13434 static SDValue tryAdvSIMDModImm32(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
13436 const SDValue *LHS = nullptr) {
13440 return SDValue();
13467 SDValue Mov;
13483 return SDValue();
13487 static SDValue tryAdvSIMDModImm16(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
13489 const SDValue *LHS = nullptr) {
13493 return SDValue();
13512 SDValue Mov;
13528 return SDValue();
13532 static SDValue tryAdvSIMDModImm321s(unsigned NewOp, SDValue Op,
13552 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
13559 return SDValue();
13563 static SDValue tryAdvSIMDModImm8(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
13574 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
13580 return SDValue();
13584 static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
13605 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
13611 return SDValue();
13617 static bool isAllConstantBuildVector(const SDValue &PotentialBVec,
13634 static bool isAllInactivePredicate(SDValue N) {
13642 static bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) {
13687 static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG) {
13691 return SDValue();
13695 SDValue And;
13696 SDValue Shift;
13698 SDValue FirstOp = N->getOperand(0);
13700 SDValue SecondOp = N->getOperand(1);
13722 return SDValue();
13734 return SDValue();
13737 return SDValue();
13743 return SDValue();
13750 return SDValue();
13764 return SDValue();
13769 return SDValue();
13771 SDValue X = And.getOperand(0);
13772 SDValue Y = ShiftHasPredOp ? Shift.getOperand(1) : Shift.getOperand(0);
13773 SDValue Imm = ShiftHasPredOp ? DAG.getTargetConstant(C2, DL, MVT::i32)
13777 SDValue ResultSLI = DAG.getNode(Inst, DL, VT, X, Y, Imm);
13788 SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
13795 if (SDValue Res = tryLowerToSLI(Op.getNode(), DAG))
13802 SDValue LHS = Op.getOperand(0);
13816 SDValue NewOp;
13837 static SDValue NormalizeBuildVector(SDValue Op,
13847 SmallVector<SDValue, 16> Ops;
13848 for (SDValue Lane : Op->ops()) {
13868 static SDValue ConstantBuildVector(SDValue Op, SelectionDAG &DAG,
13879 SDValue NewOp;
13900 return SDValue();
13902 if (SDValue R = TryMOVIWithBits(DefBits))
13904 if (SDValue R = TryMOVIWithBits(UndefBits))
13921 if (SDValue NewOp = TryMOVIWithBits(NegBits)) {
13929 return SDValue();
13931 SDValue R;
13938 return SDValue();
13941 SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
13949 SDValue Start = DAG.getConstant(SeqInfo->first, DL, ContainerVT);
13950 SDValue Steps = DAG.getStepVector(DL, ContainerVT, SeqInfo->second);
13951 SDValue Seq = DAG.getNode(ISD::ADD, DL, ContainerVT, Start, Steps);
13956 return SDValue();
13964 return SDValue();
13984 if (SDValue V = ConstantBuildVector(Op, DAG, Subtarget))
14009 SDValue Value;
14010 SDValue ConstantValue;
14011 SmallMapVector<SDValue, unsigned, 16> DifferentValueMap;
14013 SDValue PrevVal;
14015 SDValue V = Op.getOperand(i);
14083 SDValue V = Op.getOperand(i);
14090 SDValue N0 = N->getOperand(0);
14124 SDValue LHS =
14125 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
14127 SDValue RHS =
14128 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
14151 SDValue Lane = Value.getOperand(1);
14165 SmallVector<SDValue, 8> Ops;
14176 SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
14200 SDValue Val = DAG.getSplatBuildVector(VT, dl, ConstantValue);
14215 SDValue V = Op.getOperand(i);
14216 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
14230 return SDValue();
14236 if (SDValue M = ReconstructTruncateFromBuildVector(Op, DAG))
14241 if (SDValue Shuffle = ReconstructShuffle(Op, DAG))
14244 if (SDValue Shuffle = ReconstructShuffleWithRuntimeMask(Op, DAG))
14250 SmallVector<SDValue, 8> Ops(NumElts, Value);
14251 SDValue NewVector = LowerBUILD_VECTOR(DAG.getBuildVector(VT, dl, Ops), DAG);
14265 SmallVector<SDValue, 2> Vals;
14297 SmallVector<SDValue, 8> Ops1(NumElts / 2, Vals[0]);
14298 SmallVector<SDValue, 8> Ops2(NumElts / 2, Vals[1]);
14299 SDValue DUP1 =
14301 SDValue DUP2 =
14303 SDValue CONCAT_VECTORS =
14319 SDValue FirstLaneVal = Op.getOperand(0);
14321 SDValue Val = Op.getOperand(i);
14328 SmallVector<SDValue, 8> Ops1(NumElts, Vals[0]);
14329 SmallVector<SDValue, 8> Ops2(NumElts, Vals[1]);
14330 SDValue VEC1 = DAG.getBuildVector(VT, dl, Ops1);
14331 SDValue VEC2 = DAG.getBuildVector(VT, dl, Ops2);
14332 SDValue VECTOR_SHUFFLE =
14349 SDValue Vec = DAG.getUNDEF(VT);
14350 SDValue Op0 = Op.getOperand(0);
14371 SDValue V = Op.getOperand(i);
14374 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
14383 return SDValue();
14386 SDValue AArch64TargetLowering::LowerCONCAT_VECTORS(SDValue Op,
14405 SmallVector<SDValue> ConcatOps(Op->op_begin(), Op->op_end());
14408 SDValue V1 = ConcatOps[I];
14409 SDValue V2 = ConcatOps[I + 1];
14420 return SDValue();
14423 SDValue AArch64TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
14436 SDValue ExtendedVector =
14438 SDValue ExtendedValue =
14452 return SDValue();
14457 SDValue
14458 AArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
14468 SDValue Extend =
14471 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractTy,
14482 return SDValue();
14493 return SDValue();
14498 SDValue WideVec = WidenVector(Op.getOperand(0), DAG);
14510 SDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
14519 return SDValue();
14538 SDValue Vec = Op.getOperand(0);
14539 SDValue Idx = Op.getOperand(1);
14544 SDValue Container = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PackedVT,
14556 SDValue Splice = DAG.getNode(ISD::VECTOR_SPLICE, DL, InVT, Vec, Vec, Idx);
14560 return SDValue();
14563 SDValue AArch64TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
14571 SDValue Vec0 = Op.getOperand(0);
14572 SDValue Vec1 = Op.getOperand(1);
14578 return SDValue();
14585 SDValue Lo, Hi;
14602 return SDValue();
14622 SDValue Narrow;
14624 SDValue HiVec0 = DAG.getNode(AArch64ISD::UUNPKHI, DL, WideVT, Vec0);
14629 SDValue LoVec0 = DAG.getNode(AArch64ISD::UUNPKLO, DL, WideVT, Vec0);
14644 SDValue PTrue = getPTrue(DAG, DL, PredTy, *PredPattern);
14645 SDValue ScalableVec1 = convertToScalableVector(DAG, VT, Vec1);
14649 return SDValue();
14652 static bool isPow2Splat(SDValue Op, uint64_t &SplatVal, bool &Negated) {
14683 SDValue AArch64TargetLowering::LowerDIV(SDValue Op, SelectionDAG &DAG) const {
14698 SDValue Pg = getPredicateForScalableVector(DAG, dl, VT);
14699 SDValue Res =
14723 SDValue Op0Lo = DAG.getNode(UnpkLo, dl, WidenedVT, Op.getOperand(0));
14724 SDValue Op1Lo = DAG.getNode(UnpkLo, dl, WidenedVT, Op.getOperand(1));
14725 SDValue Op0Hi = DAG.getNode(UnpkHi, dl, WidenedVT, Op.getOperand(0));
14726 SDValue Op1Hi = DAG.getNode(UnpkHi, dl, WidenedVT, Op.getOperand(1));
14727 SDValue ResultLo = DAG.getNode(Op.getOpcode(), dl, WidenedVT, Op0Lo, Op1Lo);
14728 SDValue ResultHi = DAG.getNode(Op.getOpcode(), dl, WidenedVT, Op0Hi, Op1Hi);
14781 static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
14801 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
14812 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) {
14820 SDValue AArch64TargetLowering::LowerTRUNCATE(SDValue Op,
14828 SDValue Zero = DAG.getConstant(0, dl, OpVT);
14829 SDValue One = DAG.getConstant(1, dl, OpVT);
14830 SDValue And = DAG.getNode(ISD::AND, dl, OpVT, Op.getOperand(0), One);
14835 return SDValue();
14841 return SDValue();
14847 static bool canLowerSRLToRoundingShiftForVT(SDValue Shift, EVT ResVT,
14850 SDValue &RShOperand) {
14866 SDValue Add = Shift->getOperand(0);
14889 SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
14917 SDValue RShOperand;
14946 SDValue NegShift = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
14948 SDValue NegShiftLeft =
14958 static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS,
14982 return SDValue();
14984 SDValue Fcmeq;
15005 return SDValue();
15014 return SDValue();
15026 return SDValue();
15028 SDValue Cmeq;
15070 SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
15080 SDValue LHS = Op.getOperand(0);
15081 SDValue RHS = Op.getOperand(1);
15088 SDValue Cmp =
15120 SDValue NewSetcc = DAG.getSetCC(dl, MVT::v4i16, LHS, RHS, CC);
15124 return SDValue();
15138 SDValue Cmp =
15141 return SDValue();
15144 SDValue Cmp2 =
15147 return SDValue();
15160 static SDValue getReductionSDNode(unsigned Op, SDLoc DL, SDValue ScalarOp,
15162 SDValue VecOp = ScalarOp.getOperand(0);
15168 static SDValue getVectorBitwiseReduce(unsigned Opcode, SDValue Vec, EVT VT,
15183 return SDValue();
15192 SDValue Result;
15199 SDValue Lo, Hi;
15202 SDValue HalfVec = DAG.getNode(ScalarOpcode, DL, HalfVT, Lo, Hi);
15215 SDValue Extended = DAG.getNode(
15236 SDValue Lo, Hi;
15248 SDValue Scalar = DAG.getBitcast(ScalarVT, Vec);
15255 SDValue ShiftAmount =
15257 SDValue Shifted =
15268 SDValue AArch64TargetLowering::LowerVECREDUCE(SDValue Op,
15270 SDValue Src = Op.getOperand(0);
15343 SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op,
15348 return SDValue();
15354 SDValue RHS = Op.getOperand(2);
15362 SDValue
15363 AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(SDValue Op,
15369 SDValue Chain = Op.getOperand(0);
15370 SDValue Size = Op.getOperand(1);
15377 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
15384 SDValue Ops[2] = {SP, Chain};
15391 SDValue Callee = DAG.getTargetExternalSymbol(Subtarget->getChkStkName(),
15401 Chain = DAG.getCopyToReg(Chain, dl, AArch64::X15, Size, SDValue());
15414 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
15422 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
15424 SDValue Ops[2] = {SP, Chain};
15428 SDValue
15429 AArch64TargetLowering::LowerInlineDYNAMIC_STACKALLOC(SDValue Op,
15433 SDValue Chain = Op.getOperand(0);
15434 SDValue Size = Op.getOperand(1);
15442 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
15451 SDValue Ops[2] = {SP, Chain};
15455 SDValue
15456 AArch64TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
15465 return SDValue();
15468 SDValue AArch64TargetLowering::LowerAVG(SDValue Op, SelectionDAG &DAG,
15474 return SDValue();
15477 SDValue AArch64TargetLowering::LowerVSCALE(SDValue Op,
15707 const SDValue &Base = Mem->getBasePtr();
15727 bool AArch64TargetLowering::shouldRemoveRedundantExtend(SDValue Extend) const {
15730 SDValue Extract = Extend.getOperand(0);
15802 bool AArch64TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
17307 SDValue AddNode, SDValue ConstNode) const {
17495 SDValue ShiftLHS = N->getOperand(0);
17505 SDValue AndLHS = ShiftLHS.getOperand(0);
17604 static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
17608 return SDValue();
17612 SDValue Shift = N->getOperand(0);
17613 SDValue Ones = N->getOperand(1);
17616 return SDValue();
17622 return SDValue();
17642 static SDValue performVecReduceAddCombineWithUADDLP(SDNode *N,
17646 return SDValue();
17648 SDValue VecReduceOp0 = N->getOperand(0);
17652 return SDValue();
17654 SDValue ABS = VecReduceOp0;
17658 return SDValue();
17660 SDValue SUB = ABS->getOperand(0);
17666 return SDValue();
17675 return SDValue();
17677 SDValue EXT0 = SUB->getOperand(0);
17678 SDValue EXT1 = SUB->getOperand(1);
17682 return SDValue();
17688 SDValue UABDHigh8Op0 =
17691 SDValue UABDHigh8Op1 =
17694 SDValue UABDHigh8 = DAG.getNode(IsZExt ? ISD::ABDU : ISD::ABDS, DL, MVT::v8i8,
17696 SDValue UABDL = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, UABDHigh8);
17699 SDValue UABDLo8Op0 =
17702 SDValue UABDLo8Op1 =
17705 SDValue UABDLo8 = DAG.getNode(IsZExt ? ISD::ABDU : ISD::ABDS, DL, MVT::v8i8,
17707 SDValue ZExtUABD = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, UABDLo8);
17708 SDValue UABAL = DAG.getNode(ISD::ADD, DL, MVT::v8i16, UABDL, ZExtUABD);
17711 SDValue UADDLP = DAG.getNode(AArch64ISD::UADDLP, DL, MVT::v4i32, UABAL);
17723 static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG,
17726 return SDValue();
17731 SDValue Op0 = N->getOperand(0);
17734 return SDValue();
17737 SDValue A = Op0;
17738 SDValue B;
17744 return SDValue();
17748 return SDValue();
17754 return SDValue();
17778 SDValue Zeros = DAG.getConstant(0, DL, TargetType);
17779 SDValue Dot = DAG.getNode(DotOpcode, DL, Zeros.getValueType(), Zeros,
17785 SmallVector<SDValue, 4> SDotVec16;
17788 SDValue Zeros = DAG.getConstant(0, DL, MVT::v4i32);
17789 SDValue Op0 =
17792 SDValue Op1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v16i8, B,
17794 SDValue Dot =
17801 SDValue ConcatSDot16 =
17803 SDValue VecReduceAdd16 =
17810 SmallVector<SDValue, 4> SDotVec8;
17811 SDValue Zeros = DAG.getConstant(0, DL, MVT::v2i32);
17812 SDValue Vec8Op0 =
17815 SDValue Vec8Op1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, B,
17817 SDValue Dot =
17819 SDValue VecReudceAdd8 =
17829 static SDValue performUADDVAddCombine(SDValue A, SelectionDAG &DAG) {
17830 auto DetectAddExtract = [&](SDValue A) {
17835 SDValue Op0 = A.getOperand(0);
17836 SDValue Op1 = A.getOperand(1);
17840 return SDValue();
17841 SDValue Ext0 = Op0.getOperand(0);
17842 SDValue Ext1 = Op1.getOperand(0);
17846 return SDValue();
17851 return SDValue();
17856 return SDValue();
17862 if (SDValue R = DetectAddExtract(A))
17866 if (SDValue R = performUADDVAddCombine(A.getOperand(0), DAG))
17870 if (SDValue R = performUADDVAddCombine(A.getOperand(1), DAG))
17873 return SDValue();
17878 static SDValue performUADDVZextCombine(SDValue A, SelectionDAG &DAG) {
17884 return SDValue();
17885 SDValue Op0 = A.getOperand(0);
17886 SDValue Op1 = A.getOperand(1);
17888 return SDValue();
17889 SDValue Ext0 = Op0.getOperand(0);
17890 SDValue Ext1 = Op1.getOperand(0);
17896 return SDValue();
17899 SDValue Concat =
17907 SDValue Uaddlv =
17916 static SDValue performUADDVCombine(SDNode *N, SelectionDAG &DAG) {
17917 SDValue A = N->getOperand(0);
17919 if (SDValue R = performUADDVAddCombine(A, DAG))
17921 else if (SDValue R = performUADDVZextCombine(A, DAG))
17924 return SDValue();
17927 static SDValue performXorCombine(SDNode *N, SelectionDAG &DAG,
17931 return SDValue();
17936 SDValue
17942 return SDValue(N, 0); // Lower SDIV as SDIV
17950 return SDValue(N, 0);
17955 return SDValue();
17961 return SDValue();
17966 SDValue
17972 return SDValue(N, 0); // Lower SREM as SREM
17979 return SDValue(N, 0);
17984 return SDValue();
17988 return SDValue();
17991 SDValue N0 = N->getOperand(0);
17992 SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
17993 SDValue Zero = DAG.getConstant(0, DL, VT);
17994 SDValue CCVal, CSNeg;
17996 SDValue Cmp = getAArch64Cmp(N0, Zero, ISD::SETGE, CCVal, DAG, DL);
17997 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, Pow2MinusOne);
18003 SDValue CCVal = DAG.getConstant(AArch64CC::MI, DL, MVT_CC);
18006 SDValue Negs = DAG.getNode(AArch64ISD::SUBS, DL, VTs, Zero, N0);
18007 SDValue AndPos = DAG.getNode(ISD::AND, DL, VT, N0, Pow2MinusOne);
18008 SDValue AndNeg = DAG.getNode(ISD::AND, DL, VT, Negs, Pow2MinusOne);
18020 static std::optional<unsigned> IsSVECntIntrinsic(SDValue S) {
18047 static EVT calculatePreExtendType(SDValue Extend) {
18085 static SDValue performBuildShuffleExtendCombine(SDValue BV, SelectionDAG &DAG) {
18089 return SDValue();
18093 SDValue Extend = BV->getOperand(0);
18100 return SDValue();
18105 return SDValue();
18111 return SDValue();
18114 for (SDValue Op : drop_begin(BV->ops())) {
18121 return SDValue();
18124 SDValue NBV;
18130 SmallVector<SDValue, 8> NewOps;
18131 for (SDValue Op : BV->ops())
18149 static SDValue performMulVectorExtendCombine(SDNode *Mul, SelectionDAG &DAG) {
18153 return SDValue();
18155 SDValue Op0 = performBuildShuffleExtendCombine(Mul->getOperand(0), DAG);
18156 SDValue Op1 = performBuildShuffleExtendCombine(Mul->getOperand(1), DAG);
18160 return SDValue();
18169 static SDValue performMulVectorCmpZeroCombine(SDNode *N, SelectionDAG &DAG) {
18173 return SDValue();
18176 return SDValue();
18178 SDValue And = N->getOperand(0);
18179 SDValue Srl = And.getOperand(0);
18185 return SDValue();
18190 return SDValue();
18197 SDValue In = DAG.getNode(AArch64ISD::NVCAST, DL, HalfVT, Srl.getOperand(0));
18198 SDValue CM = DAG.getNode(AArch64ISD::CMLTz, DL, HalfVT, In);
18206 static SDValue performVectorExtCombine(SDNode *N, SelectionDAG &DAG) {
18215 return SDValue();
18219 return SDValue();
18221 SDValue N0 = N->getOperand(0).getOperand(0);
18222 SDValue N1 = N->getOperand(1).getOperand(0);
18233 SDValue NewN0 = DAG.getNode(N->getOperand(0).getOpcode(), DL, HalfVT, N0);
18234 SDValue NewN1 = DAG.getNode(N->getOperand(1).getOpcode(), DL, HalfVT, N1);
18235 SDValue NewOp = DAG.getNode(N->getOpcode(), DL, HalfVT, NewN0, NewN1);
18240 return SDValue();
18243 static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG,
18247 if (SDValue Ext = performMulVectorExtendCombine(N, DAG))
18249 if (SDValue Ext = performMulVectorCmpZeroCombine(N, DAG))
18251 if (SDValue Ext = performVectorExtCombine(N, DAG))
18255 return SDValue();
18262 SDValue N0 = N->getOperand(0);
18263 SDValue N1 = N->getOperand(1);
18264 SDValue MulOper;
18267 auto IsAddSubWith1 = [&](SDValue V) -> bool {
18270 SDValue Opnd = V->getOperand(1);
18281 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper);
18286 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper);
18292 return SDValue();
18303 return SDValue();
18323 return SDValue();
18328 return SDValue();
18335 auto Shl = [&](SDValue N0, unsigned N1) {
18337 return SDValue();
18340 return SDValue();
18341 SDValue RHS = DAG.getConstant(N1, DL, MVT::i64);
18344 auto Add = [&](SDValue N0, SDValue N1) {
18346 return SDValue();
18349 auto Sub = [&](SDValue N0, SDValue N1) {
18351 return SDValue();
18354 auto Negate = [&](SDValue N) {
18356 return SDValue();
18357 SDValue Zero = DAG.getConstant(0, DL, VT);
18446 SDValue MVal = Add(Shl(N0, ShiftM1), N0);
18456 SDValue MVal = Add(Shl(N0, CVM.getZExtValue()), N0);
18467 SDValue MVal = Sub(N0, Shl(N0, CVM.getZExtValue()));
18490 return SDValue();
18493 static SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N,
18510 return SDValue();
18520 return SDValue();
18527 SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
18529 SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst);
18530 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT,
18532 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd);
18536 return SDValue();
18539 static SDValue performIntToFpCombine(SDNode *N, SelectionDAG &DAG,
18543 if (SDValue Res = performVectorCompareAndMaskUnaryOpCombine(N, DAG))
18548 return SDValue();
18552 return SDValue();
18557 SDValue N0 = N->getOperand(0);
18563 SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(),
18569 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), Load.getValue(1));
18576 return SDValue();
18581 static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG,
18585 return SDValue();
18588 return SDValue();
18590 SDValue Op = N->getOperand(0);
18592 return SDValue();
18595 return SDValue();
18597 SDValue ConstVec = Op->getOperand(1);
18599 return SDValue();
18605 return SDValue();
18610 return SDValue();
18614 return SDValue();
18621 return SDValue();
18625 return SDValue();
18631 return SDValue();
18639 SDValue FixConv =
18650 static SDValue tryCombineToBSL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
18658 return SDValue();
18661 return SDValue();
18665 return SDValue();
18667 SDValue N0 = N->getOperand(0);
18669 return SDValue();
18671 SDValue N1 = N->getOperand(1);
18673 return SDValue();
18680 SDValue O0 = N0->getOperand(i);
18681 SDValue O1 = N1->getOperand(j);
18682 SDValue Sub, Add, SubSibling, AddSibling;
18747 return SDValue();
18760 static SDValue performANDORCSELCombine(SDNode *N, SelectionDAG &DAG) {
18762 SDValue CSel0 = N->getOperand(0);
18763 SDValue CSel1 = N->getOperand(1);
18767 return SDValue();
18770 return SDValue();
18776 return SDValue();
18778 SDValue Cmp0 = CSel0.getOperand(3);
18779 SDValue Cmp1 = CSel1.getOperand(3);
18783 return SDValue();
18791 return SDValue();
18794 SDValue CCmp, Condition;
18807 SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
18815 SDValue AbsOp1 =
18828 static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
18834 if (SDValue R = performANDORCSELCombine(N, DAG))
18838 return SDValue();
18840 if (SDValue Res = tryCombineToBSL(N, DCI, TLI))
18843 return SDValue();
18873 static SDValue performReinterpretCastCombine(SDNode *N) {
18874 SDValue LeafOp = SDValue(N, 0);
18875 SDValue Op = N->getOperand(0);
18881 return SDValue();
18884 static SDValue performSVEAndCombine(SDNode *N,
18887 SDValue Src = N->getOperand(0);
18892 SDValue UnpkOp = Src->getOperand(0);
18893 SDValue Dup = N->getOperand(1);
18896 return SDValue();
18901 return SDValue();
18935 SDValue And = DAG.getNode(ISD::AND, DL,
18942 return SDValue();
18952 return SDValue();
18954 SDValue Mask = N->getOperand(1);
18957 return SDValue();
18987 return SDValue();
18993 return SDValue();
18997 static SDValue performANDSETCCCombine(SDNode *N,
19003 SDValue SetCC = N->getOperand(0);
19008 // returns an empty SDValue to avoid applying the optimization to prevent
19012 return SDValue();
19018 SDValue Cmp;
19023 (Cmp = emitConjunction(DAG, SDValue(N, 0), CC))) {
19033 return SDValue();
19036 static SDValue performANDCombine(SDNode *N,
19039 SDValue LHS = N->getOperand(0);
19040 SDValue RHS = N->getOperand(1);
19043 if (SDValue R = performANDORCSELCombine(N, DAG))
19046 if (SDValue R = performANDSETCCCombine(N,DCI))
19050 return SDValue();
19058 return SDValue();
19062 return SDValue();
19071 SDValue NewOp;
19082 if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG,
19084 (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG,
19089 if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG,
19091 (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG,
19096 return SDValue();
19099 static SDValue performFADDCombine(SDNode *N,
19102 SDValue LHS = N->getOperand(0);
19103 SDValue RHS = N->getOperand(1);
19108 return SDValue();
19111 auto ReassocComplex = [&](SDValue A, SDValue B) {
19113 return SDValue();
19119 return SDValue();
19120 SDValue VCMLA = DAG.getNode(
19127 if (SDValue R = ReassocComplex(LHS, RHS))
19129 if (SDValue R = ReassocComplex(RHS, LHS))
19132 return SDValue();
19147 static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op,
19150 static bool isPredicateCCSettingOp(SDValue N) {
19170 static SDValue
19177 return SDValue();
19179 SDValue N0 = N->getOperand(0);
19184 return SDValue();
19189 return SDValue();
19193 SDValue Pg = getPTrue(DAG, SDLoc(N), VT, AArch64SVEPredPattern::all);
19200 static SDValue
19207 return SDValue();
19209 SDValue N0 = N->getOperand(0);
19213 return SDValue();
19216 SDValue Idx = N->getOperand(1);
19218 return SDValue();
19220 SDValue VS = Idx.getOperand(0);
19222 return SDValue();
19226 return SDValue();
19230 SDValue Pg = getPTrue(DAG, SDLoc(N), OpVT, AArch64SVEPredPattern::all);
19234 static SDValue
19238 if (SDValue Res = performFirstTrueTestVectorCombine(N, DCI, Subtarget))
19240 if (SDValue Res = performLastTrueTestVectorCombine(N, DCI, Subtarget))
19244 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
19267 SDValue N00 = N0->getOperand(IsStrict ? 1 : 0);
19268 SDValue N01 = N0->getOperand(IsStrict ? 2 : 1);
19271 SDValue Other = N00;
19281 SDValue Extract1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other,
19283 SDValue Extract2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other,
19292 SDValue Ret = DAG.getNode(N0->getOpcode(), DL,
19295 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Ret);
19297 return SDValue(N, 0);
19301 return SDValue();
19304 static SDValue performConcatVectorsCombine(SDNode *N,
19309 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
19313 return SDValue();
19328 SDValue N00 = N0->getOperand(0);
19329 SDValue N10 = N1->getOperand(0);
19355 all_of(N->op_values(), [SrcVT](SDValue V) {
19366 SmallVector<SDValue> Ops;
19369 SDValue V = N->getOperand(i);
19374 SDValue NewLoad = DAG.getLoad(FVT, dl, LD->getChain(),
19376 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLoad.getValue(1));
19396 auto isBitwiseVectorNegate = [](SDValue V) {
19400 SDValue N00 = N0->getOperand(0);
19401 SDValue N10 = N1->getOperand(0);
19418 return SDValue();
19426 SDValue N00 = N0->getOperand(0);
19427 SDValue N01 = N0->getOperand(1);
19428 SDValue N10 = N1->getOperand(0);
19429 SDValue N11 = N1->getOperand(1);
19432 SDValue Concat0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N00, N10);
19433 SDValue Concat1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N01, N11);
19438 auto IsRSHRN = [](SDValue Shr) {
19441 SDValue Op = Shr.getOperand(0);
19469 SDValue X = N0.getOperand(0).getOperand(0);
19470 SDValue Y = N1.isUndef() ? DAG.getUNDEF(X.getValueType())
19474 SDValue CC = DAG.getNode(ISD::CONCAT_VECTORS, dl, BVT, X, Y);
19475 SDValue Add = DAG.getNode(
19478 SDValue Shr =
19487 SDValue E0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N0.getOperand(0),
19489 SDValue E1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N0.getOperand(1),
19513 return SDValue();
19514 SDValue RHS = N1->getOperand(0);
19518 return SDValue();
19531 static SDValue
19535 return SDValue();
19539 return SDValue();
19541 SDValue V = N->getOperand(0);
19551 return SDValue();
19554 static SDValue
19558 SDValue Vec = N->getOperand(0);
19559 SDValue SubVec = N->getOperand(1);
19568 return SDValue();
19572 return SDValue();
19578 return SDValue();
19583 SDValue Lo, Hi;
19596 static SDValue tryCombineFixedPointConvert(SDNode *N,
19602 return SDValue();
19613 SDValue Op1 = N->getOperand(1);
19615 return SDValue();
19618 SDValue IID = N->getOperand(0);
19619 SDValue Shift = N->getOperand(2);
19620 SDValue Vec = Op1.getOperand(0);
19621 SDValue Lane = Op1.getOperand(1);
19630 return SDValue();
19637 return SDValue();
19639 SDValue Convert =
19659 static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG) {
19682 return SDValue();
19686 return SDValue();
19700 static bool isEssentiallyExtractHighSubvector(SDValue N) {
19713 const SDValue *Opnd0;
19714 const SDValue *Opnd1;
19720 const SDValue *Cmp;
19744 static bool isSetCC(SDValue Op, SetCCInfoAndKind &SetCCInfo) {
19787 static bool isSetCCOrZExtSetCC(const SDValue& Op, SetCCInfoAndKind &Info) {
19800 static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) {
19802 SDValue LHS = Op->getOperand(0);
19803 SDValue RHS = Op->getOperand(1);
19811 return SDValue();
19817 return SDValue();
19825 return SDValue();
19827 SDValue CCVal;
19828 SDValue Cmp;
19847 static SDValue performAddUADDVCombine(SDNode *N, SelectionDAG &DAG) {
19851 return SDValue();
19853 SDValue LHS = N->getOperand(0);
19854 SDValue RHS = N->getOperand(1);
19857 return SDValue();
19862 return SDValue();
19864 SDValue Op1 = LHS->getOperand(0);
19865 SDValue Op2 = RHS->getOperand(0);
19871 return SDValue();
19873 SDValue Val1 = Op1.getOperand(0);
19874 SDValue Val2 = Op2.getOperand(0);
19877 SDValue AddVal = DAG.getNode(ISD::ADD, DL, ValVT, Val1, Val2);
19886 static SDValue performAddCSelIntoCSinc(SDNode *N, SelectionDAG &DAG) {
19889 return SDValue();
19891 SDValue LHS = N->getOperand(0);
19892 SDValue RHS = N->getOperand(1);
19900 return SDValue();
19905 return SDValue();
19915 return SDValue();
19921 return SDValue();
19945 return SDValue();
19951 SDValue NewNode = DAG.getNode(ISD::ADD, DL, VT, RHS, SDValue(CTVal, 0));
19952 SDValue CCVal = DAG.getConstant(AArch64CC, DL, MVT::i32);
19953 SDValue Cmp = LHS.getOperand(3);
19959 static SDValue performAddDotCombine(SDNode *N, SelectionDAG &DAG) {
19962 return SDValue();
19964 SDValue Dot = N->getOperand(0);
19965 SDValue A = N->getOperand(1);
19967 auto isZeroDot = [](SDValue Dot) {
19975 return SDValue();
19981 static bool isNegatedInteger(SDValue Op) {
19985 static SDValue getNegatedInteger(SDValue Op, SelectionDAG &DAG) {
19988 SDValue Zero = DAG.getConstant(0, DL, VT);
19999 static SDValue performNegCSelCombine(SDNode *N, SelectionDAG &DAG) {
20000 if (!isNegatedInteger(SDValue(N, 0)))
20001 return SDValue();
20003 SDValue CSel = N->getOperand(1);
20005 return SDValue();
20007 SDValue N0 = CSel.getOperand(0);
20008 SDValue N1 = CSel.getOperand(1);
20013 return SDValue();
20015 SDValue N0N = getNegatedInteger(N0, DAG);
20016 SDValue N1N = getNegatedInteger(N1, DAG);
20035 static SDValue performAddSubLongCombine(SDNode *N,
20039 return SDValue();
20045 return SDValue();
20049 SDValue LHS = N->getOperand(0);
20050 SDValue RHS = N->getOperand(1);
20054 return SDValue();
20063 return SDValue();
20069 return SDValue();
20077 static bool isCMP(SDValue Op) {
20084 static std::optional<AArch64CC::CondCode> getCSETCondCode(SDValue Op) {
20090 SDValue OpLHS = Op.getOperand(0);
20091 SDValue OpRHS = Op.getOperand(1);
20102 static SDValue foldOverflowCheck(SDNode *Op, SelectionDAG &DAG, bool IsAdd) {
20103 SDValue CmpOp = Op->getOperand(2);
20105 return SDValue();
20109 return SDValue();
20112 return SDValue();
20115 SDValue CsetOp = CmpOp->getOperand(IsAdd ? 0 : 1);
20118 return SDValue();
20126 static SDValue foldADCToCINC(SDNode *N, SelectionDAG &DAG) {
20127 SDValue LHS = N->getOperand(0);
20128 SDValue RHS = N->getOperand(1);
20129 SDValue Cond = N->getOperand(2);
20132 return SDValue();
20138 SDValue CC = DAG.getConstant(AArch64CC::LO, DL, MVT::i32);
20142 static SDValue performBuildVectorCombine(SDNode *N,
20150 SDValue Elt0 = N->getOperand(0), Elt1 = N->getOperand(1),
20166 SDValue LowLanesSrcVec = Elt0->getOperand(0)->getOperand(0);
20168 SDValue HighLanes;
20189 SDValue HighLanesSrcVec = Elt2->getOperand(0)->getOperand(0);
20194 SDValue DoubleToSingleSticky =
20196 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32,
20206 SDValue Elt0 = N->getOperand(0), Elt1 = N->getOperand(1);
20223 SDValue SrcVec = Elt0->getOperand(0)->getOperand(0);
20226 SDValue HalfToSingle =
20228 SDValue SubvectorIdx = Elt0->getOperand(0)->getOperand(1);
20229 SDValue Extract = DAG.getNode(
20247 return SDValue();
20249 SDValue Elt0 = N->getOperand(0), Elt1 = N->getOperand(1);
20263 SDValue VecToExtend = Elt0->getOperand(0);
20266 return SDValue();
20268 SDValue SubvectorIdx = DAG.getVectorIdxConstant(Elt0->getConstantOperandVal(1), DL);
20270 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, DL, ExtVT, VecToExtend);
20275 return SDValue();
20278 static SDValue performTruncateCombine(SDNode *N,
20281 SDValue N0 = N->getOperand(0);
20284 SDValue Op = N0.getOperand(0);
20291 return SDValue();
20295 static bool isExtendOrShiftOperand(SDValue N) {
20320 static SDValue performAddCombineSubShift(SDNode *N, SDValue SUB, SDValue Z,
20322 auto IsOneUseExtend = [](SDValue N) {
20331 return SDValue();
20334 return SDValue();
20336 SDValue Shift = SUB.getOperand(0);
20338 return SDValue();
20343 SDValue Y = SUB.getOperand(1);
20344 SDValue NewSub = DAG.getNode(ISD::SUB, DL, VT, Z, Y);
20348 static SDValue performAddCombineForShiftedOperands(SDNode *N,
20353 return SDValue();
20359 return SDValue();
20362 SDValue LHS = N->getOperand(0);
20363 SDValue RHS = N->getOperand(1);
20365 if (SDValue Val = performAddCombineSubShift(N, LHS, RHS, DAG))
20367 if (SDValue Val = performAddCombineSubShift(N, RHS, LHS, DAG))
20384 return SDValue();
20389 static SDValue performSubAddMULCombine(SDNode *N, SelectionDAG &DAG) {
20391 return SDValue();
20393 SDValue Add = N->getOperand(1);
20394 SDValue X = N->getOperand(0);
20396 return SDValue();
20399 return SDValue();
20401 return SDValue();
20403 SDValue M1 = Add.getOperand(0);
20404 SDValue M2 = Add.getOperand(1);
20407 return SDValue();
20410 return SDValue();
20413 SDValue Sub = DAG.getNode(ISD::SUB, SDLoc(N), VT, X, M1);
20424 static SDValue
20429 return SDValue();
20432 return SDValue();
20435 return SDValue();
20438 return SDValue();
20440 auto performOpt = [&DAG, &N](SDValue Op0, SDValue Op1) -> SDValue {
20442 return SDValue();
20445 return SDValue();
20447 SDValue MulValue = Op1->getOperand(0);
20449 return SDValue();
20452 return SDValue();
20456 return SDValue();
20458 SDValue ScaledOp = convertToScalableVector(DAG, ScalableVT, Op0);
20459 SDValue NewValue =
20464 if (SDValue res = performOpt(N->getOperand(0), N->getOperand(1)))
20469 return SDValue();
20474 static SDValue performAddSubIntoVectorOp(SDNode *N, SelectionDAG &DAG) {
20478 return SDValue();
20479 SDValue Op0 = N->getOperand(0);
20480 SDValue Op1 = N->getOperand(1);
20486 return SDValue();
20489 return SDValue();
20501 return SDValue();
20508 static bool isLoadOrMultipleLoads(SDValue B, SmallVector<LoadSDNode *> &Loads) {
20509 SDValue BV = peekThroughOneUseBitcasts(B);
20579 static bool areLoadedOffsetButOtherwiseSame(SDValue Op0, SDValue Op1,
20634 static SDValue performExtBinopLoadFold(SDNode *N, SelectionDAG &DAG) {
20639 return SDValue();
20641 SDValue Other = N->getOperand(0);
20642 SDValue Shift = N->getOperand(1);
20648 return SDValue();
20655 return SDValue();
20657 SDValue Op0 = Other.getOperand(0);
20658 SDValue Op1 = Shift.getOperand(0).getOperand(0);
20662 return SDValue();
20674 return SDValue();
20677 std::function<SDValue(SDValue, SDValue, SelectionDAG &)> GenCombinedTree =
20678 [&GenCombinedTree](SDValue Op0, SDValue Op1, SelectionDAG &DAG) {
20690 SmallVector<SDValue> NewLoads;
20692 SDValue Load = DAG.getLoad(DLoadVT, SDLoc(L0), L0->getChain(),
20702 SmallVector<SDValue> Ops;
20707 SDValue NewOp = GenCombinedTree(Op0, Op1, DAG);
20720 SDValue Ext0, Ext1;
20725 SDValue SubL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, Op0.getValueType(),
20727 SDValue SubH =
20730 SDValue Extr0 =
20732 SDValue Extr1 =
20738 SDValue Ext = DAG.getNode(Other.getOpcode(), DL, DVT, NewOp);
20739 SDValue SubL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Ext,
20741 SDValue SubH =
20747 SDValue NShift =
20752 static SDValue performAddSubCombine(SDNode *N,
20755 if (SDValue Val = performAddUADDVCombine(N, DCI.DAG))
20757 if (SDValue Val = performAddDotCombine(N, DCI.DAG))
20759 if (SDValue Val = performAddCSelIntoCSinc(N, DCI.DAG))
20761 if (SDValue Val = performNegCSelCombine(N, DCI.DAG))
20763 if (SDValue Val = performVectorExtCombine(N, DCI.DAG))
20765 if (SDValue Val = performAddCombineForShiftedOperands(N, DCI.DAG))
20767 if (SDValue Val = performSubAddMULCombine(N, DCI.DAG))
20769 if (SDValue Val = performSVEMulAddSubCombine(N, DCI))
20771 if (SDValue Val = performAddSubIntoVectorOp(N, DCI.DAG))
20774 if (SDValue Val = performExtBinopLoadFold(N, DCI.DAG))
20787 static SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N,
20791 return SDValue();
20793 SDValue LHS = N->getOperand((IID == Intrinsic::not_intrinsic) ? 0 : 1);
20794 SDValue RHS = N->getOperand((IID == Intrinsic::not_intrinsic) ? 1 : 2);
20805 return SDValue();
20809 return SDValue();
20811 return SDValue();
20820 static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG) {
20832 return SDValue();
20838 return SDValue();
20885 SDValue Op = N->getOperand(1);
20908 return SDValue();
20914 static SDValue tryCombineCRC32(unsigned Mask, SDNode *N, SelectionDAG &DAG) {
20915 SDValue AndN = N->getOperand(2);
20917 return SDValue();
20921 return SDValue();
20927 static SDValue combineAcrossLanesIntrinsic(unsigned Opc, SDNode *N,
20937 static SDValue LowerSVEIntrinsicIndex(SDNode *N, SelectionDAG &DAG) {
20939 SDValue Op1 = N->getOperand(1);
20940 SDValue Op2 = N->getOperand(2);
20946 SDValue StepVector = DAG.getStepVector(DL, N->getValueType(0));
20947 SDValue Step = DAG.getNode(ISD::SPLAT_VECTOR, DL, N->getValueType(0), Op2);
20948 SDValue Mul = DAG.getNode(ISD::MUL, DL, N->getValueType(0), StepVector, Step);
20949 SDValue Base = DAG.getNode(ISD::SPLAT_VECTOR, DL, N->getValueType(0), Op1);
20953 static SDValue LowerSVEIntrinsicDUP(SDNode *N, SelectionDAG &DAG) {
20955 SDValue Scalar = N->getOperand(3);
20961 SDValue Passthru = N->getOperand(1);
20962 SDValue Pred = N->getOperand(2);
20967 static SDValue LowerSVEIntrinsicEXT(SDNode *N, SelectionDAG &DAG) {
20976 return SDValue();
20984 SDValue Op0 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(1));
20985 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(2));
20986 SDValue Op2 = DAG.getNode(ISD::MUL, dl, MVT::i32, N->getOperand(3),
20989 SDValue EXT = DAG.getNode(AArch64ISD::EXT, dl, ByteVT, Op0, Op1, Op2);
20993 static SDValue tryConvertSVEWideCompare(SDNode *N, ISD::CondCode CC,
20997 return SDValue();
20999 SDValue Comparator = N->getOperand(3);
21005 SDValue Pred = N->getOperand(1);
21006 SDValue Imm;
21026 return SDValue();
21040 return SDValue();
21047 return SDValue();
21049 SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, CmpVT, Imm);
21054 return SDValue();
21057 static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op,
21070 SDValue TVal = DAG.getConstant(1, DL, OutVT);
21071 SDValue FVal = DAG.getConstant(0, DL, OutVT);
21084 SDValue Test = DAG.getNode(
21090 SDValue CC = DAG.getConstant(getInvertedCondCode(Cond), DL, MVT::i32);
21091 SDValue Res = DAG.getNode(AArch64ISD::CSEL, DL, OutVT, FVal, TVal, CC, Test);
21095 static SDValue combineSVEReductionInt(SDNode *N, unsigned Opc,
21099 SDValue Pred = N->getOperand(1);
21100 SDValue VecToReduce = N->getOperand(2);
21105 SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, VecToReduce);
21109 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
21114 static SDValue combineSVEReductionFP(SDNode *N, unsigned Opc,
21118 SDValue Pred = N->getOperand(1);
21119 SDValue VecToReduce = N->getOperand(2);
21122 SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, VecToReduce);
21126 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
21131 static SDValue combineSVEReductionOrderedFP(SDNode *N, unsigned Opc,
21135 SDValue Pred = N->getOperand(1);
21136 SDValue InitVal = N->getOperand(2);
21137 SDValue VecToReduce = N->getOperand(3);
21142 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
21146 SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, InitVal, VecToReduce);
21157 static SDValue convertMergedOpToPredOp(SDNode *N, unsigned Opc,
21162 SDValue Pg = N->getOperand(1);
21163 SDValue Op1 = N->getOperand(SwapOperands ? 3 : 2);
21164 SDValue Op2 = N->getOperand(SwapOperands ? 2 : 3);
21175 return SDValue();
21178 static SDValue tryCombineWhileLo(SDNode *N,
21182 return SDValue();
21185 return SDValue();
21188 return SDValue();
21192 return SDValue();
21200 return SDValue();
21211 return SDValue();
21216 return SDValue();
21220 SDValue ID =
21222 SDValue Idx = N->getOperand(1);
21223 SDValue TC = N->getOperand(2);
21235 return SDValue(N, 0);
21238 static SDValue performIntrinsicCombine(SDNode *N,
21538 return SDValue();
21541 static bool isCheapToExtend(const SDValue &N) {
21547 static SDValue
21558 const SDValue SetCC = N->getOperand(0);
21560 const SDValue CCOp0 = SetCC.getOperand(0);
21561 const SDValue CCOp1 = SetCC.getOperand(1);
21564 return SDValue();
21574 const SDValue Ext1 =
21576 const SDValue Ext2 =
21584 return SDValue();
21587 static SDValue performExtendCombine(SDNode *N,
21598 SDValue NewABD =
21601 return SDValue();
21611 return SDValue();
21614 static SDValue splitStoreSplat(SelectionDAG &DAG, StoreSDNode &St,
21615 SDValue SplatVal, unsigned NumVecElts) {
21625 SDValue BasePtr = St.getBasePtr();
21629 SDValue NewST1 =
21643 SDValue OffsetPtr =
21684 static SDValue performLD1Combine(SDNode *N, SelectionDAG &DAG, unsigned Opc) {
21689 return SDValue();
21696 SDValue Ops[] = { N->getOperand(0), // Chain
21701 SDValue Load = DAG.getNode(Opc, DL, VTs, Ops);
21702 SDValue LoadChain = SDValue(Load.getNode(), 1);
21710 static SDValue performLDNT1Combine(SDNode *N, SelectionDAG &DAG) {
21720 SDValue PassThru = DAG.getConstant(0, DL, LoadVT);
21721 SDValue L = DAG.getMaskedLoad(LoadVT, DL, MINode->getChain(),
21728 SDValue Ops[] = { DAG.getNode(ISD::BITCAST, DL, VT, L), L.getValue(1) };
21736 static SDValue performLD1ReplicateCombine(SDNode *N, SelectionDAG &DAG) {
21747 SDValue Ops[] = {N->getOperand(0), N->getOperand(2), N->getOperand(3)};
21748 SDValue Load = DAG.getNode(Opcode, DL, {LoadVT, MVT::Other}, Ops);
21749 SDValue LoadChain = SDValue(Load.getNode(), 1);
21757 static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG) {
21759 SDValue Data = N->getOperand(2);
21762 SDValue InputVT = DAG.getValueType(DataVT);
21767 SDValue SrcNew;
21773 SDValue Ops[] = { N->getOperand(0), // Chain
21783 static SDValue performSTNT1Combine(SDNode *N, SelectionDAG &DAG) {
21786 SDValue Data = N->getOperand(2);
21815 static SDValue replaceZeroVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
21816 SDValue StVal = St.getValue();
21821 return SDValue();
21830 return SDValue();
21833 return SDValue();
21839 return SDValue();
21844 return SDValue();
21851 return SDValue();
21855 SDValue EltVal = StVal.getOperand(I);
21857 return SDValue();
21872 SDValue SplatVal =
21882 static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
21883 SDValue StVal = St.getValue();
21889 return SDValue();
21894 return SDValue();
21899 return SDValue();
21905 SDValue SplatVal;
21909 return SDValue();
21915 return SDValue();
21920 return SDValue();
21923 return SDValue();
21930 return SDValue();
21935 static SDValue splitStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
21941 return SDValue();
21943 SDValue StVal = S->getValue();
21947 return SDValue();
21952 if (SDValue ReplacedZeroSplat = replaceZeroVectorStore(DAG, *S))
21960 return SDValue();
21964 return SDValue();
21969 return SDValue();
21978 return SDValue();
21983 if (SDValue ReplacedSplat = replaceSplatVectorStore(DAG, *S))
21991 SDValue SubVector0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
21993 SDValue SubVector1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
21995 SDValue BasePtr = S->getBasePtr();
21996 SDValue NewST1 =
21999 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
22006 static SDValue performSpliceCombine(SDNode *N, SelectionDAG &DAG) {
22013 return SDValue();
22016 static SDValue performUnpackCombine(SDNode *N, SelectionDAG &DAG,
22032 SDValue Mask = MLD->getMask();
22036 SDValue(MLD, 0).hasOneUse() && Mask->getOpcode() == AArch64ISD::PTRUE &&
22049 SDValue PassThru = DAG.getConstant(0, DL, VT);
22050 SDValue NewLoad = DAG.getMaskedLoad(
22055 DAG.ReplaceAllUsesOfValueWith(SDValue(MLD, 1), NewLoad.getValue(1));
22062 return SDValue();
22068 SDValue Op0 = N->getOperand(0);
22079 static SDValue tryCombineExtendRShTrunc(SDNode *N, SelectionDAG &DAG) {
22081 SDValue Op0 = N->getOperand(0);
22082 SDValue Op1 = N->getOperand(1);
22087 return SDValue();
22090 SDValue ShiftValue = Op0.getOperand(1);
22092 return SDValue();
22095 SDValue Lo = Op0.getOperand(0);
22096 SDValue Hi = Op1.getOperand(0);
22099 return SDValue();
22100 SDValue OrigArg = Lo.getOperand(0);
22102 return SDValue();
22119 static SDValue trySimplifySrlAddToRshrnb(SDValue Srl, SelectionDAG &DAG,
22123 return SDValue();
22133 return SDValue();
22137 SDValue RShOperand;
22139 return SDValue();
22140 SDValue Rshrnb = DAG.getNode(
22146 static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG,
22149 SDValue Op0 = N->getOperand(0);
22150 SDValue Op1 = N->getOperand(1);
22158 SDValue SourceVec = Op0.getOperand(0);
22165 SDValue Uzp = DAG.getNode(N->getOpcode(), DL, WidenedResVT, SourceVec,
22174 return SDValue();
22196 SDValue BC = DAG.getBitcast(BCVT, Op0);
22197 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, BC);
22203 if (SDValue Urshr = tryCombineExtendRShTrunc(N, DAG))
22206 if (SDValue Rshrnb = trySimplifySrlAddToRshrnb(Op0, DAG, Subtarget))
22209 if (SDValue Rshrnb = trySimplifySrlAddToRshrnb(Op1, DAG, Subtarget))
22215 SDValue X = Op0.getOperand(0).getOperand(0);
22223 SDValue Z = Op1.getOperand(0).getOperand(1);
22230 return SDValue();
22246 return SDValue();
22248 SDValue SourceOp0 = peekThroughBitcasts(Op0);
22249 SDValue SourceOp1 = peekThroughBitcasts(Op1);
22256 SDValue Concat =
22267 return SDValue();
22273 return SDValue();
22288 return SDValue();
22291 SDValue UzpOp0 = DAG.getNode(ISD::BITCAST, DL, ResultTy, SourceOp0);
22292 SDValue UzpOp1 = DAG.getNode(ISD::BITCAST, DL, ResultTy, SourceOp1);
22293 SDValue UzpResult =
22316 static SDValue performGLD1Combine(SDNode *N, SelectionDAG &DAG) {
22335 SDValue Chain = N->getOperand(0);
22336 SDValue Pg = N->getOperand(1);
22337 SDValue Base = N->getOperand(2);
22338 SDValue Offset = N->getOperand(3);
22339 SDValue Ty = N->getOperand(4);
22351 SDValue ExtPg = Offset.getOperand(0);
22359 SDValue UnextendedOffset = Offset.getOperand(1);
22370 return SDValue();
22375 static SDValue performVectorShiftCombine(SDNode *N,
22381 SDValue Op = N->getOperand(0);
22396 return SDValue();
22402 return SDValue(N, 0);
22404 return SDValue();
22407 static SDValue performSunpkloCombine(SDNode *N, SelectionDAG &DAG) {
22414 SDValue CC = N->getOperand(0)->getOperand(0);
22416 SDValue Unpk = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, CC,
22421 return SDValue();
22426 static SDValue performPostLD1Combine(SDNode *N,
22430 return SDValue();
22436 return SDValue();
22442 return SDValue();
22445 SDValue Lane;
22450 return SDValue();
22457 return SDValue();
22466 return SDValue();
22475 return SDValue();
22478 SDValue Addr = LD->getOperand(1);
22479 SDValue Vector = N->getOperand(0);
22489 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
22510 SmallVector<SDValue, 8> Ops;
22522 SDValue UpdN = DAG.getMemIntrinsicNode(NewOp, SDLoc(N), SDTys, Ops,
22527 SDValue NewResults[] = {
22528 SDValue(LD, 0), // The result of load
22529 SDValue(UpdN.getNode(), 2) // Chain
22532 DCI.CombineTo(N, SDValue(UpdN.getNode(), 0)); // Dup/Inserted Result
22533 DCI.CombineTo(User, SDValue(UpdN.getNode(), 1)); // Write back register
22537 return SDValue();
22542 static bool performTBISimplification(SDValue Addr,
22557 static SDValue foldTruncStoreOfExt(SelectionDAG &DAG, SDNode *N) {
22563 return SDValue();
22564 SDValue Ext = Store->getValue();
22568 return SDValue();
22569 SDValue Orig = Ext->getOperand(0);
22571 return SDValue();
22576 return SDValue();
22595 static SDValue combineV3I8LoadExt(LoadSDNode *LD, SelectionDAG &DAG) {
22599 return SDValue();
22603 SDValue Chain = LD->getChain();
22604 SDValue BasePtr = LD->getBasePtr();
22609 SDValue L16 = DAG.getLoad(MVT::i16, DL, Chain, BasePtr, MMO);
22611 SDValue L8 = DAG.getLoad(MVT::i8, DL, Chain,
22616 SDValue Ext16 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, L16);
22617 SDValue Ext8 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, L8);
22620 SDValue Shl = DAG.getNode(ISD::SHL, DL, MVT::i32, Ext8,
22622 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::i32, Ext16, Shl);
22623 SDValue Cast = DAG.getNode(ISD::BITCAST, DL, MVT::v4i8, Or);
22626 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MemVT, Cast,
22628 SDValue TokenFactor = DAG.getNode(
22630 {SDValue(cast<SDNode>(L16), 1), SDValue(cast<SDNode>(L8), 1)});
22637 static SDValue performLOADCombine(SDNode *N,
22646 return SDValue(N, 0);
22648 if (SDValue Res = combineV3I8LoadExt(LD, DAG))
22652 return SDValue(N, 0);
22658 return SDValue(N, 0);
22661 SDValue Chain = LD->getChain();
22662 SDValue BasePtr = LD->getBasePtr();
22664 SmallVector<SDValue, 4> LoadOps;
22665 SmallVector<SDValue, 4> LoadOpsChain;
22676 SDValue NewPtr = DAG.getMemBasePlusOffset(
22679 SDValue NewLoad = DAG.getLoad(
22683 LoadOpsChain.push_back(SDValue(cast<SDNode>(NewLoad), 1));
22695 SDValue NewPtr = DAG.getMemBasePlusOffset(
22698 SDValue RemainingLoad =
22702 SDValue UndefVector = DAG.getUNDEF(NewVT);
22703 SDValue InsertIdx = DAG.getVectorIdxConstant(0, DL);
22704 SDValue ExtendedReminingLoad =
22708 LoadOpsChain.push_back(SDValue(cast<SDNode>(RemainingLoad), 1));
22712 SDValue ConcatVectors =
22715 SDValue ExtractSubVector =
22718 SDValue TokenFactor =
22723 static EVT tryGetOriginalBoolVectorType(SDValue Op, int Depth = 0) {
22737 for (SDValue Operand : Op->op_values()) {
22755 static SDValue vectorToScalarBitmask(SDNode *N, SelectionDAG &DAG) {
22757 SDValue ComparisonResult(N, 0);
22763 return SDValue();
22767 return SDValue();
22784 return SDValue();
22789 SmallVector<SDValue, 16> MaskConstants;
22800 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, MaskConstants);
22801 SDValue RepresentativeBits =
22804 SDValue UpperRepresentativeBits =
22807 SDValue Zipped = DAG.getNode(AArch64ISD::ZIP1, DL, VecVT,
22819 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, MaskConstants);
22820 SDValue RepresentativeBits =
22827 static SDValue combineBoolVectorAndTruncateStore(SelectionDAG &DAG,
22830 return SDValue();
22833 SDValue VecOp = Store->getValue();
22839 return SDValue();
22844 return SDValue();
22847 SDValue VectorBits = vectorToScalarBitmask(VecOp.getNode(), DAG);
22849 return SDValue();
22853 SDValue ExtendedBits = DAG.getZExtOrTrunc(VectorBits, DL, StoreVT);
22865 static SDValue combineI8TruncStore(StoreSDNode *ST, SelectionDAG &DAG,
22867 SDValue Value = ST->getValue();
22873 return SDValue();
22880 SDValue UndefVector = DAG.getUNDEF(WideVT);
22881 SDValue WideTrunc = DAG.getNode(
22884 SDValue Cast = DAG.getNode(
22889 SDValue Chain = ST->getChain();
22892 SDValue E2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8, Cast,
22895 SDValue Ptr2 = DAG.getMemBasePlusOffset(ST->getBasePtr(), Offset2, DL);
22898 SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8, Cast,
22901 SDValue Ptr1 = DAG.getMemBasePlusOffset(ST->getBasePtr(), Offset1, DL);
22904 SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8, Cast,
22911 static SDValue performSTORECombine(SDNode *N,
22916 SDValue Chain = ST->getChain();
22917 SDValue Value = ST->getValue();
22918 SDValue Ptr = ST->getBasePtr();
22926 if (SDValue Res = combineI8TruncStore(ST, DAG, Subtarget))
22942 if (SDValue Split = splitStores(N, DCI, DAG, Subtarget))
22947 return SDValue(N, 0);
22949 if (SDValue Store = foldTruncStoreOfExt(DAG, N))
22952 if (SDValue Store = combineBoolVectorAndTruncateStore(DAG, ST))
22958 return SDValue();
22959 if (SDValue Rshrnb =
22966 return SDValue();
22969 static SDValue performMSTORECombine(SDNode *N,
22974 SDValue Value = MST->getValue();
22975 SDValue Mask = MST->getMask();
23014 return SDValue();
23015 if (SDValue Rshrnb = trySimplifySrlAddToRshrnb(Value, DAG, Subtarget)) {
23023 return SDValue();
23027 static bool foldIndexIntoBase(SDValue &BasePtr, SDValue &Index, SDValue Scale,
23057 SDValue Add = Index.getOperand(0);
23058 SDValue ShiftOp = Index.getOperand(1);
23059 SDValue OffsetOp = Add.getOperand(1);
23078 SDValue &BasePtr, SDValue &Index,
23114 SDValue RHS = Index.getOperand(1);
23147 static SDValue performMaskedGatherScatterCombine(
23153 return SDValue();
23156 SDValue Chain = MGS->getChain();
23157 SDValue Scale = MGS->getScale();
23158 SDValue Index = MGS->getIndex();
23159 SDValue Mask = MGS->getMask();
23160 SDValue BasePtr = MGS->getBasePtr();
23164 return SDValue();
23169 SDValue PassThru = MGT->getPassThru();
23170 SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale};
23176 SDValue Data = MSC->getValue();
23177 SDValue Ops[] = {Chain, Data, Mask, BasePtr, Index, Scale};
23185 static SDValue performNEONPostLDSTCombine(SDNode *N,
23189 return SDValue();
23192 SDValue Addr = N->getOperand(AddrOpIdx);
23273 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
23283 SmallVector<SDValue, 8> Ops;
23303 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, SDLoc(N), SDTys, Ops,
23308 std::vector<SDValue> NewResults;
23310 NewResults.push_back(SDValue(UpdN.getNode(), i));
23312 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs + 1));
23314 DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs));
23318 return SDValue();
23324 bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType) {
23504 static SDValue performSubsToAndsCombine(SDNode *N, SDNode *SubsNode,
23510 return SDValue();
23515 return SDValue();
23518 return SDValue();
23520 return SDValue();
23524 return SDValue();
23530 SDValue ANDS = DAG.getNode(
23533 SDValue AArch64_CC =
23545 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), AArch64_CC,
23551 SDValue performCONDCombine(SDNode *N,
23561 return SDValue();
23570 return SDValue();
23572 if (SDValue Val = performSubsToAndsCombine(N, SubsNode, AndNode, DAG, CCIndex,
23585 return SDValue();
23587 SDValue AddValue = AndNode->getOperand(0);
23590 return SDValue();
23594 SDValue AddInputValue1 = AddValue.getNode()->getOperand(0);
23595 SDValue AddInputValue2 = AddValue.getNode()->getOperand(1);
23596 SDValue SubsInputValue = SubsNode->getOperand(1);
23603 return SDValue();
23610 return SDValue();
23615 return SDValue();
23621 SDValue Ops[] = { AddValue, SubsNode->getOperand(1) };
23623 SDValue NewValue = DAG.getNode(CondOpcode, SDLoc(SubsNode), VTs, Ops);
23626 return SDValue(N, 0);
23630 static SDValue performBRCONDCombine(SDNode *N,
23638 return SDValue();
23640 if (SDValue NV = performCONDCombine(N, DCI, DAG, 2, 3))
23642 SDValue Chain = N->getOperand(0);
23643 SDValue Dest = N->getOperand(1);
23644 SDValue CCVal = N->getOperand(2);
23645 SDValue Cmp = N->getOperand(3);
23650 return SDValue();
23654 return SDValue();
23659 return SDValue();
23661 SDValue LHS = Cmp.getOperand(0);
23662 SDValue RHS = Cmp.getOperand(1);
23667 return SDValue();
23673 return SDValue();
23677 return SDValue();
23680 SDValue BR;
23689 return SDValue();
23692 static SDValue foldCSELofCTTZ(SDNode *N, SelectionDAG &DAG) {
23694 SDValue SUBS = N->getOperand(3);
23695 SDValue Zero, CTTZ;
23704 return SDValue();
23709 return SDValue();
23715 return SDValue();
23717 SDValue X = CTTZ.getOpcode() == ISD::TRUNCATE
23722 return SDValue();
23727 SDValue BitWidthMinusOne =
23740 static SDValue foldCSELOfCSEL(SDNode *Op, SelectionDAG &DAG) {
23741 SDValue L = Op->getOperand(0);
23742 SDValue R = Op->getOperand(1);
23746 SDValue OpCmp = Op->getOperand(3);
23748 return SDValue();
23750 SDValue CmpLHS = OpCmp.getOperand(0);
23751 SDValue CmpRHS = OpCmp.getOperand(1);
23756 return SDValue();
23758 SDValue X = CmpLHS->getOperand(0);
23759 SDValue Y = CmpLHS->getOperand(1);
23761 return SDValue();
23770 return SDValue();
23774 SDValue Cond = CmpLHS->getOperand(3);
23779 return SDValue();
23784 return SDValue();
23789 SDValue CCValue = DAG.getConstant(CC, DL, MVT::i32);
23794 static SDValue performCSELCombine(SDNode *N,
23801 if (SDValue R = foldCSELOfCSEL(N, DAG))
23806 if (SDValue Folded = foldCSELofCTTZ(N, DAG))
23815 static SDValue tryToWidenSetCCOperands(SDNode *Op, SelectionDAG &DAG) {
23818 return SDValue();
23824 return SDValue();
23827 return SDValue();
23831 return SDValue();
23835 return SDValue();
23838 SDValue Op0ExtV;
23839 SDValue Op1ExtV;
23848 Op0ExtV = SDValue(Op0SExt, 0);
23851 Op0ExtV = SDValue(Op0ZExt, 0);
23854 return SDValue();
23860 static SDValue
23863 SDValue Vec = N->getOperand(0);
23873 return SDValue();
23876 static SDValue performSETCCCombine(SDNode *N,
23880 SDValue LHS = N->getOperand(0);
23881 SDValue RHS = N->getOperand(1);
23886 if (SDValue V = tryToWidenSetCCOperands(N, DAG))
23900 SDValue CSEL =
23916 SDValue TST = DAG.getNode(ISD::AND, DL, TstVT, LHS->getOperand(0),
23944 if (SDValue V = performOrXorChainCombine(N, DAG))
23947 return SDValue();
23952 static SDValue performFlagSettingCombine(SDNode *N,
23956 SDValue LHS = N->getOperand(0);
23957 SDValue RHS = N->getOperand(1);
23962 SDValue Res = DCI.DAG.getNode(GenericOpcode, DL, VT, N->ops());
23970 DCI.CombineTo(Generic, SDValue(N, 0));
23972 return SDValue();
23975 static SDValue performSetCCPunpkCombine(SDNode *N, SelectionDAG &DAG) {
23979 SDValue Pred = N->getOperand(0);
23980 SDValue LHS = N->getOperand(1);
23981 SDValue RHS = N->getOperand(2);
23986 return SDValue();
23988 SDValue Extract = LHS->getOperand(0);
23992 return SDValue();
23994 SDValue InnerSetCC = Extract->getOperand(0);
23996 return SDValue();
24002 SDValue InnerPred = InnerSetCC.getOperand(0);
24010 return SDValue();
24013 static SDValue
24019 SDValue Pred = N->getOperand(0);
24020 SDValue LHS = N->getOperand(1);
24021 SDValue RHS = N->getOperand(2);
24024 if (SDValue V = performSetCCPunpkCombine(N, DAG))
24053 return SDValue();
24060 static SDValue getTestBitOperand(SDValue Op, unsigned &Bit, bool &Invert,
24133 static SDValue performTBZCombine(SDNode *N,
24138 SDValue TestSrc = N->getOperand(1);
24139 SDValue NewTestSrc = getTestBitOperand(TestSrc, Bit, Invert, DAG);
24142 return SDValue();
24164 static SDValue trySwapVSelectOperands(SDNode *N, SelectionDAG &DAG) {
24170 return SDValue();
24171 SDValue SetCC = N->getOperand(0);
24173 return SDValue();
24177 return SDValue();
24184 return SDValue();
24202 static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG) {
24206 SDValue N0 = N->getOperand(0);
24218 SDValue SetCC = N->getOperand(0);
24221 SDValue CmpLHS = SetCC.getOperand(0);
24236 SmallVector<SDValue, 8> Ops(
24239 SDValue Val = DAG.getBuildVector(VT, SDLoc(N), Ops);
24252 return SDValue();
24258 return SDValue();
24260 SDValue IfTrue = N->getOperand(1);
24261 SDValue IfFalse = N->getOperand(2);
24273 static SDValue performSelectCombine(SDNode *N,
24276 SDValue N0 = N->getOperand(0);
24280 return SDValue();
24283 return SDValue();
24301 return SDValue();
24305 return SDValue();
24314 return SDValue();
24323 SDValue LHS =
24325 SDValue RHS =
24327 SDValue SetCC = DAG.getNode(ISD::SETCC, DL, CCVT, LHS, RHS, N0.getOperand(2));
24331 SDValue Mask = DAG.getVectorShuffle(CCVT, DL, SetCC, SetCC, DUPMask);
24338 static SDValue performDUPCombine(SDNode *N,
24346 SmallVector<SDValue> Ops(N->ops());
24349 return DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SDValue(LN, 0),
24363 SDValue EXTRACT_VEC_ELT = N->getOperand(0);
24376 return SDValue();
24380 static SDValue performNVCASTCombine(SDNode *N, SelectionDAG &DAG) {
24387 return SDValue();
24393 static SDValue performGlobalAddressCombine(SDNode *N, SelectionDAG &DAG,
24399 return SDValue();
24404 return SDValue();
24409 return SDValue();
24418 return SDValue();
24430 return SDValue();
24436 return SDValue();
24439 SDValue Result = DAG.getGlobalAddress(GV, DL, MVT::i64, Offset);
24444 static SDValue performCTLZCombine(SDNode *N, SelectionDAG &DAG,
24446 SDValue BR = N->getOperand(0);
24449 return SDValue();
24457 static SDValue getScaledOffsetForBitWidth(SelectionDAG &DAG, SDValue Offset,
24462 SDValue Shift = DAG.getConstant(Log2_32(BitWidth / 8), DL, MVT::i64);
24463 SDValue SplatShift = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Shift);
24495 static bool isValidImmForSVEVecImmAddrMode(SDValue Offset,
24502 static SDValue performScatterStoreCombine(SDNode *N, SelectionDAG &DAG,
24505 const SDValue Src = N->getOperand(2);
24515 return SDValue();
24524 return SDValue();
24528 SDValue Base = N->getOperand(4);
24531 SDValue Offset = N->getOperand(5);
24575 return SDValue();
24585 return SDValue();
24593 SDValue InputVT = DAG.getValueType(SrcVT);
24598 SDValue SrcNew;
24605 SDValue Ops[] = {N->getOperand(0), // Chain
24615 static SDValue performGatherLoadCombine(SDNode *N, SelectionDAG &DAG,
24626 return SDValue();
24630 SDValue Base = N->getOperand(3);
24633 SDValue Offset = N->getOperand(4);
24683 return SDValue();
24698 SDValue OutVT = DAG.getValueType(RetVT);
24703 SDValue Ops[] = {N->getOperand(0), // Chain
24707 SDValue Load = DAG.getNode(Opcode, DL, VTs, Ops);
24708 SDValue LoadChain = SDValue(Load.getNode(), 1);
24721 static SDValue
24725 SDValue Src = N->getOperand(0);
24742 SDValue ExtOp = Src->getOperand(0);
24752 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ExtOp.getValueType(),
24759 return SDValue();
24762 return SDValue();
24827 return SDValue();
24834 return SDValue();
24839 SmallVector<SDValue, 5> Ops;
24843 SDValue ExtLoad = DAG.getNode(NewOpc, SDLoc(N), VTs, Ops);
24848 return SDValue(N, 0);
24854 static SDValue legalizeSVEGatherPrefetchOffsVec(SDNode *N, SelectionDAG &DAG) {
24856 SDValue Offset = N->getOperand(OffsetPos);
24860 return SDValue();
24865 SmallVector<SDValue, 5> Ops(N->op_begin(), N->op_end());
24877 static SDValue combineSVEPrefetchVecBaseImmOff(SDNode *N, SelectionDAG &DAG,
24882 return SDValue();
24885 SmallVector<SDValue, 5> Ops(N->op_begin(), N->op_end());
24898 static bool isLanes1toNKnownZero(SDValue Op) {
24921 static SDValue removeRedundantInsertVectorElt(SDNode *N) {
24923 SDValue InsertVec = N->getOperand(0);
24924 SDValue InsertElt = N->getOperand(1);
24925 SDValue InsertIdx = N->getOperand(2);
24929 return SDValue();
24932 return SDValue();
24935 return SDValue();
24937 SDValue ExtractVec = InsertElt.getOperand(0);
24938 SDValue ExtractIdx = InsertElt.getOperand(1);
24942 return SDValue();
24948 return SDValue();
24951 return SDValue();
24957 static SDValue
24959 if (SDValue Res = removeRedundantInsertVectorElt(N))
24965 static SDValue performFPExtendCombine(SDNode *N, SelectionDAG &DAG,
24968 SDValue N0 = N->getOperand(0);
24973 return SDValue();
24988 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
24997 return SDValue(N, 0); // Return N so it doesn't get rechecked!
25000 return SDValue();
25003 static SDValue performBSPExpandForSVE(SDNode *N, SelectionDAG &DAG,
25009 return SDValue();
25013 SDValue Mask = N->getOperand(0);
25014 SDValue In1 = N->getOperand(1);
25015 SDValue In2 = N->getOperand(2);
25017 SDValue InvMask = DAG.getNOT(DL, Mask, VT);
25018 SDValue Sel = DAG.getNode(ISD::AND, DL, VT, Mask, In1);
25019 SDValue SelInv = DAG.getNode(ISD::AND, DL, VT, InvMask, In2);
25023 static SDValue performDupLane128Combine(SDNode *N, SelectionDAG &DAG) {
25026 SDValue Insert = N->getOperand(0);
25028 return SDValue();
25031 return SDValue();
25036 return SDValue();
25038 SDValue Bitcast = Insert.getOperand(1);
25040 return SDValue();
25042 SDValue Subvec = Bitcast.getOperand(0);
25045 return SDValue();
25050 SDValue NewInsert =
25053 SDValue NewDuplane128 = DAG.getNode(AArch64ISD::DUPLANE128, DL, NewSubvecVT,
25059 static SDValue tryCombineMULLWithUZP1(SDNode *N,
25063 return SDValue();
25065 SDValue LHS = N->getOperand(0);
25066 SDValue RHS = N->getOperand(1);
25068 SDValue ExtractHigh;
25069 SDValue ExtractLow;
25070 SDValue TruncHigh;
25071 SDValue TruncLow;
25090 return SDValue();
25095 SDValue TruncHighOp = TruncHigh.getOperand(0);
25099 return SDValue();
25116 SDValue ExtractHighSrcVec = ExtractHigh.getOperand(0);
25162 SDValue TruncLowOp =
25167 return SDValue();
25175 SDValue UZP1 =
25177 SDValue HighIdxCst =
25179 SDValue NewTruncHigh =
25185 SDValue NewTruncLow = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, TruncLowVT,
25190 return SDValue(N, 0);
25193 static SDValue performMULLCombine(SDNode *N,
25196 if (SDValue Val =
25200 if (SDValue Val = tryCombineMULLWithUZP1(N, DCI, DAG))
25203 return SDValue();
25206 static SDValue
25220 return SDValue();
25224 return SDValue();
25226 SDValue ZEXT = N->getOperand(0);
25228 return SDValue();
25230 SDValue EXTRACT_VEC_ELT = ZEXT.getOperand(0);
25233 return SDValue();
25236 return SDValue();
25238 SDValue UADDLV = EXTRACT_VEC_ELT.getOperand(0);
25242 return SDValue();
25246 SDValue EXTRACT_SUBVEC =
25249 SDValue NVCAST =
25255 SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
25296 SDValue(N, 0), DemandedBits, DemandedElts, DCI))
25297 return SDValue();
25571 SDValue A = DAG.getNode(
25574 SDValue B = DAG.getNode(
25600 return SDValue();
25608 SDValue &Chain) const {
25614 SDValue TCChain = Chain;
25662 SDValue &Base,
25663 SDValue &Offset,
25682 auto IsUndefOrZero = [](SDValue V) {
25711 bool AArch64TargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
25712 SDValue &Offset,
25716 SDValue Ptr;
25733 SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset,
25736 SDValue Ptr;
25757 SmallVectorImpl<SDValue> &Results,
25760 SDValue Op = N->getOperand(0);
25778 SDValue VectorBits = vectorToScalarBitmask(Op.getNode(), DAG);
25784 SmallVectorImpl<SDValue> &Results,
25788 SDValue Op = N->getOperand(0);
25792 SDValue Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtendVT, Op);
25793 SDValue CastVal = DAG.getNode(ISD::BITCAST, DL, CastVT, Vec);
25794 SDValue IdxZero = DAG.getVectorIdxConstant(0, DL);
25800 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
25802 SDValue Op = N->getOperand(0);
25833 SDValue CastResult = getSVESafeBitCast(getSVEContainerType(VT), Op, DAG);
25851 static void ReplaceAddWithADDP(SDNode *N, SmallVectorImpl<SDValue> &Results,
25862 SDValue X = N->getOperand(0);
25883 SDValue Addp = DAG.getNode(AArch64ISD::ADDP, N, LoHi.first.getValueType(),
25900 SmallVectorImpl<SDValue> &Results,
25904 SDValue Lo, Hi;
25908 SDValue InterVal = DAG.getNode(InterOp, dl, LoVT, Lo, Hi);
25909 SDValue SplitVal = DAG.getNode(AcrossOp, dl, LoVT, InterVal);
25914 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
25915 SDValue In = N->getOperand(0);
25943 SDValue Half = DAG.getNode(Opcode, DL, ExtendedHalfVT, N->getOperand(0));
25948 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
25953 SDValue RegClass =
25955 SDValue SubReg0 = DAG.getTargetConstant(AArch64::sube64, dl, MVT::i32);
25956 SDValue SubReg1 = DAG.getTargetConstant(AArch64::subo64, dl, MVT::i32);
25957 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
25958 return SDValue(
25963 SmallVectorImpl<SDValue> &Results,
25973 SDValue Ops[] = {
26006 SDValue Lo = DAG.getTargetExtractSubreg(SubReg1, SDLoc(N), MVT::i64,
26007 SDValue(CmpSwap, 0));
26008 SDValue Hi = DAG.getTargetExtractSubreg(SubReg2, SDLoc(N), MVT::i64,
26009 SDValue(CmpSwap, 0));
26012 Results.push_back(SDValue(CmpSwap, 1)); // Chain out
26038 SDValue Ops[] = {N->getOperand(1), Desired.first, Desired.second,
26046 SDValue(CmpSwap, 0), SDValue(CmpSwap, 1)));
26047 Results.push_back(SDValue(CmpSwap, 3));
26126 SmallVectorImpl<SDValue> &Results,
26144 const SDValue &Chain = N->getOperand(0);
26145 const SDValue &Ptr = N->getOperand(1);
26146 const SDValue &Val128 = N->getOperand(2);
26147 std::pair<SDValue, SDValue> Val2x64 =
26164 SDValue Ops[] = {Val2x64.first, Val2x64.second, Ptr, Chain};
26174 SDValue Lo = SDValue(AtomicInst, 0), Hi = SDValue(AtomicInst, 1);
26179 Results.push_back(SDValue(AtomicInst, 2)); // Chain out
26183 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
26195 Results.push_back(LowerVECREDUCE(SDValue(N, 0), DAG));
26204 if (SDValue Result = LowerCTPOP_PARITY(SDValue(N, 0), DAG))
26226 if (useSVEForFixedLengthVectorVT(SDValue(N, 0).getValueType()))
26228 LowerToPredicatedOp(SDValue(N, 0), DAG, AArch64ISD::MULHS_PRED));
26231 if (useSVEForFixedLengthVectorVT(SDValue(N, 0).getValueType()))
26233 LowerToPredicatedOp(SDValue(N, 0), DAG, AArch64ISD::MULHU_PRED));
26271 SDValue Result = DAG.getMemIntrinsicNode(
26279 SDValue Pair = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), MemVT,
26292 if (SDValue(N, 0).getValueType() == MVT::i128) {
26301 SDValue Result = DAG.getMemIntrinsicNode(
26308 SDValue Pair =
26390 SDValue Chain = N->getOperand(0);
26391 SDValue SysRegName = N->getOperand(1);
26393 SDValue Result = DAG.getNode(
26399 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128,
26840 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
27143 static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL,
27188 static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
27196 static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT) {
27204 static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) {
27210 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
27215 static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) {
27221 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
27226 SDValue AArch64TargetLowering::LowerFixedLengthVectorLoadToSVE(
27227 SDValue Op, SelectionDAG &DAG) const {
27243 SDValue NewLoad = DAG.getMaskedLoad(
27248 SDValue Result = NewLoad;
27261 SDValue MergedValues[2] = {Result, NewLoad.getValue(1)};
27265 static SDValue convertFixedMaskToScalableVector(SDValue Mask,
27284 SDValue AArch64TargetLowering::LowerFixedLengthVectorMLoadToSVE(
27285 SDValue Op, SelectionDAG &DAG) const {
27292 SDValue Mask = Load->getMask();
27302 SDValue PassThru;
27317 SDValue NewLoad = DAG.getMaskedLoad(
27322 SDValue Result = NewLoad;
27324 SDValue OldPassThru =
27330 SDValue MergedValues[2] = {Result, NewLoad.getValue(1)};
27335 SDValue AArch64TargetLowering::LowerFixedLengthVectorStoreToSVE(
27336 SDValue Op, SelectionDAG &DAG) const {
27368 SDValue AArch64TargetLowering::LowerFixedLengthVectorMStoreToSVE(
27369 SDValue Op, SelectionDAG &DAG) const {
27377 SDValue Mask = convertFixedMaskToScalableVector(Store->getMask(), DAG);
27385 SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE(
27386 SDValue Op, SelectionDAG &DAG) const {
27398 SDValue Op1 = convertToScalableVector(DAG, ContainerVT, Op.getOperand(0));
27399 SDValue Op2 = DAG.getTargetConstant(Log2_64(SplatVal), dl, MVT::i32);
27401 SDValue Pg = getPredicateForFixedLengthVector(DAG, dl, VT);
27402 SDValue Res =
27423 SDValue Op0 = DAG.getNode(ExtendOpcode, dl, WideVT, Op.getOperand(0));
27424 SDValue Op1 = DAG.getNode(ExtendOpcode, dl, WideVT, Op.getOperand(1));
27425 SDValue Div = DAG.getNode(Op.getOpcode(), dl, WideVT, Op0, Op1);
27430 &ExtendOpcode](SDValue Op) {
27431 SDValue IdxZero = DAG.getConstant(0, dl, MVT::i64);
27432 SDValue IdxHalf =
27434 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HalfVT, Op, IdxZero);
27435 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HalfVT, Op, IdxHalf);
27436 return std::pair<SDValue, SDValue>(
27444 SDValue Lo = DAG.getNode(Op.getOpcode(), dl, PromVT, Op0LoExt, Op1LoExt);
27445 SDValue Hi = DAG.getNode(Op.getOpcode(), dl, PromVT, Op0HiExt, Op1HiExt);
27446 SDValue LoTrunc = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Lo);
27447 SDValue HiTrunc = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Hi);
27451 SDValue AArch64TargetLowering::LowerFixedLengthVectorIntExtendToSVE(
27452 SDValue Op, SelectionDAG &DAG) const {
27457 SDValue Val = Op.getOperand(0);
27487 SDValue AArch64TargetLowering::LowerFixedLengthVectorTruncateToSVE(
27488 SDValue Op, SelectionDAG &DAG) const {
27493 SDValue Val = Op.getOperand(0);
27523 SDValue AArch64TargetLowering::LowerFixedLengthExtractVectorElt(
27524 SDValue Op, SelectionDAG &DAG) const {
27531 SDValue Op0 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(0));
27536 SDValue AArch64TargetLowering::LowerFixedLengthInsertVectorElt(
27537 SDValue Op, SelectionDAG &DAG) const {
27544 SDValue Op0 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(0));
27555 SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
27567 SmallVector<SDValue, 4> Operands = {Pg};
27568 for (const SDValue &V : Op->op_values()) {
27595 SmallVector<SDValue, 4> Operands = {Pg};
27596 for (const SDValue &V : Op->op_values()) {
27612 SDValue AArch64TargetLowering::LowerToScalableOp(SDValue Op,
27620 SmallVector<SDValue, 4> Ops;
27621 for (const SDValue &V : Op->op_values()) {
27641 SDValue AArch64TargetLowering::LowerVECREDUCE_SEQ_FADD(SDValue ScalarOp,
27644 SDValue AccOp = ScalarOp.getOperand(0);
27645 SDValue VecOp = ScalarOp.getOperand(1);
27655 SDValue Pg = getPredicateForVector(DAG, DL, SrcVT);
27656 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
27663 SDValue Rdx = DAG.getNode(AArch64ISD::FADDA_PRED, DL, ContainerVT,
27669 SDValue AArch64TargetLowering::LowerPredReductionToSVE(SDValue ReduceOp,
27672 SDValue Op = ReduceOp.getOperand(0);
27677 return SDValue();
27679 SDValue Pg = getPredicateForVector(DAG, DL, OpVT);
27683 return SDValue();
27696 SDValue ID =
27703 SDValue Cntp =
27709 return SDValue();
27712 SDValue AArch64TargetLowering::LowerReductionToSVE(unsigned Opcode,
27713 SDValue ScalarOp,
27716 SDValue VecOp = ScalarOp.getOperand(0);
27733 SDValue Pg = getPredicateForVector(DAG, DL, SrcVT);
27734 SDValue Rdx = DAG.getNode(Opcode, DL, RdxVT, Pg, VecOp);
27735 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT,
27745 SDValue
27746 AArch64TargetLowering::LowerFixedLengthVectorSelectToSVE(SDValue Op,
27753 SDValue Op1 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(1));
27754 SDValue Op2 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(2));
27770 SDValue AArch64TargetLowering::LowerFixedLengthVectorSetccToSVE(
27771 SDValue Op, SelectionDAG &DAG) const {
27794 SDValue
27795 AArch64TargetLowering::LowerFixedLengthBitcastToSVE(SDValue Op,
27809 SDValue AArch64TargetLowering::LowerFixedLengthConcatVectorsToSVE(
27810 SDValue Op, SelectionDAG &DAG) const {
27823 SmallVector<SDValue, 4> Ops;
27834 SDValue Pg = getPredicateForFixedLengthVector(DAG, DL, SrcVT);
27843 SDValue
27844 AArch64TargetLowering::LowerFixedLengthFPExtendToSVE(SDValue Op,
27850 SDValue Val = Op.getOperand(0);
27851 SDValue Pg = getPredicateForVector(DAG, DL, VT);
27868 SDValue
27869 AArch64TargetLowering::LowerFixedLengthFPRoundToSVE(SDValue Op,
27875 SDValue Val = Op.getOperand(0);
27880 SDValue Pg = getPredicateForVector(DAG, DL, RoundVT);
27892 SDValue
27893 AArch64TargetLowering::LowerFixedLengthIntToFPToSVE(SDValue Op,
27903 SDValue Val = Op.getOperand(0);
27909 SDValue Pg = getPredicateForFixedLengthVector(DAG, DL, VT);
27924 SDValue Pg = getPredicateForFixedLengthVector(DAG, DL, SrcVT);
27936 SDValue
27937 AArch64TargetLowering::LowerVECTOR_DEINTERLEAVE(SDValue Op,
27943 SDValue Even = DAG.getNode(AArch64ISD::UZP1, DL, OpVT, Op.getOperand(0),
27945 SDValue Odd = DAG.getNode(AArch64ISD::UZP2, DL, OpVT, Op.getOperand(0),
27950 SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op,
27957 SDValue Lo = DAG.getNode(AArch64ISD::ZIP1, DL, OpVT, Op.getOperand(0),
27959 SDValue Hi = DAG.getNode(AArch64ISD::ZIP2, DL, OpVT, Op.getOperand(0),
27964 SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
27969 SDValue Chain = HG->getChain();
27970 SDValue Inc = HG->getInc();
27971 SDValue Mask = HG->getMask();
27972 SDValue Ptr = HG->getBasePtr();
27973 SDValue Index = HG->getIndex();
27974 SDValue Scale = HG->getScale();
27975 SDValue IntID = HG->getIntID();
27987 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
27988 SDValue PassThru = DAG.getSplatVector(MemVT, DL, Zero);
27989 SDValue IncSplat = DAG.getSplatVector(MemVT, DL, Inc);
27990 SDValue Ops[] = {Chain, PassThru, Mask, Ptr, Index, Scale};
27998 SDValue Gather =
28002 SDValue GChain = Gather.getValue(1);
28005 SDValue ID = DAG.getTargetConstant(Intrinsic::aarch64_sve_histcnt, DL, IncVT);
28006 SDValue HistCnt =
28008 SDValue Mul = DAG.getNode(ISD::MUL, DL, MemVT, HistCnt, IncSplat);
28009 SDValue Add = DAG.getNode(ISD::ADD, DL, MemVT, Gather, Mul);
28016 SDValue ScatterOps[] = {GChain, Add, Mask, Ptr, Index, Scale};
28017 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), MemVT, DL,
28022 SDValue
28023 AArch64TargetLowering::LowerFixedLengthFPToIntToSVE(SDValue Op,
28033 SDValue Val = Op.getOperand(0);
28041 SDValue Pg = getPredicateForFixedLengthVector(DAG, DL, VT);
28053 SDValue Pg = getPredicateForFixedLengthVector(DAG, DL, SrcVT);
28065 static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
28081 return SDValue();
28094 SmallVector<SDValue, 8> TBLMask;
28097 SmallVector<SDValue, 8> AddRuntimeVLMask;
28102 return SDValue();
28125 return SDValue();
28140 SDValue VecMask =
28142 SDValue SVEMask = convertToScalableVector(DAG, MaskContainerVT, VecMask);
28144 SDValue Shuffle;
28153 SDValue VScale = (BitsPerElt == 64)
28156 SDValue VecMask =
28158 SDValue MulByMask = DAG.getNode(
28163 SDValue UpdatedVecMask =
28177 SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
28178 SDValue Op, SelectionDAG &DAG) const {
28186 SDValue Op1 = Op.getOperand(0);
28187 SDValue Op2 = Op.getOperand(1);
28202 SDValue SplatEl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarTy, Op1,
28215 SDValue Scalar = DAG.getNode(
28332 return SDValue();
28335 SDValue AArch64TargetLowering::getSVESafeBitCast(EVT VT, SDValue Op,
28378 SDValue N) const {
28387 SDValue Op, const APInt &OriginalDemandedBits,
28395 SDValue ShiftL = Op;
28396 SDValue ShiftR = Op->getOperand(0);
28426 SDValue Op0 = Op.getOperand(0);
28466 bool AArch64TargetLowering::isTargetCanonicalConstantNode(SDValue Op) const {