Lines Matching defs:SDValue
281 static bool isZeroingInactiveLanes(SDValue Op) {
342 static std::tuple<SDValue, SDValue>
343 extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) {
345 SDValue AddrDisc;
346 SDValue ConstDisc;
2266 static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm,
2342 SDValue New;
2353 SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
2354 New = SDValue(
2362 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
2408 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
2414 SDValue SrcOp = Op.getOperand(0);
2536 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
3414 static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
3415 static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
3416 static SDValue convertFixedMaskToScalableVector(SDValue Mask,
3418 static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT);
3419 static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
3600 static bool cannotBeIntMin(SDValue CheckedVal, SelectionDAG &DAG) {
3615 static bool isCMN(SDValue Op, ISD::CondCode CC, SelectionDAG &DAG) {
3622 static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &dl,
3623 SelectionDAG &DAG, SDValue Chain,
3642 static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
3677 const SDValue ANDSNode = DAG.getNode(AArch64ISD::ANDS, dl,
3747 static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
3748 ISD::CondCode CC, SDValue CCOp,
3782 SDValue Condition = DAG.getConstant(Predicate, DL, MVT_CC);
3785 SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
3803 static bool canEmitConjunction(const SDValue Val, bool &CanNegate,
3821 SDValue O0 = Val->getOperand(0);
3822 SDValue O1 = Val->getOperand(1);
3861 /// and sets @p OutCC to the flags that should be tested or returns SDValue() if
3865 static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val,
3866 AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp,
3871 SDValue LHS = Val->getOperand(0);
3872 SDValue RHS = Val->getOperand(1);
3888 SDValue ExtraCmp;
3910 SDValue LHS = Val->getOperand(0);
3917 SDValue RHS = Val->getOperand(1);
3964 SDValue CmpR = emitConjunctionRec(DAG, RHS, RHSCC, NegateR, CCOp, Predicate);
3967 SDValue CmpL = emitConjunctionRec(DAG, LHS, OutCC, NegateL, CmpR, RHSCC);
3977 static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val,
3982 return SDValue();
3984 return emitConjunctionRec(DAG, Val, OutCC, false, SDValue(), AArch64CC::AL);
3991 static unsigned getCmpOperandFoldingProfit(SDValue Op) {
3992 auto isSupportedExtend = [&](SDValue V) {
4025 static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
4026 SDValue &AArch64cc, SelectionDAG &DAG,
4097 SDValue TheLHS = LHSIsCMN ? LHS.getOperand(1) : LHS;
4098 SDValue TheRHS = RHSIsCMN ? RHS.getOperand(1) : RHS;
4107 SDValue Cmp;
4134 SDValue SExt =
4160 static std::pair<SDValue, SDValue>
4161 getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
4164 SDValue Value, Overflow;
4166 SDValue LHS = Op.getOperand(0);
4167 SDValue RHS = Op.getOperand(1);
4198 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
4205 SDValue SExtMul = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Value);
4210 SDValue UpperBits = DAG.getConstant(0xFFFFFFFF00000000, DL, MVT::i64);
4220 SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS);
4221 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value,
4229 SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS);
4250 SDValue AArch64TargetLowering::LowerXOR(SDValue Op, SelectionDAG &DAG) const {
4255 SDValue Sel = Op.getOperand(0);
4256 SDValue Other = Op.getOperand(1);
4269 return SDValue();
4271 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
4272 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
4274 SDValue Value, Overflow;
4276 SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
4294 SDValue LHS = Sel.getOperand(0);
4295 SDValue RHS = Sel.getOperand(1);
4296 SDValue TVal = Sel.getOperand(2);
4297 SDValue FVal = Sel.getOperand(3);
4320 SDValue CCVal;
4321 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
4337 static SDValue valueToCarryFlag(SDValue Value, SelectionDAG &DAG, bool Invert) {
4340 SDValue Op0 = Invert ? DAG.getConstant(0, DL, VT) : Value;
4341 SDValue Op1 = Invert ? Value : DAG.getConstant(1, DL, VT);
4342 SDValue Cmp =
4349 static SDValue carryFlagToValue(SDValue Glue, EVT VT, SelectionDAG &DAG,
4353 SDValue Zero = DAG.getConstant(0, DL, VT);
4354 SDValue One = DAG.getConstant(1, DL, VT);
4356 SDValue CC = DAG.getConstant(Cond, DL, MVT::i32);
4361 static SDValue overflowFlagToValue(SDValue Glue, EVT VT, SelectionDAG &DAG) {
4364 SDValue Zero = DAG.getConstant(0, DL, VT);
4365 SDValue One = DAG.getConstant(1, DL, VT);
4366 SDValue CC = DAG.getConstant(AArch64CC::VS, DL, MVT::i32);
4372 static SDValue lowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG,
4378 return SDValue();
4381 SDValue OpLHS = Op.getOperand(0);
4382 SDValue OpRHS = Op.getOperand(1);
4383 SDValue OpCarryIn = valueToCarryFlag(Op.getOperand(2), DAG, InvertCarry);
4388 SDValue Sum = DAG.getNode(Opcode, DL, DAG.getVTList(VT0, MVT::Glue), OpLHS,
4391 SDValue OutFlag =
4398 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
4401 return SDValue();
4406 SDValue Value, Overflow;
4410 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
4411 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
4416 SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
4429 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
4459 static void simplifySetCCIntoEq(ISD::CondCode &CC, SDValue &LHS, SDValue &RHS,
4479 SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
4483 SDValue SrcVal = Op.getOperand(0);
4493 return SDValue();
4509 SDValue Op0 = Op.getOperand(IsStrict ? 1 : 0);
4517 SDValue Ext1 =
4526 return SDValue();
4542 SDValue Ext =
4544 SDValue Shift =
4553 return SDValue();
4557 return SDValue();
4560 SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
4564 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
4574 auto ImmV = [&](int I) -> SDValue { return DAG.getConstant(I, DL, I32); };
4576 SDValue NaN;
4577 SDValue Narrow;
4592 SDValue Pg = getPredicateForVector(DAG, DL, MVT::nxv2f32);
4596 SmallVector<SDValue, 3> NewOps;
4603 return SDValue();
4606 SDValue Lsb = DAG.getNode(ISD::SRL, DL, I32, Narrow, ImmV(16));
4608 SDValue RoundingBias = DAG.getNode(ISD::ADD, DL, I32, Lsb, ImmV(0x7fff));
4617 SDValue IsNaN = DAG.getSetCC(DL, CondVT, SrcVal, SrcVal, ISD::SETUO);
4636 SDValue Narrow = SrcVal;
4637 SDValue NaN;
4652 return SDValue();
4655 SDValue One = DAG.getConstant(1, dl, I32);
4656 SDValue Lsb = DAG.getNode(ISD::SRL, dl, I32, Narrow,
4659 SDValue RoundingBias =
4667 SDValue IsNaN = DAG.getSetCC(
4682 SDValue Result = DAG.getTargetExtractSubreg(AArch64::hsub, dl, VT, Narrow);
4690 return SDValue();
4696 return SDValue();
4699 SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
4727 SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NewVT, MVT::Other},
4743 SDValue Cv = DAG.getNode(Op.getOpcode(), dl, {InVT, MVT::Other},
4745 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, Cv);
4748 SDValue Cv =
4760 SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {ExtVT, MVT::Other},
4765 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0));
4773 SDValue Extract = DAG.getNode(
4787 SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
4790 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
4800 SDValue Ext =
4816 return SDValue();
4819 SDValue
4820 AArch64TargetLowering::LowerVectorFP_TO_INT_SAT(SDValue Op,
4824 SDValue SrcVal = Op.getOperand(0);
4839 return SDValue();
4845 SDValue SrcVal2;
4862 return SDValue();
4875 SDValue Res = DAG.getNode(Op.getOpcode(), DL, DstVT, SrcVal,
4878 SDValue Res2 = DAG.getNode(Op.getOpcode(), DL, DstVT, SrcVal2,
4890 return SDValue();
4893 SDValue NativeCvt = DAG.getNode(Op.getOpcode(), DL, IntVT, SrcVal,
4895 SDValue NativeCvt2 =
4898 : SDValue();
4899 SDValue Sat, Sat2;
4901 SDValue MinC = DAG.getConstant(
4903 SDValue Min = DAG.getNode(ISD::SMIN, DL, IntVT, NativeCvt, MinC);
4904 SDValue Min2 = SrcVal2 ? DAG.getNode(ISD::SMIN, DL, IntVT, NativeCvt2, MinC) : SDValue();
4905 SDValue MaxC = DAG.getConstant(
4908 Sat2 = SrcVal2 ? DAG.getNode(ISD::SMAX, DL, IntVT, Min2, MaxC) : SDValue();
4910 SDValue MinC = DAG.getConstant(
4913 Sat2 = SrcVal2 ? DAG.getNode(ISD::UMIN, DL, IntVT, NativeCvt2, MinC) : SDValue();
4924 SDValue AArch64TargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
4928 SDValue SrcVal = Op.getOperand(0);
4946 return SDValue();
4960 return SDValue();
4962 SDValue NativeCvt =
4964 SDValue Sat;
4966 SDValue MinC = DAG.getConstant(
4968 SDValue Min = DAG.getNode(ISD::SMIN, DL, DstVT, NativeCvt, MinC);
4969 SDValue MaxC = DAG.getConstant(
4973 SDValue MinC = DAG.getConstant(
4981 SDValue AArch64TargetLowering::LowerVectorXRINT(SDValue Op,
4984 SDValue Src = Op.getOperand(0);
4994 SDValue FOp = DAG.getNode(ISD::FRINT, DL, CastVT, Src);
5001 SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op,
5009 SDValue In = Op.getOperand(IsStrict ? 1 : 0);
5036 SDValue Val = DAG.getNode(Op.getOpcode(), dl, {F32, MVT::Other},
5078 SDValue Extract = DAG.getNode(
5091 SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
5097 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
5105 SDValue Val = DAG.getNode(Op.getOpcode(), dl, {PromoteVT, MVT::Other},
5162 SDValue SignBit;
5168 SDValue SrcHi = DAG.getNode(ISD::AND, DL, MVT::i64, SrcVal,
5170 SDValue SrcLo = DAG.getNode(ISD::AND, DL, MVT::i64, SrcVal,
5172 SDValue Highest =
5175 SDValue Zero64 = DAG.getConstant(0, DL, MVT::i64);
5176 SDValue ToRound =
5178 SDValue Rounded =
5183 SDValue RoundedBits = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Rounded);
5188 SDValue HasHighest = DAG.getSetCC(
5193 SDValue HasLo = DAG.getSetCC(
5198 SDValue NeedsAdjustment =
5202 SDValue AdjustedBits =
5204 SDValue Adjusted = DAG.getNode(ISD::BITCAST, DL, MVT::f64, AdjustedBits);
5223 return SDValue();
5229 return SDValue();
5232 SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
5237 SDValue Arg = Op.getOperand(0);
5253 SDValue Callee =
5262 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
5268 SDValue AArch64TargetLowering::LowerBITCAST(SDValue Op,
5290 return SDValue();
5292 SDValue ExtResult =
5304 return SDValue();
5310 return SDValue();
5327 getConstantLaneNumOfExtractHalfOperand(SDValue &Op) {
5340 static bool isExtendedBUILD_VECTOR(SDValue N, SelectionDAG &DAG,
5347 for (const SDValue &Elt : N->op_values()) {
5366 static SDValue skipExtensionForVectorMULL(SDValue N, SelectionDAG &DAG) {
5376 static bool isSignExtended(SDValue N, SelectionDAG &DAG) {
5382 static bool isZeroExtended(SDValue N, SelectionDAG &DAG) {
5388 static bool isAddSubSExt(SDValue N, SelectionDAG &DAG) {
5391 SDValue N0 = N.getOperand(0);
5392 SDValue N1 = N.getOperand(1);
5399 static bool isAddSubZExt(SDValue N, SelectionDAG &DAG) {
5402 SDValue N0 = N.getOperand(0);
5403 SDValue N1 = N.getOperand(1);
5410 SDValue AArch64TargetLowering::LowerGET_ROUNDING(SDValue Op,
5418 SDValue Chain = Op.getOperand(0);
5419 SDValue FPCR_64 = DAG.getNode(
5423 SDValue FPCR_32 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, FPCR_64);
5424 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPCR_32,
5426 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
5428 SDValue AND = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
5433 SDValue AArch64TargetLowering::LowerSET_ROUNDING(SDValue Op,
5436 SDValue Chain = Op->getOperand(0);
5437 SDValue RMValue = Op->getOperand(1);
5459 SDValue Ops[] = {
5461 SDValue FPCR =
5471 SDValue Ops2[] = {
5477 SDValue AArch64TargetLowering::LowerGET_FPMODE(SDValue Op,
5480 SDValue Chain = Op->getOperand(0);
5483 SDValue Ops[] = {
5485 SDValue FPCR =
5491 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPCR);
5496 SDValue AArch64TargetLowering::LowerSET_FPMODE(SDValue Op,
5499 SDValue Chain = Op->getOperand(0);
5500 SDValue Mode = Op->getOperand(1);
5503 SDValue FPCR = DAG.getZExtOrTrunc(Mode, DL, MVT::i64);
5506 SDValue Ops2[] = {
5511 SDValue AArch64TargetLowering::LowerRESET_FPMODE(SDValue Op,
5514 SDValue Chain = Op->getOperand(0);
5517 SDValue Ops[] = {
5519 SDValue FPCR =
5525 SDValue FPSCRMasked = DAG.getNode(
5530 SDValue Ops2[] = {Chain,
5536 static unsigned selectUmullSmull(SDValue &N0, SDValue &N1, SelectionDAG &DAG,
5592 SDValue AArch64TargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
5603 SDValue N0 = Op.getOperand(0);
5604 SDValue N1 = Op.getOperand(1);
5620 return SDValue();
5637 return SDValue();
5644 SDValue Op0;
5645 SDValue Op1 = skipExtensionForVectorMULL(N1, DAG);
5658 SDValue N00 = skipExtensionForVectorMULL(N0.getOperand(0), DAG);
5659 SDValue N01 = skipExtensionForVectorMULL(N0.getOperand(1), DAG);
5671 static inline SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT,
5679 static SDValue optimizeIncrementingWhile(SDValue Op, SelectionDAG &DAG,
5683 return SDValue();
5700 return SDValue();
5707 return SDValue();
5719 return SDValue();
5724 static SDValue getSVEPredicateBitCast(EVT VT, SDValue Op, SelectionDAG &DAG) {
5748 SDValue Reinterpret = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op);
5763 SDValue Mask = DAG.getConstant(1, DL, InVT);
5768 SDValue AArch64TargetLowering::getRuntimePStateSM(SelectionDAG &DAG,
5769 SDValue Chain, SDLoc DL,
5771 SDValue Callee = DAG.getExternalSymbol("__arm_sme_state",
5780 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
5781 SDValue Mask = DAG.getConstant(/*PSTATE.SM*/ 1, DL, MVT::i64);
5823 SDValue LowerSMELdrStr(SDValue N, SelectionDAG &DAG, bool IsLoad) {
5826 SDValue TileSlice = N->getOperand(2);
5827 SDValue Base = N->getOperand(3);
5828 SDValue VecNum = N->getOperand(4);
5830 SDValue VarAddend = VecNum;
5839 VarAddend = SDValue();
5844 SDValue CVal = DAG.getTargetConstant(C, DL, MVT::i32);
5856 SDValue Mul = DAG.getNode(
5870 SDValue LowerVectorMatch(SDValue Op, SelectionDAG &DAG) {
5872 SDValue ID =
5926 SDValue Match = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, Mask.getValueType(),
5936 SDValue AArch64TargetLowering::LowerINTRINSIC_VOID(SDValue Op,
5942 return SDValue(); // Don't custom lower most intrinsics.
5944 SDValue Chain = Op.getOperand(0);
5945 SDValue Addr = Op.getOperand(2);
5978 SDValue AArch64TargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
5984 return SDValue(); // Don't custom lower most intrinsics.
5987 SDValue Chain = Node->getChain();
5988 SDValue Dst = Op.getOperand(2);
5989 SDValue Val = Op.getOperand(3);
5991 SDValue Size = Op.getOperand(4);
5998 SDValue MS = SDI.EmitMOPS(AArch64::MOPSMemorySetTaggingPseudo, DAG, DL,
6011 SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
6016 default: return SDValue(); // Don't custom lower most intrinsics.
6024 SDValue Result = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64,
6035 SDValue LHS = Op.getOperand(1);
6036 SDValue RHS = Op.getOperand(2);
6050 auto TryVectorizeOperand = [](SDValue N, std::optional<uint64_t> NLane,
6053 SelectionDAG &DAG) -> SDValue {
6114 return SDValue();
6131 return SDValue();
6138 return SDValue();
6145 return SDValue();
6153 return SDValue();
6161 return SDValue();
6168 return SDValue();
6241 SDValue One = DAG.getConstant(1, dl, MVT::i32);
6242 SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, dl, Op.getValueType(), One);
6246 SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, dl, Op.getValueType(),
6252 SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, dl, Op.getValueType(),
6258 SDValue Data = Op.getOperand(3);
6343 SDValue Scalar = Op.getOperand(2);
6415 SDValue FnOp = Op.getOperand(1);
6416 SDValue IncomingFPOp = Op.getOperand(2);
6481 SDValue ID =
6496 SDValue Mask = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, WhileVT, ID,
6498 SDValue MaskAsInt = DAG.getNode(ISD::SIGN_EXTEND, dl, ContainerVT, Mask);
6513 SDValue ADDLV = DAG.getNode(
6517 SDValue EXTRACT_VEC_ELT = DAG.getNode(
6523 SDValue CttzOp = Op.getOperand(1);
6531 SDValue Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, NewVT, CttzOp);
6535 SDValue NewCttzElts =
6554 bool AArch64TargetLowering::shouldRemoveExtendFromGSIndex(SDValue Extend,
6570 bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
6645 SDValue AArch64TargetLowering::LowerMGATHER(SDValue Op,
6650 SDValue Chain = MGT->getChain();
6651 SDValue PassThru = MGT->getPassThru();
6652 SDValue Mask = MGT->getMask();
6653 SDValue BasePtr = MGT->getBasePtr();
6654 SDValue Index = MGT->getIndex();
6655 SDValue Scale = MGT->getScale();
6664 SDValue Ops[] = {Chain, DAG.getUNDEF(VT), Mask, BasePtr, Index, Scale};
6665 SDValue Load =
6668 SDValue Select = DAG.getSelect(DL, VT, Mask, Load, PassThru);
6685 SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale};
6726 SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale};
6727 SDValue Load =
6732 SDValue Result = convertFromScalableVector(DAG, PromotedVT, Load);
6744 SDValue AArch64TargetLowering::LowerMSCATTER(SDValue Op,
6749 SDValue Chain = MSC->getChain();
6750 SDValue StoreVal = MSC->getValue();
6751 SDValue Mask = MSC->getMask();
6752 SDValue BasePtr = MSC->getBasePtr();
6753 SDValue Index = MSC->getIndex();
6754 SDValue Scale = MSC->getScale();
6773 SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale};
6816 SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale};
6825 SDValue AArch64TargetLowering::LowerMLOAD(SDValue Op, SelectionDAG &DAG) const {
6834 SDValue PassThru = LoadNode->getPassThru();
6835 SDValue Mask = LoadNode->getMask();
6840 SDValue Load = DAG.getMaskedLoad(
6846 SDValue Result = DAG.getSelect(DL, VT, Mask, Load, PassThru);
6852 static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST,
6858 SDValue Value = ST->getValue();
6867 SDValue Undef = DAG.getUNDEF(MVT::i16);
6868 SDValue UndefVec = DAG.getBuildVector(MVT::v4i16, DL,
6871 SDValue TruncExt = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16,
6873 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, TruncExt);
6876 SDValue ExtractTrunc = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32,
6886 SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
6892 SDValue Value = StoreNode->getValue();
6926 SDValue Lo =
6930 SDValue Hi =
6935 SDValue Result = DAG.getMemIntrinsicNode(
6944 SDValue Value = StoreNode->getValue();
6946 SDValue Chain = StoreNode->getChain();
6947 SDValue Base = StoreNode->getBasePtr();
6950 SDValue Part = DAG.getNode(AArch64ISD::LS64_EXTRACT, Dl, MVT::i64,
6952 SDValue Ptr = DAG.getNode(ISD::ADD, Dl, PtrVT, Base,
6960 return SDValue();
6964 SDValue AArch64TargetLowering::LowerStore128(SDValue Op,
6978 SDValue Value = (StoreNode->getOpcode() == ISD::STORE ||
6987 SDValue Result = DAG.getMemIntrinsicNode(
6995 SDValue AArch64TargetLowering::LowerLOAD(SDValue Op,
7002 SmallVector<SDValue, 8> Ops;
7003 SDValue Base = LoadNode->getBasePtr();
7004 SDValue Chain = LoadNode->getChain();
7007 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Base,
7009 SDValue Part = DAG.getLoad(MVT::i64, DL, Chain, Ptr,
7013 Chain = SDValue(Part.getNode(), 1);
7015 SDValue Loaded = DAG.getNode(AArch64ISD::LS64_BUILD, DL, MVT::i64x8, Ops);
7024 return SDValue();
7028 return SDValue();
7037 return SDValue();
7039 SDValue Load = DAG.getLoad(MVT::f32, DL, LoadNode->getChain(),
7041 SDValue Chain = Load.getValue(1);
7042 SDValue Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f32, Load);
7043 SDValue BC = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Vec);
7044 SDValue Ext = DAG.getNode(ExtType, DL, MVT::v8i16, BC);
7052 SDValue AArch64TargetLowering::LowerVECTOR_COMPRESS(SDValue Op,
7055 SDValue Vec = Op.getOperand(0);
7056 SDValue Mask = Op.getOperand(1);
7057 SDValue Passthru = Op.getOperand(2);
7069 return SDValue();
7072 return SDValue();
7076 return SDValue();
7112 SDValue Compressed = DAG.getNode(
7118 SDValue Offset = DAG.getNode(
7122 SDValue IndexMask = DAG.getNode(
7151 SDValue AArch64TargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const {
7158 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
7161 SDValue Cmp =
7169 static SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) {
7170 SDValue Chain = Op.getOperand(0);
7171 SDValue Cond = Op.getOperand(1);
7172 SDValue Dest = Op.getOperand(2);
7175 if (SDValue Cmp = emitConjunction(DAG, Cond, CC)) {
7177 SDValue CCVal = DAG.getConstant(CC, dl, MVT::i32);
7182 return SDValue();
7187 static SDValue LowerFunnelShift(SDValue Op, SelectionDAG &DAG) {
7188 SDValue Shifts = Op.getOperand(2);
7206 return SDValue();
7209 static SDValue LowerFLDEXP(SDValue Op, SelectionDAG &DAG) {
7210 SDValue X = Op.getOperand(0);
7212 SDValue Exp = Op.getOperand(1);
7218 return SDValue();
7234 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
7235 SDValue VX =
7237 SDValue VExp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ExpVT,
7239 SDValue VPg = getPTrue(DAG, DL, XVT.changeVectorElementType(MVT::i1),
7241 SDValue FScale =
7245 SDValue Final =
7253 SDValue AArch64TargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
7263 SDValue AArch64TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
7270 SDValue Chain = Op.getOperand(0);
7271 SDValue Trmp = Op.getOperand(1); // trampoline
7272 SDValue FPtr = Op.getOperand(2); // nested function
7273 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
7306 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
7310 SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
7318 return SDValue();
7507 return SDValue();
7550 return SDValue();
7617 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
7628 SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
7638 SDValue Chain = Op.getOperand(0);
7639 SDValue SysRegName = Op.getOperand(1);
7640 std::pair<SDValue, SDValue> Pair =
7644 SDValue Result = DAG.getNode(AArch64ISD::MSRR, DL, MVT::Other, Chain,
7727 bool AArch64TargetLowering::isReassocProfitable(SelectionDAG &DAG, SDValue N0,
7728 SDValue N1) const {
7825 SDValue AArch64TargetLowering::LowerFormalArguments(
7826 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
7828 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
7846 DenseMap<unsigned, SDValue> CopiedRegs;
7888 SDValue Glue = Chain.getValue(1);
7890 SmallVector<SDValue, 16> ArgValues;
7906 SDValue FrameIdxN = DAG.getFrameIndex(FrameIdx, PtrVT);
7915 SDValue ArgValue;
8010 SDValue FIN;
8021 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
8078 SDValue Ptr = ArgValue;
8087 SDValue BytesIncrement;
8128 SDValue PStateSM;
8204 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, Reg, InVals[I]);
8239 SDValue SVL = DAG.getNode(AArch64ISD::RDSVL, DL, MVT::i64,
8242 SDValue Buffer;
8247 SDValue Size = DAG.getNode(ISD::MUL, DL, MVT::i64, SVL, SVL);
8258 SDValue BufferSize =
8263 SDValue Buffer;
8303 SDValue &Chain) const {
8312 SmallVector<SDValue, 8> MemOps;
8334 SDValue FIN;
8341 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
8350 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
8351 SDValue Store =
8374 SDValue FIN = DAG.getFrameIndex(FPRIdx, PtrVT);
8378 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f128);
8380 SDValue Store = DAG.getStore(Val.getValue(1), DL, Val, FIN,
8398 SDValue AArch64TargetLowering::LowerCallResult(
8399 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,
8401 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
8402 SDValue ThisVal, bool RequiresSMChange) const {
8403 DenseMap<unsigned, SDValue> CopiedRegs;
8419 SDValue Val = CopiedRegs.lookup(VA.getLocReg());
8553 SDValue Callee = CLI.Callee;
8556 const SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
8704 SDValue AArch64TargetLowering::addTokenForArgument(SDValue Chain,
8708 SmallVector<SDValue, 8> ArgChains;
8728 ArgChains.push_back(SDValue(L, 1));
8742 static bool checkZExtBool(SDValue Arg, const SelectionDAG &DAG) {
8866 SDValue AArch64TargetLowering::changeStreamingMode(SelectionDAG &DAG, SDLoc DL,
8867 bool Enable, SDValue Chain,
8868 SDValue InGlue,
8870 SDValue PStateSM) const {
8876 SDValue RegMask = DAG.getRegisterMask(TRI->getSMStartStopCallPreservedMask());
8877 SDValue MSROp =
8879 SDValue ConditionOp = DAG.getTargetConstant(Condition, DL, MVT::i64);
8880 SmallVector<SDValue> Ops = {Chain, MSROp, ConditionOp};
8895 static SDValue emitSMEStateSaveRestore(const AArch64TargetLowering &TLI,
8898 SDValue Chain, bool IsSave) {
8910 SDValue Callee =
8936 SDValue
8938 SmallVectorImpl<SDValue> &InVals) const {
8942 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
8944 SDValue Chain = CLI.Chain;
8945 SDValue Callee = CLI.Callee;
9087 SDValue TPIDR2ObjAddr = DAG.getFrameIndex(
9090 SDValue NumZaSaveSlicesAddr =
9093 SDValue NumZaSaveSlices = DAG.getNode(AArch64ISD::RDSVL, DL, MVT::i64,
9116 SDValue PStateSM;
9136 SDValue ZTFrameIdx;
9169 SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP,
9172 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
9174 SmallVector<SDValue, 8> MemOpChains;
9180 SDValue Val = DAG.getCopyFromReg(Chain, DL, F.VReg, F.VT);
9189 SDValue Arg = OutVals[i];
9263 SDValue Ptr = DAG.getFrameIndex(
9265 SDValue SpillSlot = Ptr;
9270 SDValue Store = DAG.getStore(Chain, DL, OutVals[i], Ptr, MPI);
9275 SDValue BytesIncrement;
9311 SDValue &Bits =
9313 [=](const std::pair<unsigned, SDValue> &Elt) {
9341 SDValue DstAddr;
9362 SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
9377 SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
9384 SDValue SizeNode =
9386 SDValue Cpy = DAG.getMemcpy(
9401 SDValue Store = DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo);
9408 SDValue ParamPtr = StackPtr;
9428 SDValue InGlue;
9436 SDValue NewChain = changeStreamingMode(
9491 std::vector<SDValue> Ops;
9527 SDValue AddrDisc, IntDisc;
9575 SDValue Ret = DAG.getNode(Opc, DL, MVT::Other, Ops);
9605 SDValue Result = LowerCallResult(
9607 IsThisReturn ? OutVals[0] : SDValue(), RequiresSMChange);
9641 SDValue RegMask = DAG.getRegisterMask(
9643 SDValue RestoreRoutine = DAG.getTargetExternalSymbol(
9645 SDValue TPIDR2_EL0 = DAG.getNode(
9651 SDValue Glue;
9652 SDValue TPIDR2Block = DAG.getFrameIndex(
9682 SDValue X = DAG.getCopyToReg(Result, DL, Reg, InVals[I]);
9715 SDValue
9716 AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
9719 const SmallVectorImpl<SDValue> &OutVals,
9730 SDValue Glue;
9731 SmallVector<std::pair<unsigned, SDValue>, 4> RetVals;
9737 SDValue Arg = OutVals[realRVLocIdx];
9767 SDValue &Bits =
9768 llvm::find_if(RetVals, [=](const std::pair<unsigned, SDValue> &Elt) {
9786 SDValue PStateSM = DAG.getCopyFromReg(Chain, DL, Reg, MVT::i64);
9788 /*Glue*/ SDValue(),
9792 /*Glue*/ SDValue(), AArch64SME::Always);
9796 SmallVector<SDValue, 4> RetOps(1, Chain);
9813 SDValue Val = DAG.getCopyFromReg(RetOps[0], DL, SRetReg,
9848 SDValue Arm64ECRetDest =
9866 SDValue AArch64TargetLowering::getTargetNode(GlobalAddressSDNode *N, EVT Ty,
9873 SDValue AArch64TargetLowering::getTargetNode(JumpTableSDNode *N, EVT Ty,
9879 SDValue AArch64TargetLowering::getTargetNode(ConstantPoolSDNode *N, EVT Ty,
9886 SDValue AArch64TargetLowering::getTargetNode(BlockAddressSDNode* N, EVT Ty,
9892 SDValue AArch64TargetLowering::getTargetNode(ExternalSymbolSDNode *N, EVT Ty,
9900 SDValue AArch64TargetLowering::getGOT(NodeTy *N, SelectionDAG &DAG,
9905 SDValue GotAddr = getTargetNode(N, Ty, DAG, AArch64II::MO_GOT | Flags);
9911 return SDValue(DAG.getMachineNode(AArch64::LOADgotAUTH, DL, Ty, GotAddr),
9918 SDValue AArch64TargetLowering::getAddrLarge(NodeTy *N, SelectionDAG &DAG,
9934 SDValue AArch64TargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
9939 SDValue Hi = getTargetNode(N, Ty, DAG, AArch64II::MO_PAGE | Flags);
9940 SDValue Lo = getTargetNode(N, Ty, DAG,
9942 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, Ty, Hi);
9948 SDValue AArch64TargetLowering::getAddrTiny(NodeTy *N, SelectionDAG &DAG,
9953 SDValue Sym = getTargetNode(N, Ty, DAG, Flags);
9957 SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
9973 SDValue Result;
9992 /// return an SDValue containing the final node.
10018 SDValue
10019 AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op,
10029 SDValue TLVPAddr =
10031 SDValue DescAddr = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TLVPAddr);
10035 SDValue Chain = DAG.getEntryNode();
10036 SDValue FuncTLVGet = DAG.getLoad(
10060 Chain = DAG.getCopyToReg(Chain, DL, AArch64::X0, DescAddr, SDValue());
10063 SmallVector<SDValue, 8> Ops;
10085 SDValue AArch64TargetLowering::LowerELFTLSLocalExec(const GlobalValue *GV,
10086 SDValue ThreadBase,
10090 SDValue TPOff, Addr;
10099 SDValue Var = DAG.getTargetGlobalAddress(
10101 return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
10111 SDValue HiVar = DAG.getTargetGlobalAddress(
10113 SDValue LoVar = DAG.getTargetGlobalAddress(
10116 Addr = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
10120 return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, Addr,
10131 SDValue HiVar = DAG.getTargetGlobalAddress(
10133 SDValue LoVar = DAG.getTargetGlobalAddress(
10136 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
10139 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
10151 SDValue HiVar = DAG.getTargetGlobalAddress(
10153 SDValue MiVar = DAG.getTargetGlobalAddress(
10156 SDValue LoVar = DAG.getTargetGlobalAddress(
10159 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
10162 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, MiVar,
10165 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
10191 SDValue AArch64TargetLowering::LowerELFTLSDescCallSeq(SDValue SymAddr,
10196 SDValue Chain = DAG.getEntryNode();
10204 SDValue Glue = Chain.getValue(1);
10209 SDValue
10210 AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op,
10238 SDValue TPOff;
10243 SDValue ThreadBase = DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT);
10262 SDValue SymAddr = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT,
10271 SDValue HiVar = DAG.getTargetGlobalAddress(
10273 SDValue LoVar = DAG.getTargetGlobalAddress(
10277 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, HiVar,
10280 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, LoVar,
10287 SDValue SymAddr =
10298 SDValue
10299 AArch64TargetLowering::LowerWindowsGlobalTLSAddress(SDValue Op,
10303 SDValue Chain = DAG.getEntryNode();
10307 SDValue TEB = DAG.getRegister(AArch64::X18, MVT::i64);
10311 SDValue TLSArray =
10320 SDValue TLSIndexHi =
10322 SDValue TLSIndexLo = DAG.getTargetExternalSymbol(
10324 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, TLSIndexHi);
10325 SDValue TLSIndex =
10333 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
10335 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
10342 SDValue TGAHi = DAG.getTargetGlobalAddress(
10344 SDValue TGALo = DAG.getTargetGlobalAddress(
10349 SDValue Addr =
10350 SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TLS, TGAHi,
10357 SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
10402 static SDValue LowerPtrAuthGlobalAddressStatically(
10403 SDValue TGA, SDLoc DL, EVT VT, AArch64PACKey::ID KeyC,
10404 SDValue Discriminator, SDValue AddrDiscriminator, SelectionDAG &DAG) {
10419 SDValue Key = DAG.getTargetConstant(KeyC, DL, MVT::i32);
10420 return SDValue(DAG.getMachineNode(AArch64::LOADauthptrstatic, DL, MVT::i64,
10425 SDValue
10426 AArch64TargetLowering::LowerPtrAuthGlobalAddress(SDValue Op,
10428 SDValue Ptr = Op.getOperand(0);
10430 SDValue AddrDiscriminator = Op.getOperand(2);
10465 SDValue TPtr = DAG.getTargetGlobalAddress(PtrGV, DL, VT, PtrOffsetC,
10470 SDValue Key = DAG.getTargetConstant(KeyC, DL, MVT::i32);
10471 SDValue Discriminator = DAG.getTargetConstant(DiscriminatorC, DL, MVT::i64);
10472 SDValue TAddrDiscriminator = !isNullConstant(AddrDiscriminator)
10479 return SDValue(
10488 return SDValue(
10502 std::pair<SDValue, uint64_t> lookThroughSignExtension(SDValue Val) {
10515 SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
10516 SDValue Chain = Op.getOperand(0);
10518 SDValue LHS = Op.getOperand(2);
10519 SDValue RHS = Op.getOperand(3);
10520 SDValue Dest = Op.getOperand(4);
10550 return SDValue();
10554 SDValue Value, Overflow;
10559 SDValue CCVal = DAG.getConstant(OFCC, dl, MVT::i32);
10581 SDValue Test = LHS.getOperand(0);
10597 SDValue Test = LHS.getOperand(0);
10626 SDValue CCVal;
10627 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
10637 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
10640 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
10641 SDValue BR1 =
10644 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
10652 SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
10656 return SDValue();
10662 SDValue In1 = Op.getOperand(0);
10663 SDValue In2 = Op.getOperand(1);
10680 SDValue Res = DAG.getNode(ISD::FCOPYSIGN, DL, ContainerVT, In1, In2);
10684 auto BitCast = [this](EVT VT, SDValue Op, SelectionDAG &DAG) {
10691 SDValue VecVal1, VecVal2;
10721 SDValue SignMaskV = DAG.getConstant(~APInt::getSignMask(BitWidth), DL, VecVT);
10733 SDValue BSP =
10745 SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
10749 return SDValue();
10757 return SDValue();
10760 SDValue Val = Op.getOperand(0);
10766 return SDValue();
10781 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val);
10782 SDValue AddV = DAG.getNode(AArch64ISD::UADDV, DL, MVT::v8i8, CtPop);
10793 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v16i8, Val);
10794 SDValue AddV = DAG.getNode(AArch64ISD::UADDV, DL, MVT::v16i8, CtPop);
10814 SDValue Zeros = DAG.getConstant(0, DL, DT);
10815 SDValue Ones = DAG.getConstant(1, DL, VT8Bit);
10844 SDValue AArch64TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
10851 SDValue RBIT = DAG.getNode(ISD::BITREVERSE, DL, VT, Op.getOperand(0));
10855 SDValue AArch64TargetLowering::LowerMinMax(SDValue Op,
10896 SDValue Op0 = Op.getOperand(0);
10897 SDValue Op1 = Op.getOperand(1);
10898 SDValue Cond = DAG.getSetCC(DL, VT, Op0, Op1, CC);
10902 SDValue AArch64TargetLowering::LowerBitreverse(SDValue Op,
10912 SDValue REVB;
10954 isOrXorChain(SDValue N, unsigned &Num,
10955 SmallVector<std::pair<SDValue, SDValue>, 16> &WorkList) {
10981 static SDValue performOrXorChainCombine(SDNode *N, SelectionDAG &DAG) {
10982 SDValue LHS = N->getOperand(0);
10983 SDValue RHS = N->getOperand(1);
10986 SmallVector<std::pair<SDValue, SDValue>, 16> WorkList;
10990 return SDValue();
10999 SDValue XOR0, XOR1;
11002 SDValue Cmp = DAG.getSetCC(DL, VT, XOR0, XOR1, Cond);
11005 SDValue CmpChain = DAG.getSetCC(DL, VT, XOR0, XOR1, Cond);
11013 return SDValue();
11016 SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
11024 SDValue Chain;
11027 SDValue LHS = Op.getOperand(OpNo + 0);
11028 SDValue RHS = Op.getOperand(OpNo + 1);
11034 SDValue TVal = DAG.getConstant(1, dl, VT);
11035 SDValue FVal = DAG.getConstant(0, dl, VT);
11055 SDValue CCVal;
11056 SDValue Cmp = getAArch64Cmp(
11062 SDValue Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp);
11072 SDValue Cmp;
11080 SDValue Res;
11084 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
11097 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
11098 SDValue CS1 =
11101 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
11107 SDValue AArch64TargetLowering::LowerSETCCCARRY(SDValue Op,
11110 SDValue LHS = Op.getOperand(0);
11111 SDValue RHS = Op.getOperand(1);
11114 return SDValue();
11117 SDValue Carry = Op.getOperand(2);
11119 SDValue InvCarry = valueToCarryFlag(Carry, DAG, true);
11120 SDValue Cmp = DAG.getNode(AArch64ISD::SBCS, DL, DAG.getVTList(VT, MVT::Glue),
11124 SDValue TVal = DAG.getConstant(1, DL, OpVT);
11125 SDValue FVal = DAG.getConstant(0, DL, OpVT);
11129 SDValue CCVal =
11137 SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
11138 SDValue RHS, SDValue TVal,
11139 SDValue FVal, const SDLoc &dl,
11176 SDValue Shift =
11190 SDValue Shift =
11312 SDValue CCVal;
11313 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
11323 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
11349 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
11350 SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
11355 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
11363 SDValue AArch64TargetLowering::LowerVECTOR_SPLICE(SDValue Op,
11386 SDValue Pred = getPTrue(DAG, DL, PredVT, *PredPattern);
11399 return SDValue();
11402 SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
11405 SDValue LHS = Op.getOperand(0);
11406 SDValue RHS = Op.getOperand(1);
11407 SDValue TVal = Op.getOperand(2);
11408 SDValue FVal = Op.getOperand(3);
11413 SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
11415 SDValue CCVal = Op->getOperand(0);
11416 SDValue TVal = Op->getOperand(1);
11417 SDValue FVal = Op->getOperand(2);
11424 SDValue Sel =
11431 SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, PredVT, CCVal);
11441 SDValue SplatVal = DAG.getSExtOrTrunc(CCVal, DL, SplatValVT);
11442 SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, PredVT, SplatVal);
11451 return SDValue();
11454 SDValue Value, Overflow;
11456 SDValue CCVal = DAG.getConstant(OFCC, DL, MVT::i32);
11464 SDValue LHS, RHS;
11484 SDValue Res = LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
11493 SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
11508 SDValue AArch64TargetLowering::LowerBR_JT(SDValue Op,
11513 SDValue JT = Op.getOperand(1);
11514 SDValue Entry = Op.getOperand(2);
11536 SDValue X16Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, AArch64::X16,
11537 Entry, SDValue());
11541 return SDValue(B, 0);
11547 SDValue JTInfo = DAG.getJumpTableDebugInfo(JTI, Op.getOperand(0), DL);
11548 return DAG.getNode(ISD::BRIND, DL, MVT::Other, JTInfo, SDValue(Dest, 0));
11551 SDValue AArch64TargetLowering::LowerBRIND(SDValue Op, SelectionDAG &DAG) const {
11552 SDValue Chain = Op.getOperand(0);
11553 SDValue Dest = Op.getOperand(1);
11559 return SDValue();
11565 return SDValue();
11569 SDValue Disc = DAG.getTargetConstant(*BADisc, DL, MVT::i64);
11570 SDValue Key = DAG.getTargetConstant(AArch64PACKey::IA, DL, MVT::i32);
11571 SDValue AddrDisc = DAG.getRegister(AArch64::XZR, MVT::i64);
11575 return SDValue(BrA, 0);
11578 SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
11595 SDValue AArch64TargetLowering::LowerBlockAddress(SDValue Op,
11606 SDValue TargetBA = DAG.getTargetBlockAddress(BA, BAN->getValueType(0));
11608 SDValue Disc = DAG.getTargetConstant(*BADisc, DL, MVT::i64);
11610 SDValue Key = DAG.getTargetConstant(AArch64PACKey::IA, DL, MVT::i32);
11611 SDValue AddrDisc = DAG.getRegister(AArch64::XZR, MVT::i64);
11616 return DAG.getCopyFromReg(SDValue(MOV, 0), DL, AArch64::X16, MVT::i64,
11617 SDValue(MOV, 1));
11630 SDValue AArch64TargetLowering::LowerDarwin_VASTART(SDValue Op,
11636 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(),
11644 SDValue AArch64TargetLowering::LowerWin64_VASTART(SDValue Op,
11650 SDValue FR;
11656 SDValue Val = DAG.getCopyFromReg(DAG.getEntryNode(), DL, VReg, MVT::i64);
11675 SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
11686 SDValue Chain = Op.getOperand(0);
11687 SDValue VAList = Op.getOperand(1);
11689 SmallVector<SDValue, 4> MemOps;
11693 SDValue Stack = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), PtrVT);
11702 SDValue GRTop, GRTopAddr;
11721 SDValue VRTop, VRTopAddr;
11737 SDValue GROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
11745 SDValue VROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
11754 SDValue AArch64TargetLowering::LowerVASTART(SDValue Op,
11767 SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
11787 SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
11794 SDValue Chain = Op.getOperand(0);
11795 SDValue Addr = Op.getOperand(1);
11800 SDValue VAList =
11832 SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
11837 SDValue APStore =
11843 SDValue WideFP =
11846 SDValue NarrowFP =
11849 SDValue Ops[] = { NarrowFP, WideFP.getValue(1) };
11857 SDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op,
11865 SDValue FrameAddr =
11878 SDValue AArch64TargetLowering::LowerSPONENTRY(SDValue Op,
11909 SDValue AArch64TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
11916 SDValue FrameAddr =
11918 SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
11923 SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op,
11932 SDValue ReturnAddress;
11934 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
11935 SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
11955 SDValue Chain =
11959 return SDValue(St, 0);
11964 SDValue AArch64TargetLowering::LowerShiftParts(SDValue Op,
11966 SDValue Lo, Hi;
12027 static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode,
12028 SDValue Operand, SelectionDAG &DAG,
12053 return SDValue();
12056 SDValue
12057 AArch64TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
12062 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
12066 SDValue
12067 AArch64TargetLowering::getSqrtResultForDenormInput(SDValue Op,
12072 SDValue AArch64TargetLowering::getSqrtEstimate(SDValue Operand,
12079 if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRSQRTE, Operand,
12089 SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate,
12101 return SDValue();
12104 SDValue AArch64TargetLowering::getRecipEstimate(SDValue Operand,
12108 if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRECPE, Operand,
12118 SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand,
12127 return SDValue();
12292 static SDValue getSETCC(AArch64CC::CondCode CC, SDValue NZCV, const SDLoc &DL,
12301 SDValue AArch64TargetLowering::LowerAsmOutputForConstraint(
12302 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
12306 return SDValue();
12319 SDValue CC = getSETCC(Cond, Glue, DL, DAG);
12321 SDValue Result;
12527 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
12529 SDValue Result;
12673 static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG) {
12686 static unsigned getExtFactor(SDValue &V) {
12694 SDValue ReconstructShuffleWithRuntimeMask(SDValue Op, SelectionDAG &DAG) {
12704 return SDValue();
12710 SDValue SourceVec;
12711 SDValue MaskSourceVec;
12712 SmallVector<SDValue, 16> AndMaskConstants;
12715 SDValue V = Op.getOperand(i);
12717 return SDValue();
12719 SDValue OperandSourceVec = V.getOperand(0);
12723 return SDValue();
12728 SDValue MaskSource = V.getOperand(1);
12731 return SDValue();
12737 return SDValue();
12746 return SDValue();
12748 SDValue MaskIdx = MaskSource.getOperand(1);
12751 return SDValue();
12758 return SDValue();
12760 return SDValue();
12785 SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
12796 SDValue Vec;
12803 SDValue ShuffleVec;
12810 ShuffleSourceInfo(SDValue Vec)
12814 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
12821 SDValue V = Op.getOperand(i);
12832 return SDValue();
12836 SDValue SourceVec = V.getOperand(0);
12855 SDValue V = Op.getOperand(I);
12878 SmallVector<SDValue, 16> TBLOperands;
12884 SDValue Src = Sources[i].Vec;
12895 SmallVector<SDValue, 16> TBLMask;
12903 SDValue Shuffle =
12913 return SDValue();
12960 return SDValue();
12966 return SDValue();
12982 SDValue VEXTSrc1 =
12985 SDValue VEXTSrc2 =
12994 return SDValue();
13033 SDValue Entry = Op.getOperand(i);
13061 return SDValue();
13064 SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
13068 SDValue Shuffle = DAG.getVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
13070 SDValue V;
13117 static SDValue ReconstructTruncateFromBuildVector(SDValue V, SelectionDAG &DAG) {
13119 return SDValue();
13125 SDValue BaseExt = V.getOperand(X * 4);
13131 return SDValue();
13132 SDValue Base = BaseExt.getOperand(0);
13135 SDValue Ext = V.getOperand(X * 4 + Y);
13140 return SDValue();
13149 SDValue Trunc[4] = {
13152 for (SDValue &V : Trunc)
13155 SDValue Concat0 =
13157 SDValue Concat1 =
13159 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, Concat0);
13160 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, Concat1);
13394 static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG) {
13397 SDValue V0 = Op.getOperand(0);
13398 SDValue V1 = Op.getOperand(1);
13403 return SDValue();
13408 return SDValue();
13427 static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1,
13428 SDValue V2, unsigned PFEntry, SDValue LHS,
13429 SDValue RHS, SelectionDAG &DAG,
13476 SDValue OpLHS = GeneratePerfectShuffle(
13481 SDValue Input;
13512 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
13515 SDValue Ins =
13521 SDValue OpLHS, OpRHS;
13563 SDValue Lane = DAG.getConstant(OpNum - OP_VDUP0, dl, MVT::i64);
13588 static SDValue GenerateTBL(SDValue Op, ArrayRef<int> ShuffleMask,
13591 SDValue V1 = Op.getOperand(0);
13592 SDValue V2 = Op.getOperand(1);
13615 SmallVector<SDValue, 8> TBLMask;
13627 SDValue V1Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V1);
13628 SDValue V2Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V2);
13630 SDValue Shuffle;
13675 static SDValue constructDup(SDValue V, int Lane, SDLoc dl, EVT VT,
13678 auto getScaledOffsetDup = [](SDValue BitCast, int &LaneC, MVT &CastVT) {
13686 SDValue Extract = BitCast.getOperand(0);
13744 static SDValue tryWidenMaskForShuffle(SDValue Op, SelectionDAG &DAG) {
13749 SDValue V0 = Op.getOperand(0);
13750 SDValue V1 = Op.getOperand(1);
13757 return SDValue();
13773 return SDValue();
13777 static SDValue tryToConvertShuffleOfTbl2ToTbl4(SDValue Op,
13780 SDValue Tbl1 = Op->getOperand(0);
13781 SDValue Tbl2 = Op->getOperand(1);
13783 SDValue Tbl2ID =
13791 return SDValue();
13795 return SDValue();
13797 SDValue Mask1 = Tbl1->getOperand(3);
13798 SDValue Mask2 = Tbl2->getOperand(3);
13799 SmallVector<SDValue, 16> TBLMaskParts(16, SDValue());
13807 return SDValue();
13812 SDValue TBLMask = DAG.getBuildVector(VT, dl, TBLMaskParts);
13813 SDValue ID =
13824 SDValue
13825 AArch64TargetLowering::LowerZERO_EXTEND_VECTOR_INREG(SDValue Op,
13829 SDValue SrcOp = Op.getOperand(0);
13836 return SDValue();
13837 SDValue Zeros = DAG.getConstant(0, dl, SrcVT);
13842 SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
13858 SDValue V1 = Op.getOperand(0);
13859 SDValue V2 = Op.getOperand(1);
13865 if (SDValue Res = tryToConvertShuffleOfTbl2ToTbl4(Op, ShuffleMask, DAG))
13918 SDValue Rev = DAG.getNode(AArch64ISD::REV64, dl, VT, V1);
13964 if (SDValue Concat = tryFormConcatFromShuffle(Op, DAG))
13971 SDValue DstVec = DstIsLeft ? V1 : V2;
13972 SDValue DstLaneV = DAG.getConstant(Anomaly, dl, MVT::i64);
13974 SDValue SrcVec = V1;
13980 SDValue SrcLaneV = DAG.getConstant(SrcLane, dl, MVT::i64);
13993 if (SDValue NewSD = tryWidenMaskForShuffle(Op, DAG))
14020 SmallVector<SDValue> MaskElts;
14025 SDValue MaskConst = DAG.getBuildVector(IVT, dl, MaskElts);
14035 SDValue AArch64TargetLowering::LowerSPLAT_VECTOR(SDValue Op,
14052 SDValue SplatVal = DAG.getAnyExtOrTrunc(Op.getOperand(0), DL, MVT::i64);
14055 SDValue ID =
14057 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
14066 SDValue AArch64TargetLowering::LowerDUPQLane(SDValue Op,
14072 return SDValue();
14076 return SDValue();
14079 SDValue Idx128 = Op.getOperand(2);
14084 SDValue CI = DAG.getTargetConstant(CIdx->getZExtValue(), DL, MVT::i64);
14088 SDValue V = DAG.getNode(ISD::BITCAST, DL, MVT::nxv2i64, Op.getOperand(1));
14094 SDValue One = DAG.getConstant(1, DL, MVT::i64);
14095 SDValue SplatOne = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, One);
14098 SDValue SV = DAG.getStepVector(DL, MVT::nxv2i64);
14102 SDValue Idx64 = DAG.getNode(ISD::ADD, DL, MVT::i64, Idx128, Idx128);
14103 SDValue SplatIdx64 = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Idx64);
14104 SDValue ShuffleMask = DAG.getNode(ISD::ADD, DL, MVT::nxv2i64, SV, SplatIdx64);
14107 SDValue TBL = DAG.getNode(AArch64ISD::TBL, DL, MVT::nxv2i64, V, ShuffleMask);
14135 static SDValue tryAdvSIMDModImm64(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
14146 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
14152 return SDValue();
14156 static SDValue tryAdvSIMDModImm32(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
14158 const SDValue *LHS = nullptr) {
14162 return SDValue();
14189 SDValue Mov;
14205 return SDValue();
14209 static SDValue tryAdvSIMDModImm16(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
14211 const SDValue *LHS = nullptr) {
14215 return SDValue();
14234 SDValue Mov;
14250 return SDValue();
14254 static SDValue tryAdvSIMDModImm321s(unsigned NewOp, SDValue Op,
14274 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
14281 return SDValue();
14285 static SDValue tryAdvSIMDModImm8(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
14296 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
14302 return SDValue();
14306 static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
14327 SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
14333 return SDValue();
14339 static bool isAllConstantBuildVector(const SDValue &PotentialBVec,
14356 static bool isAllInactivePredicate(SDValue N) {
14364 static bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) {
14409 static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG) {
14413 return SDValue();
14417 SDValue And;
14418 SDValue Shift;
14420 SDValue FirstOp = N->getOperand(0);
14422 SDValue SecondOp = N->getOperand(1);
14444 return SDValue();
14456 return SDValue();
14459 return SDValue();
14465 return SDValue();
14472 return SDValue();
14486 return SDValue();
14491 return SDValue();
14493 SDValue X = And.getOperand(0);
14494 SDValue Y = ShiftHasPredOp ? Shift.getOperand(1) : Shift.getOperand(0);
14495 SDValue Imm = ShiftHasPredOp ? DAG.getTargetConstant(C2, DL, MVT::i32)
14499 SDValue ResultSLI = DAG.getNode(Inst, DL, VT, X, Y, Imm);
14510 SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
14517 if (SDValue Res = tryLowerToSLI(Op.getNode(), DAG))
14524 SDValue LHS = Op.getOperand(0);
14538 SDValue NewOp;
14559 static SDValue NormalizeBuildVector(SDValue Op,
14569 SmallVector<SDValue, 16> Ops;
14570 for (SDValue Lane : Op->ops()) {
14590 static SDValue ConstantBuildVector(SDValue Op, SelectionDAG &DAG,
14601 SDValue NewOp;
14622 return SDValue();
14624 if (SDValue R = TryMOVIWithBits(DefBits))
14626 if (SDValue R = TryMOVIWithBits(UndefBits))
14643 if (SDValue NewOp = TryMOVIWithBits(NegBits)) {
14651 return SDValue();
14653 SDValue R;
14660 return SDValue();
14663 SDValue AArch64TargetLowering::LowerFixedLengthBuildVectorToSVE(
14664 SDValue Op, SelectionDAG &DAG) const {
14671 SDValue Start = DAG.getConstant(SeqInfo->first, DL, ContainerVT);
14672 SDValue Steps = DAG.getStepVector(DL, ContainerVT, SeqInfo->second);
14673 SDValue Seq = DAG.getNode(ISD::ADD, DL, ContainerVT, Start, Steps);
14680 return SDValue();
14682 auto IsExtractElt = [](SDValue Op) {
14690 return SDValue();
14693 SDValue ZeroI64 = DAG.getConstant(0, DL, MVT::i64);
14694 SmallVector<SDValue, 16> Intermediates = map_to_vector<16>(
14695 Op->op_values(), [&, Undef = DAG.getUNDEF(ContainerVT)](SDValue Op) {
14706 SDValue Op0 = DAG.getBitcast(ZipVT, Intermediates[I + 0]);
14707 SDValue Op1 = DAG.getBitcast(ZipVT, Intermediates[I + 1]);
14718 SDValue Vec = DAG.getBitcast(ContainerVT, Intermediates[0]);
14722 SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
14736 return SDValue();
14756 if (SDValue V = ConstantBuildVector(Op, DAG, Subtarget))
14781 SDValue Value;
14782 SDValue ConstantValue;
14783 SmallMapVector<SDValue, unsigned, 16> DifferentValueMap;
14785 SDValue PrevVal;
14787 SDValue V = Op.getOperand(i);
14855 SDValue V = Op.getOperand(i);
14862 SDValue N0 = N->getOperand(0);
14896 SDValue LHS =
14897 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
14899 SDValue RHS =
14900 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
14923 SDValue Lane = Value.getOperand(1);
14937 SmallVector<SDValue, 8> Ops;
14948 SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
14972 SDValue Val = DAG.getSplatBuildVector(VT, dl, ConstantValue);
14987 SDValue V = Op.getOperand(i);
14988 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
15002 return SDValue();
15008 if (SDValue M = ReconstructTruncateFromBuildVector(Op, DAG))
15013 if (SDValue Shuffle = ReconstructShuffle(Op, DAG))
15016 if (SDValue Shuffle = ReconstructShuffleWithRuntimeMask(Op, DAG))
15022 SmallVector<SDValue, 8> Ops(NumElts, Value);
15023 SDValue NewVector = LowerBUILD_VECTOR(DAG.getBuildVector(VT, dl, Ops), DAG);
15037 SmallVector<SDValue, 2> Vals;
15069 SmallVector<SDValue, 8> Ops1(NumElts / 2, Vals[0]);
15070 SmallVector<SDValue, 8> Ops2(NumElts / 2, Vals[1]);
15071 SDValue DUP1 =
15073 SDValue DUP2 =
15075 SDValue CONCAT_VECTORS =
15091 SDValue FirstLaneVal = Op.getOperand(0);
15093 SDValue Val = Op.getOperand(i);
15100 SmallVector<SDValue, 8> Ops1(NumElts, Vals[0]);
15101 SmallVector<SDValue, 8> Ops2(NumElts, Vals[1]);
15102 SDValue VEC1 = DAG.getBuildVector(VT, dl, Ops1);
15103 SDValue VEC2 = DAG.getBuildVector(VT, dl, Ops2);
15104 SDValue VECTOR_SHUFFLE =
15121 SDValue Vec = DAG.getUNDEF(VT);
15122 SDValue Op0 = Op.getOperand(0);
15145 SDValue V = Op.getOperand(i);
15148 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
15157 return SDValue();
15160 SDValue AArch64TargetLowering::LowerCONCAT_VECTORS(SDValue Op,
15179 SmallVector<SDValue> ConcatOps(Op->ops());
15182 SDValue V1 = ConcatOps[I];
15183 SDValue V2 = ConcatOps[I + 1];
15194 return SDValue();
15197 SDValue AArch64TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
15210 SDValue ExtendedVector =
15212 SDValue ExtendedValue =
15226 return SDValue();
15231 SDValue
15232 AArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
15242 SDValue Extend =
15245 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractTy,
15256 return SDValue();
15267 return SDValue();
15272 SDValue WideVec = WidenVector(Op.getOperand(0), DAG);
15284 SDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
15293 return SDValue();
15312 SDValue Vec = Op.getOperand(0);
15313 SDValue Idx = Op.getOperand(1);
15318 SDValue Container = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PackedVT,
15330 SDValue Splice = DAG.getNode(ISD::VECTOR_SPLICE, DL, InVT, Vec, Vec, Idx);
15334 return SDValue();
15337 SDValue AArch64TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
15345 SDValue Vec0 = Op.getOperand(0);
15346 SDValue Vec1 = Op.getOperand(1);
15352 return SDValue();
15359 SDValue Lo, Hi;
15380 return SDValue();
15401 SDValue Narrow;
15403 SDValue HiVec0 = DAG.getNode(AArch64ISD::UUNPKHI, DL, WideVT, Vec0);
15409 SDValue LoVec0 = DAG.getNode(AArch64ISD::UUNPKLO, DL, WideVT, Vec0);
15425 SDValue PTrue = getPTrue(DAG, DL, PredTy, *PredPattern);
15426 SDValue ScalableVec1 = convertToScalableVector(DAG, VT, Vec1);
15430 return SDValue();
15433 static bool isPow2Splat(SDValue Op, uint64_t &SplatVal, bool &Negated) {
15464 SDValue AArch64TargetLowering::LowerDIV(SDValue Op, SelectionDAG &DAG) const {
15479 SDValue Pg = getPredicateForScalableVector(DAG, dl, VT);
15480 SDValue Res =
15504 SDValue Op0Lo = DAG.getNode(UnpkLo, dl, WidenedVT, Op.getOperand(0));
15505 SDValue Op1Lo = DAG.getNode(UnpkLo, dl, WidenedVT, Op.getOperand(1));
15506 SDValue Op0Hi = DAG.getNode(UnpkHi, dl, WidenedVT, Op.getOperand(0));
15507 SDValue Op1Hi = DAG.getNode(UnpkHi, dl, WidenedVT, Op.getOperand(1));
15508 SDValue ResultLo = DAG.getNode(Op.getOpcode(), dl, WidenedVT, Op0Lo, Op1Lo);
15509 SDValue ResultHi = DAG.getNode(Op.getOpcode(), dl, WidenedVT, Op0Hi, Op1Hi);
15510 SDValue ResultLoCast = DAG.getNode(AArch64ISD::NVCAST, dl, VT, ResultLo);
15511 SDValue ResultHiCast = DAG.getNode(AArch64ISD::NVCAST, dl, VT, ResultHi);
15564 static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
15584 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
15595 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) {
15603 SDValue AArch64TargetLowering::LowerTRUNCATE(SDValue Op,
15611 SDValue Zero = DAG.getConstant(0, dl, OpVT);
15612 SDValue One = DAG.getConstant(1, dl, OpVT);
15613 SDValue And = DAG.getNode(ISD::AND, dl, OpVT, Op.getOperand(0), One);
15618 return SDValue();
15624 return SDValue();
15630 static bool canLowerSRLToRoundingShiftForVT(SDValue Shift, EVT ResVT,
15633 SDValue &RShOperand) {
15649 SDValue Add = Shift->getOperand(0);
15672 SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
15700 SDValue RShOperand;
15729 SDValue NegShift = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
15731 SDValue NegShiftLeft =
15741 static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS,
15765 return SDValue();
15767 SDValue Fcmeq;
15788 return SDValue();
15797 return SDValue();
15809 return SDValue();
15811 SDValue Cmeq;
15853 SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
15863 SDValue LHS = Op.getOperand(0);
15864 SDValue RHS = Op.getOperand(1);
15871 SDValue Cmp =
15903 SDValue NewSetcc = DAG.getSetCC(dl, MVT::v4i16, LHS, RHS, CC);
15907 return SDValue();
15921 SDValue Cmp =
15924 return SDValue();
15927 SDValue Cmp2 =
15930 return SDValue();
15943 static SDValue getReductionSDNode(unsigned Op, SDLoc DL, SDValue ScalarOp,
15945 SDValue VecOp = ScalarOp.getOperand(0);
15951 static SDValue getVectorBitwiseReduce(unsigned Opcode, SDValue Vec, EVT VT,
15966 return SDValue();
15975 SDValue Result;
15982 SDValue Lo, Hi;
15985 SDValue HalfVec = DAG.getNode(ScalarOpcode, DL, HalfVT, Lo, Hi);
16005 SDValue Extended = DAG.getNode(
16034 SDValue Lo, Hi;
16046 SDValue Scalar = DAG.getBitcast(ScalarVT, Vec);
16053 SDValue ShiftAmount =
16055 SDValue Shifted =
16066 SDValue AArch64TargetLowering::LowerVECREDUCE(SDValue Op,
16068 SDValue Src = Op.getOperand(0);
16141 SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op,
16146 return SDValue();
16152 SDValue RHS = Op.getOperand(2);
16160 SDValue
16161 AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(SDValue Op,
16167 SDValue Chain = Op.getOperand(0);
16168 SDValue Size = Op.getOperand(1);
16175 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
16182 SDValue Ops[2] = {SP, Chain};
16189 SDValue Callee = DAG.getTargetExternalSymbol(Subtarget->getChkStkName(),
16199 Chain = DAG.getCopyToReg(Chain, dl, AArch64::X15, Size, SDValue());
16212 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
16220 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
16222 SDValue Ops[2] = {SP, Chain};
16226 SDValue
16227 AArch64TargetLowering::LowerInlineDYNAMIC_STACKALLOC(SDValue Op,
16231 SDValue Chain = Op.getOperand(0);
16232 SDValue Size = Op.getOperand(1);
16240 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
16249 SDValue Ops[2] = {SP, Chain};
16253 SDValue
16254 AArch64TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
16263 return SDValue();
16266 SDValue AArch64TargetLowering::LowerAVG(SDValue Op, SelectionDAG &DAG,
16272 return SDValue();
16275 SDValue AArch64TargetLowering::LowerVSCALE(SDValue Op,
16505 const SDValue &Base = Mem->getBasePtr();
16525 bool AArch64TargetLowering::shouldRemoveRedundantExtend(SDValue Extend) const {
16528 SDValue Extract = Extend.getOperand(0);
16600 bool AArch64TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
17719 SDValue AddNode, SDValue ConstNode) const {
17907 SDValue ShiftLHS = N->getOperand(0);
17924 SDValue AndLHS = ShiftLHS.getOperand(0);
18040 static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
18044 return SDValue();
18048 SDValue Shift = N->getOperand(0);
18049 SDValue Ones = N->getOperand(1);
18052 return SDValue();
18058 return SDValue();
18078 static SDValue performVecReduceAddCombineWithUADDLP(SDNode *N,
18082 return SDValue();
18084 SDValue VecReduceOp0 = N->getOperand(0);
18088 return SDValue();
18090 SDValue ABS = VecReduceOp0;
18094 return SDValue();
18096 SDValue SUB = ABS->getOperand(0);
18102 return SDValue();
18111 return SDValue();
18113 SDValue EXT0 = SUB->getOperand(0);
18114 SDValue EXT1 = SUB->getOperand(1);
18118 return SDValue();
18124 SDValue UABDHigh8Op0 =
18127 SDValue UABDHigh8Op1 =
18130 SDValue UABDHigh8 = DAG.getNode(IsZExt ? ISD::ABDU : ISD::ABDS, DL, MVT::v8i8,
18132 SDValue UABDL = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, UABDHigh8);
18135 SDValue UABDLo8Op0 =
18138 SDValue UABDLo8Op1 =
18141 SDValue UABDLo8 = DAG.getNode(IsZExt ? ISD::ABDU : ISD::ABDS, DL, MVT::v8i8,
18143 SDValue ZExtUABD = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, UABDLo8);
18144 SDValue UABAL = DAG.getNode(ISD::ADD, DL, MVT::v8i16, UABDL, ZExtUABD);
18147 SDValue UADDLP = DAG.getNode(AArch64ISD::UADDLP, DL, MVT::v4i32, UABAL);
18159 static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG,
18162 return SDValue();
18167 SDValue Op0 = N->getOperand(0);
18170 return SDValue();
18173 SDValue A = Op0;
18174 SDValue B;
18180 return SDValue();
18183 return SDValue();
18187 return SDValue();
18195 return SDValue();
18205 return SDValue();
18212 return SDValue();
18234 SDValue Zeros = DAG.getConstant(0, DL, TargetType);
18235 SDValue Dot = DAG.getNode(DotOpcode, DL, Zeros.getValueType(), Zeros,
18241 SmallVector<SDValue, 4> SDotVec16;
18244 SDValue Zeros = DAG.getConstant(0, DL, MVT::v4i32);
18245 SDValue Op0 =
18248 SDValue Op1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v16i8, B,
18250 SDValue Dot =
18257 SDValue ConcatSDot16 =
18259 SDValue VecReduceAdd16 =
18266 SmallVector<SDValue, 4> SDotVec8;
18267 SDValue Zeros = DAG.getConstant(0, DL, MVT::v2i32);
18268 SDValue Vec8Op0 =
18271 SDValue Vec8Op1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, B,
18273 SDValue Dot =
18275 SDValue VecReudceAdd8 =
18285 static SDValue performUADDVAddCombine(SDValue A, SelectionDAG &DAG) {
18286 auto DetectAddExtract = [&](SDValue A) {
18291 SDValue Op0 = A.getOperand(0);
18292 SDValue Op1 = A.getOperand(1);
18296 return SDValue();
18297 SDValue Ext0 = Op0.getOperand(0);
18298 SDValue Ext1 = Op1.getOperand(0);
18302 return SDValue();
18307 return SDValue();
18312 return SDValue();
18318 if (SDValue R = DetectAddExtract(A))
18322 if (SDValue R = performUADDVAddCombine(A.getOperand(0), DAG))
18326 if (SDValue R = performUADDVAddCombine(A.getOperand(1), DAG))
18329 return SDValue();
18334 static SDValue performUADDVZextCombine(SDValue A, SelectionDAG &DAG) {
18340 return SDValue();
18341 SDValue Op0 = A.getOperand(0);
18342 SDValue Op1 = A.getOperand(1);
18344 return SDValue();
18345 SDValue Ext0 = Op0.getOperand(0);
18346 SDValue Ext1 = Op1.getOperand(0);
18352 return SDValue();
18355 SDValue Concat =
18363 SDValue Uaddlv =
18372 static SDValue performUADDVCombine(SDNode *N, SelectionDAG &DAG) {
18373 SDValue A = N->getOperand(0);
18375 if (SDValue R = performUADDVAddCombine(A, DAG))
18377 else if (SDValue R = performUADDVZextCombine(A, DAG))
18380 return SDValue();
18383 static SDValue performXorCombine(SDNode *N, SelectionDAG &DAG,
18387 return SDValue();
18392 SDValue
18398 return SDValue(N, 0); // Lower SDIV as SDIV
18406 return SDValue(N, 0);
18411 return SDValue();
18417 return SDValue();
18422 SDValue
18428 return SDValue(N, 0); // Lower SREM as SREM
18435 return SDValue(N, 0);
18440 return SDValue();
18444 return SDValue();
18447 SDValue N0 = N->getOperand(0);
18448 SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
18449 SDValue Zero = DAG.getConstant(0, DL, VT);
18450 SDValue CCVal, CSNeg;
18452 SDValue Cmp = getAArch64Cmp(N0, Zero, ISD::SETGE, CCVal, DAG, DL);
18453 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, Pow2MinusOne);
18459 SDValue CCVal = DAG.getConstant(AArch64CC::MI, DL, MVT_CC);
18462 SDValue Negs = DAG.getNode(AArch64ISD::SUBS, DL, VTs, Zero, N0);
18463 SDValue AndPos = DAG.getNode(ISD::AND, DL, VT, N0, Pow2MinusOne);
18464 SDValue AndNeg = DAG.getNode(ISD::AND, DL, VT, Negs, Pow2MinusOne);
18476 static std::optional<unsigned> IsSVECntIntrinsic(SDValue S) {
18503 static EVT calculatePreExtendType(SDValue Extend) {
18542 static SDValue performBuildShuffleExtendCombine(SDValue BV, SelectionDAG &DAG) {
18546 return SDValue();
18550 SDValue Extend = BV->getOperand(0);
18558 return SDValue();
18563 return SDValue();
18569 return SDValue();
18573 for (SDValue Op : drop_begin(BV->ops())) {
18578 return SDValue();
18588 return SDValue();
18594 SDValue NBV;
18600 SmallVector<SDValue, 8> NewOps;
18601 for (SDValue Op : BV->ops())
18622 static SDValue performMulVectorExtendCombine(SDNode *Mul, SelectionDAG &DAG) {
18626 return SDValue();
18628 SDValue Op0 = performBuildShuffleExtendCombine(Mul->getOperand(0), DAG);
18629 SDValue Op1 = performBuildShuffleExtendCombine(Mul->getOperand(1), DAG);
18633 return SDValue();
18642 static SDValue performMulVectorCmpZeroCombine(SDNode *N, SelectionDAG &DAG) {
18646 return SDValue();
18649 return SDValue();
18651 SDValue And = N->getOperand(0);
18652 SDValue Srl = And.getOperand(0);
18658 return SDValue();
18663 return SDValue();
18670 SDValue In = DAG.getNode(AArch64ISD::NVCAST, DL, HalfVT, Srl.getOperand(0));
18671 SDValue CM = DAG.getNode(AArch64ISD::CMLTz, DL, HalfVT, In);
18679 static SDValue performVectorExtCombine(SDNode *N, SelectionDAG &DAG) {
18688 return SDValue();
18692 return SDValue();
18694 SDValue N0 = N->getOperand(0).getOperand(0);
18695 SDValue N1 = N->getOperand(1).getOperand(0);
18706 SDValue NewN0 = DAG.getNode(N->getOperand(0).getOpcode(), DL, HalfVT, N0);
18707 SDValue NewN1 = DAG.getNode(N->getOperand(1).getOpcode(), DL, HalfVT, N1);
18708 SDValue NewOp = DAG.getNode(N->getOpcode(), DL, HalfVT, NewN0, NewN1);
18713 return SDValue();
18716 static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG,
18720 if (SDValue Ext = performMulVectorExtendCombine(N, DAG))
18722 if (SDValue Ext = performMulVectorCmpZeroCombine(N, DAG))
18724 if (SDValue Ext = performVectorExtCombine(N, DAG))
18728 return SDValue();
18735 SDValue N0 = N->getOperand(0);
18736 SDValue N1 = N->getOperand(1);
18737 SDValue MulOper;
18740 auto IsAddSubWith1 = [&](SDValue V) -> bool {
18743 SDValue Opnd = V->getOperand(1);
18754 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper);
18759 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper);
18765 return SDValue();
18776 return SDValue();
18796 return SDValue();
18801 return SDValue();
18808 auto Shl = [&](SDValue N0, unsigned N1) {
18810 return SDValue();
18813 return SDValue();
18814 SDValue RHS = DAG.getConstant(N1, DL, MVT::i64);
18817 auto Add = [&](SDValue N0, SDValue N1) {
18819 return SDValue();
18822 auto Sub = [&](SDValue N0, SDValue N1) {
18824 return SDValue();
18827 auto Negate = [&](SDValue N) {
18829 return SDValue();
18830 SDValue Zero = DAG.getConstant(0, DL, VT);
18919 SDValue MVal = Add(Shl(N0, ShiftM1), N0);
18929 SDValue MVal = Add(Shl(N0, CVM.getZExtValue()), N0);
18940 SDValue MVal = Sub(N0, Shl(N0, CVM.getZExtValue()));
18963 return SDValue();
18966 static SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N,
18983 return SDValue();
18993 return SDValue();
19000 SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
19002 SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst);
19003 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT,
19005 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd);
19009 return SDValue();
19014 static SDValue
19019 return SDValue();
19022 return SDValue();
19026 return SDValue();
19032 SDValue SrcVal = N->getOperand(0);
19037 return SDValue();
19051 return SDValue();
19054 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
19055 SDValue Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, SrcVecTy,
19057 SDValue Convert = DAG.getNode(N->getOpcode(), DL, DestVecTy, Vec);
19061 static SDValue performIntToFpCombine(SDNode *N, SelectionDAG &DAG,
19066 if (SDValue Res = performVectorCompareAndMaskUnaryOpCombine(N, DAG))
19069 if (SDValue Res =
19075 return SDValue();
19079 return SDValue();
19084 SDValue N0 = N->getOperand(0);
19090 SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(),
19096 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), Load.getValue(1));
19103 return SDValue();
19108 static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG,
19111 if (SDValue Res =
19116 return SDValue();
19119 return SDValue();
19121 SDValue Op = N->getOperand(0);
19123 return SDValue();
19126 return SDValue();
19128 SDValue ConstVec = Op->getOperand(1);
19130 return SDValue();
19136 return SDValue();
19141 return SDValue();
19145 return SDValue();
19152 return SDValue();
19156 return SDValue();
19162 return SDValue();
19170 SDValue FixConv =
19181 static SDValue tryCombineToBSL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
19189 return SDValue();
19192 return SDValue();
19196 return SDValue();
19198 SDValue N0 = N->getOperand(0);
19200 return SDValue();
19202 SDValue N1 = N->getOperand(1);
19204 return SDValue();
19211 SDValue O0 = N0->getOperand(i);
19212 SDValue O1 = N1->getOperand(j);
19213 SDValue Sub, Add, SubSibling, AddSibling;
19278 return SDValue();
19291 static SDValue performANDORCSELCombine(SDNode *N, SelectionDAG &DAG) {
19293 SDValue CSel0 = N->getOperand(0);
19294 SDValue CSel1 = N->getOperand(1);
19298 return SDValue();
19301 return SDValue();
19307 return SDValue();
19309 SDValue Cmp0 = CSel0.getOperand(3);
19310 SDValue Cmp1 = CSel1.getOperand(3);
19314 return SDValue();
19322 return SDValue();
19325 SDValue CCmp, Condition;
19338 SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
19346 SDValue AbsOp1 =
19359 static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
19365 if (SDValue R = performANDORCSELCombine(N, DAG))
19369 return SDValue();
19371 if (SDValue Res = tryCombineToBSL(N, DCI, TLI))
19374 return SDValue();
19404 static SDValue performReinterpretCastCombine(SDNode *N) {
19405 SDValue LeafOp = SDValue(N, 0);
19406 SDValue Op = N->getOperand(0);
19412 return SDValue();
19415 static SDValue performSVEAndCombine(SDNode *N,
19418 SDValue Src = N->getOperand(0);
19423 SDValue UnpkOp = Src->getOperand(0);
19424 SDValue Dup = N->getOperand(1);
19427 return SDValue();
19432 return SDValue();
19466 SDValue And = DAG.getNode(ISD::AND, DL,
19473 return SDValue();
19483 return SDValue();
19485 SDValue Mask = N->getOperand(1);
19488 return SDValue();
19518 return SDValue();
19524 return SDValue();
19528 static SDValue performANDSETCCCombine(SDNode *N,
19534 SDValue SetCC = N->getOperand(0);
19539 // returns an empty SDValue to avoid applying the optimization to prevent
19543 return SDValue();
19549 SDValue Cmp;
19554 (Cmp = emitConjunction(DAG, SDValue(N, 0), CC))) {
19564 return SDValue();
19567 static SDValue performANDCombine(SDNode *N,
19570 SDValue LHS = N->getOperand(0);
19571 SDValue RHS = N->getOperand(1);
19574 if (SDValue R = performANDORCSELCombine(N, DAG))
19577 if (SDValue R = performANDSETCCCombine(N,DCI))
19581 return SDValue();
19589 return SDValue();
19593 return SDValue();
19602 SDValue NewOp;
19613 if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG,
19615 (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG,
19620 if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, SDValue(N, 0), DAG,
19622 (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, SDValue(N, 0), DAG,
19627 return SDValue();
19630 static SDValue performFADDCombine(SDNode *N,
19633 SDValue LHS = N->getOperand(0);
19634 SDValue RHS = N->getOperand(1);
19639 return SDValue();
19642 auto ReassocComplex = [&](SDValue A, SDValue B) {
19644 return SDValue();
19650 return SDValue();
19651 SDValue VCMLA = DAG.getNode(
19658 if (SDValue R = ReassocComplex(LHS, RHS))
19660 if (SDValue R = ReassocComplex(RHS, LHS))
19663 return SDValue();
19678 static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op,
19681 static bool isPredicateCCSettingOp(SDValue N) {
19701 static SDValue
19708 return SDValue();
19710 SDValue N0 = N->getOperand(0);
19715 return SDValue();
19720 return SDValue();
19724 SDValue Pg = getPTrue(DAG, SDLoc(N), VT, AArch64SVEPredPattern::all);
19731 static SDValue
19738 return SDValue();
19740 SDValue N0 = N->getOperand(0);
19744 return SDValue();
19747 SDValue Idx = N->getOperand(1);
19749 return SDValue();
19751 SDValue VS = Idx.getOperand(0);
19753 return SDValue();
19757 return SDValue();
19761 SDValue Pg = getPTrue(DAG, SDLoc(N), OpVT, AArch64SVEPredPattern::all);
19765 static SDValue
19769 if (SDValue Res = performFirstTrueTestVectorCombine(N, DCI, Subtarget))
19771 if (SDValue Res = performLastTrueTestVectorCombine(N, DCI, Subtarget))
19775 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
19798 SDValue N00 = N0->getOperand(IsStrict ? 1 : 0);
19799 SDValue N01 = N0->getOperand(IsStrict ? 2 : 1);
19802 SDValue Other = N00;
19812 SDValue Extract1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other,
19814 SDValue Extract2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other,
19823 SDValue Ret = DAG.getNode(N0->getOpcode(), DL,
19826 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Ret);
19828 return SDValue(N, 0);
19832 return SDValue();
19835 static SDValue performConcatVectorsCombine(SDNode *N,
19840 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
19844 return SDValue();
19848 SDValue N00 = N0->getOperand(0);
19849 SDValue N10 = N1->getOperand(0);
19893 SDValue N000 = N00->getOperand(0);
19894 SDValue N100 = N10->getOperand(0);
19902 SDValue Uzp = DAG.getNode(AArch64ISD::UZP2, dl, VT, N000, N100);
19903 SDValue NewShiftConstant =
19919 all_of(N->op_values(), [SrcVT](SDValue V) {
19930 SmallVector<SDValue> Ops;
19933 SDValue V = N->getOperand(i);
19938 SDValue NewLoad = DAG.getLoad(FVT, dl, LD->getChain(),
19940 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLoad.getValue(1));
19960 auto isBitwiseVectorNegate = [](SDValue V) {
19964 SDValue N00 = N0->getOperand(0);
19965 SDValue N10 = N1->getOperand(0);
19982 return SDValue();
19990 SDValue N00 = N0->getOperand(0);
19991 SDValue N01 = N0->getOperand(1);
19992 SDValue N10 = N1->getOperand(0);
19993 SDValue N11 = N1->getOperand(1);
19996 SDValue Concat0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N00, N10);
19997 SDValue Concat1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N01, N11);
20002 auto IsRSHRN = [](SDValue Shr) {
20005 SDValue Op = Shr.getOperand(0);
20033 SDValue X = N0.getOperand(0).getOperand(0);
20034 SDValue Y = N1.isUndef() ? DAG.getUNDEF(X.getValueType())
20038 SDValue CC = DAG.getNode(ISD::CONCAT_VECTORS, dl, BVT, X, Y);
20039 SDValue Add = DAG.getNode(
20042 SDValue Shr =
20051 SDValue E0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N0.getOperand(0),
20053 SDValue E1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N0.getOperand(1),
20077 return SDValue();
20078 SDValue RHS = N1->getOperand(0);
20082 return SDValue();
20095 static SDValue
20099 return SDValue();
20103 return SDValue();
20105 SDValue V = N->getOperand(0);
20115 return SDValue();
20118 static SDValue
20122 SDValue Vec = N->getOperand(0);
20123 SDValue SubVec = N->getOperand(1);
20132 return SDValue();
20136 return SDValue();
20142 return SDValue();
20147 SDValue Lo, Hi;
20160 static SDValue tryCombineFixedPointConvert(SDNode *N,
20166 return SDValue();
20177 SDValue Op1 = N->getOperand(1);
20179 return SDValue();
20182 SDValue IID = N->getOperand(0);
20183 SDValue Shift = N->getOperand(2);
20184 SDValue Vec = Op1.getOperand(0);
20185 SDValue Lane = Op1.getOperand(1);
20194 return SDValue();
20201 return SDValue();
20203 SDValue Convert =
20223 static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG) {
20246 return SDValue();
20250 return SDValue();
20264 static bool isEssentiallyExtractHighSubvector(SDValue N) {
20277 const SDValue *Opnd0;
20278 const SDValue *Opnd1;
20284 const SDValue *Cmp;
20308 static bool isSetCC(SDValue Op, SetCCInfoAndKind &SetCCInfo) {
20351 static bool isSetCCOrZExtSetCC(const SDValue& Op, SetCCInfoAndKind &Info) {
20364 static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) {
20366 SDValue LHS = Op->getOperand(0);
20367 SDValue RHS = Op->getOperand(1);
20375 return SDValue();
20381 return SDValue();
20389 return SDValue();
20391 SDValue CCVal;
20392 SDValue Cmp;
20411 static SDValue performAddUADDVCombine(SDNode *N, SelectionDAG &DAG) {
20415 return SDValue();
20417 SDValue LHS = N->getOperand(0);
20418 SDValue RHS = N->getOperand(1);
20421 return SDValue();
20426 return SDValue();
20428 SDValue Op1 = LHS->getOperand(0);
20429 SDValue Op2 = RHS->getOperand(0);
20435 return SDValue();
20437 SDValue Val1 = Op1.getOperand(0);
20438 SDValue Val2 = Op2.getOperand(0);
20441 SDValue AddVal = DAG.getNode(ISD::ADD, DL, ValVT, Val1, Val2);
20450 static SDValue performAddCSelIntoCSinc(SDNode *N, SelectionDAG &DAG) {
20453 return SDValue();
20455 SDValue LHS = N->getOperand(0);
20456 SDValue RHS = N->getOperand(1);
20464 return SDValue();
20469 return SDValue();
20479 return SDValue();
20485 return SDValue();
20509 return SDValue();
20515 SDValue NewNode = DAG.getNode(ISD::ADD, DL, VT, RHS, SDValue(CTVal, 0));
20516 SDValue CCVal = DAG.getConstant(AArch64CC, DL, MVT::i32);
20517 SDValue Cmp = LHS.getOperand(3);
20523 static SDValue performAddDotCombine(SDNode *N, SelectionDAG &DAG) {
20526 return SDValue();
20528 SDValue Dot = N->getOperand(0);
20529 SDValue A = N->getOperand(1);
20531 auto isZeroDot = [](SDValue Dot) {
20539 return SDValue();
20545 static bool isNegatedInteger(SDValue Op) {
20549 static SDValue getNegatedInteger(SDValue Op, SelectionDAG &DAG) {
20552 SDValue Zero = DAG.getConstant(0, DL, VT);
20563 static SDValue performNegCSelCombine(SDNode *N, SelectionDAG &DAG) {
20564 if (!isNegatedInteger(SDValue(N, 0)))
20565 return SDValue();
20567 SDValue CSel = N->getOperand(1);
20569 return SDValue();
20571 SDValue N0 = CSel.getOperand(0);
20572 SDValue N1 = CSel.getOperand(1);
20577 return SDValue();
20579 SDValue N0N = getNegatedInteger(N0, DAG);
20580 SDValue N1N = getNegatedInteger(N1, DAG);
20599 static SDValue performAddSubLongCombine(SDNode *N,
20603 return SDValue();
20609 return SDValue();
20613 SDValue LHS = N->getOperand(0);
20614 SDValue RHS = N->getOperand(1);
20618 return SDValue();
20627 return SDValue();
20633 return SDValue();
20641 static bool isCMP(SDValue Op) {
20648 static std::optional<AArch64CC::CondCode> getCSETCondCode(SDValue Op) {
20654 SDValue OpLHS = Op.getOperand(0);
20655 SDValue OpRHS = Op.getOperand(1);
20666 static SDValue foldOverflowCheck(SDNode *Op, SelectionDAG &DAG, bool IsAdd) {
20667 SDValue CmpOp = Op->getOperand(2);
20669 return SDValue();
20673 return SDValue();
20676 return SDValue();
20679 SDValue CsetOp = CmpOp->getOperand(IsAdd ? 0 : 1);
20682 return SDValue();
20690 static SDValue foldADCToCINC(SDNode *N, SelectionDAG &DAG) {
20691 SDValue LHS = N->getOperand(0);
20692 SDValue RHS = N->getOperand(1);
20693 SDValue Cond = N->getOperand(2);
20696 return SDValue();
20702 SDValue CC = DAG.getConstant(AArch64CC::LO, DL, MVT::i32);
20706 static SDValue performBuildVectorCombine(SDNode *N,
20714 SDValue Elt0 = N->getOperand(0), Elt1 = N->getOperand(1),
20730 SDValue LowLanesSrcVec = Elt0->getOperand(0)->getOperand(0);
20732 SDValue HighLanes;
20753 SDValue HighLanesSrcVec = Elt2->getOperand(0)->getOperand(0);
20758 SDValue DoubleToSingleSticky =
20760 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32,
20770 SDValue Elt0 = N->getOperand(0), Elt1 = N->getOperand(1);
20787 SDValue SrcVec = Elt0->getOperand(0)->getOperand(0);
20790 SDValue HalfToSingle =
20792 SDValue SubvectorIdx = Elt0->getOperand(0)->getOperand(1);
20793 SDValue Extract = DAG.getNode(
20811 return SDValue();
20813 SDValue Elt0 = N->getOperand(0), Elt1 = N->getOperand(1);
20827 SDValue VecToExtend = Elt0->getOperand(0);
20830 return SDValue();
20832 SDValue SubvectorIdx = DAG.getVectorIdxConstant(Elt0->getConstantOperandVal(1), DL);
20834 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, DL, ExtVT, VecToExtend);
20839 return SDValue();
20842 static SDValue performTruncateCombine(SDNode *N, SelectionDAG &DAG,
20846 SDValue N0 = N->getOperand(0);
20849 SDValue Op = N0.getOperand(0);
20860 SDValue Op = N0.getOperand(0);
20861 SDValue ExtractIndexNode = N0.getOperand(1);
20863 return SDValue();
20885 return SDValue();
20889 static bool isExtendOrShiftOperand(SDValue N) {
20914 static SDValue performAddCombineSubShift(SDNode *N, SDValue SUB, SDValue Z,
20916 auto IsOneUseExtend = [](SDValue N) {
20925 return SDValue();
20928 return SDValue();
20930 SDValue Shift = SUB.getOperand(0);
20932 return SDValue();
20937 SDValue Y = SUB.getOperand(1);
20938 SDValue NewSub = DAG.getNode(ISD::SUB, DL, VT, Z, Y);
20942 static SDValue performAddCombineForShiftedOperands(SDNode *N,
20947 return SDValue();
20953 return SDValue();
20956 SDValue LHS = N->getOperand(0);
20957 SDValue RHS = N->getOperand(1);
20959 if (SDValue Val = performAddCombineSubShift(N, LHS, RHS, DAG))
20961 if (SDValue Val = performAddCombineSubShift(N, RHS, LHS, DAG))
20978 return SDValue();
20983 static SDValue performSubAddMULCombine(SDNode *N, SelectionDAG &DAG) {
20985 return SDValue();
20987 SDValue Add = N->getOperand(1);
20988 SDValue X = N->getOperand(0);
20990 return SDValue();
20993 return SDValue();
20995 return SDValue();
20997 SDValue M1 = Add.getOperand(0);
20998 SDValue M2 = Add.getOperand(1);
21001 return SDValue();
21004 return SDValue();
21007 SDValue Sub = DAG.getNode(ISD::SUB, SDLoc(N), VT, X, M1);
21018 static SDValue
21023 return SDValue();
21026 return SDValue();
21029 return SDValue();
21032 return SDValue();
21034 auto performOpt = [&DAG, &N](SDValue Op0, SDValue Op1) -> SDValue {
21036 return SDValue();
21039 return SDValue();
21041 SDValue MulValue = Op1->getOperand(0);
21043 return SDValue();
21046 return SDValue();
21050 return SDValue();
21052 SDValue ScaledOp = convertToScalableVector(DAG, ScalableVT, Op0);
21053 SDValue NewValue =
21058 if (SDValue res = performOpt(N->getOperand(0), N->getOperand(1)))
21063 return SDValue();
21068 static SDValue performAddSubIntoVectorOp(SDNode *N, SelectionDAG &DAG) {
21072 return SDValue();
21073 SDValue Op0 = N->getOperand(0);
21074 SDValue Op1 = N->getOperand(1);
21080 return SDValue();
21083 return SDValue();
21095 return SDValue();
21102 static bool isLoadOrMultipleLoads(SDValue B, SmallVector<LoadSDNode *> &Loads) {
21103 SDValue BV = peekThroughOneUseBitcasts(B);
21173 static bool areLoadedOffsetButOtherwiseSame(SDValue Op0, SDValue Op1,
21228 static SDValue performExtBinopLoadFold(SDNode *N, SelectionDAG &DAG) {
21233 return SDValue();
21235 SDValue Other = N->getOperand(0);
21236 SDValue Shift = N->getOperand(1);
21242 return SDValue();
21249 return SDValue();
21251 SDValue Op0 = Other.getOperand(0);
21252 SDValue Op1 = Shift.getOperand(0).getOperand(0);
21256 return SDValue();
21268 return SDValue();
21271 std::function<SDValue(SDValue, SDValue, SelectionDAG &)> GenCombinedTree =
21272 [&GenCombinedTree](SDValue Op0, SDValue Op1, SelectionDAG &DAG) {
21284 SmallVector<SDValue> NewLoads;
21286 SDValue Load = DAG.getLoad(DLoadVT, SDLoc(L0), L0->getChain(),
21296 SmallVector<SDValue> Ops;
21301 SDValue NewOp = GenCombinedTree(Op0, Op1, DAG);
21314 SDValue Ext0, Ext1;
21319 SDValue SubL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, Op0.getValueType(),
21321 SDValue SubH =
21324 SDValue Extr0 =
21326 SDValue Extr1 =
21332 SDValue Ext = DAG.getNode(Other.getOpcode(), DL, DVT, NewOp);
21333 SDValue SubL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Ext,
21335 SDValue SubH =
21341 SDValue NShift =
21346 static SDValue performAddSubCombine(SDNode *N,
21349 if (SDValue Val = performAddUADDVCombine(N, DCI.DAG))
21351 if (SDValue Val = performAddDotCombine(N, DCI.DAG))
21353 if (SDValue Val = performAddCSelIntoCSinc(N, DCI.DAG))
21355 if (SDValue Val = performNegCSelCombine(N, DCI.DAG))
21357 if (SDValue Val = performVectorExtCombine(N, DCI.DAG))
21359 if (SDValue Val = performAddCombineForShiftedOperands(N, DCI.DAG))
21361 if (SDValue Val = performSubAddMULCombine(N, DCI.DAG))
21363 if (SDValue Val = performSVEMulAddSubCombine(N, DCI))
21365 if (SDValue Val = performAddSubIntoVectorOp(N, DCI.DAG))
21368 if (SDValue Val = performExtBinopLoadFold(N, DCI.DAG))
21381 static SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N,
21385 return SDValue();
21387 SDValue LHS = N->getOperand((IID == Intrinsic::not_intrinsic) ? 0 : 1);
21388 SDValue RHS = N->getOperand((IID == Intrinsic::not_intrinsic) ? 1 : 2);
21399 return SDValue();
21403 return SDValue();
21405 return SDValue();
21414 static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG) {
21426 return SDValue();
21432 return SDValue();
21479 SDValue Op = N->getOperand(1);
21502 return SDValue();
21508 static SDValue tryCombineCRC32(unsigned Mask, SDNode *N, SelectionDAG &DAG) {
21509 SDValue AndN = N->getOperand(2);
21511 return SDValue();
21515 return SDValue();
21521 static SDValue combineAcrossLanesIntrinsic(unsigned Opc, SDNode *N,
21531 static SDValue LowerSVEIntrinsicIndex(SDNode *N, SelectionDAG &DAG) {
21533 SDValue Op1 = N->getOperand(1);
21534 SDValue Op2 = N->getOperand(2);
21540 SDValue StepVector = DAG.getStepVector(DL, N->getValueType(0));
21541 SDValue Step = DAG.getNode(ISD::SPLAT_VECTOR, DL, N->getValueType(0), Op2);
21542 SDValue Mul = DAG.getNode(ISD::MUL, DL, N->getValueType(0), StepVector, Step);
21543 SDValue Base = DAG.getNode(ISD::SPLAT_VECTOR, DL, N->getValueType(0), Op1);
21547 static SDValue LowerSVEIntrinsicDUP(SDNode *N, SelectionDAG &DAG) {
21549 SDValue Scalar = N->getOperand(3);
21555 SDValue Passthru = N->getOperand(1);
21556 SDValue Pred = N->getOperand(2);
21561 static SDValue LowerSVEIntrinsicEXT(SDNode *N, SelectionDAG &DAG) {
21570 return SDValue();
21578 SDValue Op0 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(1));
21579 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(2));
21580 SDValue Op2 = DAG.getNode(ISD::MUL, dl, MVT::i32, N->getOperand(3),
21583 SDValue EXT = DAG.getNode(AArch64ISD::EXT, dl, ByteVT, Op0, Op1, Op2);
21587 static SDValue tryConvertSVEWideCompare(SDNode *N, ISD::CondCode CC,
21591 return SDValue();
21593 SDValue Comparator = N->getOperand(3);
21599 SDValue Pred = N->getOperand(1);
21600 SDValue Imm;
21620 return SDValue();
21634 return SDValue();
21641 return SDValue();
21643 SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, CmpVT, Imm);
21648 return SDValue();
21651 static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op,
21664 SDValue TVal = DAG.getConstant(1, DL, OutVT);
21665 SDValue FVal = DAG.getConstant(0, DL, OutVT);
21678 SDValue Test = DAG.getNode(
21684 SDValue CC = DAG.getConstant(getInvertedCondCode(Cond), DL, MVT::i32);
21685 SDValue Res = DAG.getNode(AArch64ISD::CSEL, DL, OutVT, FVal, TVal, CC, Test);
21689 static SDValue combineSVEReductionInt(SDNode *N, unsigned Opc,
21693 SDValue Pred = N->getOperand(1);
21694 SDValue VecToReduce = N->getOperand(2);
21699 SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, VecToReduce);
21703 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
21708 static SDValue combineSVEReductionFP(SDNode *N, unsigned Opc,
21712 SDValue Pred = N->getOperand(1);
21713 SDValue VecToReduce = N->getOperand(2);
21716 SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, VecToReduce);
21720 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
21725 static SDValue combineSVEReductionOrderedFP(SDNode *N, unsigned Opc,
21729 SDValue Pred = N->getOperand(1);
21730 SDValue InitVal = N->getOperand(2);
21731 SDValue VecToReduce = N->getOperand(3);
21736 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
21740 SDValue Reduce = DAG.getNode(Opc, DL, ReduceVT, Pred, InitVal, VecToReduce);
21751 static SDValue convertMergedOpToPredOp(SDNode *N, unsigned Opc,
21756 SDValue Pg = N->getOperand(1);
21757 SDValue Op1 = N->getOperand(SwapOperands ? 3 : 2);
21758 SDValue Op2 = N->getOperand(SwapOperands ? 2 : 3);
21769 return SDValue();
21772 static SDValue tryCombineWhileLo(SDNode *N,
21776 return SDValue();
21779 return SDValue();
21782 return SDValue();
21786 return SDValue();
21794 return SDValue();
21805 return SDValue();
21810 return SDValue();
21814 SDValue ID =
21816 SDValue Idx = N->getOperand(1);
21817 SDValue TC = N->getOperand(2);
21829 return SDValue(N, 0);
21832 SDValue tryLowerPartialReductionToDot(SDNode *N,
21843 return SDValue();
21845 return SDValue();
21849 SDValue Op2 = N->getOperand(2);
21851 SDValue MulOpLHS, MulOpRHS;
21858 SDValue ExtMulOpLHS = Op2->getOperand(0);
21859 SDValue ExtMulOpRHS = Op2->getOperand(1);
21865 return SDValue();
21874 return SDValue();
21876 return SDValue();
21878 SDValue Acc = N->getOperand(1);
21890 return SDValue();
21896 return SDValue();
21901 return SDValue();
21917 SDValue DotI32 =
21920 SDValue Extended = DAG.getSExtOrTrunc(DotI32, DL, ReducedVT);
21927 SDValue tryLowerPartialReductionToWideAdd(SDNode *N,
21937 return SDValue();
21942 return SDValue();
21943 SDValue Acc = N->getOperand(1);
21944 SDValue Ext = N->getOperand(2);
21948 return SDValue();
21950 SDValue ExtOp = Ext->getOperand(0);
21956 return SDValue();
21962 SDValue BottomNode = DAG.getNode(BottomOpcode, DL, AccVT, Acc, ExtOp);
21966 static SDValue performIntrinsicCombine(SDNode *N,
21975 if (SDValue Dot = tryLowerPartialReductionToDot(N, Subtarget, DAG))
21977 if (SDValue WideAdd = tryLowerPartialReductionToWideAdd(N, Subtarget, DAG))
22286 return SDValue();
22289 static bool isCheapToExtend(const SDValue &N) {
22295 static SDValue
22306 const SDValue SetCC = N->getOperand(0);
22308 const SDValue CCOp0 = SetCC.getOperand(0);
22309 const SDValue CCOp1 = SetCC.getOperand(1);
22312 return SDValue();
22322 const SDValue Ext1 =
22324 const SDValue Ext2 =
22332 return SDValue();
22338 static SDValue performZExtDeinterleaveShuffleCombine(SDNode *N,
22344 return SDValue();
22348 return SDValue();
22355 return SDValue();
22372 return SDValue();
22374 SDValue BC1 = DAG.getNode(AArch64ISD::NVCAST, DL, VT,
22376 SDValue BC2 = DAG.getNode(AArch64ISD::NVCAST, DL, VT,
22378 SDValue UZP = DAG.getNode(Idx < 2 ? AArch64ISD::UZP1 : AArch64ISD::UZP2, DL,
22395 static SDValue performZExtUZPCombine(SDNode *N, SelectionDAG &DAG) {
22399 return SDValue();
22401 SDValue Op = N->getOperand(0);
22433 return SDValue();
22436 return SDValue();
22439 return SDValue();
22444 SDValue BC = DAG.getNode(AArch64ISD::NVCAST, DL, VT,
22452 static SDValue performExtendCombine(SDNode *N,
22464 SDValue NewABD =
22467 return SDValue();
22472 if (SDValue R = performZExtDeinterleaveShuffleCombine(N, DAG))
22474 if (SDValue R = performZExtUZPCombine(N, DAG))
22490 SDValue Bswap = N->getOperand(0);
22495 SDValue NewAnyExtend = DAG.getNode(ISD::ANY_EXTEND, DL, N->getValueType(0),
22501 return SDValue();
22504 static SDValue splitStoreSplat(SelectionDAG &DAG, StoreSDNode &St,
22505 SDValue SplatVal, unsigned NumVecElts) {
22515 SDValue BasePtr = St.getBasePtr();
22519 SDValue NewST1 =
22533 SDValue OffsetPtr =
22574 static SDValue performLD1Combine(SDNode *N, SelectionDAG &DAG, unsigned Opc) {
22579 return SDValue();
22586 SDValue Ops[] = { N->getOperand(0), // Chain
22591 SDValue Load = DAG.getNode(Opc, DL, VTs, Ops);
22592 SDValue LoadChain = SDValue(Load.getNode(), 1);
22600 static SDValue performLDNT1Combine(SDNode *N, SelectionDAG &DAG) {
22610 SDValue PassThru = DAG.getConstant(0, DL, LoadVT);
22611 SDValue L = DAG.getMaskedLoad(LoadVT, DL, MINode->getChain(),
22618 SDValue Ops[] = { DAG.getNode(ISD::BITCAST, DL, VT, L), L.getValue(1) };
22626 static SDValue performLD1ReplicateCombine(SDNode *N, SelectionDAG &DAG) {
22637 SDValue Ops[] = {N->getOperand(0), N->getOperand(2), N->getOperand(3)};
22638 SDValue Load = DAG.getNode(Opcode, DL, {LoadVT, MVT::Other}, Ops);
22639 SDValue LoadChain = SDValue(Load.getNode(), 1);
22647 static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG) {
22649 SDValue Data = N->getOperand(2);
22652 SDValue InputVT = DAG.getValueType(DataVT);
22657 SDValue SrcNew;
22663 SDValue Ops[] = { N->getOperand(0), // Chain
22673 static SDValue performSTNT1Combine(SDNode *N, SelectionDAG &DAG) {
22676 SDValue Data = N->getOperand(2);
22705 static SDValue replaceZeroVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
22706 SDValue StVal = St.getValue();
22711 return SDValue();
22720 return SDValue();
22723 return SDValue();
22729 return SDValue();
22734 return SDValue();
22741 return SDValue();
22745 SDValue EltVal = StVal.getOperand(I);
22747 return SDValue();
22762 SDValue SplatVal =
22772 static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
22773 SDValue StVal = St.getValue();
22779 return SDValue();
22784 return SDValue();
22789 return SDValue();
22795 SDValue SplatVal;
22799 return SDValue();
22805 return SDValue();
22810 return SDValue();
22813 return SDValue();
22820 return SDValue();
22825 static SDValue splitStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
22831 return SDValue();
22833 SDValue StVal = S->getValue();
22837 return SDValue();
22842 if (SDValue ReplacedZeroSplat = replaceZeroVectorStore(DAG, *S))
22850 return SDValue();
22854 return SDValue();
22859 return SDValue();
22868 return SDValue();
22873 if (SDValue ReplacedSplat = replaceSplatVectorStore(DAG, *S))
22881 SDValue SubVector0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
22883 SDValue SubVector1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
22885 SDValue BasePtr = S->getBasePtr();
22886 SDValue NewST1 =
22889 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
22896 static SDValue performSpliceCombine(SDNode *N, SelectionDAG &DAG) {
22903 return SDValue();
22906 static SDValue performUnpackCombine(SDNode *N, SelectionDAG &DAG,
22922 SDValue Mask = MLD->getMask();
22926 SDValue(MLD, 0).hasOneUse() && Mask->getOpcode() == AArch64ISD::PTRUE &&
22939 SDValue PassThru = DAG.getConstant(0, DL, VT);
22940 SDValue NewLoad = DAG.getMaskedLoad(
22945 DAG.ReplaceAllUsesOfValueWith(SDValue(MLD, 1), NewLoad.getValue(1));
22952 return SDValue();
22958 SDValue Op0 = N->getOperand(0);
22969 static SDValue tryCombineExtendRShTrunc(SDNode *N, SelectionDAG &DAG) {
22971 SDValue Op0 = N->getOperand(0);
22972 SDValue Op1 = N->getOperand(1);
22977 return SDValue();
22980 SDValue ShiftValue = Op0.getOperand(1);
22982 return SDValue();
22985 SDValue Lo = Op0.getOperand(0);
22986 SDValue Hi = Op1.getOperand(0);
22989 return SDValue();
22990 SDValue OrigArg = Lo.getOperand(0);
22992 return SDValue();
23009 static SDValue trySimplifySrlAddToRshrnb(SDValue Srl, SelectionDAG &DAG,
23013 return SDValue();
23023 return SDValue();
23027 SDValue RShOperand;
23029 return SDValue();
23030 SDValue Rshrnb = DAG.getNode(
23036 static SDValue isNVCastToHalfWidthElements(SDValue V) {
23038 return SDValue();
23040 SDValue Op = V.getOperand(0);
23043 return SDValue();
23048 static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG,
23051 SDValue Op0 = N->getOperand(0);
23052 SDValue Op1 = N->getOperand(1);
23060 SDValue SourceVec = Op0.getOperand(0);
23067 SDValue Uzp = DAG.getNode(N->getOpcode(), DL, WidenedResVT, SourceVec,
23076 return SDValue();
23098 SDValue BC = DAG.getBitcast(BCVT, Op0);
23099 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, BC);
23105 if (SDValue Urshr = tryCombineExtendRShTrunc(N, DAG))
23108 if (SDValue PreCast = isNVCastToHalfWidthElements(Op0)) {
23109 if (SDValue Rshrnb = trySimplifySrlAddToRshrnb(PreCast, DAG, Subtarget)) {
23115 if (SDValue PreCast = isNVCastToHalfWidthElements(Op1)) {
23116 if (SDValue Rshrnb = trySimplifySrlAddToRshrnb(PreCast, DAG, Subtarget)) {
23123 if (SDValue PreCast = isNVCastToHalfWidthElements(Op0)) {
23126 SDValue X = PreCast.getOperand(0).getOperand(0);
23133 if (SDValue PreCast = isNVCastToHalfWidthElements(Op1)) {
23136 SDValue Z = PreCast.getOperand(0).getOperand(1);
23144 return SDValue();
23160 return SDValue();
23162 SDValue SourceOp0 = peekThroughBitcasts(Op0);
23163 SDValue SourceOp1 = peekThroughBitcasts(Op1);
23170 SDValue Concat =
23181 return SDValue();
23187 return SDValue();
23202 return SDValue();
23205 SDValue UzpOp0 = DAG.getNode(ISD::BITCAST, DL, ResultTy, SourceOp0);
23206 SDValue UzpOp1 = DAG.getNode(ISD::BITCAST, DL, ResultTy, SourceOp1);
23207 SDValue UzpResult =
23230 static SDValue performGLD1Combine(SDNode *N, SelectionDAG &DAG) {
23249 SDValue Chain = N->getOperand(0);
23250 SDValue Pg = N->getOperand(1);
23251 SDValue Base = N->getOperand(2);
23252 SDValue Offset = N->getOperand(3);
23253 SDValue Ty = N->getOperand(4);
23265 SDValue ExtPg = Offset.getOperand(0);
23273 SDValue UnextendedOffset = Offset.getOperand(1);
23284 return SDValue();
23289 static SDValue performVectorShiftCombine(SDNode *N,
23295 SDValue Op = N->getOperand(0);
23310 return SDValue();
23316 return SDValue(N, 0);
23318 return SDValue();
23321 static SDValue performSunpkloCombine(SDNode *N, SelectionDAG &DAG) {
23328 SDValue CC = N->getOperand(0)->getOperand(0);
23330 SDValue Unpk = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, CC,
23335 return SDValue();
23340 static SDValue performPostLD1Combine(SDNode *N,
23344 return SDValue();
23350 return SDValue();
23356 return SDValue();
23361 return SDValue();
23364 SDValue Lane;
23369 return SDValue();
23376 return SDValue();
23384 return SDValue();
23393 return SDValue();
23396 SDValue Addr = LD->getOperand(1);
23397 SDValue Vector = N->getOperand(0);
23405 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
23426 SmallVector<SDValue, 8> Ops;
23438 SDValue UpdN = DAG.getMemIntrinsicNode(NewOp, SDLoc(N), SDTys, Ops,
23443 SDValue NewResults[] = {
23444 SDValue(LD, 0), // The result of load
23445 SDValue(UpdN.getNode(), 2) // Chain
23448 DCI.CombineTo(N, SDValue(UpdN.getNode(), 0)); // Dup/Inserted Result
23449 DCI.CombineTo(User, SDValue(UpdN.getNode(), 1)); // Write back register
23453 return SDValue();
23458 static bool performTBISimplification(SDValue Addr,
23473 static SDValue foldTruncStoreOfExt(SelectionDAG &DAG, SDNode *N) {
23479 return SDValue();
23480 SDValue Ext = Store->getValue();
23484 return SDValue();
23485 SDValue Orig = Ext->getOperand(0);
23487 return SDValue();
23492 return SDValue();
23511 static SDValue combineV3I8LoadExt(LoadSDNode *LD, SelectionDAG &DAG) {
23515 return SDValue();
23519 SDValue Chain = LD->getChain();
23520 SDValue BasePtr = LD->getBasePtr();
23525 SDValue L16 = DAG.getLoad(MVT::i16, DL, Chain, BasePtr, MMO);
23527 SDValue L8 = DAG.getLoad(MVT::i8, DL, Chain,
23532 SDValue Ext16 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, L16);
23533 SDValue Ext8 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, L8);
23536 SDValue Shl = DAG.getNode(ISD::SHL, DL, MVT::i32, Ext8,
23538 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::i32, Ext16, Shl);
23539 SDValue Cast = DAG.getNode(ISD::BITCAST, DL, MVT::v4i8, Or);
23542 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MemVT, Cast,
23544 SDValue TokenFactor = DAG.getNode(
23546 {SDValue(cast<SDNode>(L16), 1), SDValue(cast<SDNode>(L8), 1)});
23553 static SDValue performLOADCombine(SDNode *N,
23562 return SDValue(N, 0);
23564 if (SDValue Res = combineV3I8LoadExt(LD, DAG))
23568 return SDValue(N, 0);
23574 return SDValue(N, 0);
23577 SDValue Chain = LD->getChain();
23578 SDValue BasePtr = LD->getBasePtr();
23580 SmallVector<SDValue, 4> LoadOps;
23581 SmallVector<SDValue, 4> LoadOpsChain;
23592 SDValue NewPtr = DAG.getMemBasePlusOffset(
23595 SDValue NewLoad = DAG.getLoad(
23599 LoadOpsChain.push_back(SDValue(cast<SDNode>(NewLoad), 1));
23611 SDValue NewPtr = DAG.getMemBasePlusOffset(
23614 SDValue RemainingLoad =
23618 SDValue UndefVector = DAG.getUNDEF(NewVT);
23619 SDValue InsertIdx = DAG.getVectorIdxConstant(0, DL);
23620 SDValue ExtendedReminingLoad =
23624 LoadOpsChain.push_back(SDValue(cast<SDNode>(RemainingLoad), 1));
23628 SDValue ConcatVectors =
23631 SDValue ExtractSubVector =
23634 SDValue TokenFactor =
23639 static EVT tryGetOriginalBoolVectorType(SDValue Op, int Depth = 0) {
23653 for (SDValue Operand : Op->op_values()) {
23671 static SDValue vectorToScalarBitmask(SDNode *N, SelectionDAG &DAG) {
23673 SDValue ComparisonResult(N, 0);
23679 return SDValue();
23683 return SDValue();
23700 return SDValue();
23705 SmallVector<SDValue, 16> MaskConstants;
23716 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, MaskConstants);
23717 SDValue RepresentativeBits =
23720 SDValue UpperRepresentativeBits =
23723 SDValue Zipped = DAG.getNode(AArch64ISD::ZIP1, DL, VecVT,
23735 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, MaskConstants);
23736 SDValue RepresentativeBits =
23743 static SDValue combineBoolVectorAndTruncateStore(SelectionDAG &DAG,
23746 return SDValue();
23749 SDValue VecOp = Store->getValue();
23755 return SDValue();
23760 return SDValue();
23763 SDValue VectorBits = vectorToScalarBitmask(VecOp.getNode(), DAG);
23765 return SDValue();
23769 SDValue ExtendedBits = DAG.getZExtOrTrunc(VectorBits, DL, StoreVT);
23781 static SDValue combineI8TruncStore(StoreSDNode *ST, SelectionDAG &DAG,
23783 SDValue Value = ST->getValue();
23789 return SDValue();
23796 SDValue UndefVector = DAG.getUNDEF(WideVT);
23797 SDValue WideTrunc = DAG.getNode(
23800 SDValue Cast = DAG.getNode(
23805 SDValue Chain = ST->getChain();
23808 SDValue E2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8, Cast,
23811 SDValue Ptr2 = DAG.getMemBasePlusOffset(ST->getBasePtr(), Offset2, DL);
23814 SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8, Cast,
23817 SDValue Ptr1 = DAG.getMemBasePlusOffset(ST->getBasePtr(), Offset1, DL);
23820 SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8, Cast,
23827 static SDValue performSTORECombine(SDNode *N,
23832 SDValue Chain = ST->getChain();
23833 SDValue Value = ST->getValue();
23834 SDValue Ptr = ST->getBasePtr();
23842 if (SDValue Res = combineI8TruncStore(ST, DAG, Subtarget))
23858 if (SDValue Split = splitStores(N, DCI, DAG, Subtarget))
23863 return SDValue(N, 0);
23865 if (SDValue Store = foldTruncStoreOfExt(DAG, N))
23868 if (SDValue Store = combineBoolVectorAndTruncateStore(DAG, ST))
23874 return SDValue();
23875 if (SDValue Rshrnb =
23882 return SDValue();
23885 static SDValue performMSTORECombine(SDNode *N,
23890 SDValue Value = MST->getValue();
23891 SDValue Mask = MST->getMask();
23930 return SDValue();
23931 if (SDValue Rshrnb = trySimplifySrlAddToRshrnb(Value, DAG, Subtarget)) {
23939 return SDValue();
23943 static bool foldIndexIntoBase(SDValue &BasePtr, SDValue &Index, SDValue Scale,
23973 SDValue Add = Index.getOperand(0);
23974 SDValue ShiftOp = Index.getOperand(1);
23975 SDValue OffsetOp = Add.getOperand(1);
23994 SDValue &BasePtr, SDValue &Index,
24030 SDValue RHS = Index.getOperand(1);
24063 static SDValue performMaskedGatherScatterCombine(
24066 return SDValue();
24070 SDValue Chain = MGS->getChain();
24071 SDValue Scale = MGS->getScale();
24072 SDValue Index = MGS->getIndex();
24073 SDValue Mask = MGS->getMask();
24074 SDValue BasePtr = MGS->getBasePtr();
24078 return SDValue();
24083 SDValue PassThru = MGT->getPassThru();
24084 SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale};
24090 SDValue Data = MSC->getValue();
24091 SDValue Ops[] = {Chain, Data, Mask, BasePtr, Index, Scale};
24097 SDValue Ops[] = {Chain, HG->getInc(), Mask, BasePtr,
24105 static SDValue performNEONPostLDSTCombine(SDNode *N,
24109 return SDValue();
24112 SDValue Addr = N->getOperand(AddrOpIdx);
24191 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
24201 SmallVector<SDValue, 8> Ops;
24221 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, SDLoc(N), SDTys, Ops,
24226 std::vector<SDValue> NewResults;
24228 NewResults.push_back(SDValue(UpdN.getNode(), i));
24230 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs + 1));
24232 DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs));
24236 return SDValue();
24242 bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType) {
24422 static SDValue performSubsToAndsCombine(SDNode *N, SDNode *SubsNode,
24428 return SDValue();
24433 return SDValue();
24436 return SDValue();
24438 return SDValue();
24442 return SDValue();
24448 SDValue ANDS = DAG.getNode(
24451 SDValue AArch64_CC =
24463 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), AArch64_CC,
24469 SDValue performCONDCombine(SDNode *N,
24479 return SDValue();
24488 return SDValue();
24490 if (SDValue Val = performSubsToAndsCombine(N, SubsNode, AndNode, DAG, CCIndex,
24503 return SDValue();
24505 SDValue AddValue = AndNode->getOperand(0);
24508 return SDValue();
24512 SDValue AddInputValue1 = AddValue.getNode()->getOperand(0);
24513 SDValue AddInputValue2 = AddValue.getNode()->getOperand(1);
24514 SDValue SubsInputValue = SubsNode->getOperand(1);
24521 return SDValue();
24528 return SDValue();
24533 return SDValue();
24539 SDValue Ops[] = { AddValue, SubsNode->getOperand(1) };
24541 SDValue NewValue = DAG.getNode(CondOpcode, SDLoc(SubsNode), VTs, Ops);
24544 return SDValue(N, 0);
24548 static SDValue performBRCONDCombine(SDNode *N,
24556 return SDValue();
24558 if (SDValue NV = performCONDCombine(N, DCI, DAG, 2, 3))
24560 SDValue Chain = N->getOperand(0);
24561 SDValue Dest = N->getOperand(1);
24562 SDValue CCVal = N->getOperand(2);
24563 SDValue Cmp = N->getOperand(3);
24568 return SDValue();
24572 return SDValue();
24577 return SDValue();
24579 SDValue LHS = Cmp.getOperand(0);
24580 SDValue RHS = Cmp.getOperand(1);
24585 return SDValue();
24591 return SDValue();
24595 return SDValue();
24598 SDValue BR;
24607 return SDValue();
24610 static SDValue foldCSELofCTTZ(SDNode *N, SelectionDAG &DAG) {
24612 SDValue SUBS = N->getOperand(3);
24613 SDValue Zero, CTTZ;
24622 return SDValue();
24627 return SDValue();
24633 return SDValue();
24635 SDValue X = CTTZ.getOpcode() == ISD::TRUNCATE
24640 return SDValue();
24645 SDValue BitWidthMinusOne =
24658 static SDValue foldCSELOfCSEL(SDNode *Op, SelectionDAG &DAG) {
24659 SDValue L = Op->getOperand(0);
24660 SDValue R = Op->getOperand(1);
24664 SDValue OpCmp = Op->getOperand(3);
24666 return SDValue();
24668 SDValue CmpLHS = OpCmp.getOperand(0);
24669 SDValue CmpRHS = OpCmp.getOperand(1);
24674 return SDValue();
24676 SDValue X = CmpLHS->getOperand(0);
24677 SDValue Y = CmpLHS->getOperand(1);
24679 return SDValue();
24688 return SDValue();
24692 SDValue Cond = CmpLHS->getOperand(3);
24697 return SDValue();
24702 return SDValue();
24707 SDValue CCValue = DAG.getConstant(CC, DL, MVT::i32);
24716 static SDValue reassociateCSELOperandsForCSE(SDNode *N, SelectionDAG &DAG) {
24717 SDValue SubsNode = N->getOperand(3);
24719 return SDValue();
24721 SDValue CmpOpToMatch = SubsNode.getOperand(1);
24722 SDValue CmpOpOther = SubsNode.getOperand(0);
24726 SDValue ExpectedOp;
24727 SDValue SubsOp;
24743 auto GetReassociationOp = [&](SDValue Op, SDValue ExpectedOp) {
24745 return SDValue();
24748 return SDValue();
24749 SDValue X = Op.getOperand(0).getOperand(0);
24750 SDValue Y = Op.getOperand(0).getOperand(1);
24754 return SDValue();
24756 return SDValue();
24761 auto Fold = [&](AArch64CC::CondCode NewCC, SDValue ExpectedOp,
24762 SDValue SubsOp) {
24763 SDValue TReassocOp = GetReassociationOp(N->getOperand(0), ExpectedOp);
24764 SDValue FReassocOp = GetReassociationOp(N->getOperand(1), ExpectedOp);
24766 return SDValue();
24768 SDValue NewCmp = DAG.getNode(AArch64ISD::SUBS, SDLoc(SubsNode),
24771 auto Reassociate = [&](SDValue ReassocOp, unsigned OpNum) {
24774 SDValue Res = DAG.getNode(ISD::ADD, SDLoc(N->getOperand(OpNum)), VT,
24780 SDValue TValReassoc = Reassociate(TReassocOp, 0);
24781 SDValue FValReassoc = Reassociate(FReassocOp, 1);
24791 if (SDValue R = Fold(CC, ExpectedOp, SubsOp))
24799 if (SDValue R = Fold(getSwappedCondition(CC), CmpOpToMatch, CmpOpToMatch))
24801 return SDValue();
24805 return SDValue();
24819 return Check ? Fold(NewCC, ExpectedOp, SubsOp) : SDValue();
24849 return SDValue();
24854 static SDValue performCSELCombine(SDNode *N,
24861 if (SDValue R = foldCSELOfCSEL(N, DAG))
24866 if (SDValue R = reassociateCSELOperandsForCSE(N, DAG))
24871 if (SDValue Folded = foldCSELofCTTZ(N, DAG))
24876 SDValue Cond = N->getOperand(3);
24889 SDValue Sub = DAG.getNode(AArch64ISD::SUBS, DL, Cond->getVTList(),
24904 static SDValue tryToWidenSetCCOperands(SDNode *Op, SelectionDAG &DAG) {
24907 return SDValue();
24913 return SDValue();
24916 return SDValue();
24920 return SDValue();
24924 return SDValue();
24927 SDValue Op0ExtV;
24928 SDValue Op1ExtV;
24937 Op0ExtV = SDValue(Op0SExt, 0);
24940 Op0ExtV = SDValue(Op0ZExt, 0);
24943 return SDValue();
24949 static SDValue
24952 SDValue Vec = N->getOperand(0);
24962 return SDValue();
24965 static SDValue performSETCCCombine(SDNode *N,
24969 SDValue LHS = N->getOperand(0);
24970 SDValue RHS = N->getOperand(1);
24975 if (SDValue V = tryToWidenSetCCOperands(N, DAG))
24989 SDValue CSEL =
25005 SDValue TST = DAG.getNode(ISD::AND, DL, TstVT, LHS->getOperand(0),
25033 if (SDValue V = performOrXorChainCombine(N, DAG))
25036 return SDValue();
25041 static SDValue performFlagSettingCombine(SDNode *N,
25045 SDValue LHS = N->getOperand(0);
25046 SDValue RHS = N->getOperand(1);
25051 SDValue Res = DCI.DAG.getNode(GenericOpcode, DL, VT, N->ops());
25059 DCI.CombineTo(Generic, SDValue(N, 0));
25061 return SDValue();
25064 static SDValue performSetCCPunpkCombine(SDNode *N, SelectionDAG &DAG) {
25068 SDValue Pred = N->getOperand(0);
25069 SDValue LHS = N->getOperand(1);
25070 SDValue RHS = N->getOperand(2);
25075 return SDValue();
25077 SDValue Extract = LHS->getOperand(0);
25081 return SDValue();
25083 SDValue InnerSetCC = Extract->getOperand(0);
25085 return SDValue();
25091 SDValue InnerPred = InnerSetCC.getOperand(0);
25099 return SDValue();
25102 static SDValue
25108 SDValue Pred = N->getOperand(0);
25109 SDValue LHS = N->getOperand(1);
25110 SDValue RHS = N->getOperand(2);
25113 if (SDValue V = performSetCCPunpkCombine(N, DAG))
25142 return SDValue();
25149 static SDValue getTestBitOperand(SDValue Op, unsigned &Bit, bool &Invert,
25222 static SDValue performTBZCombine(SDNode *N,
25227 SDValue TestSrc = N->getOperand(1);
25228 SDValue NewTestSrc = getTestBitOperand(TestSrc, Bit, Invert, DAG);
25231 return SDValue();
25253 static SDValue trySwapVSelectOperands(SDNode *N, SelectionDAG &DAG) {
25259 return SDValue();
25260 SDValue SetCC = N->getOperand(0);
25262 return SDValue();
25266 return SDValue();
25273 return SDValue();
25291 static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG) {
25295 SDValue N0 = N->getOperand(0);
25307 SDValue SetCC = N->getOperand(0);
25310 SDValue CmpLHS = SetCC.getOperand(0);
25325 SmallVector<SDValue, 8> Ops(
25328 SDValue Val = DAG.getBuildVector(VT, SDLoc(N), Ops);
25341 return SDValue();
25347 return SDValue();
25349 SDValue IfTrue = N->getOperand(1);
25350 SDValue IfFalse = N->getOperand(2);
25362 static SDValue performSelectCombine(SDNode *N,
25365 SDValue N0 = N->getOperand(0);
25369 return SDValue();
25372 return SDValue();
25390 return SDValue();
25394 return SDValue();
25403 return SDValue();
25412 SDValue LHS =
25414 SDValue RHS =
25416 SDValue SetCC = DAG.getNode(ISD::SETCC, DL, CCVT, LHS, RHS, N0.getOperand(2));
25420 SDValue Mask = DAG.getVectorShuffle(CCVT, DL, SetCC, SetCC, DUPMask);
25427 static SDValue performDUPCombine(SDNode *N,
25435 SmallVector<SDValue> Ops(N->ops());
25438 return DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SDValue(LN, 0),
25452 SDValue EXTRACT_VEC_ELT = N->getOperand(0);
25465 return SDValue();
25469 static SDValue performNVCASTCombine(SDNode *N, SelectionDAG &DAG) {
25476 return SDValue();
25482 static SDValue performGlobalAddressCombine(SDNode *N, SelectionDAG &DAG,
25488 return SDValue();
25493 return SDValue();
25498 return SDValue();
25507 return SDValue();
25519 return SDValue();
25525 return SDValue();
25528 SDValue Result = DAG.getGlobalAddress(GV, DL, MVT::i64, Offset);
25533 static SDValue performCTLZCombine(SDNode *N, SelectionDAG &DAG,
25535 SDValue BR = N->getOperand(0);
25538 return SDValue();
25546 static SDValue getScaledOffsetForBitWidth(SelectionDAG &DAG, SDValue Offset,
25551 SDValue Shift = DAG.getConstant(Log2_32(BitWidth / 8), DL, MVT::i64);
25552 SDValue SplatShift = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Shift);
25584 static bool isValidImmForSVEVecImmAddrMode(SDValue Offset,
25591 static SDValue performScatterStoreCombine(SDNode *N, SelectionDAG &DAG,
25594 const SDValue Src = N->getOperand(2);
25604 return SDValue();
25613 return SDValue();
25617 SDValue Base = N->getOperand(4);
25620 SDValue Offset = N->getOperand(5);
25664 return SDValue();
25674 return SDValue();
25682 SDValue InputVT = DAG.getValueType(SrcVT);
25687 SDValue SrcNew;
25694 SDValue Ops[] = {N->getOperand(0), // Chain
25704 static SDValue performGatherLoadCombine(SDNode *N, SelectionDAG &DAG,
25715 return SDValue();
25719 SDValue Base = N->getOperand(3);
25722 SDValue Offset = N->getOperand(4);
25772 return SDValue();
25787 SDValue OutVT = DAG.getValueType(RetVT);
25792 SDValue Ops[] = {N->getOperand(0), // Chain
25796 SDValue Load = DAG.getNode(Opcode, DL, VTs, Ops);
25797 SDValue LoadChain = SDValue(Load.getNode(), 1);
25810 static SDValue
25814 SDValue Src = N->getOperand(0);
25831 SDValue ExtOp = Src->getOperand(0);
25841 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ExtOp.getValueType(),
25848 return SDValue();
25851 return SDValue();
25916 return SDValue();
25923 return SDValue();
25928 SmallVector<SDValue, 5> Ops;
25932 SDValue ExtLoad = DAG.getNode(NewOpc, SDLoc(N), VTs, Ops);
25937 return SDValue(N, 0);
25943 static SDValue legalizeSVEGatherPrefetchOffsVec(SDNode *N, SelectionDAG &DAG) {
25945 SDValue Offset = N->getOperand(OffsetPos);
25949 return SDValue();
25954 SmallVector<SDValue, 5> Ops(N->ops());
25966 static SDValue combineSVEPrefetchVecBaseImmOff(SDNode *N, SelectionDAG &DAG,
25971 return SDValue();
25974 SmallVector<SDValue, 5> Ops(N->ops());
25987 static bool isLanes1toNKnownZero(SDValue Op) {
26010 static SDValue removeRedundantInsertVectorElt(SDNode *N) {
26012 SDValue InsertVec = N->getOperand(0);
26013 SDValue InsertElt = N->getOperand(1);
26014 SDValue InsertIdx = N->getOperand(2);
26018 return SDValue();
26021 return SDValue();
26024 return SDValue();
26026 SDValue ExtractVec = InsertElt.getOperand(0);
26027 SDValue ExtractIdx = InsertElt.getOperand(1);
26031 return SDValue();
26037 return SDValue();
26040 return SDValue();
26046 static SDValue
26048 if (SDValue Res = removeRedundantInsertVectorElt(N))
26054 static SDValue performFPExtendCombine(SDNode *N, SelectionDAG &DAG,
26057 SDValue N0 = N->getOperand(0);
26062 return SDValue();
26077 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
26086 return SDValue(N, 0); // Return N so it doesn't get rechecked!
26089 return SDValue();
26092 static SDValue performBSPExpandForSVE(SDNode *N, SelectionDAG &DAG,
26098 return SDValue();
26102 SDValue Mask = N->getOperand(0);
26103 SDValue In1 = N->getOperand(1);
26104 SDValue In2 = N->getOperand(2);
26106 SDValue InvMask = DAG.getNOT(DL, Mask, VT);
26107 SDValue Sel = DAG.getNode(ISD::AND, DL, VT, Mask, In1);
26108 SDValue SelInv = DAG.getNode(ISD::AND, DL, VT, InvMask, In2);
26112 static SDValue performDupLane128Combine(SDNode *N, SelectionDAG &DAG) {
26115 SDValue Insert = N->getOperand(0);
26117 return SDValue();
26120 return SDValue();
26125 return SDValue();
26127 SDValue Bitcast = Insert.getOperand(1);
26129 return SDValue();
26131 SDValue Subvec = Bitcast.getOperand(0);
26134 return SDValue();
26139 SDValue NewInsert =
26142 SDValue NewDuplane128 = DAG.getNode(AArch64ISD::DUPLANE128, DL, NewSubvecVT,
26148 static SDValue tryCombineMULLWithUZP1(SDNode *N,
26152 return SDValue();
26154 SDValue LHS = N->getOperand(0);
26155 SDValue RHS = N->getOperand(1);
26157 SDValue ExtractHigh;
26158 SDValue ExtractLow;
26159 SDValue TruncHigh;
26160 SDValue TruncLow;
26179 return SDValue();
26184 SDValue TruncHighOp = TruncHigh.getOperand(0);
26188 return SDValue();
26205 SDValue ExtractHighSrcVec = ExtractHigh.getOperand(0);
26251 SDValue TruncLowOp =
26256 return SDValue();
26264 SDValue UZP1 =
26266 SDValue HighIdxCst =
26268 SDValue NewTruncHigh =
26274 SDValue NewTruncLow = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, TruncLowVT,
26279 return SDValue(N, 0);
26282 static SDValue performMULLCombine(SDNode *N,
26285 if (SDValue Val =
26289 if (SDValue Val = tryCombineMULLWithUZP1(N, DCI, DAG))
26292 return SDValue();
26295 static SDValue
26309 return SDValue();
26313 return SDValue();
26315 SDValue ZEXT = N->getOperand(0);
26317 return SDValue();
26319 SDValue EXTRACT_VEC_ELT = ZEXT.getOperand(0);
26322 return SDValue();
26325 return SDValue();
26327 SDValue UADDLV = EXTRACT_VEC_ELT.getOperand(0);
26331 return SDValue();
26335 SDValue EXTRACT_SUBVEC =
26338 SDValue NVCAST =
26350 static SDValue performSHLCombine(SDNode *N,
26354 return SDValue();
26356 SDValue Op0 = N->getOperand(0);
26358 return SDValue();
26360 SDValue C1 = Op0->getOperand(1);
26361 SDValue C2 = N->getOperand(1);
26363 return SDValue();
26370 return SDValue();
26379 SDValue NewRHS = DAG.getNode(ISD::SHL, DL, VT, C1, C2);
26381 return SDValue();
26383 SDValue X = Op0->getOperand(0);
26384 SDValue NewShift = DAG.getNode(ISD::SHL, DL, VT, X, C2);
26388 SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
26429 SDValue(N, 0), DemandedBits, DemandedElts, DCI))
26430 return SDValue();
26705 SDValue A = DAG.getNode(
26708 SDValue B = DAG.getNode(
26736 return SDValue();
26744 SDValue &Chain) const {
26750 SDValue TCChain = Chain;
26798 SDValue &Base,
26799 SDValue &Offset,
26817 auto IsUndefOrZero = [](SDValue V) {
26846 bool AArch64TargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
26847 SDValue &Offset,
26851 SDValue Ptr;
26868 SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset,
26871 SDValue Ptr;
26892 SmallVectorImpl<SDValue> &Results,
26895 SDValue Op = N->getOperand(0);
26913 SDValue VectorBits = vectorToScalarBitmask(Op.getNode(), DAG);
26919 SmallVectorImpl<SDValue> &Results,
26923 SDValue Op = N->getOperand(0);
26927 SDValue Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtendVT, Op);
26928 SDValue CastVal = DAG.getNode(ISD::BITCAST, DL, CastVT, Vec);
26929 SDValue IdxZero = DAG.getVectorIdxConstant(0, DL);
26935 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
26937 SDValue Op = N->getOperand(0);
26968 SDValue CastResult = getSVESafeBitCast(getSVEContainerType(VT), Op, DAG);
26986 static void ReplaceAddWithADDP(SDNode *N, SmallVectorImpl<SDValue> &Results,
26997 SDValue X = N->getOperand(0);
27018 SDValue Addp = DAG.getNode(AArch64ISD::ADDP, N, LoHi.first.getValueType(),
27035 SmallVectorImpl<SDValue> &Results,
27039 SDValue Lo, Hi;
27043 SDValue InterVal = DAG.getNode(InterOp, dl, LoVT, Lo, Hi);
27044 SDValue SplitVal = DAG.getNode(AcrossOp, dl, LoVT, InterVal);
27049 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
27050 SDValue In = N->getOperand(0);
27078 SDValue Half = DAG.getNode(Opcode, DL, ExtendedHalfVT, N->getOperand(0));
27083 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
27088 SDValue RegClass =
27090 SDValue SubReg0 = DAG.getTargetConstant(AArch64::sube64, dl, MVT::i32);
27091 SDValue SubReg1 = DAG.getTargetConstant(AArch64::subo64, dl, MVT::i32);
27092 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
27093 return SDValue(
27098 SmallVectorImpl<SDValue> &Results,
27108 SDValue Ops[] = {
27141 SDValue Lo = DAG.getTargetExtractSubreg(SubReg1, SDLoc(N), MVT::i64,
27142 SDValue(CmpSwap, 0));
27143 SDValue Hi = DAG.getTargetExtractSubreg(SubReg2, SDLoc(N), MVT::i64,
27144 SDValue(CmpSwap, 0));
27147 Results.push_back(SDValue(CmpSwap, 1)); // Chain out
27173 SDValue Ops[] = {N->getOperand(1), Desired.first, Desired.second,
27181 SDValue(CmpSwap, 0), SDValue(CmpSwap, 1)));
27182 Results.push_back(SDValue(CmpSwap, 3));
27261 SmallVectorImpl<SDValue> &Results,
27279 const SDValue &Chain = N->getOperand(0);
27280 const SDValue &Ptr = N->getOperand(1);
27281 const SDValue &Val128 = N->getOperand(2);
27282 std::pair<SDValue, SDValue> Val2x64 =
27299 SDValue Ops[] = {Val2x64.first, Val2x64.second, Ptr, Chain};
27309 SDValue Lo = SDValue(AtomicInst, 0), Hi = SDValue(AtomicInst, 1);
27314 Results.push_back(SDValue(AtomicInst, 2)); // Chain out
27318 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
27330 Results.push_back(LowerVECREDUCE(SDValue(N, 0), DAG));
27333 if (SDValue Res = LowerVECTOR_COMPRESS(SDValue(N, 0), DAG))
27343 if (SDValue Result = LowerCTPOP_PARITY(SDValue(N, 0), DAG))
27365 if (useSVEForFixedLengthVectorVT(SDValue(N, 0).getValueType()))
27367 LowerToPredicatedOp(SDValue(N, 0), DAG, AArch64ISD::MULHS_PRED));
27370 if (useSVEForFixedLengthVectorVT(SDValue(N, 0).getValueType()))
27372 LowerToPredicatedOp(SDValue(N, 0), DAG, AArch64ISD::MULHU_PRED));
27410 SDValue Result = DAG.getMemIntrinsicNode(
27418 SDValue Pair = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), MemVT,
27431 if (SDValue(N, 0).getValueType() == MVT::i128) {
27440 SDValue Result = DAG.getMemIntrinsicNode(
27447 SDValue Pair =
27511 SDValue Chain = DAG.getEntryNode();
27512 SDValue RuntimePStateSM =
27539 SDValue Chain = N->getOperand(0);
27540 SDValue SysRegName = N->getOperand(1);
27542 SDValue Result = DAG.getNode(
27548 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128,
28028 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
28353 static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL,
28398 static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
28406 static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT) {
28414 static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) {
28420 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
28425 static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) {
28431 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
28436 SDValue AArch64TargetLowering::LowerFixedLengthVectorLoadToSVE(
28437 SDValue Op, SelectionDAG &DAG) const {
28453 SDValue NewLoad = DAG.getMaskedLoad(
28458 SDValue Result = NewLoad;
28471 SDValue MergedValues[2] = {Result, NewLoad.getValue(1)};
28475 static SDValue convertFixedMaskToScalableVector(SDValue Mask,
28494 SDValue AArch64TargetLowering::LowerFixedLengthVectorMLoadToSVE(
28495 SDValue Op, SelectionDAG &DAG) const {
28502 SDValue Mask = Load->getMask();
28512 SDValue PassThru;
28527 SDValue NewLoad = DAG.getMaskedLoad(
28532 SDValue Result = NewLoad;
28534 SDValue OldPassThru =
28540 SDValue MergedValues[2] = {Result, NewLoad.getValue(1)};
28545 SDValue AArch64TargetLowering::LowerFixedLengthVectorStoreToSVE(
28546 SDValue Op, SelectionDAG &DAG) const {
28578 SDValue AArch64TargetLowering::LowerFixedLengthVectorMStoreToSVE(
28579 SDValue Op, SelectionDAG &DAG) const {
28587 SDValue Mask = convertFixedMaskToScalableVector(Store->getMask(), DAG);
28595 SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE(
28596 SDValue Op, SelectionDAG &DAG) const {
28608 SDValue Op1 = convertToScalableVector(DAG, ContainerVT, Op.getOperand(0));
28609 SDValue Op2 = DAG.getTargetConstant(Log2_64(SplatVal), dl, MVT::i32);
28611 SDValue Pg = getPredicateForFixedLengthVector(DAG, dl, VT);
28612 SDValue Res =
28633 SDValue Op0 = DAG.getNode(ExtendOpcode, dl, WideVT, Op.getOperand(0));
28634 SDValue Op1 = DAG.getNode(ExtendOpcode, dl, WideVT, Op.getOperand(1));
28635 SDValue Div = DAG.getNode(Op.getOpcode(), dl, WideVT, Op0, Op1);
28640 &ExtendOpcode](SDValue Op) {
28641 SDValue IdxZero = DAG.getConstant(0, dl, MVT::i64);
28642 SDValue IdxHalf =
28644 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HalfVT, Op, IdxZero);
28645 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HalfVT, Op, IdxHalf);
28646 return std::pair<SDValue, SDValue>(
28654 SDValue Lo = DAG.getNode(Op.getOpcode(), dl, PromVT, Op0LoExt, Op1LoExt);
28655 SDValue Hi = DAG.getNode(Op.getOpcode(), dl, PromVT, Op0HiExt, Op1HiExt);
28656 SDValue LoTrunc = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Lo);
28657 SDValue HiTrunc = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Hi);
28661 SDValue AArch64TargetLowering::LowerFixedLengthVectorIntExtendToSVE(
28662 SDValue Op, SelectionDAG &DAG) const {
28667 SDValue Val = Op.getOperand(0);
28697 SDValue AArch64TargetLowering::LowerFixedLengthVectorTruncateToSVE(
28698 SDValue Op, SelectionDAG &DAG) const {
28703 SDValue Val = Op.getOperand(0);
28733 SDValue AArch64TargetLowering::LowerFixedLengthExtractVectorElt(
28734 SDValue Op, SelectionDAG &DAG) const {
28741 SDValue Op0 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(0));
28746 SDValue AArch64TargetLowering::LowerFixedLengthInsertVectorElt(
28747 SDValue Op, SelectionDAG &DAG) const {
28754 SDValue Op0 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(0));
28765 SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
28777 SmallVector<SDValue, 4> Operands = {Pg};
28778 for (const SDValue &V : Op->op_values()) {
28805 SmallVector<SDValue, 4> Operands = {Pg};
28806 for (const SDValue &V : Op->op_values()) {
28822 SDValue AArch64TargetLowering::LowerToScalableOp(SDValue Op,
28830 SmallVector<SDValue, 4> Ops;
28831 for (const SDValue &V : Op->op_values()) {
28851 SDValue AArch64TargetLowering::LowerVECREDUCE_SEQ_FADD(SDValue ScalarOp,
28854 SDValue AccOp = ScalarOp.getOperand(0);
28855 SDValue VecOp = ScalarOp.getOperand(1);
28865 SDValue Pg = getPredicateForVector(DAG, DL, SrcVT);
28866 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
28873 SDValue Rdx = DAG.getNode(AArch64ISD::FADDA_PRED, DL, ContainerVT,
28879 SDValue AArch64TargetLowering::LowerPredReductionToSVE(SDValue ReduceOp,
28882 SDValue Op = ReduceOp.getOperand(0);
28887 return SDValue();
28889 SDValue Pg = getPredicateForVector(DAG, DL, OpVT);
28893 return SDValue();
28906 SDValue ID =
28913 SDValue Cntp =
28919 return SDValue();
28922 SDValue AArch64TargetLowering::LowerReductionToSVE(unsigned Opcode,
28923 SDValue ScalarOp,
28926 SDValue VecOp = ScalarOp.getOperand(0);
28939 SDValue BoolVec = VecOp.getOperand(0);
28942 SDValue CntpOp = DAG.getNode(
28957 SDValue Pg = getPredicateForVector(DAG, DL, SrcVT);
28958 SDValue Rdx = DAG.getNode(Opcode, DL, RdxVT, Pg, VecOp);
28959 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT,
28969 SDValue
28970 AArch64TargetLowering::LowerFixedLengthVectorSelectToSVE(SDValue Op,
28977 SDValue Op1 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(1));
28978 SDValue Op2 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(2));
28994 SDValue AArch64TargetLowering::LowerFixedLengthVectorSetccToSVE(
28995 SDValue Op, SelectionDAG &DAG) const {
29018 SDValue
29019 AArch64TargetLowering::LowerFixedLengthBitcastToSVE(SDValue Op,
29033 SDValue AArch64TargetLowering::LowerFixedLengthConcatVectorsToSVE(
29034 SDValue Op, SelectionDAG &DAG) const {
29047 SmallVector<SDValue, 4> Ops;
29058 SDValue Pg = getPredicateForFixedLengthVector(DAG, DL, SrcVT);
29067 SDValue
29068 AArch64TargetLowering::LowerFixedLengthFPExtendToSVE(SDValue Op,
29074 SDValue Val = Op.getOperand(0);
29075 SDValue Pg = getPredicateForVector(DAG, DL, VT);
29092 SDValue
29093 AArch64TargetLowering::LowerFixedLengthFPRoundToSVE(SDValue Op,
29099 SDValue Val = Op.getOperand(0);
29104 SDValue Pg = getPredicateForVector(DAG, DL, RoundVT);
29116 SDValue
29117 AArch64TargetLowering::LowerFixedLengthIntToFPToSVE(SDValue Op,
29127 SDValue Val = Op.getOperand(0);
29133 SDValue Pg = getPredicateForFixedLengthVector(DAG, DL, VT);
29148 SDValue Pg = getPredicateForFixedLengthVector(DAG, DL, SrcVT);
29160 SDValue
29161 AArch64TargetLowering::LowerVECTOR_DEINTERLEAVE(SDValue Op,
29167 SDValue Even = DAG.getNode(AArch64ISD::UZP1, DL, OpVT, Op.getOperand(0),
29169 SDValue Odd = DAG.getNode(AArch64ISD::UZP2, DL, OpVT, Op.getOperand(0),
29174 SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op,
29181 SDValue Lo = DAG.getNode(AArch64ISD::ZIP1, DL, OpVT, Op.getOperand(0),
29183 SDValue Hi = DAG.getNode(AArch64ISD::ZIP2, DL, OpVT, Op.getOperand(0),
29188 SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
29193 SDValue Chain = HG->getChain();
29194 SDValue Inc = HG->getInc();
29195 SDValue Mask = HG->getMask();
29196 SDValue Ptr = HG->getBasePtr();
29197 SDValue Index = HG->getIndex();
29198 SDValue Scale = HG->getScale();
29199 SDValue IntID = HG->getIntID();
29216 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
29217 SDValue PassThru = DAG.getSplatVector(IncSplatVT, DL, Zero);
29218 SDValue IncSplat = DAG.getSplatVector(
29220 SDValue Ops[] = {Chain, PassThru, Mask, Ptr, Index, Scale};
29228 SDValue Gather = DAG.getMaskedGather(
29232 SDValue GChain = Gather.getValue(1);
29235 SDValue ID =
29237 SDValue HistCnt =
29239 SDValue Mul = DAG.getNode(ISD::MUL, DL, IncSplatVT, HistCnt, IncSplat);
29240 SDValue Add = DAG.getNode(ISD::ADD, DL, IncSplatVT, Gather, Mul);
29247 SDValue ScatterOps[] = {GChain, Add, Mask, Ptr, Index, Scale};
29248 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), MemVT, DL,
29253 SDValue
29254 AArch64TargetLowering::LowerFixedLengthFPToIntToSVE(SDValue Op,
29264 SDValue Val = Op.getOperand(0);
29272 SDValue Pg = getPredicateForFixedLengthVector(DAG, DL, VT);
29284 SDValue Pg = getPredicateForFixedLengthVector(DAG, DL, SrcVT);
29296 static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2,
29312 return SDValue();
29325 SmallVector<SDValue, 8> TBLMask;
29328 SmallVector<SDValue, 8> AddRuntimeVLMask;
29333 return SDValue();
29356 return SDValue();
29371 SDValue VecMask =
29373 SDValue SVEMask = convertToScalableVector(DAG, MaskContainerVT, VecMask);
29375 SDValue Shuffle;
29384 SDValue VScale = (BitsPerElt == 64)
29387 SDValue VecMask =
29389 SDValue MulByMask = DAG.getNode(
29394 SDValue UpdatedVecMask =
29408 SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
29409 SDValue Op, SelectionDAG &DAG) const {
29417 SDValue Op1 = Op.getOperand(0);
29418 SDValue Op2 = Op.getOperand(1);
29433 SDValue SplatEl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarTy, Op1,
29446 SDValue Scalar = DAG.getNode(
29558 if (SDValue WideOp = tryWidenMaskForShuffle(Op, DAG))
29568 return SDValue();
29571 SDValue AArch64TargetLowering::getSVESafeBitCast(EVT VT, SDValue Op,
29629 SDValue N) const {
29638 SDValue Op, const APInt &OriginalDemandedBits,
29646 SDValue ShiftL = Op;
29647 SDValue ShiftR = Op->getOperand(0);
29677 SDValue Op0 = Op.getOperand(0);
29715 bool AArch64TargetLowering::isTargetCanonicalConstantNode(SDValue Op) const {