Lines Matching defs:DAG
1 //===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===//
10 // selection DAG.
161 SelectionDAG &DAG, const SDLoc &DL) {
164 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, Arg.ArgVT, Value);
166 DAG.getNode(Arg.Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
876 // In another words, find a way when "copysign" appears in DAG with vector
2043 static bool isS16(const SDValue &Op, SelectionDAG &DAG) {
2046 return DAG.ComputeNumSignBits(Op) == 17;
2049 /// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC
2066 /// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
2182 SDValue ARMTargetLowering::MoveToHPR(const SDLoc &dl, SelectionDAG &DAG,
2184 Val = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocVT.getSizeInBits()),
2187 Val = DAG.getNode(ARMISD::VMOVhr, dl, ValVT, Val);
2189 Val = DAG.getNode(ISD::TRUNCATE, dl,
2191 Val = DAG.getNode(ISD::BITCAST, dl, ValVT, Val);
2196 SDValue ARMTargetLowering::MoveFromHPR(const SDLoc &dl, SelectionDAG &DAG,
2200 Val = DAG.getNode(ARMISD::VMOVrh, dl,
2203 Val = DAG.getNode(ISD::BITCAST, dl,
2205 Val = DAG.getNode(ISD::ZERO_EXTEND, dl,
2208 return DAG.getNode(ISD::BITCAST, dl, LocVT, Val);
2216 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
2220 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2221 *DAG.getContext());
2241 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
2246 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
2252 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
2255 SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
2256 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
2257 DAG.getConstant(0, dl, MVT::i32));
2260 Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InGlue);
2264 Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InGlue);
2269 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
2270 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
2271 DAG.getConstant(1, dl, MVT::i32));
2274 Val = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), VA.getLocVT(),
2284 Val = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), Val);
2293 Val = MoveToHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Val);
2302 Val = handleCMSEValue(Val, Arg, DAG, dl);
2311 const SDLoc &dl, SelectionDAG &DAG, const CCValAssign &VA, SDValue StackPtr,
2316 MachineFunction &MF = DAG.getMachineFunction();
2320 auto PtrVT = getPointerTy(DAG.getDataLayout());
2323 DstAddr = DAG.getFrameIndex(FI, PtrVT);
2325 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
2327 SDValue PtrOff = DAG.getIntPtrConstant(Offset, dl);
2328 DstAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
2331 MachinePointerInfo::getStack(DAG.getMachineFunction(), Offset);
2344 SelectionDAG &DAG, SDValue Src, SDValue Dst, ISD::ArgFlagsTy Flags) const {
2345 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2346 ARMFunctionInfo *AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
2390 void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG,
2398 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
2399 DAG.getVTList(MVT::i32, MVT::i32), Arg);
2408 StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP,
2409 getPointerTy(DAG.getDataLayout()));
2414 computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff);
2416 DAG.getStore(Chain, dl, fmrrd.getValue(1 - id), DstAddr, DstInfo));
2431 SelectionDAG &DAG = CLI.DAG;
2443 MachineFunction &MF = DAG.getMachineFunction();
2445 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2456 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
2457 *DAG.getContext());
2528 MaybeAlign StackAlign = DAG.getDataLayout().getStackAlignment();
2549 Chain = DAG.getCALLSEQ_START(Chain, isTailCall ? 0 : NumBytes, 0, dl);
2553 DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy(DAG.getDataLayout()));
2579 computeAddrForCallArg(dl, DAG, VA, SDValue(), true, SPDiff);
2580 ByValCopyKind Copy = ByValNeedsCopyForTailCall(DAG, Src, Dst, Flags);
2600 DAG.getFrameIndex(TempFrameIdx, getPointerTy(DAG.getDataLayout()));
2602 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
2604 DAG.getConstant(Flags.getNonZeroByValAlign().value(), dl, MVT::i32);
2606 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
2609 DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs, Ops));
2615 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, ByValCopyChains);
2640 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
2643 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
2646 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
2649 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
2654 Chain = DAG.getStackArgumentTokenFactor(Chain);
2656 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chain,
2666 Arg = MoveFromHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Arg);
2675 DAG.getConstant(MaskValue, dl, MVT::getIntegerVT(LocBits));
2676 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocBits), Arg);
2677 Arg = DAG.getNode(ISD::AND, dl, MVT::getIntegerVT(LocBits), Arg, Mask);
2678 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
2684 SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2685 DAG.getConstant(0, dl, MVT::i32));
2686 SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2687 DAG.getConstant(1, dl, MVT::i32));
2689 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
2694 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
2701 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2702 MemOpChains.push_back(DAG.getStore(Chain, dl, Op1, DstAddr, DstInfo));
2705 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
2716 const TargetOptions &Options = DAG.getTarget().Options;
2745 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2748 SDValue Const = DAG.getConstant(4*i, dl, MVT::i32);
2749 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, ByValSrc, Const);
2751 DAG.getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo(),
2752 DAG.InferPtrAlign(AddArg));
2767 auto PtrVT = getPointerTy(DAG.getDataLayout());
2771 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2772 SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl);
2773 SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, ByValSrc, SrcOffset);
2774 SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl,
2777 DAG.getConstant(Flags.getNonZeroByValAlign().value(), dl, MVT::i32);
2779 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
2781 MemOpChains.push_back(DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs,
2789 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2791 SDValue Store = DAG.getStore(Chain, dl, Arg, DstAddr, DstInfo);
2797 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
2803 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
2821 auto PtrVt = getPointerTy(DAG.getDataLayout());
2833 Callee = DAG.getNode(ARMISD::Wrapper, dl, PtrVt,
2834 DAG.getTargetGlobalAddress(GVal, dl, PtrVt));
2842 SDValue Addr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2843 Addr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Addr);
2844 Callee = DAG.getLoad(
2845 PtrVt, dl, DAG.getEntryNode(), Addr,
2846 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
2854 Callee = DAG.getNode(ARMISD::Wrapper, dl, PtrVt,
2855 DAG.getTargetGlobalAddress(GVal, dl, PtrVt));
2860 *DAG.getContext(), Sym, ARMPCLabelIndex, 0);
2863 SDValue Addr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2864 Addr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Addr);
2865 Callee = DAG.getLoad(
2866 PtrVt, dl, DAG.getEntryNode(), Addr,
2867 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
2880 Callee = DAG.getNode(
2882 DAG.getTargetGlobalAddress(GVal, dl, PtrVt, 0, ARMII::MO_NONLAZY));
2883 Callee = DAG.getLoad(
2884 PtrVt, dl, DAG.getEntryNode(), Callee,
2885 MachinePointerInfo::getGOT(DAG.getMachineFunction()), MaybeAlign(),
2896 Callee = DAG.getTargetGlobalAddress(GVal, dl, PtrVt, /*offset=*/0,
2900 DAG.getLoad(PtrVt, dl, DAG.getEntryNode(),
2901 DAG.getNode(ARMISD::Wrapper, dl, PtrVt, Callee),
2902 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
2904 Callee = DAG.getTargetGlobalAddress(GVal, dl, PtrVt, 0, 0);
2914 ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym,
2916 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2917 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
2918 Callee = DAG.getLoad(
2919 PtrVt, dl, DAG.getEntryNode(), CPAddr,
2920 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
2921 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
2922 Callee = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVt, Callee, PICLabel);
2924 Callee = DAG.getTargetExternalSymbol(Sym, PtrVt, 0);
2932 DiagnosticInfoUnsupported Diag(DAG.getMachineFunction().getFunction(),
2936 DAG.getContext()->diagnose(Diag);
2940 DAG.getMachineFunction().getFunction(),
2943 DAG.getContext()->diagnose(Diag);
2975 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, dl);
2984 Ops.push_back(DAG.getSignedTargetConstant(SPDiff, dl, MVT::i32));
2990 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
3010 Ops.push_back(DAG.getRegisterMask(Mask));
3017 SDValue Ret = DAG.getNode(ARMISD::TC_RETURN, dl, MVT::Other, Ops);
3018 DAG.addNoMergeSiteInfo(Ret.getNode(), CLI.NoMerge);
3019 DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo));
3024 Chain = DAG.getNode(CallOpc, dl, {MVT::Other, MVT::Glue}, Ops);
3025 DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
3027 DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));
3036 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, dl);
3042 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
3115 const SelectionDAG &DAG = CLI.DAG;
3116 MachineFunction &MF = DAG.getMachineFunction();
3194 LLVMContext &C = *DAG.getContext();
3251 const SDLoc &DL, SelectionDAG &DAG) {
3252 const MachineFunction &MF = DAG.getMachineFunction();
3278 DAG.getConstant(LROffset, DL, MVT::i32, false));
3280 return DAG.getNode(ARMISD::INTRET_GLUE, DL, MVT::Other, RetOps);
3288 const SDLoc &dl, SelectionDAG &DAG) const {
3293 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
3294 *DAG.getContext());
3304 MachineFunction &MF = DAG.getMachineFunction();
3313 DAG.getMachineFunction().getFunction(),
3316 DAG.getContext()->diagnose(Diag);
3357 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
3365 Arg = MoveFromHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Arg);
3370 DAG.getConstant(MaskValue, dl, MVT::getIntegerVT(LocBits));
3371 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocBits), Arg);
3372 Arg = DAG.getNode(ISD::AND, dl, MVT::getIntegerVT(LocBits), Arg, Mask);
3373 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
3381 SDValue Half = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
3382 DAG.getConstant(0, dl, MVT::i32));
3383 SDValue HalfGPRs = DAG.getNode(ARMISD::VMOVRRD, dl,
3384 DAG.getVTList(MVT::i32, MVT::i32), Half);
3387 DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3390 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3393 DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3396 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3400 Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
3401 DAG.getConstant(1, dl, MVT::i32));
3405 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
3406 DAG.getVTList(MVT::i32, MVT::i32), Arg);
3407 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3410 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3412 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3415 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Glue);
3420 RetOps.push_back(DAG.getRegister(
3425 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
3429 RetOps.push_back(DAG.getRegister(*I, MVT::i32));
3431 RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64)));
3448 if (DAG.getMachineFunction().getFunction().hasFnAttribute("interrupt") &&
3452 return LowerInterruptReturn(RetOps, dl, DAG);
3457 return DAG.getNode(RetNode, dl, MVT::Other, RetOps);
3544 static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) {
3553 std::tie(Lo, Hi) = DAG.SplitScalar(WriteValue, DL, MVT::i32, MVT::i32);
3555 return DAG.getNode(ISD::WRITE_REGISTER, DL, MVT::Other, Ops);
3565 SelectionDAG &DAG) const {
3577 auto AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
3580 auto M = const_cast<Module*>(DAG.getMachineFunction().
3584 Twine(DAG.getDataLayout().getPrivateGlobalPrefix()) + "CP" +
3585 Twine(DAG.getMachineFunction().getFunctionNumber()) + "_" +
3588 SDValue GA = DAG.getTargetGlobalAddress(dyn_cast<GlobalValue>(GV),
3590 return LowerGlobalAddress(GA, DAG);
3600 DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, CPAlign);
3602 Res = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CPAlign);
3603 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Res);
3616 SelectionDAG &DAG) const {
3617 MachineFunction &MF = DAG.getMachineFunction();
3621 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3626 CPAddr = DAG.getTargetConstantPool(BA, PtrVT, Align(4));
3633 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3635 CPAddr = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, CPAddr);
3636 SDValue Result = DAG.getLoad(
3637 PtrVT, DL, DAG.getEntryNode(), CPAddr,
3638 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3641 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, DL, MVT::i32);
3642 return DAG.getNode(ARMISD::PIC_ADD, DL, PtrVT, Result, PICLabel);
3671 SelectionDAG &DAG) const {
3678 SDValue DescAddr = LowerGlobalAddressDarwin(Op, DAG);
3682 SDValue Chain = DAG.getEntryNode();
3683 SDValue FuncTLVGet = DAG.getLoad(
3685 MachinePointerInfo::getGOT(DAG.getMachineFunction()), Align(4),
3690 MachineFunction &F = DAG.getMachineFunction();
3700 const uint32_t *Mask = ARI->getTLSCallPreservedMask(DAG.getMachineFunction());
3705 Chain = DAG.getCopyToReg(Chain, DL, ARM::R0, DescAddr, SDValue());
3707 DAG.getNode(ARMISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue),
3708 Chain, FuncTLVGet, DAG.getRegister(ARM::R0, MVT::i32),
3709 DAG.getRegisterMask(Mask), Chain.getValue(1));
3710 return DAG.getCopyFromReg(Chain, DL, ARM::R0, MVT::i32, Chain.getValue(1));
3715 SelectionDAG &DAG) const {
3718 SDValue Chain = DAG.getEntryNode();
3719 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3724 DAG.getTargetConstant(Intrinsic::arm_mrc, DL, MVT::i32),
3725 DAG.getTargetConstant(15, DL, MVT::i32),
3726 DAG.getTargetConstant(0, DL, MVT::i32),
3727 DAG.getTargetConstant(13, DL, MVT::i32),
3728 DAG.getTargetConstant(0, DL, MVT::i32),
3729 DAG.getTargetConstant(2, DL, MVT::i32)};
3730 SDValue CurrentTEB = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
3731 DAG.getVTList(MVT::i32, MVT::Other), Ops);
3739 DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x2c, DL));
3740 TLSArray = DAG.getLoad(PtrVT, DL, Chain, TLSArray, MachinePointerInfo());
3747 DAG.getTargetExternalSymbol("_tls_index", PtrVT, ARMII::MO_NO_FLAG);
3748 TLSIndex = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, TLSIndex);
3749 TLSIndex = DAG.getLoad(PtrVT, DL, Chain, TLSIndex, MachinePointerInfo());
3751 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
3752 DAG.getConstant(2, DL, MVT::i32));
3753 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
3754 DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot),
3760 SDValue Offset = DAG.getLoad(
3762 DAG.getNode(ARMISD::Wrapper, DL, MVT::i32,
3763 DAG.getTargetConstantPool(CPV, PtrVT, Align(4))),
3764 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3766 return DAG.getNode(ISD::ADD, DL, PtrVT, TLS, Offset);
3772 SelectionDAG &DAG) const {
3774 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3776 MachineFunction &MF = DAG.getMachineFunction();
3782 SDValue Argument = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3783 Argument = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Argument);
3784 Argument = DAG.getLoad(
3785 PtrVT, dl, DAG.getEntryNode(), Argument,
3786 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3789 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3790 Argument = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Argument, PICLabel);
3796 Entry.Ty = (Type *) Type::getInt32Ty(*DAG.getContext());
3800 TargetLowering::CallLoweringInfo CLI(DAG);
3802 CallingConv::C, Type::getInt32Ty(*DAG.getContext()),
3803 DAG.getExternalSymbol("__tls_get_addr", PtrVT), std::move(Args));
3813 SelectionDAG &DAG,
3818 SDValue Chain = DAG.getEntryNode();
3819 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3821 SDValue ThreadPointer = DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
3824 MachineFunction &MF = DAG.getMachineFunction();
3833 Offset = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3834 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
3835 Offset = DAG.getLoad(
3837 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3840 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3841 Offset = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Offset, PICLabel);
3843 Offset = DAG.getLoad(
3845 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3851 Offset = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3852 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
3853 Offset = DAG.getLoad(
3855 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3860 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
3864 ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
3866 if (DAG.getTarget().useEmulatedTLS())
3867 return LowerToTLSEmulatedModel(GA, DAG);
3870 return LowerGlobalTLSAddressDarwin(Op, DAG);
3873 return LowerGlobalTLSAddressWindows(Op, DAG);
3882 return LowerToTLSGeneralDynamicModel(GA, DAG);
3885 return LowerToTLSExecModels(GA, DAG, model);
3909 const GlobalValue *GV, SelectionDAG &DAG,
3918 const Function &F = DAG.getMachineFunction().getFunction();
3927 DAG.getMachineFunction().getTarget().Options.EnableFastISel)
3950 unsigned Size = DAG.getDataLayout().getTypeAllocSize(Init->getType());
3951 Align PrefAlign = DAG.getDataLayout().getPreferredAlign(GVar);
3960 MachineFunction &MF = DAG.getMachineFunction();
3990 Init = ConstantDataArray::get(*DAG.getContext(), V);
3994 SDValue CPAddr = DAG.getTargetConstantPool(CPVal, PtrVT, Align(4));
4001 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
4014 SelectionDAG &DAG) const {
4018 return LowerGlobalAddressWindows(Op, DAG);
4020 return LowerGlobalAddressELF(Op, DAG);
4022 return LowerGlobalAddressDarwin(Op, DAG);
4027 SelectionDAG &DAG) const {
4028 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4035 if (SDValue V = promoteToConstantPool(this, GV, DAG, PtrVT, dl))
4039 SDValue G = DAG.getTargetGlobalAddress(
4041 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
4044 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
4045 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
4049 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT);
4050 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
4057 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_SBREL);
4058 RelAddr = DAG.getNode(ARMISD::Wrapper, dl, PtrVT, G);
4062 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
4063 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
4064 RelAddr = DAG.getLoad(
4065 PtrVT, dl, DAG.getEntryNode(), CPAddr,
4066 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
4068 SDValue SB = DAG.getCopyFromReg(DAG.getEntryNode(), dl, ARM::R9, PtrVT);
4069 SDValue Result = DAG.getNode(ISD::ADD, dl, PtrVT, SB, RelAddr);
4082 return DAG.getNode(ARMISD::Wrapper, dl, PtrVT,
4083 DAG.getTargetGlobalAddress(GV, dl, PtrVT));
4085 SDValue CPAddr = DAG.getTargetConstantPool(GV, PtrVT, Align(4));
4086 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
4087 return DAG.getLoad(
4088 PtrVT, dl, DAG.getEntryNode(), CPAddr,
4089 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
4094 SelectionDAG &DAG) const {
4097 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4109 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_NONLAZY);
4110 SDValue Result = DAG.getNode(Wrapper, dl, PtrVT, G);
4113 Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
4114 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
4119 SelectionDAG &DAG) const {
4133 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4141 Result = DAG.getNode(ARMISD::Wrapper, DL, PtrVT,
4142 DAG.getTargetGlobalAddress(GV, DL, PtrVT, /*offset=*/0,
4145 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
4146 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
4151 ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const {
4153 SDValue Val = DAG.getConstant(0, dl, MVT::i32);
4154 return DAG.getNode(ARMISD::EH_SJLJ_SETJMP, dl,
4155 DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0),
4160 ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const {
4162 return DAG.getNode(ARMISD::EH_SJLJ_LONGJMP, dl, MVT::Other, Op.getOperand(0),
4163 Op.getOperand(1), DAG.getConstant(0, dl, MVT::i32));
4167 SelectionDAG &DAG) const {
4169 return DAG.getNode(ARMISD::EH_SJLJ_SETUP_DISPATCH, dl, MVT::Other,
4174 SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget) const {
4181 MachineFunction &MF = DAG.getMachineFunction();
4182 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4188 ARI->getCallPreservedMask(DAG.getMachineFunction(), CallingConv::C);
4193 DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, PtrVT);
4196 DAG.getTargetExternalSymbol("\01__gnu_mcount_nc", PtrVT, 0);
4197 SDValue RegisterMask = DAG.getRegisterMask(Mask);
4200 DAG.getMachineNode(
4202 {ReturnAddress, DAG.getTargetConstant(ARMCC::AL, dl, PtrVT),
4203 DAG.getRegister(0, PtrVT), Callee, RegisterMask, Chain}),
4206 DAG.getMachineNode(ARM::BL_PUSHLR, dl, ResultTys,
4214 ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
4221 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4222 return DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
4228 DAG.getNode(ISD::SRA, dl, VTy, Operand, DAG.getConstant(31, dl, VTy));
4229 SDValue XOR = DAG.getNode(ISD::XOR, dl, VTy, SRA, Operand);
4231 DAG.getNode(ISD::SHL, dl, VTy, XOR, DAG.getConstant(1, dl, VTy));
4233 DAG.getNode(ISD::OR, dl, VTy, SHL, DAG.getConstant(1, dl, VTy));
4234 SDValue Result = DAG.getNode(ISD::CTLZ, dl, VTy, OR);
4243 std::tie(Lo, Hi) = DAG.SplitScalar(Operand, dl, VTy, VTy);
4244 SDValue Constant0 = DAG.getConstant(0, dl, VTy);
4245 SDValue Constant1 = DAG.getConstant(1, dl, VTy);
4246 SDValue Constant31 = DAG.getConstant(31, dl, VTy);
4247 SDValue SRAHi = DAG.getNode(ISD::SRA, dl, VTy, Hi, Constant31);
4248 SDValue XORHi = DAG.getNode(ISD::XOR, dl, VTy, SRAHi, Hi);
4249 SDValue SHLHi = DAG.getNode(ISD::SHL, dl, VTy, XORHi, Constant1);
4250 SDValue ORHi = DAG.getNode(ISD::OR, dl, VTy, SHLHi, Constant1);
4251 SDValue CLSHi = DAG.getNode(ISD::CTLZ, dl, VTy, ORHi);
4253 DAG.getSetCC(dl, MVT::i1, CLSHi, Constant31, ISD::CondCode::SETEQ);
4255 DAG.getSetCC(dl, MVT::i1, Hi, Constant0, ISD::CondCode::SETEQ);
4257 DAG.getSelect(dl, VTy, HiIsZero, Lo, DAG.getNOT(dl, Lo, VTy));
4258 SDValue CLZAdjustedLo = DAG.getNode(ISD::CTLZ, dl, VTy, AdjustedLo);
4260 DAG.getSelect(dl, VTy, CheckLo,
4261 DAG.getNode(ISD::ADD, dl, VTy, CLZAdjustedLo, Constant31), CLSHi);
4265 MachineFunction &MF = DAG.getMachineFunction();
4268 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4275 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
4276 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
4277 SDValue Result = DAG.getLoad(
4278 PtrVT, dl, DAG.getEntryNode(), CPAddr,
4279 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
4282 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
4283 Result = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
4288 return DAG.getNode(ISD::ABS, SDLoc(Op), Op.getValueType(),
4292 return DAG.getNode(ISD::ABDS, SDLoc(Op), Op.getValueType(),
4296 return DAG.getNode(ISD::ABDU, SDLoc(Op), Op.getValueType(),
4302 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
4309 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
4318 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
4327 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
4332 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
4336 return DAG.getNode(ARMISD::VTBL1, SDLoc(Op), Op.getValueType(),
4339 return DAG.getNode(ARMISD::VTBL2, SDLoc(Op), Op.getValueType(),
4343 return DAG.getNode(ARMISD::PREDICATE_CAST, SDLoc(Op), Op.getValueType(),
4346 return DAG.getNode(ARMISD::VECTOR_REG_CAST, SDLoc(Op), Op.getValueType(),
4349 return DAG.getNode(ARMISD::LSLL, SDLoc(Op), Op->getVTList(),
4352 return DAG.getNode(ARMISD::ASRL, SDLoc(Op), Op->getVTList(),
4357 static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
4370 return DAG.getNode(ARMISD::MEMBARRIER_MCR, dl, MVT::Other, Op.getOperand(0),
4371 DAG.getConstant(0, dl, MVT::i32));
4388 return DAG.getNode(ISD::INTRINSIC_VOID, dl, MVT::Other, Op.getOperand(0),
4389 DAG.getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
4390 DAG.getConstant(Domain, dl, MVT::i32));
4393 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
4415 return DAG.getNode(ARMISD::PRELOAD, dl, MVT::Other, Op.getOperand(0),
4416 Op.getOperand(1), DAG.getConstant(isRead, dl, MVT::i32),
4417 DAG.getConstant(isData, dl, MVT::i32));
4420 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
4421 MachineFunction &MF = DAG.getMachineFunction();
4427 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
4428 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
4430 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1),
4437 SelectionDAG &DAG,
4439 MachineFunction &MF = DAG.getMachineFunction();
4450 SDValue ArgValue = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
4458 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
4459 ArgValue2 = DAG.getLoad(
4461 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
4464 ArgValue2 = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
4468 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, ArgValue, ArgValue2);
4479 int ARMTargetLowering::StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG,
4495 MachineFunction &MF = DAG.getMachineFunction();
4510 auto PtrVT = getPointerTy(DAG.getDataLayout());
4512 SDValue FIN = DAG.getFrameIndex(FrameIndex, PtrVT);
4520 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
4521 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
4524 FIN = DAG.getNode(ISD::ADD, dl, PtrVT, FIN, DAG.getConstant(4, dl, PtrVT));
4528 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
4533 void ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
4538 MachineFunction &MF = DAG.getMachineFunction();
4547 CCInfo, DAG, dl, Chain, nullptr, CCInfo.getInRegsParamsCount(),
4553 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
4559 Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(ValueBits), Val);
4560 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::getIntegerVT(PartBits), Val);
4561 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
4569 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
4576 Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(PartBits), Val);
4577 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::getIntegerVT(ValueBits), Val);
4578 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
4587 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4588 MachineFunction &MF = DAG.getMachineFunction();
4595 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4596 *DAG.getContext());
4640 auto PtrVT = getPointerTy(DAG.getDataLayout());
4658 GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4663 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4664 ArgValue2 = DAG.getLoad(
4666 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
4668 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4670 ArgValue = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
4671 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, ArgValue,
4672 ArgValue1, DAG.getIntPtrConstant(0, dl));
4673 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, ArgValue,
4674 ArgValue2, DAG.getIntPtrConstant(1, dl));
4676 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4698 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
4714 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue);
4723 ArgValue = MoveToHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), ArgValue);
4732 ArgValue = handleCMSEValue(ArgValue, Arg, DAG, dl);
4758 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
4760 InVals.push_back(DAG.getFrameIndex(FrameIndex, PtrVT));
4768 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4769 InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
4771 DAG.getMachineFunction(), FI)));
4780 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.getStackSize(),
4784 DAG.getMachineFunction().getFunction(),
4786 DAG.getContext()->diagnose(Diag);
4795 MaybeAlign StackAlign = DAG.getDataLayout().getStackAlignment();
4805 DAG.getMachineFunction().getFunction(),
4807 DAG.getContext()->diagnose(Diag);
4840 SDValue &ARMcc, SelectionDAG &DAG,
4852 RHS = DAG.getConstant(C - 1, dl, MVT::i32);
4859 RHS = DAG.getConstant(C - 1, dl, MVT::i32);
4866 RHS = DAG.getConstant(C + 1, dl, MVT::i32);
4873 RHS = DAG.getConstant(C + 1, dl, MVT::i32);
4917 SDValue ShiftAmt = DAG.getConstant(ShiftBits, dl, MVT::i32);
4918 LHS = DAG.getNode(ISD::SHL, dl, MVT::i32, LHS.getOperand(0), ShiftAmt);
4919 RHS = DAG.getConstant(RHSV << ShiftBits, dl, MVT::i32);
4936 DAG.getNode(ARMISD::LSLS, dl, DAG.getVTList(MVT::i32, FlagsVT),
4937 LHS.getOperand(0), DAG.getConstant(ShiftAmt, dl, MVT::i32));
4938 ARMcc = DAG.getConstant(ARMCC::HI, dl, MVT::i32);
4970 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
4971 return DAG.getNode(CompareType, dl, FlagsVT, LHS, RHS);
4976 SelectionDAG &DAG, const SDLoc &dl,
4981 Flags = DAG.getNode(Signaling ? ARMISD::CMPFPE : ARMISD::CMPFP, dl, FlagsVT,
4984 Flags = DAG.getNode(Signaling ? ARMISD::CMPFPEw0 : ARMISD::CMPFPw0, dl,
4986 return DAG.getNode(ARMISD::FMSTAT, dl, FlagsVT, Flags);
4994 ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
5012 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
5013 Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS);
5014 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, Value, LHS);
5017 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
5020 Value = DAG.getNode(ARMISD::ADDC, dl,
5021 DAG.getVTList(Op.getValueType(), MVT::i32), LHS, RHS)
5023 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, Value, LHS);
5026 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
5027 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
5028 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, LHS, RHS);
5031 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
5032 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
5033 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, LHS, RHS);
5037 ARMcc = DAG.getConstant(ARMCC::EQ, dl, MVT::i32);
5038 Value = DAG.getNode(ISD::UMUL_LOHI, dl,
5039 DAG.getVTList(Op.getValueType(), Op.getValueType()),
5041 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, Value.getValue(1),
5042 DAG.getConstant(0, dl, MVT::i32));
5048 ARMcc = DAG.getConstant(ARMCC::EQ, dl, MVT::i32);
5049 Value = DAG.getNode(ISD::SMUL_LOHI, dl,
5050 DAG.getVTList(Op.getValueType(), Op.getValueType()),
5052 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, Value.getValue(1),
5053 DAG.getNode(ISD::SRA, dl, Op.getValueType(),
5055 DAG.getConstant(31, dl, MVT::i32)));
5064 ARMTargetLowering::LowerSignedALUO(SDValue Op, SelectionDAG &DAG) const {
5066 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
5071 std::tie(Value, OverflowCmp) = getARMXALUOOp(Op, DAG, ARMcc);
5074 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
5075 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
5079 DAG.getNode(ARMISD::CMOV, dl, VT, TVal, FVal, ARMcc, OverflowCmp);
5081 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
5082 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
5086 SelectionDAG &DAG) {
5092 SDValue Carry = DAG.getNode(ARMISD::SUBC, DL,
5093 DAG.getVTList(CarryVT, MVT::i32),
5094 BoolCarry, DAG.getConstant(1, DL, CarryVT));
5099 SelectionDAG &DAG) {
5104 return DAG.getNode(ARMISD::ADDE, DL, DAG.getVTList(VT, MVT::i32),
5105 DAG.getConstant(0, DL, MVT::i32),
5106 DAG.getConstant(0, DL, MVT::i32), Flags);
5110 SelectionDAG &DAG) const {
5112 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
5120 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
5127 Value = DAG.getNode(ARMISD::ADDC, dl, VTs, LHS, RHS);
5129 Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG);
5132 Value = DAG.getNode(ARMISD::SUBC, dl, VTs, LHS, RHS);
5134 Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG);
5137 Overflow = DAG.getNode(ISD::SUB, dl, MVT::i32,
5138 DAG.getConstant(1, dl, MVT::i32), Overflow);
5143 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
5146 static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG,
5194 DAG.getNode(NewOpcode, dl, MVT::i32,
5195 DAG.getSExtOrTrunc(Op->getOperand(0), dl, MVT::i32),
5196 DAG.getSExtOrTrunc(Op->getOperand(1), dl, MVT::i32));
5197 return DAG.getNode(ISD::TRUNCATE, dl, VT, Add);
5200 SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
5210 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0)))
5215 std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc);
5218 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, OverflowCmp, DAG);
5248 Cond.getOperand(3), DAG);
5254 Cond = DAG.getNode(ISD::AND, dl, Cond.getValueType(), Cond,
5255 DAG.getConstant(1, dl, Cond.getValueType()));
5257 return DAG.getSelectCC(dl, Cond,
5258 DAG.getConstant(0, dl, Cond.getValueType()),
5314 SDValue Flags, SelectionDAG &DAG) const {
5316 FalseVal = DAG.getNode(ARMISD::VMOVRRD, dl,
5317 DAG.getVTList(MVT::i32, MVT::i32), FalseVal);
5318 TrueVal = DAG.getNode(ARMISD::VMOVRRD, dl,
5319 DAG.getVTList(MVT::i32, MVT::i32), TrueVal);
5326 SDValue Low = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseLow, TrueLow,
5328 SDValue High = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseHigh, TrueHigh,
5331 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Low, High);
5333 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, Flags);
5376 static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG) {
5425 return DAG.getNode(ARMISD::SSAT, dl, VT, V2Tmp,
5426 DAG.getConstant(llvm::countr_one(K), dl, VT));
5428 return DAG.getNode(ARMISD::USAT, dl, VT, V2Tmp,
5429 DAG.getConstant(llvm::countr_one(K), dl, VT));
5486 SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
5492 if (SDValue SatValue = LowerSaturatingConditional(Op, DAG))
5505 SDValue ShiftV = DAG.getNode(ISD::SRA, dl, VT, SatValue,
5506 DAG.getConstant(31, dl, VT));
5508 SDValue NotShiftV = DAG.getNode(ISD::XOR, dl, VT, ShiftV,
5509 DAG.getAllOnesConstant(dl, VT));
5510 return DAG.getNode(ISD::AND, dl, VT, SatValue, NotShiftV);
5512 return DAG.getNode(ISD::OR, dl, VT, SatValue, ShiftV);
5565 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5567 return DAG.getNode(Opcode, dl, VT, TrueVal, FalseVal, ARMcc, Cmp);
5572 DAG.getTargetLoweringInfo().softenSetCCOperands(
5573 DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS);
5578 RHS = DAG.getConstant(0, dl, LHS.getValueType());
5606 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5609 ARMcc = DAG.getConstant(ARMCC::GE, dl, MVT::i32);
5610 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Cmp, DAG);
5638 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5639 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5640 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Cmp, DAG);
5642 SDValue ARMcc2 = DAG.getConstant(CondCode2, dl, MVT::i32);
5643 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, Cmp, DAG);
5671 static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) {
5673 return DAG.getConstant(0, SDLoc(Op), MVT::i32);
5676 return DAG.getLoad(MVT::i32, SDLoc(Op), Ld->getChain(), Ld->getBasePtr(),
5683 static void expandf64Toi32(SDValue Op, SelectionDAG &DAG,
5688 RetVal1 = DAG.getConstant(0, dl, MVT::i32);
5689 RetVal2 = DAG.getConstant(0, dl, MVT::i32);
5696 DAG.getLoad(MVT::i32, dl, Ld->getChain(), Ptr, Ld->getPointerInfo(),
5700 SDValue NewPtr = DAG.getNode(ISD::ADD, dl,
5701 PtrType, Ptr, DAG.getConstant(4, dl, PtrType));
5702 RetVal2 = DAG.getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
5715 ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
5736 SDValue Mask = DAG.getConstant(0x7fffffff, dl, MVT::i32);
5739 LHS = DAG.getNode(ISD::AND, dl, MVT::i32,
5740 bitcastf32Toi32(LHS, DAG), Mask);
5741 RHS = DAG.getNode(ISD::AND, dl, MVT::i32,
5742 bitcastf32Toi32(RHS, DAG), Mask);
5743 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5744 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
5750 expandf64Toi32(LHS, DAG, LHS1, LHS2);
5751 expandf64Toi32(RHS, DAG, RHS1, RHS2);
5752 LHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, LHS2, Mask);
5753 RHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, RHS2, Mask);
5755 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5757 return DAG.getNode(ARMISD::BCC_i64, dl, MVT::Other, Ops);
5763 SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
5778 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0)))
5784 std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc);
5790 ARMcc = DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32);
5792 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
5799 SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
5808 DAG.getTargetLoweringInfo().softenSetCCOperands(
5809 DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS);
5814 RHS = DAG.getConstant(0, dl, LHS.getValueType());
5829 if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0)))
5835 std::tie(Value, OverflowCmp) = getARMXALUOOp(LHS.getValue(0), DAG, ARMcc);
5842 ARMcc = DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32);
5845 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
5851 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5852 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, Cmp);
5858 if (SDValue Result = OptimizeVFPBrcond(Op, DAG))
5865 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5866 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5868 SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Ops);
5870 ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32);
5872 Res = DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Ops);
5877 SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
5883 EVT PTy = getPointerTy(DAG.getDataLayout());
5885 SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy);
5886 Table = DAG.getNode(ARMISD::WrapperJT, dl, MVT::i32, JTI);
5887 Index = DAG.getNode(ISD::MUL, dl, PTy, Index, DAG.getConstant(4, dl, PTy));
5888 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Index);
5894 return DAG.getNode(ARMISD::BR2_JT, dl, MVT::Other, Chain,
5899 DAG.getLoad((EVT)MVT::i32, dl, Chain, Addr,
5900 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()));
5902 Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Addr);
5903 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
5906 DAG.getLoad(PTy, dl, Chain, Addr,
5907 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()));
5909 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
5913 static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
5920 return DAG.UnrollVectorOp(Op.getNode());
5923 const bool HasFullFP16 = DAG.getSubtarget<ARMSubtarget>().hasFullFP16();
5937 return DAG.UnrollVectorOp(Op.getNode());
5939 Op = DAG.getNode(Op.getOpcode(), dl, NewTy, Op.getOperand(0));
5940 return DAG.getNode(ISD::TRUNCATE, dl, VT, Op);
5943 SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
5946 return LowerVectorFP_TO_INT(Op, DAG);
5964 std::tie(Result, Chain) = makeLibCall(DAG, LC, Op.getValueType(), SrcVal,
5966 return IsStrict ? DAG.getMergeValues({Result, Chain}, Loc) : Result;
5973 DAG.getNode(Op.getOpcode() == ISD::STRICT_FP_TO_SINT ? ISD::FP_TO_SINT
5976 return DAG.getMergeValues({Result, Op.getOperand(0)}, Loc);
5982 static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
6009 SDValue CVT = DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
6010 DAG.getValueType(VT.getScalarType()));
6011 SDValue Max = DAG.getNode(IsSigned ? ISD::SMIN : ISD::UMIN, DL, VT, CVT,
6012 DAG.getConstant((1 << BW) - 1, DL, VT));
6014 Max = DAG.getNode(ISD::SMAX, DL, VT, Max,
6015 DAG.getSignedConstant(-(1 << BW), DL, VT));
6019 static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
6026 return DAG.UnrollVectorOp(Op.getNode());
6033 const bool HasFullFP16 = DAG.getSubtarget<ARMSubtarget>().hasFullFP16();
6043 return DAG.UnrollVectorOp(Op.getNode());
6059 Op = DAG.getNode(CastOpc, dl, DestVecType, Op.getOperand(0));
6060 return DAG.getNode(Opc, dl, VT, Op);
6063 SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const {
6066 return LowerVectorINT_TO_FP(Op, DAG);
6076 return makeLibCall(DAG, LC, Op.getValueType(), Op.getOperand(0),
6083 SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
6097 SDValue Mask = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v2i32,
6098 DAG.getTargetConstant(EncodedVal, dl, MVT::i32));
6101 Mask = DAG.getNode(ARMISD::VSHLIMM, dl, OpVT,
6102 DAG.getNode(ISD::BITCAST, dl, OpVT, Mask),
6103 DAG.getConstant(32, dl, MVT::i32));
6105 Tmp0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp0);
6107 Tmp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp1);
6109 Tmp1 = DAG.getNode(ARMISD::VSHLIMM, dl, OpVT,
6110 DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1),
6111 DAG.getConstant(32, dl, MVT::i32));
6113 Tmp1 = DAG.getNode(ARMISD::VSHRuIMM, dl, MVT::v1i64,
6114 DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, Tmp1),
6115 DAG.getConstant(32, dl, MVT::i32));
6116 Tmp0 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp0);
6117 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1);
6119 SDValue AllOnes = DAG.getTargetConstant(ARM_AM::createVMOVModImm(0xe, 0xff),
6121 AllOnes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v8i8, AllOnes);
6122 SDValue MaskNot = DAG.getNode(ISD::XOR, dl, OpVT, Mask,
6123 DAG.getNode(ISD::BITCAST, dl, OpVT, AllOnes));
6125 SDValue Res = DAG.getNode(ISD::OR, dl, OpVT,
6126 DAG.getNode(ISD::AND, dl, OpVT, Tmp1, Mask),
6127 DAG.getNode(ISD::AND, dl, OpVT, Tmp0, MaskNot));
6129 Res = DAG.getNode(ISD::BITCAST, dl, MVT::v2f32, Res);
6130 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
6131 DAG.getConstant(0, dl, MVT::i32));
6133 Res = DAG.getNode(ISD::BITCAST, dl, MVT::f64, Res);
6141 Tmp1 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32),
6143 Tmp1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp1);
6146 SDValue Mask1 = DAG.getConstant(0x80000000, dl, MVT::i32);
6147 SDValue Mask2 = DAG.getConstant(0x7fffffff, dl, MVT::i32);
6148 Tmp1 = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp1, Mask1);
6150 Tmp0 = DAG.getNode(ISD::AND, dl, MVT::i32,
6151 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp0), Mask2);
6152 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
6153 DAG.getNode(ISD::OR, dl, MVT::i32, Tmp0, Tmp1));
6157 Tmp0 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32),
6160 SDValue Hi = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp0.getValue(1), Mask2);
6161 Hi = DAG.getNode(ISD::OR, dl, MVT::i32, Hi, Tmp1);
6162 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
6165 SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
6166 MachineFunction &MF = DAG.getMachineFunction();
6170 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
6177 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
6178 SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
6179 return DAG.getLoad(VT, dl, DAG.getEntryNode(),
6180 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),
6186 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
6189 SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
6192 MachineFunction &MF = DAG.getMachineFunction();
6200 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
6202 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
6223 SelectionDAG &DAG) {
6230 SDValue Read = DAG.getNode(ISD::READ_REGISTER, DL,
6231 DAG.getVTList(MVT::i32, MVT::i32, MVT::Other),
6235 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Read.getValue(0),
6248 SelectionDAG &DAG) {
6283 *DAG.getContext(), DstVT.getScalarType(),
6285 SDValue BitCast = DAG.getNode(ISD::BITCAST, dl, VecVT, ExtractSrc);
6286 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DstVT, BitCast,
6287 DAG.getConstant(NewIndex.getZExtValue(), dl, MVT::i32));
6295 SDValue ARMTargetLowering::ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
6297 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6308 return MoveToHPR(SDLoc(N), DAG, MVT::i32, DstVT.getSimpleVT(),
6309 DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), MVT::i32, Op));
6314 Op = DAG.getBitcast(MVT::f16, Op);
6315 return DAG.getNode(
6317 MoveFromHPR(SDLoc(N), DAG, MVT::i32, SrcVT.getSimpleVT(), Op));
6327 if (SDValue Val = CombineVMOVDRRCandidateWithVecOp(N, DAG))
6330 std::tie(Lo, Hi) = DAG.SplitScalar(Op, dl, MVT::i32, MVT::i32);
6331 return DAG.getNode(ISD::BITCAST, dl, DstVT,
6332 DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi));
6338 if (DAG.getDataLayout().isBigEndian() && SrcVT.isVector() &&
6340 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
6341 DAG.getVTList(MVT::i32, MVT::i32),
6342 DAG.getNode(ARMISD::VREV64, dl, SrcVT, Op));
6344 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
6345 DAG.getVTList(MVT::i32, MVT::i32), Op);
6347 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Cvt, Cvt.getValue(1));
6359 static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
6362 SDValue EncodedVal = DAG.getTargetConstant(0, dl, MVT::i32);
6364 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, EncodedVal);
6365 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
6371 SelectionDAG &DAG) const {
6384 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
6385 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
6386 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
6387 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
6388 DAG.getConstant(VTBits, dl, MVT::i32));
6389 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
6390 SDValue LoSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
6391 SDValue LoBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
6392 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6393 ISD::SETGE, ARMcc, DAG, dl);
6395 DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, LoBigShift, ARMcc, CmpLo);
6397 SDValue HiSmallShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
6399 ? DAG.getNode(Opc, dl, VT, ShOpHi,
6400 DAG.getConstant(VTBits - 1, dl, VT))
6401 : DAG.getConstant(0, dl, VT);
6402 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6403 ISD::SETGE, ARMcc, DAG, dl);
6405 DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, ARMcc, CmpHi);
6408 return DAG.getMergeValues(Ops, dl);
6414 SelectionDAG &DAG) const {
6425 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
6426 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
6427 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
6428 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
6429 SDValue HiSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
6431 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
6432 DAG.getConstant(VTBits, dl, MVT::i32));
6433 SDValue HiBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
6434 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6435 ISD::SETGE, ARMcc, DAG, dl);
6437 DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, ARMcc, CmpHi);
6439 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6440 ISD::SETGE, ARMcc, DAG, dl);
6441 SDValue LoSmallShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
6442 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift,
6443 DAG.getConstant(0, dl, VT), ARMcc, CmpLo);
6446 return DAG.getMergeValues(Ops, dl);
6450 SelectionDAG &DAG) const {
6458 DAG.getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32)};
6461 DAG.getNode(ISD::INTRINSIC_W_CHAIN, dl, {MVT::i32, MVT::Other}, Ops);
6463 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPSCR,
6464 DAG.getConstant(1U << 22, dl, MVT::i32));
6465 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
6466 DAG.getConstant(22, dl, MVT::i32));
6467 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
6468 DAG.getConstant(3, dl, MVT::i32));
6469 return DAG.getMergeValues({And, Chain}, dl);
6473 SelectionDAG &DAG) const {
6489 RMValue = DAG.getNode(ISD::SUB, DL, MVT::i32, RMValue,
6490 DAG.getConstant(1, DL, MVT::i32));
6491 RMValue = DAG.getNode(ISD::AND, DL, MVT::i32, RMValue,
6492 DAG.getConstant(0x3, DL, MVT::i32));
6493 RMValue = DAG.getNode(ISD::SHL, DL, MVT::i32, RMValue,
6494 DAG.getConstant(ARM::RoundingBitsPos, DL, MVT::i32));
6498 DAG.getConstant(Intrinsic::arm_get_fpscr, DL, MVT::i32)};
6500 DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops);
6506 FPSCR = DAG.getNode(ISD::AND, DL, MVT::i32, FPSCR,
6507 DAG.getConstant(RMMask, DL, MVT::i32));
6508 FPSCR = DAG.getNode(ISD::OR, DL, MVT::i32, FPSCR, RMValue);
6510 Chain, DAG.getConstant(Intrinsic::arm_set_fpscr, DL, MVT::i32), FPSCR};
6511 return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
6515 SelectionDAG &DAG) const {
6523 DAG.getConstant(Intrinsic::arm_get_fpscr, DL, MVT::i32)};
6525 DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops);
6530 DAG.getNode(ISD::AND, DL, MVT::i32, FPSCR,
6531 DAG.getConstant(ARM::FPStatusBits, DL, MVT::i32));
6533 DAG.getNode(ISD::AND, DL, MVT::i32, Mode,
6534 DAG.getConstant(~ARM::FPStatusBits, DL, MVT::i32));
6535 FPSCR = DAG.getNode(ISD::OR, DL, MVT::i32, FPSCRMasked, InputMasked);
6538 Chain, DAG.getConstant(Intrinsic::arm_set_fpscr, DL, MVT::i32), FPSCR};
6539 return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
6543 SelectionDAG &DAG) const {
6550 DAG.getConstant(Intrinsic::arm_get_fpscr, DL, MVT::i32)};
6552 DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops);
6556 SDValue FPSCRMasked = DAG.getNode(
6558 DAG.getConstant(ARM::FPStatusBits | ARM::FPReservedBits, DL, MVT::i32));
6560 DAG.getConstant(Intrinsic::arm_set_fpscr, DL, MVT::i32),
6562 return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
6565 static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
6573 SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X);
6574 SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX);
6580 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6581 DAG.getTargetConstant(1, dl, ElemTy));
6582 SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
6583 return DAG.getNode(ISD::CTPOP, dl, VT, Bits);
6591 DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6592 DAG.getTargetConstant(NumBits - 1, dl, ElemTy));
6593 SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB);
6594 return DAG.getNode(ISD::SUB, dl, VT, WidthMinus1, CTLZ);
6603 SDValue FF = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6604 DAG.getTargetConstant(0x1eff, dl, MVT::i32));
6605 Bits = DAG.getNode(ISD::ADD, dl, VT, LSB, FF);
6607 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6608 DAG.getTargetConstant(1, dl, ElemTy));
6609 Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
6611 return DAG.getNode(ISD::CTPOP, dl, VT, Bits);
6617 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0));
6618 return DAG.getNode(ISD::CTLZ, dl, VT, rbit);
6621 static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG,
6631 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6633 SDValue Res = DAG.getBitcast(VT8Bit, N->getOperand(0));
6634 Res = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Res);
6641 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddlu, DL,
6642 TLI.getPointerTy(DAG.getDataLayout())));
6648 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, WidenVT, Ops);
6707 static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
6724 return DAG.getNode(ARMISD::VSHLIMM, dl, VT, N->getOperand(0),
6725 DAG.getConstant(Cnt, dl, MVT::i32));
6726 return DAG.getNode(ARMISD::VSHLu, dl, VT, N->getOperand(0),
6736 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0),
6737 DAG.getConstant(Cnt, dl, MVT::i32));
6743 SDValue NegatedCount = DAG.getNode(
6744 ISD::SUB, dl, ShiftVT, getZeroVector(ShiftVT, DAG, dl), N->getOperand(1));
6747 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0), NegatedCount);
6750 static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG,
6777 ShAmt = DAG.getZExtOrTrunc(ShAmt, dl, MVT::i32);
6783 ShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
6784 DAG.getConstant(0, dl, MVT::i32), ShAmt);
6794 DAG.SplitScalar(N->getOperand(0), dl, MVT::i32, MVT::i32);
6796 Lo = DAG.getNode(ShPartsOpc, dl, DAG.getVTList(MVT::i32, MVT::i32), Lo, Hi,
6800 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
6813 std::tie(Lo, Hi) = DAG.SplitScalar(N->getOperand(0), dl, MVT::i32, MVT::i32);
6818 Hi = DAG.getNode(Opc, dl, DAG.getVTList(MVT::i32, FlagsVT), Hi);
6821 Lo = DAG.getNode(ARMISD::RRX, dl, MVT::i32, Lo, Hi.getValue(1));
6824 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
6827 static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG,
6863 EVT SplitVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, CmpElements);
6864 SDValue CastOp0 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op0);
6865 SDValue CastOp1 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op1);
6866 SDValue Cmp = DAG.getNode(ISD::SETCC, dl, SplitVT, CastOp0, CastOp1,
6867 DAG.getCondCode(ISD::SETEQ));
6868 SDValue Reversed = DAG.getNode(ARMISD::VREV64, dl, SplitVT, Cmp);
6869 SDValue Merged = DAG.getNode(ISD::AND, dl, SplitVT, Cmp, Reversed);
6870 Merged = DAG.getNode(ISD::BITCAST, dl, CmpVT, Merged);
6872 Merged = DAG.getNOT(dl, Merged, CmpVT);
6873 Merged = DAG.getSExtOrTrunc(Merged, dl, VT);
6908 SDValue TmpOp0 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op1, Op0,
6909 DAG.getConstant(ARMCC::GT, dl, MVT::i32));
6910 SDValue TmpOp1 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6911 DAG.getConstant(ARMCC::GT, dl, MVT::i32));
6912 SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1);
6914 Result = DAG.getNOT(dl, Result, VT);
6920 SDValue TmpOp0 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op1, Op0,
6921 DAG.getConstant(ARMCC::GT, dl, MVT::i32));
6922 SDValue TmpOp1 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6923 DAG.getConstant(ARMCC::GE, dl, MVT::i32));
6924 SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1);
6926 Result = DAG.getNOT(dl, Result, VT);
6964 Op0 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(0));
6965 Op1 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(1));
6966 SDValue Result = DAG.getNode(ARMISD::VTST, dl, CmpVT, Op0, Op1);
6968 Result = DAG.getNOT(dl, Result, VT);
6993 Result = DAG.getNode(ARMISD::VCMPZ, dl, CmpVT, Op0,
6994 DAG.getConstant(Opc, dl, MVT::i32));
6996 Result = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6997 DAG.getConstant(Opc, dl, MVT::i32));
6999 Result = DAG.getSExtOrTrunc(Result, dl, VT);
7002 Result = DAG.getNOT(dl, Result, VT);
7007 static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) {
7018 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
7019 DAG.getConstant(1, DL, MVT::i32), Carry);
7021 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG);
7023 SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
7024 SDValue Cmp = DAG.getNode(ARMISD::SUBE, DL, VTs, LHS, RHS, Carry);
7026 SDValue FVal = DAG.getConstant(0, DL, MVT::i32);
7027 SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
7028 SDValue ARMcc = DAG.getConstant(
7030 return DAG.getNode(ARMISD::CMOV, DL, Op.getValueType(), FVal, TVal, ARMcc,
7038 unsigned SplatBitSize, SelectionDAG &DAG,
7169 return DAG.getTargetConstant(EncodedVal, dl, MVT::i32);
7172 SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
7197 SDValue Lo = DAG.getConstant(INTVal.trunc(32), DL, MVT::i32);
7198 SDValue Hi = DAG.getConstant(INTVal.lshr(32).trunc(32), DL, MVT::i32);
7199 return DAG.getNode(ARMISD::VMOVDRR, DL, MVT::f64, Lo, Hi);
7202 return DAG.getNode(ARMISD::VMOVSR, DL, VT,
7203 DAG.getConstant(INTVal, DL, MVT::i32));
7228 SDValue NewVal = DAG.getTargetConstant(ImmVal, DL, MVT::i32);
7229 SDValue VecConstant = DAG.getNode(ARMISD::VMOVFPIMM, DL, MVT::v2f32,
7231 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecConstant,
7232 DAG.getConstant(0, DL, MVT::i32));
7250 SDValue NewVal = isVMOVModifiedImm(iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op),
7254 SDValue VecConstant = DAG.getNode(ARMISD::VMOVIMM, DL, VMovVT,
7257 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant);
7260 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
7262 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
7263 DAG.getConstant(0, DL, MVT::i32));
7267 NewVal = isVMOVModifiedImm(~iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op), VMovVT,
7271 SDValue VecConstant = DAG.getNode(ARMISD::VMVNIMM, DL, VMovVT, NewVal);
7274 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant);
7277 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
7279 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
7280 DAG.getConstant(0, DL, MVT::i32));
7698 static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG,
7740 SDValue N1 = DAG.getNode(ARMISD::VCVTN, dl, VT, DAG.getUNDEF(VT), Op0,
7741 DAG.getConstant(0, dl, MVT::i32));
7742 return DAG.getNode(ARMISD::VCVTN, dl, VT, N1, Op1,
7743 DAG.getConstant(1, dl, MVT::i32));
7751 static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG,
7786 return DAG.getNode(ARMISD::VCVTL, dl, VT, Op0,
7787 DAG.getConstant(Offset, dl, MVT::i32));
7793 static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG,
7802 return DAG.getConstant(Val, dl, MVT::i32);
7805 return DAG.getConstant(Val, dl, MVT::i32);
7810 static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG,
7842 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32, FirstOp,
7843 DAG.getValueType(MVT::i1));
7844 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, Op.getValueType(), Ext);
7859 SDValue Base = DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT,
7860 DAG.getConstant(Bits32, dl, MVT::i32));
7865 Base = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Base, V,
7866 DAG.getConstant(i, dl, MVT::i32));
7872 static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG,
7901 return DAG.getNode(ARMISD::VIDUP, DL, DAG.getVTList(VT, MVT::i32), Op0,
7902 DAG.getConstant(N, DL, MVT::i32));
7949 SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
7956 return LowerBUILD_VECTOR_i1(Op, DAG, ST);
7958 if (SDValue R = LowerBUILD_VECTORToVIDUP(Op, DAG, ST))
7966 return DAG.getUNDEF(VT);
7977 SDValue Const = DAG.getConstant(SplatBits.getZExtValue(), dl, MVT::i32);
7978 SDValue VDup = DAG.getNode(ARMISD::VDUP, dl, DupVT, Const);
7979 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup);
7988 SplatBitSize, DAG, dl, VmovVT, VT, VMOVModImm);
7991 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, Val);
7992 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Vmov);
7998 NegatedImm, SplatUndef.getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
8001 SDValue Vmov = DAG.getNode(ARMISD::VMVNIMM, dl, VmovVT, Val);
8002 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Vmov);
8009 SDValue Val = DAG.getTargetConstant(ImmVal, dl, MVT::i32);
8010 return DAG.getNode(ARMISD::VMOVFPIMM, dl, VT, Val);
8021 SDValue Const = DAG.getConstant(SplatBits.getZExtValue(), dl, MVT::i32);
8022 SDValue VDup = DAG.getNode(ARMISD::VDUP, dl, DupVT, Const);
8023 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup);
8068 return DAG.getUNDEF(VT);
8073 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value);
8098 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
8099 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DAG.getUNDEF(VT),
8100 Value, DAG.getConstant(index, dl, MVT::i32)),
8101 DAG.getConstant(index, dl, MVT::i32));
8103 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
8106 N = DAG.getNode(ARMISD::VDUP, dl, VT, Value);
8117 Ops.push_back(DAG.getConstant(I, dl, MVT::i32));
8118 N = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ops);
8129 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, IVT,
8131 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), IVT, NumElts);
8132 SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
8133 Val = LowerBUILD_VECTOR(Val, DAG, ST);
8135 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
8138 SDValue Val = IsSingleInstrConstant(Value, DAG, ST, dl);
8140 return DAG.getNode(ARMISD::VDUP, dl, VT, Val);
8154 if (SDValue shuffle = ReconstructShuffle(Op, DAG))
8159 if (SDValue VCVT = LowerBuildVectorOfFPTrunc(Op, DAG, Subtarget))
8161 if (SDValue VCVT = LowerBuildVectorOfFPExt(Op, DAG, Subtarget))
8169 EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElts / 2);
8170 SDValue Lower = DAG.getBuildVector(HVT, dl, ArrayRef(&Ops[0], NumElts / 2));
8172 Lower = LowerBUILD_VECTOR(Lower, DAG, ST);
8174 DAG.getBuildVector(HVT, dl, ArrayRef(&Ops[NumElts / 2], NumElts / 2));
8176 Upper = LowerBUILD_VECTOR(Upper, DAG, ST);
8178 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lower, Upper);
8188 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
8191 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, EltVT, Op.getOperand(i)));
8192 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
8193 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
8203 SDValue Vec = DAG.getUNDEF(VT);
8208 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i32);
8209 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
8220 SelectionDAG &DAG) const {
8291 EVT ShuffleVT = EVT::getVectorVT(*DAG.getContext(), SmallestEltTy, NumElts);
8308 EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumSrcElts);
8316 DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec,
8317 DAG.getUNDEF(Src.ShuffleVec.getValueType()));
8332 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8333 DAG.getConstant(NumSrcElts, dl, MVT::i32));
8338 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8339 DAG.getConstant(0, dl, MVT::i32));
8343 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8344 DAG.getConstant(0, dl, MVT::i32));
8346 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
8347 DAG.getConstant(NumSrcElts, dl, MVT::i32));
8349 Src.ShuffleVec = DAG.getNode(ARMISD::VEXT, dl, DestVT, VEXTSrc1,
8351 DAG.getConstant(Src.MinElt, dl, MVT::i32));
8364 Src.ShuffleVec = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, ShuffleVT, Src.ShuffleVec);
8409 SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
8414 ShuffleOps[1], Mask, DAG);
8417 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Shuffle);
8512 SDValue RHS, SelectionDAG &DAG,
8525 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
8526 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
8534 return DAG.getNode(ARMISD::VREV64, dl, VT, OpLHS);
8537 return DAG.getNode(ARMISD::VREV32, dl, VT, OpLHS);
8540 return DAG.getNode(ARMISD::VREV16, dl, VT, OpLHS);
8545 return DAG.getNode(ARMISD::VDUPLANE, dl, VT,
8546 OpLHS, DAG.getConstant(OpNum-OP_VDUP0, dl, MVT::i32));
8550 return DAG.getNode(ARMISD::VEXT, dl, VT,
8552 DAG.getConstant(OpNum - OP_VEXT1 + 1, dl, MVT::i32));
8555 return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT),
8559 return DAG.getNode(ARMISD::VZIP, dl, DAG.getVTList(VT, VT),
8563 return DAG.getNode(ARMISD::VTRN, dl, DAG.getVTList(VT, VT),
8570 SelectionDAG &DAG) {
8578 VTBLMask.push_back(DAG.getSignedConstant(I, DL, MVT::i32));
8581 return DAG.getNode(ARMISD::VTBL1, DL, MVT::v8i8, V1,
8582 DAG.getBuildVector(MVT::v8i8, DL, VTBLMask));
8584 return DAG.getNode(ARMISD::VTBL2, DL, MVT::v8i8, V1, V2,
8585 DAG.getBuildVector(MVT::v8i8, DL, VTBLMask));
8588 static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
8594 SDValue OpLHS = DAG.getNode(ARMISD::VREV64, DL, VT, Op.getOperand(0));
8604 return DAG.getVectorShuffle(VT, DL, OpLHS, OpLHS, NewMask);
8623 SelectionDAG &DAG) {
8628 DAG.getTargetConstant(ARM_AM::createVMOVModImm(0xe, 0xff), dl, MVT::i32);
8629 AllOnes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v16i8, AllOnes);
8632 DAG.getTargetConstant(ARM_AM::createVMOVModImm(0xe, 0x0), dl, MVT::i32);
8633 AllZeroes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v16i8, AllZeroes);
8644 RecastV1 = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v16i1, Pred);
8650 DAG.getNode(ISD::VSELECT, dl, MVT::v16i8, RecastV1, AllOnes, AllZeroes);
8654 return DAG.getNode(ISD::BITCAST, dl, NewVT, PredAsVector);
8657 static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG,
8670 SDValue cast = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, V1);
8671 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, cast);
8672 SDValue srl = DAG.getNode(ISD::SRL, dl, MVT::i32, rbit,
8673 DAG.getConstant(16, dl, MVT::i32));
8674 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, srl);
8685 SDValue PredAsVector1 = PromoteMVEPredVector(dl, V1, VT, DAG);
8687 SDValue PredAsVector2 = V2.isUndef() ? DAG.getUNDEF(NewVT)
8688 : PromoteMVEPredVector(dl, V2, VT, DAG);
8693 SDValue Shuffled = DAG.getVectorShuffle(NewVT, dl, PredAsVector1,
8700 SDValue BC = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Shuffled);
8701 SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, BC,
8702 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
8703 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp);
8705 return DAG.getNode(ARMISD::VCMPZ, dl, VT, Shuffled,
8706 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
8711 SelectionDAG &DAG) {
8762 SDValue BitCast = DAG.getBitcast(MVT::v4f32, Input);
8763 Parts[Part] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, BitCast,
8764 DAG.getConstant(Elt, dl, MVT::i32));
8780 SDValue NewShuffle = DAG.getVectorShuffle(
8782 SDValue BitCast = DAG.getBitcast(MVT::v4f32, NewShuffle);
8786 Parts[Part] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32,
8787 BitCast, DAG.getConstant(Part, dl, MVT::i32));
8791 SDValue NewVec = DAG.getNode(ARMISD::BUILD_VECTOR, dl, MVT::v4f32, Parts);
8792 return DAG.getBitcast(VT, NewVec);
8797 SelectionDAG &DAG) {
8838 SDValue Elt = DAG.getNode(
8841 DAG.getVectorIdxConstant(ShuffleMask[OffElement] % NumElts, dl));
8842 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, VInput, Elt,
8843 DAG.getVectorIdxConstant(OffElement % NumElts, dl));
8846 static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
8856 return LowerVECTOR_SHUFFLE_i1(Op, DAG, ST);
8859 // DAG nodes, instead of keeping them as shuffles and matching them again
8874 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0));
8888 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0));
8890 return DAG.getNode(ARMISD::VDUPLANE, dl, VT, V1,
8891 DAG.getConstant(Lane, dl, MVT::i32));
8899 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V2,
8900 DAG.getConstant(Imm, dl, MVT::i32));
8904 return DAG.getNode(ARMISD::VREV64, dl, VT, V1);
8906 return DAG.getNode(ARMISD::VREV32, dl, VT, V1);
8908 return DAG.getNode(ARMISD::VREV16, dl, VT, V1);
8911 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V1,
8912 DAG.getConstant(Imm, dl, MVT::i32));
8918 // these operations, DAG memoization will ensure that a single node is
8927 return DAG.getNode(ShuffleOpc, dl, DAG.getVTList(VT, VT), V1, V2)
8933 return DAG.getNode(ARMISD::VMOVN, dl, VT, V2, V1,
8934 DAG.getConstant(0, dl, MVT::i32));
8936 return DAG.getNode(ARMISD::VMOVN, dl, VT, V1, V2,
8937 DAG.getConstant(1, dl, MVT::i32));
8939 return DAG.getNode(ARMISD::VMOVN, dl, VT, V1, V1,
8940 DAG.getConstant(1, dl, MVT::i32));
8974 SDValue Res = DAG.getNode(ShuffleOpc, dl, DAG.getVTList(SubVT, SubVT),
8976 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Res.getValue(0),
8983 if (SDValue V = LowerVECTOR_SHUFFLEUsingOneOff(Op, ShuffleMask, DAG))
8991 SDValue Lo = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, FromVT, V1);
8992 SDValue Hi = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, FromVT,
8995 SDValue Amt = DAG.getConstant(EltSize, dl, FromVT);
8996 Lo = DAG.getNode(ISD::SRL, dl, FromVT, Lo, Amt);
8997 Hi = DAG.getNode(ISD::SRL, dl, FromVT, Hi, Amt);
8999 return DAG.getNode(ARMISD::MVETRUNC, dl, VT, Lo, Hi);
9025 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
9032 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
9042 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
9043 V1 = DAG.getNode(ISD::BITCAST, dl, VecVT, V1);
9044 V2 = DAG.getNode(ISD::BITCAST, dl, VecVT, V2);
9048 Ops.push_back(DAG.getUNDEF(EltVT));
9050 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
9052 DAG.getConstant(ShuffleMask[i] & (NumElts-1),
9055 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
9056 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
9061 return LowerReverse_VECTOR_SHUFFLE(Op, DAG);
9064 if (SDValue NewOp = LowerVECTOR_SHUFFLEv8i8(Op, ShuffleMask, DAG))
9068 if (SDValue NewOp = LowerVECTOR_SHUFFLEUsingMovs(Op, ShuffleMask, DAG))
9074 static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
9083 DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Op->getOperand(0));
9088 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32,
9089 Op.getOperand(1), DAG.getValueType(MVT::i1));
9090 SDValue BFI = DAG.getNode(ARMISD::BFI, dl, MVT::i32, Conv, Ext,
9091 DAG.getConstant(~Mask, dl, MVT::i32));
9092 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, Op.getValueType(), BFI);
9096 SelectionDAG &DAG) const {
9107 return LowerINSERT_VECTOR_ELT_i1(Op, DAG, Subtarget);
9109 if (getTypeAction(*DAG.getContext(), EltVT) ==
9119 assert(getTypeAction(*DAG.getContext(), IEltVT) !=
9124 EVT IVecVT = EVT::getVectorVT(*DAG.getContext(), IEltVT,
9127 SDValue IElt = DAG.getNode(ISD::BITCAST, dl, IEltVT, Elt);
9128 SDValue IVecIn = DAG.getNode(ISD::BITCAST, dl, IVecVT, VecIn);
9129 SDValue IVecOut = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVecVT,
9131 return DAG.getNode(ISD::BITCAST, dl, VecVT, IVecOut);
9137 static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
9146 DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Op->getOperand(0));
9150 SDValue Shift = DAG.getNode(ISD::SRL, dl, MVT::i32, Conv,
9151 DAG.getConstant(Lane * LaneWidth, dl, MVT::i32));
9155 static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG,
9166 return LowerEXTRACT_VECTOR_ELT_i1(Op, DAG, ST);
9170 return DAG.getNode(ARMISD::VGETLANEu, dl, MVT::i32, Vec, Lane);
9176 static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG,
9192 EVT VT = Op1VT.getDoubleNumVectorElementsVT(*DAG.getContext());
9194 SDValue NewV1 = PromoteMVEPredVector(dl, V1, Op1VT, DAG);
9195 SDValue NewV2 = PromoteMVEPredVector(dl, V2, Op2VT, DAG);
9208 DAG.getNode(ARMISD::MVETRUNC, dl, ConcatVT, NewV1, NewV2);
9209 return DAG.getNode(ARMISD::VCMPZ, dl, VT, ConVec,
9210 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
9218 auto ExtractInto = [&DAG, &dl](SDValue NewV, SDValue ConVec, unsigned &j) {
9223 NewV = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, NewV);
9227 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV,
9228 DAG.getIntPtrConstant(i * ExtScale, dl));
9229 ConVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ConcatVT, ConVec, Elt,
9230 DAG.getConstant(j, dl, MVT::i32));
9235 SDValue ConVec = DAG.getNode(ISD::UNDEF, dl, ConcatVT);
9241 return DAG.getNode(ARMISD::VCMPZ, dl, VT, ConVec,
9242 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
9258 static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
9262 return LowerCONCAT_VECTORS_i1(Op, DAG, ST);
9269 SDValue Val = DAG.getUNDEF(MVT::v2f64);
9273 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
9274 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op0),
9275 DAG.getIntPtrConstant(0, dl));
9277 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
9278 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op1),
9279 DAG.getIntPtrConstant(1, dl));
9280 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Val);
9283 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG,
9298 SDValue NewV1 = PromoteMVEPredVector(dl, V1, Op1VT, DAG);
9307 SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT);
9309 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV1,
9310 DAG.getIntPtrConstant(i, dl));
9311 SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt,
9312 DAG.getConstant(j, dl, MVT::i32));
9313 SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt,
9314 DAG.getConstant(j + 1, dl, MVT::i32));
9316 SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, SubVec,
9317 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
9318 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp);
9322 SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT);
9324 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV1,
9325 DAG.getIntPtrConstant(i, dl));
9326 SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt,
9327 DAG.getConstant(j, dl, MVT::i32));
9332 return DAG.getNode(ARMISD::VCMPZ, dl, VT, SubVec,
9333 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
9337 static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG,
9348 DAG.getNode(ISD::AND, DL, FromVT, Op, DAG.getConstant(1, DL, FromVT));
9349 return DAG.getNode(ISD::SETCC, DL, VT, And, DAG.getConstant(0, DL, FromVT),
9350 DAG.getCondCode(ISD::SETNE));
9353 static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG,
9360 return LowerTruncatei1(N, DAG, Subtarget);
9408 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
9410 return DAG.getNode(ARMISD::MVETRUNC, DL, ToVT, Lo, Hi);
9413 static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG,
9429 EVT ExtVT = ToVT.getHalfNumVectorElementsVT(*DAG.getContext());
9435 SDValue Ext = DAG.getNode(Opcode, DL, DAG.getVTList(ExtVT, ExtVT), Op);
9439 Ext = DAG.getNode(N->getOpcode(), DL, MVT::v8i32, Ext);
9440 Ext1 = DAG.getNode(N->getOpcode(), DL, MVT::v8i32, Ext1);
9443 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ToVT, Ext, Ext1);
9449 static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
9458 unsigned LoElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9502 static bool isSignExtended(SDNode *N, SelectionDAG &DAG) {
9505 if (isExtendedBUILD_VECTOR(N, DAG, true))
9512 static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) {
9516 if (isExtendedBUILD_VECTOR(N, DAG, false))
9541 static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG,
9555 return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N);
9563 static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG& DAG) {
9568 return DAG.getLoad(LD->getMemoryVT(), SDLoc(LD), LD->getChain(),
9575 return DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD), ExtendedTy,
9587 static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) {
9590 return AddRequiredExtensionForVMULL(N->getOperand(0), DAG,
9599 SDValue newLoad = SkipLoadExtensionForVMULL(LD, DAG);
9600 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), newLoad.getValue(1));
9603 DAG.getNode(Opcode, SDLoc(newLoad), LD->getValueType(0), newLoad);
9604 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 0), extLoad);
9615 unsigned LowElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9616 return DAG.getBuildVector(
9632 Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32));
9634 return DAG.getBuildVector(MVT::getVectorVT(TruncVT, NumElts), dl, Ops);
9637 static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) {
9643 isSignExtended(N0, DAG) && isSignExtended(N1, DAG);
9648 static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) {
9654 isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG);
9659 static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
9669 bool isN0SExt = isSignExtended(N0, DAG);
9670 bool isN1SExt = isSignExtended(N1, DAG);
9674 bool isN0ZExt = isZeroExtended(N0, DAG);
9675 bool isN1ZExt = isZeroExtended(N1, DAG);
9681 if (isN1SExt && isAddSubSExt(N0, DAG)) {
9684 } else if (isN1ZExt && isAddSubZExt(N0, DAG)) {
9687 } else if (isN0ZExt && isAddSubZExt(N1, DAG)) {
9707 SDValue Op1 = SkipExtensionForVMULL(N1, DAG);
9709 Op0 = SkipExtensionForVMULL(N0, DAG);
9713 return DAG.getNode(NewOpc, DL, VT, Op0, Op1);
9724 SDValue N00 = SkipExtensionForVMULL(N0->getOperand(0).getNode(), DAG);
9725 SDValue N01 = SkipExtensionForVMULL(N0->getOperand(1).getNode(), DAG);
9727 return DAG.getNode(N0->getOpcode(), DL, VT,
9728 DAG.getNode(NewOpc, DL, VT,
9729 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1),
9730 DAG.getNode(NewOpc, DL, VT,
9731 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1));
9735 SelectionDAG &DAG) {
9741 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, X);
9742 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Y);
9743 X = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, X);
9744 Y = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, Y);
9747 Y = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9748 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9754 X = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, X, Y);
9755 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, X);
9756 Y = DAG.getConstant(0xb000, dl, MVT::v4i32);
9757 X = DAG.getNode(ISD::ADD, dl, MVT::v4i32, X, Y);
9758 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, X);
9760 X = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, X);
9761 X = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, X);
9766 SelectionDAG &DAG) {
9773 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N0);
9774 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N1);
9775 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0);
9776 N1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1);
9781 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9782 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9784 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9785 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9787 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
9792 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2);
9793 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
9794 N1 = DAG.getConstant(0x89, dl, MVT::v4i32);
9795 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1);
9796 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
9799 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0);
9800 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0);
9804 static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG,
9816 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N0);
9817 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N1);
9819 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9820 DAG.getIntPtrConstant(4, dl));
9821 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9822 DAG.getIntPtrConstant(4, dl));
9823 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9824 DAG.getIntPtrConstant(0, dl));
9825 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9826 DAG.getIntPtrConstant(0, dl));
9828 N0 = LowerSDIV_v4i8(N0, N1, dl, DAG); // v4i16
9829 N2 = LowerSDIV_v4i8(N2, N3, dl, DAG); // v4i16
9831 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2);
9832 N0 = LowerCONCAT_VECTORS(N0, DAG, ST);
9834 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v8i8, N0);
9837 return LowerSDIV_v4i16(N0, N1, dl, DAG);
9840 static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG,
9853 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N0);
9854 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N1);
9856 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9857 DAG.getIntPtrConstant(4, dl));
9858 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9859 DAG.getIntPtrConstant(4, dl));
9860 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9861 DAG.getIntPtrConstant(0, dl));
9862 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9863 DAG.getIntPtrConstant(0, dl));
9865 N0 = LowerSDIV_v4i16(N0, N1, dl, DAG); // v4i16
9866 N2 = LowerSDIV_v4i16(N2, N3, dl, DAG); // v4i16
9868 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2);
9869 N0 = LowerCONCAT_VECTORS(N0, DAG, ST);
9871 N0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v8i8,
9872 DAG.getConstant(Intrinsic::arm_neon_vqmovnsu, dl,
9881 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N0);
9882 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N1);
9883 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0);
9884 SDValue BN1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1);
9890 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9891 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9893 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9894 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9896 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
9897 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9898 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9900 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
9905 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2);
9906 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
9907 N1 = DAG.getConstant(2, dl, MVT::v4i32);
9908 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1);
9909 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
9912 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0);
9913 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0);
9917 static SDValue LowerUADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) {
9920 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
9929 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG);
9932 Result = DAG.getNode(ARMISD::ADDE, DL, VTs, Op.getOperand(0),
9936 Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG);
9940 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
9941 DAG.getConstant(1, DL, MVT::i32), Carry);
9943 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG);
9946 Result = DAG.getNode(ARMISD::SUBE, DL, VTs, Op.getOperand(0),
9950 Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG);
9953 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
9954 DAG.getConstant(1, DL, MVT::i32), Carry);
9958 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Carry);
9961 SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
9969 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
9970 auto PtrVT = getPointerTy(DAG.getDataLayout());
9972 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
9973 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9977 auto &DL = DAG.getDataLayout();
9987 SRet = DAG.getFrameIndex(FrameIdx, TLI.getPointerTy(DL));
9996 RetTy = Type::getVoidTy(*DAG.getContext());
10010 SDValue Callee = DAG.getExternalSymbol(LibcallName, getPointerTy(DL));
10012 TargetLowering::CallLoweringInfo CLI(DAG);
10014 .setChain(DAG.getEntryNode())
10023 DAG.getLoad(ArgVT, dl, CallResult.second, SRet, MachinePointerInfo());
10026 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, SRet,
10027 DAG.getIntPtrConstant(ArgVT.getStoreSize(), dl));
10029 DAG.getLoad(ArgVT, dl, LoadSin.getValue(1), Add, MachinePointerInfo());
10031 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
10032 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys,
10036 SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG,
10044 const auto &DL = DAG.getDataLayout();
10045 const auto &TLI = DAG.getTargetLoweringInfo();
10053 SDValue ES = DAG.getExternalSymbol(Name, TLI.getPointerTy(DL));
10060 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.getContext());
10064 CallLoweringInfo CLI(DAG);
10067 .setCallee(CallingConv::ARM_AAPCS_VFP, VT.getTypeForEVT(*DAG.getContext()),
10079 SelectionDAG &DAG,
10085 const auto &ST = DAG.getSubtarget<ARMSubtarget>();
10116 SDValue ARMTargetLowering::LowerDIV_Windows(SDValue Op, SelectionDAG &DAG,
10122 SDValue DBZCHK = DAG.getNode(ARMISD::WIN__DBZCHK, dl, MVT::Other,
10123 DAG.getEntryNode(), Op.getOperand(1));
10125 return LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK);
10128 static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain) {
10132 return DAG.getNode(ARMISD::WIN__DBZCHK, DL, MVT::Other, InChain, Op);
10134 std::tie(Lo, Hi) = DAG.SplitScalar(Op, DL, MVT::i32, MVT::i32);
10135 return DAG.getNode(ARMISD::WIN__DBZCHK, DL, MVT::Other, InChain,
10136 DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi));
10140 SDValue Op, SelectionDAG &DAG, bool Signed,
10142 const auto &DL = DAG.getDataLayout();
10143 const auto &TLI = DAG.getTargetLoweringInfo();
10149 SDValue DBZCHK = WinDBZCheckDenominator(DAG, Op.getNode(), DAG.getEntryNode());
10151 SDValue Result = LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK);
10153 SDValue Lower = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Result);
10154 SDValue Upper = DAG.getNode(ISD::SRL, dl, MVT::i64, Result,
10155 DAG.getConstant(32, dl, TLI.getPointerTy(DL)));
10156 Upper = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Upper);
10158 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lower, Upper));
10161 static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG) {
10183 SDValue Load = DAG.getExtLoad(
10185 EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()),
10188 if (DAG.getDataLayout().isBigEndian())
10189 Val = DAG.getNode(ISD::SRL, dl, MVT::i32,
10190 DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, Load),
10191 DAG.getConstant(32 - MemVT.getSizeInBits(), dl, MVT::i32));
10192 SDValue Pred = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v16i1, Val);
10194 Pred = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MemVT, Pred,
10195 DAG.getConstant(0, dl, MVT::i32));
10196 return DAG.getMergeValues({Pred, Load.getValue(1)}, dl);
10200 SelectionDAG &DAG) const {
10209 SDValue Result = DAG.getMemIntrinsicNode(
10210 ARMISD::LDRD, dl, DAG.getVTList({MVT::i32, MVT::i32, MVT::Other}),
10212 SDValue Lo = Result.getValue(DAG.getDataLayout().isLittleEndian() ? 0 : 1);
10213 SDValue Hi = Result.getValue(DAG.getDataLayout().isLittleEndian() ? 1 : 0);
10214 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
10219 static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) {
10236 unsigned Elt = DAG.getDataLayout().isBigEndian()
10239 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, Build,
10240 DAG.getConstant(Elt, dl, MVT::i32)));
10243 Ops.push_back(DAG.getUNDEF(MVT::i32));
10244 Build = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i1, Ops);
10246 SDValue GRP = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Build);
10247 if (MemVT == MVT::v16i1 && DAG.getDataLayout().isBigEndian())
10248 GRP = DAG.getNode(ISD::SRL, dl, MVT::i32,
10249 DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, GRP),
10250 DAG.getConstant(16, dl, MVT::i32));
10251 return DAG.getTruncStore(
10253 EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()),
10257 static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG,
10269 SDValue Lo = DAG.getNode(
10271 DAG.getTargetConstant(DAG.getDataLayout().isLittleEndian() ? 0 : 1, dl,
10273 SDValue Hi = DAG.getNode(
10275 DAG.getTargetConstant(DAG.getDataLayout().isLittleEndian() ? 1 : 0, dl,
10278 return DAG.getMemIntrinsicNode(ARMISD::STRD, dl, DAG.getVTList(MVT::Other),
10284 return LowerPredicateStore(Op, DAG);
10296 static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) {
10308 SDValue ZeroVec = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
10309 DAG.getTargetConstant(0, dl, MVT::i32));
10310 SDValue NewLoad = DAG.getMaskedLoad(
10319 Combo = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
10320 return DAG.getMergeValues({Combo, NewLoad.getValue(1)}, dl);
10323 static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG,
10356 SDValue Rev = DAG.getNode(RevOpcode, dl, VT, Op0);
10357 Op0 = DAG.getNode(BaseOpcode, dl, VT, Op0, Rev);
10364 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10365 DAG.getConstant(0 * NumElts / 4, dl, MVT::i32));
10366 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10367 DAG.getConstant(1 * NumElts / 4, dl, MVT::i32));
10368 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10369 DAG.getConstant(2 * NumElts / 4, dl, MVT::i32));
10370 SDValue Ext3 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10371 DAG.getConstant(3 * NumElts / 4, dl, MVT::i32));
10372 SDValue Res0 = DAG.getNode(BaseOpcode, dl, EltVT, Ext0, Ext1, Op->getFlags());
10373 SDValue Res1 = DAG.getNode(BaseOpcode, dl, EltVT, Ext2, Ext3, Op->getFlags());
10374 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res0, Res1, Op->getFlags());
10376 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10377 DAG.getConstant(0, dl, MVT::i32));
10378 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10379 DAG.getConstant(1, dl, MVT::i32));
10380 Res = DAG.getNode(BaseOpcode, dl, EltVT, Ext0, Ext1, Op->getFlags());
10385 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Op->getValueType(0), Res);
10389 static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG,
10393 return LowerVecReduce(Op, DAG, ST);
10396 static SDValue LowerVecReduceMinMax(SDValue Op, SelectionDAG &DAG,
10423 SDValue PairwiseOp = DAG.getConstant(PairwiseIntrinsic, dl, MVT::i32);
10435 std::tie(Lo, Hi) = DAG.SplitVector(Op0, dl);
10437 Op0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, {PairwiseOp, Lo, Hi});
10443 Op0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, {PairwiseOp, Op0, Op0});
10447 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
10448 DAG.getConstant(0, dl, MVT::i32));
10465 Res = DAG.getNode(Extend, dl, Op.getValueType(), Res);
10470 static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) {
10482 SelectionDAG &DAG,
10488 DAG.getTargetConstant(Intrinsic::arm_mrc, DL, MVT::i32),
10489 DAG.getTargetConstant(15, DL, MVT::i32),
10490 DAG.getTargetConstant(0, DL, MVT::i32),
10491 DAG.getTargetConstant(9, DL, MVT::i32),
10492 DAG.getTargetConstant(13, DL, MVT::i32),
10493 DAG.getTargetConstant(0, DL, MVT::i32)
10496 SDValue Cycles32 = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
10497 DAG.getVTList(MVT::i32, MVT::Other), Ops);
10498 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Cycles32,
10499 DAG.getConstant(0, DL, MVT::i32)));
10503 static SDValue createGPRPairNode2xi32(SelectionDAG &DAG, SDValue V0,
10507 DAG.getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
10508 SDValue SubReg0 = DAG.getTargetConstant(ARM::gsub_0, dl, MVT::i32);
10509 SDValue SubReg1 = DAG.getTargetConstant(ARM::gsub_1, dl, MVT::i32);
10512 DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
10515 static SDValue createGPRPairNodei64(SelectionDAG &DAG, SDValue V) {
10517 auto [VLo, VHi] = DAG.SplitScalar(V, dl, MVT::i32, MVT::i32);
10518 bool isBigEndian = DAG.getDataLayout().isBigEndian();
10521 return createGPRPairNode2xi32(DAG, VLo, VHi);
10526 SelectionDAG &DAG) {
10530 createGPRPairNode2xi32(DAG, N->getOperand(1),
10531 DAG.getUNDEF(MVT::i32)), // pointer, temp
10532 createGPRPairNodei64(DAG, N->getOperand(2)), // expected
10533 createGPRPairNodei64(DAG, N->getOperand(3)), // new
10536 SDNode *CmpSwap = DAG.getMachineNode(
10538 DAG.getVTList(MVT::Untyped, MVT::Untyped, MVT::Other), Ops);
10541 DAG.setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
10543 bool isBigEndian = DAG.getDataLayout().isBigEndian();
10546 DAG.getTargetExtractSubreg(isBigEndian ? ARM::gsub_1 : ARM::gsub_0,
10549 DAG.getTargetExtractSubreg(isBigEndian ? ARM::gsub_0 : ARM::gsub_1,
10551 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i64, Lo, Hi));
10555 SDValue ARMTargetLowering::LowerFSETCC(SDValue Op, SelectionDAG &DAG) const {
10567 DAG.getTargetLoweringInfo().softenSetCCOperands(
10568 DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS, Chain, IsSignaling);
10570 RHS = DAG.getConstant(0, dl, LHS.getValueType());
10573 SDValue Result = DAG.getNode(ISD::SETCC, dl, VT, LHS, RHS,
10574 DAG.getCondCode(CC));
10575 return DAG.getMergeValues({Result, Chain}, dl);
10581 SDValue True = DAG.getConstant(1, dl, VT);
10582 SDValue False = DAG.getConstant(0, dl, VT);
10583 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
10584 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10585 SDValue Result = getCMOV(dl, VT, False, True, ARMcc, Cmp, DAG);
10587 ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32);
10588 Result = getCMOV(dl, VT, Result, True, ARMcc, Cmp, DAG);
10590 return DAG.getMergeValues({Result, Chain}, dl);
10593 SDValue ARMTargetLowering::LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const {
10594 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
10596 EVT VT = getPointerTy(DAG.getDataLayout());
10599 return DAG.getFrameIndex(FI, VT);
10603 SelectionDAG &DAG) const {
10609 makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first;
10610 return DAG.getBitcast(MVT::i32, Res);
10613 SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
10617 case ISD::WRITE_REGISTER: return LowerWRITE_REGISTER(Op, DAG);
10618 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
10619 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
10620 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
10621 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
10622 case ISD::SELECT: return LowerSELECT(Op, DAG);
10623 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
10624 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
10625 case ISD::BR_CC: return LowerBR_CC(Op, DAG);
10626 case ISD::BR_JT: return LowerBR_JT(Op, DAG);
10627 case ISD::VASTART: return LowerVASTART(Op, DAG);
10628 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG, Subtarget);
10629 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG, Subtarget);
10631 case ISD::UINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
10635 case ISD::FP_TO_UINT: return LowerFP_TO_INT(Op, DAG);
10637 case ISD::FP_TO_UINT_SAT: return LowerFP_TO_INT_SAT(Op, DAG, Subtarget);
10638 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
10639 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
10640 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
10641 case ISD::EH_SJLJ_SETJMP: return LowerEH_SJLJ_SETJMP(Op, DAG);
10642 case ISD::EH_SJLJ_LONGJMP: return LowerEH_SJLJ_LONGJMP(Op, DAG);
10643 case ISD::EH_SJLJ_SETUP_DISPATCH: return LowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
10644 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG, Subtarget);
10645 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG,
10647 case ISD::BITCAST: return ExpandBITCAST(Op.getNode(), DAG, Subtarget);
10650 case ISD::SRA: return LowerShift(Op.getNode(), DAG, Subtarget);
10651 case ISD::SREM: return LowerREM(Op.getNode(), DAG);
10652 case ISD::UREM: return LowerREM(Op.getNode(), DAG);
10653 case ISD::SHL_PARTS: return LowerShiftLeftParts(Op, DAG);
10655 case ISD::SRA_PARTS: return LowerShiftRightParts(Op, DAG);
10657 case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op.getNode(), DAG, Subtarget);
10658 case ISD::CTPOP: return LowerCTPOP(Op.getNode(), DAG, Subtarget);
10659 case ISD::SETCC: return LowerVSETCC(Op, DAG, Subtarget);
10660 case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG);
10661 case ISD::ConstantFP: return LowerConstantFP(Op, DAG, Subtarget);
10662 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG, Subtarget);
10663 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
10664 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG, Subtarget);
10665 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
10666 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG, Subtarget);
10667 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG, Subtarget);
10668 case ISD::TRUNCATE: return LowerTruncate(Op.getNode(), DAG, Subtarget);
10670 case ISD::ZERO_EXTEND: return LowerVectorExtend(Op.getNode(), DAG, Subtarget);
10671 case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG);
10672 case ISD::SET_ROUNDING: return LowerSET_ROUNDING(Op, DAG);
10674 return LowerSET_FPMODE(Op, DAG);
10676 return LowerRESET_FPMODE(Op, DAG);
10677 case ISD::MUL: return LowerMUL(Op, DAG);
10680 return LowerDIV_Windows(Op, DAG, /* Signed */ true);
10681 return LowerSDIV(Op, DAG, Subtarget);
10684 return LowerDIV_Windows(Op, DAG, /* Signed */ false);
10685 return LowerUDIV(Op, DAG, Subtarget);
10688 return LowerUADDSUBO_CARRY(Op, DAG);
10691 return LowerSignedALUO(Op, DAG);
10694 return LowerUnsignedALUO(Op, DAG);
10699 return LowerADDSUBSAT(Op, DAG, Subtarget);
10701 return LowerPredicateLoad(Op, DAG);
10703 return LowerSTORE(Op, DAG, Subtarget);
10705 return LowerMLOAD(Op, DAG);
10710 return LowerVecReduce(Op, DAG, Subtarget);
10715 return LowerVecReduceF(Op, DAG, Subtarget);
10720 return LowerVecReduceMinMax(Op, DAG, Subtarget);
10722 case ISD::ATOMIC_STORE: return LowerAtomicLoadStore(Op, DAG);
10723 case ISD::FSINCOS: return LowerFSINCOS(Op, DAG);
10725 case ISD::UDIVREM: return LowerDivRem(Op, DAG);
10728 return LowerDYNAMIC_STACKALLOC(Op, DAG);
10731 case ISD::FP_ROUND: return LowerFP_ROUND(Op, DAG);
10733 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
10735 case ISD::STRICT_FSETCCS: return LowerFSETCC(Op, DAG);
10737 return LowerSPONENTRY(Op, DAG);
10739 return LowerFP_TO_BF16(Op, DAG);
10745 SelectionDAG &DAG) {
10761 std::tie(Lo, Hi) = DAG.SplitScalar(N->getOperand(3), dl, MVT::i32, MVT::i32);
10763 SDValue LongMul = DAG.getNode(Opc, dl,
10764 DAG.getVTList(MVT::i32, MVT::i32),
10767 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64,
10775 SelectionDAG &DAG) const {
10781 ExpandREAD_REGISTER(N, Results, DAG);
10784 Res = ExpandBITCAST(N, DAG, Subtarget);
10789 Res = Expand64BitShift(N, DAG, Subtarget);
10793 Res = LowerREM(N, DAG);
10797 Res = LowerDivRem(SDValue(N, 0), DAG);
10806 Res = LowerADDSUBSAT(SDValue(N, 0), DAG, Subtarget);
10809 ReplaceREADCYCLECOUNTER(N, Results, DAG, Subtarget);
10814 return ExpandDIV_Windows(SDValue(N, 0), DAG, N->getOpcode() == ISD::SDIV,
10817 ReplaceCMP_SWAP_64Results(N, Results, DAG);
10820 return ReplaceLongIntrinsic(N, Results, DAG);
10822 LowerLOAD(N, Results, DAG);
10825 Res = LowerTruncate(N, DAG, Subtarget);
10829 Res = LowerVectorExtend(N, DAG, Subtarget);
10833 Res = LowerFP_TO_INT_SAT(SDValue(N, 0), DAG, Subtarget);
12569 SelectionDAG &DAG) {
12603 OtherOp = DAG.getConstant(0, dl, VT);
12606 OtherOp = DAG.getConstant(1, dl, VT);
12608 OtherOp = DAG.getAllOnesConstant(dl, VT);
12642 SelectionDAG &DAG = DCI.DAG;
12648 NonConstantVal, DAG))
12653 SDValue FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
12659 return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
12703 SelectionDAG &DAG = DCI.DAG;
12704 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12710 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpadd, dl,
12711 TLI.getPointerTy(DAG.getDataLayout())));
12715 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops);
12743 SelectionDAG &DAG = DCI.DAG;
12744 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12755 Ops.push_back(DAG.getConstant(Opcode, dl,
12756 TLI.getPointerTy(DAG.getDataLayout())));
12759 EVT ConcatVT = EVT::getVectorVT(*DAG.getContext(), ElemTy, NumElts * 2);
12760 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), ConcatVT,
12764 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops);
12837 SelectionDAG &DAG = DCI.DAG;
12838 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12844 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddls, dl,
12845 TLI.getPointerTy(DAG.getDataLayout())));
12863 SDValue tmp = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, widenType, Ops);
12865 return DAG.getNode(ExtOp, dl, VT, tmp);
12911 SelectionDAG &DAG = DCI.DAG;
12917 if (isS16(Mul.getOperand(0), DAG) && isS16(Mul.getOperand(1), DAG)) {
12921 } else if (isS16(Mul.getOperand(0), DAG) && isSRA16(Mul.getOperand(1))) {
12925 } else if (isSRA16(Mul.getOperand(0)) && isS16(Mul.getOperand(1), DAG)) {
12938 SDValue SMLAL = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32),
12944 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), LoMLALResult);
12945 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), HiMLALResult);
13069 SelectionDAG &DAG = DCI.DAG;
13091 SDValue NewNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode), MVT::i32, Ops);
13092 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeSubeNode, 0), NewNode);
13104 SDValue MLALNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode),
13105 DAG.getVTList(MVT::i32, MVT::i32), Ops);
13109 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeSubeNode, 0), HiMLALResult);
13112 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcSubcNode, 0), LoMLALResult);
13157 SelectionDAG &DAG = DCI.DAG;
13160 SDValue UMAAL = DAG.getNode(ARMISD::UMAAL, SDLoc(AddcNode),
13161 DAG.getVTList(MVT::i32, MVT::i32), Ops);
13164 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), SDValue(UMAAL.getNode(), 1));
13165 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), SDValue(UMAAL.getNode(), 0));
13173 static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG,
13187 return DAG.getNode(ARMISD::UMAAL, SDLoc(N),
13188 DAG.getVTList(MVT::i32, MVT::i32),
13198 SelectionDAG &DAG(DCI.DAG);
13217 RHS = DAG.getConstant(-imm, DL, MVT::i32);
13220 return DAG.getNode(Opcode, DL, N->getVTList(), N->getOperand(0), RHS);
13232 SelectionDAG &DAG = DCI.DAG;
13242 RHS = DAG.getConstant(~imm, DL, MVT::i32);
13246 return DAG.getNode(Opcode, DL, N->getVTList(),
13348 LHS = DCI.DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS);
13352 DCI.DAG.getNode(Opcode, dl, MVT::i32, LHS, RHS->getOperand(0));
13356 Reduction = DCI.DAG.getNode(ISD::TRUNCATE, dl, VectorScalarType, Reduction);
13367 static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG) {
13446 DAG.getNode(ISD::ANY_EXTEND, DL, ExtVecVT, Ext0.getOperand(0));
13448 DAG.getNode(ISD::ANY_EXTEND, DL, ExtVecVT, Ext1.getOperand(0));
13449 Inp0 = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, LegalVecVT, Inp0);
13450 Inp1 = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, LegalVecVT, Inp1);
13451 SDValue VQDMULH = DAG.getNode(ARMISD::VQDMULH, DL, LegalVecVT, Inp0, Inp1);
13452 SDValue Trunc = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, ExtVecVT, VQDMULH);
13453 Trunc = DAG.getNode(ISD::TRUNCATE, DL, VecVT, Trunc);
13454 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Trunc);
13463 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LegalVecVT, Ext0.getOperand(0),
13464 DAG.getVectorIdxConstant(I * LegalLanes, DL));
13466 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LegalVecVT, Ext1.getOperand(0),
13467 DAG.getVectorIdxConstant(I * LegalLanes, DL));
13468 SDValue VQDMULH = DAG.getNode(ARMISD::VQDMULH, DL, LegalVecVT, Inp0, Inp1);
13471 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT,
13472 DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Parts));
13481 if (SDValue V = PerformVQDMULHCombine(N, DCI.DAG))
13511 return DCI.DAG.getNode(ISD::VSELECT, SDLoc(N), Type, Cond, RHS, LHS);
13524 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
13545 SDValue Op1S = DCI.DAG.getSplatValue(Op1);
13568 return DCI.DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
13569 DCI.DAG.getConstant(Opc, DL, MVT::i32),
13570 DCI.DAG.getZExtOrTrunc(Op1S, DL, MVT::i32));
13589 /// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
13614 static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG) {
13639 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, N0, N1.getOperand(0));
13640 return DAG.getNode(ISD::ADD, dl, VT, Add0, N1.getOperand(1));
13659 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, N0.getOperand(1 - N0RedOp),
13662 DAG.getNode(ISD::ADD, dl, VT, Add0, N0.getOperand(N0RedOp));
13663 return DAG.getNode(ISD::ADD, dl, VT, Add1, N1.getOperand(N1RedOp));
13697 auto BaseLocDecomp0 = BaseIndexOffset::match(Load0, DAG);
13698 auto BaseLocDecomp1 = BaseIndexOffset::match(Load1, DAG);
13738 return DAG.getNode(ISD::ADD, dl, VT, N1, N0);
13749 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, X, N1);
13750 return DAG.getNode(ISD::ADD, dl, VT, Add0, N0);
13759 static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG,
13764 if (SDValue R = TryDistrubutionADDVecReduce(N, DAG))
13795 SDValue Inp = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64,
13797 NA = DAG.getNode(ISD::ADD, dl, MVT::i64, Inp, NA);
13801 std::tie(Ops[0], Ops[1]) = DAG.SplitScalar(NA, dl, MVT::i32, MVT::i32);
13807 DAG.getNode(OpcodeA, dl, DAG.getVTList({MVT::i32, MVT::i32}), Ops);
13808 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Red,
13977 // DAG combiner will fold:
14061 SelectionDAG &DAG = DCI.DAG;
14064 SDValue BinOp = DAG.getNode(N->getOpcode(), dl, MVT::i32, X,
14065 DAG.getConstant(C1Int, dl, MVT::i32));
14067 SDValue Res = DAG.getNode(ISD::SHL, dl, MVT::i32, BinOp, SHL.getOperand(1));
14088 if (SDValue Result = PerformADDVecReduce(N, DCI.DAG, Subtarget))
14101 static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG) {
14112 return DAG.getNode(ARMISD::CSINV, SDLoc(N), MVT::i32,
14113 DAG.getNode(ISD::SUB, SDLoc(N), MVT::i32, N->getOperand(0),
14132 if (SDValue R = PerformSubCSINCCombine(N, DCI.DAG))
14153 SDValue Negate = DCI.DAG.getNode(ISD::SUB, dl, MVT::i32,
14154 DCI.DAG.getConstant(0, dl, MVT::i32),
14156 return DCI.DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0), Negate);
14180 SelectionDAG &DAG = DCI.DAG;
14200 return DAG.getNode(Opcode, DL, VT,
14201 DAG.getNode(ISD::MUL, DL, VT, N00, N1),
14202 DAG.getNode(ISD::MUL, DL, VT, N01, N1));
14205 static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG,
14254 SDValue New0a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op0);
14255 SDValue New1a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op1);
14256 return DAG.getNode(ARMISD::VMULLs, dl, VT, New0a, New1a);
14261 SDValue New0a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op0);
14262 SDValue New1a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op1);
14263 return DAG.getNode(ARMISD::VMULLu, dl, VT, New0a, New1a);
14273 SelectionDAG &DAG = DCI.DAG;
14277 return PerformMVEVMULLCombine(N, DAG, Subtarget);
14307 Res = DAG.getNode(ISD::ADD, DL, VT,
14309 DAG.getNode(ISD::SHL, DL, VT,
14311 DAG.getConstant(Log2_32(MulAmt - 1), DL,
14315 Res = DAG.getNode(ISD::SUB, DL, VT,
14316 DAG.getNode(ISD::SHL, DL, VT,
14318 DAG.getConstant(Log2_32(MulAmt + 1), DL,
14327 Res = DAG.getNode(ISD::SUB, DL, VT,
14329 DAG.getNode(ISD::SHL, DL, VT,
14331 DAG.getConstant(Log2_32(MulAmtAbs + 1), DL,
14335 Res = DAG.getNode(ISD::ADD, DL, VT,
14337 DAG.getNode(ISD::SHL, DL, VT,
14339 DAG.getConstant(Log2_32(MulAmtAbs - 1), DL,
14341 Res = DAG.getNode(ISD::SUB, DL, VT,
14342 DAG.getConstant(0, DL, MVT::i32), Res);
14348 Res = DAG.getNode(ISD::SHL, DL, VT,
14349 Res, DAG.getConstant(ShiftAmt, DL, MVT::i32));
14351 // Do not add new nodes to DAG combiner worklist.
14398 SelectionDAG &DAG = DCI.DAG;
14410 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
14411 DAG.getConstant(C3 - C2, DL, MVT::i32));
14412 return DAG.getNode(ISD::SRL, DL, MVT::i32, SHL,
14413 DAG.getConstant(C3, DL, MVT::i32));
14421 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0),
14422 DAG.getConstant(C3 - C2, DL, MVT::i32));
14423 return DAG.getNode(ISD::SHL, DL, MVT::i32, SHL,
14424 DAG.getConstant(C3, DL, MVT::i32));
14434 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
14435 DAG.getConstant(C2 + C3, DL, MVT::i32));
14436 return DAG.getNode(ISD::SRL, DL, MVT::i32, SHL,
14437 DAG.getConstant(C3, DL, MVT::i32));
14447 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0),
14448 DAG.getConstant(C2 + C3, DL, MVT::i32));
14449 return DAG.getNode(ISD::SHL, DL, MVT::i32, SHL,
14450 DAG.getConstant(C3, DL, MVT::i32));
14459 SDValue And = DAG.getNode(ISD::AND, DL, MVT::i32, N0->getOperand(0),
14460 DAG.getConstant(C1 >> C2, DL, MVT::i32));
14461 return DAG.getNode(ISD::SHL, DL, MVT::i32, And,
14462 DAG.getConstant(C2, DL, MVT::i32));
14475 SelectionDAG &DAG = DCI.DAG;
14477 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || VT == MVT::v2i1 ||
14491 DAG, dl, VbicVT, VT, OtherModImm);
14494 DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VbicVT, N->getOperand(0));
14495 SDValue Vbic = DAG.getNode(ARMISD::VBICIMM, dl, VbicVT, Input, Val);
14496 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Vbic);
14556 SelectionDAG &DAG = DCI.DAG;
14557 if (!isS16(OpS16, DAG) && !isSRA16(OpS16)) {
14564 if (isS16(OpS16, DAG))
14573 SDValue Res = DAG.getNode(Opcode, dl, MVT::i32, OpS32, OpS16);
14574 DAG.ReplaceAllUsesOfValueWith(SDValue(OR, 0), Res);
14588 SelectionDAG &DAG = DCI.DAG;
14626 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00,
14627 DAG.getConstant(Val, DL, MVT::i32),
14628 DAG.getConstant(Mask, DL, MVT::i32));
14653 Res = DAG.getNode(ISD::SRL, DL, VT, N1.getOperand(0),
14654 DAG.getConstant(amt, DL, MVT::i32));
14655 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00, Res,
14656 DAG.getConstant(Mask, DL, MVT::i32));
14670 Res = DAG.getNode(ISD::SRL, DL, VT, N00,
14671 DAG.getConstant(lsb, DL, MVT::i32));
14672 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1.getOperand(0), Res,
14673 DAG.getConstant(Mask2, DL, MVT::i32));
14681 if (DAG.MaskedValueIsZero(N1, MaskC->getAPIntValue()) &&
14692 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1, N00.getOperand(0),
14693 DAG.getConstant(~Mask, DL, MVT::i32));
14735 static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG,
14754 SDValue NewN0 = DAG.getLogicalNOT(DL, N0, VT);
14755 SDValue NewN1 = DAG.getLogicalNOT(DL, N1, VT);
14756 SDValue And = DAG.getNode(ISD::AND, DL, VT, NewN0, NewN1);
14757 return DAG.getLogicalNOT(DL, And, VT);
14768 SelectionDAG &DAG = DCI.DAG;
14770 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
14775 return PerformORCombine_i1(N, DAG, Subtarget);
14787 SplatBitSize, DAG, dl, VorrVT, VT, OtherModImm);
14790 DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VorrVT, N->getOperand(0));
14791 SDValue Vorr = DAG.getNode(ARMISD::VORRIMM, dl, VorrVT, Input, Val);
14792 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Vorr);
14810 DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
14838 SDValue Result = DAG.getNode(ARMISD::VBSP, dl, CanonicalVT,
14842 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Result);
14865 SelectionDAG &DAG = DCI.DAG;
14867 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
14894 Ops.push_back(DAG.getConstant(CC, DL, MVT::i32));
14895 return DAG.getNode(N0->getOpcode(), DL, N0->getValueType(0), Ops);
14966 static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG) {
14985 return DAG.getNode(ARMISD::BFI, SDLoc(N), N->getValueType(0),
15009 From1 = DAG.getNode(ISD::SRL, dl, VT, From1,
15010 DAG.getConstant(NewFromMask.countr_zero(), dl, VT));
15011 return DAG.getNode(ARMISD::BFI, dl, VT, CombineBFI.getOperand(0), From1,
15012 DAG.getConstant(~NewToMask, dl, VT));
15029 SDValue BFI1 = DAG.getNode(ARMISD::BFI, dl, VT, N0.getOperand(0),
15031 return DAG.getNode(ARMISD::BFI, dl, VT, BFI1, N0.getOperand(1),
15074 static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG) {
15087 static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG) {
15095 return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0),
15097 DAG.getConstant(Cond, SDLoc(N), MVT::i32), C);
15099 return DAG.getNode(
15102 DAG.getConstant(ARMCC::getOppositeCondition(Cond), SDLoc(N), MVT::i32), C);
15126 SelectionDAG &DAG = DCI.DAG;
15130 DAG.getLoad(MVT::i32, DL, LD->getChain(), BasePtr, LD->getPointerInfo(),
15133 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
15134 DAG.getConstant(4, DL, MVT::i32));
15136 SDValue NewLD2 = DAG.getLoad(MVT::i32, DL, LD->getChain(), OffsetPtr,
15141 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLD2.getValue(1));
15142 if (DCI.DAG.getDataLayout().isBigEndian())
15175 return DCI.DAG.getMergeValues({Op0, Op1}, SDLoc(N));
15193 return DCI.DAG.getMergeValues({Op0, Op1}, SDLoc(N));
15201 static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG) {
15212 return DAG.getNode(ISD::BITCAST, SDLoc(N),
15242 DCI.DAG.getNode(ISD::CopyFromReg, SDLoc(N),
15243 DCI.DAG.getVTList(ArrayRef(OutTys, HasGlue ? 3 : 2)),
15247 DCI.DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), NewCopy.getValue(0));
15248 DCI.DAG.ReplaceAllUsesOfValueWith(Copy.getValue(1), NewCopy.getValue(1));
15250 DCI.DAG.ReplaceAllUsesOfValueWith(Copy.getValue(2),
15262 DCI.DAG.getLoad(N->getValueType(0), SDLoc(N), LN0->getChain(),
15264 DCI.DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Load.getValue(0));
15265 DCI.DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), Load.getValue(1));
15272 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
15279 static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG) {
15286 return DAG.getConstant(V.bitcastToAPInt().getZExtValue(), SDLoc(N), VT);
15294 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT, LN0->getChain(),
15296 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Load.getValue(0));
15297 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
15304 return DAG.getNode(ARMISD::VGETLANEu, SDLoc(N), VT, N0->getOperand(0),
15333 SelectionDAG &DAG = DCI.DAG;
15335 if (SDValue RV = PerformVMOVDRRCombine(N, DAG))
15347 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(i));
15352 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, NumElts);
15353 SDValue BV = DAG.getBuildVector(FloatVT, dl, Ops);
15354 return DAG.getNode(ISD::BITCAST, dl, VT, BV);
15412 SelectionDAG &DAG = DCI.DAG;
15414 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts);
15416 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15425 SDValue Vec = DAG.getUNDEF(VecVT);
15436 V = DAG.getNode(ISD::BITCAST, SDLoc(V), MVT::i32, V);
15440 SDValue LaneIdx = DAG.getConstant(Idx, dl, MVT::i32);
15441 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecVT, Vec, V, LaneIdx);
15443 Vec = DAG.getNode(ISD::BITCAST, dl, VT, Vec);
15460 return DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, Op->getOperand(0));
15467 DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, Op->getOperand(0));
15468 SDValue C = DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT,
15469 DCI.DAG.getConstant(65535, dl, MVT::i32));
15470 return DCI.DAG.getNode(ISD::XOR, dl, VT, X, C);
15476 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
15483 static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG,
15491 return DAG.getNode(ISD::BITCAST, dl, VT, Op);
15498 return DAG.getUNDEF(VT);
15505 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Op->getOperand(0));
15511 static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG,
15524 return DAG.getNode(ARMISD::VCMPZ, dl, VT, Op0, N->getOperand(2));
15530 return DAG.getNode(ARMISD::VCMPZ, dl, VT, Op1,
15531 DAG.getConstant(SwappedCond, dl, MVT::i32));
15534 return DAG.getNode(ARMISD::VCMP, dl, VT, Op1, Op0,
15535 DAG.getConstant(SwappedCond, dl, MVT::i32));
15553 SelectionDAG &DAG = DCI.DAG;
15555 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64,
15557 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, N->getOperand(0));
15558 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(1));
15562 SDValue InsElt = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, FloatVT,
15564 return DAG.getNode(ISD::BITCAST, dl, VT, InsElt);
15577 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(MVT::f64))
15621 SDValue F64 = DCI.DAG.getNode(
15623 DCI.DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v2f64, Op0),
15624 DCI.DAG.getConstant(Ext.getConstantOperandVal(1) / 2, dl, MVT::i32));
15626 DCI.DAG.getNode(ARMISD::VMOVRRD, dl, {MVT::i32, MVT::i32}, F64);
15643 return DCI.DAG.getNode(ARMISD::VMOVhr, dl, VT, X);
15645 return DCI.DAG.getNode(ARMISD::VMOVrh, dl, VT, X);
15647 return DCI.DAG.getNode(ISD::BITCAST, dl, VT, X);
15686 return DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Op0.getOperand(Vec),
15687 DCI.DAG.getConstant(SubIdx, dl, MVT::i32));
15693 static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG) {
15701 return DAG.getNode(ARMISD::VGETLANEs, SDLoc(N), VT, Op.getOperand(0),
15717 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VecVT) ||
15718 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(SubVT))
15738 Hi = DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, Vec,
15739 DCI.DAG.getVectorIdxConstant(NumSubElts, DL));
15741 Lo = DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, Vec,
15742 DCI.DAG.getVectorIdxConstant(0, DL));
15745 return DCI.DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Lo, Hi);
15750 SelectionDAG &DAG) {
15758 return DAG.getNode(
15760 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(0)),
15761 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(1)),
15762 DAG.getConstant(1, DL, MVT::i32));
15764 return DAG.getNode(
15766 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(1)),
15767 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(0)),
15768 DAG.getConstant(1, DL, MVT::i32));
15774 static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG) {
15775 if (SDValue R = PerformShuffleVMOVNCombine(cast<ShuffleVectorSDNode>(N), DAG))
15800 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15807 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT,
15823 return DAG.getVectorShuffle(VT, SDLoc(N), NewConcat,
15824 DAG.getUNDEF(VT), NewMask);
15849 SelectionDAG &DAG = DCI.DAG;
16093 SDVTList SDTys = DAG.getVTList(ArrayRef(Tys, NumResultVecs + 2));
16114 Ops.push_back(DAG.getConstant(Alignment.value(), dl, MVT::i32));
16120 StVal = DAG.getNode(ISD::BITCAST, dl, AlignedVecTy, StVal);
16124 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys, Ops, LoadVT,
16136 LdVal = DAG.getNode(ISD::BITCAST, dl, VecTy, LdVal);
16149 SDValue Inc, const SelectionDAG &DAG) {
16159 if (DAG.haveNoCommonBitsSet(Ptr, Inc)) {
16209 /// CombineBaseUpdate - Target-specific DAG combine function for VLDDUP,
16234 getPointerConstIncrement(User->getOpcode(), Addr, Inc, DCI.DAG);
16246 getPointerConstIncrement(Addr->getOpcode(), Base, CInc, DCI.DAG);
16256 getPointerConstIncrement(User->getOpcode(), Base, UserInc, DCI.DAG);
16262 SDValue NewInc = DCI.DAG.getConstant(NewConstInc, SDLoc(N), MVT::i32);
16313 SelectionDAG &DAG = DCI.DAG;
16396 SDVTList SDTys = DAG.getVTList(ArrayRef(Tys, NumResultVecs + 2));
16407 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys, Ops, VecTy,
16430 SelectionDAG &DAG = DCI.DAG;
16475 SDVTList SDTys = DAG.getVTList(ArrayRef(Tys, NumVecs + 1));
16478 SDValue VLDDup = DAG.getMemIntrinsicNode(NewOpc, SDLoc(VLD), SDTys,
16514 if (!DCI.DAG.getTargetLoweringInfo().isTypeLegal(ExtractVT))
16516 SDValue Extract = DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), ExtractVT,
16518 return DCI.DAG.getNode(ARMISD::VDUP, SDLoc(N), VT, Extract);
16543 return DCI.DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
16547 static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG,
16556 return DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0),
16557 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op));
16559 return DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0),
16560 DAG.getNode(ARMISD::VMOVrh, dl, MVT::i32, Op));
16573 DAG.getConstant(LD->getAlign().value(), SDLoc(N), MVT::i32)};
16574 SDVTList SDTys = DAG.getVTList(N->getValueType(0), MVT::Other);
16576 DAG.getMemIntrinsicNode(ARMISD::VLD1DUP, SDLoc(N), SDTys, Ops,
16578 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), VLDDup.getValue(1));
16592 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
16602 SelectionDAG &DAG) {
16607 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16627 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(), StVT.getScalarType(),
16632 SDValue WideVec = DAG.getNode(ISD::BITCAST, DL, WideVecVT, StVal);
16635 ShuffleVec[i] = DAG.getDataLayout().isBigEndian() ? (i + 1) * SizeRatio - 1
16642 SDValue Shuff = DAG.getVectorShuffle(
16643 WideVecVT, DL, WideVec, DAG.getUNDEF(WideVec.getValueType()), ShuffleVec);
16659 EVT::getVectorVT(*DAG.getContext(), StoreType,
16662 SDValue ShuffWide = DAG.getNode(ISD::BITCAST, DL, StoreVecVT, Shuff);
16664 SDValue Increment = DAG.getConstant(StoreType.getSizeInBits() / 8, DL,
16665 TLI.getPointerTy(DAG.getDataLayout()));
16671 SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreType,
16672 ShuffWide, DAG.getIntPtrConstant(I, DL));
16674 DAG.getStore(St->getChain(), DL, SubVec, BasePtr, St->getPointerInfo(),
16677 DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr, Increment);
16680 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
16687 SelectionDAG &DAG) {
16738 LLVMContext &C = *DAG.getContext();
16757 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(NewOffset));
16760 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewFromVT, Trunc.getOperand(0),
16761 DAG.getConstant(i * NumElements, DL, MVT::i32));
16764 DAG.getNode(ARMISD::VCVTN, DL, MVT::v8f16, DAG.getUNDEF(MVT::v8f16),
16765 Extract, DAG.getConstant(0, DL, MVT::i32));
16766 Extract = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, MVT::v4i32, FPTrunc);
16768 SDValue Store = DAG.getTruncStore(
16773 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
16780 SelectionDAG &DAG) {
16789 LLVMContext &C = *DAG.getContext();
16806 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(NewOffset));
16809 SDValue Store = DAG.getTruncStore(
16814 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
16821 static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG) {
16832 DAG.getNodeIfExists(ARMISD::VGETLANEu, DAG.getVTList(MVT::i32),
16837 LLVMContext &C = *DAG.getContext();
16846 SDValue Store = DAG.getTruncStore(Ch, DL, SDValue(GetLane, 0), BasePtr,
16865 if (SDValue Store = PerformTruncatingStoreCombine(St, DCI.DAG))
16869 if (SDValue NewToken = PerformSplittingToNarrowingStores(St, DCI.DAG))
16873 if (SDValue NewChain = PerformExtractFpToIntStores(St, DCI.DAG))
16876 PerformSplittingMVETruncToNarrowingStores(St, DCI.DAG))
16887 SelectionDAG &DAG = DCI.DAG;
16888 bool isBigEndian = DAG.getDataLayout().isBigEndian();
16891 SDValue NewST1 = DAG.getStore(
16896 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
16897 DAG.getConstant(4, DL, MVT::i32));
16898 return DAG.getStore(NewST1.getValue(0), DL,
16910 SelectionDAG &DAG = DCI.DAG;
16913 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64,
16915 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, IntVec);
16916 SDValue ExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
16919 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ExtElt);
16924 return DAG.getStore(St->getChain(), dl, V, St->getBasePtr(),
16931 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
16946 static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG,
16983 SDValue FixConv = DAG.getNode(
16985 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32), Op->getOperand(0),
16986 DAG.getConstant(C, dl, MVT::i32));
16989 FixConv = DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), FixConv);
16994 static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG,
17033 DAG.getNode(ISD::FADD, DL, VT, Op0, Op1.getOperand(1), FaddFlags);
17034 return DAG.getNode(ISD::VSELECT, DL, VT, Op1.getOperand(0), FAdd, Op0, FaddFlags);
17037 static SDValue PerformFADDVCMLACombine(SDNode *N, SelectionDAG &DAG) {
17053 SDValue VCMLA = DAG.getNode(
17055 DAG.getNode(ISD::FADD, DL, VT, A.getOperand(2), B, N->getFlags()),
17068 static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG,
17070 if (SDValue S = PerformFAddVSelectCombine(N, DAG, Subtarget))
17072 if (SDValue S = PerformFADDVCMLACombine(N, DAG))
17086 static SDValue PerformVMulVCTPCombine(SDNode *N, SelectionDAG &DAG,
17134 ConvInput = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
17139 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
17140 DAG.getConstant(IntrinsicOpcode, DL, MVT::i32), ConvInput,
17141 DAG.getConstant(C, DL, MVT::i32));
17144 static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
17158 SDValue Red0 = DAG.getNode(ISD::VECREDUCE_ADD, dl, ResVT, N0.getOperand(0));
17159 SDValue Red1 = DAG.getNode(ISD::VECREDUCE_ADD, dl, ResVT, N0.getOperand(1));
17160 return DAG.getNode(ISD::ADD, dl, ResVT, Red0, Red1);
17187 A = DAG.getNode(ExtendCode, dl,
17292 DAG.getNode(IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, dl,
17293 DAG.getVTList(MVT::v8i16, MVT::v8i16), Ops[0]);
17295 DAG.getNode(IsUnsigned ? ARMISD::MVEZEXT : ARMISD::MVESEXT, dl,
17296 DAG.getVTList(MVT::v8i16, MVT::v8i16), Ops[1]);
17298 SDValue MLA0 = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32),
17301 DAG.getNode(IsUnsigned ? ARMISD::VMLALVAu : ARMISD::VMLALVAs, dl,
17302 DAG.getVTList(MVT::i32, MVT::i32), MLA0, MLA0.getValue(1),
17304 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, MLA1, MLA1.getValue(1));
17306 SDValue Node = DAG.getNode(Opcode, dl, {MVT::i32, MVT::i32}, Ops);
17307 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Node,
17314 return DAG.getNode(ARMISD::VMLAVs, dl, ResVT, A, B);
17316 return DAG.getNode(ARMISD::VMLAVu, dl, ResVT, A, B);
17324 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17325 DAG.getNode(ARMISD::VMLAVs, dl, MVT::i32, A, B));
17327 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17328 DAG.getNode(ARMISD::VMLAVu, dl, MVT::i32, A, B));
17332 return DAG.getNode(ARMISD::VMLAVps, dl, ResVT, A, B, Mask);
17335 return DAG.getNode(ARMISD::VMLAVpu, dl, ResVT, A, B, Mask);
17343 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17344 DAG.getNode(ARMISD::VMLAVps, dl, MVT::i32, A, B, Mask));
17346 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17347 DAG.getNode(ARMISD::VMLAVpu, dl, MVT::i32, A, B, Mask));
17350 return DAG.getNode(ARMISD::VADDVs, dl, ResVT, A);
17352 return DAG.getNode(ARMISD::VADDVu, dl, ResVT, A);
17358 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17359 DAG.getNode(ARMISD::VADDVs, dl, MVT::i32, A));
17361 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17362 DAG.getNode(ARMISD::VADDVu, dl, MVT::i32, A));
17365 return DAG.getNode(ARMISD::VADDVps, dl, ResVT, A, Mask);
17367 return DAG.getNode(ARMISD::VADDVpu, dl, ResVT, A, Mask);
17373 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17374 DAG.getNode(ARMISD::VADDVps, dl, MVT::i32, A, Mask));
17376 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
17377 DAG.getNode(ARMISD::VADDVpu, dl, MVT::i32, A, Mask));
17390 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, N0->getValueType(0), Mul);
17392 Ext = DAG.getNode(ISD::VSELECT, dl, N0->getValueType(0),
17394 return DAG.getNode(ISD::VECREDUCE_ADD, dl, ResVT, Ext);
17404 static SDValue PerformReduceShuffleCombine(SDNode *N, SelectionDAG &DAG) {
17434 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), Ops);
17456 return DCI.DAG.getNode(Op1->getOpcode(), SDLoc(Op1), N->getValueType(0),
17468 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
17487 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
17506 SDValue NewBinOp = DCI.DAG.getNode(N->getOpcode(), DL, VT,
17509 return DCI.DAG.getVectorShuffle(VT, DL, NewBinOp, UndefV, Shuf0->getMask());
17514 static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG) {
17524 SDValue Merge = DAG.getMergeValues({Op0, Op1}, DL);
17525 DAG.ReplaceAllUsesWith(N, Merge.getNode());
17532 SDValue NewShift = DAG.getNode(NewOpcode, DL, N->getVTList(), Op0, Op1,
17533 DAG.getConstant(-ShiftAmt, DL, MVT::i32));
17534 DAG.ReplaceAllUsesWith(N, NewShift.getNode());
17542 /// PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
17545 SelectionDAG &DAG = DCI.DAG;
17553 // Note: This is done during DAG combining instead of DAG legalizing because
17667 return DAG.getNode(VShiftOpc, dl, N->getValueType(0),
17668 N->getOperand(1), DAG.getConstant(Cnt, dl, MVT::i32));
17685 return DAG.getNode(VShiftOpc, dl, N->getValueType(0),
17687 DAG.getConstant(Cnt, dl, MVT::i32));
17697 return DAG.getNode(ARMISD::VBSP, dl, N->getValueType(0), N->getOperand(1),
17743 return DAG.getNode(Opc, SDLoc(N), N->getVTList(), N->getOperand(1));
17761 SDValue val = DAG.getNode(Opc, dl, {MVT::i32, MVT::i32}, Ops);
17762 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, val.getValue(0),
17771 /// lowers them. As with the vector shift intrinsics, this is done during DAG
17772 /// combining instead of DAG legalizing because the build_vectors for 64-bit
17778 SelectionDAG &DAG = DCI.DAG;
17806 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
17807 DAG.getConstant(MaskedBits, DL, MVT::i32));
17808 return DAG.getNode(
17810 DAG.getConstant(MaskedBits - ShiftAmt, DL, MVT::i32));
17816 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17830 return DAG.getNode(ARMISD::VSHLIMM, dl, VT, N->getOperand(0),
17831 DAG.getConstant(Cnt, dl, MVT::i32));
17841 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0),
17842 DAG.getConstant(Cnt, dl, MVT::i32));
17852 static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
17879 LLVMContext &C = *DAG.getContext();
17890 SDValue Offset = DAG.getUNDEF(BasePtr.getValueType());
17901 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(NewOffset));
17904 DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, NewPtr, Offset,
17917 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, MVT::v8f16, Loads[i]);
17918 SDValue FPExt = DAG.getNode(ARMISD::VCVTL, DL, MVT::v4f32, LoadBC,
17919 DAG.getConstant(0, DL, MVT::i32));
17926 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
17927 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewChain);
17928 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ToVT, Loads);
17931 /// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND,
17933 static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG,
17939 // handled during DAG combining because type legalization will promote them
17947 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17965 return DAG.getNode(Opc, SDLoc(N), VT, Vec, Lane);
17970 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
17976 static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG,
17979 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
17987 static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG,
18017 return DAG.getNode(ARMISD::SSAT, DL, VT, Input,
18018 DAG.getConstant(MinC.countr_one(), DL, VT));
18020 return DAG.getNode(ARMISD::USAT, DL, VT, Input,
18021 DAG.getConstant(MinC.countr_one(), DL, VT));
18026 /// PerformMinMaxCombine - Target-specific DAG combining for creating truncating
18028 static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG,
18034 return PerformMinMaxToSatCombine(SDValue(N, 0), DAG, ST);
18039 if (SDValue V = PerformVQDMULHCombine(N, DAG))
18083 DAG.getNode(ARMISD::VQMOVNs, DL, HalfVT, DAG.getUNDEF(HalfVT),
18084 N0->getOperand(0), DAG.getConstant(0, DL, MVT::i32));
18085 SDValue Bitcast = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, VQMOVN);
18086 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Bitcast,
18087 DAG.getValueType(ExtVT));
18124 DAG.getNode(ARMISD::VQMOVNu, DL, HalfVT, DAG.getUNDEF(HalfVT), N0,
18125 DAG.getConstant(0, DL, MVT::i32));
18126 SDValue Bitcast = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, VQMOVN);
18127 return DAG.getNode(ISD::AND, DL, VT, Bitcast,
18128 DAG.getConstant(ExtConst, DL, VT));
18142 SDValue ARMTargetLowering::PerformCMOVToBFICombine(SDNode *CMOV, SelectionDAG &DAG) const {
18201 KnownBits Known = DAG.computeKnownBits(Y);
18213 X = DAG.getNode(ISD::SRL, dl, VT, X,
18214 DAG.getConstant(BitInX, dl, VT));
18223 V = DAG.getNode(ARMISD::BFI, dl, VT, V, X,
18225 DAG.getConstant(~Mask, dl, VT));
18340 SelectionDAG &DAG = DCI.DAG;
18349 auto UpdateUncondBr = [](SDNode *Br, SDValue Dest, SelectionDAG &DAG) {
18351 SDValue NewBr = DAG.getNode(ISD::BR, SDLoc(Br), MVT::Other, NewBrOps);
18352 DAG.ReplaceAllUsesOfValueWith(SDValue(Br, 0), NewBr);
18357 SDValue Setup = DAG.getNode(ARMISD::WLSSETUP, dl, MVT::i32, Elements);
18361 Res = DAG.getNode(ARMISD::WLS, dl, MVT::Other, Ops);
18365 UpdateUncondBr(Br, Dest, DAG);
18368 Res = DAG.getNode(ARMISD::WLS, dl, MVT::Other, Ops);
18371 DAG.ReplaceAllUsesOfValueWith(Int.getValue(0), Setup);
18373 DAG.ReplaceAllUsesOfValueWith(Int.getValue(2), Int.getOperand(0));
18377 DAG.getTargetConstant(Int.getConstantOperandVal(3), dl, MVT::i32);
18379 SDValue LoopDec = DAG.getNode(ARMISD::LOOP_DEC, dl,
18380 DAG.getVTList(MVT::i32, MVT::Other), Args);
18381 DAG.ReplaceAllUsesWith(Int.getNode(), LoopDec.getNode());
18389 UpdateUncondBr(Br, Dest, DAG);
18391 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
18395 return DAG.getNode(ARMISD::LE, dl, MVT::Other, EndArgs);
18400 /// PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
18402 ARMTargetLowering::PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const {
18424 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, BB,
18432 /// PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
18434 ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
18451 SDValue R = PerformCMOVToBFICombine(N, DAG);
18475 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, TrueVal, ARMcc, Cmp);
18478 SDValue NewCmp = getARMCmp(LHS, RHS, ISD::SETNE, ARMcc, DAG, dl);
18479 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, FalseVal, ARMcc, NewCmp);
18487 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal,
18504 return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0),
18506 DAG.getConstant(Cond, SDLoc(N), MVT::i32), C);
18517 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
18518 Res = DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::CTLZ, dl, VT, Sub),
18519 DAG.getConstant(5, dl, MVT::i32));
18530 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
18531 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
18532 SDValue Neg = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, Sub);
18536 DAG.getNode(ISD::SUB, dl, MVT::i32,
18537 DAG.getConstant(1, dl, MVT::i32), Neg.getValue(1));
18538 Res = DAG.getNode(ISD::UADDO_CARRY, dl, VTs, Sub, Neg, Carry);
18545 DAG.getNode(ARMISD::SUBC, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS);
18546 Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, TrueVal, ARMcc,
18557 DAG.getNode(ARMISD::SUBC, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS);
18558 Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, FalseVal,
18559 DAG.getConstant(ARMCC::NE, dl, MVT::i32),
18565 // On Thumb1, the DAG above may be further combined if z is a power of 2
18584 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
18587 TrueVal = DAG.getConstant(1, dl, VT);
18588 SDValue Subc = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, TrueVal);
18589 Res = DAG.getNode(ISD::USUBO_CARRY, dl, VTs, FalseVal, Subc,
18593 Res = DAG.getNode(ISD::SHL, dl, VT, Res,
18594 DAG.getConstant(ShiftAmount, dl, MVT::i32));
18598 KnownBits Known = DAG.computeKnownBits(SDValue(N,0));
18601 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
18602 DAG.getValueType(MVT::i1));
18604 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
18605 DAG.getValueType(MVT::i8));
18607 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
18608 DAG.getValueType(MVT::i16));
18617 SelectionDAG &DAG = DCI.DAG;
18625 return DAG.getNode(ARMISD::VDUP, SDLoc(N), DstVT, Src.getOperand(0));
18642 DAG.getDataLayout().isBigEndian())
18643 return DAG.getNode(ARMISD::VECTOR_REG_CAST, SDLoc(N), DstVT, Src);
18656 SelectionDAG &DAG = DCI.DAG;
18662 return DAG.getUNDEF(VT);
18668 return DAG.getNode(ARMISD::MVETRUNC, DL, VT, N->getOperand(0).getOperand(0),
18687 return DAG.getNode(
18689 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(0)),
18690 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(1)),
18691 DAG.getConstant(1, DL, MVT::i32));
18693 return DAG.getNode(
18695 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(1)),
18696 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(0)),
18697 DAG.getConstant(1, DL, MVT::i32));
18713 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, O,
18714 DAG.getConstant(i, DL, MVT::i32));
18718 return DAG.getBuildVector(VT, DL, Extracts);
18728 SDValue StackPtr = DAG.CreateStackTemporary(TypeSize::getFixed(16), Align(4));
18733 EVT StoreVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
18735 StoreVT = StoreVT.getHalfNumVectorElementsVT(*DAG.getContext());
18739 SDValue Ptr = DAG.getNode(
18741 DAG.getConstant(I * 16 / NumIns, DL, StackPtr.getValueType()));
18743 DAG.getMachineFunction(), SPFI, I * 16 / NumIns);
18744 SDValue Ch = DAG.getTruncStore(DAG.getEntryNode(), DL, N->getOperand(I),
18749 SDValue Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
18751 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI, 0);
18752 return DAG.getLoad(VT, DL, Chain, StackPtr, MPI, Align(4));
18757 SelectionDAG &DAG) {
18785 LLVMContext &C = *DAG.getContext();
18794 SDValue Offset = DAG.getUNDEF(BasePtr.getValueType());
18805 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(NewOffset));
18808 DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, NewPtr, Offset,
18815 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
18816 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewChain);
18817 return DAG.getMergeValues(Loads, DL);
18825 SelectionDAG &DAG = DCI.DAG;
18832 *DAG.getContext());
18834 SDValue VVT = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, V);
18836 ? DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, VVT,
18837 DAG.getValueType(ExtVT))
18838 : DAG.getZeroExtendInReg(VVT, DL, ExtVT);
18844 return DAG.getMergeValues({Ext, Ext}, DL);
18867 V0 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op0));
18871 V0 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op1));
18876 V1 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op1));
18880 V1 = Extend(DAG.getNode(Rev, DL, SVN->getValueType(0), Op0));
18883 return DAG.getMergeValues({V0, V1}, DL);
18888 if (SDValue L = PerformSplittingMVEEXTToWideningLoad(N, DAG))
18896 SDValue StackPtr = DAG.CreateStackTemporary(TypeSize::getFixed(16), Align(4));
18902 *DAG.getContext());
18904 LoadVT = LoadVT.getHalfNumVectorElementsVT(*DAG.getContext());
18907 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI, 0);
18908 SDValue Chain = DAG.getStore(DAG.getEntryNode(), DL, N->getOperand(0),
18913 SDValue Ptr = DAG.getNode(
18915 DAG.getConstant(I * 16 / NumOuts, DL, StackPtr.getValueType()));
18917 DAG.getMachineFunction(), SPFI, I * 16 / NumOuts);
18918 SDValue Load = DAG.getExtLoad(
18924 return DAG.getMergeValues(Loads, DL);
18936 case ARMISD::UMLAL: return PerformUMLALCombine(N, DCI.DAG, Subtarget);
18948 case ARMISD::BFI: return PerformBFICombine(N, DCI.DAG);
18950 case ARMISD::VMOVDRR: return PerformVMOVDRRCombine(N, DCI.DAG);
18952 case ARMISD::VMOVrh: return PerformVMOVrhCombine(N, DCI.DAG);
18958 case ISD::SIGN_EXTEND_INREG: return PerformSignExtendInregCombine(N, DCI.DAG);
18960 case ISD::VECTOR_SHUFFLE: return PerformVECTOR_SHUFFLECombine(N, DCI.DAG);
18962 case ARMISD::VDUP: return PerformVDUPCombine(N, DCI.DAG, Subtarget);
18965 return PerformVCVTCombine(N, DCI.DAG, Subtarget);
18967 return PerformFADDCombine(N, DCI.DAG, Subtarget);
18969 return PerformVMulVCTPCombine(N, DCI.DAG, Subtarget);
18979 return PerformExtendCombine(N, DCI.DAG, Subtarget);
18981 return PerformFPExtendCombine(N, DCI.DAG, Subtarget);
18986 return PerformMinMaxCombine(N, DCI.DAG, Subtarget);
18988 return PerformCMOVCombine(N, DCI.DAG);
18990 return PerformBRCONDCombine(N, DCI.DAG);
18992 return PerformCMPZCombine(N, DCI.DAG);
18996 return PerformCSETCombine(N, DCI.DAG);
19011 return PerformVECTOR_REG_CASTCombine(N, DCI.DAG, Subtarget);
19018 return PerformVCMPCombine(N, DCI.DAG, Subtarget);
19020 return PerformVECREDUCE_ADDCombine(N, DCI.DAG, Subtarget);
19033 return PerformReduceShuffleCombine(N, DCI.DAG);
19044 return PerformLongShiftCombine(N, DCI.DAG);
19302 // negate values directly (fneg is free). So, we don't want to let the DAG
19305 // triggering this DAG combine rewrite, so we are avoiding that with this.
19699 SelectionDAG &DAG) {
19711 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
19725 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
19758 SelectionDAG &DAG) {
19768 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
19772 Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0));
19783 bool &isInc, SelectionDAG &DAG) {
19801 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
19805 Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0));
19840 SelectionDAG &DAG) const {
19878 Subtarget->isLittle(), Base, Offset, isInc, DAG);
19882 Offset, isInc, DAG);
19885 Offset, isInc, DAG);
19901 SelectionDAG &DAG) const {
19958 isInc, DAG);
19962 isInc, DAG);
19965 isInc, DAG);
19989 const SelectionDAG &DAG,
20013 Known = DAG.computeKnownBits(Op.getOperand(0), Depth+1);
20017 KnownBits KnownRHS = DAG.computeKnownBits(Op.getOperand(1), Depth+1);
20038 Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
20058 Known = DAG.computeKnownBits(SrcSV, DemandedElt, Depth + 1);
20075 KnownBits KnownOp = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
20083 KnownBits KnownOp0 = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
20084 KnownBits KnownOp1 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
20154 SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
20155 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
20205 Op, TLO.DAG.getNode(
20207 TLO.DAG.getConstant(32 - ShAmt, SDLoc(Op), MVT::i32)));
20424 SelectionDAG &DAG) const {
20576 Result = DAG.getSignedTargetConstant(CVal, SDLoc(Op), Op.getValueType());
20584 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
20628 SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
20642 if (expandDIVREMByConstant(Op.getNode(), Result, MVT::i32, DAG)) {
20644 DAG.getNode(ISD::BUILD_PAIR, dl, VT, Result[0], Result[1]);
20646 DAG.getNode(ISD::BUILD_PAIR, dl, VT, Result[2], Result[3]);
20647 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
20652 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
20666 SDValue Div = DAG.getNode(DivOpcode, dl, VT, Dividend, Divisor);
20667 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Div, Divisor);
20668 SDValue Rem = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);
20671 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(VT, VT), Values);
20676 SDValue InChain = DAG.getEntryNode();
20679 DAG.getContext(),
20682 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
20683 getPointerTy(DAG.getDataLayout()));
20688 InChain = WinDBZCheckDenominator(DAG, Op.getNode(), InChain);
20690 TargetLowering::CallLoweringInfo CLI(DAG);
20701 SDValue ARMTargetLowering::LowerREM(SDNode *N, SelectionDAG &DAG) const {
20706 if (expandDIVREMByConstant(N, Result, MVT::i32, DAG))
20707 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), N->getValueType(0),
20717 case MVT::i8: RetTyElement = Type::getInt8Ty(*DAG.getContext()); break;
20718 case MVT::i16: RetTyElement = Type::getInt16Ty(*DAG.getContext()); break;
20719 case MVT::i32: RetTyElement = Type::getInt32Ty(*DAG.getContext()); break;
20720 case MVT::i64: RetTyElement = Type::getInt64Ty(*DAG.getContext()); break;
20726 Type *RetTy = StructType::get(*DAG.getContext(), ret);
20730 SDValue InChain = DAG.getEntryNode();
20731 TargetLowering::ArgListTy Args = getDivRemArgList(N, DAG.getContext(),
20734 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
20735 getPointerTy(DAG.getDataLayout()));
20738 InChain = WinDBZCheckDenominator(DAG, N, InChain);
20741 CallLoweringInfo CLI(DAG);
20754 ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
20762 if (DAG.getMachineFunction().getFunction().hasFnAttribute(
20766 SDValue SP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
20768 SP = DAG.getNode(ISD::SUB, DL, MVT::i32, SP, Size);
20771 DAG.getNode(ISD::AND, DL, MVT::i32, SP.getValue(0),
20772 DAG.getConstant(-(uint64_t)Align->value(), DL, MVT::i32));
20773 Chain = DAG.getCopyToReg(Chain, DL, ARM::SP, SP);
20775 return DAG.getMergeValues(Ops, DL);
20778 SDValue Words = DAG.getNode(ISD::SRL, DL, MVT::i32, Size,
20779 DAG.getConstant(2, DL, MVT::i32));
20782 Chain = DAG.getCopyToReg(Chain, DL, ARM::R4, Words, Glue);
20785 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
20786 Chain = DAG.getNode(ARMISD::WIN__CHKSTK, DL, NodeTys, Chain, Glue);
20788 SDValue NewSP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
20792 return DAG.getMergeValues(Ops, DL);
20795 SDValue ARMTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
20814 SDValue Result = DAG.getNode(ISD::FP_EXTEND,
20816 return DAG.getMergeValues({Result, Op.getOperand(0)}, Loc);
20836 SrcVal = DAG.getNode(ISD::STRICT_FP_EXTEND, Loc,
20840 SrcVal = DAG.getNode(ISD::FP_EXTEND, Loc, DstVT, SrcVal);
20846 std::tie(SrcVal, Chain) = makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20851 return IsStrict ? DAG.getMergeValues({SrcVal, Chain}, Loc) : SrcVal;
20854 SDValue ARMTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
20882 std::tie(Result, Chain) = makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20884 return IsStrict ? DAG.getMergeValues({Result, Chain}, Loc) : Result;
21428 SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const {
21431 return TargetLowering::preferredShiftLegalizationStrategy(DAG, N,