Lines Matching defs:DL
1518 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
1522 return getPointerTy(DL);
1569 auto &DL = I.getDataLayout();
1594 Info.memVT = getValueType(DL, MemTy);
1595 Info.align = Align(DL.getTypeSizeInBits(MemTy->getScalarType()) / 8);
1837 bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
2387 static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
2406 LHS = DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS,
2407 DAG.getConstant(ShAmt, DL, LHS.getValueType()));
2419 RHS = DAG.getConstant(0, DL, RHS.getValueType());
2428 LHS = DAG.getConstant(0, DL, RHS.getValueType());
2734 SDLoc DL(V);
2735 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
2736 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
2746 SDLoc DL(V);
2747 SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
2748 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
2762 static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL,
2765 return DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
2768 static SDValue getVLOp(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
2778 return DAG.getConstant(NumElts, DL, Subtarget.getXLenVT());
2782 getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG,
2786 SDValue Mask = getAllOnesMask(VecVT, VL, DL, DAG);
2791 getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
2794 SDValue VL = getVLOp(NumElts, ContainerVT, DL, DAG, Subtarget);
2795 SDValue Mask = getAllOnesMask(ContainerVT, VL, DL, DAG);
2804 getDefaultVLOps(MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG,
2807 return getDefaultVLOps(VecVT.getVectorNumElements(), ContainerVT, DL, DAG,
2810 return getDefaultScalableVLOps(ContainerVT, DL, DAG, Subtarget);
2813 SDValue RISCVTargetLowering::computeVLMax(MVT VecVT, const SDLoc &DL,
2816 return DAG.getElementCount(DL, Subtarget.getXLenVT(),
2934 SDLoc DL(Op);
2936 Opc, DL, DstVT, Src,
2937 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()));
2940 FpToInt = DAG.getZeroExtendInReg(FpToInt, DL, MVT::i32);
2942 SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
2943 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt,
2974 SDLoc DL(Op);
2976 auto [Mask, VL] = getDefaultVLOps(DstVT, DstContainerVT, DL, DAG, Subtarget);
2978 SDValue IsNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
2987 Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterVT, Src, Mask, VL);
2992 SDValue Res = DAG.getNode(RVVOpc, DL, DstContainerVT, Src, Mask, VL);
2995 RISCVISD::VMV_V_X_VL, DL, DstContainerVT, DAG.getUNDEF(DstContainerVT),
2996 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL);
2997 Res = DAG.getNode(RISCVISD::VMERGE_VL, DL, DstContainerVT, IsNan, SplatZero,
3045 SDLoc DL(Op);
3063 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3070 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
3081 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
3082 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
3088 DAG.getNode(RISCVISD::SETCC_VL, DL, SetccVT,
3111 Truncated = DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, IntVT, Src, Mask,
3112 DAG.getTargetConstant(FRM, DL, XLenVT), VL);
3116 Truncated = DAG.getNode(RISCVISD::VFCVT_RTZ_X_F_VL, DL, IntVT, Src,
3121 Truncated = DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, IntVT, Src, Mask, VL);
3125 Truncated = DAG.getNode(RISCVISD::VFROUND_NOEXCEPT_VL, DL, ContainerVT, Src,
3132 Truncated = DAG.getNode(RISCVISD::SINT_TO_FP_VL, DL, ContainerVT, Truncated,
3136 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
3151 SDLoc DL(Op);
3162 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3169 SDValue Unorder = DAG.getNode(RISCVISD::STRICT_FSETCC_VL, DL,
3174 Src = DAG.getNode(RISCVISD::STRICT_FADD_VL, DL,
3180 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
3191 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
3192 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
3197 RISCVISD::SETCC_VL, DL, MaskVT,
3215 RISCVISD::STRICT_VFCVT_RM_X_F_VL, DL, DAG.getVTList(IntVT, MVT::Other),
3216 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3221 DAG.getNode(RISCVISD::STRICT_VFCVT_RTZ_X_F_VL, DL,
3225 Truncated = DAG.getNode(RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, DL,
3234 Truncated = DAG.getNode(RISCVISD::STRICT_SINT_TO_FP_VL, DL,
3241 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
3246 return DAG.getMergeValues({Truncated, Chain}, DL);
3259 SDLoc DL(Op);
3270 SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
3273 return DAG.getNode(RISCVISD::FROUND, DL, VT, Src, MaxValNode,
3274 DAG.getTargetConstant(FRM, DL, Subtarget.getXLenVT()));
3283 SDLoc DL(Op);
3292 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3294 DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, ContainerVT, Src, Mask, VL);
3304 const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op,
3309 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
3311 return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
3315 getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL,
3321 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
3323 return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
3471 static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
3493 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3495 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, Vec,
3520 SDLoc DL(Op);
3521 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3569 SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
3585 LastOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, LastOp);
3586 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
3598 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V,
3599 DAG.getVectorIdxConstant(OpIdx.index(), DL));
3606 return DAG.getConstant(V == V1, DL, XLenVT);
3608 Vec = DAG.getNode(ISD::VSELECT, DL, VT,
3609 DAG.getBuildVector(SelMaskTy, DL, Ops),
3610 DAG.getSplatBuildVector(VT, DL, V), Vec);
3627 SDLoc DL(Op);
3628 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3635 SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
3640 SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
3682 SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
3690 SDValue Vec = DAG.getBuildVector(IntegerViaVecVT, DL, Elts);
3698 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec,
3699 DAG.getConstant(0, DL, XLenVT));
3714 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
3716 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
3753 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VIDContainerVT, Mask, VL);
3761 SDValue SplatStep = DAG.getConstant(SplatStepVal, DL, VIDVT);
3762 VID = DAG.getNode(StepOpcode, DL, VIDVT, VID, SplatStep);
3766 DAG.getConstant(Log2_64(StepDenominator), DL, VIDVT);
3767 VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep);
3770 SDValue SplatAddend = DAG.getConstant(Addend, DL, VIDVT);
3771 VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend,
3776 VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID);
3812 SDValue Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ViaVecVT,
3814 DAG.getConstant(SplatValue, DL, XLenVT),
3815 DAG.getVectorIdxConstant(0, DL));
3817 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
3819 DAG.getConstant(0, DL, XLenVT));
3877 DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
3881 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
3883 DAG.getConstant(SplatValue, DL, XLenVT), ViaVL);
3886 Splat = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
3888 DAG.getConstant(0, DL, XLenVT));
3904 DL, Op->ops());
3907 SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL);
3943 SDLoc DL(Op);
3965 APInt::getLowBitsSet(XLenVT.getSizeInBits(), ElemSizeInBits), DL, XLenVT);
3995 return DAG.getNode(ISD::BITCAST, DL, VT,
3996 DAG.getBuildVector(WideVecVT, DL, NewOperands));
4026 SDLoc DL(Op);
4027 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4036 SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
4042 Splat = DAG.getNode(ISD::AND, DL, Splat.getValueType(), Splat,
4043 DAG.getConstant(1, DL, Splat.getValueType()));
4044 WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
4047 WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
4048 SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
4049 WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
4052 return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
4056 if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget))
4061 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
4063 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
4091 DAG.getNode(ISD::BUILD_VECTOR, DL, OneRegVT, OneVRegOfOps);
4094 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubBV,
4095 DAG.getVectorIdxConstant(InsertIdx, DL));
4136 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
4141 SDValue SubVecA = DAG.getBuildVector(VT, DL, SubVecAOps);
4142 SDValue SubVecB = DAG.getBuildVector(VT, DL, SubVecBOps);
4144 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
4145 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, SubVecA, SubVecB);
4211 Vec = DAG.getSplatVector(VT, DL, V);
4218 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
4219 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
4226 V = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), V);
4227 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
4231 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
4232 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
4238 static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
4249 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
4263 NewVL = DAG.getNode(ISD::ADD, DL, VL.getValueType(), VL, VL);
4268 auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT,
4270 return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
4279 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
4284 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
4287 return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Passthru, Lo,
4294 static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
4299 std::tie(Lo, Hi) = DAG.SplitScalar(Scalar, DL, MVT::i32, MVT::i32);
4300 return splatPartsI64WithVL(DL, VT, Passthru, Lo, Hi, VL, DAG);
4307 MVT VT, const SDLoc &DL, SelectionDAG &DAG,
4313 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Passthru, Scalar, VL);
4325 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
4326 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
4333 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru,
4334 DAG.getConstant(0, DL, XLenVT), VL);
4337 return splatSplitI64WithVL(DL, VT, Passthru, Scalar, VL, DAG);
4345 const SDLoc &DL, SelectionDAG &DAG,
4367 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru,
4368 ExtractedVal, DAG.getVectorIdxConstant(0, DL));
4369 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, ExtractedVal,
4370 DAG.getVectorIdxConstant(0, DL));
4376 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT,
4383 DAG.getConstant(1, DL, XLenVT),
4384 VT, DL, DAG, Subtarget);
4392 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
4393 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT,
4559 static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src,
4577 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4593 RISCVISD::VMV_V_X_VL, DL, IntContainerVT, DAG.getUNDEF(ContainerVT),
4594 DAG.getConstant(Shift, DL, Subtarget.getXLenVT()), VL);
4596 DAG.getNode(RISCVISD::VNSRL_VL, DL, IntContainerVT, Src, SplatShift,
4618 static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT,
4672 auto [TrueMask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
4674 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
4676 DAG.getConstant(NewMask[0], DL, XLenVT), TrueMask, VL);
4678 ISD::EXTRACT_SUBVECTOR, DL, VT,
4680 DAG.getConstant(0, DL, XLenVT));
4695 static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT,
4712 auto TrueMask = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).first;
4723 SDValue VL = DAG.getConstant(NumSubElts + Index, DL, XLenVT);
4729 Res = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, InPlace, ToInsert,
4732 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, InPlace, ToInsert,
4733 DAG.getConstant(Index, DL, XLenVT), TrueMask, VL, Policy);
4739 static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT,
4777 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4782 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Splat);
4783 auto Vec = DAG.getNode(OpCode, DL, ContainerVT,
4794 const SDLoc &DL, SelectionDAG &DAG,
4823 auto [Mask, VL] = getDefaultVLOps(VecVT, VecContainerVT, DL, DAG, Subtarget);
4832 DAG.getNode(RISCVISD::VZEXT_VL, DL, WideContainerVT, EvenV, Mask, VL);
4836 DAG.getConstant(VecVT.getScalarSizeInBits(), DL, VecContainerVT);
4837 Interleaved = DAG.getNode(RISCVISD::VWSLL_VL, DL, WideContainerVT, OddV,
4840 Interleaved = DAG.getNode(RISCVISD::VWADDU_W_VL, DL, WideContainerVT,
4844 DAG.getNode(RISCVISD::VZEXT_VL, DL, WideContainerVT, OddV, Mask, VL);
4847 DAG.getConstant(VecVT.getScalarSizeInBits(), DL, WideContainerVT);
4848 Interleaved = DAG.getNode(RISCVISD::SHL_VL, DL, WideContainerVT,
4856 Interleaved = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideContainerVT, EvenV,
4861 VecContainerVT, DL, DAG.getAllOnesConstant(DL, Subtarget.getXLenVT()));
4862 SDValue OddsMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideContainerVT,
4870 Interleaved = DAG.getNode(RISCVISD::ADD_VL, DL, WideContainerVT,
4896 SDLoc DL(SVN);
4925 V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ViaBitVT, DAG.getUNDEF(ViaBitVT),
4926 V, DAG.getVectorIdxConstant(0, DL));
4929 DAG.getNode(ISD::BITREVERSE, DL, ViaVT, DAG.getBitcast(ViaVT, V));
4934 Res = DAG.getNode(ISD::SRL, DL, ViaVT, Res,
4935 DAG.getConstant(ViaEltSize - NumElts, DL, ViaVT));
4940 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
4941 DAG.getVectorIdxConstant(0, DL));
4949 SDLoc DL(SVN);
4971 SDLoc DL(SVN);
4985 Rotate = DAG.getNode(ISD::BSWAP, DL, RotateVT, Op);
4987 Rotate = DAG.getNode(ISD::ROTL, DL, RotateVT, Op,
4988 DAG.getConstant(RotateAmt, DL, RotateVT));
4999 SDLoc DL(SVN);
5066 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, SrcVec,
5067 DAG.getVectorIdxConstant(ExtractIdx, DL));
5069 SubVec = DAG.getVectorShuffle(OneRegVT, DL, SubVec, SubVec, SrcSubMask);
5072 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubVec,
5073 DAG.getVectorIdxConstant(InsertIdx, DL));
5082 SDLoc DL(Op);
5098 V1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V1);
5100 : DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V2);
5101 SDValue Shuffled = DAG.getVectorShuffle(WidenVT, DL, V1, V2, SVN->getMask());
5102 return DAG.getSetCC(DL, VT, Shuffled, DAG.getConstant(0, DL, WidenVT),
5108 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5133 Ld->getBasePtr(), TypeSize::getFixed(Offset), DL);
5139 DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
5147 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
5166 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
5171 V = DAG.getExtLoad(ISD::EXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
5180 DAG.getNode(Opc, DL, SplatVT, DAG.getUNDEF(ContainerVT), V, VL);
5187 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT,
5188 V1, DAG.getConstant(Lane, DL, XLenVT),
5202 lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG))
5206 lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG))
5238 Res = getVSlidedown(DAG, Subtarget, DL, ContainerVT, Res, HiV,
5239 DAG.getConstant(Rotation, DL, XLenVT), TrueMask, VL);
5242 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, Res, LoV,
5243 DAG.getConstant(InvRotate, DL, XLenVT), TrueMask, VL,
5252 return getDeinterleaveViaVNSRL(DL, VT, V1.getOperand(0), Mask[0] == 0,
5257 lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG))
5271 EvenV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, EvenV,
5272 DAG.getVectorIdxConstant(EvenSrc % Size, DL));
5276 OddV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, OddV,
5277 DAG.getVectorIdxConstant(OddSrc % Size, DL));
5279 return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget);
5330 ? DAG.getConstant(MaskIndex, DL, XLenVT)
5333 SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
5336 SDValue Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
5362 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), ShuffleMaskLHS);
5363 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), ShuffleMaskRHS);
5368 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
5373 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
5376 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
5377 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V2, V1);
5408 SDLoc DL(Op);
5437 SDValue Neg = DAG.getNegative(Src, DL, VT);
5438 Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
5440 SDValue Neg = DAG.getNode(ISD::VP_SUB, DL, VT, DAG.getConstant(0, DL, VT),
5442 Src = DAG.getNode(ISD::VP_AND, DL, VT, Src, Neg, Mask, VL);
5449 FloatVal = DAG.getNode(ISD::VP_UINT_TO_FP, DL, FloatVT, Src, Mask, VL);
5451 FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
5459 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5461 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT());
5464 FloatVal = DAG.getNode(RISCVISD::VFCVT_RM_F_XU_VL, DL, ContainerFloatVT,
5477 Exp = DAG.getNode(ISD::VP_SRL, DL, IntVT, Bitcast,
5478 DAG.getConstant(ShiftAmt, DL, IntVT), Mask, VL);
5479 Exp = DAG.getVPZExtOrTrunc(DL, VT, Exp, Mask, VL);
5481 Exp = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
5482 DAG.getConstant(ShiftAmt, DL, IntVT));
5484 Exp = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Exp);
5486 Exp = DAG.getNode(ISD::TRUNCATE, DL, VT, Exp);
5493 return DAG.getNode(ISD::SUB, DL, VT, Exp,
5494 DAG.getConstant(ExponentBias, DL, VT));
5496 return DAG.getNode(ISD::VP_SUB, DL, VT, Exp,
5497 DAG.getConstant(ExponentBias, DL, VT), Mask, VL);
5504 Res = DAG.getNode(ISD::VP_SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp,
5507 Res = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp);
5512 Res = DAG.getNode(ISD::UMIN, DL, VT, Res, DAG.getConstant(EltSize, DL, VT));
5514 Res = DAG.getNode(ISD::VP_UMIN, DL, VT, Res,
5515 DAG.getConstant(EltSize, DL, VT), Mask, VL);
5521 SDLoc DL(Op);
5538 SDValue AllZero = DAG.getConstant(0, DL, SrcVT);
5540 Source = DAG.getNode(RISCVISD::SETCC_VL, DL, SrcVT,
5545 SDValue Res = DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Source, Mask, EVL);
5552 DAG.getSetCC(DL, XLenVT, Res, DAG.getConstant(0, DL, XLenVT), ISD::SETLT);
5553 Res = DAG.getSelect(DL, XLenVT, SetCC, EVL, Res);
5554 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
5571 SDLoc DL(Op);
5580 SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
5583 return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
5601 SDLoc DL(Op);
5612 return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
5692 SDLoc DL(Op);
5693 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(0));
5694 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(1));
5696 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
5700 SDValue SatMin = DAG.getConstant(MinVal, DL, MVT::i64);
5701 SDValue SatMax = DAG.getConstant(MaxVal, DL, MVT::i64);
5702 Result = DAG.getNode(ISD::SMIN, DL, MVT::i64, Result, SatMax);
5703 Result = DAG.getNode(ISD::SMAX, DL, MVT::i64, Result, SatMin);
5704 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Result);
5714 SDLoc DL(Op);
5715 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(0));
5716 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(1));
5717 SDValue WideOp = DAG.getNode(Op.getOpcode(), DL, MVT::i64, LHS, RHS);
5718 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, WideOp);
5729 SDLoc DL(Op);
5730 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(0));
5731 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(1));
5733 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
5734 SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, WideOp);
5735 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, WideOp,
5737 SDValue Ovf = DAG.getSetCC(DL, Op.getValue(1).getValueType(), WideOp, SExt,
5739 return DAG.getMergeValues({Res, Ovf}, DL);
5746 SDLoc DL(Op);
5747 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(0));
5748 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(1));
5749 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
5750 SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul);
5751 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Mul,
5753 SDValue Ovf = DAG.getSetCC(DL, Op.getValue(1).getValueType(), Mul, SExt,
5755 return DAG.getMergeValues({Res, Ovf}, DL);
5760 SDLoc DL(Op);
5788 SDValue TDCMaskV = DAG.getConstant(TDCMask, DL, XLenVT);
5796 auto [Mask, VL] = getDefaultScalableVLOps(VT0, DL, DAG, Subtarget);
5801 SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, DstVT, Op0, Mask,
5804 return DAG.getSetCC(DL, VT, FPCLASS,
5805 DAG.getConstant(TDCMask, DL, DstVT),
5807 SDValue AND = DAG.getNode(ISD::AND, DL, DstVT, FPCLASS,
5808 DAG.getConstant(TDCMask, DL, DstVT));
5809 return DAG.getSetCC(DL, VT, AND, DAG.getConstant(0, DL, DstVT),
5816 auto [Mask, VL] = getDefaultVLOps(VT0, ContainerVT0, DL, DAG, Subtarget);
5826 SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, ContainerDstVT, Op0,
5829 TDCMaskV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
5833 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
5838 SDValue AND = DAG.getNode(RISCVISD::AND_VL, DL, ContainerDstVT, FPCLASS,
5841 SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
5842 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
5845 SDValue VMSNE = DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
5851 SDValue FCLASS = DAG.getNode(RISCVISD::FCLASS, DL, XLenVT, Op.getOperand(0));
5852 SDValue AND = DAG.getNode(ISD::AND, DL, XLenVT, FCLASS, TDCMaskV);
5853 SDValue Res = DAG.getSetCC(DL, XLenVT, AND, DAG.getConstant(0, DL, XLenVT),
5855 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
5862 SDLoc DL(Op);
5878 SDValue XIsNonNan = DAG.getSetCC(DL, XLenVT, X, X, ISD::SETOEQ);
5879 NewY = DAG.getSelect(DL, VT, XIsNonNan, Y, X);
5884 SDValue YIsNonNan = DAG.getSetCC(DL, XLenVT, Y, Y, ISD::SETOEQ);
5885 NewX = DAG.getSelect(DL, VT, YIsNonNan, X, Y);
5890 return DAG.getNode(Opc, DL, VT, NewX, NewY);
5912 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5917 SDValue XIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
5920 NewY = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, XIsNonNan, Y, X,
5926 SDValue YIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
5929 NewX = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, YIsNonNan, X, Y,
5937 SDValue Res = DAG.getNode(Opc, DL, ContainerVT, NewX, NewY,
6139 SDLoc DL(Op);
6151 DAG.SplitVector(Op.getOperand(j), DL);
6155 DAG.getNode(Op.getOpcode(), DL, LoVT, LoOperands, Op->getFlags());
6157 DAG.getNode(Op.getOpcode(), DL, HiVT, HiOperands, Op->getFlags());
6159 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes);
6165 SDLoc DL(Op);
6173 DAG.SplitEVL(Op.getOperand(j), Op.getValueType(), DL);
6182 DAG.SplitVector(Op.getOperand(j), DL);
6186 DAG.getNode(Op.getOpcode(), DL, LoVT, LoOperands, Op->getFlags());
6188 DAG.getNode(Op.getOpcode(), DL, HiVT, HiOperands, Op->getFlags());
6190 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes);
6194 SDLoc DL(Op);
6196 auto [Lo, Hi] = DAG.SplitVector(Op.getOperand(1), DL);
6197 auto [MaskLo, MaskHi] = DAG.SplitVector(Op.getOperand(2), DL);
6199 DAG.SplitEVL(Op.getOperand(3), Op.getOperand(1).getValueType(), DL);
6202 DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
6204 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
6217 SDLoc DL(Op);
6229 DAG.SplitVector(Op.getOperand(j), DL);
6233 DAG.getNode(Op.getOpcode(), DL, LoVTs, LoOperands, Op->getFlags());
6236 DAG.getNode(Op.getOpcode(), DL, HiVTs, HiOperands, Op->getFlags());
6238 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, Op->getValueType(0),
6240 return DAG.getMergeValues({V, HiRes.getValue(1)}, DL);
6297 SDLoc DL(Op);
6304 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
6305 SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, NewOp0);
6310 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
6311 SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::bf16, NewOp0);
6316 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
6318 DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
6324 std::tie(Lo, Hi) = DAG.SplitScalar(Op0, DL, MVT::i32, MVT::i32);
6326 DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
6353 return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT,
6355 DAG.getVectorIdxConstant(0, DL)));
6367 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
6368 DAG.getVectorIdxConstant(0, DL));
6386 SDLoc DL(Op);
6390 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Op.getOperand(0));
6391 return DAG.getNode(RISCVISD::BREV8, DL, VT, BSwap);
6417 SDLoc DL(Op);
6421 SDValue V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, WideVT, Scalar);
6422 return DAG.getNode(ISD::TRUNCATE, DL, VT, V);
6427 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
6428 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Scalar);
6429 SDValue V = DAG.getNode(RISCVISD::VMV_S_X_VL, DL, ContainerVT,
6438 SDLoc DL(Op);
6439 SDValue Res = DAG.getNode(RISCVISD::READ_VLENB, DL, XLenVT);
6452 Res = DAG.getNode(ISD::SRL, DL, XLenVT, Res,
6453 DAG.getConstant(3 - Log2, DL, VT));
6455 Res = DAG.getNode(ISD::SHL, DL, XLenVT, Res,
6456 DAG.getConstant(Log2 - 3, DL, XLenVT));
6460 Res = DAG.getNode(ISD::MUL, DL, XLenVT, Res,
6461 DAG.getConstant(Val / 8, DL, XLenVT));
6463 SDValue VScale = DAG.getNode(ISD::SRL, DL, XLenVT, Res,
6464 DAG.getConstant(3, DL, XLenVT));
6465 Res = DAG.getNode(ISD::MUL, DL, XLenVT, VScale,
6466 DAG.getConstant(Val, DL, XLenVT));
6468 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
6475 SDLoc DL(Op);
6476 SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
6478 DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1));
6479 return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi,
6480 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
6492 SDLoc DL(Op);
6497 return DAG.getNode(RISCVISD::FP_EXTEND_BF16, DL, MVT::f32, Op0);
6500 DAG.getNode(RISCVISD::FP_EXTEND_BF16, DL, MVT::f32, Op0);
6501 return DAG.getNode(ISD::FP_EXTEND, DL, MVT::f64, FloatVal);
6509 SDLoc DL(Op);
6514 return DAG.getNode(RISCVISD::FP_ROUND_BF16, DL, MVT::bf16, Op0);
6518 DAG.getNode(ISD::FP_ROUND, DL, MVT::f32, Op0,
6519 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
6520 return DAG.getNode(RISCVISD::FP_ROUND_BF16, DL, MVT::bf16, FloatVal);
6539 SDLoc DL(Op);
6542 SDValue NC = DAG.getNode(Op.getOpcode(), DL, NVT, Op->ops());
6544 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NC,
6545 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
6558 SDLoc DL(Op);
6561 SDValue WidenVec = DAG.getNode(ISD::FP_EXTEND, DL, NVT, Op1);
6563 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), WidenVec);
6576 SDLoc DL(Op);
6598 SDValue Ext = DAG.getNode(ExtOpcode, DL, IVecVT, Src);
6600 return DAG.getNode(Op.getOpcode(), DL, Op->getVTList(),
6602 return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
6611 DAG.getStrictFPExtendOrRound(Src, Op.getOperand(0), DL, InterimFVT);
6612 return DAG.getNode(Op.getOpcode(), DL, Op->getVTList(), Chain, FExt);
6614 SDValue FExt = DAG.getFPExtendOrRound(Src, DL, InterimFVT);
6615 return DAG.getNode(Op.getOpcode(), DL, VT, FExt);
6625 SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL,
6629 return DAG.getStrictFPExtendOrRound(Int2FP, Chain, DL, VT).first;
6631 SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, InterimFVT, Src);
6632 return DAG.getFPExtendOrRound(Int2FP, DL, VT);
6641 DAG.getNode(Op.getOpcode(), DL, DAG.getVTList(IVecVT, MVT::Other),
6643 SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
6644 return DAG.getMergeValues({Res, FP2Int.getValue(1)}, DL);
6646 SDValue FP2Int = DAG.getNode(Op.getOpcode(), DL, IVecVT, Src);
6647 return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
6691 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
6695 Src = DAG.getNode(RVVOpc, DL, DAG.getVTList(ContainerVT, MVT::Other),
6698 return DAG.getMergeValues({SubVec, Src.getValue(1)}, DL);
6700 Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL);
6710 SDLoc DL(Op);
6715 makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first;
6717 return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Res);
6723 SDLoc DL(Op);
6725 ISD::SHL, DL, Op.getOperand(0).getValueType(), Op.getOperand(0),
6726 DAG.getShiftAmountConstant(16, Op.getOperand(0).getValueType(), DL));
6728 ? DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Op)
6732 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Res);
6739 SDLoc DL(Op);
6744 makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first;
6746 return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Res);
6753 SDLoc DL(Op);
6756 ? DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32,
6760 makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg, CallOptions, DL)
6850 SDLoc DL(Op);
6852 DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
6854 ISD::SPLAT_VECTOR, DL,
6857 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NewSplat,
6858 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
6868 SDLoc DL(Op);
6887 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT,
6889 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT,
6891 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
6903 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, SubVec,
6904 DAG.getVectorIdxConstant(OpIdx.index() * NumOpElts, DL));
6939 SDLoc DL(Op);
6941 DAG.getNode(ISD::SETCC, DL, CCVT, Tmp1, Tmp2, CC, Op->getFlags());
6942 return DAG.getSelect(DL, VT, Cond, True, False);
6954 SDLoc DL(Op);
6966 return DAG.getConstant(0, DL, VT);
6970 DL, VT, LHS, DAG.getConstant(Imm + 1, DL, OpVT), CCVal);
6971 return DAG.getLogicalNOT(DL, SetCC, VT);
6978 return DAG.getSetCC(DL, VT, RHS, LHS, CCVal);
7176 SDLoc DL(Op);
7179 auto NC = DAG.getNode(Op.getOpcode(), DL, NVT, Op->ops());
7181 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NC,
7182 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
7195 SDLoc DL(Op);
7198 SDValue WidenVec = DAG.getNode(ISD::FP_EXTEND, DL, NVT, Op1);
7200 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
7263 SDLoc DL(Op);
7264 SDValue Flags = DAG.getConstant(0, DL, Subtarget.getXLenVT());
7266 Op.getOperand(2), Flags, DL);
7273 SDValue Flags, SDLoc DL) const {
7277 {Start, End, Flags}, CallOptions, DL, InChain);
7283 static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty,
7285 return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
7288 static SDValue getTargetNode(BlockAddressSDNode *N, const SDLoc &DL, EVT Ty,
7294 static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty,
7300 static SDValue getTargetNode(JumpTableSDNode *N, const SDLoc &DL, EVT Ty,
7308 SDLoc DL(N);
7316 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
7321 return DAG.getNode(RISCVISD::LLA, DL, Ty, Addr);
7327 SDValue(DAG.getMachineNode(RISCV::PseudoLGA, DL, Ty, Addr), 0);
7344 SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
7345 SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
7346 SDValue MNHi = DAG.getNode(RISCVISD::HI, DL, Ty, AddrHi);
7347 return DAG.getNode(RISCVISD::ADD_LO, DL, Ty, MNHi, AddrLo);
7350 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
7357 SDValue(DAG.getMachineNode(RISCV::PseudoLGA, DL, Ty, Addr), 0);
7371 return DAG.getNode(RISCVISD::LLA, DL, Ty, Addr);
7408 SDLoc DL(N);
7418 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
7420 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
7431 return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
7439 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
7441 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
7443 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
7445 SDValue MNHi = DAG.getNode(RISCVISD::HI, DL, Ty, AddrHi);
7448 DAG.getNode(RISCVISD::ADD_TPREL, DL, Ty, MNHi, TPReg, AddrAdd);
7449 return DAG.getNode(RISCVISD::ADD_LO, DL, Ty, MNAdd, AddrLo);
7454 SDLoc DL(N);
7462 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
7464 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
7475 CLI.setDebugLoc(DL)
7486 SDLoc DL(N);
7497 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
7498 return SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLSDESC, DL, Ty, Addr), 0);
7565 SDLoc DL(N);
7570 SDValue Neg = DAG.getNegative(CondV, DL, VT);
7571 return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(FalseV));
7575 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV,
7576 DAG.getAllOnesConstant(DL, VT));
7577 return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(TrueV));
7582 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV,
7583 DAG.getAllOnesConstant(DL, VT));
7584 return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(FalseV));
7588 SDValue Neg = DAG.getNegative(CondV, DL, VT);
7589 return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(TrueV));
7598 SDValue Neg = DAG.getNegative(CondV, DL, VT);
7599 return DAG.getNode(ISD::XOR, DL, VT, Neg, FalseV);
7614 return DAG.getNode(*MatchResult ? ISD::OR : ISD::AND, DL, VT, TrueV,
7620 return DAG.getNode(*MatchResult ? ISD::AND : ISD::OR, DL, VT,
7666 SDLoc DL(Sel);
7674 DAG.FoldConstantArithmetic(BO->getOpcode(), DL, VT, NewConstOps);
7686 SDValue NewNonConstOp = DAG.getNode(BO->getOpcode(), DL, VT, NewNonConstOps);
7690 return DAG.getSelect(DL, VT, Sel.getOperand(0), NewT, NewF);
7697 SDLoc DL(Op);
7704 SDValue CondSplat = DAG.getSplat(SplatCondVT, DL, CondV);
7705 return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
7716 return DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV);
7719 return DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV);
7725 ISD::OR, DL, VT, TrueV,
7726 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV));
7731 ISD::OR, DL, VT, FalseV,
7732 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV));
7749 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal, DL, VT);
7751 DAG.getConstant(IsCZERO_NEZ ? TrueVal : FalseVal, DL, VT);
7754 DL, VT, LHSVal, CondV);
7755 return DAG.getNode(ISD::ADD, DL, VT, CMOV, RHSVal);
7762 ISD::OR, DL, VT,
7763 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV),
7764 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV));
7792 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, CondV);
7794 SDValue XOR = DAG.getNode(ISD::XOR, DL, XLenVT, CondV,
7795 DAG.getConstant(1, DL, XLenVT));
7796 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, XOR);
7806 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
7811 return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops);
7835 return DAG.getNode(ISD::ADD, DL, VT, CondV, FalseV);
7837 return DAG.getNode(ISD::SUB, DL, VT, FalseV, CondV);
7840 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
7844 LHS = DAG.getConstant(0, DL, VT);
7855 RHS = DAG.getConstant(0, DL, VT);
7868 return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops);
7873 SDLoc DL(Op);
7882 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
7885 return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
7889 return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
7890 CondV, DAG.getConstant(0, DL, XLenVT),
7898 SDLoc DL(Op);
7905 return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
7919 SDLoc DL(Op);
7920 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
7924 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
7925 DAG.getIntPtrConstant(Offset, DL));
7927 DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
7945 SDLoc DL(Op);
7950 SDValue Offset = DAG.getConstant(Off, DL, VT);
7951 return DAG.getLoad(VT, DL, DAG.getEntryNode(),
7952 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
7959 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
7964 SDLoc DL(Op);
7977 SDValue Zero = DAG.getConstant(0, DL, VT);
7978 SDValue One = DAG.getConstant(1, DL, VT);
7979 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
7980 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
7981 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
7982 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
7984 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
7985 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
7987 DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
7988 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
7989 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
7990 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
7992 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
7994 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
7995 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
7998 return DAG.getMergeValues(Parts, DL);
8003 SDLoc DL(Op);
8027 SDValue Zero = DAG.getConstant(0, DL, VT);
8028 SDValue One = DAG.getConstant(1, DL, VT);
8029 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
8030 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
8031 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
8032 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
8034 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
8035 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
8037 DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
8038 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
8039 SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
8040 SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
8042 IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
8044 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
8046 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
8047 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
8050 return DAG.getMergeValues(Parts, DL);
8057 SDLoc DL(Op);
8062 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
8063 return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL);
8066 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
8067 return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL);
8070 SplatVal = DAG.getNode(ISD::AND, DL, SplatVal.getValueType(), SplatVal,
8071 DAG.getConstant(1, DL, SplatVal.getValueType()));
8072 SDValue LHS = DAG.getSplatVector(InterVT, DL, SplatVal);
8073 SDValue Zero = DAG.getConstant(0, DL, InterVT);
8074 return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE);
8083 SDLoc DL(Op);
8096 auto VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
8099 splatPartsI64WithVL(DL, ContainerVT, SDValue(), Lo, Hi, VL, DAG);
8113 SDLoc DL(Op);
8121 SDValue SplatZero = DAG.getConstant(0, DL, VecVT);
8122 SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, VecVT);
8123 return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
8132 SDValue VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
8135 SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
8136 SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, XLenVT);
8138 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
8140 SplatTrueVal = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
8143 DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, SplatTrueVal,
8168 SDLoc DL(Op);
8169 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
8171 SDValue Ext = DAG.getNode(ExtendOpc, DL, ContainerExtVT, Op1, Mask, VL);
8182 SDLoc DL(Op);
8209 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
8212 SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT());
8213 SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
8215 SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
8217 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
8221 SDValue Trunc = DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne,
8223 Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT,
8234 SDLoc DL(Op);
8275 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
8283 Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, ResultVT, Result,
8296 SDLoc DL(Op);
8309 auto [Mask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
8323 Src = DAG.getNode(InterConvOpc, DL, DAG.getVTList(InterVT, MVT::Other),
8331 SDValue Res = DAG.getNode(ConvOpc, DL, DAG.getVTList(ContainerVT, MVT::Other),
8337 Res = DAG.getMergeValues({SubVec, Res.getValue(1)}, DL);
8352 SDLoc DL(Op);
8390 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
8395 Src = DAG.getNode(ConvOpc, DL, ContainerVT, Src, Mask, VL);
8406 DAG.getNode(InterConvOpc, DL, InterVT, Src, Mask, VL);
8408 DAG.getNode(ConvOpc, DL, ContainerVT, IntermediateConv, Mask, VL);
8420 getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG,
8448 SDLoc DL(Op);
8458 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
8459 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx);
8460 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec);
8479 DL, DAG, Subtarget)) {
8481 AlignedIdx = DAG.getVectorIdxConstant(0, DL);
8496 AlignedIdx = DAG.getVectorIdxConstant(ExtractIdx, DL);
8497 Idx = DAG.getVectorIdxConstant(RemIdx, DL);
8502 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec,
8517 Val = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
8521 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
8530 Val = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Val);
8531 Vec = DAG.getNode(Opc, DL, ContainerVT, Vec, Val, VL);
8534 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec,
8540 ValInVec = lowerScalarInsert(Val, VL, ContainerVT, DL, DAG, Subtarget);
8547 std::tie(ValLo, ValHi) = DAG.SplitScalar(Val, DL, MVT::i32, MVT::i32);
8551 getDefaultScalableVLOps(I32ContainerVT, DL, DAG, Subtarget).first;
8553 SDValue InsertI64VL = DAG.getConstant(2, DL, XLenVT);
8558 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
8563 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
8570 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec,
8579 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
8583 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
8592 DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT));
8599 SDValue Slideup = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, ValInVec,
8603 Slideup = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec,
8616 SDLoc DL(Op);
8631 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
8633 DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Vec, Mask, VL);
8634 SDValue Res = DAG.getSetCC(DL, XLenVT, Vfirst,
8635 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
8636 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res);
8653 ExtractElementIdx = DAG.getConstant(0, DL, XLenVT);
8660 ISD::SRL, DL, XLenVT, Idx,
8661 DAG.getConstant(Log2_64(WideEltVT.getSizeInBits()), DL, XLenVT));
8664 ISD::AND, DL, XLenVT, Idx,
8665 DAG.getConstant(WideEltVT.getSizeInBits() - 1, DL, XLenVT));
8668 Vec = DAG.getNode(ISD::BITCAST, DL, WideVT, Vec);
8669 SDValue ExtractElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, XLenVT,
8673 DAG.getNode(ISD::SRL, DL, XLenVT, ExtractElt, ExtractBitIdx);
8674 SDValue Res = DAG.getNode(ISD::AND, DL, XLenVT, ShiftRight,
8675 DAG.getConstant(1, DL, XLenVT));
8676 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res);
8681 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
8682 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx);
8707 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Vec,
8708 DAG.getVectorIdxConstant(ExtractIdx, DL));
8709 Idx = DAG.getVectorIdxConstant(RemIdx, DL);
8722 getSmallestVTForIndex(ContainerVT, *MaxIdx, DL, DAG, Subtarget)) {
8724 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec,
8725 DAG.getConstant(0, DL, XLenVT));
8748 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget);
8749 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT,
8755 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec,
8756 DAG.getVectorIdxConstant(0, DL));
8759 SDValue Elt0 = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
8760 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0);
8779 SDLoc DL(Op);
8806 ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
8807 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
8826 ScalarOp = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, ScalarOp);
8827 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
8844 DAG.SplitScalar(ScalarOp, DL, MVT::i32, MVT::i32);
8857 I32VL = DAG.getConstant(2 * AVLInt, DL, XLenVT);
8861 SDValue LMUL = DAG.getConstant(Lmul, DL, XLenVT);
8863 SDValue SEW = DAG.getConstant(Sew, DL, XLenVT);
8865 Intrinsic::riscv_vsetvlimax, DL, MVT::i32);
8866 I32VL = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, SETVLMAX, SEW,
8876 SDValue LMUL = DAG.getConstant(Lmul, DL, XLenVT);
8878 SDValue SEW = DAG.getConstant(Sew, DL, XLenVT);
8880 DAG.getTargetConstant(Intrinsic::riscv_vsetvli, DL, MVT::i32);
8883 SDValue VL = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, SETVL, AVL,
8886 DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT));
8889 SDValue I32Mask = getAllOnesMask(I32VT, I32VL, DL, DAG);
8901 Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Passthru, Vec,
8903 Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Passthru, Vec,
8906 Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Passthru, Vec,
8908 Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Passthru, Vec,
8927 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff,
8931 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff,
8939 ScalarOp = splatSplitI64WithVL(DL, VT, SDValue(), ScalarOp, VL, DAG);
8940 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
8975 SDLoc DL(N);
8977 SDValue LMul = DAG.getTargetConstant(VLMUL, DL, XLenVT);
8978 SDValue Sew = DAG.getTargetConstant(VSEW, DL, XLenVT);
8980 SDValue AVL = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, N->getOperand(1));
8982 SDValue ID = DAG.getTargetConstant(Intrinsic::riscv_vsetvli, DL, XLenVT);
8984 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, ID, AVL, Sew, LMul);
8985 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res);
8998 SDLoc DL(N);
8999 auto [Mask, VL] = getDefaultVLOps(OpVT, ContainerVT, DL, DAG, Subtarget);
9000 SDValue Res = DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Op0, Mask, VL);
9006 DAG.getSetCC(DL, XLenVT, Res, DAG.getConstant(0, DL, XLenVT), ISD::SETLT);
9007 VL = DAG.getElementCount(DL, XLenVT, OpVT.getVectorElementCount());
9008 return DAG.getSelect(DL, XLenVT, Setcc, VL, Res);
9020 SDLoc DL(Op);
9043 ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
9082 SDLoc DL(Op);
9114 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9115 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp);
9116 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9119 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1));
9128 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9130 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2));
9132 DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, Op.getOperand(3));
9133 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9136 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2),
9143 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1));
9148 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9150 RISCVISD::MOPR, DL, MVT::i64, NewOp,
9151 DAG.getTargetConstant(Op.getConstantOperandVal(2), DL, MVT::i64));
9152 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9154 return DAG.getNode(RISCVISD::MOPR, DL, XLenVT, Op.getOperand(1),
9161 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9163 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2));
9165 RISCVISD::MOPRR, DL, MVT::i64, NewOp0, NewOp1,
9166 DAG.getTargetConstant(Op.getConstantOperandVal(3), DL, MVT::i64));
9167 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9169 return DAG.getNode(RISCVISD::MOPRR, DL, XLenVT, Op.getOperand(1),
9175 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9177 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2));
9178 SDValue Res = DAG.getNode(RISCVISD::CLMUL, DL, MVT::i64, NewOp0, NewOp1);
9179 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9181 return DAG.getNode(RISCVISD::CLMUL, DL, XLenVT, Op.getOperand(1),
9189 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9191 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2));
9192 NewOp0 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0,
9193 DAG.getConstant(32, DL, MVT::i64));
9194 NewOp1 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp1,
9195 DAG.getConstant(32, DL, MVT::i64));
9196 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1);
9197 Res = DAG.getNode(ISD::SRL, DL, MVT::i64, Res,
9198 DAG.getConstant(32, DL, MVT::i64));
9199 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9202 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
9209 SDValue Res = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Op.getOperand(1));
9210 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
9213 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getValueType(),
9214 Op.getOperand(1), DAG.getVectorIdxConstant(0, DL));
9217 Op.getOperand(3), Op.getSimpleValueType(), DL, DAG,
9220 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, Op.getValueType(),
9226 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar);
9227 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, Op.getValueType(),
9252 SDValue SplattedVal = splatSplitI64WithVL(DL, VT, SDValue(), Scalar, VL, DAG);
9256 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
9257 DAG.getConstant(0, DL, MVT::i32), VL);
9260 SDValue Mask = getAllOnesMask(VT, VL, DL, DAG);
9261 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
9263 DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT,
9266 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, SelectCond, SplattedVal,
9270 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, Op.getSimpleValueType(),
9340 SDValue NewNode = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, RetVT, Operands);
9359 SDLoc DL(Op);
9381 SDValue NewNode = DAG.getNode(Type, DL, VTs, Operands);
9389 NewNode = DAG.getMergeValues({NewNode, Chain}, DL);
9410 SDLoc DL(Op);
9433 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
9443 DAG.getExtLoad(ISD::EXTLOAD, DL, XLenVT, Load->getChain(), Ptr,
9446 Result = lowerScalarSplat(SDValue(), ScalarLoad, VL, ContainerVT, DL, DAG,
9449 SDValue ScalarLoad = DAG.getLoad(ScalarVT, DL, Load->getChain(), Ptr,
9452 Result = DAG.getSplat(ContainerVT, DL, ScalarLoad);
9455 IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask, DL,
9470 DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT);
9476 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
9482 return DAG.getMergeValues({Result, Chain}, DL);
9491 SDLoc DL(Op);
9503 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
9505 SDValue IntID = DAG.getTargetConstant(VlsegInts[NF - 2], DL, XLenVT);
9515 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
9522 return DAG.getMergeValues(Results, DL);
9564 SDLoc DL(Op);
9585 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
9588 IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask, DL,
9600 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, Store->getVTList(),
9611 SDLoc DL(Op);
9624 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
9626 SDValue IntID = DAG.getTargetConstant(VssegInts[NF - 2], DL, XLenVT);
9637 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
9714 SDLoc DL(Op);
9739 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
9744 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
9752 SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
9753 Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec, TrueMask, VL);
9754 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
9762 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
9769 SDValue One = DAG.getConstant(1, DL, XLenVT);
9770 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
9771 Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One);
9778 SDValue SetCC = DAG.getSetCC(DL, XLenVT, Vec, Zero, CC);
9779 SetCC = DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), SetCC);
9790 return DAG.getNode(BaseOpc, DL, Op.getValueType(), SetCC, Op.getOperand(0));
9804 SDValue VL, const SDLoc &DL, SelectionDAG &DAG,
9817 auto InnerVL = NonZeroAVL ? VL : DAG.getConstant(1, DL, XLenVT);
9818 SDValue InitialValue = lowerScalarInsert(StartValue, InnerVL, InnerVT, DL,
9822 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, M1VT, DAG.getUNDEF(M1VT),
9823 InitialValue, DAG.getVectorIdxConstant(0, DL));
9825 SDValue Policy = DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT);
9827 SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, Ops);
9828 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Reduction,
9829 DAG.getVectorIdxConstant(0, DL));
9834 SDLoc DL(Op);
9844 auto [Lo, Hi] = DAG.SplitVector(Vec, DL);
9846 Vec = DAG.getNode(BaseOpc, DL, VecEVT, Lo, Hi);
9864 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
9866 SDValue StartV = DAG.getNeutralElement(BaseOpc, DL, VecEltVT, SDNodeFlags());
9874 StartV = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Vec,
9875 DAG.getVectorIdxConstant(0, DL));
9878 Mask, VL, DL, DAG, Subtarget);
9887 SDLoc DL(Op);
9896 DAG.getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, EltVT);
9907 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op.getOperand(0),
9908 DAG.getVectorIdxConstant(0, DL));
9920 SDLoc DL(Op);
9936 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
9938 VL, DL, DAG, Subtarget);
9948 DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
9952 SDValue CPop = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, IsNan, Mask, VL);
9953 SDValue NoNaNs = DAG.getSetCC(DL, XLenVT, CPop,
9954 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
9956 DL, ResVT, NoNaNs, Res,
9957 DAG.getConstantFP(APFloat::getNaN(DAG.EVTToAPFloatSemantics(ResVT)), DL,
9963 SDLoc DL(Op);
9987 Vec, Mask, VL, DL, DAG, Subtarget);
9996 RISCVISD::SETCC_VL, DL, PredVT,
9998 SDValue VCPop = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, IsNaN, Mask, VL);
10000 SDValue StartIsNaN = DAG.getSetCC(DL, XLenVT, Start, Start, ISD::SETUO);
10001 VCPop = DAG.getNode(ISD::OR, DL, XLenVT, VCPop, StartIsNaN);
10002 SDValue NoNaNs = DAG.getSetCC(DL, XLenVT, VCPop,
10003 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
10006 DL, ResVT, NoNaNs, Res,
10007 DAG.getConstantFP(APFloat::getNaN(DAG.EVTToAPFloatSemantics(ResVT)), DL,
10018 SDLoc DL(Op);
10051 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
10052 SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec);
10053 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec,
10055 SDValue SplatZero = DAG.getConstant(0, DL, ExtVecVT);
10056 return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE);
10076 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
10078 DAG.getVectorIdxConstant(0, DL));
10083 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
10085 DAG.getVectorIdxConstant(0, DL));
10087 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
10091 SDValue VL = getVLOp(EndIndex, ContainerVT, DL, DAG, Subtarget);
10102 DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, Vec, SubVec, VL);
10104 SDValue SlideupAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
10105 SubVec = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, SubVec,
10178 DAG.getTargetInsertSubreg(SubRegIdx, DL, ContainerVecVT, Vec, SubVec);
10200 AlignedExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
10201 DAG.getVectorIdxConstant(AlignedIdx, DL));
10204 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InterSubVT,
10206 DAG.getVectorIdxConstant(0, DL));
10208 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVecVT, DL, DAG, Subtarget);
10211 VL = DAG.getElementCount(DL, XLenVT, SubVecVT.getVectorElementCount());
10222 SubVec = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, InterSubVT, AlignedExtract,
10225 SDValue SlideupAmt = DAG.getElementCount(DL, XLenVT, RemIdx);
10228 VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL);
10230 SubVec = getVSlideup(DAG, Subtarget, DL, InterSubVT, AlignedExtract, SubVec,
10237 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVecVT, Vec, SubVec,
10238 DAG.getVectorIdxConstant(AlignedIdx, DL));
10254 SDLoc DL(Op);
10288 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
10289 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec,
10291 SDValue SplatZero = DAG.getConstant(0, DL, ExtSubVecVT);
10292 return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE);
10318 getSmallestVTForIndex(ContainerVT, LastIdx, DL, DAG, Subtarget)) {
10320 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec,
10321 DAG.getVectorIdxConstant(0, DL));
10325 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
10328 SDValue VL = getVLOp(SubVecVT.getVectorNumElements(), ContainerVT, DL, DAG,
10330 SDValue SlidedownAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
10332 getVSlidedown(DAG, Subtarget, DL, ContainerVT,
10335 Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
10336 DAG.getVectorIdxConstant(0, DL));
10376 Vec = DAG.getTargetExtractSubreg(SubRegIdx, DL, ContainerSubVecVT, Vec);
10396 Vec = DAG.getTargetExtractSubreg(SubRegIdx, DL, InterSubVT, Vec);
10401 SDValue SlidedownAmt = DAG.getElementCount(DL, XLenVT, RemIdx);
10402 auto [Mask, VL] = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget);
10404 VL = getVLOp(SubVecVT.getVectorNumElements(), InterSubVT, DL, DAG,
10407 getVSlidedown(DAG, Subtarget, DL, InterSubVT, DAG.getUNDEF(InterSubVT),
10412 Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
10413 DAG.getVectorIdxConstant(0, DL));
10422 static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL,
10430 WideOps.push_back(DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Op));
10437 SDValue WideN = DAG.getNode(N.getOpcode(), DL, VTs, WideOps);
10441 DAG.getSetCC(DL, N->getSimpleValueType(I), WideN.getValue(I),
10442 DAG.getConstant(0, DL, WideVT), ISD::SETNE));
10446 return DAG.getMergeValues(TruncVals, DL);
10452 SDLoc DL(Op);
10460 return widenVectorOpsToi8(Op, DL, DAG);
10469 SDValue ResLo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
10471 SDValue ResHi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
10474 SDValue Even = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
10476 SDValue Odd = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, ResLo.getValue(1),
10478 return DAG.getMergeValues({Even, Odd}, DL);
10485 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT,
10489 auto [Mask, VL] = getDefaultScalableVLOps(ConcatVT, DL, DAG, Subtarget);
10496 getDeinterleaveViaVNSRL(DL, VecVT, Concat, true, Subtarget, DAG);
10498 getDeinterleaveViaVNSRL(DL, VecVT, Concat, false, Subtarget, DAG);
10499 return DAG.getMergeValues({Even, Odd}, DL);
10506 DAG.getStepVector(DL, IdxVT, APInt(IdxVT.getScalarSizeInBits(), 2));
10509 DAG.getNode(ISD::ADD, DL, IdxVT, EvenIdx, DAG.getConstant(1, DL, IdxVT));
10512 SDValue EvenWide = DAG.getNode(RISCVISD::VRGATHER_VV_VL, DL, ConcatVT,
10514 SDValue OddWide = DAG.getNode(RISCVISD::VRGATHER_VV_VL, DL, ConcatVT,
10518 SDValue Even = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, EvenWide,
10519 DAG.getVectorIdxConstant(0, DL));
10520 SDValue Odd = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, OddWide,
10521 DAG.getVectorIdxConstant(0, DL));
10523 return DAG.getMergeValues({Even, Odd}, DL);
10528 SDLoc DL(Op);
10536 return widenVectorOpsToi8(Op, DL, DAG);
10547 SDValue ResLo = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
10549 SDValue ResHi = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
10552 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
10554 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
10556 return DAG.getMergeValues({Lo, Hi}, DL);
10564 Interleaved = getWideningInterleave(Op.getOperand(0), Op.getOperand(1), DL,
10571 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT,
10577 SDValue StepVec = DAG.getStepVector(DL, IdxVT);
10580 SDValue Ones = DAG.getSplatVector(IdxVT, DL, DAG.getConstant(1, DL, XLenVT));
10583 SDValue OddMask = DAG.getNode(ISD::AND, DL, IdxVT, StepVec, Ones);
10585 DL, IdxVT.changeVectorElementType(MVT::i1), OddMask,
10586 DAG.getSplatVector(IdxVT, DL, DAG.getConstant(0, DL, XLenVT)),
10589 SDValue VLMax = DAG.getSplatVector(IdxVT, DL, computeVLMax(VecVT, DL, DAG));
10593 SDValue Idx = DAG.getNode(ISD::SRL, DL, IdxVT, StepVec, Ones);
10596 DAG.getNode(RISCVISD::ADD_VL, DL, IdxVT, Idx, VLMax, Idx, OddMask, VL);
10600 SDValue TrueMask = getAllOnesMask(IdxVT, VL, DL, DAG);
10601 Interleaved = DAG.getNode(RISCVISD::VRGATHEREI16_VV_VL, DL, ConcatVT,
10606 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, Interleaved,
10607 DAG.getVectorIdxConstant(0, DL));
10609 ISD::EXTRACT_SUBVECTOR, DL, VecVT, Interleaved,
10610 DAG.getVectorIdxConstant(VecVT.getVectorMinNumElements(), DL));
10612 return DAG.getMergeValues({Lo, Hi}, DL);
10619 SDLoc DL(Op);
10623 auto [Mask, VL] = getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
10624 SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
10629 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
10630 DAG.getConstant(Log2_64(StepValImm), DL, XLenVT), VL);
10631 StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal);
10634 SDValue(), DAG.getConstant(StepValImm, DL, VT.getVectorElementType()),
10635 VL, VT, DL, DAG, Subtarget);
10636 StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal);
10649 SDLoc DL(Op);
10653 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, Op.getOperand(0));
10654 SDValue Op2 = DAG.getNode(ISD::VECTOR_REVERSE, DL, WidenVT, Op1);
10655 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Op2);
10678 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
10679 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
10683 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, DAG.getUNDEF(VecVT), Hi,
10684 DAG.getVectorIdxConstant(0, DL));
10686 ISD::INSERT_SUBVECTOR, DL, VecVT, Res, Lo,
10687 DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
10696 auto [Mask, VL] = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
10699 SDValue VLMinus1 = DAG.getNode(ISD::SUB, DL, XLenVT,
10700 computeVLMax(VecVT, DL, DAG),
10701 DAG.getConstant(1, DL, XLenVT));
10708 SplatVL = DAG.getSplatVector(IntVT, DL, VLMinus1);
10710 SplatVL = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT, DAG.getUNDEF(IntVT),
10713 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
10714 SDValue Indices = DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID,
10717 return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices,
10723 SDLoc DL(Op);
10729 SDValue VLMax = computeVLMax(VecVT, DL, DAG);
10736 DownOffset = DAG.getConstant(ImmValue, DL, XLenVT);
10737 UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DownOffset);
10741 UpOffset = DAG.getConstant(-ImmValue, DL, XLenVT);
10742 DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, UpOffset);
10745 SDValue TrueMask = getAllOnesMask(VecVT, VLMax, DL, DAG);
10748 getVSlidedown(DAG, Subtarget, DL, VecVT, DAG.getUNDEF(VecVT), V1,
10750 return getVSlideup(DAG, Subtarget, DL, VecVT, SlideDown, V2, UpOffset,
10758 SDLoc DL(Op);
10778 DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(),
10782 return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL);
10785 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, Subtarget);
10789 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle, DL, XLenVT);
10797 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
10801 return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL);
10807 SDLoc DL(Op);
10823 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getConstant(0, DL, VT),
10824 StoreVal, DAG.getVectorIdxConstant(0, DL));
10840 return DAG.getStore(Store->getChain(), DL, NewValue, Store->getBasePtr(),
10845 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
10850 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse, DL, XLenVT);
10852 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other),
10859 SDLoc DL(Op);
10894 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
10898 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
10908 Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
10913 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
10919 return DAG.getMergeValues({Result, Chain}, DL);
10924 SDLoc DL(Op);
10963 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
10966 Val = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
10967 DAG.getConstant(Intrinsic::riscv_vcompress, DL, XLenVT),
10970 DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Mask,
10971 getAllOnesMask(Mask.getSimpleValueType(), VL, DL, DAG), VL);
10976 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
10983 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
11000 SDLoc DL(Op);
11001 auto [Mask, VL] = getDefaultVLOps(VT.getVectorNumElements(), ContainerVT, DL,
11006 DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT,
11015 SDLoc DL(Op);
11032 SDValue Tmp1 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op1,
11034 SDValue Tmp2 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op2,
11036 SDValue OutChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
11040 Tmp1 = DAG.getNode(ISD::AND, DL, VT, Tmp1, Tmp2);
11041 return DAG.getMergeValues({Tmp1, OutChain}, DL);
11047 SDValue OEQ = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op1,
11049 SDValue Res = DAG.getNOT(DL, OEQ, VT);
11050 return DAG.getMergeValues({Res, OEQ.getValue(1)}, DL);
11062 auto [Mask, VL] = getDefaultVLOps(InVT, ContainerInVT, DL, DAG, Subtarget);
11070 SDValue True = getAllOnesMask(ContainerInVT, VL, DL, DAG);
11072 RISCVISD::STRICT_FSETCC_VL, DL, DAG.getVTList(MaskVT, MVT::Other),
11076 RISCVISD::STRICT_FSETCC_VL, DL, DAG.getVTList(MaskVT, MVT::Other),
11080 DAG.getNode(RISCVISD::VMAND_VL, DL, MaskVT, OrderMask1, OrderMask2, VL);
11083 Res = DAG.getNode(RISCVISD::STRICT_FSETCCS_VL, DL,
11089 Res = DAG.getNode(RVVOpc, DL, DAG.getVTList(MaskVT, MVT::Other),
11095 return DAG.getMergeValues({SubVec, Res.getValue(1)}, DL);
11102 SDLoc DL(Op);
11123 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
11126 RISCVISD::VMV_V_X_VL, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
11127 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL);
11128 SDValue NegX = DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X,
11130 SDValue Max = DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX,
11140 SDLoc DL(Op);
11151 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
11153 SDValue CopySign = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Mag,
11174 SDLoc DL(Op);
11175 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
11177 SDValue Select = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, Op1,
11209 SDLoc DL(Op);
11210 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
11221 DAG.getNode(NewOpc, DL, DAG.getVTList(ContainerVT, MVT::Other), Ops,
11224 return DAG.getMergeValues({SubVec, ScalableRes.getValue(1)}, DL);
11228 DAG.getNode(NewOpc, DL, ContainerVT, Ops, Op->getFlags());
11241 SDLoc DL(Op);
11285 return DAG.getNode(RISCVISDOpc, DL, VT, Ops, Op->getFlags());
11287 SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops, Op->getFlags());
11294 SDLoc DL(Op);
11309 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
11310 SDValue ZeroSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11314 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1, DL, XLenVT);
11315 SDValue Splat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11318 SDValue Result = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Src, Splat,
11327 SDLoc DL(Op);
11344 SDValue AllOneMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
11351 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, Op2, VL);
11356 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, Op2, VL);
11358 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, AllOneMask, VL);
11366 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, AllOneMask, VL);
11367 Result = DAG.getNode(RISCVISD::VMAND_VL, DL, ContainerVT, Temp, Op2, VL);
11375 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op2, AllOneMask, VL);
11376 Result = DAG.getNode(RISCVISD::VMAND_VL, DL, ContainerVT, Op1, Temp, VL);
11384 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, AllOneMask, VL);
11385 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, Op2, VL);
11393 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op2, AllOneMask, VL);
11394 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, Op1, VL);
11407 SDLoc DL(Op);
11440 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
11441 SDValue ZeroSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT,
11444 RISCVISDExtOpc == RISCVISD::VZEXT_VL ? 1 : -1, DL, XLenVT);
11445 SDValue OneSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT,
11447 Src = DAG.getNode(RISCVISD::VMERGE_VL, DL, IntVT, Src, OneSplat,
11453 Src = DAG.getNode(RISCVISDExtOpc, DL, IntVT, Src, Mask, VL);
11456 Result = DAG.getNode(RISCVISDOpc, DL, DstVT, Src, Mask, VL);
11467 DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterimFVT, Src, Mask, VL);
11470 Result = DAG.getNode(RISCVISDOpc, DL, DstVT, Src, Mask, VL);
11485 Result = DAG.getNode(RISCVISDOpc, DL, InterimFVT, Src, Mask, VL);
11489 Result = DAG.getNode(RISCVISD::FP_ROUND_VL, DL, DstVT, Src, Mask, VL);
11503 Result = DAG.getNode(RISCVISDOpc, DL, InterimIVT, Src, Mask, VL);
11508 SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
11509 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterimIVT,
11511 Result = DAG.getNode(RISCVISD::SETCC_VL, DL, DstVT,
11518 Result = DAG.getNode(RISCVISDOpc, DL, InterimIVT, Src, Mask, VL);
11525 Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, InterimIVT,
11541 SDLoc DL(Op);
11562 EVL1 = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, EVL1);
11569 SDValue SplatOneOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11571 DAG.getConstant(1, DL, XLenVT), EVL1);
11572 SDValue SplatZeroOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11574 DAG.getConstant(0, DL, XLenVT), EVL1);
11575 Op1 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op1, SplatOneOp1,
11578 SDValue SplatOneOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11580 DAG.getConstant(1, DL, XLenVT), EVL2);
11581 SDValue SplatZeroOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11583 DAG.getConstant(0, DL, XLenVT), EVL2);
11584 Op2 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op2, SplatOneOp2,
11593 DownOffset = DAG.getConstant(ImmValue, DL, XLenVT);
11594 UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, DownOffset);
11598 UpOffset = DAG.getConstant(-ImmValue, DL, XLenVT);
11599 DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, UpOffset);
11603 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
11605 SDValue Result = getVSlideup(DAG, Subtarget, DL, ContainerVT, SlideDown, Op2,
11611 RISCVISD::SETCC_VL, DL, ContainerVT.changeVectorElementType(MVT::i1),
11612 {Result, DAG.getConstant(0, DL, ContainerVT),
11624 SDLoc DL(Op);
11638 lowerScalarSplat(SDValue(), Val, VL, ContainerVT, DL, DAG, Subtarget);
11648 SDLoc DL(Op);
11672 SDValue SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
11674 DAG.getConstant(1, DL, XLenVT), EVL);
11675 SDValue SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
11677 DAG.getConstant(0, DL, XLenVT), EVL);
11678 Op1 = DAG.getNode(RISCVISD::VMERGE_VL, DL, IndicesVT, Op1, SplatOne,
11702 auto [Lo, Hi] = DAG.SplitVector(Op1, DL);
11704 SDValue LoRev = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
11705 SDValue HiRev = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
11712 DAG.getNode(ISD::CONCAT_VECTORS, DL, GatherVT, HiRev, LoRev);
11718 DAG.getVScale(DL, XLenVT, APInt(XLenVT.getSizeInBits(), MinElts));
11719 SDValue Diff = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, EVL);
11721 Result = getVSlidedown(DAG, Subtarget, DL, GatherVT,
11727 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
11728 {Result, DAG.getConstant(0, DL, GatherVT),
11743 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IndicesVT, Mask, EVL);
11745 DAG.getNode(ISD::SUB, DL, XLenVT, EVL, DAG.getConstant(1, DL, XLenVT));
11746 SDValue VecLenSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
11748 SDValue VRSUB = DAG.getNode(RISCVISD::SUB_VL, DL, IndicesVT, VecLenSplat, VID,
11750 SDValue Result = DAG.getNode(GatherOpc, DL, GatherVT, Op1, VRSUB,
11756 RISCVISD::SETCC_VL, DL, ContainerVT,
11757 {Result, DAG.getConstant(0, DL, GatherVT), DAG.getCondCode(ISD::SETNE),
11785 SDLoc DL(Op);
11786 SDValue Val = DAG.getNode(getRISCVVLOp(Op), DL, ContainerVT, Op1, Op2, VL);
11794 SDLoc DL(Op);
11810 DL, XLenVT);
11823 SDValue Policy = DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT);
11828 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
11835 return DAG.getMergeValues({Result, Chain}, DL);
11840 SDLoc DL(Op);
11858 DL, XLenVT);
11870 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, VPNode->getVTList(),
11883 SDLoc DL(Op);
11941 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
11945 Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
11950 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
11961 Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
11965 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
11971 return DAG.getMergeValues({Result, Chain}, DL);
11982 SDLoc DL(Op);
12039 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
12043 Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
12048 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
12056 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
12063 SDLoc DL(Op);
12066 RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
12068 SDValue RM = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
12082 DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT));
12083 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
12084 DAG.getConstant(Table, DL, XLenVT), Shift);
12085 SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
12086 DAG.getConstant(7, DL, XLenVT));
12088 return DAG.getMergeValues({Masked, Chain}, DL);
12094 SDLoc DL(Op);
12098 RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
12111 RMValue = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, RMValue);
12113 SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue,
12114 DAG.getConstant(2, DL, XLenVT));
12115 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
12116 DAG.getConstant(Table, DL, XLenVT), Shift);
12117 RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
12118 DAG.getConstant(0x7, DL, XLenVT));
12119 return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
12166 SDLoc DL(N);
12168 SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
12169 SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
12170 SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
12172 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
12178 SDLoc DL(N);
12179 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12180 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12181 SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
12182 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
12184 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
12190 SDLoc DL(N);
12213 Op0 = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
12221 Opc, DL, VTs, Chain, Op0,
12222 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
12223 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12232 Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0);
12236 DAG.getNode(Opc, DL, MVT::i64, Op0,
12237 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
12238 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12256 makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain);
12272 Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0);
12275 DAG.getNode(RISCVISD::FCVT_W_RV64, DL, MVT::i64, Op0,
12276 DAG.getTargetConstant(RISCVFPRndMode::RMM, DL, MVT::i64));
12277 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12288 SDValue Result = makeLibCall(DAG, LC, MVT::i64, Op0, CallOptions, DL).first;
12289 Result = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Result);
12302 RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding, DL, XLenVT);
12304 RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding, DL, XLenVT);
12307 RISCVSysReg::lookupSysRegByName("TIME")->Encoding, DL, XLenVT);
12309 RISCVSysReg::lookupSysRegByName("TIMEH")->Encoding, DL, XLenVT);
12312 SDValue RCW = DAG.getNode(RISCVISD::READ_COUNTER_WIDE, DL, VTs,
12316 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1)));
12354 S = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, S);
12355 U = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, U);
12356 SDValue Lo = DAG.getNode(ISD::MUL, DL, XLenVT, S, U);
12357 SDValue Hi = DAG.getNode(RISCVISD::MULHSU, DL, XLenVT, S, U);
12358 return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi);
12399 SDLoc DL(N);
12401 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12403 DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1));
12404 SDValue NewWOp = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, NewOp1);
12405 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
12407 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
12431 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12435 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0);
12436 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12473 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
12474 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
12475 SDValue Res = DAG.getNode(ISD::ADD, DL, MVT::i64, LHS, RHS);
12476 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
12479 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
12488 SDValue ResultLowerThanLHS = DAG.getSetCC(DL, OType, Res, LHS, ISD::SETLT);
12489 SDValue ConditionRHS = DAG.getSetCC(DL, OType, RHS, Zero, ISD::SETLT);
12492 DAG.getNode(ISD::XOR, DL, OType, ConditionRHS, ResultLowerThanLHS);
12493 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12503 SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12504 SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12506 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
12507 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
12517 Overflow = DAG.getSetCC(DL, N->getValueType(1), Res,
12518 DAG.getConstant(0, DL, MVT::i64), ISD::SETEQ);
12521 Overflow = DAG.getSetCC(DL, N->getValueType(1), N->getOperand(0),
12522 DAG.getConstant(0, DL, MVT::i32), ISD::SETNE);
12527 LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
12528 Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, LHS,
12532 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12545 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
12547 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
12548 SDValue Res = DAG.getNode(N->getOpcode(), DL, MVT::i64, LHS, RHS);
12549 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12573 SDValue Src = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64,
12575 SDValue Abs = DAG.getNode(RISCVISD::ABSW, DL, MVT::i64, Src);
12576 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Abs));
12581 SDValue Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12587 SDValue SignFill = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Src,
12589 SignFill = DAG.getNode(ISD::SRA, DL, MVT::i64, SignFill,
12590 DAG.getConstant(31, DL, MVT::i64));
12592 SDValue NewRes = DAG.getNode(ISD::XOR, DL, MVT::i64, Src, SignFill);
12593 NewRes = DAG.getNode(ISD::SUB, DL, MVT::i64, NewRes, SignFill);
12597 NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewRes,
12599 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
12610 SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
12611 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
12614 SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
12615 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
12619 DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
12620 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
12623 SDValue NewReg = DAG.getNode(RISCVISD::SplitF64, DL,
12625 SDValue RetReg = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64,
12637 Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
12638 DAG.getVectorIdxConstant(0, DL)));
12652 SDValue NewOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
12653 SDValue NewRes = DAG.getNode(N->getOpcode(), DL, XLenVT, NewOp);
12656 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NewRes));
12691 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget);
12696 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT,
12701 SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
12705 SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
12707 DAG.getConstant(32, DL, XLenVT), VL);
12709 DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec, ThirtyTwoV,
12712 SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
12714 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
12725 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12731 DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res));
12757 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12758 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp);
12759 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12767 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12769 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12771 DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, N->getOperand(3));
12772 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12779 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12781 RISCVISD::MOPR, DL, MVT::i64, NewOp,
12782 DAG.getTargetConstant(N->getConstantOperandVal(2), DL, MVT::i64));
12783 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12790 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12792 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12794 RISCVISD::MOPRR, DL, MVT::i64, NewOp0, NewOp1,
12795 DAG.getTargetConstant(N->getConstantOperandVal(3), DL, MVT::i64));
12796 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12804 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12806 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12807 SDValue Res = DAG.getNode(RISCVISD::CLMUL, DL, MVT::i64, NewOp0, NewOp1);
12808 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12826 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12828 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12829 NewOp0 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0,
12830 DAG.getConstant(32, DL, MVT::i64));
12831 NewOp1 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp1,
12832 DAG.getConstant(32, DL, MVT::i64));
12835 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1);
12836 Res = DAG.getNode(ISD::SRL, DL, MVT::i64, Res,
12837 DAG.getConstant(32, DL, MVT::i64));
12838 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12846 SDValue Extract = DAG.getNode(RISCVISD::VMV_X_S, DL,
12848 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract));
12860 SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
12864 auto [Mask, VL] = getDefaultVLOps(1, VecVT, DL, DAG, Subtarget);
12867 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT, DAG.getUNDEF(VecVT),
12868 DAG.getConstant(32, DL, XLenVT), VL);
12869 SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV,
12871 SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
12874 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
12904 SDValue Res = DAG.getNode(ISD::GET_ROUNDING, DL, VTs, N->getOperand(0));
12961 const SDLoc DL(N);
13009 SDValue Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ReduceVT, SrcVec,
13010 DAG.getVectorIdxConstant(0, DL));
13011 return DAG.getNode(ReduceOpc, DL, VT, Vec, N->getFlags());
13031 SDValue Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ReduceVT, SrcVec,
13032 DAG.getVectorIdxConstant(0, DL));
13035 return DAG.getNode(ReduceOpc, DL, VT, Vec, Flags);
13127 SDLoc DL(N);
13130 ScalarV.getSimpleValueType(), DL, DAG, Subtarget);
13135 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ScalarVT, DAG.getUNDEF(ScalarVT),
13136 NewScalarV, DAG.getVectorIdxConstant(0, DL));
13142 DAG.getNode(Reduce.getOpcode(), DL, Reduce.getValueType(), Ops);
13143 return DAG.getNode(Extract.getOpcode(), DL, Extract.getValueType(), NewReduce,
13184 SDLoc DL(N);
13187 SDValue SHADD = DAG.getNode(RISCVISD::SHL_ADD, DL, VT, NL,
13188 DAG.getConstant(Diff, DL, VT), NS);
13189 return DAG.getNode(ISD::SHL, DL, VT, SHADD, DAG.getConstant(Bits, DL, VT));
13340 SDLoc DL(N);
13341 SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
13342 DAG.getConstant(CA, DL, VT));
13344 DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT));
13345 return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT));
13405 SDLoc DL(N);
13421 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
13453 SDLoc DL(N);
13485 SDValue NewRHS = DAG.getConstant(ImmValMinus1, DL, VT);
13486 return DAG.getNode(ISD::ADD, DL, VT, NewLHS, NewRHS);
13531 SDLoc DL(N);
13533 return DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0),
13534 DAG.getConstant(ShAmt, DL, VT));
13589 SDLoc DL(N);
13591 SDValue Logic = DAG.getNode(Opc, DL, VT, N00, N10);
13592 return DAG.getNode(ISD::XOR, DL, VT, Logic, DAG.getConstant(1, DL, VT));
13656 SDLoc DL(N);
13658 DAG.getNode(ISD::SMAX, DL, SrcVT, True, DAG.getConstant(0, DL, SrcVT));
13660 DAG.getNode(ISD::SMIN, DL, SrcVT, Max,
13661 DAG.getConstant((1ULL << ScalarBits) - 1, DL, SrcVT));
13662 return DAG.getNode(ISD::TRUNCATE, DL, VT, Min);
13678 SDLoc DL(N0);
13679 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
13680 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
13681 SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1);
13706 SDLoc DL(N);
13707 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
13708 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
13709 SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1);
13710 SDValue And = DAG.getNode(ISD::AND, DL, MVT::i64, Srl,
13711 DAG.getConstant(1, DL, MVT::i64));
13712 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, And);
13755 SDLoc DL(N);
13757 SDValue NewN0 = DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV.getOperand(0),
13759 SDValue NewN1 = DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV.getOperand(0),
13761 SDValue NewOr = DAG.getNode(ISD::OR, DL, VT, NewN0, NewN1);
13762 return DAG.getNode(ISD::XOR, DL, VT, NewOr, TrueV.getOperand(1));
13804 SDLoc DL(N);
13805 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
13806 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
13807 SDValue Shl = DAG.getNode(ISD::SHL, DL, MVT::i64, Op0, Op1);
13808 SDValue And = DAG.getNOT(DL, Shl, MVT::i64);
13809 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, And);
13818 SDLoc DL(N);
13819 return DAG.getNode(RISCVISD::ROLW, DL, MVT::i64,
13820 DAG.getConstant(~1, DL, MVT::i64), N0.getOperand(1));
13829 SDLoc DL(N0);
13832 return DAG.getSetCC(DL, VT, N0.getOperand(1),
13833 DAG.getConstant(Imm + 1, DL, VT), CC);
13843 SDLoc DL(N);
13902 SDLoc DL(N);
13908 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, X,
13909 DAG.getConstant(Log2_64(MulAmt2), DL, VT));
13910 return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, Shl,
13911 DAG.getConstant(Log2_64(Divisor - 1), DL, VT),
13917 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
13918 DAG.getConstant(Log2_64(Divisor - 1), DL, VT), X);
13919 return DAG.getNode(ISD::SHL, DL, VT, Mul359,
13920 DAG.getConstant(Log2_64(MulAmt2), DL, VT));
13925 SDLoc DL(N);
13927 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
13928 DAG.getConstant(Log2_64(Divisor - 1), DL, VT), X);
13929 return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, Mul359,
13930 DAG.getConstant(Log2_64(MulAmt2 - 1), DL, VT),
13942 SDLoc DL(N);
13944 DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShiftAmt, DL, VT));
13945 return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
13946 DAG.getConstant(ScaleShift, DL, VT), Shift1);
13961 SDLoc DL(N);
13963 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
13964 DAG.getConstant(Log2_64(Divisor - 1), DL, VT), X);
13965 return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, Mul359,
13966 DAG.getConstant(TZ, DL, VT), X);
13975 SDLoc DL(N);
13977 DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShiftAmt, DL, VT));
13978 return DAG.getNode(ISD::ADD, DL, VT, Shift1,
13979 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
13980 DAG.getConstant(ScaleShift, DL, VT), X));
13987 SDLoc DL(N);
13989 DAG.getNode(ISD::SHL, DL, VT, X,
13990 DAG.getConstant(Log2_64(MulAmt + Offset), DL, VT));
13992 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
13993 DAG.getConstant(Log2_64(Offset - 1), DL, VT), X);
13994 return DAG.getNode(ISD::SUB, DL, VT, Shift1, Mul359);
14003 SDLoc DL(N);
14004 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
14005 DAG.getConstant(Log2_64(ShiftAmt1), DL, VT));
14007 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
14008 DAG.getConstant(Log2_64(MulAmtLowBit), DL, VT));
14009 return DAG.getNode(ISD::SUB, DL, VT, Shift1, Shift2);
14047 SDLoc DL(N);
14048 SDValue Cast = DAG.getNode(ISD::BITCAST, DL, HalfVT, Srl.getOperand(0));
14049 SDValue Sra = DAG.getNode(ISD::SRA, DL, HalfVT, Cast,
14050 DAG.getConstant(HalfSize - 1, DL, HalfVT));
14051 return DAG.getNode(ISD::BITCAST, DL, VT, Sra);
14061 SDLoc DL(N);
14085 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper);
14086 return DAG.getNode(AddSubOpc, DL, VT, N1, MulVal);
14090 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper);
14091 return DAG.getNode(AddSubOpc, DL, VT, N0, MulVal);
14113 SDLoc DL(N);
14128 N = DAG.getNode(ISD::TRUNCATE, DL,
14164 SDValue NewExt = DAG.getNode(N0->getOpcode(), DL, NewVT, N0->ops());
14165 SDValue NewShAmtVec = DAG.getConstant(ShAmtV, DL, NewVT);
14166 N = DAG.getNode(ISD::SHL, DL, NewVT, NewExt, NewShAmtVec);
14336 SDLoc DL(OrigOperand);
14344 return DAG.getNode(ExtOpc, DL, NarrowVT, Source, Mask, VL);
14346 return DAG.getSplat(NarrowVT, DL, Source.getOperand(0));
14348 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, NarrowVT,
14355 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, NarrowVT,
14696 SDLoc DL(Root);
14698 return getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
15310 SDLoc DL(N);
15337 getDefaultVLOps(SrcVT, SrcContainerVT, DL, DAG, Subtarget);
15345 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, VL);
15349 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, VL);
15353 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask,
15354 DAG.getTargetConstant(FRM, DL, XLenVT), VL);
15375 SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src.getOperand(0),
15376 DAG.getTargetConstant(FRM, DL, XLenVT));
15377 return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt);
15433 SDLoc DL(N);
15434 SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src,
15435 DAG.getTargetConstant(FRM, DL, XLenVT));
15440 FpToInt = DAG.getZeroExtendInReg(FpToInt, DL, MVT::i32);
15444 SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
15445 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
15463 SDLoc DL(N);
15464 return DAG.getNode(RISCVISD::BREV8, DL, VT, Src.getOperand(0));
15633 SDLoc DL(N);
15634 return DAG.getNode(ISD::SRA, DL, MVT::i64, Shl,
15635 DAG.getConstant(ShAmt + 32, DL, MVT::i64));
15688 SDLoc DL(N);
15695 DAG.getConstant(AddC->getAPIntValue().lshr(32), DL, MVT::i64);
15697 In = DAG.getNode(ISD::ADD, DL, MVT::i64, In, ShiftedAddC);
15699 In = DAG.getNode(ISD::SUB, DL, MVT::i64, ShiftedAddC, In);
15702 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, In,
15708 ISD::SHL, DL, MVT::i64, SExt,
15709 DAG.getConstant(32 - ShAmt, DL, MVT::i64));
15775 static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
15805 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
15834 DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS0.getOperand(0),
15835 DAG.getConstant(ShAmt, DL, LHS.getValueType()));
15847 RHS = DAG.getConstant(0, DL, LHS.getValueType());
15900 SDLoc DL(N);
15904 DAG.getNeutralElement(Opc, DL, OtherOpVT, N->getFlags());
15906 IdentityOperand = DAG.getConstant(0, DL, OtherOpVT);
15912 DAG.getSelect(DL, OtherOpVT, N->getOperand(0), OtherOp, IdentityOperand);
15913 return DAG.getNode(TrueVal.getOpcode(), DL, VT, FalseVal, NewSel);
15982 SDLoc DL(N);
16000 return DAG.getSelect(DL, VT,
16001 DAG.getSetCC(DL, CondVT, LHS, RHS, ISD::SETNE),
16034 SDLoc DL(N);
16085 return DAG.getNode(Opcode, DL, VT, DAG.getBuildVector(VT, DL, LHSOps),
16086 DAG.getBuildVector(VT, DL, RHSOps));
16095 SDLoc DL(N);
16123 SDValue LHS = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
16125 SDValue RHS = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
16127 return DAG.getNode(InVecOpcode, DL, VT, LHS, RHS);
16144 SDValue NewIdx = DAG.getVectorIdxConstant(Elt % ConcatNumElts, DL);
16148 ConcatOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ConcatVT,
16154 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps);
16164 SDLoc DL(N);
16251 : DAG.getConstant(std::get<int64_t>(StrideVariant), DL,
16254 Stride = DAG.getNegative(Stride, DL, Stride.getValueType());
16257 DAG.getSplat(WideVecVT.changeVectorElementType(MVT::i1), DL,
16258 DAG.getConstant(1, DL, MVT::i1));
16276 WideVecVT, DL, BaseLd->getChain(), BaseLd->getBasePtr(), Stride,
16278 DAG.getConstant(N->getNumOperands(), DL, Subtarget.getXLenVT()), MMO);
16328 SDLoc DL(N);
16329 return getDefaultScalableVLOps(N->getSimpleValueType(0), DL, DAG,
16351 SDLoc DL(N);
16355 return DAG.getNode(Opc, DL, VT, Ops);
16358 static bool legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index,
16380 Index = DAG.getNode(ISD::SIGN_EXTEND, DL,
16569 SDLoc DL(N);
16595 return DAG.getNode(RISCVISD::SMAX_VL, DL, V.getValueType(), SMinOp,
16648 ClipOpc, DL, ValVT,
16649 {Val, DAG.getConstant(0, DL, ValVT), DAG.getUNDEF(VT), Mask,
16650 DAG.getTargetConstant(RISCVVXRndMode::RNU, DL, Subtarget.getXLenVT()),
16661 SDLoc DL(N);
16698 SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
16699 SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
16711 DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
16717 SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
16718 DAG.getConstant(SignBit, DL, MVT::i32));
16722 SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
16723 DAG.getConstant(~SignBit, DL, MVT::i32));
16756 SDLoc DL(N);
16778 SDValue NewFMV = DAG.getNode(N->getOpcode(), DL, VT, Op0.getOperand(0));
16782 return DAG.getNode(ISD::XOR, DL, VT, NewFMV,
16783 DAG.getConstant(SignBit, DL, VT));
16786 return DAG.getNode(ISD::AND, DL, VT, NewFMV,
16787 DAG.getConstant(~SignBit, DL, VT));
16796 SDLoc DL(N);
16797 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT,
16798 DAG.getNode(ISD::ABS, DL, Src.getValueType(), Src));
16923 SDLoc DL(N);
16945 DAG.getNode(ISD::SRA, DL, VT, LHS,
16946 DAG.getConstant(Subtarget.getXLen() - 1, DL, VT));
16948 DAG.getNode(ISD::AND, DL, VT, SRA,
16949 DAG.getConstant(TrueSImm - FalseSImm, DL, VT));
16950 return DAG.getNode(ISD::ADD, DL, VT, AND, FalseV);
16957 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
16958 return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
16964 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, CCVal);
16965 SDValue Neg = DAG.getNegative(C, DL, VT);
16966 return DAG.getNode(ISD::OR, DL, VT, Neg, FalseV);
16971 DAG.getSetCC(DL, VT, LHS, RHS, ISD::getSetCCInverse(CCVal, VT));
16972 SDValue Neg = DAG.getNegative(C, DL, VT);
16973 return DAG.getNode(ISD::OR, DL, VT, Neg, TrueV);
16979 DAG.getSetCC(DL, VT, LHS, RHS, ISD::getSetCCInverse(CCVal, VT));
16980 SDValue Neg = DAG.getNegative(C, DL, VT);
16981 return DAG.getNode(ISD::AND, DL, VT, Neg, FalseV);
16985 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, CCVal);
16986 SDValue Neg = DAG.getNegative(C, DL, VT);
16987 return DAG.getNode(ISD::AND, DL, VT, Neg, TrueV);
16998 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, ISD::CondCode::SETEQ);
16999 return DAG.getNode(ISD::ADD, DL, VT, LHS, C);
17011 SDValue NewSel = DAG.getNode(RISCVISD::SELECT_CC, DL, VT, LHS, RHS, CC,
17013 return DAG.getNode(ISD::XOR, DL, VT, NewSel, TrueV.getOperand(1));
17022 SDLoc DL(N);
17024 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
17025 return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
17057 SDLoc DL(N);
17058 SDValue NewFPExtRound = DAG.getFPExtendOrRound(In2.getOperand(0), DL, VT);
17059 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
17060 DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound));
17071 SDLoc DL(N);
17072 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
17074 N->getVTList(), MGN->getMemoryVT(), DL,
17081 N->getVTList(), MGN->getMemoryVT(), DL,
17103 SDValue BasePtr = DAG.getNode(ISD::ADD, DL, PtrVT, MGN->getBasePtr(),
17104 DAG.getConstant(Addend, DL, PtrVT));
17106 SDValue EVL = DAG.getElementCount(DL, Subtarget.getXLenVT(),
17109 DAG.getStridedLoadVP(VT, DL, MGN->getChain(), BasePtr,
17110 DAG.getConstant(StepNumerator, DL, XLenVT),
17112 SDValue VPSelect = DAG.getNode(ISD::VP_SELECT, DL, VT, MGN->getMask(),
17115 DL);
17122 SDValue Load = DAG.getMaskedLoad(VT, DL, MGN->getChain(),
17128 DAG.getVectorShuffle(VT, DL, Load, DAG.getUNDEF(VT), ShuffleMask);
17129 return DAG.getMergeValues({Shuffle, Load.getValue(1)}, DL);
17140 Index = DAG.getBuildVector(IndexVT, DL, NewIndices);
17151 SDValue Mask = DAG.getSplat(MaskVT, DL, DAG.getConstant(1, DL, MVT::i1));
17154 DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other), WideVT, DL,
17159 return DAG.getMergeValues({Result, Gather.getValue(1)}, DL);
17171 SDLoc DL(N);
17172 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
17174 N->getVTList(), MSN->getMemoryVT(), DL,
17181 N->getVTList(), MSN->getMemoryVT(), DL,
17190 SDValue Shuffle = DAG.getVectorShuffle(VT, DL, MSN->getValue(),
17192 return DAG.getMaskedStore(MSN->getChain(), DL, Shuffle, MSN->getBasePtr(),
17207 SDLoc DL(N);
17208 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
17209 return DAG.getGatherVP(N->getVTList(), VPGN->getMemoryVT(), DL,
17216 return DAG.getGatherVP(N->getVTList(), VPGN->getMemoryVT(), DL,
17232 SDLoc DL(N);
17233 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
17234 return DAG.getScatterVP(N->getVTList(), VPSN->getMemoryVT(), DL,
17241 return DAG.getScatterVP(N->getVTList(), VPSN->getMemoryVT(), DL,
17257 SDLoc DL(N);
17260 ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
17262 return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt,
17280 SDLoc DL(N);
17282 ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
17285 return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt);
17333 SDLoc DL(N);
17368 SDValue NewV = DAG.getConstant(NewC, DL, NewVT);
17369 return DAG.getStore(Chain, DL, NewV, Store->getBasePtr(),
17389 SDValue NewL = DAG.getLoad(NewVT, DL, L->getChain(), L->getBasePtr(),
17392 return DAG.getStore(Chain, DL, NewL, Store->getBasePtr(),
17410 SDLoc DL(N);
17413 Store->getChain(), DL, Src, Store->getBasePtr(), Store->getOffset(),
17414 DAG.getConstant(1, DL, MaskVT),
17415 DAG.getConstant(1, DL, Subtarget.getXLenVT()), MemVT,
17453 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, Passthru, Scalar, VL);
17476 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru, Scalar, VL);
17513 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Passthru,
17514 DAG.getVectorIdxConstant(0, DL));
17516 DAG.getNode(N->getOpcode(), DL, M1VT, M1Passthru, Scalar, VL);
17517 Result = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru, Result,
17518 DAG.getVectorIdxConstant(0, DL));
17528 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
17537 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Vec,
17538 DAG.getVectorIdxConstant(0, DL));
17539 return DAG.getNode(RISCVISD::VMV_X_S, DL, N->getSimpleValueType(0), Vec);
17565 return DAG.getMaskedLoad(VT, DL, Load->getChain(), Base,
17583 return DAG.getMaskedStore(Store->getChain(), DL, Value, Base,
17600 SDLoc DL(N);
17604 return DAG.getConstant(-1, DL, VT);
17605 return DAG.getConstant(0, DL, VT);
17621 SDLoc DL(N);
17622 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i1, Ops);
17624 return DAG.getNode(ISD::TRUNCATE, DL, VT, N0);
17739 SDLoc DL(Op);
17740 SDValue NewC = TLO.DAG.getConstant(NewMask, DL, Op.getValueType());
17741 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
18138 DebugLoc DL = MI.getDebugLoc();
18141 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
18144 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
18147 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
18151 BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
18170 DebugLoc DL = MI.getDebugLoc();
18187 BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
18191 BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg)
18206 DebugLoc DL = MI.getDebugLoc();
18221 BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
18226 BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
18256 DebugLoc DL = MI.getDebugLoc();
18265 BuildMI(*BB, MI, DL, TII.get(RISCV::ReadFFLAGS), SavedFFlags);
18267 auto MIB = BuildMI(*BB, MI, DL, TII.get(RelOpcode), DstReg)
18274 BuildMI(*BB, MI, DL, TII.get(RISCV::WriteFFLAGS))
18278 auto MIB2 = BuildMI(*BB, MI, DL, TII.get(EqOpcode), RISCV::X0)
18329 const DebugLoc &DL = First.getDebugLoc();
18359 BuildMI(FirstMBB, DL, TII.getBrCond(FirstCC))
18371 BuildMI(ThisMBB, DL, TII.getBrCond(SecondCC))
18378 BuildMI(*SinkMBB, SinkMBB->begin(), DL, TII.get(RISCV::PHI), DestReg)
18473 DebugLoc DL = MI.getDebugLoc();
18506 BuildMI(HeadMBB, DL, TII.getBrCond(CC, MI.getOperand(2).isImm()))
18511 BuildMI(HeadMBB, DL, TII.getBrCond(CC))
18557 DebugLoc DL = MI.getDebugLoc();
18565 BuildMI(*BB, MI, DL, TII.get(RISCV::ReadFFLAGS), SavedFFLAGS);
18574 BuildMI(*BB, MI, DL, TII.get(CVTXOpc), Tmp)
18595 BuildMI(*BB, MI, DL, TII.get(CVTFOpc))
18609 BuildMI(*BB, MI, DL, TII.get(RISCV::WriteFFLAGS))
18677 DebugLoc DL = MI.getDebugLoc();
18705 BuildMI(MBB, DL, TII.get(FSGNJXOpc), FabsReg).addReg(SrcReg).addReg(SrcReg);
18710 BuildMI(MBB, DL, TII.get(CmpOpc), CmpReg).addReg(FabsReg).addReg(MaxReg);
18715 BuildMI(MBB, DL, TII.get(RISCV::BEQ))
18724 MIB = BuildMI(CvtMBB, DL, TII.get(F2IOpc), F2IReg).addReg(SrcReg).addImm(FRM);
18730 MIB = BuildMI(CvtMBB, DL, TII.get(I2FOpc), I2FReg).addReg(F2IReg).addImm(FRM);
18736 BuildMI(CvtMBB, DL, TII.get(FSGNJOpc), CvtReg).addReg(I2FReg).addReg(SrcReg);
18739 BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(RISCV::PHI), DstReg)
18994 bool RISCV::CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
18999 unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
19076 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes &&
19317 const CCValAssign &VA, const SDLoc &DL,
19329 Val = DAG.getNode(RISCVISD::FMV_H_X, DL, VA.getValVT(), Val);
19332 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Val);
19333 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
19335 Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
19338 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
19348 const CCValAssign &VA, const SDLoc &DL,
19358 Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
19377 return convertLocVTToValVT(DAG, Val, VA, DL, TLI.getSubtarget());
19381 const CCValAssign &VA, const SDLoc &DL,
19395 Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, LocVT, Val);
19398 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
19399 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Val);
19401 Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
19404 Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
19414 const CCValAssign &VA, const SDLoc &DL) {
19442 ExtType, DL, LocVT, Chain, FIN,
19450 const SDLoc &DL) {
19461 SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
19468 Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
19474 Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
19476 return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
19481 bool RISCV::CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI,
19662 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
19722 ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, ArgLocs[++i], DL);
19724 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, Ins[InsIdx], *this);
19726 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
19734 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
19742 SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
19744 Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
19745 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset);
19746 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
19798 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
19800 Chain, DL, ArgValue, FIN,
19804 DAG.getMemBasePlusOffset(FIN, TypeSize::getFixed(XLenInBytes), DL);
19818 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
19896 SDLoc &DL = CLI.DL;
19950 SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
19952 Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
19960 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
19977 RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
19990 StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
19992 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
19993 DAG.getIntPtrConstant(HiVA.getLocMemOffset(), DL));
19996 DAG.getStore(Chain, DL, Hi, Address, MachinePointerInfo()));
20028 SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
20031 Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
20041 DAG.getStore(Chain, DL, ArgValue, SpillSlot,
20047 DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset);
20049 DAG.getStore(Chain, DL, PartValue, Address,
20054 ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL, Subtarget);
20071 StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
20073 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
20074 DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
20078 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
20084 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
20090 Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
20109 Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, RISCVII::MO_CALL);
20144 SDValue Ret = DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
20151 Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
20158 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, Glue, DL);
20171 DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
20178 SDValue RetValue2 = DAG.getCopyFromReg(Chain, DL, RVLocs[++i].getLocReg(),
20182 RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
20186 RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL, Subtarget);
20219 const SDLoc &DL, SelectionDAG &DAG) const {
20249 SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
20262 Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
20265 Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
20270 Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget);
20271 Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
20313 return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
21078 const DataLayout &DL = AI->getDataLayout();
21080 DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
21479 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
21487 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val);
21488 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val);
21489 Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val,
21490 DAG.getConstant(0xFFFF0000, DL, MVT::i32));
21491 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
21516 Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SameEltTypeVT,
21518 DAG.getVectorIdxConstant(0, DL));
21520 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
21523 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
21524 Val, DAG.getVectorIdxConstant(0, DL));
21534 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
21542 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
21543 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val);
21544 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
21569 Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val);
21571 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
21572 DAG.getVectorIdxConstant(0, DL));
21621 const DataLayout &DL) const {
21622 EVT VT = getValueType(DL, VTy);
21628 !allowsMemoryAccessForAlignment(VTy->getContext(), DL, VT, AddrSpace,
22102 const DataLayout &DL = MF->getDataLayout();
22111 EVT VT = TLI->getValueType(DL, ElemTy);
22121 ComputeValueVTs(*TLI, DL, Ty, ValueVTs);