Lines Matching defs:VT
40 EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) {
41 unsigned StoreSize = VT.getStoreSizeInBits();
48 return VT;
183 for (MVT VT : MVT::integer_valuetypes())
184 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i64, VT,
187 for (MVT VT : MVT::integer_valuetypes()) {
188 if (VT == MVT::i64)
192 setLoadExtAction(Op, VT, MVT::i1, Promote);
193 setLoadExtAction(Op, VT, MVT::i8, Legal);
194 setLoadExtAction(Op, VT, MVT::i16, Legal);
195 setLoadExtAction(Op, VT, MVT::i32, Expand);
199 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
202 setLoadExtAction({ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, VT, MemVT,
467 for (MVT VT : ScalarIntVTs) {
469 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT,
473 setOperationAction({ISD::SDIVREM, ISD::UDIVREM}, VT, Custom);
476 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand);
478 setOperationAction({ISD::BSWAP, ISD::CTTZ, ISD::CTLZ}, VT, Expand);
481 setOperationAction({ISD::ADDC, ISD::SUBC, ISD::ADDE, ISD::SUBE}, VT, Legal);
506 for (auto VT : {MVT::i8, MVT::i16})
507 setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, VT, Custom);
513 for (MVT VT : VectorIntTypes) {
527 VT, Expand);
534 for (MVT VT : FloatVectorTypes) {
547 VT, Expand);
709 static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) {
711 VT == MVT::f64;
770 MVT VT = N->getValueType(0).getScalarType().getSimpleVT();
779 if (!opMustUseVOP3Encoding(U, VT)) {
788 EVT AMDGPUTargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,
790 assert(!VT.isVector() && "only scalar expected");
793 unsigned Size = VT.getSizeInBits();
809 bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
811 EVT ScalarVT = VT.getScalarType();
817 bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
818 EVT ScalarVT = VT.getScalarType();
932 EVT VT = Op.getValueType();
938 return DAG.getNode(AMDGPUISD::RCP, SL, VT, NegSrc, Op->getFlags());
953 bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const {
954 assert(VT.isFloatingPoint());
957 return VT == MVT::f32 || VT == MVT::f64 ||
958 (Subtarget->has16BitInsts() && (VT == MVT::f16 || VT == MVT::bf16));
961 bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
962 assert(VT.isFloatingPoint());
964 VT = VT.getScalarType();
965 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
1179 /// represents a single value that will be stored in registers. Ins[x].VT is
1184 /// argument. Since Ins[x].VT gives us the size of the register that will
1194 /// calling convention analysis function and the register type (Ins[x].VT) as
1392 InVals.push_back(DAG.getUNDEF(Arg.VT));
1559 EVT VT = Op.getValueType();
1560 if (VT.getVectorElementType().getSizeInBits() < 32) {
1579 return DAG.getNode(ISD::BITCAST, SL, VT, BV);
1594 EVT VT = Op.getValueType();
1597 if (VT.getScalarSizeInBits() == 16 && Start % 2 == 0) {
1598 unsigned NumElt = VT.getVectorNumElements();
1615 return DAG.getNode(ISD::BITCAST, SL, VT, Tmp);
1619 VT.getVectorNumElements());
1643 const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
1664 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS);
1665 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS);
1683 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS);
1684 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS);
1689 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS);
1690 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS);
1701 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS);
1702 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS);
1711 SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(const SDLoc &DL, EVT VT,
1717 return combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True, False, CC, DCI);
1739 combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, NegTrue, False, CC, DCI);
1741 return DAG.getNode(ISD::FNEG, DL, VT, Combined);
1784 AMDGPUTargetLowering::getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const {
1786 EVT EltVT = VT.getVectorElementType();
1787 unsigned NumElts = VT.getVectorNumElements();
1817 EVT VT = Op.getValueType();
1823 if (VT.getVectorNumElements() == 2) {
1838 std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG);
1858 Join = DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad);
1860 Join = DAG.getNode(ISD::INSERT_SUBVECTOR, SL, VT, DAG.getUNDEF(VT), LoLoad,
1864 VT, Join, HiLoad,
1877 EVT VT = Op.getValueType();
1895 EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 4);
1902 {DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, VT, WideLoad,
1912 EVT VT = Val.getValueType();
1916 if (VT.getVectorNumElements() == 2)
1928 std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG);
1955 EVT VT = Op.getValueType();
1969 unsigned BitSize = VT.getSizeInBits();
1982 jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
1985 jq = DAG.getNode(ISD::SRA, DL, VT, jq,
1986 DAG.getConstant(BitSize - 2, DL, VT));
1989 jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT));
2037 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2043 jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT));
2046 SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq);
2049 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS);
2050 Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem);
2056 Div = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Div, InRegSize);
2057 Rem = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Rem, InRegSize);
2059 SDValue TruncMask = DAG.getConstant((UINT64_C(1) << DivBits) - 1, DL, VT);
2060 Div = DAG.getNode(ISD::AND, DL, VT, Div, TruncMask);
2061 Rem = DAG.getNode(ISD::AND, DL, VT, Rem, TruncMask);
2071 EVT VT = Op.getValueType();
2073 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64");
2075 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
2133 SDValue Rcp64 = DAG.getBitcast(VT,
2136 SDValue Zero64 = DAG.getConstant(0, DL, VT);
2137 SDValue One64 = DAG.getConstant(1, DL, VT);
2142 SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS);
2143 SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64);
2144 SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1);
2152 SDValue Add1 = DAG.getBitcast(VT,
2156 SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1);
2157 SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2);
2165 SDValue Add2 = DAG.getBitcast(VT,
2168 SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2);
2170 SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3);
2179 SDValue Sub1 = DAG.getBitcast(VT,
2200 SDValue Sub2 = DAG.getBitcast(VT,
2203 SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64);
2212 SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64);
2220 SDValue Sub3 = DAG.getBitcast(VT,
2258 HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit);
2261 REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT));
2263 REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit);
2271 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS);
2284 EVT VT = Op.getValueType();
2286 if (VT == MVT::i64) {
2292 if (VT == MVT::i32) {
2304 SDValue Z = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Y);
2307 SDValue NegY = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Y);
2308 SDValue NegYZ = DAG.getNode(ISD::MUL, DL, VT, NegY, Z);
2309 Z = DAG.getNode(ISD::ADD, DL, VT, Z,
2310 DAG.getNode(ISD::MULHU, DL, VT, Z, NegYZ));
2313 SDValue Q = DAG.getNode(ISD::MULHU, DL, VT, X, Z);
2315 DAG.getNode(ISD::SUB, DL, VT, X, DAG.getNode(ISD::MUL, DL, VT, Q, Y));
2318 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2319 SDValue One = DAG.getConstant(1, DL, VT);
2321 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond,
2322 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q);
2323 R = DAG.getNode(ISD::SELECT, DL, VT, Cond,
2324 DAG.getNode(ISD::SUB, DL, VT, R, Y), R);
2328 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond,
2329 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q);
2330 R = DAG.getNode(ISD::SELECT, DL, VT, Cond,
2331 DAG.getNode(ISD::SUB, DL, VT, R, Y), R);
2339 EVT VT = Op.getValueType();
2344 SDValue Zero = DAG.getConstant(0, DL, VT);
2345 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
2347 if (VT == MVT::i32) {
2352 if (VT == MVT::i64 &&
2355 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
2363 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)),
2364 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1))
2371 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign);
2374 LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign);
2375 RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign);
2377 LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign);
2378 RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign);
2380 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS);
2383 Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign);
2384 Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign);
2386 Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign);
2387 Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign);
2399 EVT VT = Op.getValueType();
2404 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y, Flags);
2405 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, VT, Div, Flags);
2406 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Trunc, Flags);
2408 return DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags);
2535 auto VT = Op.getValueType();
2537 return DAG.getNode(ISD::FROUNDEVEN, SDLoc(Op), VT, Arg);
2548 EVT VT = Op.getValueType();
2550 SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X);
2554 SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T);
2556 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff);
2558 const SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
2559 const SDValue One = DAG.getConstantFP(1.0, SL, VT);
2562 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2564 const SDValue Half = DAG.getConstantFP(0.5, SL, VT);
2566 SDValue OneOrZeroFP = DAG.getNode(ISD::SELECT, SL, VT, Cmp, One, Zero);
2568 SDValue SignedOffset = DAG.getNode(ISD::FCOPYSIGN, SL, VT, OneOrZeroFP, X);
2569 return DAG.getNode(ISD::FADD, SL, VT, T, SignedOffset);
2642 EVT VT = Src.getValueType();
2643 const fltSemantics &Semantics = VT.getFltSemantics();
2645 DAG.getConstantFP(APFloat::getSmallestNormalized(Semantics), SL, VT);
2650 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Src,
2659 EVT VT = Src.getValueType();
2660 const fltSemantics &Semantics = VT.getFltSemantics();
2661 SDValue Inf = DAG.getConstantFP(APFloat::getInf(Semantics), SL, VT);
2663 SDValue Fabs = DAG.getNode(ISD::FABS, SL, VT, Src, Flags);
2665 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Fabs,
2678 MVT VT = MVT::f32;
2681 DAG.getConstantFP(APFloat::getSmallestNormalized(Semantics), SL, VT);
2684 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Src,
2687 SDValue Scale32 = DAG.getConstantFP(0x1.0p+32, SL, VT);
2688 SDValue One = DAG.getConstantFP(1.0, SL, VT);
2690 DAG.getNode(ISD::SELECT, SL, VT, IsLtSmallestNormal, Scale32, One, Flags);
2692 SDValue ScaledInput = DAG.getNode(ISD::FMUL, SL, VT, Src, ScaleFactor, Flags);
2704 EVT VT = Op.getValueType();
2708 if (VT == MVT::f16) {
2713 return DAG.getNode(ISD::FP_ROUND, SL, VT, Log,
2720 return DAG.getNode(AMDGPUISD::LOG, SL, VT, Src, Flags);
2722 SDValue Log2 = DAG.getNode(AMDGPUISD::LOG, SL, VT, ScaledInput, Flags);
2724 SDValue ThirtyTwo = DAG.getConstantFP(32.0, SL, VT);
2725 SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
2727 DAG.getNode(ISD::SELECT, SL, VT, IsLtSmallestNormal, ThirtyTwo, Zero);
2728 return DAG.getNode(ISD::FSUB, SL, VT, Log2, ResultOffset, Flags);
2731 static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X,
2733 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, X, Y, Flags);
2734 return DAG.getNode(ISD::FADD, SL, VT, Mul, C, Flags);
2740 EVT VT = Op.getValueType();
2748 if (VT == MVT::f16 || Flags.hasApproximateFuncs() ||
2751 if (VT == MVT::f16 && !Subtarget->has16BitInsts()) {
2757 if (VT == MVT::f16 && !Subtarget->has16BitInsts()) {
2758 return DAG.getNode(ISD::FP_ROUND, DL, VT, Lowered,
2769 SDValue Y = DAG.getNode(AMDGPUISD::LOG, DL, VT, X, Flags);
2781 SDValue C = DAG.getConstantFP(IsLog10 ? c_log10 : c_log, DL, VT);
2782 SDValue CC = DAG.getConstantFP(IsLog10 ? cc_log10 : cc_log, DL, VT);
2784 R = DAG.getNode(ISD::FMUL, DL, VT, Y, C, Flags);
2785 SDValue NegR = DAG.getNode(ISD::FNEG, DL, VT, R, Flags);
2786 SDValue FMA0 = DAG.getNode(ISD::FMA, DL, VT, Y, C, NegR, Flags);
2787 SDValue FMA1 = DAG.getNode(ISD::FMA, DL, VT, Y, CC, FMA0, Flags);
2788 R = DAG.getNode(ISD::FADD, DL, VT, R, FMA1, Flags);
2798 SDValue CH = DAG.getConstantFP(IsLog10 ? ch_log10 : ch_log, DL, VT);
2799 SDValue CT = DAG.getConstantFP(IsLog10 ? ct_log10 : ct_log, DL, VT);
2805 SDValue YT = DAG.getNode(ISD::FSUB, DL, VT, Y, YH, Flags);
2807 SDValue YTCT = DAG.getNode(ISD::FMUL, DL, VT, YT, CT, Flags);
2808 SDValue Mad0 = getMad(DAG, DL, VT, YH, CT, YTCT, Flags);
2809 SDValue Mad1 = getMad(DAG, DL, VT, YT, CH, Mad0, Flags);
2810 R = getMad(DAG, DL, VT, YH, CH, Mad1);
2819 R = DAG.getNode(ISD::SELECT, DL, VT, IsFinite, R, Y, Flags);
2823 SDValue Zero = DAG.getConstantFP(0.0f, DL, VT);
2825 DAG.getConstantFP(IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f, DL, VT);
2827 DAG.getNode(ISD::SELECT, DL, VT, IsScaled, ShiftK, Zero, Flags);
2828 R = DAG.getNode(ISD::FSUB, DL, VT, R, Shift, Flags);
2843 EVT VT = Src.getValueType();
2845 VT == MVT::f32 ? (unsigned)AMDGPUISD::LOG : (unsigned)ISD::FLOG2;
2850 if (VT == MVT::f32) {
2853 SDValue LogSrc = DAG.getNode(AMDGPUISD::LOG, SL, VT, ScaledInput, Flags);
2855 DAG.getConstantFP(-32.0 * Log2BaseInverted, SL, VT);
2857 SDValue Zero = DAG.getConstantFP(0.0f, SL, VT);
2859 SDValue ResultOffset = DAG.getNode(ISD::SELECT, SL, VT, IsScaled,
2862 SDValue Log2Inv = DAG.getConstantFP(Log2BaseInverted, SL, VT);
2865 return DAG.getNode(ISD::FMA, SL, VT, LogSrc, Log2Inv, ResultOffset,
2867 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, LogSrc, Log2Inv, Flags);
2868 return DAG.getNode(ISD::FADD, SL, VT, Mul, ResultOffset);
2872 SDValue Log2Operand = DAG.getNode(LogOp, SL, VT, Src, Flags);
2873 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Log2BaseInverted, SL, VT);
2875 return DAG.getNode(ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand,
2884 EVT VT = Op.getValueType();
2888 if (VT == MVT::f16) {
2893 return DAG.getNode(ISD::FP_ROUND, SL, VT, Log,
2897 assert(VT == MVT::f32);
2906 SDValue RangeCheckConst = DAG.getConstantFP(-0x1.f80000p+6f, SL, VT);
2908 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2913 SDValue SixtyFour = DAG.getConstantFP(0x1.0p+6f, SL, VT);
2914 SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
2917 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, SixtyFour, Zero);
2919 SDValue AddInput = DAG.getNode(ISD::FADD, SL, VT, Src, AddOffset, Flags);
2920 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, AddInput, Flags);
2922 SDValue TwoExpNeg64 = DAG.getConstantFP(0x1.0p-64f, SL, VT);
2923 SDValue One = DAG.getConstantFP(1.0, SL, VT);
2925 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, TwoExpNeg64, One);
2927 return DAG.getNode(ISD::FMUL, SL, VT, Exp2, ResultScale, Flags);
2933 EVT VT = X.getValueType();
2934 const SDValue Log2E = DAG.getConstantFP(numbers::log2e, SL, VT);
2936 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, X, Flags)) {
2938 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, X, Log2E, Flags);
2939 return DAG.getNode(VT == MVT::f32 ? (unsigned)AMDGPUISD::EXP
2941 SL, VT, Mul, Flags);
2944 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2946 SDValue Threshold = DAG.getConstantFP(-0x1.5d58a0p+6f, SL, VT);
2949 SDValue ScaleOffset = DAG.getConstantFP(0x1.0p+6f, SL, VT);
2951 SDValue ScaledX = DAG.getNode(ISD::FADD, SL, VT, X, ScaleOffset, Flags);
2954 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, ScaledX, X);
2956 SDValue ExpInput = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, Log2E, Flags);
2958 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, ExpInput, Flags);
2960 SDValue ResultScaleFactor = DAG.getConstantFP(0x1.969d48p-93f, SL, VT);
2962 DAG.getNode(ISD::FMUL, SL, VT, Exp2, ResultScaleFactor, Flags);
2964 return DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, AdjustedResult, Exp2,
2973 const EVT VT = X.getValueType();
2974 const unsigned Exp2Op = VT == MVT::f32 ? AMDGPUISD::EXP : ISD::FEXP2;
2976 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, X, Flags)) {
2978 SDValue K0 = DAG.getConstantFP(0x1.a92000p+1f, SL, VT);
2979 SDValue K1 = DAG.getConstantFP(0x1.4f0978p-11f, SL, VT);
2981 SDValue Mul0 = DAG.getNode(ISD::FMUL, SL, VT, X, K0, Flags);
2982 SDValue Exp2_0 = DAG.getNode(Exp2Op, SL, VT, Mul0, Flags);
2983 SDValue Mul1 = DAG.getNode(ISD::FMUL, SL, VT, X, K1, Flags);
2984 SDValue Exp2_1 = DAG.getNode(Exp2Op, SL, VT, Mul1, Flags);
2985 return DAG.getNode(ISD::FMUL, SL, VT, Exp2_0, Exp2_1);
2994 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2996 SDValue Threshold = DAG.getConstantFP(-0x1.2f7030p+5f, SL, VT);
2999 SDValue ScaleOffset = DAG.getConstantFP(0x1.0p+5f, SL, VT);
3000 SDValue ScaledX = DAG.getNode(ISD::FADD, SL, VT, X, ScaleOffset, Flags);
3002 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, ScaledX, X);
3004 SDValue K0 = DAG.getConstantFP(0x1.a92000p+1f, SL, VT);
3005 SDValue K1 = DAG.getConstantFP(0x1.4f0978p-11f, SL, VT);
3007 SDValue Mul0 = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, K0, Flags);
3008 SDValue Exp2_0 = DAG.getNode(Exp2Op, SL, VT, Mul0, Flags);
3009 SDValue Mul1 = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, K1, Flags);
3010 SDValue Exp2_1 = DAG.getNode(Exp2Op, SL, VT, Mul1, Flags);
3012 SDValue MulExps = DAG.getNode(ISD::FMUL, SL, VT, Exp2_0, Exp2_1, Flags);
3014 SDValue ResultScaleFactor = DAG.getConstantFP(0x1.9f623ep-107f, SL, VT);
3016 DAG.getNode(ISD::FMUL, SL, VT, MulExps, ResultScaleFactor, Flags);
3018 return DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, AdjustedResult, MulExps,
3023 EVT VT = Op.getValueType();
3029 if (VT.getScalarType() == MVT::f16) {
3034 if (VT.isVector())
3043 return DAG.getNode(ISD::FP_ROUND, SL, VT, Lowered,
3047 assert(VT == MVT::f32);
3090 SDValue C = DAG.getConstantFP(IsExp10 ? c_exp10 : c_exp, SL, VT);
3091 SDValue CC = DAG.getConstantFP(IsExp10 ? cc_exp10 : cc_exp, SL, VT);
3093 PH = DAG.getNode(ISD::FMUL, SL, VT, X, C, Flags);
3094 SDValue NegPH = DAG.getNode(ISD::FNEG, SL, VT, PH, Flags);
3095 SDValue FMA0 = DAG.getNode(ISD::FMA, SL, VT, X, C, NegPH, Flags);
3096 PL = DAG.getNode(ISD::FMA, SL, VT, X, CC, FMA0, Flags);
3104 SDValue CH = DAG.getConstantFP(IsExp10 ? ch_exp10 : ch_exp, SL, VT);
3105 SDValue CL = DAG.getConstantFP(IsExp10 ? cl_exp10 : cl_exp, SL, VT);
3110 SDValue XH = DAG.getNode(ISD::BITCAST, SL, VT, XHAsInt);
3111 SDValue XL = DAG.getNode(ISD::FSUB, SL, VT, X, XH, Flags);
3113 PH = DAG.getNode(ISD::FMUL, SL, VT, XH, CH, Flags);
3115 SDValue XLCL = DAG.getNode(ISD::FMUL, SL, VT, XL, CL, Flags);
3116 SDValue Mad0 = getMad(DAG, SL, VT, XL, CH, XLCL, Flags);
3117 PL = getMad(DAG, SL, VT, XH, CL, Mad0, Flags);
3120 SDValue E = DAG.getNode(ISD::FROUNDEVEN, SL, VT, PH, Flags);
3123 SDValue PHSubE = DAG.getNode(ISD::FSUB, SL, VT, PH, E, FlagsNoContract);
3125 SDValue A = DAG.getNode(ISD::FADD, SL, VT, PHSubE, PL, Flags);
3127 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, A, Flags);
3129 SDValue R = DAG.getNode(ISD::FLDEXP, SL, VT, Exp2, IntE, Flags);
3132 DAG.getConstantFP(IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f, SL, VT);
3134 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
3135 SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
3139 R = DAG.getNode(ISD::SELECT, SL, VT, Underflow, Zero, R);
3144 DAG.getConstantFP(IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f, SL, VT);
3148 DAG.getConstantFP(APFloat::getInf(APFloat::IEEEsingle()), SL, VT);
3149 R = DAG.getNode(ISD::SELECT, SL, VT, Overflow, Inf, R);
3714 MVT VT = Op.getSimpleValueType();
3715 MVT ScalarVT = VT.getScalarType();
3717 assert(VT.isVector());
3723 unsigned NElts = VT.getVectorNumElements();
3731 return DAG.getBuildVector(VT, DL, Args);
3743 EVT VT = Op.getValueType();
3744 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
3827 bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const {
3829 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT))
3832 if (!VT.isByteSized())
3835 unsigned Size = VT.getStoreSize();
3837 if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector())
3859 EVT VT = LN->getMemoryVT();
3861 unsigned Size = VT.getStoreSize();
3863 if (Alignment < Size && isTypeLegal(VT)) {
3871 VT, AS, Alignment, LN->getMemOperand()->getFlags(), &IsFast)) {
3872 if (VT.isVector())
3885 if (!shouldCombineMemoryType(VT))
3888 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
3894 SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad);
3910 EVT VT = SN->getMemoryVT();
3911 unsigned Size = VT.getStoreSize();
3916 if (Alignment < Size && isTypeLegal(VT)) {
3925 VT, AS, Alignment, SN->getMemOperand()->getFlags(), &IsFast)) {
3926 if (VT.isVector())
3936 if (!shouldCombineMemoryType(VT))
3939 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
3947 SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal);
4042 EVT VT = N->getValueType(0);
4064 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 &&
4074 if (VT != MVT::i64)
4082 return DAG.getZExtOrTrunc(Shl, SL, VT);
4086 if (VT != MVT::i64)
4149 EVT VT = N->getValueType(0);
4163 ISD::AND, SL, VT,
4164 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), N->getOperand(1)),
4165 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), N->getOperand(1)));
4170 if (VT != MVT::i64)
4195 EVT VT = N->getValueType(0);
4199 if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) {
4204 if (VT.getFixedSizeInBits() <= EltVT.getFixedSizeInBits()) {
4210 return DAG.getNode(ISD::TRUNCATE, SL, VT, Elt0);
4218 if (Src.getOpcode() == ISD::SRL && !VT.isVector()) {
4231 return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt);
4241 if (VT.getScalarSizeInBits() < 32) {
4255 (Src.getOpcode() == ISD::SHL) ? 31 : (32 - VT.getScalarSizeInBits());
4257 EVT MidVT = VT.isVector() ?
4259 VT.getVectorNumElements()) : MVT::i32;
4273 return DAG.getNode(ISD::TRUNCATE, SL, VT, ShrunkShift);
4313 EVT VT = N->getValueType(0);
4322 unsigned Size = VT.getSizeInBits();
4323 if (VT.isVector() || Size > 64)
4352 SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N1, MulOper);
4353 return DAG.getNode(ISD::ADD, DL, VT, MulVal, N1);
4357 SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N0, MulOper);
4358 return DAG.getNode(ISD::ADD, DL, VT, MulVal, N0);
4362 if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16))
4391 return DAG.getSExtOrTrunc(Mul, DL, VT);
4446 EVT VT = N->getValueType(0);
4448 if (!Subtarget->hasMulI24() || VT.isVector())
4474 return DAG.getSExtOrTrunc(Mulhi, DL, VT);
4479 EVT VT = N->getValueType(0);
4481 if (!Subtarget->hasMulU24() || VT.isVector() || VT.getSizeInBits() > 32)
4507 return DAG.getZExtOrTrunc(Mulhi, DL, VT);
4514 EVT VT = Op.getValueType();
4515 EVT LegalVT = getTypeToTransformTo(*DAG.getContext(), VT);
4520 if (VT != MVT::i32)
4524 if (VT != MVT::i32)
4525 FFBX = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBX);
4578 EVT VT = N1.getValueType();
4580 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond,
4583 return DAG.getNode(Op, SL, VT, NewSelect);
4601 EVT VT = N.getValueType();
4656 NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
4661 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT,
4664 return DAG.getNode(LHS.getOpcode(), SL, VT, NewSelect);
4680 EVT VT = N->getValueType(0);
4701 return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True);
4704 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
4706 = combineFMinMaxLegacy(SDLoc(N), VT, LHS, RHS, True, False, CC, DCI);
4800 EVT VT = N->getValueType(0);
4818 LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
4823 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
4827 SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags());
4831 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
4846 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
4848 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags());
4852 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
4871 MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS);
4874 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
4878 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS);
4882 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
4906 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
4907 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
4910 SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags());
4914 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
4920 Ops[I] = DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags());
4922 SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags());
4927 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Res);
4951 return DAG.getNode(Opc, SL, VT, CvtSrc.getOperand(0));
4960 return DAG.getNode(Opc, SL, VT, Neg, N0->getFlags());
4967 return DAG.getNode(ISD::FP_ROUND, SL, VT,
4976 return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1));
5025 SDValue Result = DAG.getNode(ISD::BITCAST, SL, VT, Build);
5028 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Result));
5032 if (BCSrc.getOpcode() == ISD::SELECT && VT == MVT::f32 &&
5308 EVT VT = N->getValueType(0);
5330 return DAG.getConstantFP(FTZ(V0), DL, VT);
5344 Register Reg, EVT VT,
5359 return DAG.getRegister(VReg, VT);
5361 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, VReg, VT);
5379 EVT VT,
5384 int FI = getOrCreateFixedStackObject(MFI, VT.getStoreSize(), Offset);
5389 return DAG.getLoad(VT, SL, DAG.getEntryNode(), Ptr, SrcPtrInfo, Align(4),
5415 EVT VT, const SDLoc &SL,
5420 CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL) :
5421 loadStackInputValue(DAG, VT, SL, Arg.getStackOffset());
5428 V = DAG.getNode(ISD::SRL, SL, VT, V,
5429 DAG.getShiftAmountConstant(Shift, VT, SL));
5430 return DAG.getNode(ISD::AND, SL, VT, V,
5431 DAG.getConstant(Mask >> Shift, SL, VT));
5622 EVT VT = Operand.getValueType();
5624 if (VT == MVT::f32) {
5626 return DAG.getNode(AMDGPUISD::RSQ, SDLoc(Operand), VT, Operand);
5638 EVT VT = Operand.getValueType();
5640 if (VT == MVT::f32) {
5647 return DAG.getNode(AMDGPUISD::RCP, SDLoc(Operand), VT, Operand);