Lines Matching defs:DAG
1 //===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
51 unsigned AMDGPUTargetLowering::numBitsUnsigned(SDValue Op, SelectionDAG &DAG) {
52 return DAG.computeKnownBits(Op).countMaxActiveBits();
55 unsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) {
58 return DAG.ComputeMaxSignificantBits(Op);
70 // Enable ganging up loads and stores in the memcpy DAG lowering.
863 const SelectionDAG &DAG,
878 return allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
919 SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize,
935 SDValue NegSrc = getNegatedExpression(Src, DAG, LegalOperations,
938 return DAG.getNode(AMDGPUISD::RCP, SL, VT, NegSrc, Op->getFlags());
945 return TargetLowering::getNegatedExpression(Op, DAG, LegalOperations,
1315 const SDLoc &DL, SelectionDAG &DAG) const {
1319 return DAG.getNode(AMDGPUISD::ENDPGM, DL, MVT::Other, Chain);
1338 SelectionDAG &DAG,
1351 for (SDNode *U : DAG.getEntryNode().getNode()->users()) {
1368 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
1375 SelectionDAG &DAG = CLI.DAG;
1377 const Function &Fn = DAG.getMachineFunction().getFunction();
1388 DAG.getContext()->diagnose(NoCalls);
1392 InVals.push_back(DAG.getUNDEF(Arg.VT));
1395 return DAG.getEntryNode();
1404 SelectionDAG &DAG) const {
1405 const Function &Fn = DAG.getMachineFunction().getFunction();
1409 DAG.getContext()->diagnose(NoDynamicAlloca);
1410 auto Ops = {DAG.getConstant(0, SDLoc(), Op.getValueType()), Op.getOperand(0)};
1411 return DAG.getMergeValues(Ops, SDLoc());
1415 SelectionDAG &DAG) const {
1418 Op->print(errs(), &DAG);
1422 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
1423 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
1424 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
1425 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
1426 case ISD::SDIVREM: return LowerSDIVREM(Op, DAG);
1427 case ISD::FREM: return LowerFREM(Op, DAG);
1428 case ISD::FCEIL: return LowerFCEIL(Op, DAG);
1429 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
1430 case ISD::FRINT: return LowerFRINT(Op, DAG);
1431 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
1433 return LowerFROUNDEVEN(Op, DAG);
1434 case ISD::FROUND: return LowerFROUND(Op, DAG);
1435 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
1437 return LowerFLOG2(Op, DAG);
1440 return LowerFLOGCommon(Op, DAG);
1443 return lowerFEXP(Op, DAG);
1445 return lowerFEXP2(Op, DAG);
1446 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
1447 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
1448 case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
1451 return LowerFP_TO_INT(Op, DAG);
1456 return LowerCTLZ_CTTZ(Op, DAG);
1457 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
1464 SelectionDAG &DAG) const {
1475 if (SDValue Lowered = LowerFLOG2(SDValue(N, 0), DAG))
1480 if (SDValue Lowered = LowerFLOGCommon(SDValue(N, 0), DAG))
1484 if (SDValue Lowered = lowerFEXP2(SDValue(N, 0), DAG))
1489 if (SDValue Lowered = lowerFEXP(SDValue(N, 0), DAG))
1494 if (auto Lowered = lowerCTLZResults(SDValue(N, 0u), DAG))
1504 SelectionDAG &DAG) const {
1506 const DataLayout &DL = DAG.getDataLayout();
1513 return DAG.getConstant(*Address, SDLoc(Op), Op.getValueType());
1523 const Function &Fn = DAG.getMachineFunction().getFunction();
1527 DAG.getContext()->diagnose(BadLDSDecl);
1534 SDValue Trap = DAG.getNode(ISD::TRAP, DL, MVT::Other, DAG.getEntryNode());
1535 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
1536 Trap, DAG.getRoot());
1537 DAG.setRoot(OutputChain);
1538 return DAG.getUNDEF(Op.getValueType());
1549 return DAG.getConstant(Offset, SDLoc(Op), Op.getValueType());
1555 SelectionDAG &DAG) const {
1565 : EVT::getVectorVT(*DAG.getContext(),
1569 SDValue NewIn = DAG.getNode(ISD::BITCAST, SL, NewEltVT, In);
1571 DAG.ExtractVectorElements(NewIn, Args);
1576 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
1578 SDValue BV = DAG.getBuildVector(NewVT, SL, Args);
1579 return DAG.getNode(ISD::BITCAST, SL, VT, BV);
1584 DAG.ExtractVectorElements(U.get(), Args);
1586 return DAG.getBuildVector(Op.getValueType(), SL, Args);
1590 SelectionDAG &DAG) const {
1603 EVT NewSrcVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumSrcElt / 2);
1606 : EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElt / 2);
1607 SDValue Tmp = DAG.getNode(ISD::BITCAST, SL, NewSrcVT, Op.getOperand(0));
1609 DAG.ExtractVectorElements(Tmp, Args, Start / 2, NumElt / 2);
1613 Tmp = DAG.getBuildVector(NewVT, SL, Args);
1615 return DAG.getNode(ISD::BITCAST, SL, VT, Tmp);
1618 DAG.ExtractVectorElements(Op.getOperand(0), Args, Start,
1621 return DAG.getBuildVector(Op.getValueType(), SL, Args);
1645 SelectionDAG &DAG = DCI.DAG;
1664 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS);
1665 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS);
1683 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS);
1684 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS);
1689 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS);
1690 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS);
1701 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS);
1702 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS);
1719 SelectionDAG &DAG = DCI.DAG;
1741 return DAG.getNode(ISD::FNEG, DL, VT, Combined);
1750 AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const {
1753 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1755 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1756 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1758 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
1759 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
1764 SDValue AMDGPUTargetLowering::getLoHalf64(SDValue Op, SelectionDAG &DAG) const {
1767 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1768 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1769 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
1772 SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const {
1775 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1776 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1777 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
1784 AMDGPUTargetLowering::getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const {
1789 LoVT = EVT::getVectorVT(*DAG.getContext(), EltVT, LoNumElts);
1792 : EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts - LoNumElts);
1801 SelectionDAG &DAG) const {
1806 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
1807 DAG.getVectorIdxConstant(0, DL));
1808 SDValue Hi = DAG.getNode(
1810 HiVT, N, DAG.getVectorIdxConstant(LoVT.getVectorNumElements(), DL));
1815 SelectionDAG &DAG) const {
1825 std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(Load, DAG);
1826 return DAG.getMergeValues(Ops, SL);
1838 std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG);
1839 std::tie(LoMemVT, HiMemVT) = getSplitDestVTs(MemVT, DAG);
1840 std::tie(Lo, Hi) = splitVector(Op, SL, LoVT, HiVT, DAG);
1846 SDValue LoLoad = DAG.getExtLoad(Load->getExtensionType(), SL, LoVT,
1849 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Size));
1851 DAG.getExtLoad(Load->getExtensionType(), SL, HiVT, Load->getChain(),
1858 Join = DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad);
1860 Join = DAG.getNode(ISD::INSERT_SUBVECTOR, SL, VT, DAG.getUNDEF(VT), LoLoad,
1861 DAG.getVectorIdxConstant(0, SL));
1862 Join = DAG.getNode(
1865 DAG.getVectorIdxConstant(LoVT.getVectorNumElements(), SL));
1868 SDValue Ops[] = {Join, DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
1871 return DAG.getMergeValues(Ops, SL);
1875 SelectionDAG &DAG) const {
1889 !SrcValue.isDereferenceable(16, *DAG.getContext(), DAG.getDataLayout())))
1890 return SplitVectorLoad(Op, DAG);
1895 EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 4);
1897 EVT::getVectorVT(*DAG.getContext(), MemVT.getVectorElementType(), 4);
1898 SDValue WideLoad = DAG.getExtLoad(
1901 return DAG.getMergeValues(
1902 {DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, VT, WideLoad,
1903 DAG.getVectorIdxConstant(0, SL)),
1909 SelectionDAG &DAG) const {
1917 return scalarizeVectorStore(Store, DAG);
1928 std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG);
1929 std::tie(LoMemVT, HiMemVT) = getSplitDestVTs(MemVT, DAG);
1930 std::tie(Lo, Hi) = splitVector(Val, SL, LoVT, HiVT, DAG);
1932 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, LoMemVT.getStoreSize());
1940 DAG.getTruncStore(Chain, SL, Lo, BasePtr, SrcValue, LoMemVT, BaseAlign,
1943 DAG.getTruncStore(Chain, SL, Hi, HiPtr, SrcValue.getWithOffset(Size),
1946 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore);
1952 SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG,
1961 unsigned LHSSignBits = DAG.ComputeNumSignBits(LHS);
1965 unsigned RHSSignBits = DAG.ComputeNumSignBits(RHS);
1978 SDValue jq = DAG.getConstant(1, DL, IntVT);
1982 jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
1985 jq = DAG.getNode(ISD::SRA, DL, VT, jq,
1986 DAG.getConstant(BitSize - 2, DL, VT));
1989 jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT));
1999 SDValue fa = DAG.getNode(ToFp, DL, FltVT, ia);
2002 SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib);
2004 SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT,
2005 fa, DAG.getNode(AMDGPUISD::RCP, DL, FltVT, fb));
2008 fq = DAG.getNode(ISD::FTRUNC, DL, FltVT, fq);
2011 SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq);
2013 MachineFunction &MF = DAG.getMachineFunction();
2026 SDValue fr = DAG.getNode(OpCode, DL, FltVT, fqneg, fb, fa);
2029 SDValue iq = DAG.getNode(ToInt, DL, IntVT, fq);
2032 fr = DAG.getNode(ISD::FABS, DL, FltVT, fr);
2035 fb = DAG.getNode(ISD::FABS, DL, FltVT, fb);
2037 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2040 SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE);
2043 jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT));
2046 SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq);
2049 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS);
2050 Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem);
2055 = DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), DivBits));
2056 Div = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Div, InRegSize);
2057 Rem = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Rem, InRegSize);
2059 SDValue TruncMask = DAG.getConstant((UINT64_C(1) << DivBits) - 1, DL, VT);
2060 Div = DAG.getNode(ISD::AND, DL, VT, Div, TruncMask);
2061 Rem = DAG.getNode(ISD::AND, DL, VT, Rem, TruncMask);
2064 return DAG.getMergeValues({ Div, Rem }, DL);
2068 SelectionDAG &DAG,
2075 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
2077 SDValue One = DAG.getConstant(1, DL, HalfVT);
2078 SDValue Zero = DAG.getConstant(0, DL, HalfVT);
2083 std::tie(LHS_Lo, LHS_Hi) = DAG.SplitScalar(LHS, DL, HalfVT, HalfVT);
2087 std::tie(RHS_Lo, RHS_Hi) = DAG.SplitScalar(RHS, DL, HalfVT, HalfVT);
2089 if (DAG.MaskedValueIsZero(RHS, APInt::getHighBitsSet(64, 32)) &&
2090 DAG.MaskedValueIsZero(LHS, APInt::getHighBitsSet(64, 32))) {
2092 SDValue Res = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
2095 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(0), Zero});
2096 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(1), Zero});
2098 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV));
2099 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM));
2107 MachineFunction &MF = DAG.getMachineFunction();
2117 SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo);
2118 SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi);
2119 SDValue Mad1 = DAG.getNode(FMAD, DL, MVT::f32, Cvt_Hi,
2120 DAG.getConstantFP(APInt(32, 0x4f800000).bitsToFloat(), DL, MVT::f32),
2122 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, DL, MVT::f32, Mad1);
2123 SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp,
2124 DAG.getConstantFP(APInt(32, 0x5f7ffffc).bitsToFloat(), DL, MVT::f32));
2125 SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1,
2126 DAG.getConstantFP(APInt(32, 0x2f800000).bitsToFloat(), DL, MVT::f32));
2127 SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2);
2128 SDValue Mad2 = DAG.getNode(FMAD, DL, MVT::f32, Trunc,
2129 DAG.getConstantFP(APInt(32, 0xcf800000).bitsToFloat(), DL, MVT::f32),
2131 SDValue Rcp_Lo = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Mad2);
2132 SDValue Rcp_Hi = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Trunc);
2133 SDValue Rcp64 = DAG.getBitcast(VT,
2134 DAG.getBuildVector(MVT::v2i32, DL, {Rcp_Lo, Rcp_Hi}));
2136 SDValue Zero64 = DAG.getConstant(0, DL, VT);
2137 SDValue One64 = DAG.getConstant(1, DL, VT);
2138 SDValue Zero1 = DAG.getConstant(0, DL, MVT::i1);
2139 SDVTList HalfCarryVT = DAG.getVTList(HalfVT, MVT::i1);
2142 SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS);
2143 SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64);
2144 SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1);
2147 DAG.SplitScalar(Mulhi1, DL, HalfVT, HalfVT);
2148 SDValue Add1_Lo = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Rcp_Lo,
2150 SDValue Add1_Hi = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Rcp_Hi,
2152 SDValue Add1 = DAG.getBitcast(VT,
2153 DAG.getBuildVector(MVT::v2i32, DL, {Add1_Lo, Add1_Hi}));
2156 SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1);
2157 SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2);
2160 DAG.SplitScalar(Mulhi2, DL, HalfVT, HalfVT);
2161 SDValue Add2_Lo = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Add1_Lo,
2163 SDValue Add2_Hi = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Add1_Hi,
2165 SDValue Add2 = DAG.getBitcast(VT,
2166 DAG.getBuildVector(MVT::v2i32, DL, {Add2_Lo, Add2_Hi}));
2168 SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2);
2170 SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3);
2173 std::tie(Mul3_Lo, Mul3_Hi) = DAG.SplitScalar(Mul3, DL, HalfVT, HalfVT);
2174 SDValue Sub1_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Lo,
2176 SDValue Sub1_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Hi,
2178 SDValue Sub1_Mi = DAG.getNode(ISD::SUB, DL, HalfVT, LHS_Hi, Mul3_Hi);
2179 SDValue Sub1 = DAG.getBitcast(VT,
2180 DAG.getBuildVector(MVT::v2i32, DL, {Sub1_Lo, Sub1_Hi}));
2182 SDValue MinusOne = DAG.getConstant(0xffffffffu, DL, HalfVT);
2183 SDValue C1 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, MinusOne, Zero,
2185 SDValue C2 = DAG.getSelectCC(DL, Sub1_Lo, RHS_Lo, MinusOne, Zero,
2187 SDValue C3 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, C2, C1, ISD::SETEQ);
2194 SDValue Sub2_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Lo,
2196 SDValue Sub2_Mi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Mi,
2198 SDValue Sub2_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Mi,
2200 SDValue Sub2 = DAG.getBitcast(VT,
2201 DAG.getBuildVector(MVT::v2i32, DL, {Sub2_Lo, Sub2_Hi}));
2203 SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64);
2205 SDValue C4 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, MinusOne, Zero,
2207 SDValue C5 = DAG.getSelectCC(DL, Sub2_Lo, RHS_Lo, MinusOne, Zero,
2209 SDValue C6 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, C5, C4, ISD::SETEQ);
2212 SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64);
2214 SDValue Sub3_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Lo,
2216 SDValue Sub3_Mi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Mi,
2218 SDValue Sub3_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub3_Mi,
2220 SDValue Sub3 = DAG.getBitcast(VT,
2221 DAG.getBuildVector(MVT::v2i32, DL, {Sub3_Lo, Sub3_Hi}));
2226 SDValue Sel1 = DAG.getSelectCC(DL, C6, Zero, Add4, Add3, ISD::SETNE);
2227 SDValue Div = DAG.getSelectCC(DL, C3, Zero, Sel1, Mulhi3, ISD::SETNE);
2229 SDValue Sel2 = DAG.getSelectCC(DL, C6, Zero, Sub3, Sub2, ISD::SETNE);
2230 SDValue Rem = DAG.getSelectCC(DL, C3, Zero, Sel2, Sub1, ISD::SETNE);
2240 SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo);
2241 SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo);
2243 SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, Zero, REM_Part, LHS_Hi, ISD::SETEQ);
2244 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, Zero});
2245 REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM);
2247 SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, Zero, DIV_Part, Zero, ISD::SETEQ);
2254 SDValue POS = DAG.getConstant(bitPos, DL, HalfVT);
2256 SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS);
2257 HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, One);
2258 HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit);
2261 REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT));
2263 REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit);
2265 SDValue BIT = DAG.getConstant(1ULL << bitPos, DL, HalfVT);
2266 SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, Zero, ISD::SETUGE);
2268 DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT);
2271 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS);
2272 REM = DAG.getSelectCC(DL, REM, RHS, REM_sub, REM, ISD::SETUGE);
2275 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {DIV_Lo, DIV_Hi});
2276 DIV = DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV);
2282 SelectionDAG &DAG) const {
2288 LowerUDIVREM64(Op, DAG, Results);
2289 return DAG.getMergeValues(Results, DL);
2293 if (SDValue Res = LowerDIVREM24(Op, DAG, false))
2304 SDValue Z = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Y);
2307 SDValue NegY = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Y);
2308 SDValue NegYZ = DAG.getNode(ISD::MUL, DL, VT, NegY, Z);
2309 Z = DAG.getNode(ISD::ADD, DL, VT, Z,
2310 DAG.getNode(ISD::MULHU, DL, VT, Z, NegYZ));
2313 SDValue Q = DAG.getNode(ISD::MULHU, DL, VT, X, Z);
2315 DAG.getNode(ISD::SUB, DL, VT, X, DAG.getNode(ISD::MUL, DL, VT, Q, Y));
2318 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2319 SDValue One = DAG.getConstant(1, DL, VT);
2320 SDValue Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE);
2321 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond,
2322 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q);
2323 R = DAG.getNode(ISD::SELECT, DL, VT, Cond,
2324 DAG.getNode(ISD::SUB, DL, VT, R, Y), R);
2327 Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE);
2328 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond,
2329 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q);
2330 R = DAG.getNode(ISD::SELECT, DL, VT, Cond,
2331 DAG.getNode(ISD::SUB, DL, VT, R, Y), R);
2333 return DAG.getMergeValues({Q, R}, DL);
2337 SelectionDAG &DAG) const {
2344 SDValue Zero = DAG.getConstant(0, DL, VT);
2345 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
2348 if (SDValue Res = LowerDIVREM24(Op, DAG, true))
2353 DAG.ComputeNumSignBits(LHS) > 32 &&
2354 DAG.ComputeNumSignBits(RHS) > 32) {
2355 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
2358 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
2359 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
2360 SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
2363 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)),
2364 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1))
2366 return DAG.getMergeValues(Res, DL);
2369 SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT);
2370 SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT);
2371 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign);
2374 LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign);
2375 RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign);
2377 LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign);
2378 RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign);
2380 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS);
2383 Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign);
2384 Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign);
2386 Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign);
2387 Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign);
2393 return DAG.getMergeValues(Res, DL);
2397 SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const {
2404 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y, Flags);
2405 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, VT, Div, Flags);
2406 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Trunc, Flags);
2408 return DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags);
2411 SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
2419 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2421 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
2422 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
2425 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
2427 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT);
2428 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
2429 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
2431 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero);
2433 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
2437 SelectionDAG &DAG) {
2441 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
2443 DAG.getConstant(FractBits - 32, SL, MVT::i32),
2444 DAG.getConstant(ExpBits, SL, MVT::i32));
2445 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart,
2446 DAG.getConstant(1023, SL, MVT::i32));
2451 SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
2457 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2461 SDValue Hi = getHiHalf64(Src, DAG);
2463 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
2468 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32);
2469 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask);
2472 SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit});
2473 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64);
2475 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src);
2477 = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64);
2479 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp);
2480 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64);
2481 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not);
2484 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32);
2486 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32);
2488 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
2489 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
2491 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0);
2492 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1);
2494 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2);
2498 SelectionDAG &DAG) const {
2505 SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64);
2506 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src);
2510 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign);
2511 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign);
2513 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src);
2516 SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64);
2519 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
2520 SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT);
2522 return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2);
2526 SelectionDAG &DAG) const {
2530 return DAG.getNode(ISD::FROUNDEVEN, SDLoc(Op), Op.getValueType(),
2534 SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
2537 return DAG.getNode(ISD::FROUNDEVEN, SDLoc(Op), VT, Arg);
2545 SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2550 SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X);
2554 SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T);
2556 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff);
2558 const SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
2559 const SDValue One = DAG.getConstantFP(1.0, SL, VT);
2562 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2564 const SDValue Half = DAG.getConstantFP(0.5, SL, VT);
2565 SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE);
2566 SDValue OneOrZeroFP = DAG.getNode(ISD::SELECT, SL, VT, Cmp, One, Zero);
2568 SDValue SignedOffset = DAG.getNode(ISD::FCOPYSIGN, SL, VT, OneOrZeroFP, X);
2569 return DAG.getNode(ISD::FADD, SL, VT, T, SignedOffset);
2572 SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
2580 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2582 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
2583 const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64);
2586 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
2588 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT);
2589 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
2590 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
2592 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero);
2594 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
2621 bool AMDGPUTargetLowering::allowApproxFunc(const SelectionDAG &DAG,
2625 auto &Options = DAG.getTarget().Options;
2629 bool AMDGPUTargetLowering::needsDenormHandlingF32(const SelectionDAG &DAG,
2633 DAG.getMachineFunction()
2638 SDValue AMDGPUTargetLowering::getIsLtSmallestNormal(SelectionDAG &DAG,
2645 DAG.getConstantFP(APFloat::getSmallestNormalized(Semantics), SL, VT);
2649 SDValue IsLtSmallestNormal = DAG.getSetCC(
2650 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Src,
2656 SDValue AMDGPUTargetLowering::getIsFinite(SelectionDAG &DAG, SDValue Src,
2661 SDValue Inf = DAG.getConstantFP(APFloat::getInf(Semantics), SL, VT);
2663 SDValue Fabs = DAG.getNode(ISD::FABS, SL, VT, Src, Flags);
2664 SDValue IsFinite = DAG.getSetCC(
2665 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Fabs,
2673 AMDGPUTargetLowering::getScaledLogInput(SelectionDAG &DAG, const SDLoc SL,
2675 if (!needsDenormHandlingF32(DAG, Src, Flags))
2681 DAG.getConstantFP(APFloat::getSmallestNormalized(Semantics), SL, VT);
2683 SDValue IsLtSmallestNormal = DAG.getSetCC(
2684 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Src,
2687 SDValue Scale32 = DAG.getConstantFP(0x1.0p+32, SL, VT);
2688 SDValue One = DAG.getConstantFP(1.0, SL, VT);
2690 DAG.getNode(ISD::SELECT, SL, VT, IsLtSmallestNormal, Scale32, One, Flags);
2692 SDValue ScaledInput = DAG.getNode(ISD::FMUL, SL, VT, Src, ScaleFactor, Flags);
2696 SDValue AMDGPUTargetLowering::LowerFLOG2(SDValue Op, SelectionDAG &DAG) const {
2711 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src, Flags);
2712 SDValue Log = DAG.getNode(AMDGPUISD::LOG, SL, MVT::f32, Ext, Flags);
2713 return DAG.getNode(ISD::FP_ROUND, SL, VT, Log,
2714 DAG.getTargetConstant(0, SL, MVT::i32), Flags);
2718 getScaledLogInput(DAG, SL, Src, Flags);
2720 return DAG.getNode(AMDGPUISD::LOG, SL, VT, Src, Flags);
2722 SDValue Log2 = DAG.getNode(AMDGPUISD::LOG, SL, VT, ScaledInput, Flags);
2724 SDValue ThirtyTwo = DAG.getConstantFP(32.0, SL, VT);
2725 SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
2727 DAG.getNode(ISD::SELECT, SL, VT, IsLtSmallestNormal, ThirtyTwo, Zero);
2728 return DAG.getNode(ISD::FSUB, SL, VT, Log2, ResultOffset, Flags);
2731 static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X,
2733 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, X, Y, Flags);
2734 return DAG.getNode(ISD::FADD, SL, VT, Mul, C, Flags);
2738 SelectionDAG &DAG) const {
2753 X = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, X, Flags);
2756 SDValue Lowered = LowerFLOGUnsafe(X, DL, DAG, IsLog10, Flags);
2758 return DAG.getNode(ISD::FP_ROUND, DL, VT, Lowered,
2759 DAG.getTargetConstant(0, DL, MVT::i32), Flags);
2765 auto [ScaledInput, IsScaled] = getScaledLogInput(DAG, DL, X, Flags);
2769 SDValue Y = DAG.getNode(AMDGPUISD::LOG, DL, VT, X, Flags);
2781 SDValue C = DAG.getConstantFP(IsLog10 ? c_log10 : c_log, DL, VT);
2782 SDValue CC = DAG.getConstantFP(IsLog10 ? cc_log10 : cc_log, DL, VT);
2784 R = DAG.getNode(ISD::FMUL, DL, VT, Y, C, Flags);
2785 SDValue NegR = DAG.getNode(ISD::FNEG, DL, VT, R, Flags);
2786 SDValue FMA0 = DAG.getNode(ISD::FMA, DL, VT, Y, C, NegR, Flags);
2787 SDValue FMA1 = DAG.getNode(ISD::FMA, DL, VT, Y, CC, FMA0, Flags);
2788 R = DAG.getNode(ISD::FADD, DL, VT, R, FMA1, Flags);
2798 SDValue CH = DAG.getConstantFP(IsLog10 ? ch_log10 : ch_log, DL, VT);
2799 SDValue CT = DAG.getConstantFP(IsLog10 ? ct_log10 : ct_log, DL, VT);
2801 SDValue YAsInt = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Y);
2802 SDValue MaskConst = DAG.getConstant(0xfffff000, DL, MVT::i32);
2803 SDValue YHInt = DAG.getNode(ISD::AND, DL, MVT::i32, YAsInt, MaskConst);
2804 SDValue YH = DAG.getNode(ISD::BITCAST, DL, MVT::f32, YHInt);
2805 SDValue YT = DAG.getNode(ISD::FSUB, DL, VT, Y, YH, Flags);
2807 SDValue YTCT = DAG.getNode(ISD::FMUL, DL, VT, YT, CT, Flags);
2808 SDValue Mad0 = getMad(DAG, DL, VT, YH, CT, YTCT, Flags);
2809 SDValue Mad1 = getMad(DAG, DL, VT, YT, CH, Mad0, Flags);
2810 R = getMad(DAG, DL, VT, YH, CH, Mad1);
2818 SDValue IsFinite = getIsFinite(DAG, Y, Flags);
2819 R = DAG.getNode(ISD::SELECT, DL, VT, IsFinite, R, Y, Flags);
2823 SDValue Zero = DAG.getConstantFP(0.0f, DL, VT);
2825 DAG.getConstantFP(IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f, DL, VT);
2827 DAG.getNode(ISD::SELECT, DL, VT, IsScaled, ShiftK, Zero, Flags);
2828 R = DAG.getNode(ISD::FSUB, DL, VT, R, Shift, Flags);
2834 SDValue AMDGPUTargetLowering::LowerFLOG10(SDValue Op, SelectionDAG &DAG) const {
2835 return LowerFLOGCommon(Op, DAG);
2841 SelectionDAG &DAG, bool IsLog10,
2851 auto [ScaledInput, IsScaled] = getScaledLogInput(DAG, SL, Src, Flags);
2853 SDValue LogSrc = DAG.getNode(AMDGPUISD::LOG, SL, VT, ScaledInput, Flags);
2855 DAG.getConstantFP(-32.0 * Log2BaseInverted, SL, VT);
2857 SDValue Zero = DAG.getConstantFP(0.0f, SL, VT);
2859 SDValue ResultOffset = DAG.getNode(ISD::SELECT, SL, VT, IsScaled,
2862 SDValue Log2Inv = DAG.getConstantFP(Log2BaseInverted, SL, VT);
2865 return DAG.getNode(ISD::FMA, SL, VT, LogSrc, Log2Inv, ResultOffset,
2867 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, LogSrc, Log2Inv, Flags);
2868 return DAG.getNode(ISD::FADD, SL, VT, Mul, ResultOffset);
2872 SDValue Log2Operand = DAG.getNode(LogOp, SL, VT, Src, Flags);
2873 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Log2BaseInverted, SL, VT);
2875 return DAG.getNode(ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand,
2879 SDValue AMDGPUTargetLowering::lowerFEXP2(SDValue Op, SelectionDAG &DAG) const {
2891 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src, Flags);
2892 SDValue Log = DAG.getNode(AMDGPUISD::EXP, SL, MVT::f32, Ext, Flags);
2893 return DAG.getNode(ISD::FP_ROUND, SL, VT, Log,
2894 DAG.getTargetConstant(0, SL, MVT::i32), Flags);
2899 if (!needsDenormHandlingF32(DAG, Src, Flags))
2900 return DAG.getNode(AMDGPUISD::EXP, SL, MVT::f32, Src, Flags);
2906 SDValue RangeCheckConst = DAG.getConstantFP(-0x1.f80000p+6f, SL, VT);
2908 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2911 DAG.getSetCC(SL, SetCCVT, Src, RangeCheckConst, ISD::SETOLT);
2913 SDValue SixtyFour = DAG.getConstantFP(0x1.0p+6f, SL, VT);
2914 SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
2917 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, SixtyFour, Zero);
2919 SDValue AddInput = DAG.getNode(ISD::FADD, SL, VT, Src, AddOffset, Flags);
2920 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, AddInput, Flags);
2922 SDValue TwoExpNeg64 = DAG.getConstantFP(0x1.0p-64f, SL, VT);
2923 SDValue One = DAG.getConstantFP(1.0, SL, VT);
2925 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, TwoExpNeg64, One);
2927 return DAG.getNode(ISD::FMUL, SL, VT, Exp2, ResultScale, Flags);
2931 SelectionDAG &DAG,
2934 const SDValue Log2E = DAG.getConstantFP(numbers::log2e, SL, VT);
2936 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, X, Flags)) {
2938 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, X, Log2E, Flags);
2939 return DAG.getNode(VT == MVT::f32 ? (unsigned)AMDGPUISD::EXP
2944 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2946 SDValue Threshold = DAG.getConstantFP(-0x1.5d58a0p+6f, SL, VT);
2947 SDValue NeedsScaling = DAG.getSetCC(SL, SetCCVT, X, Threshold, ISD::SETOLT);
2949 SDValue ScaleOffset = DAG.getConstantFP(0x1.0p+6f, SL, VT);
2951 SDValue ScaledX = DAG.getNode(ISD::FADD, SL, VT, X, ScaleOffset, Flags);
2954 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, ScaledX, X);
2956 SDValue ExpInput = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, Log2E, Flags);
2958 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, ExpInput, Flags);
2960 SDValue ResultScaleFactor = DAG.getConstantFP(0x1.969d48p-93f, SL, VT);
2962 DAG.getNode(ISD::FMUL, SL, VT, Exp2, ResultScaleFactor, Flags);
2964 return DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, AdjustedResult, Exp2,
2971 SelectionDAG &DAG,
2976 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, X, Flags)) {
2978 SDValue K0 = DAG.getConstantFP(0x1.a92000p+1f, SL, VT);
2979 SDValue K1 = DAG.getConstantFP(0x1.4f0978p-11f, SL, VT);
2981 SDValue Mul0 = DAG.getNode(ISD::FMUL, SL, VT, X, K0, Flags);
2982 SDValue Exp2_0 = DAG.getNode(Exp2Op, SL, VT, Mul0, Flags);
2983 SDValue Mul1 = DAG.getNode(ISD::FMUL, SL, VT, X, K1, Flags);
2984 SDValue Exp2_1 = DAG.getNode(Exp2Op, SL, VT, Mul1, Flags);
2985 return DAG.getNode(ISD::FMUL, SL, VT, Exp2_0, Exp2_1);
2994 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2996 SDValue Threshold = DAG.getConstantFP(-0x1.2f7030p+5f, SL, VT);
2997 SDValue NeedsScaling = DAG.getSetCC(SL, SetCCVT, X, Threshold, ISD::SETOLT);
2999 SDValue ScaleOffset = DAG.getConstantFP(0x1.0p+5f, SL, VT);
3000 SDValue ScaledX = DAG.getNode(ISD::FADD, SL, VT, X, ScaleOffset, Flags);
3002 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, ScaledX, X);
3004 SDValue K0 = DAG.getConstantFP(0x1.a92000p+1f, SL, VT);
3005 SDValue K1 = DAG.getConstantFP(0x1.4f0978p-11f, SL, VT);
3007 SDValue Mul0 = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, K0, Flags);
3008 SDValue Exp2_0 = DAG.getNode(Exp2Op, SL, VT, Mul0, Flags);
3009 SDValue Mul1 = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, K1, Flags);
3010 SDValue Exp2_1 = DAG.getNode(Exp2Op, SL, VT, Mul1, Flags);
3012 SDValue MulExps = DAG.getNode(ISD::FMUL, SL, VT, Exp2_0, Exp2_1, Flags);
3014 SDValue ResultScaleFactor = DAG.getConstantFP(0x1.9f623ep-107f, SL, VT);
3016 DAG.getNode(ISD::FMUL, SL, VT, MulExps, ResultScaleFactor, Flags);
3018 return DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, AdjustedResult, MulExps,
3022 SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const {
3031 if (allowApproxFunc(DAG, Flags)) // TODO: Does this really require fast?
3032 return lowerFEXPUnsafe(X, SL, DAG, Flags);
3041 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, X, Flags);
3042 SDValue Lowered = lowerFEXPUnsafe(Ext, SL, DAG, Flags);
3043 return DAG.getNode(ISD::FP_ROUND, SL, VT, Lowered,
3044 DAG.getTargetConstant(0, SL, MVT::i32), Flags);
3051 if (allowApproxFunc(DAG, Flags)) {
3052 return IsExp10 ? lowerFEXP10Unsafe(X, SL, DAG, Flags)
3053 : lowerFEXPUnsafe(X, SL, DAG, Flags);
3090 SDValue C = DAG.getConstantFP(IsExp10 ? c_exp10 : c_exp, SL, VT);
3091 SDValue CC = DAG.getConstantFP(IsExp10 ? cc_exp10 : cc_exp, SL, VT);
3093 PH = DAG.getNode(ISD::FMUL, SL, VT, X, C, Flags);
3094 SDValue NegPH = DAG.getNode(ISD::FNEG, SL, VT, PH, Flags);
3095 SDValue FMA0 = DAG.getNode(ISD::FMA, SL, VT, X, C, NegPH, Flags);
3096 PL = DAG.getNode(ISD::FMA, SL, VT, X, CC, FMA0, Flags);
3104 SDValue CH = DAG.getConstantFP(IsExp10 ? ch_exp10 : ch_exp, SL, VT);
3105 SDValue CL = DAG.getConstantFP(IsExp10 ? cl_exp10 : cl_exp, SL, VT);
3107 SDValue XAsInt = DAG.getNode(ISD::BITCAST, SL, MVT::i32, X);
3108 SDValue MaskConst = DAG.getConstant(0xfffff000, SL, MVT::i32);
3109 SDValue XHAsInt = DAG.getNode(ISD::AND, SL, MVT::i32, XAsInt, MaskConst);
3110 SDValue XH = DAG.getNode(ISD::BITCAST, SL, VT, XHAsInt);
3111 SDValue XL = DAG.getNode(ISD::FSUB, SL, VT, X, XH, Flags);
3113 PH = DAG.getNode(ISD::FMUL, SL, VT, XH, CH, Flags);
3115 SDValue XLCL = DAG.getNode(ISD::FMUL, SL, VT, XL, CL, Flags);
3116 SDValue Mad0 = getMad(DAG, SL, VT, XL, CH, XLCL, Flags);
3117 PL = getMad(DAG, SL, VT, XH, CL, Mad0, Flags);
3120 SDValue E = DAG.getNode(ISD::FROUNDEVEN, SL, VT, PH, Flags);
3123 SDValue PHSubE = DAG.getNode(ISD::FSUB, SL, VT, PH, E, FlagsNoContract);
3125 SDValue A = DAG.getNode(ISD::FADD, SL, VT, PHSubE, PL, Flags);
3126 SDValue IntE = DAG.getNode(ISD::FP_TO_SINT, SL, MVT::i32, E);
3127 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, A, Flags);
3129 SDValue R = DAG.getNode(ISD::FLDEXP, SL, VT, Exp2, IntE, Flags);
3132 DAG.getConstantFP(IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f, SL, VT);
3134 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
3135 SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
3137 DAG.getSetCC(SL, SetCCVT, X, UnderflowCheckConst, ISD::SETOLT);
3139 R = DAG.getNode(ISD::SELECT, SL, VT, Underflow, Zero, R);
3144 DAG.getConstantFP(IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f, SL, VT);
3146 DAG.getSetCC(SL, SetCCVT, X, OverflowCheckConst, ISD::SETOGT);
3148 DAG.getConstantFP(APFloat::getInf(APFloat::IEEEsingle()), SL, VT);
3149 R = DAG.getNode(ISD::SELECT, SL, VT, Overflow, Inf, R);
3164 SelectionDAG &DAG) const {
3177 SDValue NumExtBits = DAG.getConstant(32u - NumBits, SL, MVT::i32);
3181 NewOp = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Arg);
3182 NewOp = DAG.getNode(ISD::SHL, SL, MVT::i32, NewOp, NumExtBits);
3183 NewOp = DAG.getNode(Opc, SL, MVT::i32, NewOp);
3185 NewOp = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Arg);
3186 NewOp = DAG.getNode(Opc, SL, MVT::i32, NewOp);
3187 NewOp = DAG.getNode(ISD::SUB, SL, MVT::i32, NewOp, NumExtBits);
3190 return DAG.getNode(ISD::TRUNCATE, SL, ResultVT, NewOp);
3193 SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const {
3216 SDValue NewOpr = DAG.getNode(NewOpc, SL, MVT::i32, Src);
3218 const SDValue ConstVal = DAG.getConstant(
3220 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, ConstVal);
3222 return DAG.getNode(ISD::ZERO_EXTEND, SL, Src.getValueType(), NewOpr);
3226 std::tie(Lo, Hi) = split64BitValue(Src, DAG);
3228 SDValue OprLo = DAG.getNode(NewOpc, SL, MVT::i32, Lo);
3229 SDValue OprHi = DAG.getNode(NewOpc, SL, MVT::i32, Hi);
3237 const SDValue Const32 = DAG.getConstant(32, SL, MVT::i32);
3239 OprLo = DAG.getNode(AddOpc, SL, MVT::i32, OprLo, Const32);
3241 OprHi = DAG.getNode(AddOpc, SL, MVT::i32, OprHi, Const32);
3244 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, OprLo, OprHi);
3246 const SDValue Const64 = DAG.getConstant(64, SL, MVT::i32);
3247 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const64);
3250 return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr);
3253 SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG,
3285 std::tie(Lo, Hi) = split64BitValue(Src, DAG);
3310 SDValue OppositeSign = DAG.getNode(
3311 ISD::SRA, SL, MVT::i32, DAG.getNode(ISD::XOR, SL, MVT::i32, Lo, Hi),
3312 DAG.getConstant(31, SL, MVT::i32));
3314 DAG.getNode(ISD::ADD, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32),
3317 ShAmt = DAG.getNode(AMDGPUISD::FFBH_I32, SL, MVT::i32, Hi);
3320 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, ShAmt,
3321 DAG.getConstant(1, SL, MVT::i32));
3322 ShAmt = DAG.getNode(ISD::UMIN, SL, MVT::i32, ShAmt, MaxShAmt);
3327 Sign = DAG.getNode(ISD::SRA, SL, MVT::i64, Src,
3328 DAG.getConstant(63, SL, MVT::i64));
3330 DAG.getNode(ISD::XOR, SL, MVT::i64,
3331 DAG.getNode(ISD::ADD, SL, MVT::i64, Src, Sign), Sign);
3332 std::tie(Lo, Hi) = split64BitValue(Abs, DAG);
3335 ShAmt = DAG.getNode(ISD::CTLZ, SL, MVT::i32, Hi);
3339 SDValue Norm = DAG.getNode(ISD::SHL, SL, MVT::i64, Src, ShAmt);
3341 std::tie(Lo, Hi) = split64BitValue(Norm, DAG);
3344 SDValue Adjust = DAG.getNode(ISD::UMIN, SL, MVT::i32,
3345 DAG.getConstant(1, SL, MVT::i32), Lo);
3347 Norm = DAG.getNode(ISD::OR, SL, MVT::i32, Hi, Adjust);
3351 SDValue FVal = DAG.getNode(Opc, SL, MVT::f32, Norm);
3355 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32),
3359 return DAG.getNode(ISD::FLDEXP, SL, MVT::f32, FVal, ShAmt);
3364 SDValue Exp = DAG.getNode(ISD::SHL, SL, MVT::i32, ShAmt,
3365 DAG.getConstant(23, SL, MVT::i32));
3367 DAG.getNode(ISD::ADD, SL, MVT::i32,
3368 DAG.getNode(ISD::BITCAST, SL, MVT::i32, FVal), Exp);
3371 Sign = DAG.getNode(ISD::SHL, SL, MVT::i32,
3372 DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Sign),
3373 DAG.getConstant(31, SL, MVT::i32));
3374 IVal = DAG.getNode(ISD::OR, SL, MVT::i32, IVal, Sign);
3376 return DAG.getNode(ISD::BITCAST, SL, MVT::f32, IVal);
3379 SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
3385 std::tie(Lo, Hi) = split64BitValue(Src, DAG);
3387 SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
3390 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo);
3392 SDValue LdExp = DAG.getNode(ISD::FLDEXP, SL, MVT::f64, CvtHi,
3393 DAG.getConstant(32, SL, MVT::i32));
3395 return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo);
3399 SelectionDAG &DAG) const {
3411 SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Src);
3412 return DAG.getNode(ISD::UINT_TO_FP, DL, DestVT, Ext);
3417 SDValue ToF32 = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f32, Src);
3418 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SL, /*isTarget=*/true);
3419 return DAG.getNode(ISD::FP_ROUND, SL, MVT::bf16, ToF32, FPRoundFlag);
3428 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
3430 DAG.getIntPtrConstant(0, SDLoc(Op), /*isTarget=*/true);
3432 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag);
3438 return LowerINT_TO_FP32(Op, DAG, false);
3441 return LowerINT_TO_FP64(Op, DAG, false);
3445 SelectionDAG &DAG) const {
3457 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, Src);
3458 return DAG.getNode(ISD::SINT_TO_FP, DL, DestVT, Ext);
3463 SDValue ToF32 = DAG.getNode(ISD::SINT_TO_FP, SL, MVT::f32, Src);
3464 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SL, /*isTarget=*/true);
3465 return DAG.getNode(ISD::FP_ROUND, SL, MVT::bf16, ToF32, FPRoundFlag);
3477 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
3479 DAG.getIntPtrConstant(0, SDLoc(Op), /*isTarget=*/true);
3481 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag);
3487 return LowerINT_TO_FP32(Op, DAG, true);
3490 return LowerINT_TO_FP64(Op, DAG, true);
3493 SDValue AMDGPUTargetLowering::LowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG,
3511 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, SrcVT, Src);
3519 Sign = DAG.getNode(ISD::SRA, SL, MVT::i32,
3520 DAG.getNode(ISD::BITCAST, SL, MVT::i32, Trunc),
3521 DAG.getConstant(31, SL, MVT::i32));
3522 Trunc = DAG.getNode(ISD::FABS, SL, SrcVT, Trunc);
3527 K0 = DAG.getConstantFP(
3530 K1 = DAG.getConstantFP(
3534 K0 = DAG.getConstantFP(
3536 K1 = DAG.getConstantFP(
3540 SDValue Mul = DAG.getNode(ISD::FMUL, SL, SrcVT, Trunc, K0);
3542 SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, SrcVT, Mul);
3544 SDValue Fma = DAG.getNode(ISD::FMA, SL, SrcVT, FloorMul, K1, Trunc);
3546 SDValue Hi = DAG.getNode((Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT
3549 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma);
3551 SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64,
3552 DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi}));
3557 Sign = DAG.getNode(ISD::BITCAST, SL, MVT::i64,
3558 DAG.getBuildVector(MVT::v2i32, SL, {Sign, Sign}));
3561 DAG.getNode(ISD::SUB, SL, MVT::i64,
3562 DAG.getNode(ISD::XOR, SL, MVT::i64, Result, Sign), Sign);
3568 SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
3574 return DAG.getNode(AMDGPUISD::FP_TO_FP16, DL, Op.getValueType(), N0);
3587 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
3588 SDValue One = DAG.getConstant(1, DL, MVT::i32);
3589 SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0);
3590 SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U,
3591 DAG.getConstant(32, DL, MVT::i64));
3592 UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32);
3593 U = DAG.getZExtOrTrunc(U, DL, MVT::i32);
3594 SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
3595 DAG.getConstant(20, DL, MVT::i64));
3596 E = DAG.getNode(ISD::AND, DL, MVT::i32, E,
3597 DAG.getConstant(ExpMask, DL, MVT::i32));
3600 E = DAG.getNode(ISD::ADD, DL, MVT::i32, E,
3601 DAG.getConstant(-ExpBiasf64 + ExpBiasf16, DL, MVT::i32));
3603 SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
3604 DAG.getConstant(8, DL, MVT::i32));
3605 M = DAG.getNode(ISD::AND, DL, MVT::i32, M,
3606 DAG.getConstant(0xffe, DL, MVT::i32));
3608 SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH,
3609 DAG.getConstant(0x1ff, DL, MVT::i32));
3610 MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U);
3612 SDValue Lo40Set = DAG.getSelectCC(DL, MaskedSig, Zero, Zero, One, ISD::SETEQ);
3613 M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set);
3616 SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32,
3617 DAG.getSelectCC(DL, M, Zero, DAG.getConstant(0x0200, DL, MVT::i32),
3618 Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32));
3621 SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M,
3622 DAG.getNode(ISD::SHL, DL, MVT::i32, E,
3623 DAG.getConstant(12, DL, MVT::i32)));
3626 SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32,
3628 SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero);
3629 B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B,
3630 DAG.getConstant(13, DL, MVT::i32));
3632 SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M,
3633 DAG.getConstant(0x1000, DL, MVT::i32));
3635 SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B);
3636 SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B);
3637 SDValue D1 = DAG.getSelectCC(DL, D0, SigSetHigh, One, Zero, ISD::SETNE);
3638 D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1);
3640 SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT);
3641 SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V,
3642 DAG.getConstant(0x7, DL, MVT::i32));
3643 V = DAG.getNode(ISD::SRL, DL, MVT::i32, V,
3644 DAG.getConstant(2, DL, MVT::i32));
3645 SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32),
3647 SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32),
3649 V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1);
3650 V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1);
3652 V = DAG.getSelectCC(DL, E, DAG.getConstant(30, DL, MVT::i32),
3653 DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT);
3654 V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32),
3658 SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
3659 DAG.getConstant(16, DL, MVT::i32));
3660 Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign,
3661 DAG.getConstant(0x8000, DL, MVT::i32));
3663 V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V);
3664 return DAG.getZExtOrTrunc(V, DL, Op.getValueType());
3668 SelectionDAG &DAG) const {
3680 SDValue PromotedSrc = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
3681 return DAG.getNode(Op.getOpcode(), DL, DestVT, PromotedSrc);
3688 SDValue FpToInt32 = DAG.getNode(OpOpcode, DL, MVT::i32, Src);
3689 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToInt32);
3699 SDValue FpToInt32 = DAG.getNode(OpOpcode, DL, MVT::i32, Src);
3702 return DAG.getNode(Ext, DL, MVT::i64, FpToInt32);
3706 return LowerFP_TO_INT64(Op, DAG, OpOpcode == ISD::FP_TO_SINT);
3712 SelectionDAG &DAG) const {
3725 DAG.ExtractVectorElements(Src, Args, 0, NElts);
3727 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
3729 Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp);
3731 return DAG.getBuildVector(VT, DL, Args);
3735 // Custom DAG optimizations
3738 static bool isU24(SDValue Op, SelectionDAG &DAG) {
3739 return AMDGPUTargetLowering::numBitsUnsigned(Op, DAG) <= 24;
3742 static bool isI24(SDValue Op, SelectionDAG &DAG) {
3746 AMDGPUTargetLowering::numBitsSigned(Op, DAG) <= 24;
3751 SelectionDAG &DAG = DCI.DAG;
3752 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3783 SDValue DemandedLHS = TLI.SimplifyMultipleUseDemandedBits(LHS, Demanded, DAG);
3784 SDValue DemandedRHS = TLI.SimplifyMultipleUseDemandedBits(RHS, Demanded, DAG);
3786 return DAG.getNode(NewOpcode, SDLoc(Node24), Node24->getVTList(),
3801 static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset,
3807 return DAG.getSignedConstant(Result, DL, MVT::i32);
3809 return DAG.getConstant(Result, DL, MVT::i32);
3813 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32);
3858 SelectionDAG &DAG = DCI.DAG;
3873 return SplitVectorLoad(SDValue(LN, 0), DAG);
3876 std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(LN, DAG);
3878 return DAG.getMergeValues(Ops, SDLoc(N));
3888 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
3891 = DAG.getLoad(NewVT, SL, LN->getChain(),
3894 SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad);
3914 SelectionDAG &DAG = DCI.DAG;
3927 return SplitVectorStore(SDValue(SN, 0), DAG);
3929 return expandUnalignedStore(SN, DAG);
3939 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
3945 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NewVT, Val);
3947 SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal);
3948 DAG.ReplaceAllUsesOfValueWith(Val, CastBack);
3951 return DAG.getStore(SN->getChain(), SL, CastVal,
3955 // FIXME: This should go in generic DAG combiner with an isTruncateFree check,
3960 SelectionDAG &DAG = DCI.DAG;
3973 SDValue NewInReg = DAG.getNode(N->getOpcode(), SL, SrcVT, Src, N1);
3974 return DAG.getNode(ISD::TRUNCATE, SL, N->getValueType(0), NewInReg);
4007 return SDValue(DCI.DAG.UpdateNodeOperands(N, N->getOperand(0), PeekSign),
4021 SelectionDAG &DAG = DCI.DAG;
4023 std::tie(Lo, Hi) = split64BitValue(LHS, DAG);
4025 SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32);
4026 SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32);
4028 SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS);
4029 SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS);
4036 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd});
4037 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
4054 SelectionDAG &DAG = DCI.DAG;
4068 SDValue Vec = DAG.getBuildVector(MVT::v2i16, SL,
4069 { DAG.getConstant(0, SL, MVT::i16), LHS->getOperand(0) });
4070 return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec);
4076 KnownBits Known = DAG.computeKnownBits(X);
4081 SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(RHS, 0));
4082 return DAG.getZExtOrTrunc(Shl, SL, VT);
4097 SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32);
4099 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
4100 SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt);
4102 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
4104 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift});
4105 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
4117 SelectionDAG &DAG = DCI.DAG;
4123 SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
4124 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
4125 DAG.getConstant(31, SL, MVT::i32));
4127 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift});
4128 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
4133 SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
4134 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
4135 DAG.getConstant(31, SL, MVT::i32));
4136 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift});
4137 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
4152 SelectionDAG &DAG = DCI.DAG;
4162 return DAG.getNode(
4164 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), N->getOperand(1)),
4165 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), N->getOperand(1)));
4179 SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
4181 SDValue Hi = getHiHalf64(LHS, DAG);
4183 SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32);
4184 SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst);
4186 SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero});
4188 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair);
4194 SelectionDAG &DAG = DCI.DAG;
4206 Elt0 = DAG.getNode(ISD::BITCAST, SL,
4210 return DAG.getNode(ISD::TRUNCATE, SL, VT, Elt0);
4227 SrcElt = DAG.getNode(ISD::BITCAST, SL,
4231 return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt);
4248 KnownBits Known = DAG.computeKnownBits(Amt);
4258 EVT::getVectorVT(*DAG.getContext(), MVT::i32,
4261 EVT NewShiftVT = getShiftAmountTy(MidVT, DAG.getDataLayout());
4262 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MidVT,
4267 Amt = DAG.getZExtOrTrunc(Amt, SL, NewShiftVT);
4271 SDValue ShrunkShift = DAG.getNode(Src.getOpcode(), SL, MidVT,
4273 return DAG.getNode(ISD::TRUNCATE, SL, VT, ShrunkShift);
4285 static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
4289 return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1);
4295 SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1);
4296 SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1);
4298 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, MulLo, MulHi);
4326 SelectionDAG &DAG = DCI.DAG;
4352 SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N1, MulOper);
4353 return DAG.getNode(ISD::ADD, DL, VT, MulVal, N1);
4357 SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N0, MulOper);
4358 return DAG.getNode(ISD::ADD, DL, VT, MulVal, N0);
4377 if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
4378 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
4379 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
4380 Mul = getMul24(DAG, DL, N0, N1, Size, false);
4381 } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
4382 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
4383 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
4384 Mul = getMul24(DAG, DL, N0, N1, Size, true);
4391 return DAG.getSExtOrTrunc(Mul, DL, VT);
4400 SelectionDAG &DAG = DCI.DAG;
4421 if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
4422 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
4423 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
4428 if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
4429 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
4430 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
4438 SDValue Lo = DAG.getNode(LoOpcode, DL, MVT::i32, N0, N1);
4439 SDValue Hi = DAG.getNode(HiOpcode, DL, MVT::i32, N0, N1);
4460 SelectionDAG &DAG = DCI.DAG;
4466 if (!isI24(N0, DAG) || !isI24(N1, DAG))
4469 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
4470 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
4472 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1);
4474 return DAG.getSExtOrTrunc(Mulhi, DL, VT);
4493 SelectionDAG &DAG = DCI.DAG;
4499 if (!isU24(N0, DAG) || !isU24(N1, DAG))
4502 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
4503 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
4505 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1);
4507 return DAG.getZExtOrTrunc(Mulhi, DL, VT);
4510 SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
4515 EVT LegalVT = getTypeToTransformTo(*DAG.getContext(), VT);
4521 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op);
4523 SDValue FFBX = DAG.getNode(Opc, DL, MVT::i32, Op);
4525 FFBX = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBX);
4543 SelectionDAG &DAG = DCI.DAG;
4554 return getFFBX_U32(DAG, CmpLHS, SL, Opc);
4565 return getFFBX_U32(DAG, CmpLHS, SL, Opc);
4577 SelectionDAG &DAG = DCI.DAG;
4580 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond,
4583 return DAG.getNode(Op, SL, VT, NewSelect);
4596 SelectionDAG &DAG = DCI.DAG;
4656 NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
4661 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT,
4664 return DAG.getNode(LHS.getOpcode(), SL, VT, NewSelect);
4689 SelectionDAG &DAG = DCI.DAG;
4690 if (DAG.isConstantValueOfAnyType(True) &&
4691 !DAG.isConstantValueOfAnyType(False)) {
4700 SDValue NewCond = DAG.getSetCC(SL, Cond.getValueType(), LHS, RHS, NewCC);
4701 return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True);
4798 SelectionDAG &DAG = DCI.DAG;
4818 LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
4823 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
4827 SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags());
4831 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
4846 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
4848 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags());
4852 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
4871 MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS);
4874 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
4878 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS);
4882 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
4906 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
4907 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
4910 SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags());
4914 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
4920 Ops[I] = DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags());
4922 SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags());
4927 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Res);
4928 DAG.ReplaceAllUsesWith(N0, Neg);
4951 return DAG.getNode(Opc, SL, VT, CvtSrc.getOperand(0));
4959 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
4960 return DAG.getNode(Opc, SL, VT, Neg, N0->getFlags());
4967 return DAG.getNode(ISD::FP_ROUND, SL, VT,
4975 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
4976 return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1));
4988 SDValue IntFNeg = DAG.getNode(ISD::XOR, SL, SrcVT, Src,
4989 DAG.getConstant(0x8000, SL, SrcVT));
4990 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFNeg);
5015 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::f32, HighBits);
5016 SDValue NegHi = DAG.getNode(ISD::FNEG, SL, MVT::f32, CastHi);
5018 DAG.getNode(ISD::BITCAST, SL, HighBits.getValueType(), NegHi);
5024 DAG.getNode(ISD::BUILD_VECTOR, SL, BCSrc.getValueType(), Ops);
5025 SDValue Result = DAG.getNode(ISD::BITCAST, SL, VT, Build);
5028 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Result));
5040 DAG.getNode(ISD::BITCAST, SL, MVT::f32, BCSrc.getOperand(1));
5042 DAG.getNode(ISD::BITCAST, SL, MVT::f32, BCSrc.getOperand(2));
5044 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, MVT::f32, LHS);
5045 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, MVT::f32, RHS);
5047 return DAG.getNode(ISD::SELECT, SL, MVT::f32, BCSrc.getOperand(0), NegLHS,
5060 SelectionDAG &DAG = DCI.DAG;
5074 SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src,
5075 DAG.getConstant(0x7fff, SL, SrcVT));
5076 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFAbs);
5092 return DCI.DAG.getConstantFP(One / Val, SDLoc(N), N->getValueType(0));
5097 SelectionDAG &DAG = DCI.DAG;
5126 CastedElts.push_back(DAG.getNode(ISD::BITCAST, DL, DestEltVT, Elt));
5129 return DAG.getBuildVector(DestVT, SL, CastedElts);
5145 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
5146 DAG.getConstant(Lo_32(CVal), SL, MVT::i32),
5147 DAG.getConstant(Hi_32(CVal), SL, MVT::i32));
5148 return DAG.getNode(ISD::BITCAST, SL, DestVT, BV);
5155 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
5156 DAG.getConstant(Lo_32(CVal), SL, MVT::i32),
5157 DAG.getConstant(Hi_32(CVal), SL, MVT::i32));
5159 return DAG.getNode(ISD::BITCAST, SL, DestVT, Vec);
5218 return DAG.getConstant(0, DL, MVT::i32);
5233 unsigned OpSignBits = DAG.ComputeNumSignBits(BitsFrom);
5237 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), WidthVal);
5240 // DAG Combines. If not eliminated, we will match back to BFE during
5245 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom,
5246 DAG.getValueType(SmallVT));
5249 return DAG.getZeroExtendInReg(BitsFrom, DL, SmallVT);
5254 return constantFoldBFE<int32_t>(DAG,
5261 return constantFoldBFE<uint32_t>(DAG,
5270 SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32);
5271 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32,
5281 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
5283 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5330 return DAG.getConstantFP(FTZ(V0), DL, VT);
5342 SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
5347 MachineFunction &MF = DAG.getMachineFunction();
5359 return DAG.getRegister(VReg, VT);
5361 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, VReg, VT);
5378 SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG,
5382 MachineFunction &MF = DAG.getMachineFunction();
5387 SDValue Ptr = DAG.getFrameIndex(FI, MVT::i32);
5389 return DAG.getLoad(VT, SL, DAG.getEntryNode(), Ptr, SrcPtrInfo, Align(4),
5394 SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG,
5399 MachineFunction &MF = DAG.getMachineFunction();
5403 SDValue Ptr = DAG.getConstant(Offset, SL, MVT::i32);
5406 DAG.getCopyFromReg(Chain, SL, Info->getStackPtrOffsetReg(), MVT::i32);
5407 Ptr = DAG.getNode(ISD::ADD, SL, MVT::i32, SP, Ptr);
5408 SDValue Store = DAG.getStore(Chain, SL, ArgVal, Ptr, DstInfo, Align(4),
5413 SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
5420 CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL) :
5421 loadStackInputValue(DAG, VT, SL, Arg.getStackOffset());
5428 V = DAG.getNode(ISD::SRL, SL, VT, V,
5429 DAG.getShiftAmountConstant(Shift, VT, SL));
5430 return DAG.getNode(ISD::AND, SL, VT, V,
5431 DAG.getConstant(Mask >> Shift, SL, VT));
5464 // AMDIL DAG nodes
5467 // AMDGPU DAG nodes
5618 SelectionDAG &DAG, int Enabled,
5626 return DAG.getNode(AMDGPUISD::RSQ, SDLoc(Operand), VT, Operand);
5636 SelectionDAG &DAG, int Enabled,
5647 return DAG.getNode(AMDGPUISD::RCP, SDLoc(Operand), VT, Operand);
5671 const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
5708 KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
5709 KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
5754 KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
5755 KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
5787 Align Alignment = GA->getGlobal()->getPointerAlignment(DAG.getDataLayout());
5799 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(2), Depth + 1);
5803 KnownBits Known1 = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
5807 KnownBits Known0 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
5823 DAG.getMachineFunction().getFunction(), workitemIntrinsicDim(IID));
5835 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
5848 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
5876 unsigned Tmp2 = DAG.ComputeNumSignBits(Op.getOperand(2), Depth + 1);
5880 unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth + 1);
5884 unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
5933 const SelectionDAG &DAG,
5951 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
5952 DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
5962 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
5963 DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
5964 DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
5986 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
6008 return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
6013 return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
6014 DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
6034 return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
6035 DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1) &&
6036 DAG.isKnownNeverNaN(Op.getOperand(3), SNaN, Depth + 1);