Lines Matching defs:SL
1964 const SDLoc &SL,
1982 return DAG.getConstant(Offset, SL, PtrVT);
1985 SDValue BasePtr = DAG.getCopyFromReg(Chain, SL,
1988 return DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Offset));
1992 const SDLoc &SL) const {
1995 return lowerKernArgParameterPtr(DAG, SL, DAG.getEntryNode(), Offset);
1999 const SDLoc &SL) const {
2005 return DAG.getConstant(*KnownSize, SL, MVT::i32);
2010 const SDLoc &SL, SDValue Val,
2019 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, NarrowedVT, Val,
2020 DAG.getConstant(0, SL, MVT::i32));
2027 Val = DAG.getNode(Opc, SL, MemVT, Val, DAG.getValueType(VT));
2031 Val = getFPExtOrFPRound(DAG, Val, SL, VT);
2033 Val = DAG.getSExtOrTrunc(Val, SL, VT);
2035 Val = DAG.getZExtOrTrunc(Val, SL, VT);
2041 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
2058 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset);
2059 SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, Align(4),
2063 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32);
2064 SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt);
2066 SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract);
2067 ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal);
2068 ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg);
2071 return DAG.getMergeValues({ ArgVal, Load.getValue(1) }, SL);
2074 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
2075 SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Alignment,
2079 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg);
2080 return DAG.getMergeValues({ Val, Load.getValue(1) }, SL);
2084 const SDLoc &SL, SDValue Chain,
2126 ExtType, SL, VA.getLocVT(), Chain, FIN,
3441 SDLoc SL;
3460 Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y,
3461 DAG.getShiftAmountConstant(10, MVT::i32, SL));
3463 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y) : Y;
3469 Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z,
3470 DAG.getShiftAmountConstant(20, MVT::i32, SL));
3472 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z) : Z;
4024 SDLoc SL(Op);
4026 SDValue CopyFromSP = DAG.getCopyFromReg(Op->getOperand(0), SL, SP, MVT::i32);
4032 DAG.getNode(AMDGPUISD::WAVE_ADDRESS, SL, MVT::i32, CopyFromSP);
4033 return DAG.getMergeValues({VectorAddress, CopyFromSP.getValue(1)}, SL);
4038 SDLoc SL(Op);
4043 SDValue GetRoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32);
4046 DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32);
4047 SDValue GetReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, Op->getVTList(),
4079 DAG.getConstant(AMDGPU::FltRoundConversionTable, SL, MVT::i64);
4081 SDValue Two = DAG.getConstant(2, SL, MVT::i32);
4083 DAG.getNode(ISD::SHL, SL, MVT::i32, GetReg, Two);
4088 DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits);
4089 SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue);
4091 SDValue EntryMask = DAG.getConstant(0xf, SL, MVT::i32);
4093 DAG.getNode(ISD::AND, SL, MVT::i32, TruncTable, EntryMask);
4097 SDValue Four = DAG.getConstant(4, SL, MVT::i32);
4099 DAG.getSetCC(SL, MVT::i1, TableEntry, Four, ISD::SETULT);
4100 SDValue EnumOffset = DAG.getNode(ISD::ADD, SL, MVT::i32, TableEntry, Four);
4101 SDValue Result = DAG.getNode(ISD::SELECT, SL, MVT::i32, IsStandardValue,
4104 return DAG.getMergeValues({Result, GetReg.getValue(1)}, SL);
4109 SDLoc SL(Op);
4121 AMDGPU::decodeFltRoundToHWConversionTable(ClampedVal), SL, MVT::i32);
4133 AMDGPU::FltRoundToHWConversionTable & 0xffff, SL, MVT::i32);
4135 SDValue Two = DAG.getConstant(2, SL, MVT::i32);
4137 DAG.getNode(ISD::SHL, SL, MVT::i32, NewMode, Two);
4140 DAG.getNode(ISD::SRL, SL, MVT::i32, BitTable, RoundModeTimesNumBits);
4148 DAG.getConstant(AMDGPU::FltRoundToHWConversionTable, SL, MVT::i64);
4150 SDValue Four = DAG.getConstant(4, SL, MVT::i32);
4151 SDValue OffsetEnum = DAG.getNode(ISD::SUB, SL, MVT::i32, NewMode, Four);
4153 DAG.getNode(ISD::UMIN, SL, MVT::i32, NewMode, OffsetEnum);
4155 SDValue Two = DAG.getConstant(2, SL, MVT::i32);
4157 DAG.getNode(ISD::SHL, SL, MVT::i32, IndexVal, Two);
4160 DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits);
4161 SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue);
4172 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32);
4173 NewMode = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
4180 DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32);
4183 SDValue RoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32);
4186 DAG.getNode(ISD::INTRINSIC_VOID, SL, Op->getVTList(), Op.getOperand(0),
4218 SDLoc SL(Op);
4220 DAG.getNode(ISD::BITCAST, SL, SrcVT.changeTypeToInteger(), Src);
4226 return DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast);
4230 SDLoc SL(Op);
4236 SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32);
4239 SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32);
4243 DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32);
4244 SDValue GetModeReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, VTList,
4246 SDValue GetTrapReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, VTList,
4249 DAG.getNode(ISD::TokenFactor, SL, MVT::Other, GetModeReg.getValue(1),
4253 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, GetModeReg, GetTrapReg);
4254 SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr);
4256 return DAG.getMergeValues({Result, TokenReg}, SL);
4260 SDLoc SL(Op);
4264 SDValue Input = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op.getOperand(1));
4265 SDValue NewModeReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input,
4266 DAG.getConstant(0, SL, MVT::i32));
4267 SDValue NewTrapReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input,
4268 DAG.getConstant(1, SL, MVT::i32));
4271 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32);
4272 NewModeReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
4274 NewTrapReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
4279 SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32);
4282 SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32);
4285 DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32);
4287 DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0),
4290 DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0),
4292 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, SetTrapReg, SetModeReg);
5684 SDLoc SL(Op);
5685 SDValue OpLo = DAG.getNode(Opc, SL, Lo.getValueType(), Lo,
5687 SDValue OpHi = DAG.getNode(Opc, SL, Hi.getValueType(), Hi,
5709 SDLoc SL(Op);
5711 SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1,
5713 SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1,
5740 SDLoc SL(Op);
5743 SDValue OpLo = DAG.getNode(Opc, SL, ResVT.first, Lo0, Lo1, Lo2,
5745 SDValue OpHi = DAG.getNode(Opc, SL, ResVT.second, Hi0, Hi1, Hi2,
6058 SDLoc SL(N);
6061 Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0);
6062 Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1);
6069 SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, SL, CCVT, Src0,
6073 return DAG.getZExtOrTrunc(SetCC, SL, VT);
6080 SDLoc SL(N);
6084 return DAG.getNode(AMDGPUISD::SETCC, SL, VT, Src.getOperand(0),
6090 return DAG.getConstant(0, SL, VT);
6102 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, Exec, VT);
6109 AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32),
6110 DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE));
6120 SDLoc SL(N);
6123 auto createLaneOp = [&DAG, &SL, N, IID](SDValue Src0, SDValue Src1,
6147 Operands.push_back(DAG.getTargetConstant(IID, SL, MVT::i32));
6153 Operands.push_back(DAG.getNode(ISD::CONVERGENCECTRL_GLUE, SL, MVT::Glue,
6157 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, ValT, Operands);
6177 SL, MVT::i32);
6181 SL, MVT::i32);
6186 SL, MVT::i32);
6190 SDValue Trunc = DAG.getAnyExtOrTrunc(LaneOp, SL, IntVT);
6197 auto unrollLaneOp = [&DAG, &SL](SDNode *N) -> SDValue {
6217 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, OperandEltVT,
6218 Operand, DAG.getVectorIdxConstant(i, SL));
6227 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, SL, MVT::Glue,
6230 Scalars.push_back(DAG.getNode(N->getOpcode(), SL, EltVT, Operands));
6234 return DAG.getBuildVector(VecVT, SL, Scalars);
6252 Src0SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src0,
6253 DAG.getConstant(EltIdx, SL, MVT::i32));
6256 Src1SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src1,
6257 DAG.getConstant(EltIdx, SL, MVT::i32));
6260 Src2SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src2,
6261 DAG.getConstant(EltIdx, SL, MVT::i32));
6269 return DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, Pieces);
6314 SDLoc SL(N);
6315 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32,
6317 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt));
6326 SDLoc SL(N);
6340 Results.push_back(DAG.getNode(Opcode, SL, VT, Src0, Src1));
6342 SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1);
6343 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt));
6416 SDLoc SL(N);
6419 SDValue LHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(1));
6420 SDValue RHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(2));
6424 LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS);
6425 RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS);
6429 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, SelectVT,
6433 NewSelect = DAG.getNode(ISD::TRUNCATE, SL, NewVT, NewSelect);
6434 Results.push_back(DAG.getNode(ISD::BITCAST, SL, VT, NewSelect));
6441 SDLoc SL(N);
6442 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
6444 SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32,
6446 DAG.getConstant(0x80008000, SL, MVT::i32));
6447 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op));
6454 SDLoc SL(N);
6455 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
6457 SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32,
6459 DAG.getConstant(0x7fff7fff, SL, MVT::i32));
6460 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op));
6792 SDLoc SL(Op);
6795 DAG.getMachineNode(AMDGPU::S_MUL_U64_U32_PSEUDO, SL, VT, Op0, Op1), 0);
6800 DAG.getMachineNode(AMDGPU::S_MUL_I64_I32_PSEUDO, SL, VT, Op0, Op1), 0);
6807 SDLoc SL(Op);
6818 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), SL, MVT::i32);
6819 SDValue Result = DAG.getNode(ISD::SHL, SL, VT, LHS, ShiftAmt);
6820 SDValue Overflow = DAG.getSetCC(SL, MVT::i1,
6822 SL, VT, Result, ShiftAmt),
6824 return DAG.getMergeValues({ Result, Overflow }, SL);
6828 SDValue Result = DAG.getNode(ISD::MUL, SL, VT, LHS, RHS);
6830 SL, VT, LHS, RHS);
6833 ? DAG.getNode(ISD::SRA, SL, VT, Result,
6834 DAG.getConstant(VT.getScalarSizeInBits() - 1, SL, MVT::i32))
6835 : DAG.getConstant(0, SL, VT);
6836 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, Top, Sign, ISD::SETNE);
6838 return DAG.getMergeValues({ Result, Overflow }, SL);
6867 SDLoc SL(Op);
6869 return DAG.getNode(AMDGPUISD::ENDPGM_TRAP, SL, MVT::Other, Chain);
6885 SDLoc SL(Op);
6893 loadImplicitKernelArgument(DAG, MVT::i64, SL, Align(8), QUEUE_PTR);
6903 QueuePtr = DAG.getConstant(0, SL, MVT::i64);
6911 SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01,
6917 DAG.getTargetConstant(TrapID, SL, MVT::i16),
6921 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
6926 SDLoc SL(Op);
6932 return DAG.getNode(AMDGPUISD::SIMULATED_TRAP, SL, MVT::Other, Chain);
6937 DAG.getTargetConstant(TrapID, SL, MVT::i16)
6939 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
6943 SDLoc SL(Op);
6961 DAG.getTargetConstant(TrapID, SL, MVT::i16)
6963 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
7053 SDLoc SL(Op);
7075 SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64);
7081 SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src);
7087 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32);
7088 SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE);
7090 return DAG.getNode(ISD::SELECT, SL, MVT::i32, NonNull, Ptr,
7100 SDValue Aperture = getSegmentAperture(SrcAS, SL, DAG);
7102 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Aperture);
7103 CvtPtr = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr);
7109 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32);
7112 = DAG.getSetCC(SL, MVT::i1, Src, SegmentNullPtr, ISD::SETNE);
7114 return DAG.getNode(ISD::SELECT, SL, MVT::i64, NonNull, CvtPtr,
7123 SDValue Hi = DAG.getConstant(Info->get32BitAddressHighBits(), SL, MVT::i32);
7124 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Hi);
7125 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
7130 return DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src);
7136 MF.getFunction(), "invalid addrspacecast", SL.getDebugLoc());
7157 SDLoc SL(Op);
7170 Vec = DAG.getNode(ISD::BITCAST, SL, NewVecVT, Vec);
7171 Ins = DAG.getNode(ISD::BITCAST, SL, NewInsVT, Ins);
7178 Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Ins,
7179 DAG.getConstant(I, SL, MVT::i32));
7181 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NewVecVT, Vec, Elt,
7182 DAG.getConstant(IdxVal / 2 + I, SL, MVT::i32));
7185 return DAG.getNode(ISD::BITCAST, SL, VecVT, Vec);
7189 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Ins,
7190 DAG.getConstant(I, SL, MVT::i32));
7191 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, VecVT, Vec, Elt,
7192 DAG.getConstant(IdxVal + I, SL, MVT::i32));
7206 SDLoc SL(Op);
7212 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec);
7214 SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec,
7215 DAG.getConstant(0, SL, MVT::i32));
7216 SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec,
7217 DAG.getConstant(1, SL, MVT::i32));
7219 SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf);
7220 SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf);
7224 SDValue InsHalf = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16,
7226 DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal),
7227 DAG.getConstant(InsertLo ? Idx : (Idx - 2), SL, MVT::i32));
7229 InsHalf = DAG.getNode(ISD::BITCAST, SL, MVT::i32, InsHalf);
7232 DAG.getBuildVector(MVT::v2i32, SL, { InsHalf, HiHalf }) :
7233 DAG.getBuildVector(MVT::v2i32, SL, { LoHalf, InsHalf });
7235 return DAG.getNode(ISD::BITCAST, SL, VecVT, Concat);
7253 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
7254 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor);
7255 SDValue BFM = DAG.getNode(ISD::SHL, SL, IntVT,
7256 DAG.getConstant(EltMask, SL, IntVT), ScaledIdx);
7259 SDValue ExtVal = DAG.getNode(ISD::BITCAST, SL, IntVT,
7260 DAG.getSplatBuildVector(VecVT, SL, InsVal));
7263 SDValue LHS = DAG.getNode(ISD::AND, SL, IntVT, BFM, ExtVal);
7266 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec);
7267 SDValue RHS = DAG.getNode(ISD::AND, SL, IntVT,
7268 DAG.getNOT(SL, BFM, IntVT), BCVec);
7271 SDValue BFI = DAG.getNode(ISD::OR, SL, IntVT, LHS, RHS);
7273 return DAG.getNode(ISD::BITCAST, SL, VecVT, BFI);
7278 SDLoc SL(Op);
7304 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
7305 DAG.getConstant(0, SL, MVT::i32)));
7307 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
7308 DAG.getConstant(1, SL, MVT::i32)));
7313 Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
7314 DAG.getConstant(P, SL, MVT::i32));
7317 Lo = DAG.getBitcast(LoVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64,
7319 Hi = DAG.getBitcast(HiVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64,
7327 Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
7328 DAG.getConstant(P, SL, MVT::i32));
7332 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64,
7335 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64,
7342 SDValue IdxMask = DAG.getConstant(NElem / 2 - 1, SL, IdxVT);
7343 SDValue NewIdx = DAG.getNode(ISD::AND, SL, IdxVT, Idx, IdxMask);
7344 SDValue Half = DAG.getSelectCC(SL, Idx, IdxMask, Hi, Lo, ISD::SETUGT);
7345 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Half, NewIdx);
7357 Vec = DAG.getAnyExtOrTrunc(Src, SL, IntVT);
7363 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
7366 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor);
7368 SDValue BC = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec);
7369 SDValue Elt = DAG.getNode(ISD::SRL, SL, IntVT, BC, ScaledIdx);
7372 SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt);
7373 return DAG.getNode(ISD::BITCAST, SL, ResultVT, Result);
7376 return DAG.getAnyExtOrTrunc(Elt, SL, ResultVT);
7386 SDLoc SL(Op);
7410 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL,
7412 DAG.getConstant(EltIdx, SL, MVT::i32));
7423 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
7424 Vec0, DAG.getConstant(EltIdx0, SL, MVT::i32));
7427 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
7428 Vec1, DAG.getConstant(EltIdx1, SL, MVT::i32));
7429 Pieces.push_back(DAG.getBuildVector(PackVT, SL, { Elt0, Elt1 }));
7433 return DAG.getNode(ISD::CONCAT_VECTORS, SL, ResultVT, Pieces);
7442 SDLoc SL(Op);
7449 return DAG.getBuildVector(ResultVT, SL, VElts);
7454 SDLoc SL(Op);
7470 SDValue Lo = DAG.getBuildVector(HalfVT, SL, LoOps);
7471 SDValue Hi = DAG.getBuildVector(HalfVT, SL, HiOps);
7473 SDValue CastLo = DAG.getNode(ISD::BITCAST, SL, HalfIntVT, Lo);
7474 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, HalfIntVT, Hi);
7476 SDValue Blend = DAG.getBuildVector(MVT::getVectorVT(HalfIntVT, 2), SL,
7478 return DAG.getNode(ISD::BITCAST, SL, VT, Blend);
7493 SDValue Vec = DAG.getBuildVector(QuarterVT, SL, Parts[P]);
7494 Casts[P] = DAG.getNode(ISD::BITCAST, SL, QuarterIntVT, Vec);
7498 DAG.getBuildVector(MVT::getVectorVT(QuarterIntVT, 4), SL, Casts);
7499 return DAG.getNode(ISD::BITCAST, SL, VT, Blend);
7514 SDValue Vec = DAG.getBuildVector(QuarterVT, SL, Parts[P]);
7515 Casts[P] = DAG.getNode(ISD::BITCAST, SL, QuarterIntVT, Vec);
7519 DAG.getBuildVector(MVT::getVectorVT(QuarterIntVT, 8), SL, Casts);
7520 return DAG.getNode(ISD::BITCAST, SL, VT, Blend);
7531 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo);
7532 SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo);
7533 return DAG.getNode(ISD::BITCAST, SL, VT, ExtLo);
7536 Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Hi);
7537 Hi = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Hi);
7539 SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi,
7540 DAG.getConstant(16, SL, MVT::i32));
7542 return DAG.getNode(ISD::BITCAST, SL, VT, ShlHi);
7544 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo);
7545 Lo = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Lo);
7547 SDValue Or = DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi);
7548 return DAG.getNode(ISD::BITCAST, SL, VT, Or);
7701 SDLoc SL(Op);
7703 DAG, MVT::i32, MVT::i32, SL, DAG.getEntryNode(), Offset, Align(4), false);
7705 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Param,
8364 SDLoc SL(Op);
8366 SDValue TTMP8 = DAG.getCopyFromReg(DAG.getEntryNode(), SL, AMDGPU::TTMP8, VT);
8367 return DAG.getNode(AMDGPUISD::BFE_U32, SL, VT, TTMP8,
8368 DAG.getConstant(25, SL, VT), DAG.getConstant(5, SL, VT));
8374 SDLoc SL(Op);
8378 return DAG.getConstant(0, SL, MVT::i32);
8392 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Val,
8701 SDLoc SL(Op);
8704 SDValue Aperture = getSegmentAperture(AS, SL, DAG);
8708 SDValue SrcHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, SrcVec,
8709 DAG.getConstant(1, SL, MVT::i32));
8710 return DAG.getSetCC(SL, MVT::i1, SrcHi, Aperture, ISD::SETEQ);
8736 SDLoc SL(Op);
8737 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op.getOperand(4), SL, MVT::i32);
8738 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, Op.getValueType(),
8748 SDLoc SL(Op);
8749 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op.getOperand(6), SL, MVT::i32);
8750 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, Op.getValueType(),
10154 const SDLoc &SL, EVT VT) {
10156 return DAG.getNode(ISD::TRUNCATE, SL, VT, Op);
10160 return DAG.getNode(ISD::SIGN_EXTEND, SL, VT, Op);
10162 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, Op);
10164 return DAG.getNode(ISD::ANY_EXTEND, SL, VT, Op);
10194 SDLoc SL(Ld);
10202 ISD::UNINDEXED, ISD::NON_EXTLOAD, MVT::i32, SL, Ld->getChain(), Ptr,
10216 Cvt = DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, NewLoad,
10220 Cvt = DAG.getZeroExtendInReg(NewLoad, SL, TruncVT);
10232 Cvt = getLoadExtOrTrunc(DAG, Ld->getExtensionType(), Cvt, SL, IntVT);
10236 Cvt = DAG.getNode(ISD::BITCAST, SL, VT, Cvt);
10238 return DAG.getMergeValues({ Cvt, NewLoad.getValue(1) }, SL);
10446 SDLoc SL(Op);
10475 return DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
10481 SDValue FNegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
10482 return DAG.getNode(AMDGPUISD::RCP, SL, VT, FNegRHS);
10493 SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
10494 return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip, Flags);
10499 SDLoc SL(Op);
10510 SDValue NegY = DAG.getNode(ISD::FNEG, SL, VT, Y);
10511 SDValue One = DAG.getConstantFP(1.0, SL, VT);
10513 SDValue R = DAG.getNode(AMDGPUISD::RCP, SL, VT, Y);
10514 SDValue Tmp0 = DAG.getNode(ISD::FMA, SL, VT, NegY, R, One);
10516 R = DAG.getNode(ISD::FMA, SL, VT, Tmp0, R, R);
10517 SDValue Tmp1 = DAG.getNode(ISD::FMA, SL, VT, NegY, R, One);
10518 R = DAG.getNode(ISD::FMA, SL, VT, Tmp1, R, R);
10519 SDValue Ret = DAG.getNode(ISD::FMUL, SL, VT, X, R);
10520 SDValue Tmp2 = DAG.getNode(ISD::FMA, SL, VT, NegY, Ret, X);
10521 return DAG.getNode(ISD::FMA, SL, VT, Tmp2, R, Ret);
10524 static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
10528 return DAG.getNode(Opcode, SL, VT, A, B, Flags);
10541 return DAG.getNode(Opcode, SL, VTList,
10546 static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
10550 return DAG.getNode(Opcode, SL, VT, {A, B, C}, Flags);
10563 return DAG.getNode(Opcode, SL, VTList,
10572 SDLoc SL(Op);
10576 SDValue CvtSrc0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0);
10577 SDValue CvtSrc1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1);
10579 SDValue RcpSrc1 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, CvtSrc1);
10580 SDValue Quot = DAG.getNode(ISD::FMUL, SL, MVT::f32, CvtSrc0, RcpSrc1);
10582 SDValue FPRoundFlag = DAG.getTargetConstant(0, SL, MVT::i32);
10583 SDValue BestQuot = DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Quot, FPRoundFlag);
10585 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f16, BestQuot, Src1, Src0);
10591 SDLoc SL(Op);
10595 SDValue r1 = DAG.getNode(ISD::FABS, SL, MVT::f32, RHS, Flags);
10598 const SDValue K0 = DAG.getConstantFP(K0Val, SL, MVT::f32);
10601 const SDValue K1 = DAG.getConstantFP(K1Val, SL, MVT::f32);
10603 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32);
10608 SDValue r2 = DAG.getSetCC(SL, SetCCVT, r1, K0, ISD::SETOGT);
10610 SDValue r3 = DAG.getNode(ISD::SELECT, SL, MVT::f32, r2, K1, One, Flags);
10612 r1 = DAG.getNode(ISD::FMUL, SL, MVT::f32, RHS, r3, Flags);
10615 SDValue r0 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, r1, Flags);
10617 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f32, LHS, r0, Flags);
10619 return DAG.getNode(ISD::FMUL, SL, MVT::f32, r3, Mul, Flags);
10644 SDLoc SL(Op);
10648 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32);
10652 SDValue DenominatorScaled = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT,
10654 SDValue NumeratorScaled = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT,
10658 SDValue ApproxRcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32,
10660 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f32,
10665 const SDValue BitField = DAG.getTargetConstant(Denorm32Reg, SL, MVT::i32);
10687 SDNode *GetReg = DAG.getMachineNode(AMDGPU::S_GETREG_B32, SL,
10693 {DAG.getEntryNode(), SDValue(GetReg, 0), SDValue(GetReg, 1)}, SL);
10701 EnableDenorm = DAG.getNode(AMDGPUISD::DENORM_MODE, SL, BindParamVTs, Glue,
10706 SL, MVT::i32);
10707 EnableDenorm = DAG.getMachineNode(AMDGPU::S_SETREG_B32, SL, BindParamVTs,
10717 NegDivScale0 = DAG.getMergeValues(Ops, SL);
10720 SDValue Fma0 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0,
10723 SDValue Fma1 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma0, ApproxRcp,
10726 SDValue Mul = getFPBinOp(DAG, ISD::FMUL, SL, MVT::f32, NumeratorScaled,
10729 SDValue Fma2 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Mul,
10732 SDValue Fma3 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32,
10735 SDValue Fma4 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Fma3,
10744 DisableDenorm = DAG.getNode(AMDGPUISD::DENORM_MODE, SL, MVT::Other,
10752 : DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, SL, MVT::i32);
10755 AMDGPU::S_SETREG_B32, SL, MVT::Other,
10759 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
10765 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f32,
10768 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f32, Fmas, RHS, LHS, Flags);
10775 SDLoc SL(Op);
10779 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
10783 SDValue DivScale0 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, Y, Y, X);
10785 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f64, DivScale0);
10787 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f64, DivScale0);
10789 SDValue Fma0 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Rcp, One);
10791 SDValue Fma1 = DAG.getNode(ISD::FMA, SL, MVT::f64, Rcp, Fma0, Rcp);
10793 SDValue Fma2 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Fma1, One);
10795 SDValue DivScale1 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, X, Y, X);
10797 SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1);
10798 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3);
10800 SDValue Fma4 = DAG.getNode(ISD::FMA, SL, MVT::f64,
10809 const SDValue Hi = DAG.getConstant(1, SL, MVT::i32);
10812 SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X);
10813 SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y);
10814 SDValue Scale0BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale0);
10815 SDValue Scale1BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale1);
10817 SDValue NumHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, NumBC, Hi);
10818 SDValue DenHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, DenBC, Hi);
10821 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale0BC, Hi);
10823 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale1BC, Hi);
10825 SDValue CmpDen = DAG.getSetCC(SL, MVT::i1, DenHi, Scale0Hi, ISD::SETEQ);
10826 SDValue CmpNum = DAG.getSetCC(SL, MVT::i1, NumHi, Scale1Hi, ISD::SETEQ);
10827 Scale = DAG.getNode(ISD::XOR, SL, MVT::i1, CmpNum, CmpDen);
10832 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f64,
10835 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f64, Fmas, Y, X);
10964 SDLoc SL(Op);
10968 DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Op.getOperand(0), Flags);
10970 SDValue SqrtID = DAG.getTargetConstant(Intrinsic::amdgcn_sqrt, SL, MVT::i32);
10972 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32, SqrtID, Ext, Flags);
10974 return DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Sqrt,
10975 DAG.getTargetConstant(0, SL, MVT::i32), Flags);
11340 SDLoc SL(N);
11343 SDValue ShlX = DAG.getNode(ISD::SHL, SL, VT, N0.getOperand(0), N1);
11344 SDValue COffset = DAG.getConstant(Offset, SL, VT);
11351 return DAG.getNode(ISD::ADD, SL, VT, ShlX, COffset, Flags);
11371 SDLoc SL(N);
11404 const SDLoc &SL,
11418 return splitBinaryBitConstantOpImpl(DCI, SL, Opc, LHS, ValLo, ValHi);
11536 SDLoc SL(N);
11537 SDValue BFE = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
11539 DAG.getConstant(Offset, SL, MVT::i32),
11540 DAG.getConstant(Bits, SL, MVT::i32));
11542 SDValue Ext = DAG.getNode(ISD::AssertZext, SL, VT, BFE,
12128 static SDValue getDWordFromOffset(SelectionDAG &DAG, SDLoc SL, SDValue Src,
12137 return DAG.getBitcastedAnyExtOrTrunc(Src, SL, MVT::i32);
12143 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Src,
12144 DAG.getConstant(DWordOffset, SL, MVT::i32));
12148 ISD::EXTRACT_VECTOR_ELT, SL, ScalarTy, Src,
12149 DAG.getConstant(DWordOffset / (ScalarTySize / 32), SL, MVT::i32));
12152 Ret = DAG.getNode(ISD::SRL, SL, Ret.getValueType(), Ret,
12153 DAG.getConstant(ShiftVal, SL, MVT::i32));
12154 return DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32);
12171 MVT::getVectorVT(MVT::getIntegerVT(ScalarTySize), NumAvailElements), SL,
12173 return Ret = DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32);
12178 Ret = DAG.getNode(ISD::SRL, SL, Src.getValueType(), Src,
12179 DAG.getConstant(ShiftVal, SL, MVT::i32));
12180 return DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32);
12408 SDLoc SL(N);
12411 SDValue LowOr = DAG.getNode(ISD::OR, SL, MVT::i32, LowLHS, ExtSrc);
12416 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
12418 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
12915 SelectionDAG &DAG, const SDLoc &SL, EVT VT, const APFloat &C) const {
12922 APFloat::getZero(C.getSemantics(), C.isNegative()), SL, VT);
12934 return DAG.getConstantFP(CanonicalQNaN, SL, VT);
12942 return DAG.getConstantFP(CanonicalQNaN, SL, VT);
12946 return DAG.getConstantFP(C, SL, VT);
12980 SDLoc SL(N);
12990 NewElts[I] = getCanonicalConstantFP(DAG, SL, EltVT,
12996 NewElts[I] = DAG.getNode(ISD::FCANONICALIZE, SL, EltVT, Op);
13006 NewElts[1]: DAG.getConstantFP(0.0f, SL, EltVT);
13011 NewElts[0] : DAG.getConstantFP(0.0f, SL, EltVT);
13014 return DAG.getBuildVector(VT, SL, NewElts);
13047 const SDLoc &SL, SDValue Src,
13075 return DAG.getNode(Med3Opc, SL, VT, Src, MaxVal, MinVal);
13097 const SDLoc &SL,
13122 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Op0.getOperand(0));
13139 return DAG.getNode(AMDGPUISD::FMED3, SL, K0->getValueType(0),
13273 SDLoc SL(N);
13283 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Src2);
13304 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Src0);
13386 SDLoc SL(N);
13389 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec.getOperand(0), Idx);
13390 return DAG.getNode(Vec.getOpcode(), SL, ResVT, Elt);
13399 SDLoc SL(N);
13421 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
13423 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
13428 return DAG.getNode(Opc, SL, ResVT, Elt0, Elt1, Vec->getFlags());
13435 SDLoc SL(N);
13439 SDValue IC = DAG.getVectorIdxConstant(I, SL);
13440 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec, IC);
13444 V = DAG.getSelectCC(SL, Idx, IC, Elt, V, ISD::SETEQ);
13463 SDLoc SL(N);
13465 SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec);
13468 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast,
13469 DAG.getConstant(EltIdx, SL, MVT::i32));
13471 SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt,
13472 DAG.getConstant(LeftoverBitIdx, SL, MVT::i32));
13476 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VecEltAsIntVT, Srl);
13480 return DAG.getNode(ISD::BITCAST, SL, VecEltVT, Trunc);
13484 return DAG.getAnyExtOrTrunc(Trunc, SL, ResVT);
13504 SDLoc SL(N);
13510 SDValue IC = DAG.getConstant(I, SL, IdxVT);
13511 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec, IC);
13512 SDValue V = DAG.getSelectCC(SL, Idx, IC, Ins, Elt, ISD::SETEQ);
13516 return DAG.getBuildVector(VecVT, SL, Ops);
13553 SDLoc SL(N);
13577 SDValue A1 = DAG.getNode(ISD::FMINNUM_IEEE, SL, VT, A, B);
13578 SDValue B1 = DAG.getNode(ISD::FMAXNUM_IEEE, SL, VT, A, B);
13579 SDValue C1 = DAG.getNode(ISD::FMAXNUM_IEEE, SL, VT, A1, C);
13580 return DAG.getNode(ISD::FMINNUM_IEEE, SL, VT, B1, C1);
13640 SDLoc SL(N);
13641 SDValue Add1 = DAG.getNode(Opc, SL, VT, Op0, Op1);
13642 return DAG.getNode(Opc, SL, VT, Add1, Op2);
13645 static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL,
13651 SDValue Mad = DAG.getNode(MadOpc, SL, VTs, N0, N1, N2);
13652 return DAG.getNode(ISD::TRUNCATE, SL, VT, Mad);
13669 SDLoc SL(N);
13730 MulLHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, MulLHS);
13731 MulRHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, MulRHS);
13732 AddRHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, AddRHS);
13747 SDValue One = DAG.getConstant(1, SL, MVT::i32);
13749 auto MulLHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulLHS);
13750 auto MulRHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulRHS);
13752 getMad64_32(DAG, SL, MVT::i64, MulLHSLo, MulRHSLo, AddRHS, MulSignedLo);
13756 std::tie(AccumLo, AccumHi) = DAG.SplitScalar(Accum, SL, MVT::i32, MVT::i32);
13760 DAG.getNode(ISD::EXTRACT_ELEMENT, SL, MVT::i32, MulLHS, One);
13761 SDValue MulHi = DAG.getNode(ISD::MUL, SL, MVT::i32, MulLHSHi, MulRHSLo);
13762 AccumHi = DAG.getNode(ISD::ADD, SL, MVT::i32, MulHi, AccumHi);
13767 DAG.getNode(ISD::EXTRACT_ELEMENT, SL, MVT::i32, MulRHS, One);
13768 SDValue MulHi = DAG.getNode(ISD::MUL, SL, MVT::i32, MulLHSLo, MulRHSHi);
13769 AccumHi = DAG.getNode(ISD::ADD, SL, MVT::i32, MulHi, AccumHi);
13772 Accum = DAG.getBuildVector(MVT::v2i32, SL, {AccumLo, AccumHi});
13777 Accum = DAG.getNode(ISD::TRUNCATE, SL, VT, Accum);
13894 static SDValue resolveSources(SelectionDAG &DAG, SDLoc SL,
13901 auto EltOp = getDWordFromOffset(DAG, SL, Elt->SrcOp, Elt->DWordOffset);
13907 return DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, EltOp, EltOp,
13908 DAG.getConstant(Elt->PermMask, SL, MVT::i32));
13930 getDWordFromOffset(DAG, SL, FirstElt->SrcOp, FirstElt->DWordOffset);
13932 getDWordFromOffset(DAG, SL, SecondElt->SrcOp, SecondElt->DWordOffset);
13934 Perms.push_back(DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, FirstVal,
13936 DAG.getConstant(PermMask, SL, MVT::i32)));
13947 getDWordFromOffset(DAG, SL, FirstElt->SrcOp, FirstElt->DWordOffset);
13950 DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, EltOp, EltOp,
13951 DAG.getConstant(FirstElt->PermMask, SL, MVT::i32)));
13958 ? DAG.getNode(ISD::OR, SL, MVT::i32, Perms[0], Perms[1])
14044 SDLoc SL(N);
14114 Src2s.push_back(DAG.getConstant(0, SL, MVT::i32));
14168 getDWordFromOffset(DAG, SL, FirstElt->SrcOp, FirstElt->DWordOffset);
14171 auto SecondEltOp = getDWordFromOffset(DAG, SL, SecondElt->SrcOp,
14174 Src0 = DAG.getBitcastedAnyExtOrTrunc(FirstEltOp, SL,
14176 Src1 = DAG.getBitcastedAnyExtOrTrunc(SecondEltOp, SL,
14182 Src0 = resolveSources(DAG, SL, Src0s, false, true);
14183 Src1 = resolveSources(DAG, SL, Src1s, false, true);
14188 DAG.getExtOrTrunc(*IsSigned, Src2s[ChainLength - 1], SL, MVT::i32);
14192 SL, MVT::i64);
14195 auto Dot = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, IID, Src0,
14196 Src1, Src2, DAG.getTargetConstant(0, SL, MVT::i1));
14198 return DAG.getExtOrTrunc(*IsSigned, Dot, SL, VT);
14223 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond };
14225 return DAG.getNode(Opc, SL, VTList, Args);
14246 SDLoc SL(N);
14264 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond };
14266 return DAG.getNode(Opc, SL, VTList, Args);
14312 SDLoc SL(N);
14325 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
14326 return DAG.getNode(FusedOp, SL, VT, A, Two, RHS);
14337 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
14338 return DAG.getNode(FusedOp, SL, VT, A, Two, LHS);
14352 SDLoc SL(N);
14369 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
14370 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
14372 return DAG.getNode(FusedOp, SL, VT, A, Two, NegRHS);
14384 const SDValue NegTwo = DAG.getConstantFP(-2.0, SL, VT);
14385 return DAG.getNode(FusedOp, SL, VT, A, NegTwo, LHS);
14396 SDLoc SL(N);
14419 DAG.getNode(AMDGPUISD::RSQ, SL, VT, RHS.getOperand(0), Flags);
14420 return IsNegative ? DAG.getNode(ISD::FNEG, SL, VT, Rsq, Flags) : Rsq;
14432 SDLoc SL(N);
14496 return DAG.getNode(AMDGPUISD::FDOT2, SL, MVT::f32, Vec1, Vec2, FMAAcc,
14497 DAG.getTargetConstant(0, SL, MVT::i1));
14506 SDLoc SL(N);
14533 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0),
14534 DAG.getConstant(-1, SL, MVT::i1));
14559 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0),
14560 DAG.getConstant(-1, SL, MVT::i1));
14591 return DAG.getNode(AMDGPUISD::FP_CLASS, SL, MVT::i1, LHS.getOperand(0),
14592 DAG.getConstant(Mask, SL, MVT::i32));
14602 SDLoc SL(N);
14629 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0 + ShiftOffset / 8, SL,
14648 return DAG.getNode(N->getOpcode(), SL, MVT::f32, DemandedSrc);
14768 SDLoc SL(N);
14772 Src = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src);
14774 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Src);
14775 return DAG.getNode(ISD::BITCAST, SL, VT, Ext);
15005 SDLoc SL(Node);
15012 = DAG.getCopyToReg(Node->getOperand(0), SL, VReg, SrcVal,
15015 = DAG.getCopyToReg(ToVReg, SL, SDValue(DestReg, 0),