Lines Matching defs:SL
1990 const SDLoc &SL,
2004 return DAG.getConstant(Offset, SL, PtrVT);
2008 Chain, SL, MRI.getLiveInVirtReg(InputPtrReg->getRegister()), PtrVT);
2010 return DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Offset));
2014 const SDLoc &SL) const {
2017 return lowerKernArgParameterPtr(DAG, SL, DAG.getEntryNode(), Offset);
2021 const SDLoc &SL) const {
2027 return DAG.getConstant(*KnownSize, SL, MVT::i32);
2032 const SDLoc &SL, SDValue Val,
2041 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, NarrowedVT, Val,
2042 DAG.getConstant(0, SL, MVT::i32));
2048 Val = DAG.getNode(Opc, SL, MemVT, Val, DAG.getValueType(VT));
2052 Val = getFPExtOrFPRound(DAG, Val, SL, VT);
2054 Val = DAG.getSExtOrTrunc(Val, SL, VT);
2056 Val = DAG.getZExtOrTrunc(Val, SL, VT);
2062 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
2079 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset);
2080 SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, Align(4),
2084 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32);
2085 SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt);
2087 SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract);
2088 ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal);
2089 ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg);
2091 return DAG.getMergeValues({ArgVal, Load.getValue(1)}, SL);
2094 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
2095 SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Alignment,
2099 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg);
2100 return DAG.getMergeValues({Val, Load.getValue(1)}, SL);
2104 CCValAssign &VA, const SDLoc &SL,
2147 ExtType, SL, VA.getLocVT(), Chain, FIN,
3462 SDLoc SL;
3481 Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y,
3482 DAG.getShiftAmountConstant(10, MVT::i32, SL));
3484 ? DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y)
3491 Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z,
3492 DAG.getShiftAmountConstant(20, MVT::i32, SL));
3494 ? DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z)
4114 SDLoc SL(Op);
4116 SDValue CopyFromSP = DAG.getCopyFromReg(Op->getOperand(0), SL, SP, MVT::i32);
4122 DAG.getNode(AMDGPUISD::WAVE_ADDRESS, SL, MVT::i32, CopyFromSP);
4123 return DAG.getMergeValues({VectorAddress, CopyFromSP.getValue(1)}, SL);
4128 SDLoc SL(Op);
4133 SDValue GetRoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32);
4136 DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32);
4137 SDValue GetReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, Op->getVTList(),
4169 DAG.getConstant(AMDGPU::FltRoundConversionTable, SL, MVT::i64);
4171 SDValue Two = DAG.getConstant(2, SL, MVT::i32);
4173 DAG.getNode(ISD::SHL, SL, MVT::i32, GetReg, Two);
4178 DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits);
4179 SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue);
4181 SDValue EntryMask = DAG.getConstant(0xf, SL, MVT::i32);
4183 DAG.getNode(ISD::AND, SL, MVT::i32, TruncTable, EntryMask);
4187 SDValue Four = DAG.getConstant(4, SL, MVT::i32);
4189 DAG.getSetCC(SL, MVT::i1, TableEntry, Four, ISD::SETULT);
4190 SDValue EnumOffset = DAG.getNode(ISD::ADD, SL, MVT::i32, TableEntry, Four);
4191 SDValue Result = DAG.getNode(ISD::SELECT, SL, MVT::i32, IsStandardValue,
4194 return DAG.getMergeValues({Result, GetReg.getValue(1)}, SL);
4199 SDLoc SL(Op);
4211 AMDGPU::decodeFltRoundToHWConversionTable(ClampedVal), SL, MVT::i32);
4223 AMDGPU::FltRoundToHWConversionTable & 0xffff, SL, MVT::i32);
4225 SDValue Two = DAG.getConstant(2, SL, MVT::i32);
4227 DAG.getNode(ISD::SHL, SL, MVT::i32, NewMode, Two);
4230 DAG.getNode(ISD::SRL, SL, MVT::i32, BitTable, RoundModeTimesNumBits);
4238 DAG.getConstant(AMDGPU::FltRoundToHWConversionTable, SL, MVT::i64);
4240 SDValue Four = DAG.getConstant(4, SL, MVT::i32);
4241 SDValue OffsetEnum = DAG.getNode(ISD::SUB, SL, MVT::i32, NewMode, Four);
4243 DAG.getNode(ISD::UMIN, SL, MVT::i32, NewMode, OffsetEnum);
4245 SDValue Two = DAG.getConstant(2, SL, MVT::i32);
4247 DAG.getNode(ISD::SHL, SL, MVT::i32, IndexVal, Two);
4250 DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits);
4251 SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue);
4262 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32);
4263 NewMode = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
4270 DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32);
4273 SDValue RoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32);
4276 DAG.getNode(ISD::INTRINSIC_VOID, SL, Op->getVTList(), Op.getOperand(0),
4308 SDLoc SL(Op);
4310 DAG.getNode(ISD::BITCAST, SL, SrcVT.changeTypeToInteger(), Src);
4316 return DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast);
4320 SDLoc SL(Op);
4326 SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32);
4329 SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32);
4333 DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32);
4334 SDValue GetModeReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, VTList,
4336 SDValue GetTrapReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, VTList,
4339 DAG.getNode(ISD::TokenFactor, SL, MVT::Other, GetModeReg.getValue(1),
4343 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, GetModeReg, GetTrapReg);
4344 SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr);
4346 return DAG.getMergeValues({Result, TokenReg}, SL);
4350 SDLoc SL(Op);
4354 SDValue Input = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op.getOperand(1));
4355 SDValue NewModeReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input,
4356 DAG.getConstant(0, SL, MVT::i32));
4357 SDValue NewTrapReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input,
4358 DAG.getConstant(1, SL, MVT::i32));
4361 DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32);
4362 NewModeReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
4364 NewTrapReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
4369 SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32);
4372 SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32);
4375 DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32);
4377 DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0),
4380 DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0),
4382 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, SetTrapReg, SetModeReg);
5829 SDLoc SL(Op);
5830 SDValue OpLo = DAG.getNode(Opc, SL, Lo.getValueType(), Lo, Op->getFlags());
5831 SDValue OpHi = DAG.getNode(Opc, SL, Hi.getValueType(), Hi, Op->getFlags());
5850 SDLoc SL(Op);
5853 DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1, Op->getFlags());
5855 DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1, Op->getFlags());
5879 SDLoc SL(Op);
5883 DAG.getNode(Opc, SL, ResVT.first, Lo0, Lo1, Lo2, Op->getFlags());
5885 DAG.getNode(Opc, SL, ResVT.second, Hi0, Hi1, Hi2, Op->getFlags());
6188 SDLoc SL(N);
6191 Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0);
6192 Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1);
6199 SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, SL, CCVT, Src0, Src1,
6203 return DAG.getZExtOrTrunc(SetCC, SL, VT);
6210 SDLoc SL(N);
6214 return DAG.getNode(AMDGPUISD::SETCC, SL, VT, Src.getOperand(0),
6220 return DAG.getConstant(0, SL, VT);
6232 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, Exec, VT);
6239 AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32),
6240 DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE));
6252 SDLoc SL(N);
6261 auto createLaneOp = [&DAG, &SL, N, IID](SDValue Src0, SDValue Src1,
6289 Operands.push_back(DAG.getTargetConstant(IID, SL, MVT::i32));
6295 Operands.push_back(DAG.getNode(ISD::CONVERGENCECTRL_GLUE, SL, MVT::Glue,
6299 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, ValT, Operands);
6321 SL, MVT::i32);
6325 SL, MVT::i32);
6330 SL, MVT::i32);
6334 SDValue Trunc = DAG.getAnyExtOrTrunc(LaneOp, SL, IntVT);
6341 auto unrollLaneOp = [&DAG, &SL](SDNode *N) -> SDValue {
6361 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, OperandEltVT,
6362 Operand, DAG.getVectorIdxConstant(i, SL));
6371 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, SL, MVT::Glue,
6374 Scalars.push_back(DAG.getNode(N->getOpcode(), SL, EltVT, Operands));
6378 return DAG.getBuildVector(VecVT, SL, Scalars);
6400 Src0SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src0,
6401 DAG.getConstant(EltIdx, SL, MVT::i32));
6405 Src1SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src1,
6406 DAG.getConstant(EltIdx, SL, MVT::i32));
6409 Src2SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src2,
6410 DAG.getConstant(EltIdx, SL, MVT::i32));
6418 return DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, Pieces);
6464 SDLoc SL(N);
6466 DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32, Src0, Src1);
6467 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt));
6476 SDLoc SL(N);
6490 Results.push_back(DAG.getNode(Opcode, SL, VT, Src0, Src1));
6492 SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1);
6493 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt));
6566 SDLoc SL(N);
6569 SDValue LHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(1));
6570 SDValue RHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(2));
6574 LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS);
6575 RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS);
6580 DAG.getNode(ISD::SELECT, SL, SelectVT, N->getOperand(0), LHS, RHS);
6583 NewSelect = DAG.getNode(ISD::TRUNCATE, SL, NewVT, NewSelect);
6584 Results.push_back(DAG.getNode(ISD::BITCAST, SL, VT, NewSelect));
6591 SDLoc SL(N);
6592 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
6594 SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32, BC,
6595 DAG.getConstant(0x80008000, SL, MVT::i32));
6596 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op));
6603 SDLoc SL(N);
6604 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
6606 SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32, BC,
6607 DAG.getConstant(0x7fff7fff, SL, MVT::i32));
6608 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op));
7010 SDLoc SL(Op);
7013 DAG.getMachineNode(AMDGPU::S_MUL_U64_U32_PSEUDO, SL, VT, Op0, Op1), 0);
7018 DAG.getMachineNode(AMDGPU::S_MUL_I64_I32_PSEUDO, SL, VT, Op0, Op1), 0);
7025 SDLoc SL(Op);
7036 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), SL, MVT::i32);
7037 SDValue Result = DAG.getNode(ISD::SHL, SL, VT, LHS, ShiftAmt);
7039 DAG.getSetCC(SL, MVT::i1,
7040 DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL, SL, VT,
7043 return DAG.getMergeValues({Result, Overflow}, SL);
7047 SDValue Result = DAG.getNode(ISD::MUL, SL, VT, LHS, RHS);
7049 DAG.getNode(isSigned ? ISD::MULHS : ISD::MULHU, SL, VT, LHS, RHS);
7052 ? DAG.getNode(ISD::SRA, SL, VT, Result,
7054 SL, MVT::i32))
7055 : DAG.getConstant(0, SL, VT);
7056 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, Top, Sign, ISD::SETNE);
7058 return DAG.getMergeValues({Result, Overflow}, SL);
7086 SDLoc SL(Op);
7088 return DAG.getNode(AMDGPUISD::ENDPGM_TRAP, SL, MVT::Other, Chain);
7106 SDLoc SL(Op);
7114 loadImplicitKernelArgument(DAG, MVT::i64, SL, Align(8), QUEUE_PTR);
7124 QueuePtr = DAG.getConstant(0, SL, MVT::i64);
7132 SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01, QueuePtr, SDValue());
7135 SDValue Ops[] = {ToReg, DAG.getTargetConstant(TrapID, SL, MVT::i16), SGPR01,
7137 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
7141 SDLoc SL(Op);
7147 return DAG.getNode(AMDGPUISD::SIMULATED_TRAP, SL, MVT::Other, Chain);
7150 SDValue Ops[] = {Chain, DAG.getTargetConstant(TrapID, SL, MVT::i16)};
7151 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
7155 SDLoc SL(Op);
7171 SDValue Ops[] = {Chain, DAG.getTargetConstant(TrapID, SL, MVT::i16)};
7172 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
7262 SDLoc SL(Op);
7284 SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64);
7290 SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src);
7296 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32);
7297 SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE);
7299 return DAG.getNode(ISD::SELECT, SL, MVT::i32, NonNull, Ptr,
7309 SDValue Aperture = getSegmentAperture(SrcAS, SL, DAG);
7311 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Aperture);
7312 CvtPtr = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr);
7318 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32);
7321 DAG.getSetCC(SL, MVT::i1, Src, SegmentNullPtr, ISD::SETNE);
7323 return DAG.getNode(ISD::SELECT, SL, MVT::i64, NonNull, CvtPtr,
7332 SDValue Hi = DAG.getConstant(Info->get32BitAddressHighBits(), SL, MVT::i32);
7333 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Hi);
7334 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
7339 return DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src);
7345 MF.getFunction(), "invalid addrspacecast", SL.getDebugLoc());
7366 SDLoc SL(Op);
7379 Vec = DAG.getNode(ISD::BITCAST, SL, NewVecVT, Vec);
7380 Ins = DAG.getNode(ISD::BITCAST, SL, NewInsVT, Ins);
7387 Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Ins,
7388 DAG.getConstant(I, SL, MVT::i32));
7390 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NewVecVT, Vec, Elt,
7391 DAG.getConstant(IdxVal / 2 + I, SL, MVT::i32));
7394 return DAG.getNode(ISD::BITCAST, SL, VecVT, Vec);
7398 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Ins,
7399 DAG.getConstant(I, SL, MVT::i32));
7400 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, VecVT, Vec, Elt,
7401 DAG.getConstant(IdxVal + I, SL, MVT::i32));
7415 SDLoc SL(Op);
7421 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec);
7423 SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec,
7424 DAG.getConstant(0, SL, MVT::i32));
7425 SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec,
7426 DAG.getConstant(1, SL, MVT::i32));
7428 SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf);
7429 SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf);
7434 ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16, InsertLo ? LoVec : HiVec,
7435 DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal),
7436 DAG.getConstant(InsertLo ? Idx : (Idx - 2), SL, MVT::i32));
7438 InsHalf = DAG.getNode(ISD::BITCAST, SL, MVT::i32, InsHalf);
7441 InsertLo ? DAG.getBuildVector(MVT::v2i32, SL, {InsHalf, HiHalf})
7442 : DAG.getBuildVector(MVT::v2i32, SL, {LoHalf, InsHalf});
7444 return DAG.getNode(ISD::BITCAST, SL, VecVT, Concat);
7462 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
7463 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor);
7464 SDValue BFM = DAG.getNode(ISD::SHL, SL, IntVT,
7465 DAG.getConstant(EltMask, SL, IntVT), ScaledIdx);
7468 SDValue ExtVal = DAG.getNode(ISD::BITCAST, SL, IntVT,
7469 DAG.getSplatBuildVector(VecVT, SL, InsVal));
7472 SDValue LHS = DAG.getNode(ISD::AND, SL, IntVT, BFM, ExtVal);
7475 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec);
7477 DAG.getNode(ISD::AND, SL, IntVT, DAG.getNOT(SL, BFM, IntVT), BCVec);
7481 DAG.getNode(ISD::OR, SL, IntVT, LHS, RHS, SDNodeFlags::Disjoint);
7483 return DAG.getNode(ISD::BITCAST, SL, VecVT, BFI);
7488 SDLoc SL(Op);
7513 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
7514 DAG.getConstant(0, SL, MVT::i32)));
7516 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
7517 DAG.getConstant(1, SL, MVT::i32)));
7522 Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
7523 DAG.getConstant(P, SL, MVT::i32));
7526 Lo = DAG.getBitcast(LoVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64,
7528 Hi = DAG.getBitcast(HiVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64,
7536 Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
7537 DAG.getConstant(P, SL, MVT::i32));
7541 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64,
7544 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64,
7551 SDValue IdxMask = DAG.getConstant(NElem / 2 - 1, SL, IdxVT);
7552 SDValue NewIdx = DAG.getNode(ISD::AND, SL, IdxVT, Idx, IdxMask);
7553 SDValue Half = DAG.getSelectCC(SL, Idx, IdxMask, Hi, Lo, ISD::SETUGT);
7554 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Half, NewIdx);
7566 Vec = DAG.getAnyExtOrTrunc(Src, SL, IntVT);
7572 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
7575 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor);
7577 SDValue BC = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec);
7578 SDValue Elt = DAG.getNode(ISD::SRL, SL, IntVT, BC, ScaledIdx);
7581 SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt);
7582 return DAG.getNode(ISD::BITCAST, SL, ResultVT, Result);
7585 return DAG.getAnyExtOrTrunc(Elt, SL, ResultVT);
7601 SDLoc SL(Op);
7650 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, PackVT,
7652 DAG.getConstant(EltIdx, SL, MVT::i32));
7675 SDValue SubVec0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, PackVT, SrcOp0,
7676 DAG.getConstant(AlignedIdx0, SL, MVT::i32));
7677 SDValue SubVec1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, PackVT, SrcOp1,
7678 DAG.getConstant(AlignedIdx1, SL, MVT::i32));
7693 SDValue Shuf = DAG.getVectorShuffle(PackVT, SL, Result0, Result1,
7705 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec0,
7706 DAG.getSignedConstant(EltIdx0, SL, MVT::i32));
7709 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec1,
7710 DAG.getSignedConstant(EltIdx1, SL, MVT::i32));
7711 Pieces.push_back(DAG.getBuildVector(PackVT, SL, {Elt0, Elt1}));
7715 return DAG.getNode(ISD::CONCAT_VECTORS, SL, ResultVT, Pieces);
7724 SDLoc SL(Op);
7731 return DAG.getBuildVector(ResultVT, SL, VElts);
7736 SDLoc SL(Op);
7747 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo);
7748 SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo);
7749 return DAG.getNode(ISD::BITCAST, SL, VT, ExtLo);
7752 Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Hi);
7753 Hi = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Hi);
7755 SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi,
7756 DAG.getConstant(16, SL, MVT::i32));
7758 return DAG.getNode(ISD::BITCAST, SL, VT, ShlHi);
7760 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo);
7761 Lo = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Lo);
7764 DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi, SDNodeFlags::Disjoint);
7765 return DAG.getNode(ISD::BITCAST, SL, VT, Or);
7776 PartVT, SL, {Op.getOperand(P * 2), Op.getOperand(P * 2 + 1)});
7777 Casts.push_back(DAG.getNode(ISD::BITCAST, SL, PartIntVT, Vec));
7781 DAG.getBuildVector(MVT::getVectorVT(PartIntVT, NumParts), SL, Casts);
7782 return DAG.getNode(ISD::BITCAST, SL, VT, Blend);
7933 SDLoc SL(Op);
7935 DAG, MVT::i32, MVT::i32, SL, DAG.getEntryNode(), Offset, Align(4), false);
7937 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Param,
8606 SDLoc SL(Op);
8608 SDValue TTMP8 = DAG.getCopyFromReg(DAG.getEntryNode(), SL, AMDGPU::TTMP8, VT);
8609 return DAG.getNode(AMDGPUISD::BFE_U32, SL, VT, TTMP8,
8610 DAG.getConstant(25, SL, VT), DAG.getConstant(5, SL, VT));
8616 SDLoc SL(Op);
8620 return DAG.getConstant(0, SL, MVT::i32);
8634 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Val,
8945 SDLoc SL(Op);
8949 SDValue Aperture = getSegmentAperture(AS, SL, DAG);
8953 SDValue SrcHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, SrcVec,
8954 DAG.getConstant(1, SL, MVT::i32));
8955 return DAG.getSetCC(SL, MVT::i1, SrcHi, Aperture, ISD::SETEQ);
8981 SDLoc SL(Op);
8982 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op.getOperand(4), SL, MVT::i32);
8983 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, Op.getValueType(),
8993 SDLoc SL(Op);
8994 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op.getOperand(6), SL, MVT::i32);
8995 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, Op.getValueType(),
10461 SDValue Op, const SDLoc &SL, EVT VT) {
10463 return DAG.getNode(ISD::TRUNCATE, SL, VT, Op);
10467 return DAG.getNode(ISD::SIGN_EXTEND, SL, VT, Op);
10469 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, Op);
10471 return DAG.getNode(ISD::ANY_EXTEND, SL, VT, Op);
10502 SDLoc SL(Ld);
10510 ISD::UNINDEXED, ISD::NON_EXTLOAD, MVT::i32, SL, Ld->getChain(), Ptr,
10524 Cvt = DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, NewLoad,
10528 Cvt = DAG.getZeroExtendInReg(NewLoad, SL, TruncVT);
10540 Cvt = getLoadExtOrTrunc(DAG, Ld->getExtensionType(), Cvt, SL, IntVT);
10544 Cvt = DAG.getNode(ISD::BITCAST, SL, VT, Cvt);
10546 return DAG.getMergeValues({Cvt, NewLoad.getValue(1)}, SL);
10734 SDLoc SL(Op);
10763 return DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
10769 SDValue FNegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
10770 return DAG.getNode(AMDGPUISD::RCP, SL, VT, FNegRHS);
10781 SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
10782 return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip, Flags);
10787 SDLoc SL(Op);
10798 SDValue NegY = DAG.getNode(ISD::FNEG, SL, VT, Y);
10799 SDValue One = DAG.getConstantFP(1.0, SL, VT);
10801 SDValue R = DAG.getNode(AMDGPUISD::RCP, SL, VT, Y);
10802 SDValue Tmp0 = DAG.getNode(ISD::FMA, SL, VT, NegY, R, One);
10804 R = DAG.getNode(ISD::FMA, SL, VT, Tmp0, R, R);
10805 SDValue Tmp1 = DAG.getNode(ISD::FMA, SL, VT, NegY, R, One);
10806 R = DAG.getNode(ISD::FMA, SL, VT, Tmp1, R, R);
10807 SDValue Ret = DAG.getNode(ISD::FMUL, SL, VT, X, R);
10808 SDValue Tmp2 = DAG.getNode(ISD::FMA, SL, VT, NegY, Ret, X);
10809 return DAG.getNode(ISD::FMA, SL, VT, Tmp2, R, Ret);
10812 static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
10816 return DAG.getNode(Opcode, SL, VT, A, B, Flags);
10830 return DAG.getNode(Opcode, SL, VTList,
10835 static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
10839 return DAG.getNode(Opcode, SL, VT, {A, B, C}, Flags);
10853 return DAG.getNode(Opcode, SL, VTList,
10862 SDLoc SL(Op);
10883 SDValue LHSExt = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, LHS);
10884 SDValue RHSExt = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, RHS);
10885 SDValue NegRHSExt = DAG.getNode(ISD::FNEG, SL, MVT::f32, RHSExt);
10887 DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, RHSExt, Op->getFlags());
10889 DAG.getNode(ISD::FMUL, SL, MVT::f32, LHSExt, Rcp, Op->getFlags());
10890 SDValue Err = DAG.getNode(FMADOpCode, SL, MVT::f32, NegRHSExt, Quot, LHSExt,
10892 Quot = DAG.getNode(FMADOpCode, SL, MVT::f32, Err, Rcp, Quot, Op->getFlags());
10893 Err = DAG.getNode(FMADOpCode, SL, MVT::f32, NegRHSExt, Quot, LHSExt,
10895 SDValue Tmp = DAG.getNode(ISD::FMUL, SL, MVT::f32, Err, Rcp, Op->getFlags());
10896 SDValue TmpCast = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Tmp);
10897 TmpCast = DAG.getNode(ISD::AND, SL, MVT::i32, TmpCast,
10898 DAG.getConstant(0xff800000, SL, MVT::i32));
10899 Tmp = DAG.getNode(ISD::BITCAST, SL, MVT::f32, TmpCast);
10900 Quot = DAG.getNode(ISD::FADD, SL, MVT::f32, Tmp, Quot, Op->getFlags());
10901 SDValue RDst = DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Quot,
10902 DAG.getTargetConstant(0, SL, MVT::i32));
10903 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f16, RDst, RHS, LHS,
10910 SDLoc SL(Op);
10914 SDValue r1 = DAG.getNode(ISD::FABS, SL, MVT::f32, RHS, Flags);
10917 const SDValue K0 = DAG.getConstantFP(K0Val, SL, MVT::f32);
10920 const SDValue K1 = DAG.getConstantFP(K1Val, SL, MVT::f32);
10922 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32);
10927 SDValue r2 = DAG.getSetCC(SL, SetCCVT, r1, K0, ISD::SETOGT);
10929 SDValue r3 = DAG.getNode(ISD::SELECT, SL, MVT::f32, r2, K1, One, Flags);
10931 r1 = DAG.getNode(ISD::FMUL, SL, MVT::f32, RHS, r3, Flags);
10934 SDValue r0 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, r1, Flags);
10936 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f32, LHS, r0, Flags);
10938 return DAG.getNode(ISD::FMUL, SL, MVT::f32, r3, Mul, Flags);
10963 SDLoc SL(Op);
10967 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32);
10972 DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, {RHS, RHS, LHS}, Flags);
10974 DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, {LHS, RHS, LHS}, Flags);
10978 DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, DenominatorScaled, Flags);
10980 DAG.getNode(ISD::FNEG, SL, MVT::f32, DenominatorScaled, Flags);
10984 const SDValue BitField = DAG.getTargetConstant(Denorm32Reg, SL, MVT::i32);
11006 SDNode *GetReg = DAG.getMachineNode(AMDGPU::S_GETREG_B32, SL,
11012 {DAG.getEntryNode(), SDValue(GetReg, 0), SDValue(GetReg, 1)}, SL);
11020 EnableDenorm = DAG.getNode(AMDGPUISD::DENORM_MODE, SL, BindParamVTs, Glue,
11025 DAG.getConstant(FP_DENORM_FLUSH_NONE, SL, MVT::i32);
11026 EnableDenorm = DAG.getMachineNode(AMDGPU::S_SETREG_B32, SL, BindParamVTs,
11033 NegDivScale0 = DAG.getMergeValues(Ops, SL);
11036 SDValue Fma0 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0,
11039 SDValue Fma1 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma0, ApproxRcp,
11042 SDValue Mul = getFPBinOp(DAG, ISD::FMUL, SL, MVT::f32, NumeratorScaled, Fma1,
11045 SDValue Fma2 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Mul,
11049 getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma2, Fma1, Mul, Fma2, Flags);
11051 SDValue Fma4 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Fma3,
11061 DAG.getNode(AMDGPUISD::DENORM_MODE, SL, MVT::Other, Fma4.getValue(1),
11069 : DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, SL, MVT::i32);
11072 AMDGPU::S_SETREG_B32, SL, MVT::Other,
11076 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
11082 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f32,
11085 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f32, Fmas, RHS, LHS, Flags);
11092 SDLoc SL(Op);
11096 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
11100 SDValue DivScale0 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, Y, Y, X);
11102 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f64, DivScale0);
11104 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f64, DivScale0);
11106 SDValue Fma0 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Rcp, One);
11108 SDValue Fma1 = DAG.getNode(ISD::FMA, SL, MVT::f64, Rcp, Fma0, Rcp);
11110 SDValue Fma2 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Fma1, One);
11112 SDValue DivScale1 = DAG.getNode(AMDGPUISD::DIV_SCALE, SL, ScaleVT, X, Y, X);
11114 SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1);
11115 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3);
11118 DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Mul, DivScale1);
11126 const SDValue Hi = DAG.getConstant(1, SL, MVT::i32);
11129 SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X);
11130 SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y);
11131 SDValue Scale0BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale0);
11132 SDValue Scale1BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale1);
11135 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, NumBC, Hi);
11137 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, DenBC, Hi);
11140 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale0BC, Hi);
11142 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale1BC, Hi);
11144 SDValue CmpDen = DAG.getSetCC(SL, MVT::i1, DenHi, Scale0Hi, ISD::SETEQ);
11145 SDValue CmpNum = DAG.getSetCC(SL, MVT::i1, NumHi, Scale1Hi, ISD::SETEQ);
11146 Scale = DAG.getNode(ISD::XOR, SL, MVT::i1, CmpNum, CmpDen);
11152 DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f64, Fma4, Fma3, Mul, Scale);
11154 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f64, Fmas, Y, X);
11284 SDLoc SL(Op);
11288 DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Op.getOperand(0), Flags);
11290 SDValue SqrtID = DAG.getTargetConstant(Intrinsic::amdgcn_sqrt, SL, MVT::i32);
11292 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32, SqrtID, Ext, Flags);
11294 return DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Sqrt,
11295 DAG.getTargetConstant(0, SL, MVT::i32), Flags);
11663 SDLoc SL(N);
11666 SDValue ShlX = DAG.getNode(ISD::SHL, SL, VT, N0.getOperand(0), N1);
11667 SDValue COffset = DAG.getConstant(Offset, SL, VT);
11674 return DAG.getNode(ISD::ADD, SL, VT, ShlX, COffset, Flags);
11694 SDLoc SL(N);
11726 DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS,
11739 return splitBinaryBitConstantOpImpl(DCI, SL, Opc, LHS, ValLo, ValHi);
11860 SDLoc SL(N);
11862 DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, LHS->getOperand(0),
11863 DAG.getConstant(Offset, SL, MVT::i32),
11864 DAG.getConstant(Bits, SL, MVT::i32));
11866 SDValue Ext = DAG.getNode(ISD::AssertZext, SL, VT, BFE,
12451 static SDValue getDWordFromOffset(SelectionDAG &DAG, SDLoc SL, SDValue Src,
12460 return DAG.getBitcastedAnyExtOrTrunc(Src, SL, MVT::i32);
12466 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Src,
12467 DAG.getConstant(DWordOffset, SL, MVT::i32));
12471 ISD::EXTRACT_VECTOR_ELT, SL, ScalarTy, Src,
12472 DAG.getConstant(DWordOffset / (ScalarTySize / 32), SL, MVT::i32));
12475 Ret = DAG.getNode(ISD::SRL, SL, Ret.getValueType(), Ret,
12476 DAG.getConstant(ShiftVal, SL, MVT::i32));
12477 return DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32);
12494 MVT::getVectorVT(MVT::getIntegerVT(ScalarTySize), NumAvailElements), SL,
12496 return Ret = DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32);
12501 Ret = DAG.getNode(ISD::SRL, SL, Src.getValueType(), Src,
12502 DAG.getConstant(ShiftVal, SL, MVT::i32));
12503 return DAG.getBitcastedAnyExtOrTrunc(Ret, SL, MVT::i32);
12732 SDLoc SL(N);
12734 SDValue LowOr = DAG.getNode(ISD::OR, SL, MVT::i32, LowLHS, ExtSrc);
12740 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, LowOr, HiBits);
12741 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
13232 const SDLoc &SL, EVT VT,
13240 APFloat::getZero(C.getSemantics(), C.isNegative()), SL, VT);
13252 return DAG.getConstantFP(CanonicalQNaN, SL, VT);
13260 return DAG.getConstantFP(CanonicalQNaN, SL, VT);
13264 return DAG.getConstantFP(C, SL, VT);
13298 SDLoc SL(N);
13309 getCanonicalConstantFP(DAG, SL, EltVT, CFP->getValueAPF());
13314 NewElts[I] = DAG.getNode(ISD::FCANONICALIZE, SL, EltVT, Op);
13325 : DAG.getConstantFP(0.0f, SL, EltVT);
13331 : DAG.getConstantFP(0.0f, SL, EltVT);
13334 return DAG.getBuildVector(VT, SL, NewElts);
13367 const SDLoc &SL, SDValue Src,
13395 return DAG.getNode(Med3Opc, SL, VT, Src, MaxVal, MinVal);
13417 const SDLoc &SL, SDValue Op0,
13441 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Op0.getOperand(0));
13458 return DAG.getNode(AMDGPUISD::FMED3, SL, K0->getValueType(0), Var,
13585 SDLoc SL(N);
13595 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Src2);
13616 return DAG.getNode(AMDGPUISD::CLAMP, SL, VT, Src0);
13702 SDLoc SL(N);
13705 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec.getOperand(0), Idx);
13706 return DAG.getNode(Vec.getOpcode(), SL, ResVT, Elt);
13715 SDLoc SL(N);
13737 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
13739 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
13744 return DAG.getNode(Opc, SL, ResVT, Elt0, Elt1, Vec->getFlags());
13751 SDLoc SL(N);
13755 SDValue IC = DAG.getVectorIdxConstant(I, SL);
13756 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec, IC);
13760 V = DAG.getSelectCC(SL, Idx, IC, Elt, V, ISD::SETEQ);
13779 SDLoc SL(N);
13781 SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec);
13784 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast,
13785 DAG.getConstant(EltIdx, SL, MVT::i32));
13787 SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt,
13788 DAG.getConstant(LeftoverBitIdx, SL, MVT::i32));
13792 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VecEltAsIntVT, Srl);
13796 return DAG.getNode(ISD::BITCAST, SL, VecEltVT, Trunc);
13800 return DAG.getAnyExtOrTrunc(Trunc, SL, ResVT);
13820 SDLoc SL(N);
13826 SDValue IC = DAG.getConstant(I, SL, IdxVT);
13827 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Vec, IC);
13828 SDValue V = DAG.getSelectCC(SL, Idx, IC, Ins, Elt, ISD::SETEQ);
13832 return DAG.getBuildVector(VecVT, SL, Ops);
13869 SDLoc SL(N);
13893 SDValue A1 = DAG.getNode(ISD::FMINNUM_IEEE, SL, VT, A, B);
13894 SDValue B1 = DAG.getNode(ISD::FMAXNUM_IEEE, SL, VT, A, B);
13895 SDValue C1 = DAG.getNode(ISD::FMAXNUM_IEEE, SL, VT, A1, C);
13896 return DAG.getNode(ISD::FMINNUM_IEEE, SL, VT, B1, C1);
13956 SDLoc SL(N);
13957 SDValue Add1 = DAG.getNode(Opc, SL, VT, Op0, Op1);
13958 return DAG.getNode(Opc, SL, VT, Add1, Op2);
13961 static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, EVT VT,
13965 SDValue Mad = DAG.getNode(MadOpc, SL, VTs, N0, N1, N2);
13966 return DAG.getNode(ISD::TRUNCATE, SL, VT, Mad);
13975 static SDValue tryFoldMADwithSRL(SelectionDAG &DAG, const SDLoc &SL,
13994 DAG.getConstant(Lo_32(Const->getZExtValue()), SL, MVT::i32);
13995 return getMad64_32(DAG, SL, MVT::i64,
13996 DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulLHS), ConstMul,
13997 DAG.getZeroExtendInReg(AddRHS, SL, MVT::i32), false);
14014 SDLoc SL(N);
14058 if (SDValue FoldedMAD = tryFoldMADwithSRL(DAG, SL, MulLHS, MulRHS, AddRHS))
14078 MulLHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, MulLHS);
14079 MulRHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, MulRHS);
14080 AddRHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, AddRHS);
14095 SDValue One = DAG.getConstant(1, SL, MVT::i32);
14097 auto MulLHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulLHS);
14098 auto MulRHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulRHS);
14100 getMad64_32(DAG, SL, MVT::i64, MulLHSLo, MulRHSLo, AddRHS, MulSignedLo);
14103 auto [AccumLo, AccumHi] = DAG.SplitScalar(Accum, SL, MVT::i32, MVT::i32);
14107 DAG.getNode(ISD::EXTRACT_ELEMENT, SL, MVT::i32, MulLHS, One);
14108 SDValue MulHi = DAG.getNode(ISD::MUL, SL, MVT::i32, MulLHSHi, MulRHSLo);
14109 AccumHi = DAG.getNode(ISD::ADD, SL, MVT::i32, MulHi, AccumHi);
14114 DAG.getNode(ISD::EXTRACT_ELEMENT, SL, MVT::i32, MulRHS, One);
14115 SDValue MulHi = DAG.getNode(ISD::MUL, SL, MVT::i32, MulLHSLo, MulRHSHi);
14116 AccumHi = DAG.getNode(ISD::ADD, SL, MVT::i32, MulHi, AccumHi);
14119 Accum = DAG.getBuildVector(MVT::v2i32, SL, {AccumLo, AccumHi});
14124 Accum = DAG.getNode(ISD::TRUNCATE, SL, VT, Accum);
14141 SDLoc SL(N);
14154 SDValue ConstHi32 = DAG.getConstant(Hi_32(Val), SL, MVT::i32);
14156 DAG.getNode(N->getOpcode(), SL, MVT::i32, Hi, ConstHi32, N->getFlags());
14158 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
14159 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, Lo, AddHi);
14276 static SDValue resolveSources(SelectionDAG &DAG, SDLoc SL,
14283 auto EltOp = getDWordFromOffset(DAG, SL, Elt->SrcOp, Elt->DWordOffset);
14289 return DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, EltOp, EltOp,
14290 DAG.getConstant(Elt->PermMask, SL, MVT::i32));
14312 getDWordFromOffset(DAG, SL, FirstElt->SrcOp, FirstElt->DWordOffset);
14314 getDWordFromOffset(DAG, SL, SecondElt->SrcOp, SecondElt->DWordOffset);
14316 Perms.push_back(DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, FirstVal,
14318 DAG.getConstant(PermMask, SL, MVT::i32)));
14329 getDWordFromOffset(DAG, SL, FirstElt->SrcOp, FirstElt->DWordOffset);
14332 DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, EltOp, EltOp,
14333 DAG.getConstant(FirstElt->PermMask, SL, MVT::i32)));
14340 ? DAG.getNode(ISD::OR, SL, MVT::i32, Perms[0], Perms[1])
14426 SDLoc SL(N);
14501 Src2s.push_back(DAG.getConstant(0, SL, MVT::i32));
14555 getDWordFromOffset(DAG, SL, FirstElt->SrcOp, FirstElt->DWordOffset);
14558 auto SecondEltOp = getDWordFromOffset(DAG, SL, SecondElt->SrcOp,
14561 Src0 = DAG.getBitcastedAnyExtOrTrunc(FirstEltOp, SL,
14563 Src1 = DAG.getBitcastedAnyExtOrTrunc(SecondEltOp, SL,
14569 Src0 = resolveSources(DAG, SL, Src0s, false, true);
14570 Src1 = resolveSources(DAG, SL, Src1s, false, true);
14575 DAG.getExtOrTrunc(*IsSigned, Src2s[ChainLength - 1], SL, MVT::i32);
14579 SL, MVT::i64);
14582 auto Dot = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, IID, Src0,
14583 Src1, Src2, DAG.getTargetConstant(0, SL, MVT::i1));
14585 return DAG.getExtOrTrunc(*IsSigned, Dot, SL, VT);
14611 SDValue Args[] = {LHS, DAG.getConstant(0, SL, MVT::i32), Cond};
14613 return DAG.getNode(Opc, SL, VTList, Args);
14639 SDLoc SL(N);
14658 SDValue Args[] = {LHS, DAG.getConstant(0, SL, MVT::i32), Cond};
14660 return DAG.getNode(Opc, SL, VTList, Args);
14707 SDLoc SL(N);
14720 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
14721 return DAG.getNode(FusedOp, SL, VT, A, Two, RHS);
14732 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
14733 return DAG.getNode(FusedOp, SL, VT, A, Two, LHS);
14747 SDLoc SL(N);
14764 const SDValue Two = DAG.getConstantFP(2.0, SL, VT);
14765 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
14767 return DAG.getNode(FusedOp, SL, VT, A, Two, NegRHS);
14779 const SDValue NegTwo = DAG.getConstantFP(-2.0, SL, VT);
14780 return DAG.getNode(FusedOp, SL, VT, A, NegTwo, LHS);
14791 SDLoc SL(N);
14814 DAG.getNode(AMDGPUISD::RSQ, SL, VT, RHS.getOperand(0), Flags);
14815 return IsNegative ? DAG.getNode(ISD::FNEG, SL, VT, Rsq, Flags) : Rsq;
14867 SDLoc SL(N);
14869 DAG.getNode(ISD::SELECT, SL, IntVT, RHS.getOperand(0),
14870 DAG.getSignedConstant(TrueNodeExpVal, SL, IntVT),
14871 DAG.getSignedConstant(FalseNodeExpVal, SL, IntVT));
14874 ? DAG.getNode(ISD::FNEG, SL, VT, LHS, LHS->getFlags())
14877 return DAG.getNode(ISD::FLDEXP, SL, VT, LHS, SelectNode, N->getFlags());
14887 SDLoc SL(N);
14949 return DAG.getNode(AMDGPUISD::FDOT2, SL, MVT::f32, Vec1, Vec2, FMAAcc,
14950 DAG.getTargetConstant(0, SL, MVT::i1));
14959 SDLoc SL(N);
14986 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0),
14987 DAG.getAllOnesConstant(SL, MVT::i1));
15012 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0),
15013 DAG.getAllOnesConstant(SL, MVT::i1));
15043 return DAG.getNode(AMDGPUISD::FP_CLASS, SL, MVT::i1, LHS.getOperand(0),
15044 DAG.getConstant(Mask, SL, MVT::i32));
15055 SDLoc SL(N);
15082 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0 + ShiftOffset / 8, SL,
15101 return DAG.getNode(N->getOpcode(), SL, MVT::f32, DemandedSrc);
15246 SDLoc SL(N);
15250 Src = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src);
15252 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Src);
15253 return DAG.getNode(ISD::BITCAST, SL, VT, Ext);
15502 SDLoc SL(Node);
15509 Node->getOperand(0), SL, VReg, SrcVal,
15511 SDValue ToResultReg = DAG.getCopyToReg(ToVReg, SL, SDValue(DestReg, 0),