Lines Matching +full:tri +full:- +full:default +full:- +full:2
1 //===- AMDGPURegisterBankInfo.cpp -------------------------------*- C++ -*-==//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
17 /// sort of pseudo-register bank needed to represent SGPRs used in a vector
31 /// is naturally a bitmask with one bit per lane, in a 32 or 64-bit
37 /// SCC, which is a 1-bit unaddressable register. This will need to be copied to
38 /// a 32-bit virtual register. Taken together, this means we need to adjust the
40 /// widened to 32-bits, and all VALU booleans need to be s1 values.
42 /// A noteworthy exception to the s1-means-vcc rule is for legalization artifact
44 /// bank. A non-boolean source (such as a truncate from a 1-bit load from
52 /// most 1 SGPR or constant literal value (this to 2 in gfx10 for most
67 /// picking the optimal operand combination from a post-isel optimization pass.
69 //===----------------------------------------------------------------------===//
131 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, MRI, *RBI.TRI);
142 auto True = B.buildConstant(S32, Opc == AMDGPU::G_SEXT ? -1 : 1);
158 const RegisterBank *DstBank = RBI.getRegBank(DstReg, MRI, *RBI.TRI);
205 : Subtarget(ST), TRI(Subtarget.getRegisterInfo()),
267 if (ValMapping.NumBreakDowns >= 2 || ValMapping.BreakDown[0].Length >= 64)
270 assert(ValMapping.NumBreakDowns == 2 &&
277 // 32-bit extract of a 64-bit value is just access of a subregister, so free.
281 // TODO: 32-bit insert to a 64-bit SGPR may incur a non-free copy due to SGPR
293 // VCC-like use.
294 if (TRI->isSGPRClass(&RC)) {
296 // should be inferable from the copied to-type. We don't have many boolean
304 return TRI->isAGPRClass(&RC) ? AMDGPU::AGPRRegBank : AMDGPU::VGPRRegBank;
321 Sizes[I] = getSizeInBits(Reg, MRI, *TRI);
325 unsigned SizeI = getSizeInBits(MI.getOperand(I).getReg(), MRI, *TRI);
329 // getInstrMapping's default mapping uses ID 1, so start at 2.
330 unsigned MappingID = 2;
350 static const OpRegBankEntry<3> Table[2] = {
355 { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 }
358 const std::array<unsigned, 3> RegSrcOpIdx = { { 0, 2, 3 } };
367 { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 },
370 { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 },
377 const std::array<unsigned, 4> RegSrcOpIdx = { { 0, 2, 3, 4 } };
380 default:
391 static const OpRegBankEntry<2> Table[4] = {
406 const std::array<unsigned, 2> RegSrcOpIdx = { { 2, 3 } };
407 return addMappingFromTable<2>(MI, MRI, RegSrcOpIdx, Table);
412 static const OpRegBankEntry<3> Table[2] = {
417 { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 }
420 const std::array<unsigned, 3> RegSrcOpIdx = { { 0, 2, 3 } };
426 static const OpRegBankEntry<1> Table[2] = {
434 const std::array<unsigned, 1> RegSrcOpIdx = { { 2 } };
437 default:
449 const unsigned AS = MMO->getAddrSpace();
452 const unsigned MemSize = 8 * MMO->getSize().getValue();
454 // Require 4-byte alignment.
455 return (MMO->getAlign() >= Align(4) ||
457 ((MemSize == 16 && MMO->getAlign() >= Align(2)) ||
458 (MemSize == 8 && MMO->getAlign() >= Align(1))))) &&
460 !MMO->isAtomic() &&
461 // Don't use scalar loads for volatile accesses to non-constant address
463 (IsConst || !MMO->isVolatile()) &&
465 (IsConst || MMO->isInvariant() || (MMO->getFlags() & MONoClobber)) &&
473 const MachineFunction &MF = *MI.getParent()->getParent();
481 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
497 static const OpRegBankEntry<1> Table[2] = {
507 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
510 // s_{and|or|xor}_b32 set scc when the result of the 32-bit op is not 0.
520 2, 1, getOperandsMapping(
541 2, 2, getOperandsMapping(
552 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
564 2); // Num Operands
569 2, 1,
573 2); // Num Operands
586 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
595 const InstructionMapping &VVMapping = getInstructionMapping(2, 1,
609 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
620 const InstructionMapping &VVMapping = getInstructionMapping(2, 1,
637 2); // Num Operands
643 2); // Num Operands
653 default:
661 SmallVector<Register, 2> &Regs,
666 Register LoLHS = MRI->createGenericVirtualRegister(HalfTy);
667 Register HiLHS = MRI->createGenericVirtualRegister(HalfTy);
668 const RegisterBank *Bank = getRegBank(Reg, *MRI, *TRI);
669 MRI->setRegBank(LoLHS, *Bank);
670 MRI->setRegBank(HiLHS, *Bank);
692 assert(Ty.getElementCount().isKnownMultipleOf(2));
693 return LLT::scalarOrVector(Ty.getElementCount().divideCoefficientBy(2),
697 assert(Ty.getScalarSizeInBits() % 2 == 0);
698 return LLT::scalar(Ty.getScalarSizeInBits() / 2);
707 const RegisterBank *Bank = getRegBank(Src, MRI, *TRI);
785 const TargetRegisterClass *WaveRC = TRI->getWaveMaskRegClass();
815 MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
816 MachineBasicBlock *BodyBB = MF->CreateMachineBasicBlock();
817 MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
818 MachineBasicBlock *RestoreExecBB = MF->CreateMachineBasicBlock();
821 MF->insert(MBBI, LoopBB);
822 MF->insert(MBBI, BodyBB);
823 MF->insert(MBBI, RestoreExecBB);
824 MF->insert(MBBI, RemainderBB);
826 LoopBB->addSuccessor(BodyBB);
827 BodyBB->addSuccessor(RestoreExecBB);
828 BodyBB->addSuccessor(LoopBB);
831 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
832 RemainderBB->splice(RemainderBB->begin(), &MBB, Range.end(), MBB.end());
835 RestoreExecBB->addSuccessor(RemainderBB);
837 B.setInsertPt(*LoopBB, LoopBB->end());
852 BodyBB->splice(BodyBB->end(), &MBB, Range.begin(), MBB.end());
856 auto NewEnd = BodyBB->end();
875 Op.setReg(OldVal->second);
882 const RegisterBank *OpBank = getRegBank(OpReg, MRI, *TRI);
931 // Make sure we don't re-process this register again.
936 // The ballot becomes a no-op during instruction selection.
950 B.setInsertPt(*BodyBB, BodyBB->end());
958 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
965 BuildMI(MBB, MBB.end(), DL, TII->get(MovExecOpc), SaveExecReg)
976 B.setInsertPt(*RemainderBB, RemainderBB->begin());
991 const RegisterBank *OpBank = getRegBank(Reg, MRI, *TRI);
992 if (OpBank->getID() != AMDGPU::SGPRRegBankID)
1019 const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI);
1027 /// Split \p Ty into 2 pieces. The first will have \p FirstSize bits, and the
1032 return {LLT::scalar(FirstSize), LLT::scalar(TotalSize - FirstSize)};
1039 unsigned RemainderElts = (TotalSize - FirstSize) / EltSize;
1073 const unsigned MemSize = 8 * MMO->getSize().getValue();
1083 ((MemSize == 8 && MMO->getAlign() >= Align(1)) ||
1084 (MemSize == 16 && MMO->getAlign() >= Align(2))) &&
1094 // This is an extending load from a sub-dword size. Widen the memory
1109 // 96-bit loads are only available for vector loads. We need to split this
1110 // into a 64-bit part, and 32 (unless we can widen to a 128-bit load).
1111 if (MMO->getAlign() < Align(16)) {
1134 // 128-bit loads are supported for all instruction types.
1185 Align Alignment = assumeAligned(MI.getOperand(2).getImm());
1187 const RegisterBank *SizeBank = getRegBank(AllocSize, MRI, *TRI);
1197 Register SPReg = Info->getStackPtrOffsetReg();
1255 if (TII->splitMUBUFOffset(*Imm, SOffset, ImmOffset, Alignment)) {
1260 B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank);
1261 B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank);
1274 TII->splitMUBUFOffset(Offset, SOffset, ImmOffset, Alignment)) {
1275 if (getRegBank(Base, *MRI, *TRI) == &AMDGPU::VGPRRegBank) {
1278 B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank);
1280 return 0; // XXX - Why is this 0?
1286 B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank);
1289 return 0; // XXX - Why is this 0?
1296 Register Src0 = getSrcRegIgnoringCopies(Add->getOperand(1).getReg(), *MRI);
1297 Register Src1 = getSrcRegIgnoringCopies(Add->getOperand(2).getReg(), *MRI);
1299 const RegisterBank *Src0Bank = getRegBank(Src0, *MRI, *TRI);
1300 const RegisterBank *Src1Bank = getRegBank(Src1, *MRI, *TRI);
1317 if (getRegBank(CombinedOffset, *MRI, *TRI) == &AMDGPU::VGPRRegBank) {
1321 B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank);
1325 B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank);
1341 OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
1346 // FIXME: 96-bit case was widened during legalize. We need to narrow it back
1366 unsigned MMOOffset = setBufferOffsets(B, MI.getOperand(2).getReg(), VOffset,
1369 // TODO: 96-bit loads were widened to 128-bit results. Shrink the result if we
1386 B.getMRI()->setRegBank(VIndex, AMDGPU::VGPRRegBank);
1461 unsigned FirstOpnd = isa<GIntrinsic>(MI) ? 2 : 1;
1464 Register WidthReg = MI.getOperand(FirstOpnd + 2).getReg();
1472 // There is no 64-bit vgpr bitfield extract instructions so the operation
1482 // A 64-bit bitfield extract uses the 32-bit bitfield extract instructions
1485 // Use the 32-bit bitfield extract instruction if the width is a constant.
1486 // Depending on the width size, use either the low or high 32-bits.
1488 auto WidthImm = ConstWidth->Value.getZExtValue();
1490 // Use bitfield extract on the lower 32-bit source, and then sign-extend
1491 // or clear the upper 32-bits.
1499 // Use bitfield extract on upper 32-bit source, and combine with lower
1500 // 32-bit source.
1501 auto UpperWidth = B.buildConstant(S32, WidthImm - 32);
1512 // Expand to Src >> Offset << (64 - Width) >> (64 - Width) using 64-bit
1546 if (!constrainSelectedInstRegOperands(*MIB, *TII, *TRI, *this))
1563 Register Src0 = MI.getOperand(2).getReg();
1611 // Accumulate and produce the "carry-out" bit.
1613 // The "carry-out" is defined as bit 64 of the result when computed as a
1614 // big integer. For unsigned multiply-add, this matches the usual definition
1615 // of carry-out. For signed multiply-add, bit 64 is the sign bit of the
1617 // sign(Src0 * Src1) + sign(Src2) + carry-out from unsigned 64-bit add
1713 default:
1718 // Emit a legalized extension from <2 x s16> to 2 32-bit components, avoiding
1771 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
1811 // is copied/added for the voffset field is a large power of 2, and it
1818 ImmOffset -= Overflow;
1893 /// Implement extending a 32-bit value to a 64-bit value. \p Lo32Reg is the
1894 /// original 32-bit source value (to be inserted in the low part of the combined
1895 /// 64-bit result), and \p Hi32Reg is the high half of the combined 64-bit
1910 // Replicate sign bit from 32-bit extended part.
1912 B.getMRI()->setRegBank(ShiftAmt.getReg(0), RegBank);
1927 Register Idx = MI.getOperand(2).getReg();
1930 *OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
1957 Idx = B.buildCopy(S32, Idx)->getOperand(0).getReg();
1962 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(0));
1970 SmallVector<Register, 2> Res(NumLanes);
1976 MRI.setRegBank(IC->getOperand(0).getReg(), AMDGPU::SGPRRegBank);
1978 MRI.setRegBank(Cmp->getOperand(0).getReg(), CCBank);
1984 for (unsigned N : { 0, 2, 3 })
1985 MRI.setRegBank(S->getOperand(N).getReg(), DstBank);
1987 Res[L] = S->getOperand(0).getReg();
2047 *OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
2058 Idx = B.buildCopy(S32, Idx)->getOperand(0).getReg();
2063 SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(2));
2067 InsRegs.push_back(MI.getOperand(2).getReg());
2077 MRI.setRegBank(IC->getOperand(0).getReg(), AMDGPU::SGPRRegBank);
2079 MRI.setRegBank(Cmp->getOperand(0).getReg(), CCBank);
2098 MRI.setRegBank(Vec->getOperand(0).getReg(), DstBank);
2108 // Break s_mul_u64 into 32-bit vector operations.
2111 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0));
2112 SmallVector<Register, 2> Src0Regs(OpdMapper.getVRegs(1));
2113 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(2));
2122 assert(DefRegs.size() == 2);
2124 (Src0Regs.empty() || Src0Regs.size() == 2));
2141 split64BitValueForMapping(B, Src1Regs, HalfTy, MI.getOperand(2).getReg());
2151 // --------------------
2154 // -----------------------------------------
2157 // We drop Op1H*Op0H because the result of the multiplication is a 64-bit
2159 // The low 32-bit value is Op1L*Op0L.
2160 // The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from
2205 uint64_t ConstVal = MI.getOperand(1).getCImm()->getZExtValue();
2230 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2232 const RegisterBank *SrcBank = getRegBank(SrcReg, MRI, *TRI);
2236 B.setInsertPt(*SrcMBB, SrcMBB->getFirstTerminator());
2283 // will end up using a copy to a 32-bit vreg.
2318 const RegisterBank *CondBank = getRegBank(CondRegs[0], MRI, *TRI);
2333 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0));
2334 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(2));
2335 SmallVector<Register, 2> Src2Regs(OpdMapper.getVRegs(3));
2344 split64BitValueForMapping(B, Src1Regs, HalfTy, MI.getOperand(2).getReg());
2384 // 64-bit and is only available on the SALU, so split into 2 32-bit ops if
2395 MachineFunction *MF = MI.getParent()->getParent();
2409 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0));
2410 SmallVector<Register, 2> Src0Regs(OpdMapper.getVRegs(1));
2411 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(2));
2419 assert(DefRegs.size() == 2);
2421 (Src0Regs.empty() || Src0Regs.size() == 2));
2433 split64BitValueForMapping(B, Src1Regs, HalfTy, MI.getOperand(2).getReg());
2453 MachineFunction *MF = MI.getParent()->getParent();
2477 // s_mul_u64. Hence, we have to break down s_mul_u64 into 32-bit vector
2484 // 16-bit operations are VALU only, but can be promoted to 32-bit SALU.
2485 // Packed 16-bit operations need to be scalarized and promoted.
2486 if (DstTy != LLT::scalar(16) && DstTy != LLT::fixed_vector(2, 16))
2496 MachineFunction *MF = MBB->getParent();
2519 = unpackV2S16ToS32(B, MI.getOperand(2).getReg(), ExtendOp);
2545 // where the 33 higher bits are sign-extended and
2547 // where the 32 higher bits are zero-extended. In case scalar registers are
2557 Register SrcReg1 = MI.getOperand(2).getReg();
2561 "that handles only 64-bit operands.");
2568 MI.setDesc(TII->get(AMDGPU::S_MUL_U64));
2608 SmallVector<Register, 2> SrcRegs(OpdMapper.getVRegs(1));
2618 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(0));
2620 int Amt = MI.getOperand(2).getImm();
2638 B.buildSExtInReg(DstRegs[1], DstRegs[0], Amt - 32);
2684 // which return -1 when the input is zero:
2685 // (ctlz_zero_undef hi:lo) -> (umin (ffbh hi), (add (ffbh lo), 32))
2686 // (cttz_zero_undef hi:lo) -> (umin (add (ffbl hi), 32), (ffbl lo))
2687 // (ffbh hi:lo) -> (umin (ffbh hi), (uaddsat (ffbh lo), 32))
2688 // (ffbl hi:lo) -> (umin (uaddsat (ffbh hi), 32), (ffbh lo))
2690 SmallVector<Register, 2> SrcRegs(OpdMapper.getVRegs(1));
2730 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0));
2732 // Extend to 32-bit, and then extend the low half.
2755 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0));
2760 // 64-bit select is SGPR only
2762 SrcBank->getID() == AMDGPU::SGPRRegBankID;
2766 auto True = B.buildConstant(SelType, Signed ? -1 : 1);
2791 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(0));
2793 assert(OpdMapper.getVRegs(1).empty() && OpdMapper.getVRegs(2).empty());
2811 OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
2816 AMDGPU::getBaseWithConstantOffset(MRI, MI.getOperand(2).getReg());
2826 // Move the base register. We'll re-insert the add later.
2828 MI.getOperand(2).setReg(BaseIdxReg);
2839 executeInWaterfallLoop(B, MI, {2});
2852 // Re-insert the constant offset add inside the waterfall loop.
2854 reinsertVectorIndexAdd(B, MI, 2, ConstOffset);
2861 LLT Vec32 = LLT::fixed_vector(2 * SrcTy.getNumElements(), 32);
2868 // Split the vector index into 32-bit pieces. Prepare to move all of the
2874 // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1).
2888 if (!collectWaterfallOperands(OpsToWaterfall, MI, MRI, { 2 })) {
2901 MachineBasicBlock *LoopBB = Extract1->getParent();
2907 Extract0->getOperand(0).setReg(TmpReg0);
2908 Extract1->getOperand(0).setReg(TmpReg1);
2910 B.setInsertPt(*LoopBB, ++Extract1->getIterator());
2922 SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(2));
2940 Register InsReg = MI.getOperand(2).getReg();
2957 // Move the base register. We'll re-insert the add later.
2965 // Re-insert the constant offset add inside the waterfall loop.
2976 LLT Vec32 = LLT::fixed_vector(2 * VecTy.getNumElements(), 32);
2981 // Split the vector index into 32-bit pieces. Prepare to move all of the
2987 // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1).
2999 OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank;
3032 // Re-insert the constant offset add inside the waterfall loop.
3077 executeInWaterfallLoop(B, MI, {2, 5});
3084 executeInWaterfallLoop(B, MI, {2, 5});
3104 substituteSimpleCopyRegs(OpdMapper, 2);
3116 assert(OpdMapper.getVRegs(2).empty());
3120 constrainOpWithReadfirstlane(B, MI, 2); // Source value
3134 constrainOpWithReadfirstlane(B, MI, MI.getNumOperands() - 1); // Index
3148 substituteSimpleCopyRegs(OpdMapper, 2);
3165 constrainOpWithReadfirstlane(B, MI, 2); // Mask
3168 // Use default handling and insert copy to vcc source.
3180 assert(RSrcIntrin && RSrcIntrin->IsImage);
3181 // Non-images can have complications from operands that allow both SGPR
3184 applyMappingImage(B, MI, OpdMapper, RSrcIntrin->RsrcArg);
3188 unsigned N = MI.getNumExplicitOperands() - 2;
3202 constrainOpWithReadfirstlane(B, MI, 2); // M0
3210 constrainOpWithReadfirstlane(B, MI, 2); // M0
3222 constrainOpWithReadfirstlane(B, MI, 2); // M0
3228 constrainOpWithReadfirstlane(B, MI, 2); // M0
3232 constrainOpWithReadfirstlane(B, MI, 2);
3242 constrainOpWithReadfirstlane(B, MI, 2); // M0
3250 constrainOpWithReadfirstlane(B, MI, 2); // M0
3256 constrainOpWithReadfirstlane(B, MI, 2);
3262 constrainOpWithReadfirstlane(B, MI, MI.getNumOperands() - 1); // Index
3279 constrainOpWithReadfirstlane(B, MI, 2);
3283 constrainOpWithReadfirstlane(B, MI, 2);
3286 constrainOpWithReadfirstlane(B, MI, 2);
3289 default: {
3292 // Non-images can have complications from operands that allow both SGPR
3295 if (RSrcIntrin->IsImage) {
3296 applyMappingImage(B, MI, OpdMapper, RSrcIntrin->RsrcArg);
3320 // Move all non-copies before the copies, so that a complete range can be
3329 MBB->getParent()->getInfo<SIMachineFunctionInfo>();
3330 while (Start->getOpcode() != FrameSetupOpcode) {
3331 --Start;
3333 if (Start->getOpcode() == AMDGPU::COPY) {
3334 auto &Dst = Start->getOperand(0);
3337 if (Reg.isPhysical() && MI.readsRegister(Reg, TRI)) {
3342 auto &Src = Start->getOperand(1);
3345 IsCopy = Info->getScratchRSrcReg() == Reg;
3361 MBB->splice(LastCopy, MBB, NonCopy->getIterator());
3370 while (End->getOpcode() != FrameDestroyOpcode) {
3373 if (End->getOpcode() == AMDGPU::COPY) {
3374 auto &Src = End->getOperand(1);
3377 IsCopy = Reg.isPhysical() && MI.modifiesRegister(Reg, TRI);
3393 MBB->splice(LastCopy, MBB, NonCopy->getIterator());
3446 default:
3453 // vgpr, sgpr -> vgpr
3454 // vgpr, agpr -> vgpr
3455 // agpr, agpr -> agpr
3456 // agpr, sgpr -> vgpr
3478 // vcc, vcc -> vcc
3479 // vcc, sgpr -> vcc
3480 // vcc, vgpr -> vcc
3484 // vcc, vgpr -> vgpr
3496 if (const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI)) {
3497 RegBank = regBankUnion(RegBank, Bank->getID());
3507 const MachineFunction &MF = *MI.getParent()->getParent();
3513 if (const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI)) {
3514 if (Bank->getID() != AMDGPU::SGPRRegBankID)
3523 const MachineFunction &MF = *MI.getParent()->getParent();
3532 unsigned Size = getSizeInBits(SrcOp.getReg(), MRI, *TRI);
3541 const MachineFunction &MF = *MI.getParent()->getParent();
3554 unsigned Size = getSizeInBits(Src.getReg(), MRI, *TRI);
3565 const MachineFunction &MF = *MI.getParent()->getParent();
3574 unsigned Size = getSizeInBits(Op.getReg(), MRI, *TRI);
3604 unsigned Size = getSizeInBits(OpReg, MRI, *TRI);
3637 const RegisterBank *PtrBank = getRegBank(PtrReg, MRI, *TRI);
3638 return AMDGPU::getValueMapping(PtrBank->getID(), Size);
3644 const MachineFunction &MF = *MI.getParent()->getParent();
3646 SmallVector<const ValueMapping*, 2> OpdsMapping(2);
3647 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
3656 const RegisterBank *PtrBank = getRegBank(PtrReg, MRI, *TRI);
3691 unsigned Default) const {
3692 const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI);
3693 return Bank ? Bank->getID() : Default;
3699 const TargetRegisterInfo &TRI) const {
3703 unsigned Size = getSizeInBits(Reg, MRI, TRI);
3710 const TargetRegisterInfo &TRI) const {
3711 unsigned Size = getSizeInBits(Reg, MRI, TRI);
3718 const TargetRegisterInfo &TRI) const {
3719 unsigned Size = getSizeInBits(Reg, MRI, TRI);
3734 const MachineFunction &MF = *MI.getParent()->getParent();
3738 // The default logic bothers to analyze impossible alternative mappings. We
3741 *TRI);
3743 *TRI);
3748 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
3754 unsigned OpdsMappingSize = MI.isCopy() ? 1 : 2;
3766 // If any input is a VGPR, the result must be a VGPR. The default handling
3770 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3779 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
3787 // The default handling is broken and doesn't handle illegal SGPR->VGPR copies
3793 Register DstReg = PHI->getReg(0);
3796 if (const RegisterBank *DstBank = getRegBank(DstReg, MRI, *TRI))
3797 ResultBank = DstBank->getID();
3799 for (unsigned I = 0; I < PHI->getNumIncomingValues(); ++I) {
3800 Register Reg = PHI->getIncomingValue(I);
3801 const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI);
3804 if (!Bank || Bank->getID() == AMDGPU::VGPRRegBankID) {
3810 unsigned OpBank = Bank->getID();
3832 default:
3842 = getRegBank(MI.getOperand(0).getReg(), MRI, *TRI);
3848 TargetBankID = DstBank->getID();
3856 BankRHS = getRegBankID(MI.getOperand(2).getReg(), MRI,
3862 BankRHS = getRegBankID(MI.getOperand(2).getReg(), MRI,
3879 OpdsMapping[2] = AMDGPU::getValueMapping(BankRHS, Size);
3887 OpdsMapping[1] = OpdsMapping[2] = OpdsMapping[0];
3893 unsigned Bank2 = getRegBankID(MI.getOperand(2).getReg(), MRI /*, DefaultBankID*/);
3894 OpdsMapping[2] = AMDGPU::getValueMapping(Bank2, Size);
4013 // - Default SOP
4014 // - Default VOP
4015 // - Scalar multiply: src0 and src1 are SGPRs, the rest is VOP.
4023 if (const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI)) {
4024 if (Bank->getID() != AMDGPU::SGPRRegBankID) {
4026 if (i == 2 || i == 3) {
4037 // If the multiply-add is full-rate in VALU, use that even if the
4046 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
4091 unsigned DstSize = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
4092 unsigned SrcSize = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
4093 unsigned EltSize = getSizeInBits(MI.getOperand(2).getReg(), MRI, *TRI);
4096 OpdsMapping[2] = AMDGPU::getValueMapping(BankID, EltSize);
4102 unsigned DstSize = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
4103 unsigned SrcSize = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
4106 OpdsMapping[2] = nullptr;
4112 if (DstTy == LLT::fixed_vector(2, 16)) {
4116 unsigned Src1BankID = getRegBankID(MI.getOperand(2).getReg(), MRI);
4121 OpdsMapping[2] = AMDGPU::getValueMapping(Src1BankID, SrcSize);
4175 unsigned DstSize = getSizeInBits(Dst, MRI, *TRI);
4176 unsigned SrcSize = getSizeInBits(Src, MRI, *TRI);
4187 unsigned DstSize = getSizeInBits(Dst, MRI, *TRI);
4188 unsigned SrcSize = getSizeInBits(Src, MRI, *TRI);
4191 const RegisterBank *SrcBank = getRegBank(Src, MRI, *TRI);
4193 switch (SrcBank->getID()) {
4197 default:
4202 // Scalar extend can use 64-bit BFE, but VGPRs require extending to
4203 // 32-bits, and then to 64.
4205 OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(SrcBank->getID(),
4231 unsigned Size = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
4237 unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI);
4261 // TODO: Use 32-bit for scalar output size.
4262 // SCC results will need to be copied to a 32-bit SGPR virtual register.
4267 OpdsMapping[2] = AMDGPU::getValueMapping(SrcBank, Size);
4276 unsigned IdxSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
4277 unsigned IdxBank = getRegBankID(MI.getOperand(2).getReg(), MRI);
4284 OpdsMapping[2] = AMDGPU::getValueMapping(IdxBank, IdxSize);
4292 unsigned InsertSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
4294 unsigned InsertEltBankID = getRegBankID(MI.getOperand(2).getReg(), MRI);
4303 OpdsMapping[2] = AMDGPU::getValueMappingSplit64(InsertEltBankID,
4307 OpdsMapping[2] = AMDGPU::getValueMapping(InsertEltBankID, InsertSize);
4318 // FIXME: Shouldn't this be the default? Why do we need to handle this?
4320 unsigned Size = getSizeInBits(MI.getOperand(i).getReg(), MRI, *TRI);
4347 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4350 OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
4353 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4356 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4359 OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4381 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4384 OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
4387 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4390 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4393 OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4396 OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
4404 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4407 OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
4410 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4413 OpdsMapping[3] = getSGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4416 OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4419 OpdsMapping[5] = getVGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
4422 OpdsMapping[6] = getSGPROpMapping(MI.getOperand(6).getReg(), MRI, *TRI);
4435 OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
4436 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4440 unsigned RSrcBank = OpdsMapping[1]->BreakDown[0].RegBank->getID();
4441 unsigned OffsetBank = OpdsMapping[2]->BreakDown[0].RegBank->getID();
4451 default:
4590 OpdsMapping[0] = OpdsMapping[2]
4610 Register Src0Reg = MI.getOperand(2).getReg();
4616 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Src0Size);
4625 unsigned OpSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
4626 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, OpSize);
4640 unsigned SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
4642 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize);
4647 Register SrcReg = MI.getOperand(2).getReg();
4655 // These 2 must be SGPRs, but accept VGPRs. Readfirstlane will be inserted
4657 OpdsMapping[2] = AMDGPU::getValueMapping(SrcBank, SrcSize);
4663 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
4665 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
4671 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
4673 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
4675 OpdsMapping[4] = getSGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4676 OpdsMapping[5] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4681 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
4683 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
4727 // Default for MAI intrinsics.
4735 Info->mayNeedAGPRs()
4736 ? getAGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI)
4737 : getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4738 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4739 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4741 Info->mayNeedAGPRs()
4742 ? getAGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI)
4743 : getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4761 OpdsMapping[0] = getAGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4762 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4763 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4764 OpdsMapping[4] = getAGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
4765 OpdsMapping[5] = getVGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
4774 const int M0Idx = MI.getNumOperands() - 1;
4780 for (int I = 2; I != M0Idx && MI.getOperand(I).isReg(); ++I)
4796 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
4803 unsigned SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
4805 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, SrcSize);
4810 Register MaskReg = MI.getOperand(2).getReg();
4814 OpdsMapping[2] = AMDGPU::getValueMapping(MaskBank, MaskSize);
4819 Register MaskReg = MI.getOperand(2).getReg();
4823 OpdsMapping[2] = AMDGPU::getValueMapping(MaskBank, MaskSize);
4830 unsigned OpSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
4833 OpdsMapping[2] = AMDGPU::getValueMapping(regBankID, OpSize);
4837 Register MaskReg = MI.getOperand(2).getReg();
4840 OpdsMapping[2] = AMDGPU::getValueMapping(MaskBank, 32);
4852 // Non-images can have complications from operands that allow both SGPR
4855 assert(RSrcIntrin->IsImage);
4856 return getImageMapping(MRI, MI, RSrcIntrin->RsrcArg);
4859 unsigned N = MI.getNumExplicitOperands() - 2;
4861 OpdsMapping[N] = getSGPROpMapping(MI.getOperand(N).getReg(), MRI, *TRI);
4864 unsigned Size = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
4867 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
4870 for (unsigned I = 2; I < N; ++I) {
4912 unsigned M0Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
4914 OpdsMapping[2] = AMDGPU::getValueMapping(M0Bank, 32);
4922 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4941 OpdsMapping[8] = getSGPROpMapping(MI.getOperand(8).getReg(), MRI, *TRI);
4946 unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
4948 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
4953 unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
4955 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
4966 unsigned Size = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
4971 unsigned WaveSize = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
4994 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
4995 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
4996 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
4997 OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
5002 OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
5003 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
5004 OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
5005 OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
5014 OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
5015 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
5016 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
5017 OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
5024 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
5025 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
5026 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
5027 OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
5028 OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
5033 OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
5034 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
5035 OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
5036 OpdsMapping[5] = getVGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
5037 OpdsMapping[6] = getSGPROpMapping(MI.getOperand(6).getReg(), MRI, *TRI);
5044 OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
5045 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
5046 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
5047 OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
5048 OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
5052 unsigned Size = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
5062 unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
5064 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
5077 OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
5078 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
5082 const int M0Idx = MI.getNumOperands() - 1;
5088 for (int I = 2; I != M0Idx && MI.getOperand(I).isReg(); ++I)
5098 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
5099 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
5103 getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); // %vdst
5105 getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI); // %addr
5107 getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); // %addr
5109 getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); // %data0
5111 getVGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI); // %data1
5115 OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
5120 OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
5123 OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
5124 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
5130 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
5141 OpdsMapping[0] = getSGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
5142 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
5147 default:
5154 unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
5175 // TODO: Should report 32-bit for scalar condition type.
5179 OpdsMapping[2] = AMDGPU::getValueMappingSGPR64Only(Bank, Size);
5184 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, Size);
5195 OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
5202 unsigned Size = getSizeInBits(Reg, MRI, *TRI);
5229 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
5231 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
5235 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
5237 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
5238 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
5255 OpdsMapping[0] = getSGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);