xref: /llvm-project/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp (revision 21704a685de5f241acddf462e5f9b38d132cfcaa) 
1 //===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "SIMachineFunctionInfo.h"
10 #include "AMDGPUSubtarget.h"
11 #include "GCNSubtarget.h"
12 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
13 #include "SIRegisterInfo.h"
14 #include "Utils/AMDGPUBaseInfo.h"
15 #include "llvm/CodeGen/LiveIntervals.h"
16 #include "llvm/CodeGen/MIRParser/MIParser.h"
17 #include "llvm/CodeGen/MachineBasicBlock.h"
18 #include "llvm/CodeGen/MachineFrameInfo.h"
19 #include "llvm/CodeGen/MachineFunction.h"
20 #include "llvm/CodeGen/MachineRegisterInfo.h"
21 #include "llvm/IR/CallingConv.h"
22 #include "llvm/IR/DiagnosticInfo.h"
23 #include "llvm/IR/Function.h"
24 #include <cassert>
25 #include <optional>
26 #include <vector>
27 
28 enum { MAX_LANES = 64 };
29 
30 using namespace llvm;
31 
32 const GCNTargetMachine &getTM(const GCNSubtarget *STI) {
33   const SITargetLowering *TLI = STI->getTargetLowering();
34   return static_cast<const GCNTargetMachine &>(TLI->getTargetMachine());
35 }
36 
37 SIMachineFunctionInfo::SIMachineFunctionInfo(const Function &F,
38                                              const GCNSubtarget *STI)
39     : AMDGPUMachineFunction(F, *STI), Mode(F, *STI), GWSResourcePSV(getTM(STI)),
40       UserSGPRInfo(F, *STI), WorkGroupIDX(false), WorkGroupIDY(false),
41       WorkGroupIDZ(false), WorkGroupInfo(false), LDSKernelId(false),
42       PrivateSegmentWaveByteOffset(false), WorkItemIDX(false),
43       WorkItemIDY(false), WorkItemIDZ(false), ImplicitArgPtr(false),
44       GITPtrHigh(0xffffffff), HighBitsOf32BitAddress(0) {
45   const GCNSubtarget &ST = *static_cast<const GCNSubtarget *>(STI);
46   FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
47   WavesPerEU = ST.getWavesPerEU(F);
48   MaxNumWorkGroups = ST.getMaxNumWorkGroups(F);
49   assert(MaxNumWorkGroups.size() == 3);
50 
51   Occupancy = ST.computeOccupancy(F, getLDSSize());
52   CallingConv::ID CC = F.getCallingConv();
53 
54   VRegFlags.reserve(1024);
55 
56   const bool IsKernel = CC == CallingConv::AMDGPU_KERNEL ||
57                         CC == CallingConv::SPIR_KERNEL;
58 
59   if (IsKernel) {
60     WorkGroupIDX = true;
61     WorkItemIDX = true;
62   } else if (CC == CallingConv::AMDGPU_PS) {
63     PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
64   }
65 
66   MayNeedAGPRs = ST.hasMAIInsts();
67 
68   if (AMDGPU::isChainCC(CC)) {
69     // Chain functions don't receive an SP from their caller, but are free to
70     // set one up. For now, we can use s32 to match what amdgpu_gfx functions
71     // would use if called, but this can be revisited.
72     // FIXME: Only reserve this if we actually need it.
73     StackPtrOffsetReg = AMDGPU::SGPR32;
74 
75     ScratchRSrcReg = AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51;
76 
77     ArgInfo.PrivateSegmentBuffer =
78         ArgDescriptor::createRegister(ScratchRSrcReg);
79 
80     ImplicitArgPtr = false;
81   } else if (!isEntryFunction()) {
82     if (CC != CallingConv::AMDGPU_Gfx)
83       ArgInfo = AMDGPUArgumentUsageInfo::FixedABIFunctionInfo;
84 
85     FrameOffsetReg = AMDGPU::SGPR33;
86     StackPtrOffsetReg = AMDGPU::SGPR32;
87 
88     if (!ST.enableFlatScratch()) {
89       // Non-entry functions have no special inputs for now, other registers
90       // required for scratch access.
91       ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
92 
93       ArgInfo.PrivateSegmentBuffer =
94         ArgDescriptor::createRegister(ScratchRSrcReg);
95     }
96 
97     if (!F.hasFnAttribute("amdgpu-no-implicitarg-ptr"))
98       ImplicitArgPtr = true;
99   } else {
100     ImplicitArgPtr = false;
101     MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(),
102                                MaxKernArgAlign);
103 
104     if (ST.hasGFX90AInsts() &&
105         ST.getMaxNumVGPRs(F) <= AMDGPU::VGPR_32RegClass.getNumRegs() &&
106         !mayUseAGPRs(F))
107       MayNeedAGPRs = false; // We will select all MAI with VGPR operands.
108   }
109 
110   if (!AMDGPU::isGraphics(CC) ||
111       ((CC == CallingConv::AMDGPU_CS || CC == CallingConv::AMDGPU_Gfx) &&
112        ST.hasArchitectedSGPRs())) {
113     if (IsKernel || !F.hasFnAttribute("amdgpu-no-workgroup-id-x"))
114       WorkGroupIDX = true;
115 
116     if (!F.hasFnAttribute("amdgpu-no-workgroup-id-y"))
117       WorkGroupIDY = true;
118 
119     if (!F.hasFnAttribute("amdgpu-no-workgroup-id-z"))
120       WorkGroupIDZ = true;
121   }
122 
123   if (!AMDGPU::isGraphics(CC)) {
124     if (IsKernel || !F.hasFnAttribute("amdgpu-no-workitem-id-x"))
125       WorkItemIDX = true;
126 
127     if (!F.hasFnAttribute("amdgpu-no-workitem-id-y") &&
128         ST.getMaxWorkitemID(F, 1) != 0)
129       WorkItemIDY = true;
130 
131     if (!F.hasFnAttribute("amdgpu-no-workitem-id-z") &&
132         ST.getMaxWorkitemID(F, 2) != 0)
133       WorkItemIDZ = true;
134 
135     if (!IsKernel && !F.hasFnAttribute("amdgpu-no-lds-kernel-id"))
136       LDSKernelId = true;
137   }
138 
139   if (isEntryFunction()) {
140     // X, XY, and XYZ are the only supported combinations, so make sure Y is
141     // enabled if Z is.
142     if (WorkItemIDZ)
143       WorkItemIDY = true;
144 
145     if (!ST.flatScratchIsArchitected()) {
146       PrivateSegmentWaveByteOffset = true;
147 
148       // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
149       if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
150           (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))
151         ArgInfo.PrivateSegmentWaveByteOffset =
152             ArgDescriptor::createRegister(AMDGPU::SGPR5);
153     }
154   }
155 
156   Attribute A = F.getFnAttribute("amdgpu-git-ptr-high");
157   StringRef S = A.getValueAsString();
158   if (!S.empty())
159     S.consumeInteger(0, GITPtrHigh);
160 
161   A = F.getFnAttribute("amdgpu-32bit-address-high-bits");
162   S = A.getValueAsString();
163   if (!S.empty())
164     S.consumeInteger(0, HighBitsOf32BitAddress);
165 
166   MaxMemoryClusterDWords = F.getFnAttributeAsParsedInteger(
167       "amdgpu-max-memory-cluster-dwords", DefaultMemoryClusterDWordsLimit);
168 
169   // On GFX908, in order to guarantee copying between AGPRs, we need a scratch
170   // VGPR available at all times. For now, reserve highest available VGPR. After
171   // RA, shift it to the lowest available unused VGPR if the one exist.
172   if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
173     VGPRForAGPRCopy =
174         AMDGPU::VGPR_32RegClass.getRegister(ST.getMaxNumVGPRs(F) - 1);
175   }
176 }
177 
178 MachineFunctionInfo *SIMachineFunctionInfo::clone(
179     BumpPtrAllocator &Allocator, MachineFunction &DestMF,
180     const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
181     const {
182   return DestMF.cloneInfo<SIMachineFunctionInfo>(*this);
183 }
184 
185 void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) {
186   limitOccupancy(getMaxWavesPerEU());
187   const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
188   limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(),
189                  MF.getFunction()));
190 }
191 
192 Register SIMachineFunctionInfo::addPrivateSegmentBuffer(
193   const SIRegisterInfo &TRI) {
194   ArgInfo.PrivateSegmentBuffer =
195     ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
196     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass));
197   NumUserSGPRs += 4;
198   return ArgInfo.PrivateSegmentBuffer.getRegister();
199 }
200 
201 Register SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) {
202   ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
203     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
204   NumUserSGPRs += 2;
205   return ArgInfo.DispatchPtr.getRegister();
206 }
207 
208 Register SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) {
209   ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
210     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
211   NumUserSGPRs += 2;
212   return ArgInfo.QueuePtr.getRegister();
213 }
214 
215 Register SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) {
216   ArgInfo.KernargSegmentPtr
217     = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
218     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
219   NumUserSGPRs += 2;
220   return ArgInfo.KernargSegmentPtr.getRegister();
221 }
222 
223 Register SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) {
224   ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
225     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
226   NumUserSGPRs += 2;
227   return ArgInfo.DispatchID.getRegister();
228 }
229 
230 Register SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) {
231   ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
232     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
233   NumUserSGPRs += 2;
234   return ArgInfo.FlatScratchInit.getRegister();
235 }
236 
237 Register SIMachineFunctionInfo::addPrivateSegmentSize(const SIRegisterInfo &TRI) {
238   ArgInfo.PrivateSegmentSize = ArgDescriptor::createRegister(getNextUserSGPR());
239   NumUserSGPRs += 1;
240   return ArgInfo.PrivateSegmentSize.getRegister();
241 }
242 
243 Register SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) {
244   ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
245     getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
246   NumUserSGPRs += 2;
247   return ArgInfo.ImplicitBufferPtr.getRegister();
248 }
249 
250 Register SIMachineFunctionInfo::addLDSKernelId() {
251   ArgInfo.LDSKernelId = ArgDescriptor::createRegister(getNextUserSGPR());
252   NumUserSGPRs += 1;
253   return ArgInfo.LDSKernelId.getRegister();
254 }
255 
256 SmallVectorImpl<MCRegister> *SIMachineFunctionInfo::addPreloadedKernArg(
257     const SIRegisterInfo &TRI, const TargetRegisterClass *RC,
258     unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs) {
259   assert(!ArgInfo.PreloadKernArgs.count(KernArgIdx) &&
260          "Preload kernel argument allocated twice.");
261   NumUserSGPRs += PaddingSGPRs;
262   // If the available register tuples are aligned with the kernarg to be
263   // preloaded use that register, otherwise we need to use a set of SGPRs and
264   // merge them.
265   if (!ArgInfo.FirstKernArgPreloadReg)
266     ArgInfo.FirstKernArgPreloadReg = getNextUserSGPR();
267   Register PreloadReg =
268       TRI.getMatchingSuperReg(getNextUserSGPR(), AMDGPU::sub0, RC);
269   if (PreloadReg &&
270       (RC == &AMDGPU::SReg_32RegClass || RC == &AMDGPU::SReg_64RegClass)) {
271     ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(PreloadReg);
272     NumUserSGPRs += AllocSizeDWord;
273   } else {
274     for (unsigned I = 0; I < AllocSizeDWord; ++I) {
275       ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(getNextUserSGPR());
276       NumUserSGPRs++;
277     }
278   }
279 
280   // Track the actual number of SGPRs that HW will preload to.
281   UserSGPRInfo.allocKernargPreloadSGPRs(AllocSizeDWord + PaddingSGPRs);
282   return &ArgInfo.PreloadKernArgs[KernArgIdx].Regs;
283 }
284 
285 void SIMachineFunctionInfo::allocateWWMSpill(MachineFunction &MF, Register VGPR,
286                                              uint64_t Size, Align Alignment) {
287   // Skip if it is an entry function or the register is already added.
288   if (isEntryFunction() || WWMSpills.count(VGPR))
289     return;
290 
291   // Skip if this is a function with the amdgpu_cs_chain or
292   // amdgpu_cs_chain_preserve calling convention and this is a scratch register.
293   // We never need to allocate a spill for these because we don't even need to
294   // restore the inactive lanes for them (they're scratchier than the usual
295   // scratch registers). We only need to do this if we have calls to
296   // llvm.amdgcn.cs.chain (otherwise there's no one to save them for, since
297   // chain functions do not return) and the function did not contain a call to
298   // llvm.amdgcn.init.whole.wave (since in that case there are no inactive lanes
299   // when entering the function).
300   if (isChainFunction() &&
301       (SIRegisterInfo::isChainScratchRegister(VGPR) ||
302        !MF.getFrameInfo().hasTailCall() || hasInitWholeWave()))
303     return;
304 
305   WWMSpills.insert(std::make_pair(
306       VGPR, MF.getFrameInfo().CreateSpillStackObject(Size, Alignment)));
307 }
308 
309 // Separate out the callee-saved and scratch registers.
310 void SIMachineFunctionInfo::splitWWMSpillRegisters(
311     MachineFunction &MF,
312     SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs,
313     SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const {
314   const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
315   for (auto &Reg : WWMSpills) {
316     if (isCalleeSavedReg(CSRegs, Reg.first))
317       CalleeSavedRegs.push_back(Reg);
318     else
319       ScratchRegs.push_back(Reg);
320   }
321 }
322 
323 bool SIMachineFunctionInfo::isCalleeSavedReg(const MCPhysReg *CSRegs,
324                                              MCPhysReg Reg) const {
325   for (unsigned I = 0; CSRegs[I]; ++I) {
326     if (CSRegs[I] == Reg)
327       return true;
328   }
329 
330   return false;
331 }
332 
333 void SIMachineFunctionInfo::shiftWwmVGPRsToLowestRange(
334     MachineFunction &MF, SmallVectorImpl<Register> &WWMVGPRs,
335     BitVector &SavedVGPRs) {
336   const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
337   MachineRegisterInfo &MRI = MF.getRegInfo();
338   for (unsigned I = 0, E = WWMVGPRs.size(); I < E; ++I) {
339     Register Reg = WWMVGPRs[I];
340     Register NewReg =
341         TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
342     if (!NewReg || NewReg >= Reg)
343       break;
344 
345     MRI.replaceRegWith(Reg, NewReg);
346 
347     // Update various tables with the new VGPR.
348     WWMVGPRs[I] = NewReg;
349     WWMReservedRegs.remove(Reg);
350     WWMReservedRegs.insert(NewReg);
351     MRI.reserveReg(NewReg, TRI);
352 
353     // Replace the register in SpillPhysVGPRs. This is needed to look for free
354     // lanes while spilling special SGPRs like FP, BP, etc. during PEI.
355     auto *RegItr = std::find(SpillPhysVGPRs.begin(), SpillPhysVGPRs.end(), Reg);
356     if (RegItr != SpillPhysVGPRs.end()) {
357       unsigned Idx = std::distance(SpillPhysVGPRs.begin(), RegItr);
358       SpillPhysVGPRs[Idx] = NewReg;
359     }
360 
361     // The generic `determineCalleeSaves` might have set the old register if it
362     // is in the CSR range.
363     SavedVGPRs.reset(Reg);
364 
365     for (MachineBasicBlock &MBB : MF) {
366       MBB.removeLiveIn(Reg);
367       MBB.sortUniqueLiveIns();
368     }
369 
370     Reg = NewReg;
371   }
372 }
373 
374 bool SIMachineFunctionInfo::allocateVirtualVGPRForSGPRSpills(
375     MachineFunction &MF, int FI, unsigned LaneIndex) {
376   MachineRegisterInfo &MRI = MF.getRegInfo();
377   Register LaneVGPR;
378   if (!LaneIndex) {
379     LaneVGPR = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
380     SpillVGPRs.push_back(LaneVGPR);
381   } else {
382     LaneVGPR = SpillVGPRs.back();
383   }
384 
385   SGPRSpillsToVirtualVGPRLanes[FI].emplace_back(LaneVGPR, LaneIndex);
386   return true;
387 }
388 
389 bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills(
390     MachineFunction &MF, int FI, unsigned LaneIndex, bool IsPrologEpilog) {
391   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
392   const SIRegisterInfo *TRI = ST.getRegisterInfo();
393   MachineRegisterInfo &MRI = MF.getRegInfo();
394   Register LaneVGPR;
395   if (!LaneIndex) {
396     // Find the highest available register if called before RA to ensure the
397     // lowest registers are available for allocation. The LaneVGPR, in that
398     // case, will be shifted back to the lowest range after VGPR allocation.
399     LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF,
400                                        !IsPrologEpilog);
401     if (LaneVGPR == AMDGPU::NoRegister) {
402       // We have no VGPRs left for spilling SGPRs. Reset because we will not
403       // partially spill the SGPR to VGPRs.
404       SGPRSpillsToPhysicalVGPRLanes.erase(FI);
405       return false;
406     }
407 
408     if (IsPrologEpilog)
409       allocateWWMSpill(MF, LaneVGPR);
410 
411     reserveWWMRegister(LaneVGPR);
412     for (MachineBasicBlock &MBB : MF) {
413       MBB.addLiveIn(LaneVGPR);
414       MBB.sortUniqueLiveIns();
415     }
416     SpillPhysVGPRs.push_back(LaneVGPR);
417   } else {
418     LaneVGPR = SpillPhysVGPRs.back();
419   }
420 
421   SGPRSpillsToPhysicalVGPRLanes[FI].emplace_back(LaneVGPR, LaneIndex);
422   return true;
423 }
424 
425 bool SIMachineFunctionInfo::allocateSGPRSpillToVGPRLane(
426     MachineFunction &MF, int FI, bool SpillToPhysVGPRLane,
427     bool IsPrologEpilog) {
428   std::vector<SIRegisterInfo::SpilledReg> &SpillLanes =
429       SpillToPhysVGPRLane ? SGPRSpillsToPhysicalVGPRLanes[FI]
430                           : SGPRSpillsToVirtualVGPRLanes[FI];
431 
432   // This has already been allocated.
433   if (!SpillLanes.empty())
434     return true;
435 
436   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
437   MachineFrameInfo &FrameInfo = MF.getFrameInfo();
438   unsigned WaveSize = ST.getWavefrontSize();
439 
440   unsigned Size = FrameInfo.getObjectSize(FI);
441   unsigned NumLanes = Size / 4;
442 
443   if (NumLanes > WaveSize)
444     return false;
445 
446   assert(Size >= 4 && "invalid sgpr spill size");
447   assert(ST.getRegisterInfo()->spillSGPRToVGPR() &&
448          "not spilling SGPRs to VGPRs");
449 
450   unsigned &NumSpillLanes = SpillToPhysVGPRLane ? NumPhysicalVGPRSpillLanes
451                                                 : NumVirtualVGPRSpillLanes;
452 
453   for (unsigned I = 0; I < NumLanes; ++I, ++NumSpillLanes) {
454     unsigned LaneIndex = (NumSpillLanes % WaveSize);
455 
456     bool Allocated = SpillToPhysVGPRLane
457                          ? allocatePhysicalVGPRForSGPRSpills(MF, FI, LaneIndex,
458                                                              IsPrologEpilog)
459                          : allocateVirtualVGPRForSGPRSpills(MF, FI, LaneIndex);
460     if (!Allocated) {
461       NumSpillLanes -= I;
462       return false;
463     }
464   }
465 
466   return true;
467 }
468 
469 /// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI.
470 /// Either AGPR is spilled to VGPR to vice versa.
471 /// Returns true if a \p FI can be eliminated completely.
472 bool SIMachineFunctionInfo::allocateVGPRSpillToAGPR(MachineFunction &MF,
473                                                     int FI,
474                                                     bool isAGPRtoVGPR) {
475   MachineRegisterInfo &MRI = MF.getRegInfo();
476   MachineFrameInfo &FrameInfo = MF.getFrameInfo();
477   const GCNSubtarget &ST =  MF.getSubtarget<GCNSubtarget>();
478 
479   assert(ST.hasMAIInsts() && FrameInfo.isSpillSlotObjectIndex(FI));
480 
481   auto &Spill = VGPRToAGPRSpills[FI];
482 
483   // This has already been allocated.
484   if (!Spill.Lanes.empty())
485     return Spill.FullyAllocated;
486 
487   unsigned Size = FrameInfo.getObjectSize(FI);
488   unsigned NumLanes = Size / 4;
489   Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister);
490 
491   const TargetRegisterClass &RC =
492       isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
493   auto Regs = RC.getRegisters();
494 
495   auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
496   const SIRegisterInfo *TRI = ST.getRegisterInfo();
497   Spill.FullyAllocated = true;
498 
499   // FIXME: Move allocation logic out of MachineFunctionInfo and initialize
500   // once.
501   BitVector OtherUsedRegs;
502   OtherUsedRegs.resize(TRI->getNumRegs());
503 
504   const uint32_t *CSRMask =
505       TRI->getCallPreservedMask(MF, MF.getFunction().getCallingConv());
506   if (CSRMask)
507     OtherUsedRegs.setBitsInMask(CSRMask);
508 
509   // TODO: Should include register tuples, but doesn't matter with current
510   // usage.
511   for (MCPhysReg Reg : SpillAGPR)
512     OtherUsedRegs.set(Reg);
513   for (MCPhysReg Reg : SpillVGPR)
514     OtherUsedRegs.set(Reg);
515 
516   SmallVectorImpl<MCPhysReg>::const_iterator NextSpillReg = Regs.begin();
517   for (int I = NumLanes - 1; I >= 0; --I) {
518     NextSpillReg = std::find_if(
519         NextSpillReg, Regs.end(), [&MRI, &OtherUsedRegs](MCPhysReg Reg) {
520           return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) &&
521                  !OtherUsedRegs[Reg];
522         });
523 
524     if (NextSpillReg == Regs.end()) { // Registers exhausted
525       Spill.FullyAllocated = false;
526       break;
527     }
528 
529     OtherUsedRegs.set(*NextSpillReg);
530     SpillRegs.push_back(*NextSpillReg);
531     MRI.reserveReg(*NextSpillReg, TRI);
532     Spill.Lanes[I] = *NextSpillReg++;
533   }
534 
535   return Spill.FullyAllocated;
536 }
537 
538 bool SIMachineFunctionInfo::removeDeadFrameIndices(
539     MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs) {
540   // Remove dead frame indices from function frame, however keep FP & BP since
541   // spills for them haven't been inserted yet. And also make sure to remove the
542   // frame indices from `SGPRSpillsToVirtualVGPRLanes` data structure,
543   // otherwise, it could result in an unexpected side effect and bug, in case of
544   // any re-mapping of freed frame indices by later pass(es) like "stack slot
545   // coloring".
546   for (auto &R : make_early_inc_range(SGPRSpillsToVirtualVGPRLanes)) {
547     MFI.RemoveStackObject(R.first);
548     SGPRSpillsToVirtualVGPRLanes.erase(R.first);
549   }
550 
551   // Remove the dead frame indices of CSR SGPRs which are spilled to physical
552   // VGPR lanes during SILowerSGPRSpills pass.
553   if (!ResetSGPRSpillStackIDs) {
554     for (auto &R : make_early_inc_range(SGPRSpillsToPhysicalVGPRLanes)) {
555       MFI.RemoveStackObject(R.first);
556       SGPRSpillsToPhysicalVGPRLanes.erase(R.first);
557     }
558   }
559   bool HaveSGPRToMemory = false;
560 
561   if (ResetSGPRSpillStackIDs) {
562     // All other SGPRs must be allocated on the default stack, so reset the
563     // stack ID.
564     for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); I != E;
565          ++I) {
566       if (!checkIndexInPrologEpilogSGPRSpills(I)) {
567         if (MFI.getStackID(I) == TargetStackID::SGPRSpill) {
568           MFI.setStackID(I, TargetStackID::Default);
569           HaveSGPRToMemory = true;
570         }
571       }
572     }
573   }
574 
575   for (auto &R : VGPRToAGPRSpills) {
576     if (R.second.IsDead)
577       MFI.RemoveStackObject(R.first);
578   }
579 
580   return HaveSGPRToMemory;
581 }
582 
583 int SIMachineFunctionInfo::getScavengeFI(MachineFrameInfo &MFI,
584                                          const SIRegisterInfo &TRI) {
585   if (ScavengeFI)
586     return *ScavengeFI;
587 
588   ScavengeFI =
589       MFI.CreateStackObject(TRI.getSpillSize(AMDGPU::SGPR_32RegClass),
590                             TRI.getSpillAlign(AMDGPU::SGPR_32RegClass), false);
591   return *ScavengeFI;
592 }
593 
594 MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {
595   assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");
596   return AMDGPU::SGPR0 + NumUserSGPRs;
597 }
598 
599 MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {
600   return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
601 }
602 
603 void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(Register Reg) {
604   VRegFlags.grow(Reg);
605 }
606 
607 void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(Register NewReg,
608                                                          Register SrcReg) {
609   VRegFlags.grow(NewReg);
610   VRegFlags[NewReg] = VRegFlags[SrcReg];
611 }
612 
613 Register
614 SIMachineFunctionInfo::getGITPtrLoReg(const MachineFunction &MF) const {
615   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
616   if (!ST.isAmdPalOS())
617     return Register();
618   Register GitPtrLo = AMDGPU::SGPR0; // Low GIT address passed in
619   if (ST.hasMergedShaders()) {
620     switch (MF.getFunction().getCallingConv()) {
621     case CallingConv::AMDGPU_HS:
622     case CallingConv::AMDGPU_GS:
623       // Low GIT address is passed in s8 rather than s0 for an LS+HS or
624       // ES+GS merged shader on gfx9+.
625       GitPtrLo = AMDGPU::SGPR8;
626       return GitPtrLo;
627     default:
628       return GitPtrLo;
629     }
630   }
631   return GitPtrLo;
632 }
633 
634 static yaml::StringValue regToString(Register Reg,
635                                      const TargetRegisterInfo &TRI) {
636   yaml::StringValue Dest;
637   {
638     raw_string_ostream OS(Dest.Value);
639     OS << printReg(Reg, &TRI);
640   }
641   return Dest;
642 }
643 
644 static std::optional<yaml::SIArgumentInfo>
645 convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo,
646                     const TargetRegisterInfo &TRI) {
647   yaml::SIArgumentInfo AI;
648 
649   auto convertArg = [&](std::optional<yaml::SIArgument> &A,
650                         const ArgDescriptor &Arg) {
651     if (!Arg)
652       return false;
653 
654     // Create a register or stack argument.
655     yaml::SIArgument SA = yaml::SIArgument::createArgument(Arg.isRegister());
656     if (Arg.isRegister()) {
657       raw_string_ostream OS(SA.RegisterName.Value);
658       OS << printReg(Arg.getRegister(), &TRI);
659     } else
660       SA.StackOffset = Arg.getStackOffset();
661     // Check and update the optional mask.
662     if (Arg.isMasked())
663       SA.Mask = Arg.getMask();
664 
665     A = SA;
666     return true;
667   };
668 
669   // TODO: Need to serialize kernarg preloads.
670   bool Any = false;
671   Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
672   Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
673   Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
674   Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
675   Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
676   Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
677   Any |= convertArg(AI.LDSKernelId, ArgInfo.LDSKernelId);
678   Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);
679   Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);
680   Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);
681   Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);
682   Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);
683   Any |= convertArg(AI.PrivateSegmentWaveByteOffset,
684                     ArgInfo.PrivateSegmentWaveByteOffset);
685   Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);
686   Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);
687   Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);
688   Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);
689   Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);
690 
691   if (Any)
692     return AI;
693 
694   return std::nullopt;
695 }
696 
697 yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
698     const llvm::SIMachineFunctionInfo &MFI, const TargetRegisterInfo &TRI,
699     const llvm::MachineFunction &MF)
700     : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
701       MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()),
702       GDSSize(MFI.getGDSSize()), DynLDSAlign(MFI.getDynLDSAlign()),
703       IsEntryFunction(MFI.isEntryFunction()),
704       NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
705       MemoryBound(MFI.isMemoryBound()), WaveLimiter(MFI.needsWaveLimiter()),
706       HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
707       HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
708       HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
709       Occupancy(MFI.getOccupancy()),
710       ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)),
711       FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)),
712       StackPtrOffsetReg(regToString(MFI.getStackPtrOffsetReg(), TRI)),
713       BytesInStackArgArea(MFI.getBytesInStackArgArea()),
714       ReturnsVoid(MFI.returnsVoid()),
715       ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)),
716       PSInputAddr(MFI.getPSInputAddr()), PSInputEnable(MFI.getPSInputEnable()),
717       MaxMemoryClusterDWords(MFI.getMaxMemoryClusterDWords()),
718       Mode(MFI.getMode()), HasInitWholeWave(MFI.hasInitWholeWave()) {
719   for (Register Reg : MFI.getSGPRSpillPhysVGPRs())
720     SpillPhysVGPRS.push_back(regToString(Reg, TRI));
721 
722   for (Register Reg : MFI.getWWMReservedRegs())
723     WWMReservedRegs.push_back(regToString(Reg, TRI));
724 
725   if (MFI.getLongBranchReservedReg())
726     LongBranchReservedReg = regToString(MFI.getLongBranchReservedReg(), TRI);
727   if (MFI.getVGPRForAGPRCopy())
728     VGPRForAGPRCopy = regToString(MFI.getVGPRForAGPRCopy(), TRI);
729 
730   if (MFI.getSGPRForEXECCopy())
731     SGPRForEXECCopy = regToString(MFI.getSGPRForEXECCopy(), TRI);
732 
733   auto SFI = MFI.getOptionalScavengeFI();
734   if (SFI)
735     ScavengeFI = yaml::FrameIndex(*SFI, MF.getFrameInfo());
736 }
737 
738 void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
739   MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, *this);
740 }
741 
742 bool SIMachineFunctionInfo::initializeBaseYamlFields(
743     const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF,
744     PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) {
745   ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize;
746   MaxKernArgAlign = YamlMFI.MaxKernArgAlign;
747   LDSSize = YamlMFI.LDSSize;
748   GDSSize = YamlMFI.GDSSize;
749   DynLDSAlign = YamlMFI.DynLDSAlign;
750   PSInputAddr = YamlMFI.PSInputAddr;
751   PSInputEnable = YamlMFI.PSInputEnable;
752   MaxMemoryClusterDWords = YamlMFI.MaxMemoryClusterDWords;
753   HighBitsOf32BitAddress = YamlMFI.HighBitsOf32BitAddress;
754   Occupancy = YamlMFI.Occupancy;
755   IsEntryFunction = YamlMFI.IsEntryFunction;
756   NoSignedZerosFPMath = YamlMFI.NoSignedZerosFPMath;
757   MemoryBound = YamlMFI.MemoryBound;
758   WaveLimiter = YamlMFI.WaveLimiter;
759   HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs;
760   HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs;
761   BytesInStackArgArea = YamlMFI.BytesInStackArgArea;
762   ReturnsVoid = YamlMFI.ReturnsVoid;
763 
764   if (YamlMFI.ScavengeFI) {
765     auto FIOrErr = YamlMFI.ScavengeFI->getFI(MF.getFrameInfo());
766     if (!FIOrErr) {
767       // Create a diagnostic for a the frame index.
768       const MemoryBuffer &Buffer =
769           *PFS.SM->getMemoryBuffer(PFS.SM->getMainFileID());
770 
771       Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1, 1,
772                            SourceMgr::DK_Error, toString(FIOrErr.takeError()),
773                            "", {}, {});
774       SourceRange = YamlMFI.ScavengeFI->SourceRange;
775       return true;
776     }
777     ScavengeFI = *FIOrErr;
778   } else {
779     ScavengeFI = std::nullopt;
780   }
781   return false;
782 }
783 
784 bool SIMachineFunctionInfo::mayUseAGPRs(const Function &F) const {
785   return !F.hasFnAttribute("amdgpu-no-agpr");
786 }
787 
788 bool SIMachineFunctionInfo::usesAGPRs(const MachineFunction &MF) const {
789   if (UsesAGPRs)
790     return *UsesAGPRs;
791 
792   if (!mayNeedAGPRs()) {
793     UsesAGPRs = false;
794     return false;
795   }
796 
797   if (!AMDGPU::isEntryFunctionCC(MF.getFunction().getCallingConv()) ||
798       MF.getFrameInfo().hasCalls()) {
799     UsesAGPRs = true;
800     return true;
801   }
802 
803   const MachineRegisterInfo &MRI = MF.getRegInfo();
804 
805   for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
806     const Register Reg = Register::index2VirtReg(I);
807     const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);
808     if (RC && SIRegisterInfo::isAGPRClass(RC)) {
809       UsesAGPRs = true;
810       return true;
811     }
812     if (!RC && !MRI.use_empty(Reg) && MRI.getType(Reg).isValid()) {
813       // Defer caching UsesAGPRs, function might not yet been regbank selected.
814       return true;
815     }
816   }
817 
818   for (MCRegister Reg : AMDGPU::AGPR_32RegClass) {
819     if (MRI.isPhysRegUsed(Reg)) {
820       UsesAGPRs = true;
821       return true;
822     }
823   }
824 
825   UsesAGPRs = false;
826   return false;
827 }
828