xref: /llvm-project/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h (revision b25b4c0ab4ad1acc1490c7560970a2e80cf94b3e) 
1 //==- SIMachineFunctionInfo.h - SIMachineFunctionInfo interface --*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #ifndef LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
14 #define LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
15 
16 #include "AMDGPUArgumentUsageInfo.h"
17 #include "AMDGPUMachineFunction.h"
18 #include "AMDGPUTargetMachine.h"
19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
20 #include "SIInstrInfo.h"
21 #include "llvm/ADT/SetVector.h"
22 #include "llvm/CodeGen/MIRYamlMapping.h"
23 #include "llvm/CodeGen/PseudoSourceValue.h"
24 #include "llvm/Support/raw_ostream.h"
25 #include <optional>
26 
27 namespace llvm {
28 
29 class MachineFrameInfo;
30 class MachineFunction;
31 class SIMachineFunctionInfo;
32 class SIRegisterInfo;
33 class TargetRegisterClass;
34 
35 class AMDGPUPseudoSourceValue : public PseudoSourceValue {
36 public:
37   enum AMDGPUPSVKind : unsigned {
38     PSVImage = PseudoSourceValue::TargetCustom,
39     GWSResource
40   };
41 
42 protected:
43   AMDGPUPseudoSourceValue(unsigned Kind, const AMDGPUTargetMachine &TM)
44       : PseudoSourceValue(Kind, TM) {}
45 
46 public:
47   bool isConstant(const MachineFrameInfo *) const override {
48     // This should probably be true for most images, but we will start by being
49     // conservative.
50     return false;
51   }
52 
53   bool isAliased(const MachineFrameInfo *) const override {
54     return true;
55   }
56 
57   bool mayAlias(const MachineFrameInfo *) const override {
58     return true;
59   }
60 };
61 
62 class AMDGPUGWSResourcePseudoSourceValue final : public AMDGPUPseudoSourceValue {
63 public:
64   explicit AMDGPUGWSResourcePseudoSourceValue(const AMDGPUTargetMachine &TM)
65       : AMDGPUPseudoSourceValue(GWSResource, TM) {}
66 
67   static bool classof(const PseudoSourceValue *V) {
68     return V->kind() == GWSResource;
69   }
70 
71   // These are inaccessible memory from IR.
72   bool isAliased(const MachineFrameInfo *) const override {
73     return false;
74   }
75 
76   // These are inaccessible memory from IR.
77   bool mayAlias(const MachineFrameInfo *) const override {
78     return false;
79   }
80 
81   void printCustom(raw_ostream &OS) const override {
82     OS << "GWSResource";
83   }
84 };
85 
86 namespace yaml {
87 
88 struct SIArgument {
89   bool IsRegister;
90   union {
91     StringValue RegisterName;
92     unsigned StackOffset;
93   };
94   std::optional<unsigned> Mask;
95 
96   // Default constructor, which creates a stack argument.
97   SIArgument() : IsRegister(false), StackOffset(0) {}
98   SIArgument(const SIArgument &Other) {
99     IsRegister = Other.IsRegister;
100     if (IsRegister) {
101       ::new ((void *)std::addressof(RegisterName))
102           StringValue(Other.RegisterName);
103     } else
104       StackOffset = Other.StackOffset;
105     Mask = Other.Mask;
106   }
107   SIArgument &operator=(const SIArgument &Other) {
108     IsRegister = Other.IsRegister;
109     if (IsRegister) {
110       ::new ((void *)std::addressof(RegisterName))
111           StringValue(Other.RegisterName);
112     } else
113       StackOffset = Other.StackOffset;
114     Mask = Other.Mask;
115     return *this;
116   }
117   ~SIArgument() {
118     if (IsRegister)
119       RegisterName.~StringValue();
120   }
121 
122   // Helper to create a register or stack argument.
123   static inline SIArgument createArgument(bool IsReg) {
124     if (IsReg)
125       return SIArgument(IsReg);
126     return SIArgument();
127   }
128 
129 private:
130   // Construct a register argument.
131   SIArgument(bool) : IsRegister(true), RegisterName() {}
132 };
133 
134 template <> struct MappingTraits<SIArgument> {
135   static void mapping(IO &YamlIO, SIArgument &A) {
136     if (YamlIO.outputting()) {
137       if (A.IsRegister)
138         YamlIO.mapRequired("reg", A.RegisterName);
139       else
140         YamlIO.mapRequired("offset", A.StackOffset);
141     } else {
142       auto Keys = YamlIO.keys();
143       if (is_contained(Keys, "reg")) {
144         A = SIArgument::createArgument(true);
145         YamlIO.mapRequired("reg", A.RegisterName);
146       } else if (is_contained(Keys, "offset"))
147         YamlIO.mapRequired("offset", A.StackOffset);
148       else
149         YamlIO.setError("missing required key 'reg' or 'offset'");
150     }
151     YamlIO.mapOptional("mask", A.Mask);
152   }
153   static const bool flow = true;
154 };
155 
156 struct SIArgumentInfo {
157   std::optional<SIArgument> PrivateSegmentBuffer;
158   std::optional<SIArgument> DispatchPtr;
159   std::optional<SIArgument> QueuePtr;
160   std::optional<SIArgument> KernargSegmentPtr;
161   std::optional<SIArgument> DispatchID;
162   std::optional<SIArgument> FlatScratchInit;
163   std::optional<SIArgument> PrivateSegmentSize;
164 
165   std::optional<SIArgument> WorkGroupIDX;
166   std::optional<SIArgument> WorkGroupIDY;
167   std::optional<SIArgument> WorkGroupIDZ;
168   std::optional<SIArgument> WorkGroupInfo;
169   std::optional<SIArgument> LDSKernelId;
170   std::optional<SIArgument> PrivateSegmentWaveByteOffset;
171 
172   std::optional<SIArgument> ImplicitArgPtr;
173   std::optional<SIArgument> ImplicitBufferPtr;
174 
175   std::optional<SIArgument> WorkItemIDX;
176   std::optional<SIArgument> WorkItemIDY;
177   std::optional<SIArgument> WorkItemIDZ;
178 };
179 
180 template <> struct MappingTraits<SIArgumentInfo> {
181   static void mapping(IO &YamlIO, SIArgumentInfo &AI) {
182     YamlIO.mapOptional("privateSegmentBuffer", AI.PrivateSegmentBuffer);
183     YamlIO.mapOptional("dispatchPtr", AI.DispatchPtr);
184     YamlIO.mapOptional("queuePtr", AI.QueuePtr);
185     YamlIO.mapOptional("kernargSegmentPtr", AI.KernargSegmentPtr);
186     YamlIO.mapOptional("dispatchID", AI.DispatchID);
187     YamlIO.mapOptional("flatScratchInit", AI.FlatScratchInit);
188     YamlIO.mapOptional("privateSegmentSize", AI.PrivateSegmentSize);
189 
190     YamlIO.mapOptional("workGroupIDX", AI.WorkGroupIDX);
191     YamlIO.mapOptional("workGroupIDY", AI.WorkGroupIDY);
192     YamlIO.mapOptional("workGroupIDZ", AI.WorkGroupIDZ);
193     YamlIO.mapOptional("workGroupInfo", AI.WorkGroupInfo);
194     YamlIO.mapOptional("LDSKernelId", AI.LDSKernelId);
195     YamlIO.mapOptional("privateSegmentWaveByteOffset",
196                        AI.PrivateSegmentWaveByteOffset);
197 
198     YamlIO.mapOptional("implicitArgPtr", AI.ImplicitArgPtr);
199     YamlIO.mapOptional("implicitBufferPtr", AI.ImplicitBufferPtr);
200 
201     YamlIO.mapOptional("workItemIDX", AI.WorkItemIDX);
202     YamlIO.mapOptional("workItemIDY", AI.WorkItemIDY);
203     YamlIO.mapOptional("workItemIDZ", AI.WorkItemIDZ);
204   }
205 };
206 
207 // Default to default mode for default calling convention.
208 struct SIMode {
209   bool IEEE = true;
210   bool DX10Clamp = true;
211   bool FP32InputDenormals = true;
212   bool FP32OutputDenormals = true;
213   bool FP64FP16InputDenormals = true;
214   bool FP64FP16OutputDenormals = true;
215 
216   SIMode() = default;
217 
218   SIMode(const AMDGPU::SIModeRegisterDefaults &Mode) {
219     IEEE = Mode.IEEE;
220     DX10Clamp = Mode.DX10Clamp;
221     FP32InputDenormals = Mode.FP32InputDenormals;
222     FP32OutputDenormals = Mode.FP32OutputDenormals;
223     FP64FP16InputDenormals = Mode.FP64FP16InputDenormals;
224     FP64FP16OutputDenormals = Mode.FP64FP16OutputDenormals;
225   }
226 
227   bool operator ==(const SIMode Other) const {
228     return IEEE == Other.IEEE &&
229            DX10Clamp == Other.DX10Clamp &&
230            FP32InputDenormals == Other.FP32InputDenormals &&
231            FP32OutputDenormals == Other.FP32OutputDenormals &&
232            FP64FP16InputDenormals == Other.FP64FP16InputDenormals &&
233            FP64FP16OutputDenormals == Other.FP64FP16OutputDenormals;
234   }
235 };
236 
237 template <> struct MappingTraits<SIMode> {
238   static void mapping(IO &YamlIO, SIMode &Mode) {
239     YamlIO.mapOptional("ieee", Mode.IEEE, true);
240     YamlIO.mapOptional("dx10-clamp", Mode.DX10Clamp, true);
241     YamlIO.mapOptional("fp32-input-denormals", Mode.FP32InputDenormals, true);
242     YamlIO.mapOptional("fp32-output-denormals", Mode.FP32OutputDenormals, true);
243     YamlIO.mapOptional("fp64-fp16-input-denormals", Mode.FP64FP16InputDenormals, true);
244     YamlIO.mapOptional("fp64-fp16-output-denormals", Mode.FP64FP16OutputDenormals, true);
245   }
246 };
247 
248 struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo {
249   uint64_t ExplicitKernArgSize = 0;
250   Align MaxKernArgAlign;
251   uint32_t LDSSize = 0;
252   uint32_t GDSSize = 0;
253   Align DynLDSAlign;
254   bool IsEntryFunction = false;
255   bool NoSignedZerosFPMath = false;
256   bool MemoryBound = false;
257   bool WaveLimiter = false;
258   bool HasSpilledSGPRs = false;
259   bool HasSpilledVGPRs = false;
260   uint32_t HighBitsOf32BitAddress = 0;
261 
262   // TODO: 10 may be a better default since it's the maximum.
263   unsigned Occupancy = 0;
264 
265   SmallVector<StringValue> WWMReservedRegs;
266 
267   StringValue ScratchRSrcReg = "$private_rsrc_reg";
268   StringValue FrameOffsetReg = "$fp_reg";
269   StringValue StackPtrOffsetReg = "$sp_reg";
270 
271   unsigned BytesInStackArgArea = 0;
272   bool ReturnsVoid = true;
273 
274   std::optional<SIArgumentInfo> ArgInfo;
275   SIMode Mode;
276   std::optional<FrameIndex> ScavengeFI;
277   StringValue VGPRForAGPRCopy;
278 
279   SIMachineFunctionInfo() = default;
280   SIMachineFunctionInfo(const llvm::SIMachineFunctionInfo &,
281                         const TargetRegisterInfo &TRI,
282                         const llvm::MachineFunction &MF);
283 
284   void mappingImpl(yaml::IO &YamlIO) override;
285   ~SIMachineFunctionInfo() = default;
286 };
287 
288 template <> struct MappingTraits<SIMachineFunctionInfo> {
289   static void mapping(IO &YamlIO, SIMachineFunctionInfo &MFI) {
290     YamlIO.mapOptional("explicitKernArgSize", MFI.ExplicitKernArgSize,
291                        UINT64_C(0));
292     YamlIO.mapOptional("maxKernArgAlign", MFI.MaxKernArgAlign);
293     YamlIO.mapOptional("ldsSize", MFI.LDSSize, 0u);
294     YamlIO.mapOptional("gdsSize", MFI.GDSSize, 0u);
295     YamlIO.mapOptional("dynLDSAlign", MFI.DynLDSAlign, Align());
296     YamlIO.mapOptional("isEntryFunction", MFI.IsEntryFunction, false);
297     YamlIO.mapOptional("noSignedZerosFPMath", MFI.NoSignedZerosFPMath, false);
298     YamlIO.mapOptional("memoryBound", MFI.MemoryBound, false);
299     YamlIO.mapOptional("waveLimiter", MFI.WaveLimiter, false);
300     YamlIO.mapOptional("hasSpilledSGPRs", MFI.HasSpilledSGPRs, false);
301     YamlIO.mapOptional("hasSpilledVGPRs", MFI.HasSpilledVGPRs, false);
302     YamlIO.mapOptional("scratchRSrcReg", MFI.ScratchRSrcReg,
303                        StringValue("$private_rsrc_reg"));
304     YamlIO.mapOptional("frameOffsetReg", MFI.FrameOffsetReg,
305                        StringValue("$fp_reg"));
306     YamlIO.mapOptional("stackPtrOffsetReg", MFI.StackPtrOffsetReg,
307                        StringValue("$sp_reg"));
308     YamlIO.mapOptional("bytesInStackArgArea", MFI.BytesInStackArgArea, 0u);
309     YamlIO.mapOptional("returnsVoid", MFI.ReturnsVoid, true);
310     YamlIO.mapOptional("argumentInfo", MFI.ArgInfo);
311     YamlIO.mapOptional("mode", MFI.Mode, SIMode());
312     YamlIO.mapOptional("highBitsOf32BitAddress",
313                        MFI.HighBitsOf32BitAddress, 0u);
314     YamlIO.mapOptional("occupancy", MFI.Occupancy, 0);
315     YamlIO.mapOptional("wwmReservedRegs", MFI.WWMReservedRegs);
316     YamlIO.mapOptional("scavengeFI", MFI.ScavengeFI);
317     YamlIO.mapOptional("vgprForAGPRCopy", MFI.VGPRForAGPRCopy,
318                        StringValue()); // Don't print out when it's empty.
319   }
320 };
321 
322 } // end namespace yaml
323 
324 /// This class keeps track of the SPI_SP_INPUT_ADDR config register, which
325 /// tells the hardware which interpolation parameters to load.
326 class SIMachineFunctionInfo final : public AMDGPUMachineFunction {
327   friend class GCNTargetMachine;
328 
329   // State of MODE register, assumed FP mode.
330   AMDGPU::SIModeRegisterDefaults Mode;
331 
332   // Registers that may be reserved for spilling purposes. These may be the same
333   // as the input registers.
334   Register ScratchRSrcReg = AMDGPU::PRIVATE_RSRC_REG;
335 
336   // This is the unswizzled offset from the current dispatch's scratch wave
337   // base to the beginning of the current function's frame.
338   Register FrameOffsetReg = AMDGPU::FP_REG;
339 
340   // This is an ABI register used in the non-entry calling convention to
341   // communicate the unswizzled offset from the current dispatch's scratch wave
342   // base to the beginning of the new function's frame.
343   Register StackPtrOffsetReg = AMDGPU::SP_REG;
344 
345   AMDGPUFunctionArgInfo ArgInfo;
346 
347   // Graphics info.
348   unsigned PSInputAddr = 0;
349   unsigned PSInputEnable = 0;
350 
351   /// Number of bytes of arguments this function has on the stack. If the callee
352   /// is expected to restore the argument stack this should be a multiple of 16,
353   /// all usable during a tail call.
354   ///
355   /// The alternative would forbid tail call optimisation in some cases: if we
356   /// want to transfer control from a function with 8-bytes of stack-argument
357   /// space to a function with 16-bytes then misalignment of this value would
358   /// make a stack adjustment necessary, which could not be undone by the
359   /// callee.
360   unsigned BytesInStackArgArea = 0;
361 
362   bool ReturnsVoid = true;
363 
364   // A pair of default/requested minimum/maximum flat work group sizes.
365   // Minimum - first, maximum - second.
366   std::pair<unsigned, unsigned> FlatWorkGroupSizes = {0, 0};
367 
368   // A pair of default/requested minimum/maximum number of waves per execution
369   // unit. Minimum - first, maximum - second.
370   std::pair<unsigned, unsigned> WavesPerEU = {0, 0};
371 
372   const AMDGPUGWSResourcePseudoSourceValue GWSResourcePSV;
373 
374 private:
375   unsigned NumUserSGPRs = 0;
376   unsigned NumSystemSGPRs = 0;
377 
378   bool HasSpilledSGPRs = false;
379   bool HasSpilledVGPRs = false;
380   bool HasNonSpillStackObjects = false;
381   bool IsStackRealigned = false;
382 
383   unsigned NumSpilledSGPRs = 0;
384   unsigned NumSpilledVGPRs = 0;
385 
386   // Feature bits required for inputs passed in user SGPRs.
387   bool PrivateSegmentBuffer : 1;
388   bool DispatchPtr : 1;
389   bool QueuePtr : 1;
390   bool KernargSegmentPtr : 1;
391   bool DispatchID : 1;
392   bool FlatScratchInit : 1;
393 
394   // Feature bits required for inputs passed in system SGPRs.
395   bool WorkGroupIDX : 1; // Always initialized.
396   bool WorkGroupIDY : 1;
397   bool WorkGroupIDZ : 1;
398   bool WorkGroupInfo : 1;
399   bool LDSKernelId : 1;
400   bool PrivateSegmentWaveByteOffset : 1;
401 
402   bool WorkItemIDX : 1; // Always initialized.
403   bool WorkItemIDY : 1;
404   bool WorkItemIDZ : 1;
405 
406   // Private memory buffer
407   // Compute directly in sgpr[0:1]
408   // Other shaders indirect 64-bits at sgpr[0:1]
409   bool ImplicitBufferPtr : 1;
410 
411   // Pointer to where the ABI inserts special kernel arguments separate from the
412   // user arguments. This is an offset from the KernargSegmentPtr.
413   bool ImplicitArgPtr : 1;
414 
415   bool MayNeedAGPRs : 1;
416 
417   // The hard-wired high half of the address of the global information table
418   // for AMDPAL OS type. 0xffffffff represents no hard-wired high half, since
419   // current hardware only allows a 16 bit value.
420   unsigned GITPtrHigh;
421 
422   unsigned HighBitsOf32BitAddress;
423 
424   // Current recorded maximum possible occupancy.
425   unsigned Occupancy;
426 
427   mutable std::optional<bool> UsesAGPRs;
428 
429   MCPhysReg getNextUserSGPR() const;
430 
431   MCPhysReg getNextSystemSGPR() const;
432 
433 public:
434   struct VGPRSpillToAGPR {
435     SmallVector<MCPhysReg, 32> Lanes;
436     bool FullyAllocated = false;
437     bool IsDead = false;
438   };
439 
440 private:
441   // To track VGPR + lane index for each subregister of the SGPR spilled to
442   // frameindex key during SILowerSGPRSpills pass.
443   DenseMap<int, std::vector<SIRegisterInfo::SpilledReg>> SGPRSpillToVGPRLanes;
444   // To track VGPR + lane index for spilling special SGPRs like Frame Pointer
445   // identified during PrologEpilogInserter.
446   DenseMap<int, std::vector<SIRegisterInfo::SpilledReg>>
447       PrologEpilogSGPRSpillToVGPRLanes;
448   unsigned NumVGPRSpillLanes = 0;
449   unsigned NumVGPRPrologEpilogSpillLanes = 0;
450   SmallVector<Register, 2> SpillVGPRs;
451   using WWMSpillsMap = MapVector<Register, int>;
452   // To track the registers used in instructions that can potentially modify the
453   // inactive lanes. The WWM instructions and the writelane instructions for
454   // spilling SGPRs to VGPRs fall under such category of operations. The VGPRs
455   // modified by them should be spilled/restored at function prolog/epilog to
456   // avoid any undesired outcome. Each entry in this map holds a pair of values,
457   // the VGPR and its stack slot index.
458   WWMSpillsMap WWMSpills;
459 
460   using ReservedRegSet = SmallSetVector<Register, 8>;
461   // To track the VGPRs reserved for WWM instructions. They get stack slots
462   // later during PrologEpilogInserter and get added into the superset WWMSpills
463   // for actual spilling. A separate set makes the register reserved part and
464   // the serialization easier.
465   ReservedRegSet WWMReservedRegs;
466 
467   DenseMap<int, VGPRSpillToAGPR> VGPRToAGPRSpills;
468 
469   // AGPRs used for VGPR spills.
470   SmallVector<MCPhysReg, 32> SpillAGPR;
471 
472   // VGPRs used for AGPR spills.
473   SmallVector<MCPhysReg, 32> SpillVGPR;
474 
475   // Emergency stack slot. Sometimes, we create this before finalizing the stack
476   // frame, so save it here and add it to the RegScavenger later.
477   std::optional<int> ScavengeFI;
478 
479 private:
480   Register VGPRForAGPRCopy;
481 
482   bool allocateVGPRForSGPRSpills(MachineFunction &MF, int FI,
483                                  unsigned LaneIndex);
484   bool allocateVGPRForPrologEpilogSGPRSpills(MachineFunction &MF, int FI,
485                                              unsigned LaneIndex);
486 
487 public:
488   Register getVGPRForAGPRCopy() const {
489     return VGPRForAGPRCopy;
490   }
491 
492   void setVGPRForAGPRCopy(Register NewVGPRForAGPRCopy) {
493     VGPRForAGPRCopy = NewVGPRForAGPRCopy;
494   }
495 
496 public: // FIXME
497   /// If this is set, an SGPR used for save/restore of the register used for the
498   /// frame pointer.
499   Register SGPRForFPSaveRestoreCopy;
500   std::optional<int> FramePointerSaveIndex;
501 
502   /// If this is set, an SGPR used for save/restore of the register used for the
503   /// base pointer.
504   Register SGPRForBPSaveRestoreCopy;
505   std::optional<int> BasePointerSaveIndex;
506 
507   bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg);
508 
509 public:
510   SIMachineFunctionInfo(const MachineFunction &MF);
511   SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI) = default;
512 
513   MachineFunctionInfo *
514   clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF,
515         const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
516       const override;
517 
518   bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI,
519                                 const MachineFunction &MF,
520                                 PerFunctionMIParsingState &PFS,
521                                 SMDiagnostic &Error, SMRange &SourceRange);
522 
523   void reserveWWMRegister(Register Reg) { WWMReservedRegs.insert(Reg); }
524 
525   AMDGPU::SIModeRegisterDefaults getMode() const {
526     return Mode;
527   }
528 
529   ArrayRef<SIRegisterInfo::SpilledReg>
530   getSGPRSpillToVGPRLanes(int FrameIndex) const {
531     auto I = SGPRSpillToVGPRLanes.find(FrameIndex);
532     return (I == SGPRSpillToVGPRLanes.end())
533                ? ArrayRef<SIRegisterInfo::SpilledReg>()
534                : makeArrayRef(I->second);
535   }
536 
537   ArrayRef<Register> getSGPRSpillVGPRs() const { return SpillVGPRs; }
538   const WWMSpillsMap &getWWMSpills() const { return WWMSpills; }
539   const ReservedRegSet &getWWMReservedRegs() const { return WWMReservedRegs; }
540 
541   ArrayRef<SIRegisterInfo::SpilledReg>
542   getPrologEpilogSGPRSpillToVGPRLanes(int FrameIndex) const {
543     auto I = PrologEpilogSGPRSpillToVGPRLanes.find(FrameIndex);
544     return (I == PrologEpilogSGPRSpillToVGPRLanes.end())
545                ? ArrayRef<SIRegisterInfo::SpilledReg>()
546                : makeArrayRef(I->second);
547   }
548 
549   void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size = 4,
550                         Align Alignment = Align(4));
551 
552   ArrayRef<MCPhysReg> getAGPRSpillVGPRs() const {
553     return SpillAGPR;
554   }
555 
556   ArrayRef<MCPhysReg> getVGPRSpillAGPRs() const {
557     return SpillVGPR;
558   }
559 
560   MCPhysReg getVGPRToAGPRSpill(int FrameIndex, unsigned Lane) const {
561     auto I = VGPRToAGPRSpills.find(FrameIndex);
562     return (I == VGPRToAGPRSpills.end()) ? (MCPhysReg)AMDGPU::NoRegister
563                                          : I->second.Lanes[Lane];
564   }
565 
566   void setVGPRToAGPRSpillDead(int FrameIndex) {
567     auto I = VGPRToAGPRSpills.find(FrameIndex);
568     if (I != VGPRToAGPRSpills.end())
569       I->second.IsDead = true;
570   }
571 
572   bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI,
573                                    bool IsPrologEpilog = false);
574   bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR);
575 
576   /// If \p ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill
577   /// to the default stack.
578   bool removeDeadFrameIndices(MachineFrameInfo &MFI,
579                               bool ResetSGPRSpillStackIDs);
580 
581   int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI);
582   std::optional<int> getOptionalScavengeFI() const { return ScavengeFI; }
583 
584   unsigned getBytesInStackArgArea() const {
585     return BytesInStackArgArea;
586   }
587 
588   void setBytesInStackArgArea(unsigned Bytes) {
589     BytesInStackArgArea = Bytes;
590   }
591 
592   // Add user SGPRs.
593   Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI);
594   Register addDispatchPtr(const SIRegisterInfo &TRI);
595   Register addQueuePtr(const SIRegisterInfo &TRI);
596   Register addKernargSegmentPtr(const SIRegisterInfo &TRI);
597   Register addDispatchID(const SIRegisterInfo &TRI);
598   Register addFlatScratchInit(const SIRegisterInfo &TRI);
599   Register addImplicitBufferPtr(const SIRegisterInfo &TRI);
600   Register addLDSKernelId();
601 
602   /// Increment user SGPRs used for padding the argument list only.
603   Register addReservedUserSGPR() {
604     Register Next = getNextUserSGPR();
605     ++NumUserSGPRs;
606     return Next;
607   }
608 
609   // Add system SGPRs.
610   Register addWorkGroupIDX() {
611     ArgInfo.WorkGroupIDX = ArgDescriptor::createRegister(getNextSystemSGPR());
612     NumSystemSGPRs += 1;
613     return ArgInfo.WorkGroupIDX.getRegister();
614   }
615 
616   Register addWorkGroupIDY() {
617     ArgInfo.WorkGroupIDY = ArgDescriptor::createRegister(getNextSystemSGPR());
618     NumSystemSGPRs += 1;
619     return ArgInfo.WorkGroupIDY.getRegister();
620   }
621 
622   Register addWorkGroupIDZ() {
623     ArgInfo.WorkGroupIDZ = ArgDescriptor::createRegister(getNextSystemSGPR());
624     NumSystemSGPRs += 1;
625     return ArgInfo.WorkGroupIDZ.getRegister();
626   }
627 
628   Register addWorkGroupInfo() {
629     ArgInfo.WorkGroupInfo = ArgDescriptor::createRegister(getNextSystemSGPR());
630     NumSystemSGPRs += 1;
631     return ArgInfo.WorkGroupInfo.getRegister();
632   }
633 
634   // Add special VGPR inputs
635   void setWorkItemIDX(ArgDescriptor Arg) {
636     ArgInfo.WorkItemIDX = Arg;
637   }
638 
639   void setWorkItemIDY(ArgDescriptor Arg) {
640     ArgInfo.WorkItemIDY = Arg;
641   }
642 
643   void setWorkItemIDZ(ArgDescriptor Arg) {
644     ArgInfo.WorkItemIDZ = Arg;
645   }
646 
647   Register addPrivateSegmentWaveByteOffset() {
648     ArgInfo.PrivateSegmentWaveByteOffset
649       = ArgDescriptor::createRegister(getNextSystemSGPR());
650     NumSystemSGPRs += 1;
651     return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
652   }
653 
654   void setPrivateSegmentWaveByteOffset(Register Reg) {
655     ArgInfo.PrivateSegmentWaveByteOffset = ArgDescriptor::createRegister(Reg);
656   }
657 
658   bool hasPrivateSegmentBuffer() const {
659     return PrivateSegmentBuffer;
660   }
661 
662   bool hasDispatchPtr() const {
663     return DispatchPtr;
664   }
665 
666   bool hasQueuePtr() const {
667     return QueuePtr;
668   }
669 
670   bool hasKernargSegmentPtr() const {
671     return KernargSegmentPtr;
672   }
673 
674   bool hasDispatchID() const {
675     return DispatchID;
676   }
677 
678   bool hasFlatScratchInit() const {
679     return FlatScratchInit;
680   }
681 
682   bool hasWorkGroupIDX() const {
683     return WorkGroupIDX;
684   }
685 
686   bool hasWorkGroupIDY() const {
687     return WorkGroupIDY;
688   }
689 
690   bool hasWorkGroupIDZ() const {
691     return WorkGroupIDZ;
692   }
693 
694   bool hasWorkGroupInfo() const {
695     return WorkGroupInfo;
696   }
697 
698   bool hasLDSKernelId() const { return LDSKernelId; }
699 
700   bool hasPrivateSegmentWaveByteOffset() const {
701     return PrivateSegmentWaveByteOffset;
702   }
703 
704   bool hasWorkItemIDX() const {
705     return WorkItemIDX;
706   }
707 
708   bool hasWorkItemIDY() const {
709     return WorkItemIDY;
710   }
711 
712   bool hasWorkItemIDZ() const {
713     return WorkItemIDZ;
714   }
715 
716   bool hasImplicitArgPtr() const {
717     return ImplicitArgPtr;
718   }
719 
720   bool hasImplicitBufferPtr() const {
721     return ImplicitBufferPtr;
722   }
723 
724   AMDGPUFunctionArgInfo &getArgInfo() {
725     return ArgInfo;
726   }
727 
728   const AMDGPUFunctionArgInfo &getArgInfo() const {
729     return ArgInfo;
730   }
731 
732   std::tuple<const ArgDescriptor *, const TargetRegisterClass *, LLT>
733   getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
734     return ArgInfo.getPreloadedValue(Value);
735   }
736 
737   MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
738     auto Arg = std::get<0>(ArgInfo.getPreloadedValue(Value));
739     return Arg ? Arg->getRegister() : MCRegister();
740   }
741 
742   unsigned getGITPtrHigh() const {
743     return GITPtrHigh;
744   }
745 
746   Register getGITPtrLoReg(const MachineFunction &MF) const;
747 
748   uint32_t get32BitAddressHighBits() const {
749     return HighBitsOf32BitAddress;
750   }
751 
752   unsigned getNumUserSGPRs() const {
753     return NumUserSGPRs;
754   }
755 
756   unsigned getNumPreloadedSGPRs() const {
757     return NumUserSGPRs + NumSystemSGPRs;
758   }
759 
760   Register getPrivateSegmentWaveByteOffsetSystemSGPR() const {
761     return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
762   }
763 
764   /// Returns the physical register reserved for use as the resource
765   /// descriptor for scratch accesses.
766   Register getScratchRSrcReg() const {
767     return ScratchRSrcReg;
768   }
769 
770   void setScratchRSrcReg(Register Reg) {
771     assert(Reg != 0 && "Should never be unset");
772     ScratchRSrcReg = Reg;
773   }
774 
775   Register getFrameOffsetReg() const {
776     return FrameOffsetReg;
777   }
778 
779   void setFrameOffsetReg(Register Reg) {
780     assert(Reg != 0 && "Should never be unset");
781     FrameOffsetReg = Reg;
782   }
783 
784   void setStackPtrOffsetReg(Register Reg) {
785     assert(Reg != 0 && "Should never be unset");
786     StackPtrOffsetReg = Reg;
787   }
788 
789   // Note the unset value for this is AMDGPU::SP_REG rather than
790   // NoRegister. This is mostly a workaround for MIR tests where state that
791   // can't be directly computed from the function is not preserved in serialized
792   // MIR.
793   Register getStackPtrOffsetReg() const {
794     return StackPtrOffsetReg;
795   }
796 
797   Register getQueuePtrUserSGPR() const {
798     return ArgInfo.QueuePtr.getRegister();
799   }
800 
801   Register getImplicitBufferPtrUserSGPR() const {
802     return ArgInfo.ImplicitBufferPtr.getRegister();
803   }
804 
805   bool hasSpilledSGPRs() const {
806     return HasSpilledSGPRs;
807   }
808 
809   void setHasSpilledSGPRs(bool Spill = true) {
810     HasSpilledSGPRs = Spill;
811   }
812 
813   bool hasSpilledVGPRs() const {
814     return HasSpilledVGPRs;
815   }
816 
817   void setHasSpilledVGPRs(bool Spill = true) {
818     HasSpilledVGPRs = Spill;
819   }
820 
821   bool hasNonSpillStackObjects() const {
822     return HasNonSpillStackObjects;
823   }
824 
825   void setHasNonSpillStackObjects(bool StackObject = true) {
826     HasNonSpillStackObjects = StackObject;
827   }
828 
829   bool isStackRealigned() const {
830     return IsStackRealigned;
831   }
832 
833   void setIsStackRealigned(bool Realigned = true) {
834     IsStackRealigned = Realigned;
835   }
836 
837   unsigned getNumSpilledSGPRs() const {
838     return NumSpilledSGPRs;
839   }
840 
841   unsigned getNumSpilledVGPRs() const {
842     return NumSpilledVGPRs;
843   }
844 
845   void addToSpilledSGPRs(unsigned num) {
846     NumSpilledSGPRs += num;
847   }
848 
849   void addToSpilledVGPRs(unsigned num) {
850     NumSpilledVGPRs += num;
851   }
852 
853   unsigned getPSInputAddr() const {
854     return PSInputAddr;
855   }
856 
857   unsigned getPSInputEnable() const {
858     return PSInputEnable;
859   }
860 
861   bool isPSInputAllocated(unsigned Index) const {
862     return PSInputAddr & (1 << Index);
863   }
864 
865   void markPSInputAllocated(unsigned Index) {
866     PSInputAddr |= 1 << Index;
867   }
868 
869   void markPSInputEnabled(unsigned Index) {
870     PSInputEnable |= 1 << Index;
871   }
872 
873   bool returnsVoid() const {
874     return ReturnsVoid;
875   }
876 
877   void setIfReturnsVoid(bool Value) {
878     ReturnsVoid = Value;
879   }
880 
881   /// \returns A pair of default/requested minimum/maximum flat work group sizes
882   /// for this function.
883   std::pair<unsigned, unsigned> getFlatWorkGroupSizes() const {
884     return FlatWorkGroupSizes;
885   }
886 
887   /// \returns Default/requested minimum flat work group size for this function.
888   unsigned getMinFlatWorkGroupSize() const {
889     return FlatWorkGroupSizes.first;
890   }
891 
892   /// \returns Default/requested maximum flat work group size for this function.
893   unsigned getMaxFlatWorkGroupSize() const {
894     return FlatWorkGroupSizes.second;
895   }
896 
897   /// \returns A pair of default/requested minimum/maximum number of waves per
898   /// execution unit.
899   std::pair<unsigned, unsigned> getWavesPerEU() const {
900     return WavesPerEU;
901   }
902 
903   /// \returns Default/requested minimum number of waves per execution unit.
904   unsigned getMinWavesPerEU() const {
905     return WavesPerEU.first;
906   }
907 
908   /// \returns Default/requested maximum number of waves per execution unit.
909   unsigned getMaxWavesPerEU() const {
910     return WavesPerEU.second;
911   }
912 
913   /// \returns SGPR used for \p Dim's work group ID.
914   Register getWorkGroupIDSGPR(unsigned Dim) const {
915     switch (Dim) {
916     case 0:
917       assert(hasWorkGroupIDX());
918       return ArgInfo.WorkGroupIDX.getRegister();
919     case 1:
920       assert(hasWorkGroupIDY());
921       return ArgInfo.WorkGroupIDY.getRegister();
922     case 2:
923       assert(hasWorkGroupIDZ());
924       return ArgInfo.WorkGroupIDZ.getRegister();
925     }
926     llvm_unreachable("unexpected dimension");
927   }
928 
929   const AMDGPUGWSResourcePseudoSourceValue *
930   getGWSPSV(const AMDGPUTargetMachine &TM) {
931     return &GWSResourcePSV;
932   }
933 
934   unsigned getOccupancy() const {
935     return Occupancy;
936   }
937 
938   unsigned getMinAllowedOccupancy() const {
939     if (!isMemoryBound() && !needsWaveLimiter())
940       return Occupancy;
941     return (Occupancy < 4) ? Occupancy : 4;
942   }
943 
944   void limitOccupancy(const MachineFunction &MF);
945 
946   void limitOccupancy(unsigned Limit) {
947     if (Occupancy > Limit)
948       Occupancy = Limit;
949   }
950 
951   void increaseOccupancy(const MachineFunction &MF, unsigned Limit) {
952     if (Occupancy < Limit)
953       Occupancy = Limit;
954     limitOccupancy(MF);
955   }
956 
957   bool mayNeedAGPRs() const {
958     return MayNeedAGPRs;
959   }
960 
961   // \returns true if a function has a use of AGPRs via inline asm or
962   // has a call which may use it.
963   bool mayUseAGPRs(const Function &F) const;
964 
965   // \returns true if a function needs or may need AGPRs.
966   bool usesAGPRs(const MachineFunction &MF) const;
967 };
968 
969 } // end namespace llvm
970 
971 #endif // LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
972