xref: /llvm-project/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h (revision 5257a60ee02e5cbecb2f577b27a9c89e92b2f85f) 
1 //==- SIMachineFunctionInfo.h - SIMachineFunctionInfo interface --*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #ifndef LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
14 #define LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
15 
16 #include "AMDGPUArgumentUsageInfo.h"
17 #include "AMDGPUMachineFunction.h"
18 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
19 #include "SIInstrInfo.h"
20 #include "SIRegisterInfo.h"
21 #include "llvm/ADT/ArrayRef.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/Optional.h"
24 #include "llvm/ADT/STLExtras.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/SparseBitVector.h"
27 #include "llvm/CodeGen/MIRYamlMapping.h"
28 #include "llvm/CodeGen/PseudoSourceValue.h"
29 #include "llvm/CodeGen/TargetInstrInfo.h"
30 #include "llvm/MC/MCRegisterInfo.h"
31 #include "llvm/Support/ErrorHandling.h"
32 #include <array>
33 #include <cassert>
34 #include <utility>
35 #include <vector>
36 
37 namespace llvm {
38 
39 class MachineFrameInfo;
40 class MachineFunction;
41 class TargetRegisterClass;
42 
43 class AMDGPUPseudoSourceValue : public PseudoSourceValue {
44 public:
45   enum AMDGPUPSVKind : unsigned {
46     PSVBuffer = PseudoSourceValue::TargetCustom,
47     PSVImage,
48     GWSResource
49   };
50 
51 protected:
52   AMDGPUPseudoSourceValue(unsigned Kind, const TargetInstrInfo &TII)
53       : PseudoSourceValue(Kind, TII) {}
54 
55 public:
56   bool isConstant(const MachineFrameInfo *) const override {
57     // This should probably be true for most images, but we will start by being
58     // conservative.
59     return false;
60   }
61 
62   bool isAliased(const MachineFrameInfo *) const override {
63     return true;
64   }
65 
66   bool mayAlias(const MachineFrameInfo *) const override {
67     return true;
68   }
69 };
70 
71 class AMDGPUBufferPseudoSourceValue final : public AMDGPUPseudoSourceValue {
72 public:
73   explicit AMDGPUBufferPseudoSourceValue(const TargetInstrInfo &TII)
74       : AMDGPUPseudoSourceValue(PSVBuffer, TII) {}
75 
76   static bool classof(const PseudoSourceValue *V) {
77     return V->kind() == PSVBuffer;
78   }
79 };
80 
81 class AMDGPUImagePseudoSourceValue final : public AMDGPUPseudoSourceValue {
82 public:
83   // TODO: Is the img rsrc useful?
84   explicit AMDGPUImagePseudoSourceValue(const TargetInstrInfo &TII)
85       : AMDGPUPseudoSourceValue(PSVImage, TII) {}
86 
87   static bool classof(const PseudoSourceValue *V) {
88     return V->kind() == PSVImage;
89   }
90 };
91 
92 class AMDGPUGWSResourcePseudoSourceValue final : public AMDGPUPseudoSourceValue {
93 public:
94   explicit AMDGPUGWSResourcePseudoSourceValue(const TargetInstrInfo &TII)
95       : AMDGPUPseudoSourceValue(GWSResource, TII) {}
96 
97   static bool classof(const PseudoSourceValue *V) {
98     return V->kind() == GWSResource;
99   }
100 
101   // These are inaccessible memory from IR.
102   bool isAliased(const MachineFrameInfo *) const override {
103     return false;
104   }
105 
106   // These are inaccessible memory from IR.
107   bool mayAlias(const MachineFrameInfo *) const override {
108     return false;
109   }
110 
111   void printCustom(raw_ostream &OS) const override {
112     OS << "GWSResource";
113   }
114 };
115 
116 namespace yaml {
117 
118 struct SIArgument {
119   bool IsRegister;
120   union {
121     StringValue RegisterName;
122     unsigned StackOffset;
123   };
124   Optional<unsigned> Mask;
125 
126   // Default constructor, which creates a stack argument.
127   SIArgument() : IsRegister(false), StackOffset(0) {}
128   SIArgument(const SIArgument &Other) {
129     IsRegister = Other.IsRegister;
130     if (IsRegister) {
131       ::new ((void *)std::addressof(RegisterName))
132           StringValue(Other.RegisterName);
133     } else
134       StackOffset = Other.StackOffset;
135     Mask = Other.Mask;
136   }
137   SIArgument &operator=(const SIArgument &Other) {
138     IsRegister = Other.IsRegister;
139     if (IsRegister) {
140       ::new ((void *)std::addressof(RegisterName))
141           StringValue(Other.RegisterName);
142     } else
143       StackOffset = Other.StackOffset;
144     Mask = Other.Mask;
145     return *this;
146   }
147   ~SIArgument() {
148     if (IsRegister)
149       RegisterName.~StringValue();
150   }
151 
152   // Helper to create a register or stack argument.
153   static inline SIArgument createArgument(bool IsReg) {
154     if (IsReg)
155       return SIArgument(IsReg);
156     return SIArgument();
157   }
158 
159 private:
160   // Construct a register argument.
161   SIArgument(bool) : IsRegister(true), RegisterName() {}
162 };
163 
164 template <> struct MappingTraits<SIArgument> {
165   static void mapping(IO &YamlIO, SIArgument &A) {
166     if (YamlIO.outputting()) {
167       if (A.IsRegister)
168         YamlIO.mapRequired("reg", A.RegisterName);
169       else
170         YamlIO.mapRequired("offset", A.StackOffset);
171     } else {
172       auto Keys = YamlIO.keys();
173       if (is_contained(Keys, "reg")) {
174         A = SIArgument::createArgument(true);
175         YamlIO.mapRequired("reg", A.RegisterName);
176       } else if (is_contained(Keys, "offset"))
177         YamlIO.mapRequired("offset", A.StackOffset);
178       else
179         YamlIO.setError("missing required key 'reg' or 'offset'");
180     }
181     YamlIO.mapOptional("mask", A.Mask);
182   }
183   static const bool flow = true;
184 };
185 
186 struct SIArgumentInfo {
187   Optional<SIArgument> PrivateSegmentBuffer;
188   Optional<SIArgument> DispatchPtr;
189   Optional<SIArgument> QueuePtr;
190   Optional<SIArgument> KernargSegmentPtr;
191   Optional<SIArgument> DispatchID;
192   Optional<SIArgument> FlatScratchInit;
193   Optional<SIArgument> PrivateSegmentSize;
194 
195   Optional<SIArgument> WorkGroupIDX;
196   Optional<SIArgument> WorkGroupIDY;
197   Optional<SIArgument> WorkGroupIDZ;
198   Optional<SIArgument> WorkGroupInfo;
199   Optional<SIArgument> PrivateSegmentWaveByteOffset;
200 
201   Optional<SIArgument> ImplicitArgPtr;
202   Optional<SIArgument> ImplicitBufferPtr;
203 
204   Optional<SIArgument> WorkItemIDX;
205   Optional<SIArgument> WorkItemIDY;
206   Optional<SIArgument> WorkItemIDZ;
207 };
208 
209 template <> struct MappingTraits<SIArgumentInfo> {
210   static void mapping(IO &YamlIO, SIArgumentInfo &AI) {
211     YamlIO.mapOptional("privateSegmentBuffer", AI.PrivateSegmentBuffer);
212     YamlIO.mapOptional("dispatchPtr", AI.DispatchPtr);
213     YamlIO.mapOptional("queuePtr", AI.QueuePtr);
214     YamlIO.mapOptional("kernargSegmentPtr", AI.KernargSegmentPtr);
215     YamlIO.mapOptional("dispatchID", AI.DispatchID);
216     YamlIO.mapOptional("flatScratchInit", AI.FlatScratchInit);
217     YamlIO.mapOptional("privateSegmentSize", AI.PrivateSegmentSize);
218 
219     YamlIO.mapOptional("workGroupIDX", AI.WorkGroupIDX);
220     YamlIO.mapOptional("workGroupIDY", AI.WorkGroupIDY);
221     YamlIO.mapOptional("workGroupIDZ", AI.WorkGroupIDZ);
222     YamlIO.mapOptional("workGroupInfo", AI.WorkGroupInfo);
223     YamlIO.mapOptional("privateSegmentWaveByteOffset",
224                        AI.PrivateSegmentWaveByteOffset);
225 
226     YamlIO.mapOptional("implicitArgPtr", AI.ImplicitArgPtr);
227     YamlIO.mapOptional("implicitBufferPtr", AI.ImplicitBufferPtr);
228 
229     YamlIO.mapOptional("workItemIDX", AI.WorkItemIDX);
230     YamlIO.mapOptional("workItemIDY", AI.WorkItemIDY);
231     YamlIO.mapOptional("workItemIDZ", AI.WorkItemIDZ);
232   }
233 };
234 
235 // Default to default mode for default calling convention.
236 struct SIMode {
237   bool IEEE = true;
238   bool DX10Clamp = true;
239   bool FP32InputDenormals = true;
240   bool FP32OutputDenormals = true;
241   bool FP64FP16InputDenormals = true;
242   bool FP64FP16OutputDenormals = true;
243 
244   SIMode() = default;
245 
246   SIMode(const AMDGPU::SIModeRegisterDefaults &Mode) {
247     IEEE = Mode.IEEE;
248     DX10Clamp = Mode.DX10Clamp;
249     FP32InputDenormals = Mode.FP32InputDenormals;
250     FP32OutputDenormals = Mode.FP32OutputDenormals;
251     FP64FP16InputDenormals = Mode.FP64FP16InputDenormals;
252     FP64FP16OutputDenormals = Mode.FP64FP16OutputDenormals;
253   }
254 
255   bool operator ==(const SIMode Other) const {
256     return IEEE == Other.IEEE &&
257            DX10Clamp == Other.DX10Clamp &&
258            FP32InputDenormals == Other.FP32InputDenormals &&
259            FP32OutputDenormals == Other.FP32OutputDenormals &&
260            FP64FP16InputDenormals == Other.FP64FP16InputDenormals &&
261            FP64FP16OutputDenormals == Other.FP64FP16OutputDenormals;
262   }
263 };
264 
265 template <> struct MappingTraits<SIMode> {
266   static void mapping(IO &YamlIO, SIMode &Mode) {
267     YamlIO.mapOptional("ieee", Mode.IEEE, true);
268     YamlIO.mapOptional("dx10-clamp", Mode.DX10Clamp, true);
269     YamlIO.mapOptional("fp32-input-denormals", Mode.FP32InputDenormals, true);
270     YamlIO.mapOptional("fp32-output-denormals", Mode.FP32OutputDenormals, true);
271     YamlIO.mapOptional("fp64-fp16-input-denormals", Mode.FP64FP16InputDenormals, true);
272     YamlIO.mapOptional("fp64-fp16-output-denormals", Mode.FP64FP16OutputDenormals, true);
273   }
274 };
275 
276 struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo {
277   uint64_t ExplicitKernArgSize = 0;
278   unsigned MaxKernArgAlign = 0;
279   unsigned LDSSize = 0;
280   Align DynLDSAlign;
281   bool IsEntryFunction = false;
282   bool NoSignedZerosFPMath = false;
283   bool MemoryBound = false;
284   bool WaveLimiter = false;
285   bool HasSpilledSGPRs = false;
286   bool HasSpilledVGPRs = false;
287   uint32_t HighBitsOf32BitAddress = 0;
288 
289   StringValue ScratchRSrcReg = "$private_rsrc_reg";
290   StringValue FrameOffsetReg = "$fp_reg";
291   StringValue StackPtrOffsetReg = "$sp_reg";
292 
293   Optional<SIArgumentInfo> ArgInfo;
294   SIMode Mode;
295 
296   SIMachineFunctionInfo() = default;
297   SIMachineFunctionInfo(const llvm::SIMachineFunctionInfo &,
298                         const TargetRegisterInfo &TRI);
299 
300   void mappingImpl(yaml::IO &YamlIO) override;
301   ~SIMachineFunctionInfo() = default;
302 };
303 
304 template <> struct MappingTraits<SIMachineFunctionInfo> {
305   static void mapping(IO &YamlIO, SIMachineFunctionInfo &MFI) {
306     YamlIO.mapOptional("explicitKernArgSize", MFI.ExplicitKernArgSize,
307                        UINT64_C(0));
308     YamlIO.mapOptional("maxKernArgAlign", MFI.MaxKernArgAlign, 0u);
309     YamlIO.mapOptional("ldsSize", MFI.LDSSize, 0u);
310     YamlIO.mapOptional("dynLDSAlign", MFI.DynLDSAlign, Align());
311     YamlIO.mapOptional("isEntryFunction", MFI.IsEntryFunction, false);
312     YamlIO.mapOptional("noSignedZerosFPMath", MFI.NoSignedZerosFPMath, false);
313     YamlIO.mapOptional("memoryBound", MFI.MemoryBound, false);
314     YamlIO.mapOptional("waveLimiter", MFI.WaveLimiter, false);
315     YamlIO.mapOptional("hasSpilledSGPRs", MFI.HasSpilledSGPRs, false);
316     YamlIO.mapOptional("hasSpilledVGPRs", MFI.HasSpilledVGPRs, false);
317     YamlIO.mapOptional("scratchRSrcReg", MFI.ScratchRSrcReg,
318                        StringValue("$private_rsrc_reg"));
319     YamlIO.mapOptional("frameOffsetReg", MFI.FrameOffsetReg,
320                        StringValue("$fp_reg"));
321     YamlIO.mapOptional("stackPtrOffsetReg", MFI.StackPtrOffsetReg,
322                        StringValue("$sp_reg"));
323     YamlIO.mapOptional("argumentInfo", MFI.ArgInfo);
324     YamlIO.mapOptional("mode", MFI.Mode, SIMode());
325     YamlIO.mapOptional("highBitsOf32BitAddress",
326                        MFI.HighBitsOf32BitAddress, 0u);
327   }
328 };
329 
330 } // end namespace yaml
331 
332 /// This class keeps track of the SPI_SP_INPUT_ADDR config register, which
333 /// tells the hardware which interpolation parameters to load.
334 class SIMachineFunctionInfo final : public AMDGPUMachineFunction {
335   friend class GCNTargetMachine;
336 
337   Register TIDReg = AMDGPU::NoRegister;
338 
339   // Registers that may be reserved for spilling purposes. These may be the same
340   // as the input registers.
341   Register ScratchRSrcReg = AMDGPU::PRIVATE_RSRC_REG;
342 
343   // This is the the unswizzled offset from the current dispatch's scratch wave
344   // base to the beginning of the current function's frame.
345   Register FrameOffsetReg = AMDGPU::FP_REG;
346 
347   // This is an ABI register used in the non-entry calling convention to
348   // communicate the unswizzled offset from the current dispatch's scratch wave
349   // base to the beginning of the new function's frame.
350   Register StackPtrOffsetReg = AMDGPU::SP_REG;
351 
352   AMDGPUFunctionArgInfo ArgInfo;
353 
354   // Graphics info.
355   unsigned PSInputAddr = 0;
356   unsigned PSInputEnable = 0;
357 
358   /// Number of bytes of arguments this function has on the stack. If the callee
359   /// is expected to restore the argument stack this should be a multiple of 16,
360   /// all usable during a tail call.
361   ///
362   /// The alternative would forbid tail call optimisation in some cases: if we
363   /// want to transfer control from a function with 8-bytes of stack-argument
364   /// space to a function with 16-bytes then misalignment of this value would
365   /// make a stack adjustment necessary, which could not be undone by the
366   /// callee.
367   unsigned BytesInStackArgArea = 0;
368 
369   bool ReturnsVoid = true;
370 
371   // A pair of default/requested minimum/maximum flat work group sizes.
372   // Minimum - first, maximum - second.
373   std::pair<unsigned, unsigned> FlatWorkGroupSizes = {0, 0};
374 
375   // A pair of default/requested minimum/maximum number of waves per execution
376   // unit. Minimum - first, maximum - second.
377   std::pair<unsigned, unsigned> WavesPerEU = {0, 0};
378 
379   DenseMap<const Value *,
380            std::unique_ptr<const AMDGPUBufferPseudoSourceValue>> BufferPSVs;
381   DenseMap<const Value *,
382            std::unique_ptr<const AMDGPUImagePseudoSourceValue>> ImagePSVs;
383   std::unique_ptr<const AMDGPUGWSResourcePseudoSourceValue> GWSResourcePSV;
384 
385 private:
386   unsigned LDSWaveSpillSize = 0;
387   unsigned NumUserSGPRs = 0;
388   unsigned NumSystemSGPRs = 0;
389 
390   bool HasSpilledSGPRs = false;
391   bool HasSpilledVGPRs = false;
392   bool HasNonSpillStackObjects = false;
393   bool IsStackRealigned = false;
394 
395   unsigned NumSpilledSGPRs = 0;
396   unsigned NumSpilledVGPRs = 0;
397 
398   // Feature bits required for inputs passed in user SGPRs.
399   bool PrivateSegmentBuffer : 1;
400   bool DispatchPtr : 1;
401   bool QueuePtr : 1;
402   bool KernargSegmentPtr : 1;
403   bool DispatchID : 1;
404   bool FlatScratchInit : 1;
405 
406   // Feature bits required for inputs passed in system SGPRs.
407   bool WorkGroupIDX : 1; // Always initialized.
408   bool WorkGroupIDY : 1;
409   bool WorkGroupIDZ : 1;
410   bool WorkGroupInfo : 1;
411   bool PrivateSegmentWaveByteOffset : 1;
412 
413   bool WorkItemIDX : 1; // Always initialized.
414   bool WorkItemIDY : 1;
415   bool WorkItemIDZ : 1;
416 
417   // Private memory buffer
418   // Compute directly in sgpr[0:1]
419   // Other shaders indirect 64-bits at sgpr[0:1]
420   bool ImplicitBufferPtr : 1;
421 
422   // Pointer to where the ABI inserts special kernel arguments separate from the
423   // user arguments. This is an offset from the KernargSegmentPtr.
424   bool ImplicitArgPtr : 1;
425 
426   // The hard-wired high half of the address of the global information table
427   // for AMDPAL OS type. 0xffffffff represents no hard-wired high half, since
428   // current hardware only allows a 16 bit value.
429   unsigned GITPtrHigh;
430 
431   unsigned HighBitsOf32BitAddress;
432   unsigned GDSSize;
433 
434   // Current recorded maximum possible occupancy.
435   unsigned Occupancy;
436 
437   MCPhysReg getNextUserSGPR() const;
438 
439   MCPhysReg getNextSystemSGPR() const;
440 
441 public:
442   struct SpilledReg {
443     Register VGPR;
444     int Lane = -1;
445 
446     SpilledReg() = default;
447     SpilledReg(Register R, int L) : VGPR (R), Lane (L) {}
448 
449     bool hasLane() { return Lane != -1;}
450     bool hasReg() { return VGPR != 0;}
451   };
452 
453   struct SGPRSpillVGPRCSR {
454     // VGPR used for SGPR spills
455     Register VGPR;
456 
457     // If the VGPR is a CSR, the stack slot used to save/restore it in the
458     // prolog/epilog.
459     Optional<int> FI;
460 
461     SGPRSpillVGPRCSR(Register V, Optional<int> F) : VGPR(V), FI(F) {}
462   };
463 
464   struct VGPRSpillToAGPR {
465     SmallVector<MCPhysReg, 32> Lanes;
466     bool FullyAllocated = false;
467   };
468 
469   SparseBitVector<> WWMReservedRegs;
470 
471   void ReserveWWMRegister(Register Reg) { WWMReservedRegs.set(Reg); }
472 
473 private:
474   // Track VGPR + wave index for each subregister of the SGPR spilled to
475   // frameindex key.
476   DenseMap<int, std::vector<SpilledReg>> SGPRToVGPRSpills;
477   unsigned NumVGPRSpillLanes = 0;
478   SmallVector<SGPRSpillVGPRCSR, 2> SpillVGPRs;
479 
480   DenseMap<int, VGPRSpillToAGPR> VGPRToAGPRSpills;
481 
482   // AGPRs used for VGPR spills.
483   SmallVector<MCPhysReg, 32> SpillAGPR;
484 
485   // VGPRs used for AGPR spills.
486   SmallVector<MCPhysReg, 32> SpillVGPR;
487 
488 public: // FIXME
489   /// If this is set, an SGPR used for save/restore of the register used for the
490   /// frame pointer.
491   Register SGPRForFPSaveRestoreCopy;
492   Optional<int> FramePointerSaveIndex;
493 
494   /// If this is set, an SGPR used for save/restore of the register used for the
495   /// base pointer.
496   Register SGPRForBPSaveRestoreCopy;
497   Optional<int> BasePointerSaveIndex;
498 
499   Register VGPRReservedForSGPRSpill;
500   bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg);
501 
502 public:
503   SIMachineFunctionInfo(const MachineFunction &MF);
504 
505   bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI);
506 
507   ArrayRef<SpilledReg> getSGPRToVGPRSpills(int FrameIndex) const {
508     auto I = SGPRToVGPRSpills.find(FrameIndex);
509     return (I == SGPRToVGPRSpills.end()) ?
510       ArrayRef<SpilledReg>() : makeArrayRef(I->second);
511   }
512 
513   ArrayRef<SGPRSpillVGPRCSR> getSGPRSpillVGPRs() const {
514     return SpillVGPRs;
515   }
516 
517   void setSGPRSpillVGPRs(Register NewVGPR, Optional<int> newFI, int Index) {
518     SpillVGPRs[Index].VGPR = NewVGPR;
519     SpillVGPRs[Index].FI = newFI;
520     VGPRReservedForSGPRSpill = NewVGPR;
521   }
522 
523   bool removeVGPRForSGPRSpill(Register ReservedVGPR, MachineFunction &MF);
524 
525   ArrayRef<MCPhysReg> getAGPRSpillVGPRs() const {
526     return SpillAGPR;
527   }
528 
529   ArrayRef<MCPhysReg> getVGPRSpillAGPRs() const {
530     return SpillVGPR;
531   }
532 
533   MCPhysReg getVGPRToAGPRSpill(int FrameIndex, unsigned Lane) const {
534     auto I = VGPRToAGPRSpills.find(FrameIndex);
535     return (I == VGPRToAGPRSpills.end()) ? (MCPhysReg)AMDGPU::NoRegister
536                                          : I->second.Lanes[Lane];
537   }
538 
539   bool haveFreeLanesForSGPRSpill(const MachineFunction &MF,
540                                  unsigned NumLane) const;
541   bool allocateSGPRSpillToVGPR(MachineFunction &MF, int FI);
542   bool reserveVGPRforSGPRSpills(MachineFunction &MF);
543   bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR);
544   void removeDeadFrameIndices(MachineFrameInfo &MFI);
545 
546   bool hasCalculatedTID() const { return TIDReg != 0; };
547   Register getTIDReg() const { return TIDReg; };
548   void setTIDReg(Register Reg) { TIDReg = Reg; }
549 
550   unsigned getBytesInStackArgArea() const {
551     return BytesInStackArgArea;
552   }
553 
554   void setBytesInStackArgArea(unsigned Bytes) {
555     BytesInStackArgArea = Bytes;
556   }
557 
558   // Add user SGPRs.
559   Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI);
560   Register addDispatchPtr(const SIRegisterInfo &TRI);
561   Register addQueuePtr(const SIRegisterInfo &TRI);
562   Register addKernargSegmentPtr(const SIRegisterInfo &TRI);
563   Register addDispatchID(const SIRegisterInfo &TRI);
564   Register addFlatScratchInit(const SIRegisterInfo &TRI);
565   Register addImplicitBufferPtr(const SIRegisterInfo &TRI);
566 
567   // Add system SGPRs.
568   Register addWorkGroupIDX() {
569     ArgInfo.WorkGroupIDX = ArgDescriptor::createRegister(getNextSystemSGPR());
570     NumSystemSGPRs += 1;
571     return ArgInfo.WorkGroupIDX.getRegister();
572   }
573 
574   Register addWorkGroupIDY() {
575     ArgInfo.WorkGroupIDY = ArgDescriptor::createRegister(getNextSystemSGPR());
576     NumSystemSGPRs += 1;
577     return ArgInfo.WorkGroupIDY.getRegister();
578   }
579 
580   Register addWorkGroupIDZ() {
581     ArgInfo.WorkGroupIDZ = ArgDescriptor::createRegister(getNextSystemSGPR());
582     NumSystemSGPRs += 1;
583     return ArgInfo.WorkGroupIDZ.getRegister();
584   }
585 
586   Register addWorkGroupInfo() {
587     ArgInfo.WorkGroupInfo = ArgDescriptor::createRegister(getNextSystemSGPR());
588     NumSystemSGPRs += 1;
589     return ArgInfo.WorkGroupInfo.getRegister();
590   }
591 
592   // Add special VGPR inputs
593   void setWorkItemIDX(ArgDescriptor Arg) {
594     ArgInfo.WorkItemIDX = Arg;
595   }
596 
597   void setWorkItemIDY(ArgDescriptor Arg) {
598     ArgInfo.WorkItemIDY = Arg;
599   }
600 
601   void setWorkItemIDZ(ArgDescriptor Arg) {
602     ArgInfo.WorkItemIDZ = Arg;
603   }
604 
605   Register addPrivateSegmentWaveByteOffset() {
606     ArgInfo.PrivateSegmentWaveByteOffset
607       = ArgDescriptor::createRegister(getNextSystemSGPR());
608     NumSystemSGPRs += 1;
609     return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
610   }
611 
612   void setPrivateSegmentWaveByteOffset(Register Reg) {
613     ArgInfo.PrivateSegmentWaveByteOffset = ArgDescriptor::createRegister(Reg);
614   }
615 
616   bool hasPrivateSegmentBuffer() const {
617     return PrivateSegmentBuffer;
618   }
619 
620   bool hasDispatchPtr() const {
621     return DispatchPtr;
622   }
623 
624   bool hasQueuePtr() const {
625     return QueuePtr;
626   }
627 
628   bool hasKernargSegmentPtr() const {
629     return KernargSegmentPtr;
630   }
631 
632   bool hasDispatchID() const {
633     return DispatchID;
634   }
635 
636   bool hasFlatScratchInit() const {
637     return FlatScratchInit;
638   }
639 
640   bool hasWorkGroupIDX() const {
641     return WorkGroupIDX;
642   }
643 
644   bool hasWorkGroupIDY() const {
645     return WorkGroupIDY;
646   }
647 
648   bool hasWorkGroupIDZ() const {
649     return WorkGroupIDZ;
650   }
651 
652   bool hasWorkGroupInfo() const {
653     return WorkGroupInfo;
654   }
655 
656   bool hasPrivateSegmentWaveByteOffset() const {
657     return PrivateSegmentWaveByteOffset;
658   }
659 
660   bool hasWorkItemIDX() const {
661     return WorkItemIDX;
662   }
663 
664   bool hasWorkItemIDY() const {
665     return WorkItemIDY;
666   }
667 
668   bool hasWorkItemIDZ() const {
669     return WorkItemIDZ;
670   }
671 
672   bool hasImplicitArgPtr() const {
673     return ImplicitArgPtr;
674   }
675 
676   bool hasImplicitBufferPtr() const {
677     return ImplicitBufferPtr;
678   }
679 
680   AMDGPUFunctionArgInfo &getArgInfo() {
681     return ArgInfo;
682   }
683 
684   const AMDGPUFunctionArgInfo &getArgInfo() const {
685     return ArgInfo;
686   }
687 
688   std::tuple<const ArgDescriptor *, const TargetRegisterClass *, LLT>
689   getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
690     return ArgInfo.getPreloadedValue(Value);
691   }
692 
693   MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
694     auto Arg = std::get<0>(ArgInfo.getPreloadedValue(Value));
695     return Arg ? Arg->getRegister() : MCRegister();
696   }
697 
698   unsigned getGITPtrHigh() const {
699     return GITPtrHigh;
700   }
701 
702   Register getGITPtrLoReg(const MachineFunction &MF) const;
703 
704   uint32_t get32BitAddressHighBits() const {
705     return HighBitsOf32BitAddress;
706   }
707 
708   unsigned getGDSSize() const {
709     return GDSSize;
710   }
711 
712   unsigned getNumUserSGPRs() const {
713     return NumUserSGPRs;
714   }
715 
716   unsigned getNumPreloadedSGPRs() const {
717     return NumUserSGPRs + NumSystemSGPRs;
718   }
719 
720   Register getPrivateSegmentWaveByteOffsetSystemSGPR() const {
721     return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
722   }
723 
724   /// Returns the physical register reserved for use as the resource
725   /// descriptor for scratch accesses.
726   Register getScratchRSrcReg() const {
727     return ScratchRSrcReg;
728   }
729 
730   void setScratchRSrcReg(Register Reg) {
731     assert(Reg != 0 && "Should never be unset");
732     ScratchRSrcReg = Reg;
733   }
734 
735   Register getFrameOffsetReg() const {
736     return FrameOffsetReg;
737   }
738 
739   void setFrameOffsetReg(Register Reg) {
740     assert(Reg != 0 && "Should never be unset");
741     FrameOffsetReg = Reg;
742   }
743 
744   void setStackPtrOffsetReg(Register Reg) {
745     assert(Reg != 0 && "Should never be unset");
746     StackPtrOffsetReg = Reg;
747   }
748 
749   // Note the unset value for this is AMDGPU::SP_REG rather than
750   // NoRegister. This is mostly a workaround for MIR tests where state that
751   // can't be directly computed from the function is not preserved in serialized
752   // MIR.
753   Register getStackPtrOffsetReg() const {
754     return StackPtrOffsetReg;
755   }
756 
757   Register getQueuePtrUserSGPR() const {
758     return ArgInfo.QueuePtr.getRegister();
759   }
760 
761   Register getImplicitBufferPtrUserSGPR() const {
762     return ArgInfo.ImplicitBufferPtr.getRegister();
763   }
764 
765   bool hasSpilledSGPRs() const {
766     return HasSpilledSGPRs;
767   }
768 
769   void setHasSpilledSGPRs(bool Spill = true) {
770     HasSpilledSGPRs = Spill;
771   }
772 
773   bool hasSpilledVGPRs() const {
774     return HasSpilledVGPRs;
775   }
776 
777   void setHasSpilledVGPRs(bool Spill = true) {
778     HasSpilledVGPRs = Spill;
779   }
780 
781   bool hasNonSpillStackObjects() const {
782     return HasNonSpillStackObjects;
783   }
784 
785   void setHasNonSpillStackObjects(bool StackObject = true) {
786     HasNonSpillStackObjects = StackObject;
787   }
788 
789   bool isStackRealigned() const {
790     return IsStackRealigned;
791   }
792 
793   void setIsStackRealigned(bool Realigned = true) {
794     IsStackRealigned = Realigned;
795   }
796 
797   unsigned getNumSpilledSGPRs() const {
798     return NumSpilledSGPRs;
799   }
800 
801   unsigned getNumSpilledVGPRs() const {
802     return NumSpilledVGPRs;
803   }
804 
805   void addToSpilledSGPRs(unsigned num) {
806     NumSpilledSGPRs += num;
807   }
808 
809   void addToSpilledVGPRs(unsigned num) {
810     NumSpilledVGPRs += num;
811   }
812 
813   unsigned getPSInputAddr() const {
814     return PSInputAddr;
815   }
816 
817   unsigned getPSInputEnable() const {
818     return PSInputEnable;
819   }
820 
821   bool isPSInputAllocated(unsigned Index) const {
822     return PSInputAddr & (1 << Index);
823   }
824 
825   void markPSInputAllocated(unsigned Index) {
826     PSInputAddr |= 1 << Index;
827   }
828 
829   void markPSInputEnabled(unsigned Index) {
830     PSInputEnable |= 1 << Index;
831   }
832 
833   bool returnsVoid() const {
834     return ReturnsVoid;
835   }
836 
837   void setIfReturnsVoid(bool Value) {
838     ReturnsVoid = Value;
839   }
840 
841   /// \returns A pair of default/requested minimum/maximum flat work group sizes
842   /// for this function.
843   std::pair<unsigned, unsigned> getFlatWorkGroupSizes() const {
844     return FlatWorkGroupSizes;
845   }
846 
847   /// \returns Default/requested minimum flat work group size for this function.
848   unsigned getMinFlatWorkGroupSize() const {
849     return FlatWorkGroupSizes.first;
850   }
851 
852   /// \returns Default/requested maximum flat work group size for this function.
853   unsigned getMaxFlatWorkGroupSize() const {
854     return FlatWorkGroupSizes.second;
855   }
856 
857   /// \returns A pair of default/requested minimum/maximum number of waves per
858   /// execution unit.
859   std::pair<unsigned, unsigned> getWavesPerEU() const {
860     return WavesPerEU;
861   }
862 
863   /// \returns Default/requested minimum number of waves per execution unit.
864   unsigned getMinWavesPerEU() const {
865     return WavesPerEU.first;
866   }
867 
868   /// \returns Default/requested maximum number of waves per execution unit.
869   unsigned getMaxWavesPerEU() const {
870     return WavesPerEU.second;
871   }
872 
873   /// \returns SGPR used for \p Dim's work group ID.
874   Register getWorkGroupIDSGPR(unsigned Dim) const {
875     switch (Dim) {
876     case 0:
877       assert(hasWorkGroupIDX());
878       return ArgInfo.WorkGroupIDX.getRegister();
879     case 1:
880       assert(hasWorkGroupIDY());
881       return ArgInfo.WorkGroupIDY.getRegister();
882     case 2:
883       assert(hasWorkGroupIDZ());
884       return ArgInfo.WorkGroupIDZ.getRegister();
885     }
886     llvm_unreachable("unexpected dimension");
887   }
888 
889   unsigned getLDSWaveSpillSize() const {
890     return LDSWaveSpillSize;
891   }
892 
893   const AMDGPUBufferPseudoSourceValue *getBufferPSV(const SIInstrInfo &TII,
894                                                     const Value *BufferRsrc) {
895     assert(BufferRsrc);
896     auto PSV = BufferPSVs.try_emplace(
897       BufferRsrc,
898       std::make_unique<AMDGPUBufferPseudoSourceValue>(TII));
899     return PSV.first->second.get();
900   }
901 
902   const AMDGPUImagePseudoSourceValue *getImagePSV(const SIInstrInfo &TII,
903                                                   const Value *ImgRsrc) {
904     assert(ImgRsrc);
905     auto PSV = ImagePSVs.try_emplace(
906       ImgRsrc,
907       std::make_unique<AMDGPUImagePseudoSourceValue>(TII));
908     return PSV.first->second.get();
909   }
910 
911   const AMDGPUGWSResourcePseudoSourceValue *getGWSPSV(const SIInstrInfo &TII) {
912     if (!GWSResourcePSV) {
913       GWSResourcePSV =
914           std::make_unique<AMDGPUGWSResourcePseudoSourceValue>(TII);
915     }
916 
917     return GWSResourcePSV.get();
918   }
919 
920   unsigned getOccupancy() const {
921     return Occupancy;
922   }
923 
924   unsigned getMinAllowedOccupancy() const {
925     if (!isMemoryBound() && !needsWaveLimiter())
926       return Occupancy;
927     return (Occupancy < 4) ? Occupancy : 4;
928   }
929 
930   void limitOccupancy(const MachineFunction &MF);
931 
932   void limitOccupancy(unsigned Limit) {
933     if (Occupancy > Limit)
934       Occupancy = Limit;
935   }
936 
937   void increaseOccupancy(const MachineFunction &MF, unsigned Limit) {
938     if (Occupancy < Limit)
939       Occupancy = Limit;
940     limitOccupancy(MF);
941   }
942 };
943 
944 } // end namespace llvm
945 
946 #endif // LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
947