1 //===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "SIMachineFunctionInfo.h" 10 #include "AMDGPUSubtarget.h" 11 #include "GCNSubtarget.h" 12 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 13 #include "SIRegisterInfo.h" 14 #include "Utils/AMDGPUBaseInfo.h" 15 #include "llvm/CodeGen/LiveIntervals.h" 16 #include "llvm/CodeGen/MIRParser/MIParser.h" 17 #include "llvm/CodeGen/MachineBasicBlock.h" 18 #include "llvm/CodeGen/MachineFrameInfo.h" 19 #include "llvm/CodeGen/MachineFunction.h" 20 #include "llvm/CodeGen/MachineRegisterInfo.h" 21 #include "llvm/IR/CallingConv.h" 22 #include "llvm/IR/DiagnosticInfo.h" 23 #include "llvm/IR/Function.h" 24 #include <cassert> 25 #include <optional> 26 #include <vector> 27 28 enum { MAX_LANES = 64 }; 29 30 using namespace llvm; 31 32 const GCNTargetMachine &getTM(const GCNSubtarget *STI) { 33 const SITargetLowering *TLI = STI->getTargetLowering(); 34 return static_cast<const GCNTargetMachine &>(TLI->getTargetMachine()); 35 } 36 37 SIMachineFunctionInfo::SIMachineFunctionInfo(const Function &F, 38 const GCNSubtarget *STI) 39 : AMDGPUMachineFunction(F, *STI), Mode(F, *STI), GWSResourcePSV(getTM(STI)), 40 UserSGPRInfo(F, *STI), WorkGroupIDX(false), WorkGroupIDY(false), 41 WorkGroupIDZ(false), WorkGroupInfo(false), LDSKernelId(false), 42 PrivateSegmentWaveByteOffset(false), WorkItemIDX(false), 43 WorkItemIDY(false), WorkItemIDZ(false), ImplicitArgPtr(false), 44 GITPtrHigh(0xffffffff), HighBitsOf32BitAddress(0) { 45 const GCNSubtarget &ST = *static_cast<const GCNSubtarget *>(STI); 46 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F); 47 WavesPerEU = ST.getWavesPerEU(F); 48 MaxNumWorkGroups = ST.getMaxNumWorkGroups(F); 49 assert(MaxNumWorkGroups.size() == 3); 50 51 Occupancy = ST.computeOccupancy(F, getLDSSize()).second; 52 CallingConv::ID CC = F.getCallingConv(); 53 54 VRegFlags.reserve(1024); 55 56 const bool IsKernel = CC == CallingConv::AMDGPU_KERNEL || 57 CC == CallingConv::SPIR_KERNEL; 58 59 if (IsKernel) { 60 WorkGroupIDX = true; 61 WorkItemIDX = true; 62 } else if (CC == CallingConv::AMDGPU_PS) { 63 PSInputAddr = AMDGPU::getInitialPSInputAddr(F); 64 } 65 66 MayNeedAGPRs = ST.hasMAIInsts(); 67 68 if (AMDGPU::isChainCC(CC)) { 69 // Chain functions don't receive an SP from their caller, but are free to 70 // set one up. For now, we can use s32 to match what amdgpu_gfx functions 71 // would use if called, but this can be revisited. 72 // FIXME: Only reserve this if we actually need it. 73 StackPtrOffsetReg = AMDGPU::SGPR32; 74 75 ScratchRSrcReg = AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51; 76 77 ArgInfo.PrivateSegmentBuffer = 78 ArgDescriptor::createRegister(ScratchRSrcReg); 79 80 ImplicitArgPtr = false; 81 } else if (!isEntryFunction()) { 82 if (CC != CallingConv::AMDGPU_Gfx) 83 ArgInfo = AMDGPUArgumentUsageInfo::FixedABIFunctionInfo; 84 85 FrameOffsetReg = AMDGPU::SGPR33; 86 StackPtrOffsetReg = AMDGPU::SGPR32; 87 88 if (!ST.enableFlatScratch()) { 89 // Non-entry functions have no special inputs for now, other registers 90 // required for scratch access. 91 ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3; 92 93 ArgInfo.PrivateSegmentBuffer = 94 ArgDescriptor::createRegister(ScratchRSrcReg); 95 } 96 97 if (!F.hasFnAttribute("amdgpu-no-implicitarg-ptr")) 98 ImplicitArgPtr = true; 99 } else { 100 ImplicitArgPtr = false; 101 MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(), 102 MaxKernArgAlign); 103 104 if (ST.hasGFX90AInsts() && 105 ST.getMaxNumVGPRs(F) <= AMDGPU::VGPR_32RegClass.getNumRegs() && 106 !mayUseAGPRs(F)) 107 MayNeedAGPRs = false; // We will select all MAI with VGPR operands. 108 } 109 110 if (!AMDGPU::isGraphics(CC) || 111 ((CC == CallingConv::AMDGPU_CS || CC == CallingConv::AMDGPU_Gfx) && 112 ST.hasArchitectedSGPRs())) { 113 if (IsKernel || !F.hasFnAttribute("amdgpu-no-workgroup-id-x")) 114 WorkGroupIDX = true; 115 116 if (!F.hasFnAttribute("amdgpu-no-workgroup-id-y")) 117 WorkGroupIDY = true; 118 119 if (!F.hasFnAttribute("amdgpu-no-workgroup-id-z")) 120 WorkGroupIDZ = true; 121 } 122 123 if (!AMDGPU::isGraphics(CC)) { 124 if (IsKernel || !F.hasFnAttribute("amdgpu-no-workitem-id-x")) 125 WorkItemIDX = true; 126 127 if (!F.hasFnAttribute("amdgpu-no-workitem-id-y") && 128 ST.getMaxWorkitemID(F, 1) != 0) 129 WorkItemIDY = true; 130 131 if (!F.hasFnAttribute("amdgpu-no-workitem-id-z") && 132 ST.getMaxWorkitemID(F, 2) != 0) 133 WorkItemIDZ = true; 134 135 if (!IsKernel && !F.hasFnAttribute("amdgpu-no-lds-kernel-id")) 136 LDSKernelId = true; 137 } 138 139 if (isEntryFunction()) { 140 // X, XY, and XYZ are the only supported combinations, so make sure Y is 141 // enabled if Z is. 142 if (WorkItemIDZ) 143 WorkItemIDY = true; 144 145 if (!ST.flatScratchIsArchitected()) { 146 PrivateSegmentWaveByteOffset = true; 147 148 // HS and GS always have the scratch wave offset in SGPR5 on GFX9. 149 if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 && 150 (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS)) 151 ArgInfo.PrivateSegmentWaveByteOffset = 152 ArgDescriptor::createRegister(AMDGPU::SGPR5); 153 } 154 } 155 156 Attribute A = F.getFnAttribute("amdgpu-git-ptr-high"); 157 StringRef S = A.getValueAsString(); 158 if (!S.empty()) 159 S.consumeInteger(0, GITPtrHigh); 160 161 A = F.getFnAttribute("amdgpu-32bit-address-high-bits"); 162 S = A.getValueAsString(); 163 if (!S.empty()) 164 S.consumeInteger(0, HighBitsOf32BitAddress); 165 166 MaxMemoryClusterDWords = F.getFnAttributeAsParsedInteger( 167 "amdgpu-max-memory-cluster-dwords", DefaultMemoryClusterDWordsLimit); 168 169 // On GFX908, in order to guarantee copying between AGPRs, we need a scratch 170 // VGPR available at all times. For now, reserve highest available VGPR. After 171 // RA, shift it to the lowest available unused VGPR if the one exist. 172 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) { 173 VGPRForAGPRCopy = 174 AMDGPU::VGPR_32RegClass.getRegister(ST.getMaxNumVGPRs(F) - 1); 175 } 176 } 177 178 MachineFunctionInfo *SIMachineFunctionInfo::clone( 179 BumpPtrAllocator &Allocator, MachineFunction &DestMF, 180 const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB) 181 const { 182 return DestMF.cloneInfo<SIMachineFunctionInfo>(*this); 183 } 184 185 void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) { 186 limitOccupancy(getMaxWavesPerEU()); 187 const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>(); 188 limitOccupancy(ST.getOccupancyWithWorkGroupSizes(MF).second); 189 } 190 191 Register SIMachineFunctionInfo::addPrivateSegmentBuffer( 192 const SIRegisterInfo &TRI) { 193 ArgInfo.PrivateSegmentBuffer = 194 ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 195 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass)); 196 NumUserSGPRs += 4; 197 return ArgInfo.PrivateSegmentBuffer.getRegister(); 198 } 199 200 Register SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) { 201 ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 202 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 203 NumUserSGPRs += 2; 204 return ArgInfo.DispatchPtr.getRegister(); 205 } 206 207 Register SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) { 208 ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 209 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 210 NumUserSGPRs += 2; 211 return ArgInfo.QueuePtr.getRegister(); 212 } 213 214 Register SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) { 215 ArgInfo.KernargSegmentPtr 216 = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 217 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 218 NumUserSGPRs += 2; 219 return ArgInfo.KernargSegmentPtr.getRegister(); 220 } 221 222 Register SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) { 223 ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 224 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 225 NumUserSGPRs += 2; 226 return ArgInfo.DispatchID.getRegister(); 227 } 228 229 Register SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) { 230 ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 231 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 232 NumUserSGPRs += 2; 233 return ArgInfo.FlatScratchInit.getRegister(); 234 } 235 236 Register SIMachineFunctionInfo::addPrivateSegmentSize(const SIRegisterInfo &TRI) { 237 ArgInfo.PrivateSegmentSize = ArgDescriptor::createRegister(getNextUserSGPR()); 238 NumUserSGPRs += 1; 239 return ArgInfo.PrivateSegmentSize.getRegister(); 240 } 241 242 Register SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) { 243 ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 244 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 245 NumUserSGPRs += 2; 246 return ArgInfo.ImplicitBufferPtr.getRegister(); 247 } 248 249 Register SIMachineFunctionInfo::addLDSKernelId() { 250 ArgInfo.LDSKernelId = ArgDescriptor::createRegister(getNextUserSGPR()); 251 NumUserSGPRs += 1; 252 return ArgInfo.LDSKernelId.getRegister(); 253 } 254 255 SmallVectorImpl<MCRegister> *SIMachineFunctionInfo::addPreloadedKernArg( 256 const SIRegisterInfo &TRI, const TargetRegisterClass *RC, 257 unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs) { 258 assert(!ArgInfo.PreloadKernArgs.count(KernArgIdx) && 259 "Preload kernel argument allocated twice."); 260 NumUserSGPRs += PaddingSGPRs; 261 // If the available register tuples are aligned with the kernarg to be 262 // preloaded use that register, otherwise we need to use a set of SGPRs and 263 // merge them. 264 if (!ArgInfo.FirstKernArgPreloadReg) 265 ArgInfo.FirstKernArgPreloadReg = getNextUserSGPR(); 266 Register PreloadReg = 267 TRI.getMatchingSuperReg(getNextUserSGPR(), AMDGPU::sub0, RC); 268 if (PreloadReg && 269 (RC == &AMDGPU::SReg_32RegClass || RC == &AMDGPU::SReg_64RegClass)) { 270 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(PreloadReg); 271 NumUserSGPRs += AllocSizeDWord; 272 } else { 273 for (unsigned I = 0; I < AllocSizeDWord; ++I) { 274 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(getNextUserSGPR()); 275 NumUserSGPRs++; 276 } 277 } 278 279 // Track the actual number of SGPRs that HW will preload to. 280 UserSGPRInfo.allocKernargPreloadSGPRs(AllocSizeDWord + PaddingSGPRs); 281 return &ArgInfo.PreloadKernArgs[KernArgIdx].Regs; 282 } 283 284 void SIMachineFunctionInfo::allocateWWMSpill(MachineFunction &MF, Register VGPR, 285 uint64_t Size, Align Alignment) { 286 // Skip if it is an entry function or the register is already added. 287 if (isEntryFunction() || WWMSpills.count(VGPR)) 288 return; 289 290 // Skip if this is a function with the amdgpu_cs_chain or 291 // amdgpu_cs_chain_preserve calling convention and this is a scratch register. 292 // We never need to allocate a spill for these because we don't even need to 293 // restore the inactive lanes for them (they're scratchier than the usual 294 // scratch registers). We only need to do this if we have calls to 295 // llvm.amdgcn.cs.chain (otherwise there's no one to save them for, since 296 // chain functions do not return) and the function did not contain a call to 297 // llvm.amdgcn.init.whole.wave (since in that case there are no inactive lanes 298 // when entering the function). 299 if (isChainFunction() && 300 (SIRegisterInfo::isChainScratchRegister(VGPR) || 301 !MF.getFrameInfo().hasTailCall() || hasInitWholeWave())) 302 return; 303 304 WWMSpills.insert(std::make_pair( 305 VGPR, MF.getFrameInfo().CreateSpillStackObject(Size, Alignment))); 306 } 307 308 // Separate out the callee-saved and scratch registers. 309 void SIMachineFunctionInfo::splitWWMSpillRegisters( 310 MachineFunction &MF, 311 SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs, 312 SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const { 313 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs(); 314 for (auto &Reg : WWMSpills) { 315 if (isCalleeSavedReg(CSRegs, Reg.first)) 316 CalleeSavedRegs.push_back(Reg); 317 else 318 ScratchRegs.push_back(Reg); 319 } 320 } 321 322 bool SIMachineFunctionInfo::isCalleeSavedReg(const MCPhysReg *CSRegs, 323 MCPhysReg Reg) const { 324 for (unsigned I = 0; CSRegs[I]; ++I) { 325 if (CSRegs[I] == Reg) 326 return true; 327 } 328 329 return false; 330 } 331 332 void SIMachineFunctionInfo::shiftWwmVGPRsToLowestRange( 333 MachineFunction &MF, SmallVectorImpl<Register> &WWMVGPRs, 334 BitVector &SavedVGPRs) { 335 const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo(); 336 MachineRegisterInfo &MRI = MF.getRegInfo(); 337 for (unsigned I = 0, E = WWMVGPRs.size(); I < E; ++I) { 338 Register Reg = WWMVGPRs[I]; 339 Register NewReg = 340 TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF); 341 if (!NewReg || NewReg >= Reg) 342 break; 343 344 MRI.replaceRegWith(Reg, NewReg); 345 346 // Update various tables with the new VGPR. 347 WWMVGPRs[I] = NewReg; 348 WWMReservedRegs.remove(Reg); 349 WWMReservedRegs.insert(NewReg); 350 MRI.reserveReg(NewReg, TRI); 351 352 // Replace the register in SpillPhysVGPRs. This is needed to look for free 353 // lanes while spilling special SGPRs like FP, BP, etc. during PEI. 354 auto *RegItr = std::find(SpillPhysVGPRs.begin(), SpillPhysVGPRs.end(), Reg); 355 if (RegItr != SpillPhysVGPRs.end()) { 356 unsigned Idx = std::distance(SpillPhysVGPRs.begin(), RegItr); 357 SpillPhysVGPRs[Idx] = NewReg; 358 } 359 360 // The generic `determineCalleeSaves` might have set the old register if it 361 // is in the CSR range. 362 SavedVGPRs.reset(Reg); 363 364 for (MachineBasicBlock &MBB : MF) { 365 MBB.removeLiveIn(Reg); 366 MBB.sortUniqueLiveIns(); 367 } 368 369 Reg = NewReg; 370 } 371 } 372 373 bool SIMachineFunctionInfo::allocateVirtualVGPRForSGPRSpills( 374 MachineFunction &MF, int FI, unsigned LaneIndex) { 375 MachineRegisterInfo &MRI = MF.getRegInfo(); 376 Register LaneVGPR; 377 if (!LaneIndex) { 378 LaneVGPR = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 379 SpillVGPRs.push_back(LaneVGPR); 380 } else { 381 LaneVGPR = SpillVGPRs.back(); 382 } 383 384 SGPRSpillsToVirtualVGPRLanes[FI].emplace_back(LaneVGPR, LaneIndex); 385 return true; 386 } 387 388 bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills( 389 MachineFunction &MF, int FI, unsigned LaneIndex, bool IsPrologEpilog) { 390 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 391 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 392 MachineRegisterInfo &MRI = MF.getRegInfo(); 393 Register LaneVGPR; 394 if (!LaneIndex) { 395 // Find the highest available register if called before RA to ensure the 396 // lowest registers are available for allocation. The LaneVGPR, in that 397 // case, will be shifted back to the lowest range after VGPR allocation. 398 LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF, 399 !IsPrologEpilog); 400 if (LaneVGPR == AMDGPU::NoRegister) { 401 // We have no VGPRs left for spilling SGPRs. Reset because we will not 402 // partially spill the SGPR to VGPRs. 403 SGPRSpillsToPhysicalVGPRLanes.erase(FI); 404 return false; 405 } 406 407 if (IsPrologEpilog) 408 allocateWWMSpill(MF, LaneVGPR); 409 410 reserveWWMRegister(LaneVGPR); 411 for (MachineBasicBlock &MBB : MF) { 412 MBB.addLiveIn(LaneVGPR); 413 MBB.sortUniqueLiveIns(); 414 } 415 SpillPhysVGPRs.push_back(LaneVGPR); 416 } else { 417 LaneVGPR = SpillPhysVGPRs.back(); 418 } 419 420 SGPRSpillsToPhysicalVGPRLanes[FI].emplace_back(LaneVGPR, LaneIndex); 421 return true; 422 } 423 424 bool SIMachineFunctionInfo::allocateSGPRSpillToVGPRLane( 425 MachineFunction &MF, int FI, bool SpillToPhysVGPRLane, 426 bool IsPrologEpilog) { 427 std::vector<SIRegisterInfo::SpilledReg> &SpillLanes = 428 SpillToPhysVGPRLane ? SGPRSpillsToPhysicalVGPRLanes[FI] 429 : SGPRSpillsToVirtualVGPRLanes[FI]; 430 431 // This has already been allocated. 432 if (!SpillLanes.empty()) 433 return true; 434 435 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 436 MachineFrameInfo &FrameInfo = MF.getFrameInfo(); 437 unsigned WaveSize = ST.getWavefrontSize(); 438 439 unsigned Size = FrameInfo.getObjectSize(FI); 440 unsigned NumLanes = Size / 4; 441 442 if (NumLanes > WaveSize) 443 return false; 444 445 assert(Size >= 4 && "invalid sgpr spill size"); 446 assert(ST.getRegisterInfo()->spillSGPRToVGPR() && 447 "not spilling SGPRs to VGPRs"); 448 449 unsigned &NumSpillLanes = SpillToPhysVGPRLane ? NumPhysicalVGPRSpillLanes 450 : NumVirtualVGPRSpillLanes; 451 452 for (unsigned I = 0; I < NumLanes; ++I, ++NumSpillLanes) { 453 unsigned LaneIndex = (NumSpillLanes % WaveSize); 454 455 bool Allocated = SpillToPhysVGPRLane 456 ? allocatePhysicalVGPRForSGPRSpills(MF, FI, LaneIndex, 457 IsPrologEpilog) 458 : allocateVirtualVGPRForSGPRSpills(MF, FI, LaneIndex); 459 if (!Allocated) { 460 NumSpillLanes -= I; 461 return false; 462 } 463 } 464 465 return true; 466 } 467 468 /// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI. 469 /// Either AGPR is spilled to VGPR to vice versa. 470 /// Returns true if a \p FI can be eliminated completely. 471 bool SIMachineFunctionInfo::allocateVGPRSpillToAGPR(MachineFunction &MF, 472 int FI, 473 bool isAGPRtoVGPR) { 474 MachineRegisterInfo &MRI = MF.getRegInfo(); 475 MachineFrameInfo &FrameInfo = MF.getFrameInfo(); 476 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 477 478 assert(ST.hasMAIInsts() && FrameInfo.isSpillSlotObjectIndex(FI)); 479 480 auto &Spill = VGPRToAGPRSpills[FI]; 481 482 // This has already been allocated. 483 if (!Spill.Lanes.empty()) 484 return Spill.FullyAllocated; 485 486 unsigned Size = FrameInfo.getObjectSize(FI); 487 unsigned NumLanes = Size / 4; 488 Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister); 489 490 const TargetRegisterClass &RC = 491 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass; 492 auto Regs = RC.getRegisters(); 493 494 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR; 495 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 496 Spill.FullyAllocated = true; 497 498 // FIXME: Move allocation logic out of MachineFunctionInfo and initialize 499 // once. 500 BitVector OtherUsedRegs; 501 OtherUsedRegs.resize(TRI->getNumRegs()); 502 503 const uint32_t *CSRMask = 504 TRI->getCallPreservedMask(MF, MF.getFunction().getCallingConv()); 505 if (CSRMask) 506 OtherUsedRegs.setBitsInMask(CSRMask); 507 508 // TODO: Should include register tuples, but doesn't matter with current 509 // usage. 510 for (MCPhysReg Reg : SpillAGPR) 511 OtherUsedRegs.set(Reg); 512 for (MCPhysReg Reg : SpillVGPR) 513 OtherUsedRegs.set(Reg); 514 515 SmallVectorImpl<MCPhysReg>::const_iterator NextSpillReg = Regs.begin(); 516 for (int I = NumLanes - 1; I >= 0; --I) { 517 NextSpillReg = std::find_if( 518 NextSpillReg, Regs.end(), [&MRI, &OtherUsedRegs](MCPhysReg Reg) { 519 return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) && 520 !OtherUsedRegs[Reg]; 521 }); 522 523 if (NextSpillReg == Regs.end()) { // Registers exhausted 524 Spill.FullyAllocated = false; 525 break; 526 } 527 528 OtherUsedRegs.set(*NextSpillReg); 529 SpillRegs.push_back(*NextSpillReg); 530 MRI.reserveReg(*NextSpillReg, TRI); 531 Spill.Lanes[I] = *NextSpillReg++; 532 } 533 534 return Spill.FullyAllocated; 535 } 536 537 bool SIMachineFunctionInfo::removeDeadFrameIndices( 538 MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs) { 539 // Remove dead frame indices from function frame, however keep FP & BP since 540 // spills for them haven't been inserted yet. And also make sure to remove the 541 // frame indices from `SGPRSpillsToVirtualVGPRLanes` data structure, 542 // otherwise, it could result in an unexpected side effect and bug, in case of 543 // any re-mapping of freed frame indices by later pass(es) like "stack slot 544 // coloring". 545 for (auto &R : make_early_inc_range(SGPRSpillsToVirtualVGPRLanes)) { 546 MFI.RemoveStackObject(R.first); 547 SGPRSpillsToVirtualVGPRLanes.erase(R.first); 548 } 549 550 // Remove the dead frame indices of CSR SGPRs which are spilled to physical 551 // VGPR lanes during SILowerSGPRSpills pass. 552 if (!ResetSGPRSpillStackIDs) { 553 for (auto &R : make_early_inc_range(SGPRSpillsToPhysicalVGPRLanes)) { 554 MFI.RemoveStackObject(R.first); 555 SGPRSpillsToPhysicalVGPRLanes.erase(R.first); 556 } 557 } 558 bool HaveSGPRToMemory = false; 559 560 if (ResetSGPRSpillStackIDs) { 561 // All other SGPRs must be allocated on the default stack, so reset the 562 // stack ID. 563 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); I != E; 564 ++I) { 565 if (!checkIndexInPrologEpilogSGPRSpills(I)) { 566 if (MFI.getStackID(I) == TargetStackID::SGPRSpill) { 567 MFI.setStackID(I, TargetStackID::Default); 568 HaveSGPRToMemory = true; 569 } 570 } 571 } 572 } 573 574 for (auto &R : VGPRToAGPRSpills) { 575 if (R.second.IsDead) 576 MFI.RemoveStackObject(R.first); 577 } 578 579 return HaveSGPRToMemory; 580 } 581 582 int SIMachineFunctionInfo::getScavengeFI(MachineFrameInfo &MFI, 583 const SIRegisterInfo &TRI) { 584 if (ScavengeFI) 585 return *ScavengeFI; 586 587 ScavengeFI = 588 MFI.CreateStackObject(TRI.getSpillSize(AMDGPU::SGPR_32RegClass), 589 TRI.getSpillAlign(AMDGPU::SGPR_32RegClass), false); 590 return *ScavengeFI; 591 } 592 593 MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const { 594 assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs"); 595 return AMDGPU::SGPR0 + NumUserSGPRs; 596 } 597 598 MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const { 599 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs; 600 } 601 602 void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(Register Reg) { 603 VRegFlags.grow(Reg); 604 } 605 606 void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(Register NewReg, 607 Register SrcReg) { 608 VRegFlags.grow(NewReg); 609 VRegFlags[NewReg] = VRegFlags[SrcReg]; 610 } 611 612 Register 613 SIMachineFunctionInfo::getGITPtrLoReg(const MachineFunction &MF) const { 614 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 615 if (!ST.isAmdPalOS()) 616 return Register(); 617 Register GitPtrLo = AMDGPU::SGPR0; // Low GIT address passed in 618 if (ST.hasMergedShaders()) { 619 switch (MF.getFunction().getCallingConv()) { 620 case CallingConv::AMDGPU_HS: 621 case CallingConv::AMDGPU_GS: 622 // Low GIT address is passed in s8 rather than s0 for an LS+HS or 623 // ES+GS merged shader on gfx9+. 624 GitPtrLo = AMDGPU::SGPR8; 625 return GitPtrLo; 626 default: 627 return GitPtrLo; 628 } 629 } 630 return GitPtrLo; 631 } 632 633 static yaml::StringValue regToString(Register Reg, 634 const TargetRegisterInfo &TRI) { 635 yaml::StringValue Dest; 636 { 637 raw_string_ostream OS(Dest.Value); 638 OS << printReg(Reg, &TRI); 639 } 640 return Dest; 641 } 642 643 static std::optional<yaml::SIArgumentInfo> 644 convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo, 645 const TargetRegisterInfo &TRI) { 646 yaml::SIArgumentInfo AI; 647 648 auto convertArg = [&](std::optional<yaml::SIArgument> &A, 649 const ArgDescriptor &Arg) { 650 if (!Arg) 651 return false; 652 653 // Create a register or stack argument. 654 yaml::SIArgument SA = yaml::SIArgument::createArgument(Arg.isRegister()); 655 if (Arg.isRegister()) { 656 raw_string_ostream OS(SA.RegisterName.Value); 657 OS << printReg(Arg.getRegister(), &TRI); 658 } else 659 SA.StackOffset = Arg.getStackOffset(); 660 // Check and update the optional mask. 661 if (Arg.isMasked()) 662 SA.Mask = Arg.getMask(); 663 664 A = SA; 665 return true; 666 }; 667 668 // TODO: Need to serialize kernarg preloads. 669 bool Any = false; 670 Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer); 671 Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr); 672 Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr); 673 Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr); 674 Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID); 675 Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit); 676 Any |= convertArg(AI.LDSKernelId, ArgInfo.LDSKernelId); 677 Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize); 678 Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX); 679 Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY); 680 Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ); 681 Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo); 682 Any |= convertArg(AI.PrivateSegmentWaveByteOffset, 683 ArgInfo.PrivateSegmentWaveByteOffset); 684 Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr); 685 Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr); 686 Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX); 687 Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY); 688 Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ); 689 690 if (Any) 691 return AI; 692 693 return std::nullopt; 694 } 695 696 yaml::SIMachineFunctionInfo::SIMachineFunctionInfo( 697 const llvm::SIMachineFunctionInfo &MFI, const TargetRegisterInfo &TRI, 698 const llvm::MachineFunction &MF) 699 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()), 700 MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()), 701 GDSSize(MFI.getGDSSize()), DynLDSAlign(MFI.getDynLDSAlign()), 702 IsEntryFunction(MFI.isEntryFunction()), 703 NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()), 704 MemoryBound(MFI.isMemoryBound()), WaveLimiter(MFI.needsWaveLimiter()), 705 HasSpilledSGPRs(MFI.hasSpilledSGPRs()), 706 HasSpilledVGPRs(MFI.hasSpilledVGPRs()), 707 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()), 708 Occupancy(MFI.getOccupancy()), 709 ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)), 710 FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)), 711 StackPtrOffsetReg(regToString(MFI.getStackPtrOffsetReg(), TRI)), 712 BytesInStackArgArea(MFI.getBytesInStackArgArea()), 713 ReturnsVoid(MFI.returnsVoid()), 714 ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)), 715 PSInputAddr(MFI.getPSInputAddr()), PSInputEnable(MFI.getPSInputEnable()), 716 MaxMemoryClusterDWords(MFI.getMaxMemoryClusterDWords()), 717 Mode(MFI.getMode()), HasInitWholeWave(MFI.hasInitWholeWave()) { 718 for (Register Reg : MFI.getSGPRSpillPhysVGPRs()) 719 SpillPhysVGPRS.push_back(regToString(Reg, TRI)); 720 721 for (Register Reg : MFI.getWWMReservedRegs()) 722 WWMReservedRegs.push_back(regToString(Reg, TRI)); 723 724 if (MFI.getLongBranchReservedReg()) 725 LongBranchReservedReg = regToString(MFI.getLongBranchReservedReg(), TRI); 726 if (MFI.getVGPRForAGPRCopy()) 727 VGPRForAGPRCopy = regToString(MFI.getVGPRForAGPRCopy(), TRI); 728 729 if (MFI.getSGPRForEXECCopy()) 730 SGPRForEXECCopy = regToString(MFI.getSGPRForEXECCopy(), TRI); 731 732 auto SFI = MFI.getOptionalScavengeFI(); 733 if (SFI) 734 ScavengeFI = yaml::FrameIndex(*SFI, MF.getFrameInfo()); 735 } 736 737 void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) { 738 MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, *this); 739 } 740 741 bool SIMachineFunctionInfo::initializeBaseYamlFields( 742 const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF, 743 PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) { 744 ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize; 745 MaxKernArgAlign = YamlMFI.MaxKernArgAlign; 746 LDSSize = YamlMFI.LDSSize; 747 GDSSize = YamlMFI.GDSSize; 748 DynLDSAlign = YamlMFI.DynLDSAlign; 749 PSInputAddr = YamlMFI.PSInputAddr; 750 PSInputEnable = YamlMFI.PSInputEnable; 751 MaxMemoryClusterDWords = YamlMFI.MaxMemoryClusterDWords; 752 HighBitsOf32BitAddress = YamlMFI.HighBitsOf32BitAddress; 753 Occupancy = YamlMFI.Occupancy; 754 IsEntryFunction = YamlMFI.IsEntryFunction; 755 NoSignedZerosFPMath = YamlMFI.NoSignedZerosFPMath; 756 MemoryBound = YamlMFI.MemoryBound; 757 WaveLimiter = YamlMFI.WaveLimiter; 758 HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs; 759 HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs; 760 BytesInStackArgArea = YamlMFI.BytesInStackArgArea; 761 ReturnsVoid = YamlMFI.ReturnsVoid; 762 763 if (YamlMFI.ScavengeFI) { 764 auto FIOrErr = YamlMFI.ScavengeFI->getFI(MF.getFrameInfo()); 765 if (!FIOrErr) { 766 // Create a diagnostic for a the frame index. 767 const MemoryBuffer &Buffer = 768 *PFS.SM->getMemoryBuffer(PFS.SM->getMainFileID()); 769 770 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1, 1, 771 SourceMgr::DK_Error, toString(FIOrErr.takeError()), 772 "", {}, {}); 773 SourceRange = YamlMFI.ScavengeFI->SourceRange; 774 return true; 775 } 776 ScavengeFI = *FIOrErr; 777 } else { 778 ScavengeFI = std::nullopt; 779 } 780 return false; 781 } 782 783 bool SIMachineFunctionInfo::mayUseAGPRs(const Function &F) const { 784 return !F.hasFnAttribute("amdgpu-no-agpr"); 785 } 786 787 bool SIMachineFunctionInfo::usesAGPRs(const MachineFunction &MF) const { 788 if (UsesAGPRs) 789 return *UsesAGPRs; 790 791 if (!mayNeedAGPRs()) { 792 UsesAGPRs = false; 793 return false; 794 } 795 796 if (!AMDGPU::isEntryFunctionCC(MF.getFunction().getCallingConv()) || 797 MF.getFrameInfo().hasCalls()) { 798 UsesAGPRs = true; 799 return true; 800 } 801 802 const MachineRegisterInfo &MRI = MF.getRegInfo(); 803 804 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 805 const Register Reg = Register::index2VirtReg(I); 806 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg); 807 if (RC && SIRegisterInfo::isAGPRClass(RC)) { 808 UsesAGPRs = true; 809 return true; 810 } 811 if (!RC && !MRI.use_empty(Reg) && MRI.getType(Reg).isValid()) { 812 // Defer caching UsesAGPRs, function might not yet been regbank selected. 813 return true; 814 } 815 } 816 817 for (MCRegister Reg : AMDGPU::AGPR_32RegClass) { 818 if (MRI.isPhysRegUsed(Reg)) { 819 UsesAGPRs = true; 820 return true; 821 } 822 } 823 824 UsesAGPRs = false; 825 return false; 826 } 827