1 //===- AMDGPUReleaseVGPRs.cpp - Automatically release vgprs on GFX11+ -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// Insert S_SENDMSG instructions to release vgprs on GFX11+. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPU.h" 15 #include "AMDGPUSubtarget.h" 16 #include "GCNSubtarget.h" 17 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 18 #include "SIDefines.h" 19 #include "llvm/ADT/DepthFirstIterator.h" 20 #include "llvm/CodeGen/MachineBasicBlock.h" 21 #include "llvm/CodeGen/MachineOperand.h" 22 #include <optional> 23 using namespace llvm; 24 25 #define DEBUG_TYPE "release-vgprs" 26 27 namespace { 28 29 class AMDGPUReleaseVGPRs : public MachineFunctionPass { 30 public: 31 static char ID; 32 AMDGPUReleaseVGPRs()33 AMDGPUReleaseVGPRs() : MachineFunctionPass(ID) {} 34 getAnalysisUsage(AnalysisUsage & AU) const35 void getAnalysisUsage(AnalysisUsage &AU) const override { 36 AU.setPreservesAll(); 37 MachineFunctionPass::getAnalysisUsage(AU); 38 } 39 40 // Track if the last instruction referencing a vgpr in a MBB is a VMEM 41 // store. Because this pass is late in the pipeline, it is expected that the 42 // last vgpr use will likely be one of vmem store, ds, exp. 43 // Loads and others vgpr operations would have been 44 // deleted by this point, except for complex control flow involving loops. 45 // This is why we are just testing the type of instructions rather 46 // than the operands. 47 class LastVGPRUseIsVMEMStore { 48 BitVector BlockVMEMStore; 49 50 static std::optional<bool> lastVGPRUseIsStore(const MachineBasicBlock & MBB)51 lastVGPRUseIsStore(const MachineBasicBlock &MBB) { 52 for (auto &MI : reverse(MBB.instrs())) { 53 // If it's a VMEM store, a VGPR will be used, return true. 54 if ((SIInstrInfo::isVMEM(MI) || SIInstrInfo::isFLAT(MI)) && 55 MI.mayStore()) 56 return true; 57 58 // If it's referencing a VGPR but is not a VMEM store, return false. 59 if (SIInstrInfo::isDS(MI) || SIInstrInfo::isEXP(MI) || 60 SIInstrInfo::isVMEM(MI) || SIInstrInfo::isFLAT(MI) || 61 SIInstrInfo::isVALU(MI)) 62 return false; 63 } 64 // Wait until the values are propagated from the predecessors 65 return std::nullopt; 66 } 67 68 public: LastVGPRUseIsVMEMStore(const MachineFunction & MF)69 LastVGPRUseIsVMEMStore(const MachineFunction &MF) 70 : BlockVMEMStore(MF.getNumBlockIDs()) { 71 72 df_iterator_default_set<const MachineBasicBlock *> Visited; 73 SmallVector<const MachineBasicBlock *> EndWithVMEMStoreBlocks; 74 75 for (const auto &MBB : MF) { 76 auto LastUseIsStore = lastVGPRUseIsStore(MBB); 77 if (!LastUseIsStore.has_value()) 78 continue; 79 80 if (*LastUseIsStore) { 81 EndWithVMEMStoreBlocks.push_back(&MBB); 82 } else { 83 Visited.insert(&MBB); 84 } 85 } 86 87 for (const auto *MBB : EndWithVMEMStoreBlocks) { 88 for (const auto *Succ : depth_first_ext(MBB, Visited)) { 89 BlockVMEMStore[Succ->getNumber()] = true; 90 } 91 } 92 } 93 94 // Return true if the last instruction referencing a vgpr in this MBB 95 // is a VMEM store, otherwise return false. isLastVGPRUseVMEMStore(const MachineBasicBlock & MBB) const96 bool isLastVGPRUseVMEMStore(const MachineBasicBlock &MBB) const { 97 return BlockVMEMStore[MBB.getNumber()]; 98 } 99 }; 100 101 static bool runOnMachineBasicBlock(MachineBasicBlock & MBB,const SIInstrInfo * SII,const LastVGPRUseIsVMEMStore & BlockVMEMStore)102 runOnMachineBasicBlock(MachineBasicBlock &MBB, const SIInstrInfo *SII, 103 const LastVGPRUseIsVMEMStore &BlockVMEMStore) { 104 105 bool Changed = false; 106 107 for (auto &MI : MBB.terminators()) { 108 // Look for S_ENDPGM instructions 109 if (MI.getOpcode() == AMDGPU::S_ENDPGM || 110 MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) { 111 // If the last instruction using a VGPR in the block is a VMEM store, 112 // release VGPRs. The VGPRs release will be placed just before ending 113 // the program 114 if (BlockVMEMStore.isLastVGPRUseVMEMStore(MBB)) { 115 BuildMI(MBB, MI, DebugLoc(), SII->get(AMDGPU::S_SENDMSG)) 116 .addImm(AMDGPU::SendMsg::ID_DEALLOC_VGPRS_GFX11Plus); 117 Changed = true; 118 } 119 } 120 } 121 122 return Changed; 123 } 124 runOnMachineFunction(MachineFunction & MF)125 bool runOnMachineFunction(MachineFunction &MF) override { 126 Function &F = MF.getFunction(); 127 if (skipFunction(F) || !AMDGPU::isEntryFunctionCC(F.getCallingConv())) 128 return false; 129 130 // This pass only runs on GFX11+ 131 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 132 if (ST.getGeneration() < AMDGPUSubtarget::GFX11) 133 return false; 134 135 LLVM_DEBUG(dbgs() << "AMDGPUReleaseVGPRs running on " << MF.getName() 136 << "\n"); 137 138 const SIInstrInfo *SII = ST.getInstrInfo(); 139 LastVGPRUseIsVMEMStore BlockVMEMStore(MF); 140 141 bool Changed = false; 142 for (auto &MBB : MF) { 143 Changed |= runOnMachineBasicBlock(MBB, SII, BlockVMEMStore); 144 } 145 146 return Changed; 147 } 148 }; 149 150 } // namespace 151 152 char AMDGPUReleaseVGPRs::ID = 0; 153 154 char &llvm::AMDGPUReleaseVGPRsID = AMDGPUReleaseVGPRs::ID; 155 156 INITIALIZE_PASS(AMDGPUReleaseVGPRs, DEBUG_TYPE, "Release VGPRs", false, false) 157