1 //===- SIPreAllocateWWMRegs.cpp - WWM Register Pre-allocation -------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// Pass to pre-allocated WWM registers 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPU.h" 15 #include "AMDGPUSubtarget.h" 16 #include "SIInstrInfo.h" 17 #include "SIRegisterInfo.h" 18 #include "SIMachineFunctionInfo.h" 19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 20 #include "llvm/ADT/PostOrderIterator.h" 21 #include "llvm/CodeGen/VirtRegMap.h" 22 #include "llvm/CodeGen/LiveInterval.h" 23 #include "llvm/CodeGen/LiveIntervals.h" 24 #include "llvm/CodeGen/LiveRegMatrix.h" 25 #include "llvm/CodeGen/MachineDominators.h" 26 #include "llvm/CodeGen/MachineFunctionPass.h" 27 #include "llvm/CodeGen/RegisterClassInfo.h" 28 29 using namespace llvm; 30 31 #define DEBUG_TYPE "si-pre-allocate-wwm-regs" 32 33 namespace { 34 35 class SIPreAllocateWWMRegs : public MachineFunctionPass { 36 private: 37 const SIInstrInfo *TII; 38 const SIRegisterInfo *TRI; 39 MachineRegisterInfo *MRI; 40 LiveIntervals *LIS; 41 LiveRegMatrix *Matrix; 42 VirtRegMap *VRM; 43 RegisterClassInfo RegClassInfo; 44 45 std::vector<unsigned> RegsToRewrite; 46 47 public: 48 static char ID; 49 50 SIPreAllocateWWMRegs() : MachineFunctionPass(ID) { 51 initializeSIPreAllocateWWMRegsPass(*PassRegistry::getPassRegistry()); 52 } 53 54 bool runOnMachineFunction(MachineFunction &MF) override; 55 56 void getAnalysisUsage(AnalysisUsage &AU) const override { 57 AU.addRequired<LiveIntervals>(); 58 AU.addPreserved<LiveIntervals>(); 59 AU.addRequired<VirtRegMap>(); 60 AU.addRequired<LiveRegMatrix>(); 61 AU.addPreserved<SlotIndexes>(); 62 AU.setPreservesCFG(); 63 MachineFunctionPass::getAnalysisUsage(AU); 64 } 65 66 private: 67 bool processDef(MachineOperand &MO); 68 void rewriteRegs(MachineFunction &MF); 69 }; 70 71 } // End anonymous namespace. 72 73 INITIALIZE_PASS_BEGIN(SIPreAllocateWWMRegs, DEBUG_TYPE, 74 "SI Pre-allocate WWM Registers", false, false) 75 INITIALIZE_PASS_DEPENDENCY(LiveIntervals) 76 INITIALIZE_PASS_DEPENDENCY(VirtRegMap) 77 INITIALIZE_PASS_DEPENDENCY(LiveRegMatrix) 78 INITIALIZE_PASS_END(SIPreAllocateWWMRegs, DEBUG_TYPE, 79 "SI Pre-allocate WWM Registers", false, false) 80 81 char SIPreAllocateWWMRegs::ID = 0; 82 83 char &llvm::SIPreAllocateWWMRegsID = SIPreAllocateWWMRegs::ID; 84 85 FunctionPass *llvm::createSIPreAllocateWWMRegsPass() { 86 return new SIPreAllocateWWMRegs(); 87 } 88 89 bool SIPreAllocateWWMRegs::processDef(MachineOperand &MO) { 90 if (!MO.isReg()) 91 return false; 92 93 Register Reg = MO.getReg(); 94 95 if (!TRI->isVGPR(*MRI, Reg)) 96 return false; 97 98 if (Register::isPhysicalRegister(Reg)) 99 return false; 100 101 if (VRM->hasPhys(Reg)) 102 return false; 103 104 LiveInterval &LI = LIS->getInterval(Reg); 105 106 for (unsigned PhysReg : RegClassInfo.getOrder(MRI->getRegClass(Reg))) { 107 if (!MRI->isPhysRegUsed(PhysReg) && 108 Matrix->checkInterference(LI, PhysReg) == LiveRegMatrix::IK_Free) { 109 Matrix->assign(LI, PhysReg); 110 assert(PhysReg != 0); 111 RegsToRewrite.push_back(Reg); 112 return true; 113 } 114 } 115 116 llvm_unreachable("physreg not found for WWM expression"); 117 return false; 118 } 119 120 void SIPreAllocateWWMRegs::rewriteRegs(MachineFunction &MF) { 121 for (MachineBasicBlock &MBB : MF) { 122 for (MachineInstr &MI : MBB) { 123 for (MachineOperand &MO : MI.operands()) { 124 if (!MO.isReg()) 125 continue; 126 127 const Register VirtReg = MO.getReg(); 128 if (Register::isPhysicalRegister(VirtReg)) 129 continue; 130 131 if (!VRM->hasPhys(VirtReg)) 132 continue; 133 134 Register PhysReg = VRM->getPhys(VirtReg); 135 const unsigned SubReg = MO.getSubReg(); 136 if (SubReg != 0) { 137 PhysReg = TRI->getSubReg(PhysReg, SubReg); 138 MO.setSubReg(0); 139 } 140 141 MO.setReg(PhysReg); 142 MO.setIsRenamable(false); 143 } 144 } 145 } 146 147 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 148 149 for (unsigned Reg : RegsToRewrite) { 150 LIS->removeInterval(Reg); 151 152 const Register PhysReg = VRM->getPhys(Reg); 153 assert(PhysReg != 0); 154 MFI->ReserveWWMRegister(PhysReg); 155 } 156 157 RegsToRewrite.clear(); 158 159 // Update the set of reserved registers to include WWM ones. 160 MRI->freezeReservedRegs(MF); 161 } 162 163 bool SIPreAllocateWWMRegs::runOnMachineFunction(MachineFunction &MF) { 164 LLVM_DEBUG(dbgs() << "SIPreAllocateWWMRegs: function " << MF.getName() << "\n"); 165 166 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 167 168 TII = ST.getInstrInfo(); 169 TRI = &TII->getRegisterInfo(); 170 MRI = &MF.getRegInfo(); 171 172 LIS = &getAnalysis<LiveIntervals>(); 173 Matrix = &getAnalysis<LiveRegMatrix>(); 174 VRM = &getAnalysis<VirtRegMap>(); 175 176 RegClassInfo.runOnMachineFunction(MF); 177 178 bool RegsAssigned = false; 179 180 // We use a reverse post-order traversal of the control-flow graph to 181 // guarantee that we visit definitions in dominance order. Since WWM 182 // expressions are guaranteed to never involve phi nodes, and we can only 183 // escape WWM through the special WWM instruction, this means that this is a 184 // perfect elimination order, so we can never do any better. 185 ReversePostOrderTraversal<MachineFunction*> RPOT(&MF); 186 187 for (MachineBasicBlock *MBB : RPOT) { 188 bool InWWM = false; 189 for (MachineInstr &MI : *MBB) { 190 if (MI.getOpcode() == AMDGPU::V_SET_INACTIVE_B32 || 191 MI.getOpcode() == AMDGPU::V_SET_INACTIVE_B64) 192 RegsAssigned |= processDef(MI.getOperand(0)); 193 194 if (MI.getOpcode() == AMDGPU::ENTER_WWM) { 195 LLVM_DEBUG(dbgs() << "entering WWM region: " << MI << "\n"); 196 InWWM = true; 197 continue; 198 } 199 200 if (MI.getOpcode() == AMDGPU::EXIT_WWM) { 201 LLVM_DEBUG(dbgs() << "exiting WWM region: " << MI << "\n"); 202 InWWM = false; 203 } 204 205 if (!InWWM) 206 continue; 207 208 LLVM_DEBUG(dbgs() << "processing " << MI << "\n"); 209 210 for (MachineOperand &DefOpnd : MI.defs()) { 211 RegsAssigned |= processDef(DefOpnd); 212 } 213 } 214 } 215 216 if (!RegsAssigned) 217 return false; 218 219 rewriteRegs(MF); 220 return true; 221 } 222