1 //===-- SIOptimizeExecMaskingPreRA.cpp ------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This pass performs exec mask handling peephole optimizations which needs 11 /// to be done before register allocation to reduce register pressure. 12 /// 13 //===----------------------------------------------------------------------===// 14 15 #include "AMDGPU.h" 16 #include "AMDGPUSubtarget.h" 17 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 18 #include "SIInstrInfo.h" 19 #include "llvm/CodeGen/LiveIntervals.h" 20 #include "llvm/CodeGen/MachineFunctionPass.h" 21 #include "llvm/InitializePasses.h" 22 23 using namespace llvm; 24 25 #define DEBUG_TYPE "si-optimize-exec-masking-pre-ra" 26 27 namespace { 28 29 class SIOptimizeExecMaskingPreRA : public MachineFunctionPass { 30 private: 31 const SIRegisterInfo *TRI; 32 const SIInstrInfo *TII; 33 MachineRegisterInfo *MRI; 34 35 public: 36 static char ID; 37 38 SIOptimizeExecMaskingPreRA() : MachineFunctionPass(ID) { 39 initializeSIOptimizeExecMaskingPreRAPass(*PassRegistry::getPassRegistry()); 40 } 41 42 bool runOnMachineFunction(MachineFunction &MF) override; 43 44 StringRef getPassName() const override { 45 return "SI optimize exec mask operations pre-RA"; 46 } 47 48 void getAnalysisUsage(AnalysisUsage &AU) const override { 49 AU.addRequired<LiveIntervals>(); 50 AU.setPreservesAll(); 51 MachineFunctionPass::getAnalysisUsage(AU); 52 } 53 }; 54 55 } // End anonymous namespace. 56 57 INITIALIZE_PASS_BEGIN(SIOptimizeExecMaskingPreRA, DEBUG_TYPE, 58 "SI optimize exec mask operations pre-RA", false, false) 59 INITIALIZE_PASS_DEPENDENCY(LiveIntervals) 60 INITIALIZE_PASS_END(SIOptimizeExecMaskingPreRA, DEBUG_TYPE, 61 "SI optimize exec mask operations pre-RA", false, false) 62 63 char SIOptimizeExecMaskingPreRA::ID = 0; 64 65 char &llvm::SIOptimizeExecMaskingPreRAID = SIOptimizeExecMaskingPreRA::ID; 66 67 FunctionPass *llvm::createSIOptimizeExecMaskingPreRAPass() { 68 return new SIOptimizeExecMaskingPreRA(); 69 } 70 71 static bool isFullExecCopy(const MachineInstr& MI, const GCNSubtarget& ST) { 72 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 73 74 if (MI.isCopy() && MI.getOperand(1).getReg() == Exec) { 75 assert(MI.isFullCopy()); 76 return true; 77 } 78 79 return false; 80 } 81 82 // Optimize sequence 83 // %sel = V_CNDMASK_B32_e64 0, 1, %cc 84 // %cmp = V_CMP_NE_U32 1, %1 85 // $vcc = S_AND_B64 $exec, %cmp 86 // S_CBRANCH_VCC[N]Z 87 // => 88 // $vcc = S_ANDN2_B64 $exec, %cc 89 // S_CBRANCH_VCC[N]Z 90 // 91 // It is the negation pattern inserted by DAGCombiner::visitBRCOND() in the 92 // rebuildSetCC(). We start with S_CBRANCH to avoid exhaustive search, but 93 // only 3 first instructions are really needed. S_AND_B64 with exec is a 94 // required part of the pattern since V_CNDMASK_B32 writes zeroes for inactive 95 // lanes. 96 // 97 // Returns %cc register on success. 98 static unsigned optimizeVcndVcmpPair(MachineBasicBlock &MBB, 99 const GCNSubtarget &ST, 100 MachineRegisterInfo &MRI, 101 LiveIntervals *LIS) { 102 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 103 const SIInstrInfo *TII = ST.getInstrInfo(); 104 bool Wave32 = ST.isWave32(); 105 const unsigned AndOpc = Wave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; 106 const unsigned Andn2Opc = Wave32 ? AMDGPU::S_ANDN2_B32 : AMDGPU::S_ANDN2_B64; 107 const unsigned CondReg = Wave32 ? AMDGPU::VCC_LO : AMDGPU::VCC; 108 const unsigned ExecReg = Wave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 109 110 auto I = llvm::find_if(MBB.terminators(), [](const MachineInstr &MI) { 111 unsigned Opc = MI.getOpcode(); 112 return Opc == AMDGPU::S_CBRANCH_VCCZ || 113 Opc == AMDGPU::S_CBRANCH_VCCNZ; }); 114 if (I == MBB.terminators().end()) 115 return AMDGPU::NoRegister; 116 117 auto *And = TRI->findReachingDef(CondReg, AMDGPU::NoSubRegister, 118 *I, MRI, LIS); 119 if (!And || And->getOpcode() != AndOpc || 120 !And->getOperand(1).isReg() || !And->getOperand(2).isReg()) 121 return AMDGPU::NoRegister; 122 123 MachineOperand *AndCC = &And->getOperand(1); 124 Register CmpReg = AndCC->getReg(); 125 unsigned CmpSubReg = AndCC->getSubReg(); 126 if (CmpReg == ExecReg) { 127 AndCC = &And->getOperand(2); 128 CmpReg = AndCC->getReg(); 129 CmpSubReg = AndCC->getSubReg(); 130 } else if (And->getOperand(2).getReg() != ExecReg) { 131 return AMDGPU::NoRegister; 132 } 133 134 auto *Cmp = TRI->findReachingDef(CmpReg, CmpSubReg, *And, MRI, LIS); 135 if (!Cmp || !(Cmp->getOpcode() == AMDGPU::V_CMP_NE_U32_e32 || 136 Cmp->getOpcode() == AMDGPU::V_CMP_NE_U32_e64) || 137 Cmp->getParent() != And->getParent()) 138 return AMDGPU::NoRegister; 139 140 MachineOperand *Op1 = TII->getNamedOperand(*Cmp, AMDGPU::OpName::src0); 141 MachineOperand *Op2 = TII->getNamedOperand(*Cmp, AMDGPU::OpName::src1); 142 if (Op1->isImm() && Op2->isReg()) 143 std::swap(Op1, Op2); 144 if (!Op1->isReg() || !Op2->isImm() || Op2->getImm() != 1) 145 return AMDGPU::NoRegister; 146 147 Register SelReg = Op1->getReg(); 148 auto *Sel = TRI->findReachingDef(SelReg, Op1->getSubReg(), *Cmp, MRI, LIS); 149 if (!Sel || Sel->getOpcode() != AMDGPU::V_CNDMASK_B32_e64) 150 return AMDGPU::NoRegister; 151 152 if (TII->hasModifiersSet(*Sel, AMDGPU::OpName::src0_modifiers) || 153 TII->hasModifiersSet(*Sel, AMDGPU::OpName::src1_modifiers)) 154 return AMDGPU::NoRegister; 155 156 Op1 = TII->getNamedOperand(*Sel, AMDGPU::OpName::src0); 157 Op2 = TII->getNamedOperand(*Sel, AMDGPU::OpName::src1); 158 MachineOperand *CC = TII->getNamedOperand(*Sel, AMDGPU::OpName::src2); 159 if (!Op1->isImm() || !Op2->isImm() || !CC->isReg() || 160 Op1->getImm() != 0 || Op2->getImm() != 1) 161 return AMDGPU::NoRegister; 162 163 LLVM_DEBUG(dbgs() << "Folding sequence:\n\t" << *Sel << '\t' << *Cmp << '\t' 164 << *And); 165 166 Register CCReg = CC->getReg(); 167 LIS->RemoveMachineInstrFromMaps(*And); 168 MachineInstr *Andn2 = 169 BuildMI(MBB, *And, And->getDebugLoc(), TII->get(Andn2Opc), 170 And->getOperand(0).getReg()) 171 .addReg(ExecReg) 172 .addReg(CCReg, getUndefRegState(CC->isUndef()), CC->getSubReg()); 173 And->eraseFromParent(); 174 LIS->InsertMachineInstrInMaps(*Andn2); 175 176 LLVM_DEBUG(dbgs() << "=>\n\t" << *Andn2 << '\n'); 177 178 // Try to remove compare. Cmp value should not used in between of cmp 179 // and s_and_b64 if VCC or just unused if any other register. 180 if ((Register::isVirtualRegister(CmpReg) && MRI.use_nodbg_empty(CmpReg)) || 181 (CmpReg == CondReg && 182 std::none_of(std::next(Cmp->getIterator()), Andn2->getIterator(), 183 [&](const MachineInstr &MI) { 184 return MI.readsRegister(CondReg, TRI); 185 }))) { 186 LLVM_DEBUG(dbgs() << "Erasing: " << *Cmp << '\n'); 187 188 LIS->RemoveMachineInstrFromMaps(*Cmp); 189 Cmp->eraseFromParent(); 190 191 // Try to remove v_cndmask_b32. 192 if (Register::isVirtualRegister(SelReg) && MRI.use_nodbg_empty(SelReg)) { 193 LLVM_DEBUG(dbgs() << "Erasing: " << *Sel << '\n'); 194 195 LIS->RemoveMachineInstrFromMaps(*Sel); 196 Sel->eraseFromParent(); 197 } 198 } 199 200 return CCReg; 201 } 202 203 bool SIOptimizeExecMaskingPreRA::runOnMachineFunction(MachineFunction &MF) { 204 if (skipFunction(MF.getFunction())) 205 return false; 206 207 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 208 TRI = ST.getRegisterInfo(); 209 TII = ST.getInstrInfo(); 210 MRI = &MF.getRegInfo(); 211 212 MachineRegisterInfo &MRI = MF.getRegInfo(); 213 LiveIntervals *LIS = &getAnalysis<LiveIntervals>(); 214 DenseSet<unsigned> RecalcRegs({AMDGPU::EXEC_LO, AMDGPU::EXEC_HI}); 215 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 216 bool Changed = false; 217 218 for (MachineBasicBlock &MBB : MF) { 219 220 if (unsigned Reg = optimizeVcndVcmpPair(MBB, ST, MRI, LIS)) { 221 RecalcRegs.insert(Reg); 222 RecalcRegs.insert(AMDGPU::VCC_LO); 223 RecalcRegs.insert(AMDGPU::VCC_HI); 224 RecalcRegs.insert(AMDGPU::SCC); 225 Changed = true; 226 } 227 228 // Try to remove unneeded instructions before s_endpgm. 229 if (MBB.succ_empty()) { 230 if (MBB.empty()) 231 continue; 232 233 // Skip this if the endpgm has any implicit uses, otherwise we would need 234 // to be careful to update / remove them. 235 // S_ENDPGM always has a single imm operand that is not used other than to 236 // end up in the encoding 237 MachineInstr &Term = MBB.back(); 238 if (Term.getOpcode() != AMDGPU::S_ENDPGM || Term.getNumOperands() != 1) 239 continue; 240 241 SmallVector<MachineBasicBlock*, 4> Blocks({&MBB}); 242 243 while (!Blocks.empty()) { 244 auto CurBB = Blocks.pop_back_val(); 245 auto I = CurBB->rbegin(), E = CurBB->rend(); 246 if (I != E) { 247 if (I->isUnconditionalBranch() || I->getOpcode() == AMDGPU::S_ENDPGM) 248 ++I; 249 else if (I->isBranch()) 250 continue; 251 } 252 253 while (I != E) { 254 if (I->isDebugInstr()) { 255 I = std::next(I); 256 continue; 257 } 258 259 if (I->mayStore() || I->isBarrier() || I->isCall() || 260 I->hasUnmodeledSideEffects() || I->hasOrderedMemoryRef()) 261 break; 262 263 LLVM_DEBUG(dbgs() 264 << "Removing no effect instruction: " << *I << '\n'); 265 266 for (auto &Op : I->operands()) { 267 if (Op.isReg()) 268 RecalcRegs.insert(Op.getReg()); 269 } 270 271 auto Next = std::next(I); 272 LIS->RemoveMachineInstrFromMaps(*I); 273 I->eraseFromParent(); 274 I = Next; 275 276 Changed = true; 277 } 278 279 if (I != E) 280 continue; 281 282 // Try to ascend predecessors. 283 for (auto *Pred : CurBB->predecessors()) { 284 if (Pred->succ_size() == 1) 285 Blocks.push_back(Pred); 286 } 287 } 288 continue; 289 } 290 291 // If the only user of a logical operation is move to exec, fold it now 292 // to prevent forming of saveexec. I.e: 293 // 294 // %0:sreg_64 = COPY $exec 295 // %1:sreg_64 = S_AND_B64 %0:sreg_64, %2:sreg_64 296 // => 297 // %1 = S_AND_B64 $exec, %2:sreg_64 298 unsigned ScanThreshold = 10; 299 for (auto I = MBB.rbegin(), E = MBB.rend(); I != E 300 && ScanThreshold--; ++I) { 301 if (!isFullExecCopy(*I, ST)) 302 continue; 303 304 Register SavedExec = I->getOperand(0).getReg(); 305 if (SavedExec.isVirtual() && MRI.hasOneNonDBGUse(SavedExec) && 306 MRI.use_instr_nodbg_begin(SavedExec)->getParent() == I->getParent()) { 307 LLVM_DEBUG(dbgs() << "Redundant EXEC COPY: " << *I << '\n'); 308 LIS->RemoveMachineInstrFromMaps(*I); 309 I->eraseFromParent(); 310 MRI.replaceRegWith(SavedExec, Exec); 311 LIS->removeInterval(SavedExec); 312 Changed = true; 313 } 314 break; 315 } 316 } 317 318 if (Changed) { 319 for (auto Reg : RecalcRegs) { 320 if (Register::isVirtualRegister(Reg)) { 321 LIS->removeInterval(Reg); 322 if (!MRI.reg_empty(Reg)) 323 LIS->createAndComputeVirtRegInterval(Reg); 324 } else { 325 LIS->removeAllRegUnitsForPhysReg(Reg); 326 } 327 } 328 } 329 330 return Changed; 331 } 332