1 //===-- SIOptimizeExecMaskingPreRA.cpp ------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This pass removes redundant S_OR_B64 instructions enabling lanes in 11 /// the exec. If two SI_END_CF (lowered as S_OR_B64) come together without any 12 /// vector instructions between them we can only keep outer SI_END_CF, given 13 /// that CFG is structured and exec bits of the outer end statement are always 14 /// not less than exec bit of the inner one. 15 /// 16 /// This needs to be done before the RA to eliminate saved exec bits registers 17 /// but after register coalescer to have no vector registers copies in between 18 /// of different end cf statements. 19 /// 20 //===----------------------------------------------------------------------===// 21 22 #include "AMDGPU.h" 23 #include "AMDGPUSubtarget.h" 24 #include "SIInstrInfo.h" 25 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 26 #include "llvm/CodeGen/LiveIntervals.h" 27 #include "llvm/CodeGen/MachineFunctionPass.h" 28 29 using namespace llvm; 30 31 #define DEBUG_TYPE "si-optimize-exec-masking-pre-ra" 32 33 namespace { 34 35 class SIOptimizeExecMaskingPreRA : public MachineFunctionPass { 36 public: 37 static char ID; 38 39 public: 40 SIOptimizeExecMaskingPreRA() : MachineFunctionPass(ID) { 41 initializeSIOptimizeExecMaskingPreRAPass(*PassRegistry::getPassRegistry()); 42 } 43 44 bool runOnMachineFunction(MachineFunction &MF) override; 45 46 StringRef getPassName() const override { 47 return "SI optimize exec mask operations pre-RA"; 48 } 49 50 void getAnalysisUsage(AnalysisUsage &AU) const override { 51 AU.addRequired<LiveIntervals>(); 52 AU.setPreservesAll(); 53 MachineFunctionPass::getAnalysisUsage(AU); 54 } 55 }; 56 57 } // End anonymous namespace. 58 59 INITIALIZE_PASS_BEGIN(SIOptimizeExecMaskingPreRA, DEBUG_TYPE, 60 "SI optimize exec mask operations pre-RA", false, false) 61 INITIALIZE_PASS_DEPENDENCY(LiveIntervals) 62 INITIALIZE_PASS_END(SIOptimizeExecMaskingPreRA, DEBUG_TYPE, 63 "SI optimize exec mask operations pre-RA", false, false) 64 65 char SIOptimizeExecMaskingPreRA::ID = 0; 66 67 char &llvm::SIOptimizeExecMaskingPreRAID = SIOptimizeExecMaskingPreRA::ID; 68 69 FunctionPass *llvm::createSIOptimizeExecMaskingPreRAPass() { 70 return new SIOptimizeExecMaskingPreRA(); 71 } 72 73 static bool isEndCF(const MachineInstr& MI, const SIRegisterInfo* TRI) { 74 return MI.getOpcode() == AMDGPU::S_OR_B64 && 75 MI.modifiesRegister(AMDGPU::EXEC, TRI); 76 } 77 78 static bool isFullExecCopy(const MachineInstr& MI) { 79 return MI.getOperand(1).getReg() == AMDGPU::EXEC; 80 } 81 82 static unsigned getOrNonExecReg(const MachineInstr &MI, 83 const SIInstrInfo &TII) { 84 auto Op = TII.getNamedOperand(MI, AMDGPU::OpName::src1); 85 if (Op->isReg() && Op->getReg() != AMDGPU::EXEC) 86 return Op->getReg(); 87 Op = TII.getNamedOperand(MI, AMDGPU::OpName::src0); 88 if (Op->isReg() && Op->getReg() != AMDGPU::EXEC) 89 return Op->getReg(); 90 return AMDGPU::NoRegister; 91 } 92 93 static MachineInstr* getOrExecSource(const MachineInstr &MI, 94 const SIInstrInfo &TII, 95 const MachineRegisterInfo &MRI) { 96 auto SavedExec = getOrNonExecReg(MI, TII); 97 if (SavedExec == AMDGPU::NoRegister) 98 return nullptr; 99 auto SaveExecInst = MRI.getUniqueVRegDef(SavedExec); 100 if (!SaveExecInst || !isFullExecCopy(*SaveExecInst)) 101 return nullptr; 102 return SaveExecInst; 103 } 104 105 // Optimize sequence 106 // %sel = V_CNDMASK_B32_e64 0, 1, %cc 107 // %cmp = V_CMP_NE_U32 1, %1 108 // $vcc = S_AND_B64 $exec, %cmp 109 // S_CBRANCH_VCC[N]Z 110 // => 111 // $vcc = S_ANDN2_B64 $exec, %cc 112 // S_CBRANCH_VCC[N]Z 113 // 114 // It is the negation pattern inserted by DAGCombiner::visitBRCOND() in the 115 // rebuildSetCC(). We start with S_CBRANCH to avoid exhaustive search, but 116 // only 3 first instructions are really needed. S_AND_B64 with exec is a 117 // required part of the pattern since V_CNDMASK_B32 writes zeroes for inactive 118 // lanes. 119 // 120 // Returns %cc register on success. 121 static unsigned optimizeVcndVcmpPair(MachineBasicBlock &MBB, 122 const GCNSubtarget &ST, 123 MachineRegisterInfo &MRI, 124 LiveIntervals *LIS) { 125 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 126 const SIInstrInfo *TII = ST.getInstrInfo(); 127 const unsigned AndOpc = AMDGPU::S_AND_B64; 128 const unsigned Andn2Opc = AMDGPU::S_ANDN2_B64; 129 const unsigned CondReg = AMDGPU::VCC; 130 const unsigned ExecReg = AMDGPU::EXEC; 131 132 auto I = llvm::find_if(MBB.terminators(), [](const MachineInstr &MI) { 133 unsigned Opc = MI.getOpcode(); 134 return Opc == AMDGPU::S_CBRANCH_VCCZ || 135 Opc == AMDGPU::S_CBRANCH_VCCNZ; }); 136 if (I == MBB.terminators().end()) 137 return AMDGPU::NoRegister; 138 139 auto *And = TRI->findReachingDef(CondReg, AMDGPU::NoSubRegister, 140 *I, MRI, LIS); 141 if (!And || And->getOpcode() != AndOpc || 142 !And->getOperand(1).isReg() || !And->getOperand(2).isReg()) 143 return AMDGPU::NoRegister; 144 145 MachineOperand *AndCC = &And->getOperand(1); 146 unsigned CmpReg = AndCC->getReg(); 147 unsigned CmpSubReg = AndCC->getSubReg(); 148 if (CmpReg == ExecReg) { 149 AndCC = &And->getOperand(2); 150 CmpReg = AndCC->getReg(); 151 CmpSubReg = AndCC->getSubReg(); 152 } else if (And->getOperand(2).getReg() != ExecReg) { 153 return AMDGPU::NoRegister; 154 } 155 156 auto *Cmp = TRI->findReachingDef(CmpReg, CmpSubReg, *And, MRI, LIS); 157 if (!Cmp || !(Cmp->getOpcode() == AMDGPU::V_CMP_NE_U32_e32 || 158 Cmp->getOpcode() == AMDGPU::V_CMP_NE_U32_e64) || 159 Cmp->getParent() != And->getParent()) 160 return AMDGPU::NoRegister; 161 162 MachineOperand *Op1 = TII->getNamedOperand(*Cmp, AMDGPU::OpName::src0); 163 MachineOperand *Op2 = TII->getNamedOperand(*Cmp, AMDGPU::OpName::src1); 164 if (Op1->isImm() && Op2->isReg()) 165 std::swap(Op1, Op2); 166 if (!Op1->isReg() || !Op2->isImm() || Op2->getImm() != 1) 167 return AMDGPU::NoRegister; 168 169 unsigned SelReg = Op1->getReg(); 170 auto *Sel = TRI->findReachingDef(SelReg, Op1->getSubReg(), *Cmp, MRI, LIS); 171 if (!Sel || Sel->getOpcode() != AMDGPU::V_CNDMASK_B32_e64) 172 return AMDGPU::NoRegister; 173 174 if (TII->hasModifiersSet(*Sel, AMDGPU::OpName::src0_modifiers) || 175 TII->hasModifiersSet(*Sel, AMDGPU::OpName::src0_modifiers)) 176 return AMDGPU::NoRegister; 177 178 Op1 = TII->getNamedOperand(*Sel, AMDGPU::OpName::src0); 179 Op2 = TII->getNamedOperand(*Sel, AMDGPU::OpName::src1); 180 MachineOperand *CC = TII->getNamedOperand(*Sel, AMDGPU::OpName::src2); 181 if (!Op1->isImm() || !Op2->isImm() || !CC->isReg() || 182 Op1->getImm() != 0 || Op2->getImm() != 1) 183 return AMDGPU::NoRegister; 184 185 LLVM_DEBUG(dbgs() << "Folding sequence:\n\t" << *Sel << '\t' 186 << *Cmp << '\t' << *And); 187 188 unsigned CCReg = CC->getReg(); 189 LIS->RemoveMachineInstrFromMaps(*And); 190 MachineInstr *Andn2 = BuildMI(MBB, *And, And->getDebugLoc(), 191 TII->get(Andn2Opc), And->getOperand(0).getReg()) 192 .addReg(ExecReg) 193 .addReg(CCReg, CC->getSubReg()); 194 And->eraseFromParent(); 195 LIS->InsertMachineInstrInMaps(*Andn2); 196 197 LLVM_DEBUG(dbgs() << "=>\n\t" << *Andn2 << '\n'); 198 199 // Try to remove compare. Cmp value should not used in between of cmp 200 // and s_and_b64 if VCC or just unused if any other register. 201 if ((TargetRegisterInfo::isVirtualRegister(CmpReg) && 202 MRI.use_nodbg_empty(CmpReg)) || 203 (CmpReg == CondReg && 204 std::none_of(std::next(Cmp->getIterator()), Andn2->getIterator(), 205 [&](const MachineInstr &MI) { 206 return MI.readsRegister(CondReg, TRI); }))) { 207 LLVM_DEBUG(dbgs() << "Erasing: " << *Cmp << '\n'); 208 209 LIS->RemoveMachineInstrFromMaps(*Cmp); 210 Cmp->eraseFromParent(); 211 212 // Try to remove v_cndmask_b32. 213 if (TargetRegisterInfo::isVirtualRegister(SelReg) && 214 MRI.use_nodbg_empty(SelReg)) { 215 LLVM_DEBUG(dbgs() << "Erasing: " << *Sel << '\n'); 216 217 LIS->RemoveMachineInstrFromMaps(*Sel); 218 Sel->eraseFromParent(); 219 } 220 } 221 222 return CCReg; 223 } 224 225 bool SIOptimizeExecMaskingPreRA::runOnMachineFunction(MachineFunction &MF) { 226 if (skipFunction(MF.getFunction())) 227 return false; 228 229 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 230 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 231 const SIInstrInfo *TII = ST.getInstrInfo(); 232 MachineRegisterInfo &MRI = MF.getRegInfo(); 233 LiveIntervals *LIS = &getAnalysis<LiveIntervals>(); 234 DenseSet<unsigned> RecalcRegs({AMDGPU::EXEC_LO, AMDGPU::EXEC_HI}); 235 bool Changed = false; 236 237 for (MachineBasicBlock &MBB : MF) { 238 239 if (unsigned Reg = optimizeVcndVcmpPair(MBB, ST, MRI, LIS)) { 240 RecalcRegs.insert(Reg); 241 RecalcRegs.insert(AMDGPU::VCC_LO); 242 RecalcRegs.insert(AMDGPU::VCC_HI); 243 RecalcRegs.insert(AMDGPU::SCC); 244 Changed = true; 245 } 246 247 // Try to remove unneeded instructions before s_endpgm. 248 if (MBB.succ_empty()) { 249 if (MBB.empty()) 250 continue; 251 252 // Skip this if the endpgm has any implicit uses, otherwise we would need 253 // to be careful to update / remove them. 254 // S_ENDPGM always has a single imm operand that is not used other than to 255 // end up in the encoding 256 MachineInstr &Term = MBB.back(); 257 if (Term.getOpcode() != AMDGPU::S_ENDPGM || Term.getNumOperands() != 1) 258 continue; 259 260 SmallVector<MachineBasicBlock*, 4> Blocks({&MBB}); 261 262 while (!Blocks.empty()) { 263 auto CurBB = Blocks.pop_back_val(); 264 auto I = CurBB->rbegin(), E = CurBB->rend(); 265 if (I != E) { 266 if (I->isUnconditionalBranch() || I->getOpcode() == AMDGPU::S_ENDPGM) 267 ++I; 268 else if (I->isBranch()) 269 continue; 270 } 271 272 while (I != E) { 273 if (I->isDebugInstr()) { 274 I = std::next(I); 275 continue; 276 } 277 278 if (I->mayStore() || I->isBarrier() || I->isCall() || 279 I->hasUnmodeledSideEffects() || I->hasOrderedMemoryRef()) 280 break; 281 282 LLVM_DEBUG(dbgs() 283 << "Removing no effect instruction: " << *I << '\n'); 284 285 for (auto &Op : I->operands()) { 286 if (Op.isReg()) 287 RecalcRegs.insert(Op.getReg()); 288 } 289 290 auto Next = std::next(I); 291 LIS->RemoveMachineInstrFromMaps(*I); 292 I->eraseFromParent(); 293 I = Next; 294 295 Changed = true; 296 } 297 298 if (I != E) 299 continue; 300 301 // Try to ascend predecessors. 302 for (auto *Pred : CurBB->predecessors()) { 303 if (Pred->succ_size() == 1) 304 Blocks.push_back(Pred); 305 } 306 } 307 continue; 308 } 309 310 // Try to collapse adjacent endifs. 311 auto E = MBB.end(); 312 auto Lead = skipDebugInstructionsForward(MBB.begin(), E); 313 if (MBB.succ_size() != 1 || Lead == E || !isEndCF(*Lead, TRI)) 314 continue; 315 316 const MachineBasicBlock* Succ = *MBB.succ_begin(); 317 if (!MBB.isLayoutSuccessor(Succ)) 318 continue; 319 320 auto I = std::next(Lead); 321 322 for ( ; I != E; ++I) { 323 if (I->isDebugInstr()) 324 continue; 325 326 if (!TII->isSALU(*I) || I->readsRegister(AMDGPU::EXEC, TRI)) 327 break; 328 } 329 330 if (I != E) 331 continue; 332 333 auto NextLead = skipDebugInstructionsForward(Succ->begin(), Succ->end()); 334 if (NextLead == Succ->end() || !isEndCF(*NextLead, TRI) || 335 !getOrExecSource(*NextLead, *TII, MRI)) 336 continue; 337 338 LLVM_DEBUG(dbgs() << "Redundant EXEC = S_OR_B64 found: " << *Lead << '\n'); 339 340 auto SaveExec = getOrExecSource(*Lead, *TII, MRI); 341 unsigned SaveExecReg = getOrNonExecReg(*Lead, *TII); 342 for (auto &Op : Lead->operands()) { 343 if (Op.isReg()) 344 RecalcRegs.insert(Op.getReg()); 345 } 346 347 LIS->RemoveMachineInstrFromMaps(*Lead); 348 Lead->eraseFromParent(); 349 if (SaveExecReg) { 350 LIS->removeInterval(SaveExecReg); 351 LIS->createAndComputeVirtRegInterval(SaveExecReg); 352 } 353 354 Changed = true; 355 356 // If the only use of saved exec in the removed instruction is S_AND_B64 357 // fold the copy now. 358 if (!SaveExec || !SaveExec->isFullCopy()) 359 continue; 360 361 unsigned SavedExec = SaveExec->getOperand(0).getReg(); 362 bool SafeToReplace = true; 363 for (auto& U : MRI.use_nodbg_instructions(SavedExec)) { 364 if (U.getParent() != SaveExec->getParent()) { 365 SafeToReplace = false; 366 break; 367 } 368 369 LLVM_DEBUG(dbgs() << "Redundant EXEC COPY: " << *SaveExec << '\n'); 370 } 371 372 if (SafeToReplace) { 373 LIS->RemoveMachineInstrFromMaps(*SaveExec); 374 SaveExec->eraseFromParent(); 375 MRI.replaceRegWith(SavedExec, AMDGPU::EXEC); 376 LIS->removeInterval(SavedExec); 377 } 378 } 379 380 if (Changed) { 381 for (auto Reg : RecalcRegs) { 382 if (TargetRegisterInfo::isVirtualRegister(Reg)) { 383 LIS->removeInterval(Reg); 384 if (!MRI.reg_empty(Reg)) 385 LIS->createAndComputeVirtRegInterval(Reg); 386 } else { 387 LIS->removeAllRegUnitsForPhysReg(Reg); 388 } 389 } 390 } 391 392 return Changed; 393 } 394