1 //===-- SILowerControlFlow.cpp - Use predicates for control flow ----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This pass lowers the pseudo control flow instructions to real 11 /// machine instructions. 12 /// 13 /// All control flow is handled using predicated instructions and 14 /// a predicate stack. Each Scalar ALU controls the operations of 64 Vector 15 /// ALUs. The Scalar ALU can update the predicate for any of the Vector ALUs 16 /// by writting to the 64-bit EXEC register (each bit corresponds to a 17 /// single vector ALU). Typically, for predicates, a vector ALU will write 18 /// to its bit of the VCC register (like EXEC VCC is 64-bits, one for each 19 /// Vector ALU) and then the ScalarALU will AND the VCC register with the 20 /// EXEC to update the predicates. 21 /// 22 /// For example: 23 /// %vcc = V_CMP_GT_F32 %vgpr1, %vgpr2 24 /// %sgpr0 = SI_IF %vcc 25 /// %vgpr0 = V_ADD_F32 %vgpr0, %vgpr0 26 /// %sgpr0 = SI_ELSE %sgpr0 27 /// %vgpr0 = V_SUB_F32 %vgpr0, %vgpr0 28 /// SI_END_CF %sgpr0 29 /// 30 /// becomes: 31 /// 32 /// %sgpr0 = S_AND_SAVEEXEC_B64 %vcc // Save and update the exec mask 33 /// %sgpr0 = S_XOR_B64 %sgpr0, %exec // Clear live bits from saved exec mask 34 /// S_CBRANCH_EXECZ label0 // This instruction is an optional 35 /// // optimization which allows us to 36 /// // branch if all the bits of 37 /// // EXEC are zero. 38 /// %vgpr0 = V_ADD_F32 %vgpr0, %vgpr0 // Do the IF block of the branch 39 /// 40 /// label0: 41 /// %sgpr0 = S_OR_SAVEEXEC_B64 %sgpr0 // Restore the exec mask for the Then block 42 /// %exec = S_XOR_B64 %sgpr0, %exec // Update the exec mask 43 /// S_BRANCH_EXECZ label1 // Use our branch optimization 44 /// // instruction again. 45 /// %vgpr0 = V_SUB_F32 %vgpr0, %vgpr // Do the THEN block 46 /// label1: 47 /// %exec = S_OR_B64 %exec, %sgpr0 // Re-enable saved exec mask bits 48 //===----------------------------------------------------------------------===// 49 50 #include "AMDGPU.h" 51 #include "AMDGPUSubtarget.h" 52 #include "SIInstrInfo.h" 53 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 54 #include "llvm/ADT/SmallSet.h" 55 #include "llvm/ADT/SmallVector.h" 56 #include "llvm/ADT/StringRef.h" 57 #include "llvm/CodeGen/LiveIntervals.h" 58 #include "llvm/CodeGen/MachineBasicBlock.h" 59 #include "llvm/CodeGen/MachineFunction.h" 60 #include "llvm/CodeGen/MachineFunctionPass.h" 61 #include "llvm/CodeGen/MachineInstr.h" 62 #include "llvm/CodeGen/MachineInstrBuilder.h" 63 #include "llvm/CodeGen/MachineOperand.h" 64 #include "llvm/CodeGen/MachineRegisterInfo.h" 65 #include "llvm/CodeGen/Passes.h" 66 #include "llvm/CodeGen/SlotIndexes.h" 67 #include "llvm/CodeGen/TargetRegisterInfo.h" 68 #include "llvm/MC/MCRegisterInfo.h" 69 #include "llvm/Pass.h" 70 #include <cassert> 71 #include <iterator> 72 73 using namespace llvm; 74 75 #define DEBUG_TYPE "si-lower-control-flow" 76 77 static cl::opt<bool> 78 RemoveRedundantEndcf("amdgpu-remove-redundant-endcf", 79 cl::init(true), cl::ReallyHidden); 80 81 namespace { 82 83 class SILowerControlFlow : public MachineFunctionPass { 84 private: 85 const SIRegisterInfo *TRI = nullptr; 86 const SIInstrInfo *TII = nullptr; 87 LiveIntervals *LIS = nullptr; 88 MachineRegisterInfo *MRI = nullptr; 89 DenseSet<const MachineInstr*> LoweredEndCf; 90 DenseSet<Register> LoweredIf; 91 92 const TargetRegisterClass *BoolRC = nullptr; 93 unsigned AndOpc; 94 unsigned OrOpc; 95 unsigned XorOpc; 96 unsigned MovTermOpc; 97 unsigned Andn2TermOpc; 98 unsigned XorTermrOpc; 99 unsigned OrSaveExecOpc; 100 unsigned Exec; 101 102 void emitIf(MachineInstr &MI); 103 void emitElse(MachineInstr &MI); 104 void emitIfBreak(MachineInstr &MI); 105 void emitLoop(MachineInstr &MI); 106 void emitEndCf(MachineInstr &MI); 107 108 void findMaskOperands(MachineInstr &MI, unsigned OpNo, 109 SmallVectorImpl<MachineOperand> &Src) const; 110 111 void combineMasks(MachineInstr &MI); 112 113 // Skip to the next instruction, ignoring debug instructions, and trivial 114 // block boundaries (blocks that have one (typically fallthrough) successor, 115 // and the successor has one predecessor. 116 MachineBasicBlock::iterator 117 skipIgnoreExecInstsTrivialSucc(MachineBasicBlock &MBB, 118 MachineBasicBlock::iterator It) const; 119 120 public: 121 static char ID; 122 123 SILowerControlFlow() : MachineFunctionPass(ID) {} 124 125 bool runOnMachineFunction(MachineFunction &MF) override; 126 127 StringRef getPassName() const override { 128 return "SI Lower control flow pseudo instructions"; 129 } 130 131 void getAnalysisUsage(AnalysisUsage &AU) const override { 132 // Should preserve the same set that TwoAddressInstructions does. 133 AU.addPreserved<SlotIndexes>(); 134 AU.addPreserved<LiveIntervals>(); 135 AU.addPreservedID(LiveVariablesID); 136 AU.addPreservedID(MachineLoopInfoID); 137 AU.addPreservedID(MachineDominatorsID); 138 AU.setPreservesCFG(); 139 MachineFunctionPass::getAnalysisUsage(AU); 140 } 141 }; 142 143 } // end anonymous namespace 144 145 char SILowerControlFlow::ID = 0; 146 147 INITIALIZE_PASS(SILowerControlFlow, DEBUG_TYPE, 148 "SI lower control flow", false, false) 149 150 static void setImpSCCDefDead(MachineInstr &MI, bool IsDead) { 151 MachineOperand &ImpDefSCC = MI.getOperand(3); 152 assert(ImpDefSCC.getReg() == AMDGPU::SCC && ImpDefSCC.isDef()); 153 154 ImpDefSCC.setIsDead(IsDead); 155 } 156 157 char &llvm::SILowerControlFlowID = SILowerControlFlow::ID; 158 159 static bool isSimpleIf(const MachineInstr &MI, const MachineRegisterInfo *MRI, 160 const SIInstrInfo *TII) { 161 Register SaveExecReg = MI.getOperand(0).getReg(); 162 auto U = MRI->use_instr_nodbg_begin(SaveExecReg); 163 164 if (U == MRI->use_instr_nodbg_end() || 165 std::next(U) != MRI->use_instr_nodbg_end() || 166 U->getOpcode() != AMDGPU::SI_END_CF) 167 return false; 168 169 // Check for SI_KILL_*_TERMINATOR on path from if to endif. 170 // if there is any such terminator simplififcations are not safe. 171 auto SMBB = MI.getParent(); 172 auto EMBB = U->getParent(); 173 DenseSet<const MachineBasicBlock*> Visited; 174 SmallVector<MachineBasicBlock*, 4> Worklist(SMBB->succ_begin(), 175 SMBB->succ_end()); 176 177 while (!Worklist.empty()) { 178 MachineBasicBlock *MBB = Worklist.pop_back_val(); 179 180 if (MBB == EMBB || !Visited.insert(MBB).second) 181 continue; 182 for(auto &Term : MBB->terminators()) 183 if (TII->isKillTerminator(Term.getOpcode())) 184 return false; 185 186 Worklist.append(MBB->succ_begin(), MBB->succ_end()); 187 } 188 189 return true; 190 } 191 192 void SILowerControlFlow::emitIf(MachineInstr &MI) { 193 MachineBasicBlock &MBB = *MI.getParent(); 194 const DebugLoc &DL = MI.getDebugLoc(); 195 MachineBasicBlock::iterator I(&MI); 196 Register SaveExecReg = MI.getOperand(0).getReg(); 197 MachineOperand& Cond = MI.getOperand(1); 198 assert(Cond.getSubReg() == AMDGPU::NoSubRegister); 199 200 MachineOperand &ImpDefSCC = MI.getOperand(4); 201 assert(ImpDefSCC.getReg() == AMDGPU::SCC && ImpDefSCC.isDef()); 202 203 // If there is only one use of save exec register and that use is SI_END_CF, 204 // we can optimize SI_IF by returning the full saved exec mask instead of 205 // just cleared bits. 206 bool SimpleIf = isSimpleIf(MI, MRI, TII); 207 208 // Add an implicit def of exec to discourage scheduling VALU after this which 209 // will interfere with trying to form s_and_saveexec_b64 later. 210 Register CopyReg = SimpleIf ? SaveExecReg 211 : MRI->createVirtualRegister(BoolRC); 212 MachineInstr *CopyExec = 213 BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), CopyReg) 214 .addReg(Exec) 215 .addReg(Exec, RegState::ImplicitDefine); 216 LoweredIf.insert(CopyReg); 217 218 Register Tmp = MRI->createVirtualRegister(BoolRC); 219 220 MachineInstr *And = 221 BuildMI(MBB, I, DL, TII->get(AndOpc), Tmp) 222 .addReg(CopyReg) 223 .add(Cond); 224 225 setImpSCCDefDead(*And, true); 226 227 MachineInstr *Xor = nullptr; 228 if (!SimpleIf) { 229 Xor = 230 BuildMI(MBB, I, DL, TII->get(XorOpc), SaveExecReg) 231 .addReg(Tmp) 232 .addReg(CopyReg); 233 setImpSCCDefDead(*Xor, ImpDefSCC.isDead()); 234 } 235 236 // Use a copy that is a terminator to get correct spill code placement it with 237 // fast regalloc. 238 MachineInstr *SetExec = 239 BuildMI(MBB, I, DL, TII->get(MovTermOpc), Exec) 240 .addReg(Tmp, RegState::Kill); 241 242 // Insert the S_CBRANCH_EXECZ instruction which will be optimized later 243 // during SIRemoveShortExecBranches. 244 MachineInstr *NewBr = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ)) 245 .add(MI.getOperand(2)); 246 247 if (!LIS) { 248 MI.eraseFromParent(); 249 return; 250 } 251 252 LIS->InsertMachineInstrInMaps(*CopyExec); 253 254 // Replace with and so we don't need to fix the live interval for condition 255 // register. 256 LIS->ReplaceMachineInstrInMaps(MI, *And); 257 258 if (!SimpleIf) 259 LIS->InsertMachineInstrInMaps(*Xor); 260 LIS->InsertMachineInstrInMaps(*SetExec); 261 LIS->InsertMachineInstrInMaps(*NewBr); 262 263 LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC); 264 MI.eraseFromParent(); 265 266 // FIXME: Is there a better way of adjusting the liveness? It shouldn't be 267 // hard to add another def here but I'm not sure how to correctly update the 268 // valno. 269 LIS->removeInterval(SaveExecReg); 270 LIS->createAndComputeVirtRegInterval(SaveExecReg); 271 LIS->createAndComputeVirtRegInterval(Tmp); 272 if (!SimpleIf) 273 LIS->createAndComputeVirtRegInterval(CopyReg); 274 } 275 276 void SILowerControlFlow::emitElse(MachineInstr &MI) { 277 MachineBasicBlock &MBB = *MI.getParent(); 278 const DebugLoc &DL = MI.getDebugLoc(); 279 280 Register DstReg = MI.getOperand(0).getReg(); 281 282 bool ExecModified = MI.getOperand(3).getImm() != 0; 283 MachineBasicBlock::iterator Start = MBB.begin(); 284 285 // We are running before TwoAddressInstructions, and si_else's operands are 286 // tied. In order to correctly tie the registers, split this into a copy of 287 // the src like it does. 288 Register CopyReg = MRI->createVirtualRegister(BoolRC); 289 MachineInstr *CopyExec = 290 BuildMI(MBB, Start, DL, TII->get(AMDGPU::COPY), CopyReg) 291 .add(MI.getOperand(1)); // Saved EXEC 292 293 // This must be inserted before phis and any spill code inserted before the 294 // else. 295 Register SaveReg = ExecModified ? 296 MRI->createVirtualRegister(BoolRC) : DstReg; 297 MachineInstr *OrSaveExec = 298 BuildMI(MBB, Start, DL, TII->get(OrSaveExecOpc), SaveReg) 299 .addReg(CopyReg); 300 301 MachineBasicBlock *DestBB = MI.getOperand(2).getMBB(); 302 303 MachineBasicBlock::iterator ElsePt(MI); 304 305 if (ExecModified) { 306 MachineInstr *And = 307 BuildMI(MBB, ElsePt, DL, TII->get(AndOpc), DstReg) 308 .addReg(Exec) 309 .addReg(SaveReg); 310 311 if (LIS) 312 LIS->InsertMachineInstrInMaps(*And); 313 } 314 315 MachineInstr *Xor = 316 BuildMI(MBB, ElsePt, DL, TII->get(XorTermrOpc), Exec) 317 .addReg(Exec) 318 .addReg(DstReg); 319 320 MachineInstr *Branch = 321 BuildMI(MBB, ElsePt, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ)) 322 .addMBB(DestBB); 323 324 if (!LIS) { 325 MI.eraseFromParent(); 326 return; 327 } 328 329 LIS->RemoveMachineInstrFromMaps(MI); 330 MI.eraseFromParent(); 331 332 LIS->InsertMachineInstrInMaps(*CopyExec); 333 LIS->InsertMachineInstrInMaps(*OrSaveExec); 334 335 LIS->InsertMachineInstrInMaps(*Xor); 336 LIS->InsertMachineInstrInMaps(*Branch); 337 338 // src reg is tied to dst reg. 339 LIS->removeInterval(DstReg); 340 LIS->createAndComputeVirtRegInterval(DstReg); 341 LIS->createAndComputeVirtRegInterval(CopyReg); 342 if (ExecModified) 343 LIS->createAndComputeVirtRegInterval(SaveReg); 344 345 // Let this be recomputed. 346 LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC); 347 } 348 349 void SILowerControlFlow::emitIfBreak(MachineInstr &MI) { 350 MachineBasicBlock &MBB = *MI.getParent(); 351 const DebugLoc &DL = MI.getDebugLoc(); 352 auto Dst = MI.getOperand(0).getReg(); 353 354 // Skip ANDing with exec if the break condition is already masked by exec 355 // because it is a V_CMP in the same basic block. (We know the break 356 // condition operand was an i1 in IR, so if it is a VALU instruction it must 357 // be one with a carry-out.) 358 bool SkipAnding = false; 359 if (MI.getOperand(1).isReg()) { 360 if (MachineInstr *Def = MRI->getUniqueVRegDef(MI.getOperand(1).getReg())) { 361 SkipAnding = Def->getParent() == MI.getParent() 362 && SIInstrInfo::isVALU(*Def); 363 } 364 } 365 366 // AND the break condition operand with exec, then OR that into the "loop 367 // exit" mask. 368 MachineInstr *And = nullptr, *Or = nullptr; 369 if (!SkipAnding) { 370 Register AndReg = MRI->createVirtualRegister(BoolRC); 371 And = BuildMI(MBB, &MI, DL, TII->get(AndOpc), AndReg) 372 .addReg(Exec) 373 .add(MI.getOperand(1)); 374 Or = BuildMI(MBB, &MI, DL, TII->get(OrOpc), Dst) 375 .addReg(AndReg) 376 .add(MI.getOperand(2)); 377 if (LIS) 378 LIS->createAndComputeVirtRegInterval(AndReg); 379 } else 380 Or = BuildMI(MBB, &MI, DL, TII->get(OrOpc), Dst) 381 .add(MI.getOperand(1)) 382 .add(MI.getOperand(2)); 383 384 if (LIS) { 385 if (And) 386 LIS->InsertMachineInstrInMaps(*And); 387 LIS->ReplaceMachineInstrInMaps(MI, *Or); 388 } 389 390 MI.eraseFromParent(); 391 } 392 393 void SILowerControlFlow::emitLoop(MachineInstr &MI) { 394 MachineBasicBlock &MBB = *MI.getParent(); 395 const DebugLoc &DL = MI.getDebugLoc(); 396 397 MachineInstr *AndN2 = 398 BuildMI(MBB, &MI, DL, TII->get(Andn2TermOpc), Exec) 399 .addReg(Exec) 400 .add(MI.getOperand(0)); 401 402 MachineInstr *Branch = 403 BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ)) 404 .add(MI.getOperand(1)); 405 406 if (LIS) { 407 LIS->ReplaceMachineInstrInMaps(MI, *AndN2); 408 LIS->InsertMachineInstrInMaps(*Branch); 409 } 410 411 MI.eraseFromParent(); 412 } 413 414 MachineBasicBlock::iterator 415 SILowerControlFlow::skipIgnoreExecInstsTrivialSucc( 416 MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { 417 418 SmallSet<const MachineBasicBlock *, 4> Visited; 419 MachineBasicBlock *B = &MBB; 420 do { 421 if (!Visited.insert(B).second) 422 return MBB.end(); 423 424 auto E = B->end(); 425 for ( ; It != E; ++It) { 426 if (TII->mayReadEXEC(*MRI, *It)) 427 break; 428 } 429 430 if (It != E) 431 return It; 432 433 if (B->succ_size() != 1) 434 return MBB.end(); 435 436 // If there is one trivial successor, advance to the next block. 437 MachineBasicBlock *Succ = *B->succ_begin(); 438 439 It = Succ->begin(); 440 B = Succ; 441 } while (true); 442 } 443 444 void SILowerControlFlow::emitEndCf(MachineInstr &MI) { 445 MachineBasicBlock &MBB = *MI.getParent(); 446 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 447 unsigned CFMask = MI.getOperand(0).getReg(); 448 MachineInstr *Def = MRI.getUniqueVRegDef(CFMask); 449 const DebugLoc &DL = MI.getDebugLoc(); 450 451 // If the only instruction immediately following this END_CF is an another 452 // END_CF in the only successor we can avoid emitting exec mask restore here. 453 if (RemoveRedundantEndcf) { 454 auto Next = 455 skipIgnoreExecInstsTrivialSucc(MBB, std::next(MI.getIterator())); 456 if (Next != MBB.end() && (Next->getOpcode() == AMDGPU::SI_END_CF || 457 LoweredEndCf.count(&*Next))) { 458 // Only skip inner END_CF if outer ENDCF belongs to SI_IF. 459 // If that belongs to SI_ELSE then saved mask has an inverted value. 460 Register SavedExec = Next->getOperand(0).getReg(); 461 const MachineInstr *Def = MRI.getUniqueVRegDef(SavedExec); 462 // A lowered SI_IF turns definition into COPY of exec. 463 if (Def && (Def->getOpcode() == AMDGPU::SI_IF || 464 LoweredIf.count(SavedExec))) { 465 LLVM_DEBUG(dbgs() << "Skip redundant "; MI.dump()); 466 if (LIS) 467 LIS->RemoveMachineInstrFromMaps(MI); 468 MI.eraseFromParent(); 469 return; 470 } 471 } 472 } 473 474 MachineBasicBlock::iterator InsPt = 475 Def && Def->getParent() == &MBB ? std::next(MachineBasicBlock::iterator(Def)) 476 : MBB.begin(); 477 MachineInstr *NewMI = BuildMI(MBB, InsPt, DL, TII->get(OrOpc), Exec) 478 .addReg(Exec) 479 .add(MI.getOperand(0)); 480 481 LoweredEndCf.insert(NewMI); 482 483 if (LIS) 484 LIS->ReplaceMachineInstrInMaps(MI, *NewMI); 485 486 MI.eraseFromParent(); 487 488 if (LIS) 489 LIS->handleMove(*NewMI); 490 } 491 492 // Returns replace operands for a logical operation, either single result 493 // for exec or two operands if source was another equivalent operation. 494 void SILowerControlFlow::findMaskOperands(MachineInstr &MI, unsigned OpNo, 495 SmallVectorImpl<MachineOperand> &Src) const { 496 MachineOperand &Op = MI.getOperand(OpNo); 497 if (!Op.isReg() || !Register::isVirtualRegister(Op.getReg())) { 498 Src.push_back(Op); 499 return; 500 } 501 502 MachineInstr *Def = MRI->getUniqueVRegDef(Op.getReg()); 503 if (!Def || Def->getParent() != MI.getParent() || 504 !(Def->isFullCopy() || (Def->getOpcode() == MI.getOpcode()))) 505 return; 506 507 // Make sure we do not modify exec between def and use. 508 // A copy with implcitly defined exec inserted earlier is an exclusion, it 509 // does not really modify exec. 510 for (auto I = Def->getIterator(); I != MI.getIterator(); ++I) 511 if (I->modifiesRegister(AMDGPU::EXEC, TRI) && 512 !(I->isCopy() && I->getOperand(0).getReg() != Exec)) 513 return; 514 515 for (const auto &SrcOp : Def->explicit_operands()) 516 if (SrcOp.isReg() && SrcOp.isUse() && 517 (Register::isVirtualRegister(SrcOp.getReg()) || SrcOp.getReg() == Exec)) 518 Src.push_back(SrcOp); 519 } 520 521 // Search and combine pairs of equivalent instructions, like 522 // S_AND_B64 x, (S_AND_B64 x, y) => S_AND_B64 x, y 523 // S_OR_B64 x, (S_OR_B64 x, y) => S_OR_B64 x, y 524 // One of the operands is exec mask. 525 void SILowerControlFlow::combineMasks(MachineInstr &MI) { 526 assert(MI.getNumExplicitOperands() == 3); 527 SmallVector<MachineOperand, 4> Ops; 528 unsigned OpToReplace = 1; 529 findMaskOperands(MI, 1, Ops); 530 if (Ops.size() == 1) OpToReplace = 2; // First operand can be exec or its copy 531 findMaskOperands(MI, 2, Ops); 532 if (Ops.size() != 3) return; 533 534 unsigned UniqueOpndIdx; 535 if (Ops[0].isIdenticalTo(Ops[1])) UniqueOpndIdx = 2; 536 else if (Ops[0].isIdenticalTo(Ops[2])) UniqueOpndIdx = 1; 537 else if (Ops[1].isIdenticalTo(Ops[2])) UniqueOpndIdx = 1; 538 else return; 539 540 Register Reg = MI.getOperand(OpToReplace).getReg(); 541 MI.RemoveOperand(OpToReplace); 542 MI.addOperand(Ops[UniqueOpndIdx]); 543 if (MRI->use_empty(Reg)) 544 MRI->getUniqueVRegDef(Reg)->eraseFromParent(); 545 } 546 547 bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) { 548 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 549 TII = ST.getInstrInfo(); 550 TRI = &TII->getRegisterInfo(); 551 552 // This doesn't actually need LiveIntervals, but we can preserve them. 553 LIS = getAnalysisIfAvailable<LiveIntervals>(); 554 MRI = &MF.getRegInfo(); 555 BoolRC = TRI->getBoolRC(); 556 557 if (ST.isWave32()) { 558 AndOpc = AMDGPU::S_AND_B32; 559 OrOpc = AMDGPU::S_OR_B32; 560 XorOpc = AMDGPU::S_XOR_B32; 561 MovTermOpc = AMDGPU::S_MOV_B32_term; 562 Andn2TermOpc = AMDGPU::S_ANDN2_B32_term; 563 XorTermrOpc = AMDGPU::S_XOR_B32_term; 564 OrSaveExecOpc = AMDGPU::S_OR_SAVEEXEC_B32; 565 Exec = AMDGPU::EXEC_LO; 566 } else { 567 AndOpc = AMDGPU::S_AND_B64; 568 OrOpc = AMDGPU::S_OR_B64; 569 XorOpc = AMDGPU::S_XOR_B64; 570 MovTermOpc = AMDGPU::S_MOV_B64_term; 571 Andn2TermOpc = AMDGPU::S_ANDN2_B64_term; 572 XorTermrOpc = AMDGPU::S_XOR_B64_term; 573 OrSaveExecOpc = AMDGPU::S_OR_SAVEEXEC_B64; 574 Exec = AMDGPU::EXEC; 575 } 576 577 MachineFunction::iterator NextBB; 578 for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); 579 BI != BE; BI = NextBB) { 580 NextBB = std::next(BI); 581 MachineBasicBlock &MBB = *BI; 582 583 MachineBasicBlock::iterator I, Next, Last; 584 585 for (I = MBB.begin(), Last = MBB.end(); I != MBB.end(); I = Next) { 586 Next = std::next(I); 587 MachineInstr &MI = *I; 588 589 switch (MI.getOpcode()) { 590 case AMDGPU::SI_IF: 591 emitIf(MI); 592 break; 593 594 case AMDGPU::SI_ELSE: 595 emitElse(MI); 596 break; 597 598 case AMDGPU::SI_IF_BREAK: 599 emitIfBreak(MI); 600 break; 601 602 case AMDGPU::SI_LOOP: 603 emitLoop(MI); 604 break; 605 606 case AMDGPU::SI_END_CF: 607 emitEndCf(MI); 608 break; 609 610 case AMDGPU::S_AND_B64: 611 case AMDGPU::S_OR_B64: 612 case AMDGPU::S_AND_B32: 613 case AMDGPU::S_OR_B32: 614 // Cleanup bit manipulations on exec mask 615 combineMasks(MI); 616 Last = I; 617 continue; 618 619 default: 620 Last = I; 621 continue; 622 } 623 624 // Replay newly inserted code to combine masks 625 Next = (Last == MBB.end()) ? MBB.begin() : Last; 626 } 627 } 628 629 LoweredEndCf.clear(); 630 LoweredIf.clear(); 631 632 return true; 633 } 634