xref: /llvm-project/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp (revision ddd2f4b96f9f3967a66e744a98b6ecec25c55de8)
1 //===-- SILowerControlFlow.cpp - Use predicates for control flow ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This pass lowers the pseudo control flow instructions to real
11 /// machine instructions.
12 ///
13 /// All control flow is handled using predicated instructions and
14 /// a predicate stack.  Each Scalar ALU controls the operations of 64 Vector
15 /// ALUs.  The Scalar ALU can update the predicate for any of the Vector ALUs
16 /// by writting to the 64-bit EXEC register (each bit corresponds to a
17 /// single vector ALU).  Typically, for predicates, a vector ALU will write
18 /// to its bit of the VCC register (like EXEC VCC is 64-bits, one for each
19 /// Vector ALU) and then the ScalarALU will AND the VCC register with the
20 /// EXEC to update the predicates.
21 ///
22 /// For example:
23 /// %vcc = V_CMP_GT_F32 %vgpr1, %vgpr2
24 /// %sgpr0 = SI_IF %vcc
25 ///   %vgpr0 = V_ADD_F32 %vgpr0, %vgpr0
26 /// %sgpr0 = SI_ELSE %sgpr0
27 ///   %vgpr0 = V_SUB_F32 %vgpr0, %vgpr0
28 /// SI_END_CF %sgpr0
29 ///
30 /// becomes:
31 ///
32 /// %sgpr0 = S_AND_SAVEEXEC_B64 %vcc  // Save and update the exec mask
33 /// %sgpr0 = S_XOR_B64 %sgpr0, %exec  // Clear live bits from saved exec mask
34 /// S_CBRANCH_EXECZ label0            // This instruction is an optional
35 ///                                   // optimization which allows us to
36 ///                                   // branch if all the bits of
37 ///                                   // EXEC are zero.
38 /// %vgpr0 = V_ADD_F32 %vgpr0, %vgpr0 // Do the IF block of the branch
39 ///
40 /// label0:
41 /// %sgpr0 = S_OR_SAVEEXEC_B64 %sgpr0  // Restore the exec mask for the Then block
42 /// %exec = S_XOR_B64 %sgpr0, %exec    // Update the exec mask
43 /// S_BRANCH_EXECZ label1              // Use our branch optimization
44 ///                                    // instruction again.
45 /// %vgpr0 = V_SUB_F32 %vgpr0, %vgpr   // Do the THEN block
46 /// label1:
47 /// %exec = S_OR_B64 %exec, %sgpr0     // Re-enable saved exec mask bits
48 //===----------------------------------------------------------------------===//
49 
50 #include "AMDGPU.h"
51 #include "AMDGPUSubtarget.h"
52 #include "SIInstrInfo.h"
53 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
54 #include "llvm/ADT/SmallSet.h"
55 #include "llvm/ADT/SmallVector.h"
56 #include "llvm/ADT/StringRef.h"
57 #include "llvm/CodeGen/LiveIntervals.h"
58 #include "llvm/CodeGen/MachineBasicBlock.h"
59 #include "llvm/CodeGen/MachineFunction.h"
60 #include "llvm/CodeGen/MachineFunctionPass.h"
61 #include "llvm/CodeGen/MachineInstr.h"
62 #include "llvm/CodeGen/MachineInstrBuilder.h"
63 #include "llvm/CodeGen/MachineOperand.h"
64 #include "llvm/CodeGen/MachineRegisterInfo.h"
65 #include "llvm/CodeGen/Passes.h"
66 #include "llvm/CodeGen/SlotIndexes.h"
67 #include "llvm/CodeGen/TargetRegisterInfo.h"
68 #include "llvm/MC/MCRegisterInfo.h"
69 #include "llvm/Pass.h"
70 #include <cassert>
71 #include <iterator>
72 
73 using namespace llvm;
74 
75 #define DEBUG_TYPE "si-lower-control-flow"
76 
77 static cl::opt<bool>
78 RemoveRedundantEndcf("amdgpu-remove-redundant-endcf",
79     cl::init(true), cl::ReallyHidden);
80 
81 namespace {
82 
83 class SILowerControlFlow : public MachineFunctionPass {
84 private:
85   const SIRegisterInfo *TRI = nullptr;
86   const SIInstrInfo *TII = nullptr;
87   LiveIntervals *LIS = nullptr;
88   MachineRegisterInfo *MRI = nullptr;
89   DenseSet<const MachineInstr*> LoweredEndCf;
90   DenseSet<Register> LoweredIf;
91 
92   const TargetRegisterClass *BoolRC = nullptr;
93   unsigned AndOpc;
94   unsigned OrOpc;
95   unsigned XorOpc;
96   unsigned MovTermOpc;
97   unsigned Andn2TermOpc;
98   unsigned XorTermrOpc;
99   unsigned OrSaveExecOpc;
100   unsigned Exec;
101 
102   void emitIf(MachineInstr &MI);
103   void emitElse(MachineInstr &MI);
104   void emitIfBreak(MachineInstr &MI);
105   void emitLoop(MachineInstr &MI);
106   void emitEndCf(MachineInstr &MI);
107 
108   void findMaskOperands(MachineInstr &MI, unsigned OpNo,
109                         SmallVectorImpl<MachineOperand> &Src) const;
110 
111   void combineMasks(MachineInstr &MI);
112 
113   // Skip to the next instruction, ignoring debug instructions, and trivial
114   // block boundaries (blocks that have one (typically fallthrough) successor,
115   // and the successor has one predecessor.
116   MachineBasicBlock::iterator
117   skipIgnoreExecInstsTrivialSucc(MachineBasicBlock &MBB,
118                                  MachineBasicBlock::iterator It) const;
119 
120 public:
121   static char ID;
122 
123   SILowerControlFlow() : MachineFunctionPass(ID) {}
124 
125   bool runOnMachineFunction(MachineFunction &MF) override;
126 
127   StringRef getPassName() const override {
128     return "SI Lower control flow pseudo instructions";
129   }
130 
131   void getAnalysisUsage(AnalysisUsage &AU) const override {
132     // Should preserve the same set that TwoAddressInstructions does.
133     AU.addPreserved<SlotIndexes>();
134     AU.addPreserved<LiveIntervals>();
135     AU.addPreservedID(LiveVariablesID);
136     AU.addPreservedID(MachineLoopInfoID);
137     AU.addPreservedID(MachineDominatorsID);
138     AU.setPreservesCFG();
139     MachineFunctionPass::getAnalysisUsage(AU);
140   }
141 };
142 
143 } // end anonymous namespace
144 
145 char SILowerControlFlow::ID = 0;
146 
147 INITIALIZE_PASS(SILowerControlFlow, DEBUG_TYPE,
148                "SI lower control flow", false, false)
149 
150 static void setImpSCCDefDead(MachineInstr &MI, bool IsDead) {
151   MachineOperand &ImpDefSCC = MI.getOperand(3);
152   assert(ImpDefSCC.getReg() == AMDGPU::SCC && ImpDefSCC.isDef());
153 
154   ImpDefSCC.setIsDead(IsDead);
155 }
156 
157 char &llvm::SILowerControlFlowID = SILowerControlFlow::ID;
158 
159 static bool isSimpleIf(const MachineInstr &MI, const MachineRegisterInfo *MRI,
160                        const SIInstrInfo *TII) {
161   Register SaveExecReg = MI.getOperand(0).getReg();
162   auto U = MRI->use_instr_nodbg_begin(SaveExecReg);
163 
164   if (U == MRI->use_instr_nodbg_end() ||
165       std::next(U) != MRI->use_instr_nodbg_end() ||
166       U->getOpcode() != AMDGPU::SI_END_CF)
167     return false;
168 
169   // Check for SI_KILL_*_TERMINATOR on path from if to endif.
170   // if there is any such terminator simplififcations are not safe.
171   auto SMBB = MI.getParent();
172   auto EMBB = U->getParent();
173   DenseSet<const MachineBasicBlock*> Visited;
174   SmallVector<MachineBasicBlock*, 4> Worklist(SMBB->succ_begin(),
175                                               SMBB->succ_end());
176 
177   while (!Worklist.empty()) {
178     MachineBasicBlock *MBB = Worklist.pop_back_val();
179 
180     if (MBB == EMBB || !Visited.insert(MBB).second)
181       continue;
182     for(auto &Term : MBB->terminators())
183       if (TII->isKillTerminator(Term.getOpcode()))
184         return false;
185 
186     Worklist.append(MBB->succ_begin(), MBB->succ_end());
187   }
188 
189   return true;
190 }
191 
192 void SILowerControlFlow::emitIf(MachineInstr &MI) {
193   MachineBasicBlock &MBB = *MI.getParent();
194   const DebugLoc &DL = MI.getDebugLoc();
195   MachineBasicBlock::iterator I(&MI);
196   Register SaveExecReg = MI.getOperand(0).getReg();
197   MachineOperand& Cond = MI.getOperand(1);
198   assert(Cond.getSubReg() == AMDGPU::NoSubRegister);
199 
200   MachineOperand &ImpDefSCC = MI.getOperand(4);
201   assert(ImpDefSCC.getReg() == AMDGPU::SCC && ImpDefSCC.isDef());
202 
203   // If there is only one use of save exec register and that use is SI_END_CF,
204   // we can optimize SI_IF by returning the full saved exec mask instead of
205   // just cleared bits.
206   bool SimpleIf = isSimpleIf(MI, MRI, TII);
207 
208   // Add an implicit def of exec to discourage scheduling VALU after this which
209   // will interfere with trying to form s_and_saveexec_b64 later.
210   Register CopyReg = SimpleIf ? SaveExecReg
211                        : MRI->createVirtualRegister(BoolRC);
212   MachineInstr *CopyExec =
213     BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), CopyReg)
214     .addReg(Exec)
215     .addReg(Exec, RegState::ImplicitDefine);
216   LoweredIf.insert(CopyReg);
217 
218   Register Tmp = MRI->createVirtualRegister(BoolRC);
219 
220   MachineInstr *And =
221     BuildMI(MBB, I, DL, TII->get(AndOpc), Tmp)
222     .addReg(CopyReg)
223     .add(Cond);
224 
225   setImpSCCDefDead(*And, true);
226 
227   MachineInstr *Xor = nullptr;
228   if (!SimpleIf) {
229     Xor =
230       BuildMI(MBB, I, DL, TII->get(XorOpc), SaveExecReg)
231       .addReg(Tmp)
232       .addReg(CopyReg);
233     setImpSCCDefDead(*Xor, ImpDefSCC.isDead());
234   }
235 
236   // Use a copy that is a terminator to get correct spill code placement it with
237   // fast regalloc.
238   MachineInstr *SetExec =
239     BuildMI(MBB, I, DL, TII->get(MovTermOpc), Exec)
240     .addReg(Tmp, RegState::Kill);
241 
242   // Insert the S_CBRANCH_EXECZ instruction which will be optimized later
243   // during SIRemoveShortExecBranches.
244   MachineInstr *NewBr = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ))
245                             .add(MI.getOperand(2));
246 
247   if (!LIS) {
248     MI.eraseFromParent();
249     return;
250   }
251 
252   LIS->InsertMachineInstrInMaps(*CopyExec);
253 
254   // Replace with and so we don't need to fix the live interval for condition
255   // register.
256   LIS->ReplaceMachineInstrInMaps(MI, *And);
257 
258   if (!SimpleIf)
259     LIS->InsertMachineInstrInMaps(*Xor);
260   LIS->InsertMachineInstrInMaps(*SetExec);
261   LIS->InsertMachineInstrInMaps(*NewBr);
262 
263   LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC);
264   MI.eraseFromParent();
265 
266   // FIXME: Is there a better way of adjusting the liveness? It shouldn't be
267   // hard to add another def here but I'm not sure how to correctly update the
268   // valno.
269   LIS->removeInterval(SaveExecReg);
270   LIS->createAndComputeVirtRegInterval(SaveExecReg);
271   LIS->createAndComputeVirtRegInterval(Tmp);
272   if (!SimpleIf)
273     LIS->createAndComputeVirtRegInterval(CopyReg);
274 }
275 
276 void SILowerControlFlow::emitElse(MachineInstr &MI) {
277   MachineBasicBlock &MBB = *MI.getParent();
278   const DebugLoc &DL = MI.getDebugLoc();
279 
280   Register DstReg = MI.getOperand(0).getReg();
281 
282   bool ExecModified = MI.getOperand(3).getImm() != 0;
283   MachineBasicBlock::iterator Start = MBB.begin();
284 
285   // We are running before TwoAddressInstructions, and si_else's operands are
286   // tied. In order to correctly tie the registers, split this into a copy of
287   // the src like it does.
288   Register CopyReg = MRI->createVirtualRegister(BoolRC);
289   MachineInstr *CopyExec =
290     BuildMI(MBB, Start, DL, TII->get(AMDGPU::COPY), CopyReg)
291       .add(MI.getOperand(1)); // Saved EXEC
292 
293   // This must be inserted before phis and any spill code inserted before the
294   // else.
295   Register SaveReg = ExecModified ?
296     MRI->createVirtualRegister(BoolRC) : DstReg;
297   MachineInstr *OrSaveExec =
298     BuildMI(MBB, Start, DL, TII->get(OrSaveExecOpc), SaveReg)
299     .addReg(CopyReg);
300 
301   MachineBasicBlock *DestBB = MI.getOperand(2).getMBB();
302 
303   MachineBasicBlock::iterator ElsePt(MI);
304 
305   if (ExecModified) {
306     MachineInstr *And =
307       BuildMI(MBB, ElsePt, DL, TII->get(AndOpc), DstReg)
308       .addReg(Exec)
309       .addReg(SaveReg);
310 
311     if (LIS)
312       LIS->InsertMachineInstrInMaps(*And);
313   }
314 
315   MachineInstr *Xor =
316     BuildMI(MBB, ElsePt, DL, TII->get(XorTermrOpc), Exec)
317     .addReg(Exec)
318     .addReg(DstReg);
319 
320   MachineInstr *Branch =
321       BuildMI(MBB, ElsePt, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ))
322           .addMBB(DestBB);
323 
324   if (!LIS) {
325     MI.eraseFromParent();
326     return;
327   }
328 
329   LIS->RemoveMachineInstrFromMaps(MI);
330   MI.eraseFromParent();
331 
332   LIS->InsertMachineInstrInMaps(*CopyExec);
333   LIS->InsertMachineInstrInMaps(*OrSaveExec);
334 
335   LIS->InsertMachineInstrInMaps(*Xor);
336   LIS->InsertMachineInstrInMaps(*Branch);
337 
338   // src reg is tied to dst reg.
339   LIS->removeInterval(DstReg);
340   LIS->createAndComputeVirtRegInterval(DstReg);
341   LIS->createAndComputeVirtRegInterval(CopyReg);
342   if (ExecModified)
343     LIS->createAndComputeVirtRegInterval(SaveReg);
344 
345   // Let this be recomputed.
346   LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC);
347 }
348 
349 void SILowerControlFlow::emitIfBreak(MachineInstr &MI) {
350   MachineBasicBlock &MBB = *MI.getParent();
351   const DebugLoc &DL = MI.getDebugLoc();
352   auto Dst = MI.getOperand(0).getReg();
353 
354   // Skip ANDing with exec if the break condition is already masked by exec
355   // because it is a V_CMP in the same basic block. (We know the break
356   // condition operand was an i1 in IR, so if it is a VALU instruction it must
357   // be one with a carry-out.)
358   bool SkipAnding = false;
359   if (MI.getOperand(1).isReg()) {
360     if (MachineInstr *Def = MRI->getUniqueVRegDef(MI.getOperand(1).getReg())) {
361       SkipAnding = Def->getParent() == MI.getParent()
362           && SIInstrInfo::isVALU(*Def);
363     }
364   }
365 
366   // AND the break condition operand with exec, then OR that into the "loop
367   // exit" mask.
368   MachineInstr *And = nullptr, *Or = nullptr;
369   if (!SkipAnding) {
370     Register AndReg = MRI->createVirtualRegister(BoolRC);
371     And = BuildMI(MBB, &MI, DL, TII->get(AndOpc), AndReg)
372              .addReg(Exec)
373              .add(MI.getOperand(1));
374     Or = BuildMI(MBB, &MI, DL, TII->get(OrOpc), Dst)
375              .addReg(AndReg)
376              .add(MI.getOperand(2));
377     if (LIS)
378       LIS->createAndComputeVirtRegInterval(AndReg);
379   } else
380     Or = BuildMI(MBB, &MI, DL, TII->get(OrOpc), Dst)
381              .add(MI.getOperand(1))
382              .add(MI.getOperand(2));
383 
384   if (LIS) {
385     if (And)
386       LIS->InsertMachineInstrInMaps(*And);
387     LIS->ReplaceMachineInstrInMaps(MI, *Or);
388   }
389 
390   MI.eraseFromParent();
391 }
392 
393 void SILowerControlFlow::emitLoop(MachineInstr &MI) {
394   MachineBasicBlock &MBB = *MI.getParent();
395   const DebugLoc &DL = MI.getDebugLoc();
396 
397   MachineInstr *AndN2 =
398       BuildMI(MBB, &MI, DL, TII->get(Andn2TermOpc), Exec)
399           .addReg(Exec)
400           .add(MI.getOperand(0));
401 
402   MachineInstr *Branch =
403       BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
404           .add(MI.getOperand(1));
405 
406   if (LIS) {
407     LIS->ReplaceMachineInstrInMaps(MI, *AndN2);
408     LIS->InsertMachineInstrInMaps(*Branch);
409   }
410 
411   MI.eraseFromParent();
412 }
413 
414 MachineBasicBlock::iterator
415 SILowerControlFlow::skipIgnoreExecInstsTrivialSucc(
416   MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const {
417 
418   SmallSet<const MachineBasicBlock *, 4> Visited;
419   MachineBasicBlock *B = &MBB;
420   do {
421     if (!Visited.insert(B).second)
422       return MBB.end();
423 
424     auto E = B->end();
425     for ( ; It != E; ++It) {
426       if (TII->mayReadEXEC(*MRI, *It))
427         break;
428     }
429 
430     if (It != E)
431       return It;
432 
433     if (B->succ_size() != 1)
434       return MBB.end();
435 
436     // If there is one trivial successor, advance to the next block.
437     MachineBasicBlock *Succ = *B->succ_begin();
438 
439     It = Succ->begin();
440     B = Succ;
441   } while (true);
442 }
443 
444 void SILowerControlFlow::emitEndCf(MachineInstr &MI) {
445   MachineBasicBlock &MBB = *MI.getParent();
446   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
447   unsigned CFMask = MI.getOperand(0).getReg();
448   MachineInstr *Def = MRI.getUniqueVRegDef(CFMask);
449   const DebugLoc &DL = MI.getDebugLoc();
450 
451   // If the only instruction immediately following this END_CF is an another
452   // END_CF in the only successor we can avoid emitting exec mask restore here.
453   if (RemoveRedundantEndcf) {
454     auto Next =
455       skipIgnoreExecInstsTrivialSucc(MBB, std::next(MI.getIterator()));
456     if (Next != MBB.end() && (Next->getOpcode() == AMDGPU::SI_END_CF ||
457                               LoweredEndCf.count(&*Next))) {
458       // Only skip inner END_CF if outer ENDCF belongs to SI_IF.
459       // If that belongs to SI_ELSE then saved mask has an inverted value.
460       Register SavedExec = Next->getOperand(0).getReg();
461       const MachineInstr *Def = MRI.getUniqueVRegDef(SavedExec);
462       // A lowered SI_IF turns definition into COPY of exec.
463       if (Def && (Def->getOpcode() == AMDGPU::SI_IF ||
464                   LoweredIf.count(SavedExec))) {
465         LLVM_DEBUG(dbgs() << "Skip redundant "; MI.dump());
466         if (LIS)
467           LIS->RemoveMachineInstrFromMaps(MI);
468         MI.eraseFromParent();
469         return;
470       }
471     }
472   }
473 
474   MachineBasicBlock::iterator InsPt =
475       Def && Def->getParent() == &MBB ? std::next(MachineBasicBlock::iterator(Def))
476                                : MBB.begin();
477   MachineInstr *NewMI = BuildMI(MBB, InsPt, DL, TII->get(OrOpc), Exec)
478                             .addReg(Exec)
479                             .add(MI.getOperand(0));
480 
481   LoweredEndCf.insert(NewMI);
482 
483   if (LIS)
484     LIS->ReplaceMachineInstrInMaps(MI, *NewMI);
485 
486   MI.eraseFromParent();
487 
488   if (LIS)
489     LIS->handleMove(*NewMI);
490 }
491 
492 // Returns replace operands for a logical operation, either single result
493 // for exec or two operands if source was another equivalent operation.
494 void SILowerControlFlow::findMaskOperands(MachineInstr &MI, unsigned OpNo,
495        SmallVectorImpl<MachineOperand> &Src) const {
496   MachineOperand &Op = MI.getOperand(OpNo);
497   if (!Op.isReg() || !Register::isVirtualRegister(Op.getReg())) {
498     Src.push_back(Op);
499     return;
500   }
501 
502   MachineInstr *Def = MRI->getUniqueVRegDef(Op.getReg());
503   if (!Def || Def->getParent() != MI.getParent() ||
504       !(Def->isFullCopy() || (Def->getOpcode() == MI.getOpcode())))
505     return;
506 
507   // Make sure we do not modify exec between def and use.
508   // A copy with implcitly defined exec inserted earlier is an exclusion, it
509   // does not really modify exec.
510   for (auto I = Def->getIterator(); I != MI.getIterator(); ++I)
511     if (I->modifiesRegister(AMDGPU::EXEC, TRI) &&
512         !(I->isCopy() && I->getOperand(0).getReg() != Exec))
513       return;
514 
515   for (const auto &SrcOp : Def->explicit_operands())
516     if (SrcOp.isReg() && SrcOp.isUse() &&
517         (Register::isVirtualRegister(SrcOp.getReg()) || SrcOp.getReg() == Exec))
518       Src.push_back(SrcOp);
519 }
520 
521 // Search and combine pairs of equivalent instructions, like
522 // S_AND_B64 x, (S_AND_B64 x, y) => S_AND_B64 x, y
523 // S_OR_B64  x, (S_OR_B64  x, y) => S_OR_B64  x, y
524 // One of the operands is exec mask.
525 void SILowerControlFlow::combineMasks(MachineInstr &MI) {
526   assert(MI.getNumExplicitOperands() == 3);
527   SmallVector<MachineOperand, 4> Ops;
528   unsigned OpToReplace = 1;
529   findMaskOperands(MI, 1, Ops);
530   if (Ops.size() == 1) OpToReplace = 2; // First operand can be exec or its copy
531   findMaskOperands(MI, 2, Ops);
532   if (Ops.size() != 3) return;
533 
534   unsigned UniqueOpndIdx;
535   if (Ops[0].isIdenticalTo(Ops[1])) UniqueOpndIdx = 2;
536   else if (Ops[0].isIdenticalTo(Ops[2])) UniqueOpndIdx = 1;
537   else if (Ops[1].isIdenticalTo(Ops[2])) UniqueOpndIdx = 1;
538   else return;
539 
540   Register Reg = MI.getOperand(OpToReplace).getReg();
541   MI.RemoveOperand(OpToReplace);
542   MI.addOperand(Ops[UniqueOpndIdx]);
543   if (MRI->use_empty(Reg))
544     MRI->getUniqueVRegDef(Reg)->eraseFromParent();
545 }
546 
547 bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
548   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
549   TII = ST.getInstrInfo();
550   TRI = &TII->getRegisterInfo();
551 
552   // This doesn't actually need LiveIntervals, but we can preserve them.
553   LIS = getAnalysisIfAvailable<LiveIntervals>();
554   MRI = &MF.getRegInfo();
555   BoolRC = TRI->getBoolRC();
556 
557   if (ST.isWave32()) {
558     AndOpc = AMDGPU::S_AND_B32;
559     OrOpc = AMDGPU::S_OR_B32;
560     XorOpc = AMDGPU::S_XOR_B32;
561     MovTermOpc = AMDGPU::S_MOV_B32_term;
562     Andn2TermOpc = AMDGPU::S_ANDN2_B32_term;
563     XorTermrOpc = AMDGPU::S_XOR_B32_term;
564     OrSaveExecOpc = AMDGPU::S_OR_SAVEEXEC_B32;
565     Exec = AMDGPU::EXEC_LO;
566   } else {
567     AndOpc = AMDGPU::S_AND_B64;
568     OrOpc = AMDGPU::S_OR_B64;
569     XorOpc = AMDGPU::S_XOR_B64;
570     MovTermOpc = AMDGPU::S_MOV_B64_term;
571     Andn2TermOpc = AMDGPU::S_ANDN2_B64_term;
572     XorTermrOpc = AMDGPU::S_XOR_B64_term;
573     OrSaveExecOpc = AMDGPU::S_OR_SAVEEXEC_B64;
574     Exec = AMDGPU::EXEC;
575   }
576 
577   MachineFunction::iterator NextBB;
578   for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
579        BI != BE; BI = NextBB) {
580     NextBB = std::next(BI);
581     MachineBasicBlock &MBB = *BI;
582 
583     MachineBasicBlock::iterator I, Next, Last;
584 
585     for (I = MBB.begin(), Last = MBB.end(); I != MBB.end(); I = Next) {
586       Next = std::next(I);
587       MachineInstr &MI = *I;
588 
589       switch (MI.getOpcode()) {
590       case AMDGPU::SI_IF:
591         emitIf(MI);
592         break;
593 
594       case AMDGPU::SI_ELSE:
595         emitElse(MI);
596         break;
597 
598       case AMDGPU::SI_IF_BREAK:
599         emitIfBreak(MI);
600         break;
601 
602       case AMDGPU::SI_LOOP:
603         emitLoop(MI);
604         break;
605 
606       case AMDGPU::SI_END_CF:
607         emitEndCf(MI);
608         break;
609 
610       case AMDGPU::S_AND_B64:
611       case AMDGPU::S_OR_B64:
612       case AMDGPU::S_AND_B32:
613       case AMDGPU::S_OR_B32:
614         // Cleanup bit manipulations on exec mask
615         combineMasks(MI);
616         Last = I;
617         continue;
618 
619       default:
620         Last = I;
621         continue;
622       }
623 
624       // Replay newly inserted code to combine masks
625       Next = (Last == MBB.end()) ? MBB.begin() : Last;
626     }
627   }
628 
629   LoweredEndCf.clear();
630   LoweredIf.clear();
631 
632   return true;
633 }
634