xref: /llvm-project/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp (revision fb28bf3fb42e2e4ce9b606bcdd855c960b4592f1) 
1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements hazard recognizers for scheduling on GCN processors.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "GCNHazardRecognizer.h"
14 #include "GCNSubtarget.h"
15 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
16 #include "SIMachineFunctionInfo.h"
17 #include "llvm/CodeGen/MachineFunction.h"
18 #include "llvm/CodeGen/ScheduleDAG.h"
19 #include "llvm/Support/TargetParser.h"
20 
21 using namespace llvm;
22 
23 namespace {
24 
25 struct MFMAPaddingRatioParser : public cl::parser<unsigned> {
26   MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {}
27 
28   bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
29     if (Arg.getAsInteger(0, Value))
30       return O.error("'" + Arg + "' value invalid for uint argument!");
31 
32     if (Value > 100)
33       return O.error("'" + Arg + "' value must be in the range [0, 100]!");
34 
35     return false;
36   }
37 };
38 
39 } // end anonymous namespace
40 
41 static cl::opt<unsigned, false, MFMAPaddingRatioParser>
42     MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden,
43                      cl::desc("Fill a percentage of the latency between "
44                               "neighboring MFMA with s_nops."));
45 
46 //===----------------------------------------------------------------------===//
47 // Hazard Recognizer Implementation
48 //===----------------------------------------------------------------------===//
49 
50 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
51                                                  const GCNSubtarget &ST);
52 
53 GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) :
54   IsHazardRecognizerMode(false),
55   CurrCycleInstr(nullptr),
56   MF(MF),
57   ST(MF.getSubtarget<GCNSubtarget>()),
58   TII(*ST.getInstrInfo()),
59   TRI(TII.getRegisterInfo()),
60   ClauseUses(TRI.getNumRegUnits()),
61   ClauseDefs(TRI.getNumRegUnits()) {
62   MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5;
63   TSchedModel.init(&ST);
64   RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
65 }
66 
67 void GCNHazardRecognizer::Reset() {
68   EmittedInstrs.clear();
69 }
70 
71 void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
72   EmitInstruction(SU->getInstr());
73 }
74 
75 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
76   CurrCycleInstr = MI;
77 }
78 
79 static bool isDivFMas(unsigned Opcode) {
80   return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
81 }
82 
83 static bool isSGetReg(unsigned Opcode) {
84   return Opcode == AMDGPU::S_GETREG_B32;
85 }
86 
87 static bool isSSetReg(unsigned Opcode) {
88   switch (Opcode) {
89   case AMDGPU::S_SETREG_B32:
90   case AMDGPU::S_SETREG_B32_mode:
91   case AMDGPU::S_SETREG_IMM32_B32:
92   case AMDGPU::S_SETREG_IMM32_B32_mode:
93     return true;
94   }
95   return false;
96 }
97 
98 static bool isRWLane(unsigned Opcode) {
99   return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
100 }
101 
102 static bool isRFE(unsigned Opcode) {
103   return Opcode == AMDGPU::S_RFE_B64;
104 }
105 
106 static bool isSMovRel(unsigned Opcode) {
107   switch (Opcode) {
108   case AMDGPU::S_MOVRELS_B32:
109   case AMDGPU::S_MOVRELS_B64:
110   case AMDGPU::S_MOVRELD_B32:
111   case AMDGPU::S_MOVRELD_B64:
112     return true;
113   default:
114     return false;
115   }
116 }
117 
118 static bool isDGEMM(unsigned Opcode) {
119   return AMDGPU::getMAIIsDGEMM(Opcode);
120 }
121 
122 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) {
123   unsigned Opcode = MI.getOpcode();
124 
125   if (!SIInstrInfo::isMAI(MI) ||
126       isDGEMM(Opcode) ||
127       Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
128       Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
129     return false;
130 
131   if (!ST.hasGFX940Insts())
132     return true;
133 
134   return AMDGPU::getMAIIsGFX940XDL(Opcode);
135 }
136 
137 static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
138                                     const MachineInstr &MI) {
139   if (TII.isAlwaysGDS(MI.getOpcode()))
140     return true;
141 
142   switch (MI.getOpcode()) {
143   case AMDGPU::S_SENDMSG:
144   case AMDGPU::S_SENDMSGHALT:
145   case AMDGPU::S_TTRACEDATA:
146     return true;
147   // These DS opcodes don't support GDS.
148   case AMDGPU::DS_NOP:
149   case AMDGPU::DS_PERMUTE_B32:
150   case AMDGPU::DS_BPERMUTE_B32:
151     return false;
152   default:
153     if (TII.isDS(MI.getOpcode())) {
154       int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
155                                            AMDGPU::OpName::gds);
156       if (MI.getOperand(GDS).getImm())
157         return true;
158     }
159     return false;
160   }
161 }
162 
163 static bool isPermlane(const MachineInstr &MI) {
164   unsigned Opcode = MI.getOpcode();
165   return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
166          Opcode == AMDGPU::V_PERMLANEX16_B32_e64;
167 }
168 
169 static bool isLdsDma(const MachineInstr &MI) {
170   return SIInstrInfo::isVALU(MI) &&
171          (SIInstrInfo::isMUBUF(MI) || SIInstrInfo::isFLAT(MI));
172 }
173 
174 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
175   const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
176                                                      AMDGPU::OpName::simm16);
177   return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
178 }
179 
180 ScheduleHazardRecognizer::HazardType
181 GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
182   MachineInstr *MI = SU->getInstr();
183   // If we are not in "HazardRecognizerMode" and therefore not being run from
184   // the scheduler, track possible stalls from hazards but don't insert noops.
185   auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard;
186 
187   if (MI->isBundle())
188    return NoHazard;
189 
190   if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
191     return HazardType;
192 
193   if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
194     return HazardType;
195 
196   if (checkFPAtomicToDenormModeHazard(MI) > 0)
197     return HazardType;
198 
199   if (ST.hasNoDataDepHazard())
200     return NoHazard;
201 
202   // FIXME: Should flat be considered vmem?
203   if ((SIInstrInfo::isVMEM(*MI) ||
204        SIInstrInfo::isFLAT(*MI))
205       && checkVMEMHazards(MI) > 0)
206     return HazardType;
207 
208   if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
209     return HazardType;
210 
211   if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
212     return HazardType;
213 
214   if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
215     return HazardType;
216 
217   if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
218     return HazardType;
219 
220   if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) ||
221        SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) ||
222        SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
223     return HazardType;
224 
225   if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
226     return HazardType;
227 
228   if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
229     return HazardType;
230 
231   if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
232     return HazardType;
233 
234   if (((ST.hasReadM0MovRelInterpHazard() &&
235         (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) ||
236          MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
237          MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
238        (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) ||
239        (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) ||
240        (ST.hasReadM0LdsDirectHazard() &&
241         MI->readsRegister(AMDGPU::LDS_DIRECT))) &&
242       checkReadM0Hazards(MI) > 0)
243     return HazardType;
244 
245   if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0)
246     return HazardType;
247 
248   if ((SIInstrInfo::isVMEM(*MI) ||
249        SIInstrInfo::isFLAT(*MI) ||
250        SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0)
251     return HazardType;
252 
253   if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
254     return HazardType;
255 
256   return NoHazard;
257 }
258 
259 static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII,
260                                 unsigned Quantity) {
261   while (Quantity > 0) {
262     unsigned Arg = std::min(Quantity, 8u);
263     Quantity -= Arg;
264     BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP))
265         .addImm(Arg - 1);
266   }
267 }
268 
269 unsigned
270 GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const {
271   const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&MI);
272   assert(TSchedModel.getWriteProcResBegin(SC) !=
273          TSchedModel.getWriteProcResEnd(SC));
274   return TSchedModel.getWriteProcResBegin(SC)->Cycles;
275 }
276 
277 void GCNHazardRecognizer::processBundle() {
278   MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator());
279   MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end();
280   // Check bundled MachineInstr's for hazards.
281   for (; MI != E && MI->isInsideBundle(); ++MI) {
282     CurrCycleInstr = &*MI;
283     unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
284 
285     if (IsHazardRecognizerMode) {
286       fixHazards(CurrCycleInstr);
287 
288       insertNoopsInBundle(CurrCycleInstr, TII, WaitStates);
289     }
290 
291     // It’s unnecessary to track more than MaxLookAhead instructions. Since we
292     // include the bundled MI directly after, only add a maximum of
293     // (MaxLookAhead - 1) noops to EmittedInstrs.
294     for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i)
295       EmittedInstrs.push_front(nullptr);
296 
297     EmittedInstrs.push_front(CurrCycleInstr);
298     EmittedInstrs.resize(MaxLookAhead);
299   }
300   CurrCycleInstr = nullptr;
301 }
302 
303 void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) {
304   assert(IsHazardRecognizerMode);
305 
306   unsigned NumPreNoops = PreEmitNoops(MI);
307   EmitNoops(NumPreNoops);
308   if (MI->isInsideBundle())
309     insertNoopsInBundle(MI, TII, NumPreNoops);
310   else
311     TII.insertNoops(*MI->getParent(), MachineBasicBlock::iterator(MI),
312                     NumPreNoops);
313   EmitInstruction(MI);
314   AdvanceCycle();
315 }
316 
317 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
318   IsHazardRecognizerMode = true;
319   CurrCycleInstr = MI;
320   unsigned W = PreEmitNoopsCommon(MI);
321   fixHazards(MI);
322   CurrCycleInstr = nullptr;
323   return W;
324 }
325 
326 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) {
327   if (MI->isBundle())
328     return 0;
329 
330   int WaitStates = 0;
331 
332   if (SIInstrInfo::isSMRD(*MI))
333     return std::max(WaitStates, checkSMRDHazards(MI));
334 
335   if (ST.hasNSAtoVMEMBug())
336     WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI));
337 
338   WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI));
339 
340   if (ST.hasNoDataDepHazard())
341     return WaitStates;
342 
343   if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI))
344     WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
345 
346   if (SIInstrInfo::isVALU(*MI))
347     WaitStates = std::max(WaitStates, checkVALUHazards(MI));
348 
349   if (SIInstrInfo::isDPP(*MI))
350     WaitStates = std::max(WaitStates, checkDPPHazards(MI));
351 
352   if (isDivFMas(MI->getOpcode()))
353     WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
354 
355   if (isRWLane(MI->getOpcode()))
356     WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
357 
358   if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) ||
359        SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) ||
360        SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
361     WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI));
362 
363   if (MI->isInlineAsm())
364     return std::max(WaitStates, checkInlineAsmHazards(MI));
365 
366   if (isSGetReg(MI->getOpcode()))
367     return std::max(WaitStates, checkGetRegHazards(MI));
368 
369   if (isSSetReg(MI->getOpcode()))
370     return std::max(WaitStates, checkSetRegHazards(MI));
371 
372   if (isRFE(MI->getOpcode()))
373     return std::max(WaitStates, checkRFEHazards(MI));
374 
375   if ((ST.hasReadM0MovRelInterpHazard() &&
376        (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) ||
377         MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
378         MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
379       (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) ||
380       (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) ||
381       (ST.hasReadM0LdsDirectHazard() && MI->readsRegister(AMDGPU::LDS_DIRECT)))
382     return std::max(WaitStates, checkReadM0Hazards(MI));
383 
384   if (SIInstrInfo::isMAI(*MI))
385     return std::max(WaitStates, checkMAIHazards(MI));
386 
387   if (SIInstrInfo::isVMEM(*MI) ||
388       SIInstrInfo::isFLAT(*MI) ||
389       SIInstrInfo::isDS(*MI))
390     return std::max(WaitStates, checkMAILdStHazards(MI));
391 
392   return WaitStates;
393 }
394 
395 void GCNHazardRecognizer::EmitNoop() {
396   EmittedInstrs.push_front(nullptr);
397 }
398 
399 void GCNHazardRecognizer::AdvanceCycle() {
400   // When the scheduler detects a stall, it will call AdvanceCycle() without
401   // emitting any instructions.
402   if (!CurrCycleInstr) {
403     EmittedInstrs.push_front(nullptr);
404     return;
405   }
406 
407   if (CurrCycleInstr->isBundle()) {
408     processBundle();
409     return;
410   }
411 
412   unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
413   if (!NumWaitStates) {
414     CurrCycleInstr = nullptr;
415     return;
416   }
417 
418   // Keep track of emitted instructions
419   EmittedInstrs.push_front(CurrCycleInstr);
420 
421   // Add a nullptr for each additional wait state after the first.  Make sure
422   // not to add more than getMaxLookAhead() items to the list, since we
423   // truncate the list to that size right after this loop.
424   for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
425        i < e; ++i) {
426     EmittedInstrs.push_front(nullptr);
427   }
428 
429   // getMaxLookahead() is the largest number of wait states we will ever need
430   // to insert, so there is no point in keeping track of more than that many
431   // wait states.
432   EmittedInstrs.resize(getMaxLookAhead());
433 
434   CurrCycleInstr = nullptr;
435 }
436 
437 void GCNHazardRecognizer::RecedeCycle() {
438   llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
439 }
440 
441 //===----------------------------------------------------------------------===//
442 // Helper Functions
443 //===----------------------------------------------------------------------===//
444 
445 typedef enum { HazardFound, HazardExpired, NoHazardFound } HazardFnResult;
446 
447 typedef function_ref<bool(const MachineInstr &, int WaitStates)> IsExpiredFn;
448 typedef function_ref<unsigned int(const MachineInstr &)> GetNumWaitStatesFn;
449 
450 // Search for a hazard in a block and its predecessors.
451 template <typename StateT>
452 static bool
453 hasHazard(StateT State,
454           function_ref<HazardFnResult(StateT &, const MachineInstr &)> IsHazard,
455           function_ref<void(StateT &, const MachineInstr &)> UpdateState,
456           const MachineBasicBlock *MBB,
457           MachineBasicBlock::const_reverse_instr_iterator I,
458           DenseSet<const MachineBasicBlock *> &Visited) {
459   for (auto E = MBB->instr_rend(); I != E; ++I) {
460     // No need to look at parent BUNDLE instructions.
461     if (I->isBundle())
462       continue;
463 
464     switch (IsHazard(State, *I)) {
465     case HazardFound:
466       return true;
467     case HazardExpired:
468       return false;
469     default:
470       // Continue search
471       break;
472     }
473 
474     if (I->isInlineAsm() || I->isMetaInstruction())
475       continue;
476 
477     UpdateState(State, *I);
478   }
479 
480   for (MachineBasicBlock *Pred : MBB->predecessors()) {
481     if (!Visited.insert(Pred).second)
482       continue;
483 
484     if (hasHazard(State, IsHazard, UpdateState, Pred, Pred->instr_rbegin(),
485                   Visited))
486       return true;
487   }
488 
489   return false;
490 }
491 
492 // Returns a minimum wait states since \p I walking all predecessors.
493 // Only scans until \p IsExpired does not return true.
494 // Can only be run in a hazard recognizer mode.
495 static int getWaitStatesSince(
496     GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock *MBB,
497     MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates,
498     IsExpiredFn IsExpired, DenseSet<const MachineBasicBlock *> &Visited,
499     GetNumWaitStatesFn GetNumWaitStates = SIInstrInfo::getNumWaitStates) {
500   for (auto E = MBB->instr_rend(); I != E; ++I) {
501     // Don't add WaitStates for parent BUNDLE instructions.
502     if (I->isBundle())
503       continue;
504 
505     if (IsHazard(*I))
506       return WaitStates;
507 
508     if (I->isInlineAsm())
509       continue;
510 
511     WaitStates += GetNumWaitStates(*I);
512 
513     if (IsExpired(*I, WaitStates))
514       return std::numeric_limits<int>::max();
515   }
516 
517   int MinWaitStates = std::numeric_limits<int>::max();
518   for (MachineBasicBlock *Pred : MBB->predecessors()) {
519     if (!Visited.insert(Pred).second)
520       continue;
521 
522     int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(), WaitStates,
523                                IsExpired, Visited, GetNumWaitStates);
524 
525     MinWaitStates = std::min(MinWaitStates, W);
526   }
527 
528   return MinWaitStates;
529 }
530 
531 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
532                               const MachineInstr *MI, IsExpiredFn IsExpired) {
533   DenseSet<const MachineBasicBlock *> Visited;
534   return getWaitStatesSince(IsHazard, MI->getParent(),
535                             std::next(MI->getReverseIterator()),
536                             0, IsExpired, Visited);
537 }
538 
539 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) {
540   if (IsHazardRecognizerMode) {
541     auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) {
542       return WaitStates >= Limit;
543     };
544     return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn);
545   }
546 
547   int WaitStates = 0;
548   for (MachineInstr *MI : EmittedInstrs) {
549     if (MI) {
550       if (IsHazard(*MI))
551         return WaitStates;
552 
553       if (MI->isInlineAsm())
554         continue;
555     }
556     ++WaitStates;
557 
558     if (WaitStates >= Limit)
559       break;
560   }
561   return std::numeric_limits<int>::max();
562 }
563 
564 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
565                                                IsHazardFn IsHazardDef,
566                                                int Limit) {
567   const SIRegisterInfo *TRI = ST.getRegisterInfo();
568 
569   auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) {
570     return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI);
571   };
572 
573   return getWaitStatesSince(IsHazardFn, Limit);
574 }
575 
576 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
577                                                   int Limit) {
578   auto IsHazardFn = [IsHazard](const MachineInstr &MI) {
579     return isSSetReg(MI.getOpcode()) && IsHazard(MI);
580   };
581 
582   return getWaitStatesSince(IsHazardFn, Limit);
583 }
584 
585 //===----------------------------------------------------------------------===//
586 // No-op Hazard Detection
587 //===----------------------------------------------------------------------===//
588 
589 static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
590                         MCRegister Reg) {
591   for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI)
592     BV.set(*RUI);
593 }
594 
595 static void addRegsToSet(const SIRegisterInfo &TRI,
596                          iterator_range<MachineInstr::const_mop_iterator> Ops,
597                          BitVector &Set) {
598   for (const MachineOperand &Op : Ops) {
599     if (Op.isReg())
600       addRegUnits(TRI, Set, Op.getReg().asMCReg());
601   }
602 }
603 
604 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) {
605   // XXX: Do we need to worry about implicit operands
606   addRegsToSet(TRI, MI.defs(), ClauseDefs);
607   addRegsToSet(TRI, MI.uses(), ClauseUses);
608 }
609 
610 static bool breaksSMEMSoftClause(MachineInstr *MI) {
611   return !SIInstrInfo::isSMRD(*MI);
612 }
613 
614 static bool breaksVMEMSoftClause(MachineInstr *MI) {
615   return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI);
616 }
617 
618 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
619   // SMEM soft clause are only present on VI+, and only matter if xnack is
620   // enabled.
621   if (!ST.isXNACKEnabled())
622     return 0;
623 
624   bool IsSMRD = TII.isSMRD(*MEM);
625 
626   resetClause();
627 
628   // A soft-clause is any group of consecutive SMEM instructions.  The
629   // instructions in this group may return out of order and/or may be
630   // replayed (i.e. the same instruction issued more than once).
631   //
632   // In order to handle these situations correctly we need to make sure that
633   // when a clause has more than one instruction, no instruction in the clause
634   // writes to a register that is read by another instruction in the clause
635   // (including itself). If we encounter this situation, we need to break the
636   // clause by inserting a non SMEM instruction.
637 
638   for (MachineInstr *MI : EmittedInstrs) {
639     // When we hit a non-SMEM instruction then we have passed the start of the
640     // clause and we can stop.
641     if (!MI)
642       break;
643 
644     if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI))
645       break;
646 
647     addClauseInst(*MI);
648   }
649 
650   if (ClauseDefs.none())
651     return 0;
652 
653   // We need to make sure not to put loads and stores in the same clause if they
654   // use the same address. For now, just start a new clause whenever we see a
655   // store.
656   if (MEM->mayStore())
657     return 1;
658 
659   addClauseInst(*MEM);
660 
661   // If the set of defs and uses intersect then we cannot add this instruction
662   // to the clause, so we have a hazard.
663   return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
664 }
665 
666 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
667   int WaitStatesNeeded = 0;
668 
669   WaitStatesNeeded = checkSoftClauseHazards(SMRD);
670 
671   // This SMRD hazard only affects SI.
672   if (!ST.hasSMRDReadVALUDefHazard())
673     return WaitStatesNeeded;
674 
675   // A read of an SGPR by SMRD instruction requires 4 wait states when the
676   // SGPR was written by a VALU instruction.
677   int SmrdSgprWaitStates = 4;
678   auto IsHazardDefFn = [this](const MachineInstr &MI) {
679     return TII.isVALU(MI);
680   };
681   auto IsBufferHazardDefFn = [this](const MachineInstr &MI) {
682     return TII.isSALU(MI);
683   };
684 
685   bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
686 
687   for (const MachineOperand &Use : SMRD->uses()) {
688     if (!Use.isReg())
689       continue;
690     int WaitStatesNeededForUse =
691         SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
692                                                    SmrdSgprWaitStates);
693     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
694 
695     // This fixes what appears to be undocumented hardware behavior in SI where
696     // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
697     // needs some number of nops in between. We don't know how many we need, but
698     // let's use 4. This wasn't discovered before probably because the only
699     // case when this happens is when we expand a 64-bit pointer into a full
700     // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
701     // probably never encountered in the closed-source land.
702     if (IsBufferSMRD) {
703       int WaitStatesNeededForUse =
704         SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
705                                                    IsBufferHazardDefFn,
706                                                    SmrdSgprWaitStates);
707       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
708     }
709   }
710 
711   return WaitStatesNeeded;
712 }
713 
714 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
715   if (!ST.hasVMEMReadSGPRVALUDefHazard())
716     return 0;
717 
718   int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
719 
720   // A read of an SGPR by a VMEM instruction requires 5 wait states when the
721   // SGPR was written by a VALU Instruction.
722   const int VmemSgprWaitStates = 5;
723   auto IsHazardDefFn = [this](const MachineInstr &MI) {
724     return TII.isVALU(MI);
725   };
726   for (const MachineOperand &Use : VMEM->uses()) {
727     if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg()))
728       continue;
729 
730     int WaitStatesNeededForUse =
731         VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
732                                                    VmemSgprWaitStates);
733     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
734   }
735   return WaitStatesNeeded;
736 }
737 
738 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
739   const SIRegisterInfo *TRI = ST.getRegisterInfo();
740   const SIInstrInfo *TII = ST.getInstrInfo();
741 
742   // Check for DPP VGPR read after VALU VGPR write and EXEC write.
743   int DppVgprWaitStates = 2;
744   int DppExecWaitStates = 5;
745   int WaitStatesNeeded = 0;
746   auto IsHazardDefFn = [TII](const MachineInstr &MI) {
747     return TII->isVALU(MI);
748   };
749 
750   for (const MachineOperand &Use : DPP->uses()) {
751     if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
752       continue;
753     int WaitStatesNeededForUse =
754         DppVgprWaitStates - getWaitStatesSinceDef(
755                                 Use.getReg(),
756                                 [](const MachineInstr &) { return true; },
757                                 DppVgprWaitStates);
758     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
759   }
760 
761   WaitStatesNeeded = std::max(
762       WaitStatesNeeded,
763       DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
764                                                 DppExecWaitStates));
765 
766   return WaitStatesNeeded;
767 }
768 
769 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
770   const SIInstrInfo *TII = ST.getInstrInfo();
771 
772   // v_div_fmas requires 4 wait states after a write to vcc from a VALU
773   // instruction.
774   const int DivFMasWaitStates = 4;
775   auto IsHazardDefFn = [TII](const MachineInstr &MI) {
776     return TII->isVALU(MI);
777   };
778   int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
779                                                DivFMasWaitStates);
780 
781   return DivFMasWaitStates - WaitStatesNeeded;
782 }
783 
784 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
785   const SIInstrInfo *TII = ST.getInstrInfo();
786   unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
787 
788   const int GetRegWaitStates = 2;
789   auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) {
790     return GetRegHWReg == getHWReg(TII, MI);
791   };
792   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates);
793 
794   return GetRegWaitStates - WaitStatesNeeded;
795 }
796 
797 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
798   const SIInstrInfo *TII = ST.getInstrInfo();
799   unsigned HWReg = getHWReg(TII, *SetRegInstr);
800 
801   const int SetRegWaitStates = ST.getSetRegWaitStates();
802   auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) {
803     return HWReg == getHWReg(TII, MI);
804   };
805   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates);
806   return SetRegWaitStates - WaitStatesNeeded;
807 }
808 
809 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
810   if (!MI.mayStore())
811     return -1;
812 
813   const SIInstrInfo *TII = ST.getInstrInfo();
814   unsigned Opcode = MI.getOpcode();
815   const MCInstrDesc &Desc = MI.getDesc();
816 
817   int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
818   int VDataRCID = -1;
819   if (VDataIdx != -1)
820     VDataRCID = Desc.OpInfo[VDataIdx].RegClass;
821 
822   if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
823     // There is no hazard if the instruction does not use vector regs
824     // (like wbinvl1)
825     if (VDataIdx == -1)
826       return -1;
827     // For MUBUF/MTBUF instructions this hazard only exists if the
828     // instruction is not using a register in the soffset field.
829     const MachineOperand *SOffset =
830         TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
831     // If we have no soffset operand, then assume this field has been
832     // hardcoded to zero.
833     if (AMDGPU::getRegBitWidth(VDataRCID) > 64 &&
834         (!SOffset || !SOffset->isReg()))
835       return VDataIdx;
836   }
837 
838   // MIMG instructions create a hazard if they don't use a 256-bit T# and
839   // the store size is greater than 8 bytes and they have more than two bits
840   // of their dmask set.
841   // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
842   if (TII->isMIMG(MI)) {
843     int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
844     assert(SRsrcIdx != -1 &&
845            AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256);
846     (void)SRsrcIdx;
847   }
848 
849   if (TII->isFLAT(MI)) {
850     int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
851     if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64)
852       return DataIdx;
853   }
854 
855   return -1;
856 }
857 
858 int
859 GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
860                                             const MachineRegisterInfo &MRI) {
861   // Helper to check for the hazard where VMEM instructions that store more than
862   // 8 bytes can have there store data over written by the next instruction.
863   const SIRegisterInfo *TRI = ST.getRegisterInfo();
864 
865   const int VALUWaitStates = ST.hasGFX940Insts() ? 2 : 1;
866   int WaitStatesNeeded = 0;
867 
868   if (!TRI->isVectorRegister(MRI, Def.getReg()))
869     return WaitStatesNeeded;
870   Register Reg = Def.getReg();
871   auto IsHazardFn = [this, Reg, TRI](const MachineInstr &MI) {
872     int DataIdx = createsVALUHazard(MI);
873     return DataIdx >= 0 &&
874            TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg);
875   };
876   int WaitStatesNeededForDef =
877     VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates);
878   WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
879 
880   return WaitStatesNeeded;
881 }
882 
883 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
884   int WaitStatesNeeded = 0;
885 
886   if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(*VALU)) {
887     const int TransDefWaitstates = 1;
888 
889     auto IsTransDefFn = [this, VALU](const MachineInstr &MI) {
890       if (!SIInstrInfo::isTRANS(MI))
891         return false;
892       const SIRegisterInfo *TRI = ST.getRegisterInfo();
893       const SIInstrInfo *TII = ST.getInstrInfo();
894       Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg();
895 
896       for (const MachineOperand &Use : VALU->explicit_uses()) {
897         if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg()))
898           return true;
899       }
900 
901       return false;
902     };
903 
904     int WaitStatesNeededForDef =
905         TransDefWaitstates -
906         getWaitStatesSince(IsTransDefFn, TransDefWaitstates);
907     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
908   }
909 
910   if (ST.hasDstSelForwardingHazard()) {
911     const int Shift16DefWaitstates = 1;
912 
913     auto IsShift16BitDefFn = [this, VALU](const MachineInstr &MI) {
914       if (!SIInstrInfo::isVALU(MI))
915         return false;
916       const SIInstrInfo *TII = ST.getInstrInfo();
917       if (SIInstrInfo::isSDWA(MI)) {
918         if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel))
919           if (DstSel->getImm() == AMDGPU::SDWA::DWORD)
920             return false;
921       } else {
922         if ((AMDGPU::getNamedOperandIdx(MI.getOpcode(),
923                                         AMDGPU::OpName::op_sel) == -1) ||
924             !(TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)
925                   ->getImm() &
926               SISrcMods::DST_OP_SEL))
927           return false;
928       }
929       const SIRegisterInfo *TRI = ST.getRegisterInfo();
930       if (auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) {
931         Register Def = Dst->getReg();
932 
933         for (const MachineOperand &Use : VALU->explicit_uses()) {
934           if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg()))
935             return true;
936         }
937       }
938 
939       return false;
940     };
941 
942     int WaitStatesNeededForDef =
943         Shift16DefWaitstates -
944         getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates);
945     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
946   }
947 
948   if (ST.hasVDecCoExecHazard()) {
949     const int VALUWriteSGPRVALUReadWaitstates = 2;
950     const int VALUWriteEXECRWLane = 4;
951     const int VALUWriteVGPRReadlaneRead = 1;
952 
953     const SIRegisterInfo *TRI = ST.getRegisterInfo();
954     const MachineRegisterInfo &MRI = MF.getRegInfo();
955     Register UseReg;
956     auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) {
957       if (!SIInstrInfo::isVALU(MI))
958         return false;
959       return MI.modifiesRegister(UseReg, TRI);
960     };
961 
962     for (const MachineOperand &Use : VALU->explicit_uses()) {
963       if (!Use.isReg())
964         continue;
965 
966       UseReg = Use.getReg();
967       if (TRI->isSGPRReg(MRI, UseReg)) {
968         int WaitStatesNeededForDef =
969             VALUWriteSGPRVALUReadWaitstates -
970             getWaitStatesSince(IsVALUDefSGPRFn,
971                                VALUWriteSGPRVALUReadWaitstates);
972         WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
973       }
974     }
975 
976     if (VALU->readsRegister(AMDGPU::VCC, TRI)) {
977       UseReg = AMDGPU::VCC;
978       int WaitStatesNeededForDef =
979           VALUWriteSGPRVALUReadWaitstates -
980           getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteSGPRVALUReadWaitstates);
981       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
982     }
983 
984     switch (VALU->getOpcode()) {
985     case AMDGPU::V_READLANE_B32:
986     case AMDGPU::V_READFIRSTLANE_B32: {
987       MachineOperand *Src = TII.getNamedOperand(*VALU, AMDGPU::OpName::src0);
988       UseReg = Src->getReg();
989       int WaitStatesNeededForDef =
990           VALUWriteVGPRReadlaneRead -
991           getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteVGPRReadlaneRead);
992       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
993     }
994       [[fallthrough]];
995     case AMDGPU::V_WRITELANE_B32: {
996       UseReg = AMDGPU::EXEC;
997       int WaitStatesNeededForDef =
998           VALUWriteEXECRWLane -
999           getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteEXECRWLane);
1000       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1001       break;
1002     }
1003     default:
1004       break;
1005     }
1006   }
1007 
1008   // This checks for the hazard where VMEM instructions that store more than
1009   // 8 bytes can have there store data over written by the next instruction.
1010   if (!ST.has12DWordStoreHazard())
1011     return WaitStatesNeeded;
1012 
1013   const MachineRegisterInfo &MRI = MF.getRegInfo();
1014 
1015   for (const MachineOperand &Def : VALU->defs()) {
1016     WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
1017   }
1018 
1019   return WaitStatesNeeded;
1020 }
1021 
1022 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) {
1023   // This checks for hazards associated with inline asm statements.
1024   // Since inline asms can contain just about anything, we use this
1025   // to call/leverage other check*Hazard routines. Note that
1026   // this function doesn't attempt to address all possible inline asm
1027   // hazards (good luck), but is a collection of what has been
1028   // problematic thus far.
1029 
1030   // see checkVALUHazards()
1031   if (!ST.has12DWordStoreHazard())
1032     return 0;
1033 
1034   const MachineRegisterInfo &MRI = MF.getRegInfo();
1035   int WaitStatesNeeded = 0;
1036 
1037   for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands();
1038        I != E; ++I) {
1039     const MachineOperand &Op = IA->getOperand(I);
1040     if (Op.isReg() && Op.isDef()) {
1041       WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
1042     }
1043   }
1044 
1045   return WaitStatesNeeded;
1046 }
1047 
1048 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
1049   const SIInstrInfo *TII = ST.getInstrInfo();
1050   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1051   const MachineRegisterInfo &MRI = MF.getRegInfo();
1052 
1053   const MachineOperand *LaneSelectOp =
1054       TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
1055 
1056   if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
1057     return 0;
1058 
1059   Register LaneSelectReg = LaneSelectOp->getReg();
1060   auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); };
1061 
1062   const int RWLaneWaitStates = 4;
1063   int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
1064                                               RWLaneWaitStates);
1065   return RWLaneWaitStates - WaitStatesSince;
1066 }
1067 
1068 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
1069   if (!ST.hasRFEHazards())
1070     return 0;
1071 
1072   const SIInstrInfo *TII = ST.getInstrInfo();
1073 
1074   const int RFEWaitStates = 1;
1075 
1076   auto IsHazardFn = [TII](const MachineInstr &MI) {
1077     return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS;
1078   };
1079   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates);
1080   return RFEWaitStates - WaitStatesNeeded;
1081 }
1082 
1083 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
1084   const SIInstrInfo *TII = ST.getInstrInfo();
1085   const int ReadM0WaitStates = 1;
1086   auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); };
1087   return ReadM0WaitStates -
1088          getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, ReadM0WaitStates);
1089 }
1090 
1091 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
1092   fixVMEMtoScalarWriteHazards(MI);
1093   fixVcmpxPermlaneHazards(MI);
1094   fixSMEMtoVectorWriteHazards(MI);
1095   fixVcmpxExecWARHazard(MI);
1096   fixLdsBranchVmemWARHazard(MI);
1097   if (ST.hasLdsDirect()) {
1098     fixLdsDirectVALUHazard(MI);
1099     fixLdsDirectVMEMHazard(MI);
1100   }
1101   fixVALUPartialForwardingHazard(MI);
1102   fixVALUTransUseHazard(MI);
1103   fixWMMAHazards(MI);
1104   fixShift64HighRegBug(MI);
1105 }
1106 
1107 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
1108   if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI))
1109     return false;
1110 
1111   const SIInstrInfo *TII = ST.getInstrInfo();
1112   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1113   auto IsHazardFn = [TII, TRI](const MachineInstr &MI) {
1114     return (TII->isVOPC(MI) ||
1115             ((TII->isVOP3(MI) || TII->isSDWA(MI)) && MI.isCompare())) &&
1116            MI.modifiesRegister(AMDGPU::EXEC, TRI);
1117   };
1118 
1119   auto IsExpiredFn = [](const MachineInstr &MI, int) {
1120     unsigned Opc = MI.getOpcode();
1121     return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 &&
1122            Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa;
1123   };
1124 
1125   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1126       std::numeric_limits<int>::max())
1127     return false;
1128 
1129   // V_NOP will be discarded by SQ.
1130   // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
1131   // which is always a VGPR and available.
1132   auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0);
1133   Register Reg = Src0->getReg();
1134   bool IsUndef = Src0->isUndef();
1135   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1136           TII->get(AMDGPU::V_MOV_B32_e32))
1137     .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0))
1138     .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill);
1139 
1140   return true;
1141 }
1142 
1143 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
1144   if (!ST.hasVMEMtoScalarWriteHazard())
1145     return false;
1146 
1147   if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI))
1148     return false;
1149 
1150   if (MI->getNumDefs() == 0)
1151     return false;
1152 
1153   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1154 
1155   auto IsHazardFn = [TRI, MI](const MachineInstr &I) {
1156     if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I) &&
1157         !SIInstrInfo::isFLAT(I))
1158       return false;
1159 
1160     for (const MachineOperand &Def : MI->defs()) {
1161       const MachineOperand *Op =
1162           I.findRegisterUseOperand(Def.getReg(), false, TRI);
1163       if (!Op)
1164         continue;
1165       return true;
1166     }
1167     return false;
1168   };
1169 
1170   auto IsExpiredFn = [](const MachineInstr &MI, int) {
1171     return SIInstrInfo::isVALU(MI) ||
1172            (MI.getOpcode() == AMDGPU::S_WAITCNT &&
1173             !MI.getOperand(0).getImm()) ||
1174            (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1175             MI.getOperand(0).getImm() == 0xffe3);
1176   };
1177 
1178   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1179       std::numeric_limits<int>::max())
1180     return false;
1181 
1182   const SIInstrInfo *TII = ST.getInstrInfo();
1183   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1184           TII->get(AMDGPU::S_WAITCNT_DEPCTR))
1185       .addImm(0xffe3);
1186   return true;
1187 }
1188 
1189 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
1190   if (!ST.hasSMEMtoVectorWriteHazard())
1191     return false;
1192 
1193   if (!SIInstrInfo::isVALU(*MI))
1194     return false;
1195 
1196   unsigned SDSTName;
1197   switch (MI->getOpcode()) {
1198   case AMDGPU::V_READLANE_B32:
1199   case AMDGPU::V_READFIRSTLANE_B32:
1200     SDSTName = AMDGPU::OpName::vdst;
1201     break;
1202   default:
1203     SDSTName = AMDGPU::OpName::sdst;
1204     break;
1205   }
1206 
1207   const SIInstrInfo *TII = ST.getInstrInfo();
1208   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1209   const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU());
1210   const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName);
1211   if (!SDST) {
1212     for (const auto &MO : MI->implicit_operands()) {
1213       if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) {
1214         SDST = &MO;
1215         break;
1216       }
1217     }
1218   }
1219 
1220   if (!SDST)
1221     return false;
1222 
1223   const Register SDSTReg = SDST->getReg();
1224   auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) {
1225     return SIInstrInfo::isSMRD(I) && I.readsRegister(SDSTReg, TRI);
1226   };
1227 
1228   auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) {
1229     if (TII->isSALU(MI)) {
1230       switch (MI.getOpcode()) {
1231       case AMDGPU::S_SETVSKIP:
1232       case AMDGPU::S_VERSION:
1233       case AMDGPU::S_WAITCNT_VSCNT:
1234       case AMDGPU::S_WAITCNT_VMCNT:
1235       case AMDGPU::S_WAITCNT_EXPCNT:
1236         // These instructions cannot not mitigate the hazard.
1237         return false;
1238       case AMDGPU::S_WAITCNT_LGKMCNT:
1239         // Reducing lgkmcnt count to 0 always mitigates the hazard.
1240         return (MI.getOperand(1).getImm() == 0) &&
1241                (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1242       case AMDGPU::S_WAITCNT: {
1243         const int64_t Imm = MI.getOperand(0).getImm();
1244         AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm);
1245         return (Decoded.LgkmCnt == 0);
1246       }
1247       default:
1248         // SOPP instructions cannot mitigate the hazard.
1249         if (TII->isSOPP(MI))
1250           return false;
1251         // At this point the SALU can be assumed to mitigate the hazard
1252         // because either:
1253         // (a) it is independent of the at risk SMEM (breaking chain),
1254         // or
1255         // (b) it is dependent on the SMEM, in which case an appropriate
1256         //     s_waitcnt lgkmcnt _must_ exist between it and the at risk
1257         //     SMEM instruction.
1258         return true;
1259       }
1260     }
1261     return false;
1262   };
1263 
1264   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1265       std::numeric_limits<int>::max())
1266     return false;
1267 
1268   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1269           TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL)
1270       .addImm(0);
1271   return true;
1272 }
1273 
1274 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
1275   if (!ST.hasVcmpxExecWARHazard() || !SIInstrInfo::isVALU(*MI))
1276     return false;
1277 
1278   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1279   if (!MI->modifiesRegister(AMDGPU::EXEC, TRI))
1280     return false;
1281 
1282   auto IsHazardFn = [TRI](const MachineInstr &I) {
1283     if (SIInstrInfo::isVALU(I))
1284       return false;
1285     return I.readsRegister(AMDGPU::EXEC, TRI);
1286   };
1287 
1288   const SIInstrInfo *TII = ST.getInstrInfo();
1289   auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) {
1290     if (SIInstrInfo::isVALU(MI)) {
1291       if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst))
1292         return true;
1293       for (auto MO : MI.implicit_operands())
1294         if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg())))
1295           return true;
1296     }
1297     if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1298         (MI.getOperand(0).getImm() & 0xfffe) == 0xfffe)
1299       return true;
1300     return false;
1301   };
1302 
1303   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1304       std::numeric_limits<int>::max())
1305     return false;
1306 
1307   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1308           TII->get(AMDGPU::S_WAITCNT_DEPCTR))
1309     .addImm(0xfffe);
1310   return true;
1311 }
1312 
1313 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
1314                                                  const GCNSubtarget &ST) {
1315   if (!ST.hasLdsBranchVmemWARHazard())
1316     return false;
1317 
1318   // Check if the necessary condition for the hazard is met: both LDS and VMEM
1319   // instructions need to appear in the same function.
1320   bool HasLds = false;
1321   bool HasVmem = false;
1322   for (auto &MBB : MF) {
1323     for (auto &MI : MBB) {
1324       HasLds |= SIInstrInfo::isDS(MI);
1325       HasVmem |=
1326           SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI);
1327       if (HasLds && HasVmem)
1328         return true;
1329     }
1330   }
1331   return false;
1332 }
1333 
1334 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
1335   if (!RunLdsBranchVmemWARHazardFixup)
1336     return false;
1337 
1338   assert(ST.hasLdsBranchVmemWARHazard());
1339 
1340   auto IsHazardInst = [](const MachineInstr &MI) {
1341     if (SIInstrInfo::isDS(MI))
1342       return 1;
1343     if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI))
1344       return 2;
1345     return 0;
1346   };
1347 
1348   auto InstType = IsHazardInst(*MI);
1349   if (!InstType)
1350     return false;
1351 
1352   auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) {
1353     return IsHazardInst(I) || (I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1354                                I.getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1355                                !I.getOperand(1).getImm());
1356   };
1357 
1358   auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) {
1359     if (!I.isBranch())
1360       return false;
1361 
1362     auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) {
1363       auto InstType2 = IsHazardInst(I);
1364       return InstType2 && InstType != InstType2;
1365     };
1366 
1367     auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) {
1368       auto InstType2 = IsHazardInst(I);
1369       if (InstType == InstType2)
1370         return true;
1371 
1372       return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1373              I.getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1374              !I.getOperand(1).getImm();
1375     };
1376 
1377     return ::getWaitStatesSince(IsHazardFn, &I, IsExpiredFn) !=
1378            std::numeric_limits<int>::max();
1379   };
1380 
1381   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1382       std::numeric_limits<int>::max())
1383     return false;
1384 
1385   const SIInstrInfo *TII = ST.getInstrInfo();
1386   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1387           TII->get(AMDGPU::S_WAITCNT_VSCNT))
1388     .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1389     .addImm(0);
1390 
1391   return true;
1392 }
1393 
1394 bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) {
1395   if (!SIInstrInfo::isLDSDIR(*MI))
1396     return false;
1397 
1398   const int NoHazardWaitStates = 15;
1399   const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst);
1400   const Register VDSTReg = VDST->getReg();
1401 
1402   bool VisitedTrans = false;
1403   auto IsHazardFn = [this, VDSTReg, &VisitedTrans](const MachineInstr &I) {
1404     if (!SIInstrInfo::isVALU(I))
1405       return false;
1406     VisitedTrans = VisitedTrans || SIInstrInfo::isTRANS(I);
1407     // Cover both WAR and WAW
1408     return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI);
1409   };
1410   auto IsExpiredFn = [&](const MachineInstr &I, int WaitStates) {
1411     if (WaitStates >= NoHazardWaitStates)
1412       return true;
1413     // Instructions which cause va_vdst==0 expire hazard
1414     return SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) ||
1415            SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I);
1416   };
1417   auto GetWaitStatesFn = [](const MachineInstr &MI) {
1418     return SIInstrInfo::isVALU(MI) ? 1 : 0;
1419   };
1420 
1421   DenseSet<const MachineBasicBlock *> Visited;
1422   auto Count = ::getWaitStatesSince(IsHazardFn, MI->getParent(),
1423                                     std::next(MI->getReverseIterator()), 0,
1424                                     IsExpiredFn, Visited, GetWaitStatesFn);
1425 
1426   // Transcendentals can execute in parallel to other VALUs.
1427   // This makes va_vdst count unusable with a mixture of VALU and TRANS.
1428   if (VisitedTrans)
1429     Count = 0;
1430 
1431   MachineOperand *WaitVdstOp =
1432       TII.getNamedOperand(*MI, AMDGPU::OpName::waitvdst);
1433   WaitVdstOp->setImm(std::min(Count, NoHazardWaitStates));
1434 
1435   return true;
1436 }
1437 
1438 bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) {
1439   if (!SIInstrInfo::isLDSDIR(*MI))
1440     return false;
1441 
1442   const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst);
1443   const Register VDSTReg = VDST->getReg();
1444 
1445   auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) {
1446     if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I) &&
1447         !SIInstrInfo::isDS(I))
1448       return false;
1449     return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI);
1450   };
1451   auto IsExpiredFn = [](const MachineInstr &I, int) {
1452     return SIInstrInfo::isVALU(I) || SIInstrInfo::isEXP(I) ||
1453            (I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(0).getImm()) ||
1454            (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1455             I.getOperand(0).getImm() == 0xffe3);
1456   };
1457 
1458   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1459       std::numeric_limits<int>::max())
1460     return false;
1461 
1462   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1463           TII.get(AMDGPU::S_WAITCNT_DEPCTR))
1464       .addImm(0xffe3);
1465 
1466   return true;
1467 }
1468 
1469 bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
1470   if (!ST.isWave64())
1471     return false;
1472   if (!ST.hasVALUPartialForwardingHazard())
1473     return false;
1474   if (!SIInstrInfo::isVALU(*MI))
1475     return false;
1476 
1477   SmallSetVector<Register, 4> SrcVGPRs;
1478 
1479   for (const MachineOperand &Use : MI->explicit_uses()) {
1480     if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
1481       SrcVGPRs.insert(Use.getReg());
1482   }
1483 
1484   // Only applies with >= 2 unique VGPR sources
1485   if (SrcVGPRs.size() <= 1)
1486     return false;
1487 
1488   // Look for the following pattern:
1489   //   Va <- VALU [PreExecPos]
1490   //   intv1
1491   //   Exec <- SALU [ExecPos]
1492   //   intv2
1493   //   Vb <- VALU [PostExecPos]
1494   //   intv3
1495   //   MI Va, Vb (WaitState = 0)
1496   //
1497   // Where:
1498   // intv1 + intv2 <= 2 VALUs
1499   // intv3 <= 4 VALUs
1500   //
1501   // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
1502 
1503   const int Intv1plus2MaxVALUs = 2;
1504   const int Intv3MaxVALUs = 4;
1505   const int IntvMaxVALUs = 6;
1506   const int NoHazardVALUWaitStates = IntvMaxVALUs + 2;
1507 
1508   struct StateType {
1509     SmallDenseMap<Register, int, 4> DefPos;
1510     int ExecPos = std::numeric_limits<int>::max();
1511     int VALUs = 0;
1512   };
1513 
1514   StateType State;
1515 
1516   // This overloads expiry testing with all the hazard detection
1517   auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
1518     // Too many VALU states have passed
1519     if (State.VALUs > NoHazardVALUWaitStates)
1520       return HazardExpired;
1521 
1522     // Instructions which cause va_vdst==0 expire hazard
1523     if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) ||
1524         SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I) ||
1525         (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1526          I.getOperand(0).getImm() == 0x0fff))
1527       return HazardExpired;
1528 
1529     // Track registers writes
1530     bool Changed = false;
1531     if (SIInstrInfo::isVALU(I)) {
1532       for (Register Src : SrcVGPRs) {
1533         if (!State.DefPos.count(Src) && I.modifiesRegister(Src, &TRI)) {
1534           State.DefPos[Src] = State.VALUs;
1535           Changed = true;
1536         }
1537       }
1538     } else if (SIInstrInfo::isSALU(I)) {
1539       if (State.ExecPos == std::numeric_limits<int>::max()) {
1540         if (!State.DefPos.empty() && I.modifiesRegister(AMDGPU::EXEC, &TRI)) {
1541           State.ExecPos = State.VALUs;
1542           Changed = true;
1543         }
1544       }
1545     }
1546 
1547     // Early expiration: too many VALUs in intv3
1548     if (State.VALUs > Intv3MaxVALUs && State.DefPos.empty())
1549       return HazardExpired;
1550 
1551     // Only evaluate state if something changed
1552     if (!Changed)
1553       return NoHazardFound;
1554 
1555     // Determine positions of VALUs pre/post exec change
1556     if (State.ExecPos == std::numeric_limits<int>::max())
1557       return NoHazardFound;
1558 
1559     int PreExecPos = std::numeric_limits<int>::max();
1560     int PostExecPos = std::numeric_limits<int>::max();
1561 
1562     for (auto Entry : State.DefPos) {
1563       int DefVALUs = Entry.second;
1564       if (DefVALUs != std::numeric_limits<int>::max()) {
1565         if (DefVALUs >= State.ExecPos)
1566           PreExecPos = std::min(PreExecPos, DefVALUs);
1567         else if (DefVALUs < State.ExecPos)
1568           PostExecPos = std::min(PostExecPos, DefVALUs);
1569       }
1570     }
1571 
1572     // Need a VALUs post exec change
1573     if (PostExecPos == std::numeric_limits<int>::max())
1574       return NoHazardFound;
1575 
1576     // Too many VALUs in intv3?
1577     int Intv3VALUs = PostExecPos;
1578     if (Intv3VALUs > Intv3MaxVALUs)
1579       return HazardExpired;
1580 
1581     // Too many VALUs in intv2?
1582     int Intv2VALUs = (State.ExecPos - PostExecPos) - 1;
1583     if (Intv2VALUs > Intv1plus2MaxVALUs)
1584       return HazardExpired;
1585 
1586     // Need a VALUs pre exec change
1587     if (PreExecPos == std::numeric_limits<int>::max())
1588       return NoHazardFound;
1589 
1590     // Too many VALUs in intv1?
1591     int Intv1VALUs = PreExecPos - State.ExecPos;
1592     if (Intv1VALUs > Intv1plus2MaxVALUs)
1593       return HazardExpired;
1594 
1595     // Too many VALUs in intv1 + intv2
1596     if (Intv1VALUs + Intv2VALUs > Intv1plus2MaxVALUs)
1597       return HazardExpired;
1598 
1599     return HazardFound;
1600   };
1601   auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
1602     if (SIInstrInfo::isVALU(MI))
1603       State.VALUs += 1;
1604   };
1605 
1606   DenseSet<const MachineBasicBlock *> Visited;
1607   if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(),
1608                             std::next(MI->getReverseIterator()), Visited))
1609     return false;
1610 
1611   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1612           TII.get(AMDGPU::S_WAITCNT_DEPCTR))
1613       .addImm(0x0fff);
1614 
1615   return true;
1616 }
1617 
1618 bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
1619   if (!ST.hasVALUTransUseHazard())
1620     return false;
1621   if (!SIInstrInfo::isVALU(*MI))
1622     return false;
1623 
1624   SmallSet<Register, 4> SrcVGPRs;
1625 
1626   for (const MachineOperand &Use : MI->explicit_uses()) {
1627     if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
1628       SrcVGPRs.insert(Use.getReg());
1629   }
1630 
1631   // Look for the following pattern:
1632   //   Va <- TRANS VALU
1633   //   intv
1634   //   MI Va (WaitState = 0)
1635   //
1636   // Where:
1637   // intv <= 5 VALUs / 1 TRANS
1638   //
1639   // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
1640 
1641   const int IntvMaxVALUs = 5;
1642   const int IntvMaxTRANS = 1;
1643 
1644   struct StateType {
1645     int VALUs = 0;
1646     int TRANS = 0;
1647   };
1648 
1649   StateType State;
1650 
1651   // This overloads expiry testing with all the hazard detection
1652   auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
1653     // Too many VALU states have passed
1654     if (State.VALUs > IntvMaxVALUs || State.TRANS > IntvMaxTRANS)
1655       return HazardExpired;
1656 
1657     // Instructions which cause va_vdst==0 expire hazard
1658     if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) ||
1659         SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I) ||
1660         (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1661          I.getOperand(0).getImm() == 0x0fff))
1662       return HazardExpired;
1663 
1664     // Track registers writes
1665     if (SIInstrInfo::isTRANS(I)) {
1666       for (Register Src : SrcVGPRs) {
1667         if (I.modifiesRegister(Src, &TRI)) {
1668           return HazardFound;
1669         }
1670       }
1671     }
1672 
1673     return NoHazardFound;
1674   };
1675   auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
1676     if (SIInstrInfo::isVALU(MI))
1677       State.VALUs += 1;
1678     if (SIInstrInfo::isTRANS(MI))
1679       State.TRANS += 1;
1680   };
1681 
1682   DenseSet<const MachineBasicBlock *> Visited;
1683   if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(),
1684                             std::next(MI->getReverseIterator()), Visited))
1685     return false;
1686 
1687   // Hazard is observed - insert a wait on va_dst counter to ensure hazard is
1688   // avoided (mask 0x0fff achieves this).
1689   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1690           TII.get(AMDGPU::S_WAITCNT_DEPCTR))
1691       .addImm(0x0fff);
1692 
1693   return true;
1694 }
1695 
1696 bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
1697   if (!SIInstrInfo::isWMMA(*MI))
1698     return false;
1699 
1700   const SIInstrInfo *TII = ST.getInstrInfo();
1701   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1702 
1703   auto IsHazardFn = [MI, TII, TRI](const MachineInstr &I) {
1704     if (!SIInstrInfo::isWMMA(I))
1705       return false;
1706 
1707     // Src0 or Src1 of the current wmma instruction overlaps with the dest of
1708     // the previous wmma.
1709     const Register CurSrc0Reg =
1710         TII->getNamedOperand(*MI, AMDGPU::OpName::src0)->getReg();
1711     const Register CurSrc1Reg =
1712         TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg();
1713 
1714     const Register PrevDstReg =
1715         TII->getNamedOperand(I, AMDGPU::OpName::vdst)->getReg();
1716 
1717     if (TRI->regsOverlap(PrevDstReg, CurSrc0Reg) ||
1718         TRI->regsOverlap(PrevDstReg, CurSrc1Reg)) {
1719       return true;
1720     }
1721 
1722     // Src2 of the current wmma instruction overlaps with the dest of the
1723     // previous wmma.
1724     const MachineOperand *Src2 =
1725         TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
1726     const Register CurSrc2Reg = Src2->isReg() ? Src2->getReg() : Register();
1727 
1728     if (CurSrc2Reg != AMDGPU::NoRegister &&
1729         TRI->regsOverlap(PrevDstReg, CurSrc2Reg)) {
1730 
1731       const MachineOperand *Src2Mods =
1732           TII->getNamedOperand(*MI, AMDGPU::OpName::src2_modifiers);
1733       const bool NoSrc2Mods =
1734           (Src2Mods->getImm() & (SISrcMods::NEG | SISrcMods::NEG_HI)) == 0;
1735       // Exception: there is no hazard if the wmma instructions are of the same
1736       // type and there is no input modifier on src2 of the current instruction.
1737       return !(NoSrc2Mods && (TII->pseudoToMCOpcode(I.getOpcode()) ==
1738                               TII->pseudoToMCOpcode(MI->getOpcode())));
1739     }
1740 
1741     return false;
1742   };
1743 
1744   auto IsExpiredFn = [](const MachineInstr &I, int) {
1745     return SIInstrInfo::isVALU(I);
1746   };
1747 
1748   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1749       std::numeric_limits<int>::max())
1750     return false;
1751 
1752   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32));
1753 
1754   return true;
1755 }
1756 
1757 bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) {
1758   if (!ST.hasShift64HighRegBug())
1759     return false;
1760 
1761   switch (MI->getOpcode()) {
1762   default:
1763     return false;
1764   case AMDGPU::V_LSHLREV_B64_e64:
1765   case AMDGPU::V_LSHRREV_B64_e64:
1766   case AMDGPU::V_ASHRREV_I64_e64:
1767     break;
1768   }
1769 
1770   MachineOperand *Amt = TII.getNamedOperand(*MI, AMDGPU::OpName::src0);
1771   if (!Amt->isReg())
1772     return false;
1773 
1774   Register AmtReg = Amt->getReg();
1775   const MachineRegisterInfo &MRI = MF.getRegInfo();
1776   // Check if this is a last VGPR in the allocation block.
1777   if (!TRI.isVGPR(MRI, AmtReg) || ((AmtReg - AMDGPU::VGPR0) & 7) != 7)
1778     return false;
1779 
1780   if (AmtReg != AMDGPU::VGPR255 && MRI.isPhysRegUsed(AmtReg + 1))
1781     return false;
1782 
1783   MachineOperand *Src1 = TII.getNamedOperand(*MI, AMDGPU::OpName::src1);
1784   bool OverlappedSrc = Src1->isReg() && TRI.regsOverlap(Src1->getReg(), AmtReg);
1785   bool OverlappedDst = MI->modifiesRegister(AmtReg, &TRI);
1786   bool Overlapped = OverlappedSrc || OverlappedDst;
1787 
1788   assert(!OverlappedDst || !OverlappedSrc ||
1789          Src1->getReg() == MI->getOperand(0).getReg());
1790   assert(ST.needsAlignedVGPRs());
1791   static_assert(AMDGPU::VGPR0 + 1 == AMDGPU::VGPR1);
1792 
1793   Register NewReg;
1794   for (MCRegister Reg : Overlapped ? AMDGPU::VReg_64_Align2RegClass
1795                                    : AMDGPU::VGPR_32RegClass) {
1796     if (!MI->modifiesRegister(Reg, &TRI) && !MI->readsRegister(Reg, &TRI)) {
1797       NewReg = Reg;
1798       break;
1799     }
1800   }
1801 
1802   Register NewAmt = Overlapped ? (Register)TRI.getSubReg(NewReg, AMDGPU::sub1)
1803                                : NewReg;
1804   Register NewAmtLo;
1805 
1806   if (Overlapped)
1807     NewAmtLo = TRI.getSubReg(NewReg, AMDGPU::sub0);
1808 
1809   DebugLoc DL = MI->getDebugLoc();
1810   MachineBasicBlock *MBB = MI->getParent();
1811   // Insert a full wait count because found register might be pending a wait.
1812   BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_WAITCNT))
1813       .addImm(0);
1814 
1815   // Insert V_SWAP_B32 instruction(s) and run hazard recognizer on them.
1816   if (Overlapped)
1817     runOnInstruction(
1818         BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmtLo)
1819             .addDef(AmtReg - 1)
1820             .addReg(AmtReg - 1, RegState::Undef)
1821             .addReg(NewAmtLo, RegState::Undef));
1822   runOnInstruction(BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmt)
1823                        .addDef(AmtReg)
1824                        .addReg(AmtReg, RegState::Undef)
1825                        .addReg(NewAmt, RegState::Undef));
1826 
1827   // Instructions emitted after the current instruction will be processed by the
1828   // parent loop of the hazard recognizer in a natural way.
1829   BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32),
1830           AmtReg)
1831       .addDef(NewAmt)
1832       .addReg(NewAmt)
1833       .addReg(AmtReg);
1834   if (Overlapped)
1835     BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32),
1836             AmtReg - 1)
1837         .addDef(NewAmtLo)
1838         .addReg(NewAmtLo)
1839         .addReg(AmtReg - 1);
1840 
1841   // Re-running hazard recognizer on the modified instruction is not necessary,
1842   // inserted V_SWAP_B32 has already both read and write new registers so
1843   // hazards related to these register has already been handled.
1844   Amt->setReg(NewAmt);
1845   Amt->setIsKill(false);
1846   // We do not update liveness, so verifier may see it as undef.
1847   Amt->setIsUndef(true);
1848   if (OverlappedDst)
1849     MI->getOperand(0).setReg(NewReg);
1850   if (OverlappedSrc) {
1851     Src1->setReg(NewReg);
1852     Src1->setIsKill(false);
1853     Src1->setIsUndef(true);
1854   }
1855 
1856   return true;
1857 }
1858 
1859 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) {
1860   int NSAtoVMEMWaitStates = 1;
1861 
1862   if (!ST.hasNSAtoVMEMBug())
1863     return 0;
1864 
1865   if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI))
1866     return 0;
1867 
1868   const SIInstrInfo *TII = ST.getInstrInfo();
1869   const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset);
1870   if (!Offset || (Offset->getImm() & 6) == 0)
1871     return 0;
1872 
1873   auto IsHazardFn = [TII](const MachineInstr &I) {
1874     if (!SIInstrInfo::isMIMG(I))
1875       return false;
1876     const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode());
1877     return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
1878            TII->getInstSizeInBytes(I) >= 16;
1879   };
1880 
1881   return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1);
1882 }
1883 
1884 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) {
1885   int FPAtomicToDenormModeWaitStates = 3;
1886 
1887   if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
1888     return 0;
1889 
1890   auto IsHazardFn = [](const MachineInstr &I) {
1891     if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I))
1892       return false;
1893     return SIInstrInfo::isFPAtomic(I);
1894   };
1895 
1896   auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) {
1897     if (WaitStates >= 3 || SIInstrInfo::isVALU(MI))
1898       return true;
1899 
1900     switch (MI.getOpcode()) {
1901     case AMDGPU::S_WAITCNT:
1902     case AMDGPU::S_WAITCNT_VSCNT:
1903     case AMDGPU::S_WAITCNT_VMCNT:
1904     case AMDGPU::S_WAITCNT_EXPCNT:
1905     case AMDGPU::S_WAITCNT_LGKMCNT:
1906     case AMDGPU::S_WAIT_IDLE:
1907       return true;
1908     default:
1909       break;
1910     }
1911 
1912     return false;
1913   };
1914 
1915   return FPAtomicToDenormModeWaitStates -
1916          ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn);
1917 }
1918 
1919 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) {
1920   assert(SIInstrInfo::isMAI(*MI));
1921 
1922   return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
1923 }
1924 
1925 int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) {
1926   // Early exit if no padding is requested.
1927   if (MFMAPaddingRatio == 0)
1928     return 0;
1929 
1930   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1931   if (!SIInstrInfo::isMFMA(*MI) || MFI->getOccupancy() < 2)
1932     return 0;
1933 
1934   int NeighborMFMALatency = 0;
1935   auto IsNeighboringMFMA = [&NeighborMFMALatency,
1936                             this](const MachineInstr &MI) {
1937     if (!SIInstrInfo::isMFMA(MI))
1938       return false;
1939 
1940     NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI);
1941     return true;
1942   };
1943 
1944   const int MaxMFMAPipelineWaitStates = 16;
1945   int WaitStatesSinceNeighborMFMA =
1946       getWaitStatesSince(IsNeighboringMFMA, MaxMFMAPipelineWaitStates);
1947 
1948   int NeighborMFMAPaddingNeeded =
1949       (NeighborMFMALatency * MFMAPaddingRatio / 100) -
1950       WaitStatesSinceNeighborMFMA;
1951 
1952   return std::max(0, NeighborMFMAPaddingNeeded);
1953 }
1954 
1955 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) {
1956   int WaitStatesNeeded = 0;
1957   unsigned Opc = MI->getOpcode();
1958 
1959   auto IsVALUFn = [](const MachineInstr &MI) {
1960     return SIInstrInfo::isVALU(MI);
1961   };
1962 
1963   if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
1964     const int LegacyVALUWritesVGPRWaitStates = 2;
1965     const int VALUWritesExecWaitStates = 4;
1966     const int MaxWaitStates = 4;
1967 
1968     int WaitStatesNeededForUse = VALUWritesExecWaitStates -
1969       getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates);
1970     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1971 
1972     if (WaitStatesNeeded < MaxWaitStates) {
1973       for (const MachineOperand &Use : MI->explicit_uses()) {
1974         const int MaxWaitStates = 2;
1975 
1976         if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
1977           continue;
1978 
1979         int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
1980           getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates);
1981         WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1982 
1983         if (WaitStatesNeeded == MaxWaitStates)
1984           break;
1985       }
1986     }
1987   }
1988 
1989   for (const MachineOperand &Op : MI->explicit_operands()) {
1990     if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg()))
1991       continue;
1992 
1993     if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1994       continue;
1995 
1996     const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
1997     const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
1998     const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
1999     const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
2000     const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
2001     const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
2002     const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
2003     const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
2004     const int MaxWaitStates = 18;
2005     Register Reg = Op.getReg();
2006     unsigned HazardDefLatency = 0;
2007 
2008     auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency,
2009                                this](const MachineInstr &MI) {
2010       if (!SIInstrInfo::isMFMA(MI))
2011         return false;
2012       Register DstReg = MI.getOperand(0).getReg();
2013       if (DstReg == Reg)
2014         return false;
2015       HazardDefLatency =
2016           std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
2017       return TRI.regsOverlap(DstReg, Reg);
2018     };
2019 
2020     int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn,
2021                                                    MaxWaitStates);
2022     int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
2023     int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
2024     int OpNo = MI->getOperandNo(&Op);
2025     if (OpNo == SrcCIdx) {
2026       NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
2027     } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
2028       switch (HazardDefLatency) {
2029       case 2:  NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
2030                break;
2031       case 8:  NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
2032                break;
2033       case 16: [[fallthrough]];
2034       default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
2035                break;
2036       }
2037     } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
2038       switch (HazardDefLatency) {
2039       case 2:  NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
2040                break;
2041       case 8:  NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
2042                break;
2043       case 16: [[fallthrough]];
2044       default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
2045                break;
2046       }
2047     }
2048 
2049     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
2050     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2051 
2052     if (WaitStatesNeeded == MaxWaitStates)
2053       return WaitStatesNeeded; // Early exit.
2054 
2055     auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) {
2056       if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
2057         return false;
2058       Register DstReg = MI.getOperand(0).getReg();
2059       return TRI.regsOverlap(Reg, DstReg);
2060     };
2061 
2062     const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
2063     const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
2064     const int AccVGPRWriteAccVgprReadWaitStates = 3;
2065     NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
2066     if (OpNo == SrcCIdx)
2067       NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
2068     else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
2069       NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
2070 
2071     WaitStatesNeededForUse = NeedWaitStates -
2072       getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates);
2073     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2074 
2075     if (WaitStatesNeeded == MaxWaitStates)
2076       return WaitStatesNeeded; // Early exit.
2077   }
2078 
2079   if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
2080     const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
2081     const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
2082     const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
2083     const int MaxWaitStates = 13;
2084     Register DstReg = MI->getOperand(0).getReg();
2085     unsigned HazardDefLatency = 0;
2086 
2087     auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency,
2088                          this](const MachineInstr &MI) {
2089       if (!SIInstrInfo::isMFMA(MI))
2090         return false;
2091       Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg();
2092       HazardDefLatency =
2093           std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
2094       return TRI.regsOverlap(Reg, DstReg);
2095     };
2096 
2097     int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates);
2098     int NeedWaitStates;
2099     switch (HazardDefLatency) {
2100     case 2:  NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
2101              break;
2102     case 8:  NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
2103              break;
2104     case 16: [[fallthrough]];
2105     default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
2106              break;
2107     }
2108 
2109     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
2110     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2111   }
2112 
2113   // Pad neighboring MFMA with noops for better inter-wave performance.
2114   WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI));
2115 
2116   return WaitStatesNeeded;
2117 }
2118 
2119 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
2120   int WaitStatesNeeded = 0;
2121   unsigned Opc = MI->getOpcode();
2122 
2123   auto IsLegacyVALUFn = [](const MachineInstr &MI) {
2124     return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI);
2125   };
2126 
2127   auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) {
2128     return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI) &&
2129            !SIInstrInfo::isDOT(MI);
2130   };
2131 
2132   if (!SIInstrInfo::isMFMA(*MI))
2133     return WaitStatesNeeded;
2134 
2135   const int VALUWritesExecWaitStates = 4;
2136   int WaitStatesNeededForUse = VALUWritesExecWaitStates -
2137     getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn,
2138                           VALUWritesExecWaitStates);
2139   WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2140 
2141   int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
2142 
2143   // Loop for both DGEMM and S/HGEMM 2nd instruction.
2144   for (const MachineOperand &Use : MI->explicit_uses()) {
2145     const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
2146     const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
2147     const int GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates = 3;
2148     const int GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates = 5;
2149     const int GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates = 4;
2150     const int GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates = 9;
2151     const int GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates = 8;
2152     const int GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates = 17;
2153     const int GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates = 16;
2154     const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
2155     const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
2156     const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
2157     const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
2158     const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
2159     const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
2160     const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
2161     const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
2162     const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
2163     const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
2164     const int GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates = 4;
2165     const int GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates = 6;
2166     const int GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates = 10;
2167     const int GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates = 18;
2168     const int GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates = 5;
2169     const int GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates = 7;
2170     const int GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates = 11;
2171     const int GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates = 19;
2172     const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
2173     const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
2174     const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
2175     const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2;
2176     const int MaxWaitStates = 19;
2177 
2178     if (!Use.isReg())
2179       continue;
2180     Register Reg = Use.getReg();
2181     bool FullReg;
2182     const MachineInstr *MI1;
2183 
2184     auto IsOverlappedMFMAFn = [Reg, &FullReg, &MI1,
2185                                this](const MachineInstr &MI) {
2186       if (!SIInstrInfo::isMFMA(MI))
2187         return false;
2188       Register DstReg = MI.getOperand(0).getReg();
2189       FullReg = (DstReg == Reg);
2190       MI1 = &MI;
2191       return TRI.regsOverlap(DstReg, Reg);
2192     };
2193 
2194     WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
2195       getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates);
2196     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2197 
2198     int NumWaitStates =
2199         getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates);
2200     if (NumWaitStates == std::numeric_limits<int>::max())
2201       continue;
2202 
2203     int OpNo = MI->getOperandNo(&Use);
2204     unsigned Opc1 = MI1->getOpcode();
2205     int NeedWaitStates = 0;
2206     if (OpNo == SrcCIdx) {
2207       if (!isDGEMM(Opc) && (!ST.hasGFX940Insts() && isDGEMM(Opc1))) {
2208         NeedWaitStates = 0;
2209       } else if (FullReg) {
2210         if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
2211              Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
2212             (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
2213              Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
2214           NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
2215         else if (ST.hasGFX940Insts() &&
2216                  TSchedModel.computeInstrLatency(MI1) == 2)
2217           NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
2218       } else {
2219         switch (Opc1) {
2220         case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
2221         case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
2222         case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
2223         case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
2224           if (!isXDL(ST, *MI))
2225             NeedWaitStates = DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
2226           break;
2227         case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
2228         case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
2229           if (!isXDL(ST, *MI))
2230             NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
2231           break;
2232         default:
2233           if (ST.hasGFX940Insts() && isXDL(ST, *MI) && !isXDL(ST, *MI1))
2234             break;
2235           switch (TSchedModel.computeInstrLatency(MI1)) {
2236           case 2:
2237             NeedWaitStates = ST.hasGFX940Insts()
2238               ? isXDL(ST, *MI1)
2239                 ? GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates
2240                 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates
2241               : isDGEMM(Opc)
2242                 ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
2243                 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
2244             break;
2245           case 4:
2246             assert(ST.hasGFX940Insts());
2247             NeedWaitStates = isXDL(ST, *MI1)
2248               ? GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates
2249               : GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates;
2250             break;
2251           case 8:
2252             NeedWaitStates = ST.hasGFX940Insts()
2253               ? isXDL(ST, *MI1)
2254                 ? GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates
2255                 : GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates
2256               : isDGEMM(Opc)
2257                 ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
2258                 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
2259             break;
2260           case 16: [[fallthrough]];
2261           default:
2262             NeedWaitStates = ST.hasGFX940Insts()
2263               ? isXDL(ST, *MI1)
2264                 ? GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates
2265                 : GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates
2266               : isDGEMM(Opc)
2267                 ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
2268                 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
2269           }
2270         }
2271       }
2272     } else {
2273       switch (Opc1) {
2274       case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
2275       case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
2276       case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
2277       case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
2278         NeedWaitStates = DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
2279         break;
2280       case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
2281       case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
2282         NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
2283         break;
2284       default:
2285         switch (TSchedModel.computeInstrLatency(MI1)) {
2286         case 2:
2287           NeedWaitStates = ST.hasGFX940Insts()
2288             ? isXDL(ST, *MI1)
2289               ? GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates
2290               : GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates
2291             : SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
2292           break;
2293         case 4:
2294           assert(ST.hasGFX940Insts());
2295           NeedWaitStates = isXDL(ST, *MI1)
2296             ? GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates
2297             : GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates;
2298           break;
2299         case 8:
2300           NeedWaitStates = ST.hasGFX940Insts()
2301             ? isXDL(ST, *MI1)
2302               ? GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates
2303               : GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates
2304             : SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
2305           break;
2306         case 16: [[fallthrough]];
2307         default:
2308           NeedWaitStates = ST.hasGFX940Insts()
2309             ? isXDL(ST, *MI1)
2310               ? GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates
2311               : GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates
2312             : SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
2313         }
2314       }
2315     }
2316     if (WaitStatesNeeded >= NeedWaitStates)
2317       continue;
2318 
2319     WaitStatesNeededForUse = NeedWaitStates - NumWaitStates;
2320     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2321 
2322     if (WaitStatesNeeded == MaxWaitStates)
2323       break;
2324   }
2325 
2326   return WaitStatesNeeded;
2327 }
2328 
2329 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) {
2330   // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards()
2331   if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
2332     return 0;
2333 
2334   int WaitStatesNeeded = 0;
2335 
2336   auto IsAccVgprReadFn = [](const MachineInstr &MI) {
2337     return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
2338   };
2339 
2340   for (const MachineOperand &Op : MI->explicit_uses()) {
2341     if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg()))
2342       continue;
2343 
2344     Register Reg = Op.getReg();
2345 
2346     const int AccVgprReadLdStWaitStates = 2;
2347     const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
2348     const int MaxWaitStates = 2;
2349 
2350     int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
2351       getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates);
2352     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2353 
2354     if (WaitStatesNeeded == MaxWaitStates)
2355       return WaitStatesNeeded; // Early exit.
2356 
2357     auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) {
2358       if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
2359           MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
2360         return false;
2361       auto IsVALUFn = [](const MachineInstr &MI) {
2362         return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMAI(MI);
2363       };
2364       return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) <
2365              std::numeric_limits<int>::max();
2366     };
2367 
2368     WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
2369       getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates);
2370     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2371   }
2372 
2373   return WaitStatesNeeded;
2374 }
2375 
2376 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
2377   if (!ST.hasGFX90AInsts())
2378     return 0;
2379 
2380   auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
2381     return isDGEMM(MI.getOpcode());
2382   };
2383 
2384   // This is checked in checkMAIHazards90A()
2385   if (SIInstrInfo::isMFMA(*MI))
2386     return 0;
2387 
2388   const MachineRegisterInfo &MRI = MF.getRegInfo();
2389 
2390   int WaitStatesNeeded = 0;
2391 
2392   bool IsMem = SIInstrInfo::isVMEM(*MI) ||
2393                SIInstrInfo::isFLAT(*MI) ||
2394                SIInstrInfo::isDS(*MI);
2395   bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(*MI);
2396   bool IsVALU = SIInstrInfo::isVALU(*MI);
2397 
2398   const MachineInstr *MFMA = nullptr;
2399   unsigned Reg;
2400   auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
2401     if (!SIInstrInfo::isMFMA(MI) ||
2402         !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
2403       return false;
2404     MFMA = &MI;
2405     return true;
2406   };
2407 
2408   const MachineInstr *DOT = nullptr;
2409   auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) {
2410     if (!SIInstrInfo::isDOT(MI) ||
2411         !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
2412       return false;
2413     DOT = &MI;
2414     return true;
2415   };
2416 
2417   bool DGEMMAfterVALUWrite = false;
2418   auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) {
2419     // Found DGEMM on reverse traversal to def.
2420     if (isDGEMM(MI.getOpcode()))
2421       DGEMMAfterVALUWrite = true;
2422 
2423     // Only hazard if register is defined by a VALU and a DGEMM is found after
2424     // after the def.
2425     if (!TII.isVALU(MI) || !DGEMMAfterVALUWrite)
2426       return false;
2427 
2428     return true;
2429   };
2430 
2431   int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(),
2432                                            AMDGPU::OpName::src2);
2433 
2434   if (IsMemOrExport || IsVALU) {
2435     const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
2436     const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
2437     const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
2438     const int GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates = 4;
2439     const int GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates = 6;
2440     const int GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates = 10;
2441     const int GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates = 18;
2442     const int GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates = 5;
2443     const int GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates = 7;
2444     const int GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates = 11;
2445     const int GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates = 19;
2446     const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
2447     const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
2448     const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
2449     const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
2450     const int DotWriteSameDotReadSrcAB = 3;
2451     const int DotWriteDifferentVALURead = 3;
2452     const int DMFMABetweenVALUWriteVMEMRead = 2;
2453     const int MaxWaitStates = 19;
2454 
2455     for (const MachineOperand &Use : MI->explicit_uses()) {
2456       if (!Use.isReg())
2457         continue;
2458       Reg = Use.getReg();
2459 
2460       DOT = nullptr;
2461       int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
2462                                                      MaxWaitStates);
2463       if (DOT) {
2464         int NeedWaitStates = 0;
2465         if (DOT->getOpcode() == MI->getOpcode()) {
2466           if (&Use - &MI->getOperand(0) != SrcCIdx)
2467             NeedWaitStates = DotWriteSameDotReadSrcAB;
2468         } else {
2469           NeedWaitStates = DotWriteDifferentVALURead;
2470         }
2471 
2472         int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
2473         WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2474       }
2475 
2476       // Workaround for HW data hazard bug observed only in GFX90A. When there
2477       // is a DGEMM instruction in-between a VALU and a VMEM instruction it
2478       // causes the SQ to incorrectly not insert two wait states between the two
2479       // instructions needed to avoid data hazard.
2480       if (IsMem && ST.hasGFX90AInsts() && !ST.hasGFX940Insts()) {
2481         DGEMMAfterVALUWrite = false;
2482         if (TRI.isVectorRegister(MRI, Reg)) {
2483           int WaitStatesNeededForUse =
2484                 DMFMABetweenVALUWriteVMEMRead -
2485                 getWaitStatesSinceDef(Reg, IsDGEMMHazard,
2486                                       DMFMABetweenVALUWriteVMEMRead);
2487 
2488           WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2489         }
2490       }
2491 
2492       MFMA = nullptr;
2493       WaitStatesSinceDef =
2494           getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
2495       if (!MFMA)
2496         continue;
2497 
2498       unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
2499       int NeedWaitStates = MaxWaitStates;
2500       switch (HazardDefLatency) {
2501       case 2:
2502         NeedWaitStates =
2503           ST.hasGFX940Insts()
2504             ? isXDL(ST, *MFMA)
2505               ? GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates
2506               : GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates
2507             : SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
2508         break;
2509       case 4:
2510         assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts());
2511         NeedWaitStates =
2512           isDGEMM(MFMA->getOpcode())
2513             ? IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
2514                             : DMFMA4x4WriteVgprVALUReadWaitStates
2515             : isXDL(ST, *MFMA)
2516               ? GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates
2517               : GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates;
2518         break;
2519       case 8:
2520         NeedWaitStates =
2521           ST.hasGFX940Insts()
2522             ? isXDL(ST, *MFMA)
2523               ? GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates
2524               : GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates
2525             : SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
2526         break;
2527       case 16: [[fallthrough]];
2528       default:
2529         NeedWaitStates =
2530           isDGEMM(MFMA->getOpcode())
2531             ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates
2532                             : DMFMA16x16WriteVgprVALUReadWaitStates
2533             : ST.hasGFX940Insts()
2534               ? isXDL(ST, *MFMA)
2535                 ? GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates
2536                 : GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates
2537               : SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
2538         break;
2539       }
2540 
2541       int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
2542       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2543 
2544       if (WaitStatesNeeded == MaxWaitStates)
2545         break;
2546     }
2547   }
2548 
2549   unsigned Opc = MI->getOpcode();
2550   const int DMFMAToFMA64WaitStates = 2;
2551   if ((Opc == AMDGPU::V_FMA_F64_e64 ||
2552        Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
2553        Opc == AMDGPU::V_FMAC_F64_dpp) &&
2554       WaitStatesNeeded < DMFMAToFMA64WaitStates) {
2555     int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
2556       getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates);
2557     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2558   }
2559 
2560   if (!IsVALU && !IsMemOrExport)
2561     return WaitStatesNeeded;
2562 
2563   for (const MachineOperand &Def : MI->defs()) {
2564     const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
2565     const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
2566     const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
2567     const int GFX940_SMFMA2PassWriteVgprVALUWawWaitStates = 4;
2568     const int GFX940_SMFMA4PassWriteVgprVALUWawWaitStates = 6;
2569     const int GFX940_SMFMA8PassWriteVgprVALUWawWaitStates = 10;
2570     const int GFX940_SMFMA16PassWriteVgprVALUWawWaitStates = 18;
2571     const int GFX940_XDL2PassWriteVgprVALUWawWaitStates = 5;
2572     const int GFX940_XDL4PassWriteVgprVALUWawWaitStates = 7;
2573     const int GFX940_XDL8PassWriteVgprVALUWawWaitStates = 11;
2574     const int GFX940_XDL16PassWriteVgprVALUWawWaitStates = 19;
2575     const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
2576     const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3;
2577     const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
2578     const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
2579     const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
2580     const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
2581     const int DotWriteDifferentVALUWrite = 3;
2582     const int MaxWaitStates = 19;
2583     const int MaxWarWaitStates = 15;
2584 
2585     Reg = Def.getReg();
2586 
2587     DOT = nullptr;
2588     int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
2589                                                    MaxWaitStates);
2590     if (DOT && DOT->getOpcode() != MI->getOpcode())
2591       WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite -
2592                                                     WaitStatesSinceDef);
2593 
2594     MFMA = nullptr;
2595     WaitStatesSinceDef =
2596         getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
2597     if (MFMA) {
2598       int NeedWaitStates = MaxWaitStates;
2599       switch (TSchedModel.computeInstrLatency(MFMA)) {
2600       case 2:
2601         NeedWaitStates = ST.hasGFX940Insts()
2602           ? isXDL(ST, *MFMA)
2603             ? GFX940_XDL2PassWriteVgprVALUWawWaitStates
2604             : GFX940_SMFMA2PassWriteVgprVALUWawWaitStates
2605           : SMFMA4x4WriteVgprVALUWawWaitStates;
2606         break;
2607       case 4:
2608         assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts());
2609         NeedWaitStates = isDGEMM(MFMA->getOpcode())
2610             ? DMFMA4x4WriteVgprVALUWriteWaitStates
2611             : isXDL(ST, *MFMA)
2612               ? GFX940_XDL4PassWriteVgprVALUWawWaitStates
2613               : GFX940_SMFMA4PassWriteVgprVALUWawWaitStates;
2614         break;
2615       case 8:
2616         NeedWaitStates = ST.hasGFX940Insts()
2617           ? isXDL(ST, *MFMA)
2618             ? GFX940_XDL8PassWriteVgprVALUWawWaitStates
2619             : GFX940_SMFMA8PassWriteVgprVALUWawWaitStates
2620           : SMFMA16x16WriteVgprVALUWawWaitStates;
2621         break;
2622       case 16: [[fallthrough]];
2623       default:
2624         NeedWaitStates = isDGEMM(MFMA->getOpcode())
2625                    ? DMFMA16x16WriteVgprVALUWriteWaitStates
2626                    : ST.hasGFX940Insts()
2627                      ? isXDL(ST, *MFMA)
2628                        ? GFX940_XDL16PassWriteVgprVALUWawWaitStates
2629                        : GFX940_SMFMA16PassWriteVgprVALUWawWaitStates
2630                    : SMFMA32x32WriteVgprVALUWawWaitStates;
2631         break;
2632       }
2633 
2634       int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
2635       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2636 
2637       if (WaitStatesNeeded == MaxWaitStates)
2638         break;
2639     }
2640 
2641     auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
2642       if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) ||
2643           !MI.readsRegister(Reg, &TRI))
2644         return false;
2645 
2646       if (ST.hasGFX940Insts() && !isXDL(ST, MI))
2647         return false;
2648 
2649       const MachineOperand *SrcC =
2650           TII.getNamedOperand(MI, AMDGPU::OpName::src2);
2651       assert(SrcC);
2652       if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg))
2653         return false;
2654 
2655       MFMA = &MI;
2656       return true;
2657     };
2658 
2659     MFMA = nullptr;
2660     int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn,
2661                                                 MaxWarWaitStates);
2662     if (!MFMA)
2663       continue;
2664 
2665     unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
2666     int NeedWaitStates = MaxWaitStates;
2667     switch (HazardDefLatency) {
2668     case 2:  NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
2669              break;
2670     case 4:  assert(ST.hasGFX940Insts());
2671              NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates;
2672              break;
2673     case 8:  NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates;
2674              break;
2675     case 16: [[fallthrough]];
2676     default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates;
2677              break;
2678     }
2679 
2680     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse;
2681     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2682   }
2683 
2684   return WaitStatesNeeded;
2685 }
2686 
2687 bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) {
2688   if (!SU->isInstr())
2689     return false;
2690 
2691   const MachineInstr *MAI = nullptr;
2692 
2693   auto IsMFMAFn = [&MAI](const MachineInstr &MI) {
2694     MAI = nullptr;
2695     if (SIInstrInfo::isMFMA(MI))
2696       MAI = &MI;
2697     return MAI != nullptr;
2698   };
2699 
2700   MachineInstr *MI = SU->getInstr();
2701   if (IsMFMAFn(*MI)) {
2702     int W = getWaitStatesSince(IsMFMAFn, 16);
2703     if (MAI)
2704       return W < (int)TSchedModel.computeInstrLatency(MAI);
2705   }
2706 
2707   return false;
2708 }
2709