xref: /llvm-project/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp (revision b120dae9bb99b67d12c7b307debb222953473b7c)
1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements hazard recognizers for scheduling on GCN processors.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "GCNHazardRecognizer.h"
14 #include "GCNSubtarget.h"
15 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
16 #include "SIMachineFunctionInfo.h"
17 #include "llvm/CodeGen/MachineFunction.h"
18 #include "llvm/CodeGen/ScheduleDAG.h"
19 #include "llvm/TargetParser/TargetParser.h"
20 
21 using namespace llvm;
22 
23 namespace {
24 
25 struct MFMAPaddingRatioParser : public cl::parser<unsigned> {
26   MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {}
27 
28   bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
29     if (Arg.getAsInteger(0, Value))
30       return O.error("'" + Arg + "' value invalid for uint argument!");
31 
32     if (Value > 100)
33       return O.error("'" + Arg + "' value must be in the range [0, 100]!");
34 
35     return false;
36   }
37 };
38 
39 } // end anonymous namespace
40 
41 static cl::opt<unsigned, false, MFMAPaddingRatioParser>
42     MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden,
43                      cl::desc("Fill a percentage of the latency between "
44                               "neighboring MFMA with s_nops."));
45 
46 //===----------------------------------------------------------------------===//
47 // Hazard Recognizer Implementation
48 //===----------------------------------------------------------------------===//
49 
50 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
51                                                  const GCNSubtarget &ST);
52 
53 GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) :
54   IsHazardRecognizerMode(false),
55   CurrCycleInstr(nullptr),
56   MF(MF),
57   ST(MF.getSubtarget<GCNSubtarget>()),
58   TII(*ST.getInstrInfo()),
59   TRI(TII.getRegisterInfo()),
60   ClauseUses(TRI.getNumRegUnits()),
61   ClauseDefs(TRI.getNumRegUnits()) {
62   MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5;
63   TSchedModel.init(&ST);
64   RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
65 }
66 
67 void GCNHazardRecognizer::Reset() {
68   EmittedInstrs.clear();
69 }
70 
71 void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
72   EmitInstruction(SU->getInstr());
73 }
74 
75 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
76   CurrCycleInstr = MI;
77 }
78 
79 static bool isDivFMas(unsigned Opcode) {
80   return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
81 }
82 
83 static bool isSGetReg(unsigned Opcode) {
84   return Opcode == AMDGPU::S_GETREG_B32;
85 }
86 
87 static bool isSSetReg(unsigned Opcode) {
88   switch (Opcode) {
89   case AMDGPU::S_SETREG_B32:
90   case AMDGPU::S_SETREG_B32_mode:
91   case AMDGPU::S_SETREG_IMM32_B32:
92   case AMDGPU::S_SETREG_IMM32_B32_mode:
93     return true;
94   }
95   return false;
96 }
97 
98 static bool isRWLane(unsigned Opcode) {
99   return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
100 }
101 
102 static bool isRFE(unsigned Opcode) {
103   return Opcode == AMDGPU::S_RFE_B64;
104 }
105 
106 static bool isSMovRel(unsigned Opcode) {
107   switch (Opcode) {
108   case AMDGPU::S_MOVRELS_B32:
109   case AMDGPU::S_MOVRELS_B64:
110   case AMDGPU::S_MOVRELD_B32:
111   case AMDGPU::S_MOVRELD_B64:
112     return true;
113   default:
114     return false;
115   }
116 }
117 
118 static bool isDGEMM(unsigned Opcode) {
119   return AMDGPU::getMAIIsDGEMM(Opcode);
120 }
121 
122 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) {
123   unsigned Opcode = MI.getOpcode();
124 
125   if (!SIInstrInfo::isMAI(MI) ||
126       isDGEMM(Opcode) ||
127       Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
128       Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
129     return false;
130 
131   if (!ST.hasGFX940Insts())
132     return true;
133 
134   return AMDGPU::getMAIIsGFX940XDL(Opcode);
135 }
136 
137 static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
138                                     const MachineInstr &MI) {
139   if (TII.isAlwaysGDS(MI.getOpcode()))
140     return true;
141 
142   switch (MI.getOpcode()) {
143   case AMDGPU::S_SENDMSG:
144   case AMDGPU::S_SENDMSGHALT:
145   case AMDGPU::S_TTRACEDATA:
146     return true;
147   // These DS opcodes don't support GDS.
148   case AMDGPU::DS_NOP:
149   case AMDGPU::DS_PERMUTE_B32:
150   case AMDGPU::DS_BPERMUTE_B32:
151     return false;
152   default:
153     if (TII.isDS(MI.getOpcode())) {
154       int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
155                                            AMDGPU::OpName::gds);
156       if (MI.getOperand(GDS).getImm())
157         return true;
158     }
159     return false;
160   }
161 }
162 
163 static bool isPermlane(const MachineInstr &MI) {
164   unsigned Opcode = MI.getOpcode();
165   return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
166          Opcode == AMDGPU::V_PERMLANEX16_B32_e64 ||
167          Opcode == AMDGPU::V_PERMLANE16_VAR_B32_e64 ||
168          Opcode == AMDGPU::V_PERMLANEX16_VAR_B32_e64;
169 }
170 
171 static bool isLdsDma(const MachineInstr &MI) {
172   return SIInstrInfo::isVALU(MI) &&
173          (SIInstrInfo::isMUBUF(MI) || SIInstrInfo::isFLAT(MI));
174 }
175 
176 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
177   const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
178                                                      AMDGPU::OpName::simm16);
179   return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
180 }
181 
182 ScheduleHazardRecognizer::HazardType
183 GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
184   MachineInstr *MI = SU->getInstr();
185   // If we are not in "HazardRecognizerMode" and therefore not being run from
186   // the scheduler, track possible stalls from hazards but don't insert noops.
187   auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard;
188 
189   if (MI->isBundle())
190    return NoHazard;
191 
192   if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
193     return HazardType;
194 
195   if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
196     return HazardType;
197 
198   if (checkFPAtomicToDenormModeHazard(MI) > 0)
199     return HazardType;
200 
201   if (ST.hasNoDataDepHazard())
202     return NoHazard;
203 
204   // FIXME: Should flat be considered vmem?
205   if ((SIInstrInfo::isVMEM(*MI) ||
206        SIInstrInfo::isFLAT(*MI))
207       && checkVMEMHazards(MI) > 0)
208     return HazardType;
209 
210   if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
211     return HazardType;
212 
213   if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
214     return HazardType;
215 
216   if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
217     return HazardType;
218 
219   if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
220     return HazardType;
221 
222   if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) ||
223        SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) ||
224        SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
225     return HazardType;
226 
227   if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
228     return HazardType;
229 
230   if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
231     return HazardType;
232 
233   if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
234     return HazardType;
235 
236   if (((ST.hasReadM0MovRelInterpHazard() &&
237         (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) ||
238          MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
239          MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
240        (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) ||
241        (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) ||
242        (ST.hasReadM0LdsDirectHazard() &&
243         MI->readsRegister(AMDGPU::LDS_DIRECT))) &&
244       checkReadM0Hazards(MI) > 0)
245     return HazardType;
246 
247   if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0)
248     return HazardType;
249 
250   if ((SIInstrInfo::isVMEM(*MI) ||
251        SIInstrInfo::isFLAT(*MI) ||
252        SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0)
253     return HazardType;
254 
255   if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
256     return HazardType;
257 
258   return NoHazard;
259 }
260 
261 static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII,
262                                 unsigned Quantity) {
263   while (Quantity > 0) {
264     unsigned Arg = std::min(Quantity, 8u);
265     Quantity -= Arg;
266     BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP))
267         .addImm(Arg - 1);
268   }
269 }
270 
271 unsigned
272 GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const {
273   const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&MI);
274   assert(TSchedModel.getWriteProcResBegin(SC) !=
275          TSchedModel.getWriteProcResEnd(SC));
276   return TSchedModel.getWriteProcResBegin(SC)->ReleaseAtCycle;
277 }
278 
279 void GCNHazardRecognizer::processBundle() {
280   MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator());
281   MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end();
282   // Check bundled MachineInstr's for hazards.
283   for (; MI != E && MI->isInsideBundle(); ++MI) {
284     CurrCycleInstr = &*MI;
285     unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
286 
287     if (IsHazardRecognizerMode) {
288       fixHazards(CurrCycleInstr);
289 
290       insertNoopsInBundle(CurrCycleInstr, TII, WaitStates);
291     }
292 
293     // It’s unnecessary to track more than MaxLookAhead instructions. Since we
294     // include the bundled MI directly after, only add a maximum of
295     // (MaxLookAhead - 1) noops to EmittedInstrs.
296     for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i)
297       EmittedInstrs.push_front(nullptr);
298 
299     EmittedInstrs.push_front(CurrCycleInstr);
300     EmittedInstrs.resize(MaxLookAhead);
301   }
302   CurrCycleInstr = nullptr;
303 }
304 
305 void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) {
306   assert(IsHazardRecognizerMode);
307 
308   unsigned NumPreNoops = PreEmitNoops(MI);
309   EmitNoops(NumPreNoops);
310   if (MI->isInsideBundle())
311     insertNoopsInBundle(MI, TII, NumPreNoops);
312   else
313     TII.insertNoops(*MI->getParent(), MachineBasicBlock::iterator(MI),
314                     NumPreNoops);
315   EmitInstruction(MI);
316   AdvanceCycle();
317 }
318 
319 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
320   IsHazardRecognizerMode = true;
321   CurrCycleInstr = MI;
322   unsigned W = PreEmitNoopsCommon(MI);
323   fixHazards(MI);
324   CurrCycleInstr = nullptr;
325   return W;
326 }
327 
328 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) {
329   if (MI->isBundle())
330     return 0;
331 
332   int WaitStates = 0;
333 
334   if (SIInstrInfo::isSMRD(*MI))
335     return std::max(WaitStates, checkSMRDHazards(MI));
336 
337   if (ST.hasNSAtoVMEMBug())
338     WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI));
339 
340   WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI));
341 
342   if (ST.hasNoDataDepHazard())
343     return WaitStates;
344 
345   if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI))
346     WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
347 
348   if (SIInstrInfo::isVALU(*MI))
349     WaitStates = std::max(WaitStates, checkVALUHazards(MI));
350 
351   if (SIInstrInfo::isDPP(*MI))
352     WaitStates = std::max(WaitStates, checkDPPHazards(MI));
353 
354   if (isDivFMas(MI->getOpcode()))
355     WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
356 
357   if (isRWLane(MI->getOpcode()))
358     WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
359 
360   if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) ||
361        SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) ||
362        SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
363     WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI));
364 
365   if (MI->isInlineAsm())
366     return std::max(WaitStates, checkInlineAsmHazards(MI));
367 
368   if (isSGetReg(MI->getOpcode()))
369     return std::max(WaitStates, checkGetRegHazards(MI));
370 
371   if (isSSetReg(MI->getOpcode()))
372     return std::max(WaitStates, checkSetRegHazards(MI));
373 
374   if (isRFE(MI->getOpcode()))
375     return std::max(WaitStates, checkRFEHazards(MI));
376 
377   if ((ST.hasReadM0MovRelInterpHazard() &&
378        (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) ||
379         MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
380         MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
381       (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) ||
382       (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) ||
383       (ST.hasReadM0LdsDirectHazard() && MI->readsRegister(AMDGPU::LDS_DIRECT)))
384     return std::max(WaitStates, checkReadM0Hazards(MI));
385 
386   if (SIInstrInfo::isMAI(*MI))
387     return std::max(WaitStates, checkMAIHazards(MI));
388 
389   if (SIInstrInfo::isVMEM(*MI) ||
390       SIInstrInfo::isFLAT(*MI) ||
391       SIInstrInfo::isDS(*MI))
392     return std::max(WaitStates, checkMAILdStHazards(MI));
393 
394   return WaitStates;
395 }
396 
397 void GCNHazardRecognizer::EmitNoop() {
398   EmittedInstrs.push_front(nullptr);
399 }
400 
401 void GCNHazardRecognizer::AdvanceCycle() {
402   // When the scheduler detects a stall, it will call AdvanceCycle() without
403   // emitting any instructions.
404   if (!CurrCycleInstr) {
405     EmittedInstrs.push_front(nullptr);
406     return;
407   }
408 
409   if (CurrCycleInstr->isBundle()) {
410     processBundle();
411     return;
412   }
413 
414   unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
415   if (!NumWaitStates) {
416     CurrCycleInstr = nullptr;
417     return;
418   }
419 
420   // Keep track of emitted instructions
421   EmittedInstrs.push_front(CurrCycleInstr);
422 
423   // Add a nullptr for each additional wait state after the first.  Make sure
424   // not to add more than getMaxLookAhead() items to the list, since we
425   // truncate the list to that size right after this loop.
426   for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
427        i < e; ++i) {
428     EmittedInstrs.push_front(nullptr);
429   }
430 
431   // getMaxLookahead() is the largest number of wait states we will ever need
432   // to insert, so there is no point in keeping track of more than that many
433   // wait states.
434   EmittedInstrs.resize(getMaxLookAhead());
435 
436   CurrCycleInstr = nullptr;
437 }
438 
439 void GCNHazardRecognizer::RecedeCycle() {
440   llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
441 }
442 
443 //===----------------------------------------------------------------------===//
444 // Helper Functions
445 //===----------------------------------------------------------------------===//
446 
447 typedef enum { HazardFound, HazardExpired, NoHazardFound } HazardFnResult;
448 
449 typedef function_ref<bool(const MachineInstr &, int WaitStates)> IsExpiredFn;
450 typedef function_ref<unsigned int(const MachineInstr &)> GetNumWaitStatesFn;
451 
452 // Search for a hazard in a block and its predecessors.
453 template <typename StateT>
454 static bool
455 hasHazard(StateT State,
456           function_ref<HazardFnResult(StateT &, const MachineInstr &)> IsHazard,
457           function_ref<void(StateT &, const MachineInstr &)> UpdateState,
458           const MachineBasicBlock *MBB,
459           MachineBasicBlock::const_reverse_instr_iterator I,
460           DenseSet<const MachineBasicBlock *> &Visited) {
461   for (auto E = MBB->instr_rend(); I != E; ++I) {
462     // No need to look at parent BUNDLE instructions.
463     if (I->isBundle())
464       continue;
465 
466     switch (IsHazard(State, *I)) {
467     case HazardFound:
468       return true;
469     case HazardExpired:
470       return false;
471     default:
472       // Continue search
473       break;
474     }
475 
476     if (I->isInlineAsm() || I->isMetaInstruction())
477       continue;
478 
479     UpdateState(State, *I);
480   }
481 
482   for (MachineBasicBlock *Pred : MBB->predecessors()) {
483     if (!Visited.insert(Pred).second)
484       continue;
485 
486     if (hasHazard(State, IsHazard, UpdateState, Pred, Pred->instr_rbegin(),
487                   Visited))
488       return true;
489   }
490 
491   return false;
492 }
493 
494 // Returns a minimum wait states since \p I walking all predecessors.
495 // Only scans until \p IsExpired does not return true.
496 // Can only be run in a hazard recognizer mode.
497 static int getWaitStatesSince(
498     GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock *MBB,
499     MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates,
500     IsExpiredFn IsExpired, DenseSet<const MachineBasicBlock *> &Visited,
501     GetNumWaitStatesFn GetNumWaitStates = SIInstrInfo::getNumWaitStates) {
502   for (auto E = MBB->instr_rend(); I != E; ++I) {
503     // Don't add WaitStates for parent BUNDLE instructions.
504     if (I->isBundle())
505       continue;
506 
507     if (IsHazard(*I))
508       return WaitStates;
509 
510     if (I->isInlineAsm())
511       continue;
512 
513     WaitStates += GetNumWaitStates(*I);
514 
515     if (IsExpired(*I, WaitStates))
516       return std::numeric_limits<int>::max();
517   }
518 
519   int MinWaitStates = std::numeric_limits<int>::max();
520   for (MachineBasicBlock *Pred : MBB->predecessors()) {
521     if (!Visited.insert(Pred).second)
522       continue;
523 
524     int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(), WaitStates,
525                                IsExpired, Visited, GetNumWaitStates);
526 
527     MinWaitStates = std::min(MinWaitStates, W);
528   }
529 
530   return MinWaitStates;
531 }
532 
533 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
534                               const MachineInstr *MI, IsExpiredFn IsExpired) {
535   DenseSet<const MachineBasicBlock *> Visited;
536   return getWaitStatesSince(IsHazard, MI->getParent(),
537                             std::next(MI->getReverseIterator()),
538                             0, IsExpired, Visited);
539 }
540 
541 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) {
542   if (IsHazardRecognizerMode) {
543     auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) {
544       return WaitStates >= Limit;
545     };
546     return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn);
547   }
548 
549   int WaitStates = 0;
550   for (MachineInstr *MI : EmittedInstrs) {
551     if (MI) {
552       if (IsHazard(*MI))
553         return WaitStates;
554 
555       if (MI->isInlineAsm())
556         continue;
557     }
558     ++WaitStates;
559 
560     if (WaitStates >= Limit)
561       break;
562   }
563   return std::numeric_limits<int>::max();
564 }
565 
566 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
567                                                IsHazardFn IsHazardDef,
568                                                int Limit) {
569   const SIRegisterInfo *TRI = ST.getRegisterInfo();
570 
571   auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) {
572     return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI);
573   };
574 
575   return getWaitStatesSince(IsHazardFn, Limit);
576 }
577 
578 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
579                                                   int Limit) {
580   auto IsHazardFn = [IsHazard](const MachineInstr &MI) {
581     return isSSetReg(MI.getOpcode()) && IsHazard(MI);
582   };
583 
584   return getWaitStatesSince(IsHazardFn, Limit);
585 }
586 
587 //===----------------------------------------------------------------------===//
588 // No-op Hazard Detection
589 //===----------------------------------------------------------------------===//
590 
591 static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
592                         MCRegister Reg) {
593   for (MCRegUnit Unit : TRI.regunits(Reg))
594     BV.set(Unit);
595 }
596 
597 static void addRegsToSet(const SIRegisterInfo &TRI,
598                          iterator_range<MachineInstr::const_mop_iterator> Ops,
599                          BitVector &DefSet, BitVector &UseSet) {
600   for (const MachineOperand &Op : Ops) {
601     if (Op.isReg())
602       addRegUnits(TRI, Op.isDef() ? DefSet : UseSet, Op.getReg().asMCReg());
603   }
604 }
605 
606 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) {
607   addRegsToSet(TRI, MI.operands(), ClauseDefs, ClauseUses);
608 }
609 
610 static bool breaksSMEMSoftClause(MachineInstr *MI) {
611   return !SIInstrInfo::isSMRD(*MI);
612 }
613 
614 static bool breaksVMEMSoftClause(MachineInstr *MI) {
615   return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI);
616 }
617 
618 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
619   // SMEM soft clause are only present on VI+, and only matter if xnack is
620   // enabled.
621   if (!ST.isXNACKEnabled())
622     return 0;
623 
624   bool IsSMRD = TII.isSMRD(*MEM);
625 
626   resetClause();
627 
628   // A soft-clause is any group of consecutive SMEM instructions.  The
629   // instructions in this group may return out of order and/or may be
630   // replayed (i.e. the same instruction issued more than once).
631   //
632   // In order to handle these situations correctly we need to make sure that
633   // when a clause has more than one instruction, no instruction in the clause
634   // writes to a register that is read by another instruction in the clause
635   // (including itself). If we encounter this situation, we need to break the
636   // clause by inserting a non SMEM instruction.
637 
638   for (MachineInstr *MI : EmittedInstrs) {
639     // When we hit a non-SMEM instruction then we have passed the start of the
640     // clause and we can stop.
641     if (!MI)
642       break;
643 
644     if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI))
645       break;
646 
647     addClauseInst(*MI);
648   }
649 
650   if (ClauseDefs.none())
651     return 0;
652 
653   // We need to make sure not to put loads and stores in the same clause if they
654   // use the same address. For now, just start a new clause whenever we see a
655   // store.
656   if (MEM->mayStore())
657     return 1;
658 
659   addClauseInst(*MEM);
660 
661   // If the set of defs and uses intersect then we cannot add this instruction
662   // to the clause, so we have a hazard.
663   return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
664 }
665 
666 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
667   int WaitStatesNeeded = 0;
668 
669   WaitStatesNeeded = checkSoftClauseHazards(SMRD);
670 
671   // This SMRD hazard only affects SI.
672   if (!ST.hasSMRDReadVALUDefHazard())
673     return WaitStatesNeeded;
674 
675   // A read of an SGPR by SMRD instruction requires 4 wait states when the
676   // SGPR was written by a VALU instruction.
677   int SmrdSgprWaitStates = 4;
678   auto IsHazardDefFn = [this](const MachineInstr &MI) {
679     return TII.isVALU(MI);
680   };
681   auto IsBufferHazardDefFn = [this](const MachineInstr &MI) {
682     return TII.isSALU(MI);
683   };
684 
685   bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
686 
687   for (const MachineOperand &Use : SMRD->uses()) {
688     if (!Use.isReg())
689       continue;
690     int WaitStatesNeededForUse =
691         SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
692                                                    SmrdSgprWaitStates);
693     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
694 
695     // This fixes what appears to be undocumented hardware behavior in SI where
696     // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
697     // needs some number of nops in between. We don't know how many we need, but
698     // let's use 4. This wasn't discovered before probably because the only
699     // case when this happens is when we expand a 64-bit pointer into a full
700     // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
701     // probably never encountered in the closed-source land.
702     if (IsBufferSMRD) {
703       int WaitStatesNeededForUse =
704         SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
705                                                    IsBufferHazardDefFn,
706                                                    SmrdSgprWaitStates);
707       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
708     }
709   }
710 
711   return WaitStatesNeeded;
712 }
713 
714 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
715   if (!ST.hasVMEMReadSGPRVALUDefHazard())
716     return 0;
717 
718   int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
719 
720   // A read of an SGPR by a VMEM instruction requires 5 wait states when the
721   // SGPR was written by a VALU Instruction.
722   const int VmemSgprWaitStates = 5;
723   auto IsHazardDefFn = [this](const MachineInstr &MI) {
724     return TII.isVALU(MI);
725   };
726   for (const MachineOperand &Use : VMEM->uses()) {
727     if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg()))
728       continue;
729 
730     int WaitStatesNeededForUse =
731         VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
732                                                    VmemSgprWaitStates);
733     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
734   }
735   return WaitStatesNeeded;
736 }
737 
738 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
739   const SIRegisterInfo *TRI = ST.getRegisterInfo();
740   const SIInstrInfo *TII = ST.getInstrInfo();
741 
742   // Check for DPP VGPR read after VALU VGPR write and EXEC write.
743   int DppVgprWaitStates = 2;
744   int DppExecWaitStates = 5;
745   int WaitStatesNeeded = 0;
746   auto IsHazardDefFn = [TII](const MachineInstr &MI) {
747     return TII->isVALU(MI);
748   };
749 
750   for (const MachineOperand &Use : DPP->uses()) {
751     if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
752       continue;
753     int WaitStatesNeededForUse =
754         DppVgprWaitStates - getWaitStatesSinceDef(
755                                 Use.getReg(),
756                                 [](const MachineInstr &) { return true; },
757                                 DppVgprWaitStates);
758     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
759   }
760 
761   WaitStatesNeeded = std::max(
762       WaitStatesNeeded,
763       DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
764                                                 DppExecWaitStates));
765 
766   return WaitStatesNeeded;
767 }
768 
769 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
770   const SIInstrInfo *TII = ST.getInstrInfo();
771 
772   // v_div_fmas requires 4 wait states after a write to vcc from a VALU
773   // instruction.
774   const int DivFMasWaitStates = 4;
775   auto IsHazardDefFn = [TII](const MachineInstr &MI) {
776     return TII->isVALU(MI);
777   };
778   int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
779                                                DivFMasWaitStates);
780 
781   return DivFMasWaitStates - WaitStatesNeeded;
782 }
783 
784 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
785   const SIInstrInfo *TII = ST.getInstrInfo();
786   unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
787 
788   const int GetRegWaitStates = 2;
789   auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) {
790     return GetRegHWReg == getHWReg(TII, MI);
791   };
792   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates);
793 
794   return GetRegWaitStates - WaitStatesNeeded;
795 }
796 
797 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
798   const SIInstrInfo *TII = ST.getInstrInfo();
799   unsigned HWReg = getHWReg(TII, *SetRegInstr);
800 
801   const int SetRegWaitStates = ST.getSetRegWaitStates();
802   auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) {
803     return HWReg == getHWReg(TII, MI);
804   };
805   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates);
806   return SetRegWaitStates - WaitStatesNeeded;
807 }
808 
809 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
810   if (!MI.mayStore())
811     return -1;
812 
813   const SIInstrInfo *TII = ST.getInstrInfo();
814   unsigned Opcode = MI.getOpcode();
815   const MCInstrDesc &Desc = MI.getDesc();
816 
817   int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
818   int VDataRCID = -1;
819   if (VDataIdx != -1)
820     VDataRCID = Desc.operands()[VDataIdx].RegClass;
821 
822   if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
823     // There is no hazard if the instruction does not use vector regs
824     // (like wbinvl1)
825     if (VDataIdx == -1)
826       return -1;
827     // For MUBUF/MTBUF instructions this hazard only exists if the
828     // instruction is not using a register in the soffset field.
829     const MachineOperand *SOffset =
830         TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
831     // If we have no soffset operand, then assume this field has been
832     // hardcoded to zero.
833     if (AMDGPU::getRegBitWidth(VDataRCID) > 64 &&
834         (!SOffset || !SOffset->isReg()))
835       return VDataIdx;
836   }
837 
838   // MIMG instructions create a hazard if they don't use a 256-bit T# and
839   // the store size is greater than 8 bytes and they have more than two bits
840   // of their dmask set.
841   // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
842   if (TII->isMIMG(MI)) {
843     int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
844     assert(SRsrcIdx != -1 &&
845            AMDGPU::getRegBitWidth(Desc.operands()[SRsrcIdx].RegClass) == 256);
846     (void)SRsrcIdx;
847   }
848 
849   if (TII->isFLAT(MI)) {
850     int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
851     if (AMDGPU::getRegBitWidth(Desc.operands()[DataIdx].RegClass) > 64)
852       return DataIdx;
853   }
854 
855   return -1;
856 }
857 
858 int
859 GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
860                                             const MachineRegisterInfo &MRI) {
861   // Helper to check for the hazard where VMEM instructions that store more than
862   // 8 bytes can have there store data over written by the next instruction.
863   const SIRegisterInfo *TRI = ST.getRegisterInfo();
864 
865   const int VALUWaitStates = ST.hasGFX940Insts() ? 2 : 1;
866   int WaitStatesNeeded = 0;
867 
868   if (!TRI->isVectorRegister(MRI, Def.getReg()))
869     return WaitStatesNeeded;
870   Register Reg = Def.getReg();
871   auto IsHazardFn = [this, Reg, TRI](const MachineInstr &MI) {
872     int DataIdx = createsVALUHazard(MI);
873     return DataIdx >= 0 &&
874            TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg);
875   };
876   int WaitStatesNeededForDef =
877     VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates);
878   WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
879 
880   return WaitStatesNeeded;
881 }
882 
883 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
884   int WaitStatesNeeded = 0;
885 
886   if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(*VALU)) {
887     const int TransDefWaitstates = 1;
888 
889     auto IsTransDefFn = [this, VALU](const MachineInstr &MI) {
890       if (!SIInstrInfo::isTRANS(MI))
891         return false;
892       const SIRegisterInfo *TRI = ST.getRegisterInfo();
893       const SIInstrInfo *TII = ST.getInstrInfo();
894       Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg();
895 
896       for (const MachineOperand &Use : VALU->explicit_uses()) {
897         if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg()))
898           return true;
899       }
900 
901       return false;
902     };
903 
904     int WaitStatesNeededForDef =
905         TransDefWaitstates -
906         getWaitStatesSince(IsTransDefFn, TransDefWaitstates);
907     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
908   }
909 
910   if (ST.hasDstSelForwardingHazard()) {
911     const int Shift16DefWaitstates = 1;
912 
913     auto IsShift16BitDefFn = [this, VALU](const MachineInstr &MI) {
914       if (!SIInstrInfo::isVALU(MI))
915         return false;
916       const SIInstrInfo *TII = ST.getInstrInfo();
917       if (SIInstrInfo::isSDWA(MI)) {
918         if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel))
919           if (DstSel->getImm() == AMDGPU::SDWA::DWORD)
920             return false;
921       } else {
922         if (!AMDGPU::hasNamedOperand(MI.getOpcode(), AMDGPU::OpName::op_sel) ||
923             !(TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)
924                   ->getImm() &
925               SISrcMods::DST_OP_SEL))
926           return false;
927       }
928       const SIRegisterInfo *TRI = ST.getRegisterInfo();
929       if (auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) {
930         Register Def = Dst->getReg();
931 
932         for (const MachineOperand &Use : VALU->explicit_uses()) {
933           if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg()))
934             return true;
935         }
936       }
937 
938       return false;
939     };
940 
941     int WaitStatesNeededForDef =
942         Shift16DefWaitstates -
943         getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates);
944     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
945   }
946 
947   if (ST.hasVDecCoExecHazard()) {
948     const int VALUWriteSGPRVALUReadWaitstates = 2;
949     const int VALUWriteEXECRWLane = 4;
950     const int VALUWriteVGPRReadlaneRead = 1;
951 
952     const SIRegisterInfo *TRI = ST.getRegisterInfo();
953     const MachineRegisterInfo &MRI = MF.getRegInfo();
954     Register UseReg;
955     auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) {
956       if (!SIInstrInfo::isVALU(MI))
957         return false;
958       return MI.modifiesRegister(UseReg, TRI);
959     };
960 
961     for (const MachineOperand &Use : VALU->explicit_uses()) {
962       if (!Use.isReg())
963         continue;
964 
965       UseReg = Use.getReg();
966       if (TRI->isSGPRReg(MRI, UseReg)) {
967         int WaitStatesNeededForDef =
968             VALUWriteSGPRVALUReadWaitstates -
969             getWaitStatesSince(IsVALUDefSGPRFn,
970                                VALUWriteSGPRVALUReadWaitstates);
971         WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
972       }
973     }
974 
975     if (VALU->readsRegister(AMDGPU::VCC, TRI)) {
976       UseReg = AMDGPU::VCC;
977       int WaitStatesNeededForDef =
978           VALUWriteSGPRVALUReadWaitstates -
979           getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteSGPRVALUReadWaitstates);
980       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
981     }
982 
983     switch (VALU->getOpcode()) {
984     case AMDGPU::V_READLANE_B32:
985     case AMDGPU::V_READFIRSTLANE_B32: {
986       MachineOperand *Src = TII.getNamedOperand(*VALU, AMDGPU::OpName::src0);
987       UseReg = Src->getReg();
988       int WaitStatesNeededForDef =
989           VALUWriteVGPRReadlaneRead -
990           getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteVGPRReadlaneRead);
991       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
992     }
993       [[fallthrough]];
994     case AMDGPU::V_WRITELANE_B32: {
995       UseReg = AMDGPU::EXEC;
996       int WaitStatesNeededForDef =
997           VALUWriteEXECRWLane -
998           getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteEXECRWLane);
999       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
1000       break;
1001     }
1002     default:
1003       break;
1004     }
1005   }
1006 
1007   // This checks for the hazard where VMEM instructions that store more than
1008   // 8 bytes can have there store data over written by the next instruction.
1009   if (!ST.has12DWordStoreHazard())
1010     return WaitStatesNeeded;
1011 
1012   const MachineRegisterInfo &MRI = MF.getRegInfo();
1013 
1014   for (const MachineOperand &Def : VALU->defs()) {
1015     WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
1016   }
1017 
1018   return WaitStatesNeeded;
1019 }
1020 
1021 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) {
1022   // This checks for hazards associated with inline asm statements.
1023   // Since inline asms can contain just about anything, we use this
1024   // to call/leverage other check*Hazard routines. Note that
1025   // this function doesn't attempt to address all possible inline asm
1026   // hazards (good luck), but is a collection of what has been
1027   // problematic thus far.
1028 
1029   // see checkVALUHazards()
1030   if (!ST.has12DWordStoreHazard())
1031     return 0;
1032 
1033   const MachineRegisterInfo &MRI = MF.getRegInfo();
1034   int WaitStatesNeeded = 0;
1035 
1036   for (const MachineOperand &Op :
1037        llvm::drop_begin(IA->operands(), InlineAsm::MIOp_FirstOperand)) {
1038     if (Op.isReg() && Op.isDef()) {
1039       WaitStatesNeeded =
1040           std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
1041     }
1042   }
1043 
1044   return WaitStatesNeeded;
1045 }
1046 
1047 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
1048   const SIInstrInfo *TII = ST.getInstrInfo();
1049   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1050   const MachineRegisterInfo &MRI = MF.getRegInfo();
1051 
1052   const MachineOperand *LaneSelectOp =
1053       TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
1054 
1055   if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
1056     return 0;
1057 
1058   Register LaneSelectReg = LaneSelectOp->getReg();
1059   auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); };
1060 
1061   const int RWLaneWaitStates = 4;
1062   int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
1063                                               RWLaneWaitStates);
1064   return RWLaneWaitStates - WaitStatesSince;
1065 }
1066 
1067 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
1068   if (!ST.hasRFEHazards())
1069     return 0;
1070 
1071   const SIInstrInfo *TII = ST.getInstrInfo();
1072 
1073   const int RFEWaitStates = 1;
1074 
1075   auto IsHazardFn = [TII](const MachineInstr &MI) {
1076     return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS;
1077   };
1078   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates);
1079   return RFEWaitStates - WaitStatesNeeded;
1080 }
1081 
1082 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
1083   const SIInstrInfo *TII = ST.getInstrInfo();
1084   const int ReadM0WaitStates = 1;
1085   auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); };
1086   return ReadM0WaitStates -
1087          getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, ReadM0WaitStates);
1088 }
1089 
1090 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
1091   fixVMEMtoScalarWriteHazards(MI);
1092   fixVcmpxPermlaneHazards(MI);
1093   fixSMEMtoVectorWriteHazards(MI);
1094   fixVcmpxExecWARHazard(MI);
1095   fixLdsBranchVmemWARHazard(MI);
1096   if (ST.hasLdsDirect()) {
1097     fixLdsDirectVALUHazard(MI);
1098     fixLdsDirectVMEMHazard(MI);
1099   }
1100   fixVALUPartialForwardingHazard(MI);
1101   fixVALUTransUseHazard(MI);
1102   fixWMMAHazards(MI);
1103   fixShift64HighRegBug(MI);
1104   fixVALUMaskWriteHazard(MI);
1105 }
1106 
1107 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
1108   if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI))
1109     return false;
1110 
1111   const SIInstrInfo *TII = ST.getInstrInfo();
1112   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1113   auto IsHazardFn = [TII, TRI](const MachineInstr &MI) {
1114     return (TII->isVOPC(MI) ||
1115             ((TII->isVOP3(MI) || TII->isSDWA(MI)) && MI.isCompare())) &&
1116            MI.modifiesRegister(AMDGPU::EXEC, TRI);
1117   };
1118 
1119   auto IsExpiredFn = [](const MachineInstr &MI, int) {
1120     unsigned Opc = MI.getOpcode();
1121     return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 &&
1122            Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa;
1123   };
1124 
1125   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1126       std::numeric_limits<int>::max())
1127     return false;
1128 
1129   // V_NOP will be discarded by SQ.
1130   // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
1131   // which is always a VGPR and available.
1132   auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0);
1133   Register Reg = Src0->getReg();
1134   bool IsUndef = Src0->isUndef();
1135   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1136           TII->get(AMDGPU::V_MOV_B32_e32))
1137     .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0))
1138     .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill);
1139 
1140   return true;
1141 }
1142 
1143 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
1144   if (!ST.hasVMEMtoScalarWriteHazard())
1145     return false;
1146 
1147   if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI))
1148     return false;
1149 
1150   if (MI->getNumDefs() == 0)
1151     return false;
1152 
1153   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1154 
1155   auto IsHazardFn = [TRI, MI](const MachineInstr &I) {
1156     if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I) &&
1157         !SIInstrInfo::isFLAT(I))
1158       return false;
1159 
1160     for (const MachineOperand &Def : MI->defs()) {
1161       const MachineOperand *Op =
1162           I.findRegisterUseOperand(Def.getReg(), false, TRI);
1163       if (!Op)
1164         continue;
1165       return true;
1166     }
1167     return false;
1168   };
1169 
1170   auto IsExpiredFn = [](const MachineInstr &MI, int) {
1171     return SIInstrInfo::isVALU(MI) ||
1172            (MI.getOpcode() == AMDGPU::S_WAITCNT &&
1173             !MI.getOperand(0).getImm()) ||
1174            (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1175             AMDGPU::DepCtr::decodeFieldVmVsrc(MI.getOperand(0).getImm()) == 0);
1176   };
1177 
1178   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1179       std::numeric_limits<int>::max())
1180     return false;
1181 
1182   const SIInstrInfo *TII = ST.getInstrInfo();
1183   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1184           TII->get(AMDGPU::S_WAITCNT_DEPCTR))
1185       .addImm(AMDGPU::DepCtr::encodeFieldVmVsrc(0));
1186   return true;
1187 }
1188 
1189 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
1190   if (!ST.hasSMEMtoVectorWriteHazard())
1191     return false;
1192 
1193   if (!SIInstrInfo::isVALU(*MI))
1194     return false;
1195 
1196   unsigned SDSTName;
1197   switch (MI->getOpcode()) {
1198   case AMDGPU::V_READLANE_B32:
1199   case AMDGPU::V_READFIRSTLANE_B32:
1200     SDSTName = AMDGPU::OpName::vdst;
1201     break;
1202   default:
1203     SDSTName = AMDGPU::OpName::sdst;
1204     break;
1205   }
1206 
1207   const SIInstrInfo *TII = ST.getInstrInfo();
1208   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1209   const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU());
1210   const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName);
1211   if (!SDST) {
1212     for (const auto &MO : MI->implicit_operands()) {
1213       if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegBaseClass(MO.getReg()))) {
1214         SDST = &MO;
1215         break;
1216       }
1217     }
1218   }
1219 
1220   if (!SDST)
1221     return false;
1222 
1223   const Register SDSTReg = SDST->getReg();
1224   auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) {
1225     return SIInstrInfo::isSMRD(I) && I.readsRegister(SDSTReg, TRI);
1226   };
1227 
1228   auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) {
1229     if (TII->isSALU(MI)) {
1230       switch (MI.getOpcode()) {
1231       case AMDGPU::S_SETVSKIP:
1232       case AMDGPU::S_VERSION:
1233       case AMDGPU::S_WAITCNT_VSCNT:
1234       case AMDGPU::S_WAITCNT_VMCNT:
1235       case AMDGPU::S_WAITCNT_EXPCNT:
1236         // These instructions cannot not mitigate the hazard.
1237         return false;
1238       case AMDGPU::S_WAITCNT_LGKMCNT:
1239         // Reducing lgkmcnt count to 0 always mitigates the hazard.
1240         return (MI.getOperand(1).getImm() == 0) &&
1241                (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1242       case AMDGPU::S_WAITCNT: {
1243         const int64_t Imm = MI.getOperand(0).getImm();
1244         AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm);
1245         return (Decoded.LgkmCnt == 0);
1246       }
1247       default:
1248         // SOPP instructions cannot mitigate the hazard.
1249         if (TII->isSOPP(MI))
1250           return false;
1251         // At this point the SALU can be assumed to mitigate the hazard
1252         // because either:
1253         // (a) it is independent of the at risk SMEM (breaking chain),
1254         // or
1255         // (b) it is dependent on the SMEM, in which case an appropriate
1256         //     s_waitcnt lgkmcnt _must_ exist between it and the at risk
1257         //     SMEM instruction.
1258         return true;
1259       }
1260     }
1261     return false;
1262   };
1263 
1264   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1265       std::numeric_limits<int>::max())
1266     return false;
1267 
1268   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1269           TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL)
1270       .addImm(0);
1271   return true;
1272 }
1273 
1274 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
1275   if (!ST.hasVcmpxExecWARHazard() || !SIInstrInfo::isVALU(*MI))
1276     return false;
1277 
1278   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1279   if (!MI->modifiesRegister(AMDGPU::EXEC, TRI))
1280     return false;
1281 
1282   auto IsHazardFn = [TRI](const MachineInstr &I) {
1283     if (SIInstrInfo::isVALU(I))
1284       return false;
1285     return I.readsRegister(AMDGPU::EXEC, TRI);
1286   };
1287 
1288   const SIInstrInfo *TII = ST.getInstrInfo();
1289   auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) {
1290     if (SIInstrInfo::isVALU(MI)) {
1291       if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst))
1292         return true;
1293       for (auto MO : MI.implicit_operands())
1294         if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegBaseClass(MO.getReg())))
1295           return true;
1296     }
1297     if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1298         AMDGPU::DepCtr::decodeFieldSaSdst(MI.getOperand(0).getImm()) == 0)
1299       return true;
1300     return false;
1301   };
1302 
1303   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1304       std::numeric_limits<int>::max())
1305     return false;
1306 
1307   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1308           TII->get(AMDGPU::S_WAITCNT_DEPCTR))
1309       .addImm(AMDGPU::DepCtr::encodeFieldSaSdst(0));
1310   return true;
1311 }
1312 
1313 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
1314                                                  const GCNSubtarget &ST) {
1315   if (!ST.hasLdsBranchVmemWARHazard())
1316     return false;
1317 
1318   // Check if the necessary condition for the hazard is met: both LDS and VMEM
1319   // instructions need to appear in the same function.
1320   bool HasLds = false;
1321   bool HasVmem = false;
1322   for (auto &MBB : MF) {
1323     for (auto &MI : MBB) {
1324       HasLds |= SIInstrInfo::isDS(MI);
1325       HasVmem |=
1326           SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI);
1327       if (HasLds && HasVmem)
1328         return true;
1329     }
1330   }
1331   return false;
1332 }
1333 
1334 static bool isStoreCountWaitZero(const MachineInstr &I) {
1335   return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1336          I.getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1337          !I.getOperand(1).getImm();
1338 }
1339 
1340 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
1341   if (!RunLdsBranchVmemWARHazardFixup)
1342     return false;
1343 
1344   assert(ST.hasLdsBranchVmemWARHazard());
1345 
1346   auto IsHazardInst = [](const MachineInstr &MI) {
1347     if (SIInstrInfo::isDS(MI))
1348       return 1;
1349     if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI))
1350       return 2;
1351     return 0;
1352   };
1353 
1354   auto InstType = IsHazardInst(*MI);
1355   if (!InstType)
1356     return false;
1357 
1358   auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) {
1359     return IsHazardInst(I) || isStoreCountWaitZero(I);
1360   };
1361 
1362   auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) {
1363     if (!I.isBranch())
1364       return false;
1365 
1366     auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) {
1367       auto InstType2 = IsHazardInst(I);
1368       return InstType2 && InstType != InstType2;
1369     };
1370 
1371     auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) {
1372       auto InstType2 = IsHazardInst(I);
1373       if (InstType == InstType2)
1374         return true;
1375 
1376       return isStoreCountWaitZero(I);
1377     };
1378 
1379     return ::getWaitStatesSince(IsHazardFn, &I, IsExpiredFn) !=
1380            std::numeric_limits<int>::max();
1381   };
1382 
1383   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1384       std::numeric_limits<int>::max())
1385     return false;
1386 
1387   const SIInstrInfo *TII = ST.getInstrInfo();
1388   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1389           TII->get(AMDGPU::S_WAITCNT_VSCNT))
1390     .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1391     .addImm(0);
1392 
1393   return true;
1394 }
1395 
1396 bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) {
1397   if (!SIInstrInfo::isLDSDIR(*MI))
1398     return false;
1399 
1400   const int NoHazardWaitStates = 15;
1401   const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst);
1402   const Register VDSTReg = VDST->getReg();
1403 
1404   bool VisitedTrans = false;
1405   auto IsHazardFn = [this, VDSTReg, &VisitedTrans](const MachineInstr &I) {
1406     if (!SIInstrInfo::isVALU(I))
1407       return false;
1408     VisitedTrans = VisitedTrans || SIInstrInfo::isTRANS(I);
1409     // Cover both WAR and WAW
1410     return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI);
1411   };
1412   auto IsExpiredFn = [&](const MachineInstr &I, int WaitStates) {
1413     if (WaitStates >= NoHazardWaitStates)
1414       return true;
1415     // Instructions which cause va_vdst==0 expire hazard
1416     return SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) ||
1417            SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I);
1418   };
1419   auto GetWaitStatesFn = [](const MachineInstr &MI) {
1420     return SIInstrInfo::isVALU(MI) ? 1 : 0;
1421   };
1422 
1423   DenseSet<const MachineBasicBlock *> Visited;
1424   auto Count = ::getWaitStatesSince(IsHazardFn, MI->getParent(),
1425                                     std::next(MI->getReverseIterator()), 0,
1426                                     IsExpiredFn, Visited, GetWaitStatesFn);
1427 
1428   // Transcendentals can execute in parallel to other VALUs.
1429   // This makes va_vdst count unusable with a mixture of VALU and TRANS.
1430   if (VisitedTrans)
1431     Count = 0;
1432 
1433   MachineOperand *WaitVdstOp =
1434       TII.getNamedOperand(*MI, AMDGPU::OpName::waitvdst);
1435   WaitVdstOp->setImm(std::min(Count, NoHazardWaitStates));
1436 
1437   return true;
1438 }
1439 
1440 bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) {
1441   if (!SIInstrInfo::isLDSDIR(*MI))
1442     return false;
1443 
1444   const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst);
1445   const Register VDSTReg = VDST->getReg();
1446 
1447   auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) {
1448     if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I) &&
1449         !SIInstrInfo::isDS(I))
1450       return false;
1451     return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI);
1452   };
1453   bool LdsdirCanWait = ST.hasLdsWaitVMSRC();
1454   auto IsExpiredFn = [this, LdsdirCanWait](const MachineInstr &I, int) {
1455     return SIInstrInfo::isVALU(I) || SIInstrInfo::isEXP(I) ||
1456            (I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(0).getImm()) ||
1457            (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1458             AMDGPU::DepCtr::decodeFieldVmVsrc(I.getOperand(0).getImm()) == 0) ||
1459            (LdsdirCanWait && SIInstrInfo::isLDSDIR(I) &&
1460             !TII.getNamedOperand(I, AMDGPU::OpName::waitvsrc)->getImm());
1461   };
1462 
1463   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1464       std::numeric_limits<int>::max())
1465     return false;
1466 
1467   if (LdsdirCanWait) {
1468     TII.getNamedOperand(*MI, AMDGPU::OpName::waitvsrc)->setImm(0);
1469   } else {
1470     BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1471             TII.get(AMDGPU::S_WAITCNT_DEPCTR))
1472         .addImm(AMDGPU::DepCtr::encodeFieldVmVsrc(0));
1473   }
1474 
1475   return true;
1476 }
1477 
1478 bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
1479   if (!ST.isWave64())
1480     return false;
1481   if (!ST.hasVALUPartialForwardingHazard())
1482     return false;
1483   if (!SIInstrInfo::isVALU(*MI))
1484     return false;
1485 
1486   SmallSetVector<Register, 4> SrcVGPRs;
1487 
1488   for (const MachineOperand &Use : MI->explicit_uses()) {
1489     if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
1490       SrcVGPRs.insert(Use.getReg());
1491   }
1492 
1493   // Only applies with >= 2 unique VGPR sources
1494   if (SrcVGPRs.size() <= 1)
1495     return false;
1496 
1497   // Look for the following pattern:
1498   //   Va <- VALU [PreExecPos]
1499   //   intv1
1500   //   Exec <- SALU [ExecPos]
1501   //   intv2
1502   //   Vb <- VALU [PostExecPos]
1503   //   intv3
1504   //   MI Va, Vb (WaitState = 0)
1505   //
1506   // Where:
1507   // intv1 + intv2 <= 2 VALUs
1508   // intv3 <= 4 VALUs
1509   //
1510   // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
1511 
1512   const int Intv1plus2MaxVALUs = 2;
1513   const int Intv3MaxVALUs = 4;
1514   const int IntvMaxVALUs = 6;
1515   const int NoHazardVALUWaitStates = IntvMaxVALUs + 2;
1516 
1517   struct StateType {
1518     SmallDenseMap<Register, int, 4> DefPos;
1519     int ExecPos = std::numeric_limits<int>::max();
1520     int VALUs = 0;
1521   };
1522 
1523   StateType State;
1524 
1525   // This overloads expiry testing with all the hazard detection
1526   auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
1527     // Too many VALU states have passed
1528     if (State.VALUs > NoHazardVALUWaitStates)
1529       return HazardExpired;
1530 
1531     // Instructions which cause va_vdst==0 expire hazard
1532     if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) ||
1533         SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I) ||
1534         (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1535          AMDGPU::DepCtr::decodeFieldVaVdst(I.getOperand(0).getImm()) == 0))
1536       return HazardExpired;
1537 
1538     // Track registers writes
1539     bool Changed = false;
1540     if (SIInstrInfo::isVALU(I)) {
1541       for (Register Src : SrcVGPRs) {
1542         if (!State.DefPos.count(Src) && I.modifiesRegister(Src, &TRI)) {
1543           State.DefPos[Src] = State.VALUs;
1544           Changed = true;
1545         }
1546       }
1547     } else if (SIInstrInfo::isSALU(I)) {
1548       if (State.ExecPos == std::numeric_limits<int>::max()) {
1549         if (!State.DefPos.empty() && I.modifiesRegister(AMDGPU::EXEC, &TRI)) {
1550           State.ExecPos = State.VALUs;
1551           Changed = true;
1552         }
1553       }
1554     }
1555 
1556     // Early expiration: too many VALUs in intv3
1557     if (State.VALUs > Intv3MaxVALUs && State.DefPos.empty())
1558       return HazardExpired;
1559 
1560     // Only evaluate state if something changed
1561     if (!Changed)
1562       return NoHazardFound;
1563 
1564     // Determine positions of VALUs pre/post exec change
1565     if (State.ExecPos == std::numeric_limits<int>::max())
1566       return NoHazardFound;
1567 
1568     int PreExecPos = std::numeric_limits<int>::max();
1569     int PostExecPos = std::numeric_limits<int>::max();
1570 
1571     for (auto Entry : State.DefPos) {
1572       int DefVALUs = Entry.second;
1573       if (DefVALUs != std::numeric_limits<int>::max()) {
1574         if (DefVALUs >= State.ExecPos)
1575           PreExecPos = std::min(PreExecPos, DefVALUs);
1576         else if (DefVALUs < State.ExecPos)
1577           PostExecPos = std::min(PostExecPos, DefVALUs);
1578       }
1579     }
1580 
1581     // Need a VALUs post exec change
1582     if (PostExecPos == std::numeric_limits<int>::max())
1583       return NoHazardFound;
1584 
1585     // Too many VALUs in intv3?
1586     int Intv3VALUs = PostExecPos;
1587     if (Intv3VALUs > Intv3MaxVALUs)
1588       return HazardExpired;
1589 
1590     // Too many VALUs in intv2?
1591     int Intv2VALUs = (State.ExecPos - PostExecPos) - 1;
1592     if (Intv2VALUs > Intv1plus2MaxVALUs)
1593       return HazardExpired;
1594 
1595     // Need a VALUs pre exec change
1596     if (PreExecPos == std::numeric_limits<int>::max())
1597       return NoHazardFound;
1598 
1599     // Too many VALUs in intv1?
1600     int Intv1VALUs = PreExecPos - State.ExecPos;
1601     if (Intv1VALUs > Intv1plus2MaxVALUs)
1602       return HazardExpired;
1603 
1604     // Too many VALUs in intv1 + intv2
1605     if (Intv1VALUs + Intv2VALUs > Intv1plus2MaxVALUs)
1606       return HazardExpired;
1607 
1608     return HazardFound;
1609   };
1610   auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
1611     if (SIInstrInfo::isVALU(MI))
1612       State.VALUs += 1;
1613   };
1614 
1615   DenseSet<const MachineBasicBlock *> Visited;
1616   if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(),
1617                             std::next(MI->getReverseIterator()), Visited))
1618     return false;
1619 
1620   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1621           TII.get(AMDGPU::S_WAITCNT_DEPCTR))
1622       .addImm(0x0fff);
1623 
1624   return true;
1625 }
1626 
1627 bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
1628   if (!ST.hasVALUTransUseHazard())
1629     return false;
1630   if (!SIInstrInfo::isVALU(*MI))
1631     return false;
1632 
1633   SmallSet<Register, 4> SrcVGPRs;
1634 
1635   for (const MachineOperand &Use : MI->explicit_uses()) {
1636     if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
1637       SrcVGPRs.insert(Use.getReg());
1638   }
1639 
1640   // Look for the following pattern:
1641   //   Va <- TRANS VALU
1642   //   intv
1643   //   MI Va (WaitState = 0)
1644   //
1645   // Where:
1646   // intv <= 5 VALUs / 1 TRANS
1647   //
1648   // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
1649 
1650   const int IntvMaxVALUs = 5;
1651   const int IntvMaxTRANS = 1;
1652 
1653   struct StateType {
1654     int VALUs = 0;
1655     int TRANS = 0;
1656   };
1657 
1658   StateType State;
1659 
1660   // This overloads expiry testing with all the hazard detection
1661   auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
1662     // Too many VALU states have passed
1663     if (State.VALUs > IntvMaxVALUs || State.TRANS > IntvMaxTRANS)
1664       return HazardExpired;
1665 
1666     // Instructions which cause va_vdst==0 expire hazard
1667     if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) ||
1668         SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I) ||
1669         (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1670          I.getOperand(0).getImm() == 0x0fff))
1671       return HazardExpired;
1672 
1673     // Track registers writes
1674     if (SIInstrInfo::isTRANS(I)) {
1675       for (Register Src : SrcVGPRs) {
1676         if (I.modifiesRegister(Src, &TRI)) {
1677           return HazardFound;
1678         }
1679       }
1680     }
1681 
1682     return NoHazardFound;
1683   };
1684   auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
1685     if (SIInstrInfo::isVALU(MI))
1686       State.VALUs += 1;
1687     if (SIInstrInfo::isTRANS(MI))
1688       State.TRANS += 1;
1689   };
1690 
1691   DenseSet<const MachineBasicBlock *> Visited;
1692   if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(),
1693                             std::next(MI->getReverseIterator()), Visited))
1694     return false;
1695 
1696   // Hazard is observed - insert a wait on va_dst counter to ensure hazard is
1697   // avoided.
1698   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1699           TII.get(AMDGPU::S_WAITCNT_DEPCTR))
1700       .addImm(AMDGPU::DepCtr::encodeFieldVaVdst(0));
1701 
1702   return true;
1703 }
1704 
1705 bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
1706   if (!SIInstrInfo::isWMMA(*MI))
1707     return false;
1708 
1709   const SIInstrInfo *TII = ST.getInstrInfo();
1710   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1711 
1712   auto IsHazardFn = [MI, TII, TRI](const MachineInstr &I) {
1713     if (!SIInstrInfo::isWMMA(I))
1714       return false;
1715 
1716     // Src0 or Src1 of the current wmma instruction overlaps with the dest of
1717     // the previous wmma.
1718     const Register CurSrc0Reg =
1719         TII->getNamedOperand(*MI, AMDGPU::OpName::src0)->getReg();
1720     const Register CurSrc1Reg =
1721         TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg();
1722 
1723     const Register PrevDstReg =
1724         TII->getNamedOperand(I, AMDGPU::OpName::vdst)->getReg();
1725 
1726     if (TRI->regsOverlap(PrevDstReg, CurSrc0Reg) ||
1727         TRI->regsOverlap(PrevDstReg, CurSrc1Reg)) {
1728       return true;
1729     }
1730 
1731     // Src2 of the current wmma instruction overlaps with the dest of the
1732     // previous wmma.
1733     const MachineOperand *Src2 =
1734         TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
1735     const Register CurSrc2Reg = Src2->isReg() ? Src2->getReg() : Register();
1736 
1737     if (CurSrc2Reg != AMDGPU::NoRegister &&
1738         TRI->regsOverlap(PrevDstReg, CurSrc2Reg)) {
1739 
1740       const MachineOperand *Src2Mods =
1741           TII->getNamedOperand(*MI, AMDGPU::OpName::src2_modifiers);
1742       const bool NoSrc2Mods =
1743           (Src2Mods->getImm() & (SISrcMods::NEG | SISrcMods::NEG_HI)) == 0;
1744       // Exception: there is no hazard if the wmma instructions are of the same
1745       // type and there is no input modifier on src2 of the current instruction.
1746       return !(NoSrc2Mods && (TII->pseudoToMCOpcode(I.getOpcode()) ==
1747                               TII->pseudoToMCOpcode(MI->getOpcode())));
1748     }
1749 
1750     return false;
1751   };
1752 
1753   auto IsExpiredFn = [](const MachineInstr &I, int) {
1754     return SIInstrInfo::isVALU(I);
1755   };
1756 
1757   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1758       std::numeric_limits<int>::max())
1759     return false;
1760 
1761   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32));
1762 
1763   return true;
1764 }
1765 
1766 bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) {
1767   if (!ST.hasShift64HighRegBug())
1768     return false;
1769 
1770   switch (MI->getOpcode()) {
1771   default:
1772     return false;
1773   case AMDGPU::V_LSHLREV_B64_e64:
1774   case AMDGPU::V_LSHRREV_B64_e64:
1775   case AMDGPU::V_ASHRREV_I64_e64:
1776     break;
1777   }
1778 
1779   MachineOperand *Amt = TII.getNamedOperand(*MI, AMDGPU::OpName::src0);
1780   if (!Amt->isReg())
1781     return false;
1782 
1783   Register AmtReg = Amt->getReg();
1784   const MachineRegisterInfo &MRI = MF.getRegInfo();
1785   // Check if this is a last VGPR in the allocation block.
1786   if (!TRI.isVGPR(MRI, AmtReg) || ((AmtReg - AMDGPU::VGPR0) & 7) != 7)
1787     return false;
1788 
1789   if (AmtReg != AMDGPU::VGPR255 && MRI.isPhysRegUsed(AmtReg + 1))
1790     return false;
1791 
1792   MachineOperand *Src1 = TII.getNamedOperand(*MI, AMDGPU::OpName::src1);
1793   bool OverlappedSrc = Src1->isReg() && TRI.regsOverlap(Src1->getReg(), AmtReg);
1794   bool OverlappedDst = MI->modifiesRegister(AmtReg, &TRI);
1795   bool Overlapped = OverlappedSrc || OverlappedDst;
1796 
1797   assert(!OverlappedDst || !OverlappedSrc ||
1798          Src1->getReg() == MI->getOperand(0).getReg());
1799   assert(ST.needsAlignedVGPRs());
1800   static_assert(AMDGPU::VGPR0 + 1 == AMDGPU::VGPR1);
1801 
1802   Register NewReg;
1803   for (MCRegister Reg : Overlapped ? AMDGPU::VReg_64_Align2RegClass
1804                                    : AMDGPU::VGPR_32RegClass) {
1805     if (!MI->modifiesRegister(Reg, &TRI) && !MI->readsRegister(Reg, &TRI)) {
1806       NewReg = Reg;
1807       break;
1808     }
1809   }
1810 
1811   Register NewAmt = Overlapped ? (Register)TRI.getSubReg(NewReg, AMDGPU::sub1)
1812                                : NewReg;
1813   Register NewAmtLo;
1814 
1815   if (Overlapped)
1816     NewAmtLo = TRI.getSubReg(NewReg, AMDGPU::sub0);
1817 
1818   DebugLoc DL = MI->getDebugLoc();
1819   MachineBasicBlock *MBB = MI->getParent();
1820   // Insert a full wait count because found register might be pending a wait.
1821   BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_WAITCNT))
1822       .addImm(0);
1823 
1824   // Insert V_SWAP_B32 instruction(s) and run hazard recognizer on them.
1825   if (Overlapped)
1826     runOnInstruction(
1827         BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmtLo)
1828             .addDef(AmtReg - 1)
1829             .addReg(AmtReg - 1, RegState::Undef)
1830             .addReg(NewAmtLo, RegState::Undef));
1831   runOnInstruction(BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmt)
1832                        .addDef(AmtReg)
1833                        .addReg(AmtReg, RegState::Undef)
1834                        .addReg(NewAmt, RegState::Undef));
1835 
1836   // Instructions emitted after the current instruction will be processed by the
1837   // parent loop of the hazard recognizer in a natural way.
1838   BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32),
1839           AmtReg)
1840       .addDef(NewAmt)
1841       .addReg(NewAmt)
1842       .addReg(AmtReg);
1843   if (Overlapped)
1844     BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32),
1845             AmtReg - 1)
1846         .addDef(NewAmtLo)
1847         .addReg(NewAmtLo)
1848         .addReg(AmtReg - 1);
1849 
1850   // Re-running hazard recognizer on the modified instruction is not necessary,
1851   // inserted V_SWAP_B32 has already both read and write new registers so
1852   // hazards related to these register has already been handled.
1853   Amt->setReg(NewAmt);
1854   Amt->setIsKill(false);
1855   // We do not update liveness, so verifier may see it as undef.
1856   Amt->setIsUndef();
1857   if (OverlappedDst)
1858     MI->getOperand(0).setReg(NewReg);
1859   if (OverlappedSrc) {
1860     Src1->setReg(NewReg);
1861     Src1->setIsKill(false);
1862     Src1->setIsUndef();
1863   }
1864 
1865   return true;
1866 }
1867 
1868 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) {
1869   int NSAtoVMEMWaitStates = 1;
1870 
1871   if (!ST.hasNSAtoVMEMBug())
1872     return 0;
1873 
1874   if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI))
1875     return 0;
1876 
1877   const SIInstrInfo *TII = ST.getInstrInfo();
1878   const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset);
1879   if (!Offset || (Offset->getImm() & 6) == 0)
1880     return 0;
1881 
1882   auto IsHazardFn = [TII](const MachineInstr &I) {
1883     if (!SIInstrInfo::isMIMG(I))
1884       return false;
1885     const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode());
1886     return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
1887            TII->getInstSizeInBytes(I) >= 16;
1888   };
1889 
1890   return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1);
1891 }
1892 
1893 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) {
1894   int FPAtomicToDenormModeWaitStates = 3;
1895 
1896   if (!ST.hasFPAtomicToDenormModeHazard())
1897     return 0;
1898 
1899   if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
1900     return 0;
1901 
1902   auto IsHazardFn = [](const MachineInstr &I) {
1903     if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I))
1904       return false;
1905     return SIInstrInfo::isFPAtomic(I);
1906   };
1907 
1908   auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) {
1909     if (WaitStates >= 3 || SIInstrInfo::isVALU(MI))
1910       return true;
1911 
1912     switch (MI.getOpcode()) {
1913     case AMDGPU::S_WAITCNT:
1914     case AMDGPU::S_WAITCNT_VSCNT:
1915     case AMDGPU::S_WAITCNT_VMCNT:
1916     case AMDGPU::S_WAITCNT_EXPCNT:
1917     case AMDGPU::S_WAITCNT_LGKMCNT:
1918     case AMDGPU::S_WAIT_IDLE:
1919       return true;
1920     default:
1921       break;
1922     }
1923 
1924     return false;
1925   };
1926 
1927   return FPAtomicToDenormModeWaitStates -
1928          ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn);
1929 }
1930 
1931 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) {
1932   assert(SIInstrInfo::isMAI(*MI));
1933 
1934   return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
1935 }
1936 
1937 int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) {
1938   // Early exit if no padding is requested.
1939   if (MFMAPaddingRatio == 0)
1940     return 0;
1941 
1942   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1943   if (!SIInstrInfo::isMFMA(*MI) || MFI->getOccupancy() < 2)
1944     return 0;
1945 
1946   int NeighborMFMALatency = 0;
1947   auto IsNeighboringMFMA = [&NeighborMFMALatency,
1948                             this](const MachineInstr &MI) {
1949     if (!SIInstrInfo::isMFMA(MI))
1950       return false;
1951 
1952     NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI);
1953     return true;
1954   };
1955 
1956   const int MaxMFMAPipelineWaitStates = 16;
1957   int WaitStatesSinceNeighborMFMA =
1958       getWaitStatesSince(IsNeighboringMFMA, MaxMFMAPipelineWaitStates);
1959 
1960   int NeighborMFMAPaddingNeeded =
1961       (NeighborMFMALatency * MFMAPaddingRatio / 100) -
1962       WaitStatesSinceNeighborMFMA;
1963 
1964   return std::max(0, NeighborMFMAPaddingNeeded);
1965 }
1966 
1967 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) {
1968   int WaitStatesNeeded = 0;
1969   unsigned Opc = MI->getOpcode();
1970 
1971   auto IsVALUFn = [](const MachineInstr &MI) {
1972     return SIInstrInfo::isVALU(MI) || MI.isInlineAsm();
1973   };
1974 
1975   if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
1976     const int LegacyVALUWritesVGPRWaitStates = 2;
1977     const int VALUWritesExecWaitStates = 4;
1978     const int MaxWaitStates = 4;
1979 
1980     int WaitStatesNeededForUse = VALUWritesExecWaitStates -
1981       getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates);
1982     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1983 
1984     if (WaitStatesNeeded < MaxWaitStates) {
1985       for (const MachineOperand &Use : MI->explicit_uses()) {
1986         const int MaxWaitStates = 2;
1987 
1988         if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
1989           continue;
1990 
1991         int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
1992           getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates);
1993         WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1994 
1995         if (WaitStatesNeeded == MaxWaitStates)
1996           break;
1997       }
1998     }
1999   }
2000 
2001   for (const MachineOperand &Op : MI->explicit_operands()) {
2002     if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg()))
2003       continue;
2004 
2005     if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
2006       continue;
2007 
2008     const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
2009     const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
2010     const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
2011     const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
2012     const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
2013     const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
2014     const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
2015     const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
2016     const int MaxWaitStates = 18;
2017     Register Reg = Op.getReg();
2018     unsigned HazardDefLatency = 0;
2019 
2020     auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency,
2021                                this](const MachineInstr &MI) {
2022       if (!SIInstrInfo::isMFMA(MI))
2023         return false;
2024       Register DstReg = MI.getOperand(0).getReg();
2025       if (DstReg == Reg)
2026         return false;
2027       HazardDefLatency =
2028           std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
2029       return TRI.regsOverlap(DstReg, Reg);
2030     };
2031 
2032     int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn,
2033                                                    MaxWaitStates);
2034     int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
2035     int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
2036     int OpNo = Op.getOperandNo();
2037     if (OpNo == SrcCIdx) {
2038       NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
2039     } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
2040       switch (HazardDefLatency) {
2041       case 2:  NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
2042                break;
2043       case 8:  NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
2044                break;
2045       case 16: [[fallthrough]];
2046       default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
2047                break;
2048       }
2049     } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
2050       switch (HazardDefLatency) {
2051       case 2:  NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
2052                break;
2053       case 8:  NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
2054                break;
2055       case 16: [[fallthrough]];
2056       default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
2057                break;
2058       }
2059     }
2060 
2061     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
2062     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2063 
2064     if (WaitStatesNeeded == MaxWaitStates)
2065       return WaitStatesNeeded; // Early exit.
2066 
2067     auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) {
2068       if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
2069         return false;
2070       Register DstReg = MI.getOperand(0).getReg();
2071       return TRI.regsOverlap(Reg, DstReg);
2072     };
2073 
2074     const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
2075     const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
2076     const int AccVGPRWriteAccVgprReadWaitStates = 3;
2077     NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
2078     if (OpNo == SrcCIdx)
2079       NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
2080     else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
2081       NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
2082 
2083     WaitStatesNeededForUse = NeedWaitStates -
2084       getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates);
2085     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2086 
2087     if (WaitStatesNeeded == MaxWaitStates)
2088       return WaitStatesNeeded; // Early exit.
2089   }
2090 
2091   if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
2092     const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
2093     const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
2094     const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
2095     const int MaxWaitStates = 13;
2096     Register DstReg = MI->getOperand(0).getReg();
2097     unsigned HazardDefLatency = 0;
2098 
2099     auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency,
2100                          this](const MachineInstr &MI) {
2101       if (!SIInstrInfo::isMFMA(MI))
2102         return false;
2103       Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg();
2104       HazardDefLatency =
2105           std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
2106       return TRI.regsOverlap(Reg, DstReg);
2107     };
2108 
2109     int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates);
2110     int NeedWaitStates;
2111     switch (HazardDefLatency) {
2112     case 2:  NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
2113              break;
2114     case 8:  NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
2115              break;
2116     case 16: [[fallthrough]];
2117     default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
2118              break;
2119     }
2120 
2121     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
2122     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2123   }
2124 
2125   // Pad neighboring MFMA with noops for better inter-wave performance.
2126   WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI));
2127 
2128   return WaitStatesNeeded;
2129 }
2130 
2131 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
2132   int WaitStatesNeeded = 0;
2133   unsigned Opc = MI->getOpcode();
2134 
2135   auto IsLegacyVALUFn = [](const MachineInstr &MI) {
2136     return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI);
2137   };
2138 
2139   auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) {
2140     return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI) &&
2141            !SIInstrInfo::isDOT(MI);
2142   };
2143 
2144   if (!SIInstrInfo::isMFMA(*MI))
2145     return WaitStatesNeeded;
2146 
2147   const int VALUWritesExecWaitStates = 4;
2148   int WaitStatesNeededForUse = VALUWritesExecWaitStates -
2149     getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn,
2150                           VALUWritesExecWaitStates);
2151   WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2152 
2153   int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
2154 
2155   // Loop for both DGEMM and S/HGEMM 2nd instruction.
2156   for (const MachineOperand &Use : MI->explicit_uses()) {
2157     const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
2158     const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
2159     const int GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates = 3;
2160     const int GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates = 5;
2161     const int GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates = 4;
2162     const int GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates = 9;
2163     const int GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates = 8;
2164     const int GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates = 17;
2165     const int GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates = 16;
2166     const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
2167     const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
2168     const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
2169     const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
2170     const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
2171     const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
2172     const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
2173     const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
2174     const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
2175     const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
2176     const int GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates = 4;
2177     const int GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates = 6;
2178     const int GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates = 10;
2179     const int GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates = 18;
2180     const int GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates = 5;
2181     const int GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates = 7;
2182     const int GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates = 11;
2183     const int GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates = 19;
2184     const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
2185     const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
2186     const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
2187     const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2;
2188     const int MaxWaitStates = 19;
2189 
2190     if (!Use.isReg())
2191       continue;
2192     Register Reg = Use.getReg();
2193     bool FullReg;
2194     const MachineInstr *MI1;
2195 
2196     auto IsOverlappedMFMAFn = [Reg, &FullReg, &MI1,
2197                                this](const MachineInstr &MI) {
2198       if (!SIInstrInfo::isMFMA(MI))
2199         return false;
2200       Register DstReg = MI.getOperand(0).getReg();
2201       FullReg = (DstReg == Reg);
2202       MI1 = &MI;
2203       return TRI.regsOverlap(DstReg, Reg);
2204     };
2205 
2206     WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
2207       getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates);
2208     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2209 
2210     int NumWaitStates =
2211         getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates);
2212     if (NumWaitStates == std::numeric_limits<int>::max())
2213       continue;
2214 
2215     int OpNo = Use.getOperandNo();
2216     unsigned Opc1 = MI1->getOpcode();
2217     int NeedWaitStates = 0;
2218     if (OpNo == SrcCIdx) {
2219       if (!isDGEMM(Opc) && (!ST.hasGFX940Insts() && isDGEMM(Opc1))) {
2220         NeedWaitStates = 0;
2221       } else if (FullReg) {
2222         if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
2223              Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
2224             (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
2225              Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
2226           NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
2227         else if (ST.hasGFX940Insts() &&
2228                  TSchedModel.computeInstrLatency(MI1) == 2)
2229           NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
2230       } else {
2231         switch (Opc1) {
2232         case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
2233         case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
2234         case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
2235         case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
2236           if (!isXDL(ST, *MI))
2237             NeedWaitStates = DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
2238           break;
2239         case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
2240         case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
2241           if (!isXDL(ST, *MI))
2242             NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
2243           break;
2244         default:
2245           if (ST.hasGFX940Insts() && isXDL(ST, *MI) && !isXDL(ST, *MI1))
2246             break;
2247           switch (TSchedModel.computeInstrLatency(MI1)) {
2248           case 2:
2249             NeedWaitStates = ST.hasGFX940Insts()
2250               ? isXDL(ST, *MI1)
2251                 ? GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates
2252                 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates
2253               : isDGEMM(Opc)
2254                 ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
2255                 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
2256             break;
2257           case 4:
2258             assert(ST.hasGFX940Insts());
2259             NeedWaitStates = isXDL(ST, *MI1)
2260               ? GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates
2261               : GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates;
2262             break;
2263           case 8:
2264             NeedWaitStates = ST.hasGFX940Insts()
2265               ? isXDL(ST, *MI1)
2266                 ? GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates
2267                 : GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates
2268               : isDGEMM(Opc)
2269                 ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
2270                 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
2271             break;
2272           case 16: [[fallthrough]];
2273           default:
2274             NeedWaitStates = ST.hasGFX940Insts()
2275               ? isXDL(ST, *MI1)
2276                 ? GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates
2277                 : GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates
2278               : isDGEMM(Opc)
2279                 ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
2280                 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
2281           }
2282         }
2283       }
2284     } else {
2285       switch (Opc1) {
2286       case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
2287       case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
2288       case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
2289       case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
2290         NeedWaitStates = DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
2291         break;
2292       case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
2293       case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
2294         NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
2295         break;
2296       default:
2297         switch (TSchedModel.computeInstrLatency(MI1)) {
2298         case 2:
2299           NeedWaitStates = ST.hasGFX940Insts()
2300             ? isXDL(ST, *MI1)
2301               ? GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates
2302               : GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates
2303             : SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
2304           break;
2305         case 4:
2306           assert(ST.hasGFX940Insts());
2307           NeedWaitStates = isXDL(ST, *MI1)
2308             ? GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates
2309             : GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates;
2310           break;
2311         case 8:
2312           NeedWaitStates = ST.hasGFX940Insts()
2313             ? isXDL(ST, *MI1)
2314               ? GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates
2315               : GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates
2316             : SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
2317           break;
2318         case 16: [[fallthrough]];
2319         default:
2320           NeedWaitStates = ST.hasGFX940Insts()
2321             ? isXDL(ST, *MI1)
2322               ? GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates
2323               : GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates
2324             : SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
2325         }
2326       }
2327     }
2328     if (WaitStatesNeeded >= NeedWaitStates)
2329       continue;
2330 
2331     WaitStatesNeededForUse = NeedWaitStates - NumWaitStates;
2332     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2333 
2334     if (WaitStatesNeeded == MaxWaitStates)
2335       break;
2336   }
2337 
2338   return WaitStatesNeeded;
2339 }
2340 
2341 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) {
2342   // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards()
2343   if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
2344     return 0;
2345 
2346   int WaitStatesNeeded = 0;
2347 
2348   auto IsAccVgprReadFn = [](const MachineInstr &MI) {
2349     return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
2350   };
2351 
2352   for (const MachineOperand &Op : MI->explicit_uses()) {
2353     if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg()))
2354       continue;
2355 
2356     Register Reg = Op.getReg();
2357 
2358     const int AccVgprReadLdStWaitStates = 2;
2359     const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
2360     const int MaxWaitStates = 2;
2361 
2362     int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
2363       getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates);
2364     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2365 
2366     if (WaitStatesNeeded == MaxWaitStates)
2367       return WaitStatesNeeded; // Early exit.
2368 
2369     auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) {
2370       if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
2371           MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
2372         return false;
2373       auto IsVALUFn = [](const MachineInstr &MI) {
2374         return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMAI(MI);
2375       };
2376       return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) <
2377              std::numeric_limits<int>::max();
2378     };
2379 
2380     WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
2381       getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates);
2382     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2383   }
2384 
2385   return WaitStatesNeeded;
2386 }
2387 
2388 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
2389   if (!ST.hasGFX90AInsts())
2390     return 0;
2391 
2392   auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
2393     return isDGEMM(MI.getOpcode());
2394   };
2395 
2396   // This is checked in checkMAIHazards90A()
2397   if (SIInstrInfo::isMFMA(*MI))
2398     return 0;
2399 
2400   const MachineRegisterInfo &MRI = MF.getRegInfo();
2401 
2402   int WaitStatesNeeded = 0;
2403 
2404   bool IsMem = SIInstrInfo::isVMEM(*MI) ||
2405                SIInstrInfo::isFLAT(*MI) ||
2406                SIInstrInfo::isDS(*MI);
2407   bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(*MI);
2408   bool IsVALU = SIInstrInfo::isVALU(*MI);
2409 
2410   const MachineInstr *MFMA = nullptr;
2411   unsigned Reg;
2412   auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
2413     if (!SIInstrInfo::isMFMA(MI) ||
2414         !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
2415       return false;
2416     MFMA = &MI;
2417     return true;
2418   };
2419 
2420   const MachineInstr *DOT = nullptr;
2421   auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) {
2422     if (!SIInstrInfo::isDOT(MI) ||
2423         !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
2424       return false;
2425     DOT = &MI;
2426     return true;
2427   };
2428 
2429   bool DGEMMAfterVALUWrite = false;
2430   auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) {
2431     // Found DGEMM on reverse traversal to def.
2432     if (isDGEMM(MI.getOpcode()))
2433       DGEMMAfterVALUWrite = true;
2434 
2435     // Only hazard if register is defined by a VALU and a DGEMM is found after
2436     // after the def.
2437     if (!TII.isVALU(MI) || !DGEMMAfterVALUWrite)
2438       return false;
2439 
2440     return true;
2441   };
2442 
2443   int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(),
2444                                            AMDGPU::OpName::src2);
2445 
2446   if (IsMemOrExport || IsVALU) {
2447     const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
2448     const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
2449     const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
2450     const int GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates = 4;
2451     const int GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates = 6;
2452     const int GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates = 10;
2453     const int GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates = 18;
2454     const int GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates = 5;
2455     const int GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates = 7;
2456     const int GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates = 11;
2457     const int GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates = 19;
2458     const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
2459     const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
2460     const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
2461     const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
2462     const int DotWriteSameDotReadSrcAB = 3;
2463     const int DotWriteDifferentVALURead = 3;
2464     const int DMFMABetweenVALUWriteVMEMRead = 2;
2465     const int MaxWaitStates = 19;
2466 
2467     for (const MachineOperand &Use : MI->explicit_uses()) {
2468       if (!Use.isReg())
2469         continue;
2470       Reg = Use.getReg();
2471 
2472       DOT = nullptr;
2473       int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
2474                                                      MaxWaitStates);
2475       if (DOT) {
2476         int NeedWaitStates = 0;
2477         if (DOT->getOpcode() == MI->getOpcode()) {
2478           if (&Use - &MI->getOperand(0) != SrcCIdx)
2479             NeedWaitStates = DotWriteSameDotReadSrcAB;
2480         } else {
2481           NeedWaitStates = DotWriteDifferentVALURead;
2482         }
2483 
2484         int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
2485         WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2486       }
2487 
2488       // Workaround for HW data hazard bug observed only in GFX90A. When there
2489       // is a DGEMM instruction in-between a VALU and a VMEM instruction it
2490       // causes the SQ to incorrectly not insert two wait states between the two
2491       // instructions needed to avoid data hazard.
2492       if (IsMem && ST.hasGFX90AInsts() && !ST.hasGFX940Insts()) {
2493         DGEMMAfterVALUWrite = false;
2494         if (TRI.isVectorRegister(MRI, Reg)) {
2495           int WaitStatesNeededForUse =
2496                 DMFMABetweenVALUWriteVMEMRead -
2497                 getWaitStatesSinceDef(Reg, IsDGEMMHazard,
2498                                       DMFMABetweenVALUWriteVMEMRead);
2499 
2500           WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2501         }
2502       }
2503 
2504       MFMA = nullptr;
2505       WaitStatesSinceDef =
2506           getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
2507       if (!MFMA)
2508         continue;
2509 
2510       unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
2511       int NeedWaitStates = MaxWaitStates;
2512       switch (HazardDefLatency) {
2513       case 2:
2514         NeedWaitStates =
2515           ST.hasGFX940Insts()
2516             ? isXDL(ST, *MFMA)
2517               ? GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates
2518               : GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates
2519             : SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
2520         break;
2521       case 4:
2522         assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts());
2523         NeedWaitStates =
2524           isDGEMM(MFMA->getOpcode())
2525             ? IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
2526                             : DMFMA4x4WriteVgprVALUReadWaitStates
2527             : isXDL(ST, *MFMA)
2528               ? GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates
2529               : GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates;
2530         break;
2531       case 8:
2532         NeedWaitStates =
2533           ST.hasGFX940Insts()
2534             ? isXDL(ST, *MFMA)
2535               ? GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates
2536               : GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates
2537             : SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
2538         break;
2539       case 16: [[fallthrough]];
2540       default:
2541         NeedWaitStates =
2542           isDGEMM(MFMA->getOpcode())
2543             ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates
2544                             : DMFMA16x16WriteVgprVALUReadWaitStates
2545             : ST.hasGFX940Insts()
2546               ? isXDL(ST, *MFMA)
2547                 ? GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates
2548                 : GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates
2549               : SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
2550         break;
2551       }
2552 
2553       int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
2554       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2555 
2556       if (WaitStatesNeeded == MaxWaitStates)
2557         break;
2558     }
2559   }
2560 
2561   unsigned Opc = MI->getOpcode();
2562   const int DMFMAToFMA64WaitStates = 2;
2563   if ((Opc == AMDGPU::V_FMA_F64_e64 ||
2564        Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
2565        Opc == AMDGPU::V_FMAC_F64_dpp) &&
2566       WaitStatesNeeded < DMFMAToFMA64WaitStates) {
2567     int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
2568       getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates);
2569     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2570   }
2571 
2572   if (!IsVALU && !IsMemOrExport)
2573     return WaitStatesNeeded;
2574 
2575   for (const MachineOperand &Def : MI->defs()) {
2576     const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
2577     const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
2578     const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
2579     const int GFX940_SMFMA2PassWriteVgprVALUWawWaitStates = 4;
2580     const int GFX940_SMFMA4PassWriteVgprVALUWawWaitStates = 6;
2581     const int GFX940_SMFMA8PassWriteVgprVALUWawWaitStates = 10;
2582     const int GFX940_SMFMA16PassWriteVgprVALUWawWaitStates = 18;
2583     const int GFX940_XDL2PassWriteVgprVALUWawWaitStates = 5;
2584     const int GFX940_XDL4PassWriteVgprVALUWawWaitStates = 7;
2585     const int GFX940_XDL8PassWriteVgprVALUWawWaitStates = 11;
2586     const int GFX940_XDL16PassWriteVgprVALUWawWaitStates = 19;
2587     const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
2588     const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3;
2589     const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
2590     const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
2591     const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
2592     const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
2593     const int DotWriteDifferentVALUWrite = 3;
2594     const int MaxWaitStates = 19;
2595     const int MaxWarWaitStates = 15;
2596 
2597     Reg = Def.getReg();
2598 
2599     DOT = nullptr;
2600     int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
2601                                                    MaxWaitStates);
2602     if (DOT && DOT->getOpcode() != MI->getOpcode())
2603       WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite -
2604                                                     WaitStatesSinceDef);
2605 
2606     MFMA = nullptr;
2607     WaitStatesSinceDef =
2608         getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
2609     if (MFMA) {
2610       int NeedWaitStates = MaxWaitStates;
2611       switch (TSchedModel.computeInstrLatency(MFMA)) {
2612       case 2:
2613         NeedWaitStates = ST.hasGFX940Insts()
2614           ? isXDL(ST, *MFMA)
2615             ? GFX940_XDL2PassWriteVgprVALUWawWaitStates
2616             : GFX940_SMFMA2PassWriteVgprVALUWawWaitStates
2617           : SMFMA4x4WriteVgprVALUWawWaitStates;
2618         break;
2619       case 4:
2620         assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts());
2621         NeedWaitStates = isDGEMM(MFMA->getOpcode())
2622             ? DMFMA4x4WriteVgprVALUWriteWaitStates
2623             : isXDL(ST, *MFMA)
2624               ? GFX940_XDL4PassWriteVgprVALUWawWaitStates
2625               : GFX940_SMFMA4PassWriteVgprVALUWawWaitStates;
2626         break;
2627       case 8:
2628         NeedWaitStates = ST.hasGFX940Insts()
2629           ? isXDL(ST, *MFMA)
2630             ? GFX940_XDL8PassWriteVgprVALUWawWaitStates
2631             : GFX940_SMFMA8PassWriteVgprVALUWawWaitStates
2632           : SMFMA16x16WriteVgprVALUWawWaitStates;
2633         break;
2634       case 16: [[fallthrough]];
2635       default:
2636         NeedWaitStates = isDGEMM(MFMA->getOpcode())
2637                    ? DMFMA16x16WriteVgprVALUWriteWaitStates
2638                    : ST.hasGFX940Insts()
2639                      ? isXDL(ST, *MFMA)
2640                        ? GFX940_XDL16PassWriteVgprVALUWawWaitStates
2641                        : GFX940_SMFMA16PassWriteVgprVALUWawWaitStates
2642                    : SMFMA32x32WriteVgprVALUWawWaitStates;
2643         break;
2644       }
2645 
2646       int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
2647       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2648 
2649       if (WaitStatesNeeded == MaxWaitStates)
2650         break;
2651     }
2652 
2653     auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
2654       if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) ||
2655           !MI.readsRegister(Reg, &TRI))
2656         return false;
2657 
2658       if (ST.hasGFX940Insts() && !isXDL(ST, MI))
2659         return false;
2660 
2661       const MachineOperand *SrcC =
2662           TII.getNamedOperand(MI, AMDGPU::OpName::src2);
2663       assert(SrcC);
2664       if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg))
2665         return false;
2666 
2667       MFMA = &MI;
2668       return true;
2669     };
2670 
2671     MFMA = nullptr;
2672     int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn,
2673                                                 MaxWarWaitStates);
2674     if (!MFMA)
2675       continue;
2676 
2677     unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
2678     int NeedWaitStates = MaxWaitStates;
2679     switch (HazardDefLatency) {
2680     case 2:  NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
2681              break;
2682     case 4:  assert(ST.hasGFX940Insts());
2683              NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates;
2684              break;
2685     case 8:  NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates;
2686              break;
2687     case 16: [[fallthrough]];
2688     default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates;
2689              break;
2690     }
2691 
2692     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse;
2693     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2694   }
2695 
2696   return WaitStatesNeeded;
2697 }
2698 
2699 bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) {
2700   if (!SU->isInstr())
2701     return false;
2702 
2703   const MachineInstr *MAI = nullptr;
2704 
2705   auto IsMFMAFn = [&MAI](const MachineInstr &MI) {
2706     MAI = nullptr;
2707     if (SIInstrInfo::isMFMA(MI))
2708       MAI = &MI;
2709     return MAI != nullptr;
2710   };
2711 
2712   MachineInstr *MI = SU->getInstr();
2713   if (IsMFMAFn(*MI)) {
2714     int W = getWaitStatesSince(IsMFMAFn, 16);
2715     if (MAI)
2716       return W < (int)TSchedModel.computeInstrLatency(MAI);
2717   }
2718 
2719   return false;
2720 }
2721 
2722 bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
2723   if (!ST.isWave64())
2724     return false;
2725   if (!ST.hasVALUMaskWriteHazard())
2726     return false;
2727   if (!SIInstrInfo::isSALU(*MI))
2728     return false;
2729 
2730   // The hazard sequence is three instructions:
2731   //   1. VALU reads SGPR as mask
2732   //   2. SALU writes SGPR
2733   //   3. SALU reads SGPR
2734   // The hazard can expire if the distance between 2 and 3 is sufficient.
2735   // In practice this happens <10% of the time, hence this always assumes
2736   // the hazard exists if 1 and 2 are present to avoid searching.
2737 
2738   const MachineOperand *SDSTOp = TII.getNamedOperand(*MI, AMDGPU::OpName::sdst);
2739   if (!SDSTOp || !SDSTOp->isReg())
2740     return false;
2741 
2742   const Register HazardReg = SDSTOp->getReg();
2743   if (HazardReg == AMDGPU::EXEC ||
2744       HazardReg == AMDGPU::EXEC_LO ||
2745       HazardReg == AMDGPU::EXEC_HI ||
2746       HazardReg == AMDGPU::M0)
2747     return false;
2748 
2749   auto IsHazardFn = [HazardReg, this](const MachineInstr &I) {
2750     switch (I.getOpcode()) {
2751     case AMDGPU::V_ADDC_U32_e32:
2752     case AMDGPU::V_ADDC_U32_dpp:
2753     case AMDGPU::V_CNDMASK_B16_e32:
2754     case AMDGPU::V_CNDMASK_B16_dpp:
2755     case AMDGPU::V_CNDMASK_B32_e32:
2756     case AMDGPU::V_CNDMASK_B32_dpp:
2757     case AMDGPU::V_DIV_FMAS_F32_e64:
2758     case AMDGPU::V_DIV_FMAS_F64_e64:
2759     case AMDGPU::V_SUBB_U32_e32:
2760     case AMDGPU::V_SUBB_U32_dpp:
2761     case AMDGPU::V_SUBBREV_U32_e32:
2762     case AMDGPU::V_SUBBREV_U32_dpp:
2763       // These implicitly read VCC as mask source.
2764       return HazardReg == AMDGPU::VCC ||
2765              HazardReg == AMDGPU::VCC_LO ||
2766              HazardReg == AMDGPU::VCC_HI;
2767     case AMDGPU::V_ADDC_U32_e64:
2768     case AMDGPU::V_ADDC_U32_e64_dpp:
2769     case AMDGPU::V_CNDMASK_B16_e64:
2770     case AMDGPU::V_CNDMASK_B16_e64_dpp:
2771     case AMDGPU::V_CNDMASK_B32_e64:
2772     case AMDGPU::V_CNDMASK_B32_e64_dpp:
2773     case AMDGPU::V_SUBB_U32_e64:
2774     case AMDGPU::V_SUBB_U32_e64_dpp:
2775     case AMDGPU::V_SUBBREV_U32_e64:
2776     case AMDGPU::V_SUBBREV_U32_e64_dpp: {
2777       // Only check mask register overlaps.
2778       const MachineOperand *SSRCOp = TII.getNamedOperand(I, AMDGPU::OpName::src2);
2779       assert(SSRCOp);
2780       return TRI.regsOverlap(SSRCOp->getReg(), HazardReg);
2781     }
2782     default:
2783       return false;
2784     }
2785   };
2786 
2787   const MachineRegisterInfo &MRI = MF.getRegInfo();
2788   auto IsExpiredFn = [&MRI, this](const MachineInstr &I, int) {
2789     // s_waitcnt_depctr sa_sdst(0) mitigates hazard.
2790     if (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
2791         AMDGPU::DepCtr::decodeFieldSaSdst(I.getOperand(0).getImm()) == 0)
2792       return true;
2793 
2794     // VALU access to any SGPR or literal constant other than HazardReg
2795     // mitigates hazard. No need to check HazardReg here as this will
2796     // only be called when !IsHazardFn.
2797     if (!SIInstrInfo::isVALU(I))
2798       return false;
2799     for (int OpNo = 0, End = I.getNumOperands(); OpNo < End; ++OpNo) {
2800       const MachineOperand &Op = I.getOperand(OpNo);
2801       if (Op.isReg()) {
2802         Register OpReg = Op.getReg();
2803         // Only consider uses
2804         if (!Op.isUse())
2805           continue;
2806         // Ignore EXEC
2807         if (OpReg == AMDGPU::EXEC ||
2808             OpReg == AMDGPU::EXEC_LO ||
2809             OpReg == AMDGPU::EXEC_HI)
2810           continue;
2811         // Ignore all implicit uses except VCC
2812         if (Op.isImplicit()) {
2813           if (OpReg == AMDGPU::VCC ||
2814               OpReg == AMDGPU::VCC_LO ||
2815               OpReg == AMDGPU::VCC_HI)
2816             return true;
2817           continue;
2818         }
2819         if (TRI.isSGPRReg(MRI, OpReg))
2820           return true;
2821       } else {
2822         const MCInstrDesc &InstDesc = I.getDesc();
2823         const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
2824         if (!TII.isInlineConstant(Op, OpInfo))
2825           return true;
2826       }
2827     }
2828     return false;
2829   };
2830 
2831   // Check for hazard
2832   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
2833       std::numeric_limits<int>::max())
2834     return false;
2835 
2836   auto NextMI = std::next(MI->getIterator());
2837 
2838   // Add s_waitcnt_depctr sa_sdst(0) after SALU write.
2839   BuildMI(*MI->getParent(), NextMI, MI->getDebugLoc(),
2840           TII.get(AMDGPU::S_WAITCNT_DEPCTR))
2841       .addImm(AMDGPU::DepCtr::encodeFieldSaSdst(0));
2842 
2843   // SALU write may be s_getpc in a bundle.
2844   if (MI->getOpcode() == AMDGPU::S_GETPC_B64) {
2845     // Update offsets of any references in the bundle.
2846     while (NextMI != MI->getParent()->end() &&
2847            NextMI->isBundledWithPred()) {
2848       for (auto &Operand : NextMI->operands()) {
2849         if (Operand.isGlobal())
2850           Operand.setOffset(Operand.getOffset() + 4);
2851       }
2852       NextMI++;
2853     }
2854   }
2855 
2856   return true;
2857 }
2858