xref: /llvm-project/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp (revision bd9eed3aecc6326d0260cfe68ee7bddc5011989c) 
1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements hazard recognizers for scheduling on GCN processors.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "GCNHazardRecognizer.h"
14 #include "GCNSubtarget.h"
15 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
16 #include "SIMachineFunctionInfo.h"
17 #include "llvm/CodeGen/MachineFunction.h"
18 #include "llvm/CodeGen/ScheduleDAG.h"
19 #include "llvm/Support/TargetParser.h"
20 
21 using namespace llvm;
22 
23 namespace {
24 
25 struct MFMAPaddingRatioParser : public cl::parser<unsigned> {
26   MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {}
27 
28   bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
29     if (Arg.getAsInteger(0, Value))
30       return O.error("'" + Arg + "' value invalid for uint argument!");
31 
32     if (Value > 100)
33       return O.error("'" + Arg + "' value must be in the range [0, 100]!");
34 
35     return false;
36   }
37 };
38 
39 } // end anonymous namespace
40 
41 static cl::opt<unsigned, false, MFMAPaddingRatioParser>
42     MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden,
43                      cl::desc("Fill a percentage of the latency between "
44                               "neighboring MFMA with s_nops."));
45 
46 //===----------------------------------------------------------------------===//
47 // Hazard Recognizer Implementation
48 //===----------------------------------------------------------------------===//
49 
50 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
51                                                  const GCNSubtarget &ST);
52 
53 GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) :
54   IsHazardRecognizerMode(false),
55   CurrCycleInstr(nullptr),
56   MF(MF),
57   ST(MF.getSubtarget<GCNSubtarget>()),
58   TII(*ST.getInstrInfo()),
59   TRI(TII.getRegisterInfo()),
60   ClauseUses(TRI.getNumRegUnits()),
61   ClauseDefs(TRI.getNumRegUnits()) {
62   MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5;
63   TSchedModel.init(&ST);
64   RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
65 }
66 
67 void GCNHazardRecognizer::Reset() {
68   EmittedInstrs.clear();
69 }
70 
71 void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
72   EmitInstruction(SU->getInstr());
73 }
74 
75 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
76   CurrCycleInstr = MI;
77 }
78 
79 static bool isDivFMas(unsigned Opcode) {
80   return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
81 }
82 
83 static bool isSGetReg(unsigned Opcode) {
84   return Opcode == AMDGPU::S_GETREG_B32;
85 }
86 
87 static bool isSSetReg(unsigned Opcode) {
88   switch (Opcode) {
89   case AMDGPU::S_SETREG_B32:
90   case AMDGPU::S_SETREG_B32_mode:
91   case AMDGPU::S_SETREG_IMM32_B32:
92   case AMDGPU::S_SETREG_IMM32_B32_mode:
93     return true;
94   }
95   return false;
96 }
97 
98 static bool isRWLane(unsigned Opcode) {
99   return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
100 }
101 
102 static bool isRFE(unsigned Opcode) {
103   return Opcode == AMDGPU::S_RFE_B64;
104 }
105 
106 static bool isSMovRel(unsigned Opcode) {
107   switch (Opcode) {
108   case AMDGPU::S_MOVRELS_B32:
109   case AMDGPU::S_MOVRELS_B64:
110   case AMDGPU::S_MOVRELD_B32:
111   case AMDGPU::S_MOVRELD_B64:
112     return true;
113   default:
114     return false;
115   }
116 }
117 
118 static bool isDGEMM(unsigned Opcode) {
119   return AMDGPU::getMAIIsDGEMM(Opcode);
120 }
121 
122 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) {
123   unsigned Opcode = MI.getOpcode();
124 
125   if (!SIInstrInfo::isMAI(MI) ||
126       isDGEMM(Opcode) ||
127       Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
128       Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
129     return false;
130 
131   if (!ST.hasGFX940Insts())
132     return true;
133 
134   return AMDGPU::getMAIIsGFX940XDL(Opcode);
135 }
136 
137 static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
138                                     const MachineInstr &MI) {
139   if (TII.isAlwaysGDS(MI.getOpcode()))
140     return true;
141 
142   switch (MI.getOpcode()) {
143   case AMDGPU::S_SENDMSG:
144   case AMDGPU::S_SENDMSGHALT:
145   case AMDGPU::S_TTRACEDATA:
146     return true;
147   // These DS opcodes don't support GDS.
148   case AMDGPU::DS_NOP:
149   case AMDGPU::DS_PERMUTE_B32:
150   case AMDGPU::DS_BPERMUTE_B32:
151     return false;
152   default:
153     if (TII.isDS(MI.getOpcode())) {
154       int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
155                                            AMDGPU::OpName::gds);
156       if (MI.getOperand(GDS).getImm())
157         return true;
158     }
159     return false;
160   }
161 }
162 
163 static bool isPermlane(const MachineInstr &MI) {
164   unsigned Opcode = MI.getOpcode();
165   return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
166          Opcode == AMDGPU::V_PERMLANEX16_B32_e64;
167 }
168 
169 static bool isLdsDma(const MachineInstr &MI) {
170   return SIInstrInfo::isVALU(MI) &&
171          (SIInstrInfo::isMUBUF(MI) || SIInstrInfo::isFLAT(MI));
172 }
173 
174 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
175   const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
176                                                      AMDGPU::OpName::simm16);
177   return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
178 }
179 
180 ScheduleHazardRecognizer::HazardType
181 GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
182   MachineInstr *MI = SU->getInstr();
183   // If we are not in "HazardRecognizerMode" and therefore not being run from
184   // the scheduler, track possible stalls from hazards but don't insert noops.
185   auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard;
186 
187   if (MI->isBundle())
188    return NoHazard;
189 
190   if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
191     return HazardType;
192 
193   if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
194     return HazardType;
195 
196   if (checkFPAtomicToDenormModeHazard(MI) > 0)
197     return HazardType;
198 
199   if (ST.hasNoDataDepHazard())
200     return NoHazard;
201 
202   // FIXME: Should flat be considered vmem?
203   if ((SIInstrInfo::isVMEM(*MI) ||
204        SIInstrInfo::isFLAT(*MI))
205       && checkVMEMHazards(MI) > 0)
206     return HazardType;
207 
208   if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
209     return HazardType;
210 
211   if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
212     return HazardType;
213 
214   if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
215     return HazardType;
216 
217   if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
218     return HazardType;
219 
220   if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) ||
221        SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) ||
222        SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
223     return HazardType;
224 
225   if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
226     return HazardType;
227 
228   if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
229     return HazardType;
230 
231   if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
232     return HazardType;
233 
234   if (((ST.hasReadM0MovRelInterpHazard() &&
235         (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()))) ||
236        (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) ||
237        (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) ||
238        (ST.hasReadM0LdsDirectHazard() &&
239         MI->readsRegister(AMDGPU::LDS_DIRECT))) &&
240       checkReadM0Hazards(MI) > 0)
241     return HazardType;
242 
243   if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0)
244     return HazardType;
245 
246   if ((SIInstrInfo::isVMEM(*MI) ||
247        SIInstrInfo::isFLAT(*MI) ||
248        SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0)
249     return HazardType;
250 
251   if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
252     return HazardType;
253 
254   return NoHazard;
255 }
256 
257 static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII,
258                                 unsigned Quantity) {
259   while (Quantity > 0) {
260     unsigned Arg = std::min(Quantity, 8u);
261     Quantity -= Arg;
262     BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP))
263         .addImm(Arg - 1);
264   }
265 }
266 
267 unsigned
268 GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const {
269   const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&MI);
270   assert(TSchedModel.getWriteProcResBegin(SC) !=
271          TSchedModel.getWriteProcResEnd(SC));
272   return TSchedModel.getWriteProcResBegin(SC)->Cycles;
273 }
274 
275 void GCNHazardRecognizer::processBundle() {
276   MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator());
277   MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end();
278   // Check bundled MachineInstr's for hazards.
279   for (; MI != E && MI->isInsideBundle(); ++MI) {
280     CurrCycleInstr = &*MI;
281     unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
282 
283     if (IsHazardRecognizerMode) {
284       fixHazards(CurrCycleInstr);
285 
286       insertNoopsInBundle(CurrCycleInstr, TII, WaitStates);
287     }
288 
289     // It’s unnecessary to track more than MaxLookAhead instructions. Since we
290     // include the bundled MI directly after, only add a maximum of
291     // (MaxLookAhead - 1) noops to EmittedInstrs.
292     for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i)
293       EmittedInstrs.push_front(nullptr);
294 
295     EmittedInstrs.push_front(CurrCycleInstr);
296     EmittedInstrs.resize(MaxLookAhead);
297   }
298   CurrCycleInstr = nullptr;
299 }
300 
301 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
302   IsHazardRecognizerMode = true;
303   CurrCycleInstr = MI;
304   unsigned W = PreEmitNoopsCommon(MI);
305   fixHazards(MI);
306   CurrCycleInstr = nullptr;
307   return W;
308 }
309 
310 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) {
311   if (MI->isBundle())
312     return 0;
313 
314   int WaitStates = 0;
315 
316   if (SIInstrInfo::isSMRD(*MI))
317     return std::max(WaitStates, checkSMRDHazards(MI));
318 
319   if (ST.hasNSAtoVMEMBug())
320     WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI));
321 
322   WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI));
323 
324   if (ST.hasNoDataDepHazard())
325     return WaitStates;
326 
327   if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI))
328     WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
329 
330   if (SIInstrInfo::isVALU(*MI))
331     WaitStates = std::max(WaitStates, checkVALUHazards(MI));
332 
333   if (SIInstrInfo::isDPP(*MI))
334     WaitStates = std::max(WaitStates, checkDPPHazards(MI));
335 
336   if (isDivFMas(MI->getOpcode()))
337     WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
338 
339   if (isRWLane(MI->getOpcode()))
340     WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
341 
342   if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) ||
343        SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) ||
344        SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
345     WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI));
346 
347   if (MI->isInlineAsm())
348     return std::max(WaitStates, checkInlineAsmHazards(MI));
349 
350   if (isSGetReg(MI->getOpcode()))
351     return std::max(WaitStates, checkGetRegHazards(MI));
352 
353   if (isSSetReg(MI->getOpcode()))
354     return std::max(WaitStates, checkSetRegHazards(MI));
355 
356   if (isRFE(MI->getOpcode()))
357     return std::max(WaitStates, checkRFEHazards(MI));
358 
359   if ((ST.hasReadM0MovRelInterpHazard() &&
360        (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()))) ||
361       (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) ||
362       (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) ||
363       (ST.hasReadM0LdsDirectHazard() && MI->readsRegister(AMDGPU::LDS_DIRECT)))
364     return std::max(WaitStates, checkReadM0Hazards(MI));
365 
366   if (SIInstrInfo::isMAI(*MI))
367     return std::max(WaitStates, checkMAIHazards(MI));
368 
369   if (SIInstrInfo::isVMEM(*MI) ||
370       SIInstrInfo::isFLAT(*MI) ||
371       SIInstrInfo::isDS(*MI))
372     return std::max(WaitStates, checkMAILdStHazards(MI));
373 
374   return WaitStates;
375 }
376 
377 void GCNHazardRecognizer::EmitNoop() {
378   EmittedInstrs.push_front(nullptr);
379 }
380 
381 void GCNHazardRecognizer::AdvanceCycle() {
382   // When the scheduler detects a stall, it will call AdvanceCycle() without
383   // emitting any instructions.
384   if (!CurrCycleInstr) {
385     EmittedInstrs.push_front(nullptr);
386     return;
387   }
388 
389   if (CurrCycleInstr->isBundle()) {
390     processBundle();
391     return;
392   }
393 
394   unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
395   if (!NumWaitStates) {
396     CurrCycleInstr = nullptr;
397     return;
398   }
399 
400   // Keep track of emitted instructions
401   EmittedInstrs.push_front(CurrCycleInstr);
402 
403   // Add a nullptr for each additional wait state after the first.  Make sure
404   // not to add more than getMaxLookAhead() items to the list, since we
405   // truncate the list to that size right after this loop.
406   for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
407        i < e; ++i) {
408     EmittedInstrs.push_front(nullptr);
409   }
410 
411   // getMaxLookahead() is the largest number of wait states we will ever need
412   // to insert, so there is no point in keeping track of more than that many
413   // wait states.
414   EmittedInstrs.resize(getMaxLookAhead());
415 
416   CurrCycleInstr = nullptr;
417 }
418 
419 void GCNHazardRecognizer::RecedeCycle() {
420   llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
421 }
422 
423 //===----------------------------------------------------------------------===//
424 // Helper Functions
425 //===----------------------------------------------------------------------===//
426 
427 typedef function_ref<bool(const MachineInstr &, int WaitStates)> IsExpiredFn;
428 
429 // Returns a minimum wait states since \p I walking all predecessors.
430 // Only scans until \p IsExpired does not return true.
431 // Can only be run in a hazard recognizer mode.
432 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
433                               const MachineBasicBlock *MBB,
434                               MachineBasicBlock::const_reverse_instr_iterator I,
435                               int WaitStates, IsExpiredFn IsExpired,
436                               DenseSet<const MachineBasicBlock *> &Visited) {
437   for (auto E = MBB->instr_rend(); I != E; ++I) {
438     // Don't add WaitStates for parent BUNDLE instructions.
439     if (I->isBundle())
440       continue;
441 
442     if (IsHazard(*I))
443       return WaitStates;
444 
445     if (I->isInlineAsm())
446       continue;
447 
448     WaitStates += SIInstrInfo::getNumWaitStates(*I);
449 
450     if (IsExpired(*I, WaitStates))
451       return std::numeric_limits<int>::max();
452   }
453 
454   int MinWaitStates = std::numeric_limits<int>::max();
455   for (MachineBasicBlock *Pred : MBB->predecessors()) {
456     if (!Visited.insert(Pred).second)
457       continue;
458 
459     int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(),
460                                WaitStates, IsExpired, Visited);
461 
462     MinWaitStates = std::min(MinWaitStates, W);
463   }
464 
465   return MinWaitStates;
466 }
467 
468 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
469                               const MachineInstr *MI, IsExpiredFn IsExpired) {
470   DenseSet<const MachineBasicBlock *> Visited;
471   return getWaitStatesSince(IsHazard, MI->getParent(),
472                             std::next(MI->getReverseIterator()),
473                             0, IsExpired, Visited);
474 }
475 
476 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) {
477   if (IsHazardRecognizerMode) {
478     auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) {
479       return WaitStates >= Limit;
480     };
481     return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn);
482   }
483 
484   int WaitStates = 0;
485   for (MachineInstr *MI : EmittedInstrs) {
486     if (MI) {
487       if (IsHazard(*MI))
488         return WaitStates;
489 
490       if (MI->isInlineAsm())
491         continue;
492     }
493     ++WaitStates;
494 
495     if (WaitStates >= Limit)
496       break;
497   }
498   return std::numeric_limits<int>::max();
499 }
500 
501 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
502                                                IsHazardFn IsHazardDef,
503                                                int Limit) {
504   const SIRegisterInfo *TRI = ST.getRegisterInfo();
505 
506   auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) {
507     return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI);
508   };
509 
510   return getWaitStatesSince(IsHazardFn, Limit);
511 }
512 
513 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
514                                                   int Limit) {
515   auto IsHazardFn = [IsHazard](const MachineInstr &MI) {
516     return isSSetReg(MI.getOpcode()) && IsHazard(MI);
517   };
518 
519   return getWaitStatesSince(IsHazardFn, Limit);
520 }
521 
522 //===----------------------------------------------------------------------===//
523 // No-op Hazard Detection
524 //===----------------------------------------------------------------------===//
525 
526 static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
527                         MCRegister Reg) {
528   for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI)
529     BV.set(*RUI);
530 }
531 
532 static void addRegsToSet(const SIRegisterInfo &TRI,
533                          iterator_range<MachineInstr::const_mop_iterator> Ops,
534                          BitVector &Set) {
535   for (const MachineOperand &Op : Ops) {
536     if (Op.isReg())
537       addRegUnits(TRI, Set, Op.getReg().asMCReg());
538   }
539 }
540 
541 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) {
542   // XXX: Do we need to worry about implicit operands
543   addRegsToSet(TRI, MI.defs(), ClauseDefs);
544   addRegsToSet(TRI, MI.uses(), ClauseUses);
545 }
546 
547 static bool breaksSMEMSoftClause(MachineInstr *MI) {
548   return !SIInstrInfo::isSMRD(*MI);
549 }
550 
551 static bool breaksVMEMSoftClause(MachineInstr *MI) {
552   return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI);
553 }
554 
555 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
556   // SMEM soft clause are only present on VI+, and only matter if xnack is
557   // enabled.
558   if (!ST.isXNACKEnabled())
559     return 0;
560 
561   bool IsSMRD = TII.isSMRD(*MEM);
562 
563   resetClause();
564 
565   // A soft-clause is any group of consecutive SMEM instructions.  The
566   // instructions in this group may return out of order and/or may be
567   // replayed (i.e. the same instruction issued more than once).
568   //
569   // In order to handle these situations correctly we need to make sure that
570   // when a clause has more than one instruction, no instruction in the clause
571   // writes to a register that is read by another instruction in the clause
572   // (including itself). If we encounter this situation, we need to break the
573   // clause by inserting a non SMEM instruction.
574 
575   for (MachineInstr *MI : EmittedInstrs) {
576     // When we hit a non-SMEM instruction then we have passed the start of the
577     // clause and we can stop.
578     if (!MI)
579       break;
580 
581     if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI))
582       break;
583 
584     addClauseInst(*MI);
585   }
586 
587   if (ClauseDefs.none())
588     return 0;
589 
590   // We need to make sure not to put loads and stores in the same clause if they
591   // use the same address. For now, just start a new clause whenever we see a
592   // store.
593   if (MEM->mayStore())
594     return 1;
595 
596   addClauseInst(*MEM);
597 
598   // If the set of defs and uses intersect then we cannot add this instruction
599   // to the clause, so we have a hazard.
600   return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
601 }
602 
603 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
604   int WaitStatesNeeded = 0;
605 
606   WaitStatesNeeded = checkSoftClauseHazards(SMRD);
607 
608   // This SMRD hazard only affects SI.
609   if (!ST.hasSMRDReadVALUDefHazard())
610     return WaitStatesNeeded;
611 
612   // A read of an SGPR by SMRD instruction requires 4 wait states when the
613   // SGPR was written by a VALU instruction.
614   int SmrdSgprWaitStates = 4;
615   auto IsHazardDefFn = [this](const MachineInstr &MI) {
616     return TII.isVALU(MI);
617   };
618   auto IsBufferHazardDefFn = [this](const MachineInstr &MI) {
619     return TII.isSALU(MI);
620   };
621 
622   bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
623 
624   for (const MachineOperand &Use : SMRD->uses()) {
625     if (!Use.isReg())
626       continue;
627     int WaitStatesNeededForUse =
628         SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
629                                                    SmrdSgprWaitStates);
630     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
631 
632     // This fixes what appears to be undocumented hardware behavior in SI where
633     // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
634     // needs some number of nops in between. We don't know how many we need, but
635     // let's use 4. This wasn't discovered before probably because the only
636     // case when this happens is when we expand a 64-bit pointer into a full
637     // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
638     // probably never encountered in the closed-source land.
639     if (IsBufferSMRD) {
640       int WaitStatesNeededForUse =
641         SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
642                                                    IsBufferHazardDefFn,
643                                                    SmrdSgprWaitStates);
644       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
645     }
646   }
647 
648   return WaitStatesNeeded;
649 }
650 
651 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
652   if (!ST.hasVMEMReadSGPRVALUDefHazard())
653     return 0;
654 
655   int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
656 
657   // A read of an SGPR by a VMEM instruction requires 5 wait states when the
658   // SGPR was written by a VALU Instruction.
659   const int VmemSgprWaitStates = 5;
660   auto IsHazardDefFn = [this](const MachineInstr &MI) {
661     return TII.isVALU(MI);
662   };
663   for (const MachineOperand &Use : VMEM->uses()) {
664     if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg()))
665       continue;
666 
667     int WaitStatesNeededForUse =
668         VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
669                                                    VmemSgprWaitStates);
670     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
671   }
672   return WaitStatesNeeded;
673 }
674 
675 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
676   const SIRegisterInfo *TRI = ST.getRegisterInfo();
677   const SIInstrInfo *TII = ST.getInstrInfo();
678 
679   // Check for DPP VGPR read after VALU VGPR write and EXEC write.
680   int DppVgprWaitStates = 2;
681   int DppExecWaitStates = 5;
682   int WaitStatesNeeded = 0;
683   auto IsHazardDefFn = [TII](const MachineInstr &MI) {
684     return TII->isVALU(MI);
685   };
686 
687   for (const MachineOperand &Use : DPP->uses()) {
688     if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
689       continue;
690     int WaitStatesNeededForUse =
691         DppVgprWaitStates - getWaitStatesSinceDef(
692                                 Use.getReg(),
693                                 [](const MachineInstr &) { return true; },
694                                 DppVgprWaitStates);
695     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
696   }
697 
698   WaitStatesNeeded = std::max(
699       WaitStatesNeeded,
700       DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
701                                                 DppExecWaitStates));
702 
703   return WaitStatesNeeded;
704 }
705 
706 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
707   const SIInstrInfo *TII = ST.getInstrInfo();
708 
709   // v_div_fmas requires 4 wait states after a write to vcc from a VALU
710   // instruction.
711   const int DivFMasWaitStates = 4;
712   auto IsHazardDefFn = [TII](const MachineInstr &MI) {
713     return TII->isVALU(MI);
714   };
715   int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
716                                                DivFMasWaitStates);
717 
718   return DivFMasWaitStates - WaitStatesNeeded;
719 }
720 
721 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
722   const SIInstrInfo *TII = ST.getInstrInfo();
723   unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
724 
725   const int GetRegWaitStates = 2;
726   auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) {
727     return GetRegHWReg == getHWReg(TII, MI);
728   };
729   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates);
730 
731   return GetRegWaitStates - WaitStatesNeeded;
732 }
733 
734 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
735   const SIInstrInfo *TII = ST.getInstrInfo();
736   unsigned HWReg = getHWReg(TII, *SetRegInstr);
737 
738   const int SetRegWaitStates = ST.getSetRegWaitStates();
739   auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) {
740     return HWReg == getHWReg(TII, MI);
741   };
742   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates);
743   return SetRegWaitStates - WaitStatesNeeded;
744 }
745 
746 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
747   if (!MI.mayStore())
748     return -1;
749 
750   const SIInstrInfo *TII = ST.getInstrInfo();
751   unsigned Opcode = MI.getOpcode();
752   const MCInstrDesc &Desc = MI.getDesc();
753 
754   int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
755   int VDataRCID = -1;
756   if (VDataIdx != -1)
757     VDataRCID = Desc.OpInfo[VDataIdx].RegClass;
758 
759   if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
760     // There is no hazard if the instruction does not use vector regs
761     // (like wbinvl1)
762     if (VDataIdx == -1)
763       return -1;
764     // For MUBUF/MTBUF instructions this hazard only exists if the
765     // instruction is not using a register in the soffset field.
766     const MachineOperand *SOffset =
767         TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
768     // If we have no soffset operand, then assume this field has been
769     // hardcoded to zero.
770     if (AMDGPU::getRegBitWidth(VDataRCID) > 64 &&
771         (!SOffset || !SOffset->isReg()))
772       return VDataIdx;
773   }
774 
775   // MIMG instructions create a hazard if they don't use a 256-bit T# and
776   // the store size is greater than 8 bytes and they have more than two bits
777   // of their dmask set.
778   // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
779   if (TII->isMIMG(MI)) {
780     int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
781     assert(SRsrcIdx != -1 &&
782            AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256);
783     (void)SRsrcIdx;
784   }
785 
786   if (TII->isFLAT(MI)) {
787     int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
788     if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64)
789       return DataIdx;
790   }
791 
792   return -1;
793 }
794 
795 int
796 GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
797                                             const MachineRegisterInfo &MRI) {
798   // Helper to check for the hazard where VMEM instructions that store more than
799   // 8 bytes can have there store data over written by the next instruction.
800   const SIRegisterInfo *TRI = ST.getRegisterInfo();
801 
802   const int VALUWaitStates = ST.hasGFX940Insts() ? 2 : 1;
803   int WaitStatesNeeded = 0;
804 
805   if (!TRI->isVectorRegister(MRI, Def.getReg()))
806     return WaitStatesNeeded;
807   Register Reg = Def.getReg();
808   auto IsHazardFn = [this, Reg, TRI](const MachineInstr &MI) {
809     int DataIdx = createsVALUHazard(MI);
810     return DataIdx >= 0 &&
811            TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg);
812   };
813   int WaitStatesNeededForDef =
814     VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates);
815   WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
816 
817   return WaitStatesNeeded;
818 }
819 
820 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
821   int WaitStatesNeeded = 0;
822 
823   if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(*VALU)) {
824     const int TransDefWaitstates = 1;
825 
826     auto IsTransDefFn = [this, VALU](const MachineInstr &MI) {
827       if (!SIInstrInfo::isTRANS(MI))
828         return false;
829       const SIRegisterInfo *TRI = ST.getRegisterInfo();
830       const SIInstrInfo *TII = ST.getInstrInfo();
831       Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg();
832 
833       for (const MachineOperand &Use : VALU->explicit_uses()) {
834         if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg()))
835           return true;
836       }
837 
838       return false;
839     };
840 
841     int WaitStatesNeededForDef =
842         TransDefWaitstates -
843         getWaitStatesSince(IsTransDefFn, TransDefWaitstates);
844     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
845   }
846 
847   if (ST.hasDstSelForwardingHazard()) {
848     const int Shift16DefWaitstates = 1;
849 
850     auto IsShift16BitDefFn = [this, VALU](const MachineInstr &MI) {
851       if (!SIInstrInfo::isVALU(MI))
852         return false;
853       const SIInstrInfo *TII = ST.getInstrInfo();
854       if (SIInstrInfo::isSDWA(MI)) {
855         if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel))
856           if (DstSel->getImm() == AMDGPU::SDWA::DWORD)
857             return false;
858       } else {
859         if ((AMDGPU::getNamedOperandIdx(MI.getOpcode(),
860                                         AMDGPU::OpName::op_sel) == -1) ||
861             !(TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)
862                   ->getImm() &
863               SISrcMods::DST_OP_SEL))
864           return false;
865       }
866       const SIRegisterInfo *TRI = ST.getRegisterInfo();
867       if (auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) {
868         Register Def = Dst->getReg();
869 
870         for (const MachineOperand &Use : VALU->explicit_uses()) {
871           if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg()))
872             return true;
873         }
874       }
875 
876       return false;
877     };
878 
879     int WaitStatesNeededForDef =
880         Shift16DefWaitstates -
881         getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates);
882     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
883   }
884 
885   if (ST.hasVDecCoExecHazard()) {
886     const int VALUWriteSGPRVALUReadWaitstates = 2;
887     const int VALUWriteEXECRWLane = 4;
888     const int VALUWriteVGPRReadlaneRead = 1;
889 
890     const SIRegisterInfo *TRI = ST.getRegisterInfo();
891     const MachineRegisterInfo &MRI = MF.getRegInfo();
892     Register UseReg;
893     auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) {
894       if (!SIInstrInfo::isVALU(MI))
895         return false;
896       return MI.modifiesRegister(UseReg, TRI);
897     };
898 
899     for (const MachineOperand &Use : VALU->explicit_uses()) {
900       if (!Use.isReg())
901         continue;
902 
903       UseReg = Use.getReg();
904       if (TRI->isSGPRReg(MRI, UseReg)) {
905         int WaitStatesNeededForDef =
906             VALUWriteSGPRVALUReadWaitstates -
907             getWaitStatesSince(IsVALUDefSGPRFn,
908                                VALUWriteSGPRVALUReadWaitstates);
909         WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
910       }
911     }
912 
913     if (VALU->readsRegister(AMDGPU::VCC, TRI)) {
914       UseReg = AMDGPU::VCC;
915       int WaitStatesNeededForDef =
916           VALUWriteSGPRVALUReadWaitstates -
917           getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteSGPRVALUReadWaitstates);
918       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
919     }
920 
921     switch (VALU->getOpcode()) {
922     case AMDGPU::V_READLANE_B32:
923     case AMDGPU::V_READFIRSTLANE_B32: {
924       MachineOperand *Src = TII.getNamedOperand(*VALU, AMDGPU::OpName::src0);
925       UseReg = Src->getReg();
926       int WaitStatesNeededForDef =
927           VALUWriteVGPRReadlaneRead -
928           getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteVGPRReadlaneRead);
929       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
930     }
931       LLVM_FALLTHROUGH;
932     case AMDGPU::V_WRITELANE_B32: {
933       UseReg = AMDGPU::EXEC;
934       int WaitStatesNeededForDef =
935           VALUWriteEXECRWLane -
936           getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteEXECRWLane);
937       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
938       break;
939     }
940     default:
941       break;
942     }
943   }
944 
945   // This checks for the hazard where VMEM instructions that store more than
946   // 8 bytes can have there store data over written by the next instruction.
947   if (!ST.has12DWordStoreHazard())
948     return WaitStatesNeeded;
949 
950   const MachineRegisterInfo &MRI = MF.getRegInfo();
951 
952   for (const MachineOperand &Def : VALU->defs()) {
953     WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
954   }
955 
956   return WaitStatesNeeded;
957 }
958 
959 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) {
960   // This checks for hazards associated with inline asm statements.
961   // Since inline asms can contain just about anything, we use this
962   // to call/leverage other check*Hazard routines. Note that
963   // this function doesn't attempt to address all possible inline asm
964   // hazards (good luck), but is a collection of what has been
965   // problematic thus far.
966 
967   // see checkVALUHazards()
968   if (!ST.has12DWordStoreHazard())
969     return 0;
970 
971   const MachineRegisterInfo &MRI = MF.getRegInfo();
972   int WaitStatesNeeded = 0;
973 
974   for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands();
975        I != E; ++I) {
976     const MachineOperand &Op = IA->getOperand(I);
977     if (Op.isReg() && Op.isDef()) {
978       WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
979     }
980   }
981 
982   return WaitStatesNeeded;
983 }
984 
985 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
986   const SIInstrInfo *TII = ST.getInstrInfo();
987   const SIRegisterInfo *TRI = ST.getRegisterInfo();
988   const MachineRegisterInfo &MRI = MF.getRegInfo();
989 
990   const MachineOperand *LaneSelectOp =
991       TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
992 
993   if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
994     return 0;
995 
996   Register LaneSelectReg = LaneSelectOp->getReg();
997   auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); };
998 
999   const int RWLaneWaitStates = 4;
1000   int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
1001                                               RWLaneWaitStates);
1002   return RWLaneWaitStates - WaitStatesSince;
1003 }
1004 
1005 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
1006   if (!ST.hasRFEHazards())
1007     return 0;
1008 
1009   const SIInstrInfo *TII = ST.getInstrInfo();
1010 
1011   const int RFEWaitStates = 1;
1012 
1013   auto IsHazardFn = [TII](const MachineInstr &MI) {
1014     return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS;
1015   };
1016   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates);
1017   return RFEWaitStates - WaitStatesNeeded;
1018 }
1019 
1020 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
1021   const SIInstrInfo *TII = ST.getInstrInfo();
1022   const int ReadM0WaitStates = 1;
1023   auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); };
1024   return ReadM0WaitStates -
1025          getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, ReadM0WaitStates);
1026 }
1027 
1028 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
1029   fixVMEMtoScalarWriteHazards(MI);
1030   fixVcmpxPermlaneHazards(MI);
1031   fixSMEMtoVectorWriteHazards(MI);
1032   fixVcmpxExecWARHazard(MI);
1033   fixLdsBranchVmemWARHazard(MI);
1034 }
1035 
1036 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
1037   if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI))
1038     return false;
1039 
1040   const SIInstrInfo *TII = ST.getInstrInfo();
1041   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1042   auto IsHazardFn = [TII, TRI](const MachineInstr &MI) {
1043     return (TII->isVOPC(MI) ||
1044             ((TII->isVOP3(MI) || TII->isSDWA(MI)) && MI.isCompare())) &&
1045            MI.modifiesRegister(AMDGPU::EXEC, TRI);
1046   };
1047 
1048   auto IsExpiredFn = [](const MachineInstr &MI, int) {
1049     unsigned Opc = MI.getOpcode();
1050     return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 &&
1051            Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa;
1052   };
1053 
1054   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1055       std::numeric_limits<int>::max())
1056     return false;
1057 
1058   // V_NOP will be discarded by SQ.
1059   // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
1060   // which is always a VGPR and available.
1061   auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0);
1062   Register Reg = Src0->getReg();
1063   bool IsUndef = Src0->isUndef();
1064   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1065           TII->get(AMDGPU::V_MOV_B32_e32))
1066     .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0))
1067     .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill);
1068 
1069   return true;
1070 }
1071 
1072 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
1073   if (!ST.hasVMEMtoScalarWriteHazard())
1074     return false;
1075 
1076   if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI))
1077     return false;
1078 
1079   if (MI->getNumDefs() == 0)
1080     return false;
1081 
1082   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1083 
1084   auto IsHazardFn = [TRI, MI](const MachineInstr &I) {
1085     if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I) &&
1086         !SIInstrInfo::isFLAT(I))
1087       return false;
1088 
1089     for (const MachineOperand &Def : MI->defs()) {
1090       const MachineOperand *Op =
1091           I.findRegisterUseOperand(Def.getReg(), false, TRI);
1092       if (!Op)
1093         continue;
1094       return true;
1095     }
1096     return false;
1097   };
1098 
1099   auto IsExpiredFn = [](const MachineInstr &MI, int) {
1100     return SIInstrInfo::isVALU(MI) ||
1101            (MI.getOpcode() == AMDGPU::S_WAITCNT &&
1102             !MI.getOperand(0).getImm()) ||
1103            (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1104             MI.getOperand(0).getImm() == 0xffe3);
1105   };
1106 
1107   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1108       std::numeric_limits<int>::max())
1109     return false;
1110 
1111   const SIInstrInfo *TII = ST.getInstrInfo();
1112   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1113           TII->get(AMDGPU::S_WAITCNT_DEPCTR))
1114       .addImm(0xffe3);
1115   return true;
1116 }
1117 
1118 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
1119   if (!ST.hasSMEMtoVectorWriteHazard())
1120     return false;
1121 
1122   if (!SIInstrInfo::isVALU(*MI))
1123     return false;
1124 
1125   unsigned SDSTName;
1126   switch (MI->getOpcode()) {
1127   case AMDGPU::V_READLANE_B32:
1128   case AMDGPU::V_READFIRSTLANE_B32:
1129     SDSTName = AMDGPU::OpName::vdst;
1130     break;
1131   default:
1132     SDSTName = AMDGPU::OpName::sdst;
1133     break;
1134   }
1135 
1136   const SIInstrInfo *TII = ST.getInstrInfo();
1137   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1138   const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU());
1139   const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName);
1140   if (!SDST) {
1141     for (const auto &MO : MI->implicit_operands()) {
1142       if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) {
1143         SDST = &MO;
1144         break;
1145       }
1146     }
1147   }
1148 
1149   if (!SDST)
1150     return false;
1151 
1152   const Register SDSTReg = SDST->getReg();
1153   auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) {
1154     return SIInstrInfo::isSMRD(I) && I.readsRegister(SDSTReg, TRI);
1155   };
1156 
1157   auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) {
1158     if (TII->isSALU(MI)) {
1159       switch (MI.getOpcode()) {
1160       case AMDGPU::S_SETVSKIP:
1161       case AMDGPU::S_VERSION:
1162       case AMDGPU::S_WAITCNT_VSCNT:
1163       case AMDGPU::S_WAITCNT_VMCNT:
1164       case AMDGPU::S_WAITCNT_EXPCNT:
1165         // These instructions cannot not mitigate the hazard.
1166         return false;
1167       case AMDGPU::S_WAITCNT_LGKMCNT:
1168         // Reducing lgkmcnt count to 0 always mitigates the hazard.
1169         return (MI.getOperand(1).getImm() == 0) &&
1170                (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1171       case AMDGPU::S_WAITCNT: {
1172         const int64_t Imm = MI.getOperand(0).getImm();
1173         AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm);
1174         return (Decoded.LgkmCnt == 0);
1175       }
1176       default:
1177         // SOPP instructions cannot mitigate the hazard.
1178         if (TII->isSOPP(MI))
1179           return false;
1180         // At this point the SALU can be assumed to mitigate the hazard
1181         // because either:
1182         // (a) it is independent of the at risk SMEM (breaking chain),
1183         // or
1184         // (b) it is dependent on the SMEM, in which case an appropriate
1185         //     s_waitcnt lgkmcnt _must_ exist between it and the at risk
1186         //     SMEM instruction.
1187         return true;
1188       }
1189     }
1190     return false;
1191   };
1192 
1193   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1194       std::numeric_limits<int>::max())
1195     return false;
1196 
1197   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1198           TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL)
1199       .addImm(0);
1200   return true;
1201 }
1202 
1203 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
1204   if (!ST.hasVcmpxExecWARHazard() || !SIInstrInfo::isVALU(*MI))
1205     return false;
1206 
1207   const SIRegisterInfo *TRI = ST.getRegisterInfo();
1208   if (!MI->modifiesRegister(AMDGPU::EXEC, TRI))
1209     return false;
1210 
1211   auto IsHazardFn = [TRI](const MachineInstr &I) {
1212     if (SIInstrInfo::isVALU(I))
1213       return false;
1214     return I.readsRegister(AMDGPU::EXEC, TRI);
1215   };
1216 
1217   const SIInstrInfo *TII = ST.getInstrInfo();
1218   auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) {
1219     if (SIInstrInfo::isVALU(MI)) {
1220       if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst))
1221         return true;
1222       for (auto MO : MI.implicit_operands())
1223         if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg())))
1224           return true;
1225     }
1226     if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1227         (MI.getOperand(0).getImm() & 0xfffe) == 0xfffe)
1228       return true;
1229     return false;
1230   };
1231 
1232   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1233       std::numeric_limits<int>::max())
1234     return false;
1235 
1236   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1237           TII->get(AMDGPU::S_WAITCNT_DEPCTR))
1238     .addImm(0xfffe);
1239   return true;
1240 }
1241 
1242 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
1243                                                  const GCNSubtarget &ST) {
1244   if (!ST.hasLdsBranchVmemWARHazard())
1245     return false;
1246 
1247   // Check if the necessary condition for the hazard is met: both LDS and VMEM
1248   // instructions need to appear in the same function.
1249   bool HasLds = false;
1250   bool HasVmem = false;
1251   for (auto &MBB : MF) {
1252     for (auto &MI : MBB) {
1253       HasLds |= SIInstrInfo::isDS(MI);
1254       HasVmem |=
1255           SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI);
1256       if (HasLds && HasVmem)
1257         return true;
1258     }
1259   }
1260   return false;
1261 }
1262 
1263 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
1264   if (!RunLdsBranchVmemWARHazardFixup)
1265     return false;
1266 
1267   assert(ST.hasLdsBranchVmemWARHazard());
1268 
1269   auto IsHazardInst = [](const MachineInstr &MI) {
1270     if (SIInstrInfo::isDS(MI))
1271       return 1;
1272     if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI))
1273       return 2;
1274     return 0;
1275   };
1276 
1277   auto InstType = IsHazardInst(*MI);
1278   if (!InstType)
1279     return false;
1280 
1281   auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) {
1282     return IsHazardInst(I) || (I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1283                                I.getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1284                                !I.getOperand(1).getImm());
1285   };
1286 
1287   auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) {
1288     if (!I.isBranch())
1289       return false;
1290 
1291     auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) {
1292       auto InstType2 = IsHazardInst(I);
1293       return InstType2 && InstType != InstType2;
1294     };
1295 
1296     auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) {
1297       auto InstType2 = IsHazardInst(I);
1298       if (InstType == InstType2)
1299         return true;
1300 
1301       return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1302              I.getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1303              !I.getOperand(1).getImm();
1304     };
1305 
1306     return ::getWaitStatesSince(IsHazardFn, &I, IsExpiredFn) !=
1307            std::numeric_limits<int>::max();
1308   };
1309 
1310   if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1311       std::numeric_limits<int>::max())
1312     return false;
1313 
1314   const SIInstrInfo *TII = ST.getInstrInfo();
1315   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1316           TII->get(AMDGPU::S_WAITCNT_VSCNT))
1317     .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1318     .addImm(0);
1319 
1320   return true;
1321 }
1322 
1323 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) {
1324   int NSAtoVMEMWaitStates = 1;
1325 
1326   if (!ST.hasNSAtoVMEMBug())
1327     return 0;
1328 
1329   if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI))
1330     return 0;
1331 
1332   const SIInstrInfo *TII = ST.getInstrInfo();
1333   const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset);
1334   if (!Offset || (Offset->getImm() & 6) == 0)
1335     return 0;
1336 
1337   auto IsHazardFn = [TII](const MachineInstr &I) {
1338     if (!SIInstrInfo::isMIMG(I))
1339       return false;
1340     const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode());
1341     return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
1342            TII->getInstSizeInBytes(I) >= 16;
1343   };
1344 
1345   return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1);
1346 }
1347 
1348 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) {
1349   int FPAtomicToDenormModeWaitStates = 3;
1350 
1351   if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
1352     return 0;
1353 
1354   auto IsHazardFn = [](const MachineInstr &I) {
1355     if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I))
1356       return false;
1357     return SIInstrInfo::isFPAtomic(I);
1358   };
1359 
1360   auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) {
1361     if (WaitStates >= 3 || SIInstrInfo::isVALU(MI))
1362       return true;
1363 
1364     switch (MI.getOpcode()) {
1365     case AMDGPU::S_WAITCNT:
1366     case AMDGPU::S_WAITCNT_VSCNT:
1367     case AMDGPU::S_WAITCNT_VMCNT:
1368     case AMDGPU::S_WAITCNT_EXPCNT:
1369     case AMDGPU::S_WAITCNT_LGKMCNT:
1370     case AMDGPU::S_WAIT_IDLE:
1371       return true;
1372     default:
1373       break;
1374     }
1375 
1376     return false;
1377   };
1378 
1379   return FPAtomicToDenormModeWaitStates -
1380          ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn);
1381 }
1382 
1383 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) {
1384   assert(SIInstrInfo::isMAI(*MI));
1385 
1386   return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
1387 }
1388 
1389 int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) {
1390   // Early exit if no padding is requested.
1391   if (MFMAPaddingRatio == 0)
1392     return 0;
1393 
1394   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1395   if (!SIInstrInfo::isMFMA(*MI) || MFI->getOccupancy() < 2)
1396     return 0;
1397 
1398   int NeighborMFMALatency = 0;
1399   auto IsNeighboringMFMA = [&NeighborMFMALatency,
1400                             this](const MachineInstr &MI) {
1401     if (!SIInstrInfo::isMFMA(MI))
1402       return false;
1403 
1404     NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI);
1405     return true;
1406   };
1407 
1408   const int MaxMFMAPipelineWaitStates = 16;
1409   int WaitStatesSinceNeighborMFMA =
1410       getWaitStatesSince(IsNeighboringMFMA, MaxMFMAPipelineWaitStates);
1411 
1412   int NeighborMFMAPaddingNeeded =
1413       (NeighborMFMALatency * MFMAPaddingRatio / 100) -
1414       WaitStatesSinceNeighborMFMA;
1415 
1416   return std::max(0, NeighborMFMAPaddingNeeded);
1417 }
1418 
1419 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) {
1420   int WaitStatesNeeded = 0;
1421   unsigned Opc = MI->getOpcode();
1422 
1423   auto IsVALUFn = [](const MachineInstr &MI) {
1424     return SIInstrInfo::isVALU(MI);
1425   };
1426 
1427   if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
1428     const int LegacyVALUWritesVGPRWaitStates = 2;
1429     const int VALUWritesExecWaitStates = 4;
1430     const int MaxWaitStates = 4;
1431 
1432     int WaitStatesNeededForUse = VALUWritesExecWaitStates -
1433       getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates);
1434     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1435 
1436     if (WaitStatesNeeded < MaxWaitStates) {
1437       for (const MachineOperand &Use : MI->explicit_uses()) {
1438         const int MaxWaitStates = 2;
1439 
1440         if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
1441           continue;
1442 
1443         int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
1444           getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates);
1445         WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1446 
1447         if (WaitStatesNeeded == MaxWaitStates)
1448           break;
1449       }
1450     }
1451   }
1452 
1453   for (const MachineOperand &Op : MI->explicit_operands()) {
1454     if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg()))
1455       continue;
1456 
1457     if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1458       continue;
1459 
1460     const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
1461     const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
1462     const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
1463     const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
1464     const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
1465     const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
1466     const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
1467     const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
1468     const int MaxWaitStates = 18;
1469     Register Reg = Op.getReg();
1470     unsigned HazardDefLatency = 0;
1471 
1472     auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency,
1473                                this](const MachineInstr &MI) {
1474       if (!SIInstrInfo::isMFMA(MI))
1475         return false;
1476       Register DstReg = MI.getOperand(0).getReg();
1477       if (DstReg == Reg)
1478         return false;
1479       HazardDefLatency =
1480           std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
1481       return TRI.regsOverlap(DstReg, Reg);
1482     };
1483 
1484     int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn,
1485                                                    MaxWaitStates);
1486     int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
1487     int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
1488     int OpNo = MI->getOperandNo(&Op);
1489     if (OpNo == SrcCIdx) {
1490       NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
1491     } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
1492       switch (HazardDefLatency) {
1493       case 2:  NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
1494                break;
1495       case 8:  NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
1496                break;
1497       case 16: LLVM_FALLTHROUGH;
1498       default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
1499                break;
1500       }
1501     } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
1502       switch (HazardDefLatency) {
1503       case 2:  NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
1504                break;
1505       case 8:  NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
1506                break;
1507       case 16: LLVM_FALLTHROUGH;
1508       default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
1509                break;
1510       }
1511     }
1512 
1513     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1514     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1515 
1516     if (WaitStatesNeeded == MaxWaitStates)
1517       return WaitStatesNeeded; // Early exit.
1518 
1519     auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) {
1520       if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1521         return false;
1522       Register DstReg = MI.getOperand(0).getReg();
1523       return TRI.regsOverlap(Reg, DstReg);
1524     };
1525 
1526     const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
1527     const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
1528     const int AccVGPRWriteAccVgprReadWaitStates = 3;
1529     NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
1530     if (OpNo == SrcCIdx)
1531       NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
1532     else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
1533       NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
1534 
1535     WaitStatesNeededForUse = NeedWaitStates -
1536       getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates);
1537     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1538 
1539     if (WaitStatesNeeded == MaxWaitStates)
1540       return WaitStatesNeeded; // Early exit.
1541   }
1542 
1543   if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
1544     const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
1545     const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
1546     const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
1547     const int MaxWaitStates = 13;
1548     Register DstReg = MI->getOperand(0).getReg();
1549     unsigned HazardDefLatency = 0;
1550 
1551     auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency,
1552                          this](const MachineInstr &MI) {
1553       if (!SIInstrInfo::isMFMA(MI))
1554         return false;
1555       Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg();
1556       HazardDefLatency =
1557           std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
1558       return TRI.regsOverlap(Reg, DstReg);
1559     };
1560 
1561     int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates);
1562     int NeedWaitStates;
1563     switch (HazardDefLatency) {
1564     case 2:  NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
1565              break;
1566     case 8:  NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
1567              break;
1568     case 16: LLVM_FALLTHROUGH;
1569     default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
1570              break;
1571     }
1572 
1573     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
1574     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1575   }
1576 
1577   // Pad neighboring MFMA with noops for better inter-wave performance.
1578   WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI));
1579 
1580   return WaitStatesNeeded;
1581 }
1582 
1583 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
1584   int WaitStatesNeeded = 0;
1585   unsigned Opc = MI->getOpcode();
1586 
1587   auto IsLegacyVALUFn = [](const MachineInstr &MI) {
1588     return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI);
1589   };
1590 
1591   auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) {
1592     return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI) &&
1593            !SIInstrInfo::isDOT(MI);
1594   };
1595 
1596   if (!SIInstrInfo::isMFMA(*MI))
1597     return WaitStatesNeeded;
1598 
1599   const int VALUWritesExecWaitStates = 4;
1600   int WaitStatesNeededForUse = VALUWritesExecWaitStates -
1601     getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn,
1602                           VALUWritesExecWaitStates);
1603   WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1604 
1605   int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
1606 
1607   // Loop for both DGEMM and S/HGEMM 2nd instruction.
1608   for (const MachineOperand &Use : MI->explicit_uses()) {
1609     const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
1610     const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
1611     const int GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates = 3;
1612     const int GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates = 5;
1613     const int GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates = 4;
1614     const int GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates = 9;
1615     const int GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates = 8;
1616     const int GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates = 17;
1617     const int GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates = 16;
1618     const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
1619     const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
1620     const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
1621     const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
1622     const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
1623     const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
1624     const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
1625     const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
1626     const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
1627     const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
1628     const int GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates = 4;
1629     const int GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates = 6;
1630     const int GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates = 10;
1631     const int GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates = 18;
1632     const int GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates = 5;
1633     const int GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates = 7;
1634     const int GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates = 11;
1635     const int GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates = 19;
1636     const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
1637     const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
1638     const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
1639     const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2;
1640     const int MaxWaitStates = 19;
1641 
1642     if (!Use.isReg())
1643       continue;
1644     Register Reg = Use.getReg();
1645     bool FullReg;
1646     const MachineInstr *MI1;
1647 
1648     auto IsOverlappedMFMAFn = [Reg, &FullReg, &MI1,
1649                                this](const MachineInstr &MI) {
1650       if (!SIInstrInfo::isMFMA(MI))
1651         return false;
1652       Register DstReg = MI.getOperand(0).getReg();
1653       FullReg = (DstReg == Reg);
1654       MI1 = &MI;
1655       return TRI.regsOverlap(DstReg, Reg);
1656     };
1657 
1658     WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
1659       getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates);
1660     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1661 
1662     int NumWaitStates =
1663         getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates);
1664     if (NumWaitStates == std::numeric_limits<int>::max())
1665       continue;
1666 
1667     int OpNo = MI->getOperandNo(&Use);
1668     unsigned Opc1 = MI1->getOpcode();
1669     int NeedWaitStates = 0;
1670     if (OpNo == SrcCIdx) {
1671       if (!isDGEMM(Opc) && (!ST.hasGFX940Insts() && isDGEMM(Opc1))) {
1672         NeedWaitStates = 0;
1673       } else if (FullReg) {
1674         if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
1675              Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
1676             (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
1677              Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
1678           NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
1679         else if (ST.hasGFX940Insts() &&
1680                  TSchedModel.computeInstrLatency(MI1) == 2)
1681           NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
1682       } else {
1683         switch (Opc1) {
1684         case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
1685         case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
1686         case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
1687         case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
1688           if (!isXDL(ST, *MI))
1689             NeedWaitStates = DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
1690           break;
1691         case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
1692         case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
1693           if (!isXDL(ST, *MI))
1694             NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
1695           break;
1696         default:
1697           if (ST.hasGFX940Insts() && isXDL(ST, *MI) && !isXDL(ST, *MI1))
1698             break;
1699           switch (TSchedModel.computeInstrLatency(MI1)) {
1700           case 2:
1701             NeedWaitStates = ST.hasGFX940Insts()
1702               ? isXDL(ST, *MI1)
1703                 ? GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates
1704                 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates
1705               : isDGEMM(Opc)
1706                 ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
1707                 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
1708             break;
1709           case 4:
1710             assert(ST.hasGFX940Insts());
1711             NeedWaitStates = isXDL(ST, *MI1)
1712               ? GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates
1713               : GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates;
1714             break;
1715           case 8:
1716             NeedWaitStates = ST.hasGFX940Insts()
1717               ? isXDL(ST, *MI1)
1718                 ? GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates
1719                 : GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates
1720               : isDGEMM(Opc)
1721                 ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
1722                 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
1723             break;
1724           case 16: LLVM_FALLTHROUGH;
1725           default:
1726             NeedWaitStates = ST.hasGFX940Insts()
1727               ? isXDL(ST, *MI1)
1728                 ? GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates
1729                 : GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates
1730               : isDGEMM(Opc)
1731                 ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
1732                 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
1733           }
1734         }
1735       }
1736     } else {
1737       switch (Opc1) {
1738       case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
1739       case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
1740       case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
1741       case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
1742         NeedWaitStates = DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
1743         break;
1744       case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
1745       case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
1746         NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
1747         break;
1748       default:
1749         switch (TSchedModel.computeInstrLatency(MI1)) {
1750         case 2:
1751           NeedWaitStates = ST.hasGFX940Insts()
1752             ? isXDL(ST, *MI1)
1753               ? GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates
1754               : GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates
1755             : SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
1756           break;
1757         case 4:
1758           assert(ST.hasGFX940Insts());
1759           NeedWaitStates = isXDL(ST, *MI1)
1760             ? GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates
1761             : GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates;
1762           break;
1763         case 8:
1764           NeedWaitStates = ST.hasGFX940Insts()
1765             ? isXDL(ST, *MI1)
1766               ? GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates
1767               : GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates
1768             : SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
1769           break;
1770         case 16: LLVM_FALLTHROUGH;
1771         default:
1772           NeedWaitStates = ST.hasGFX940Insts()
1773             ? isXDL(ST, *MI1)
1774               ? GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates
1775               : GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates
1776             : SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
1777         }
1778       }
1779     }
1780     if (WaitStatesNeeded >= NeedWaitStates)
1781       continue;
1782 
1783     WaitStatesNeededForUse = NeedWaitStates - NumWaitStates;
1784     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1785 
1786     if (WaitStatesNeeded == MaxWaitStates)
1787       break;
1788   }
1789 
1790   return WaitStatesNeeded;
1791 }
1792 
1793 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) {
1794   // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards()
1795   if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
1796     return 0;
1797 
1798   int WaitStatesNeeded = 0;
1799 
1800   auto IsAccVgprReadFn = [](const MachineInstr &MI) {
1801     return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
1802   };
1803 
1804   for (const MachineOperand &Op : MI->explicit_uses()) {
1805     if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg()))
1806       continue;
1807 
1808     Register Reg = Op.getReg();
1809 
1810     const int AccVgprReadLdStWaitStates = 2;
1811     const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
1812     const int MaxWaitStates = 2;
1813 
1814     int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
1815       getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates);
1816     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1817 
1818     if (WaitStatesNeeded == MaxWaitStates)
1819       return WaitStatesNeeded; // Early exit.
1820 
1821     auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) {
1822       if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
1823           MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1824         return false;
1825       auto IsVALUFn = [](const MachineInstr &MI) {
1826         return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMAI(MI);
1827       };
1828       return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) <
1829              std::numeric_limits<int>::max();
1830     };
1831 
1832     WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
1833       getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates);
1834     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1835   }
1836 
1837   return WaitStatesNeeded;
1838 }
1839 
1840 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
1841   if (!ST.hasGFX90AInsts())
1842     return 0;
1843 
1844   auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
1845     return isDGEMM(MI.getOpcode());
1846   };
1847 
1848   // This is checked in checkMAIHazards90A()
1849   if (SIInstrInfo::isMFMA(*MI))
1850     return 0;
1851 
1852   int WaitStatesNeeded = 0;
1853 
1854   bool IsMemOrExport = SIInstrInfo::isVMEM(*MI) ||
1855                        SIInstrInfo::isFLAT(*MI) ||
1856                        SIInstrInfo::isDS(*MI) ||
1857                        SIInstrInfo::isEXP(*MI);
1858   bool IsVALU = SIInstrInfo::isVALU(*MI);
1859 
1860   const MachineInstr *MFMA = nullptr;
1861   unsigned Reg;
1862   auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
1863     if (!SIInstrInfo::isMFMA(MI) ||
1864         !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
1865       return false;
1866     MFMA = &MI;
1867     return true;
1868   };
1869 
1870   const MachineInstr *DOT = nullptr;
1871   auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) {
1872     if (!SIInstrInfo::isDOT(MI) ||
1873         !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
1874       return false;
1875     DOT = &MI;
1876     return true;
1877   };
1878 
1879   int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(),
1880                                            AMDGPU::OpName::src2);
1881 
1882   if (IsMemOrExport || IsVALU) {
1883     const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
1884     const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
1885     const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
1886     const int GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates = 4;
1887     const int GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates = 6;
1888     const int GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates = 10;
1889     const int GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates = 18;
1890     const int GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates = 5;
1891     const int GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates = 7;
1892     const int GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates = 11;
1893     const int GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates = 19;
1894     const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
1895     const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
1896     const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
1897     const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
1898     const int DotWriteSameDotReadSrcAB = 3;
1899     const int DotWriteDifferentVALURead = 3;
1900     const int MaxWaitStates = 19;
1901 
1902     for (const MachineOperand &Use : MI->explicit_uses()) {
1903       if (!Use.isReg())
1904         continue;
1905       Reg = Use.getReg();
1906 
1907       DOT = nullptr;
1908       int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
1909                                                      MaxWaitStates);
1910       if (DOT) {
1911         int NeedWaitStates = 0;
1912         if (DOT->getOpcode() == MI->getOpcode()) {
1913           if (&Use - &MI->getOperand(0) != SrcCIdx)
1914             NeedWaitStates = DotWriteSameDotReadSrcAB;
1915         } else {
1916           NeedWaitStates = DotWriteDifferentVALURead;
1917         }
1918 
1919         int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1920         WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1921       }
1922 
1923       MFMA = nullptr;
1924       WaitStatesSinceDef =
1925           getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
1926       if (!MFMA)
1927         continue;
1928 
1929       unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
1930       int NeedWaitStates = MaxWaitStates;
1931       switch (HazardDefLatency) {
1932       case 2:
1933         NeedWaitStates =
1934           ST.hasGFX940Insts()
1935             ? isXDL(ST, *MFMA)
1936               ? GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates
1937               : GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates
1938             : SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
1939         break;
1940       case 4:
1941         assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts());
1942         NeedWaitStates =
1943           isDGEMM(MFMA->getOpcode())
1944             ? IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
1945                             : DMFMA4x4WriteVgprVALUReadWaitStates
1946             : isXDL(ST, *MFMA)
1947               ? GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates
1948               : GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates;
1949         break;
1950       case 8:
1951         NeedWaitStates =
1952           ST.hasGFX940Insts()
1953             ? isXDL(ST, *MFMA)
1954               ? GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates
1955               : GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates
1956             : SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
1957         break;
1958       case 16: LLVM_FALLTHROUGH;
1959       default:
1960         NeedWaitStates =
1961           isDGEMM(MFMA->getOpcode())
1962             ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates
1963                             : DMFMA16x16WriteVgprVALUReadWaitStates
1964             : ST.hasGFX940Insts()
1965               ? isXDL(ST, *MFMA)
1966                 ? GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates
1967                 : GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates
1968               : SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
1969         break;
1970       }
1971 
1972       int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1973       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1974 
1975       if (WaitStatesNeeded == MaxWaitStates)
1976         break;
1977     }
1978   }
1979 
1980   unsigned Opc = MI->getOpcode();
1981   const int DMFMAToFMA64WaitStates = 2;
1982   if ((Opc == AMDGPU::V_FMA_F64_e64 ||
1983        Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
1984        Opc == AMDGPU::V_FMAC_F64_dpp) &&
1985       WaitStatesNeeded < DMFMAToFMA64WaitStates) {
1986     int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
1987       getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates);
1988     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1989   }
1990 
1991   if (!IsVALU && !IsMemOrExport)
1992     return WaitStatesNeeded;
1993 
1994   for (const MachineOperand &Def : MI->defs()) {
1995     const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
1996     const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
1997     const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
1998     const int GFX940_SMFMA2PassWriteVgprVALUWawWaitStates = 4;
1999     const int GFX940_SMFMA4PassWriteVgprVALUWawWaitStates = 6;
2000     const int GFX940_SMFMA8PassWriteVgprVALUWawWaitStates = 10;
2001     const int GFX940_SMFMA16PassWriteVgprVALUWawWaitStates = 18;
2002     const int GFX940_XDL2PassWriteVgprVALUWawWaitStates = 5;
2003     const int GFX940_XDL4PassWriteVgprVALUWawWaitStates = 7;
2004     const int GFX940_XDL8PassWriteVgprVALUWawWaitStates = 11;
2005     const int GFX940_XDL16PassWriteVgprVALUWawWaitStates = 19;
2006     const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
2007     const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3;
2008     const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
2009     const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
2010     const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
2011     const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
2012     const int DotWriteDifferentVALUWrite = 3;
2013     const int MaxWaitStates = 19;
2014     const int MaxWarWaitStates = 15;
2015 
2016     Reg = Def.getReg();
2017 
2018     DOT = nullptr;
2019     int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
2020                                                    MaxWaitStates);
2021     if (DOT && DOT->getOpcode() != MI->getOpcode())
2022       WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite -
2023                                                     WaitStatesSinceDef);
2024 
2025     MFMA = nullptr;
2026     WaitStatesSinceDef =
2027         getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
2028     if (MFMA) {
2029       int NeedWaitStates = MaxWaitStates;
2030       switch (TSchedModel.computeInstrLatency(MFMA)) {
2031       case 2:
2032         NeedWaitStates = ST.hasGFX940Insts()
2033           ? isXDL(ST, *MFMA)
2034             ? GFX940_XDL2PassWriteVgprVALUWawWaitStates
2035             : GFX940_SMFMA2PassWriteVgprVALUWawWaitStates
2036           : SMFMA4x4WriteVgprVALUWawWaitStates;
2037         break;
2038       case 4:
2039         assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts());
2040         NeedWaitStates = isDGEMM(MFMA->getOpcode())
2041             ? DMFMA4x4WriteVgprVALUWriteWaitStates
2042             : isXDL(ST, *MFMA)
2043               ? GFX940_XDL4PassWriteVgprVALUWawWaitStates
2044               : GFX940_SMFMA4PassWriteVgprVALUWawWaitStates;
2045         break;
2046       case 8:
2047         NeedWaitStates = ST.hasGFX940Insts()
2048           ? isXDL(ST, *MFMA)
2049             ? GFX940_XDL8PassWriteVgprVALUWawWaitStates
2050             : GFX940_SMFMA8PassWriteVgprVALUWawWaitStates
2051           : SMFMA16x16WriteVgprVALUWawWaitStates;
2052         break;
2053       case 16: LLVM_FALLTHROUGH;
2054       default:
2055         NeedWaitStates = isDGEMM(MFMA->getOpcode())
2056                    ? DMFMA16x16WriteVgprVALUWriteWaitStates
2057                    : ST.hasGFX940Insts()
2058                      ? isXDL(ST, *MFMA)
2059                        ? GFX940_XDL16PassWriteVgprVALUWawWaitStates
2060                        : GFX940_SMFMA16PassWriteVgprVALUWawWaitStates
2061                    : SMFMA32x32WriteVgprVALUWawWaitStates;
2062         break;
2063       }
2064 
2065       int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
2066       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2067 
2068       if (WaitStatesNeeded == MaxWaitStates)
2069         break;
2070     }
2071 
2072     auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
2073       if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) ||
2074           !MI.readsRegister(Reg, &TRI))
2075         return false;
2076 
2077       if (ST.hasGFX940Insts() && !isXDL(ST, MI))
2078         return false;
2079 
2080       const MachineOperand *SrcC =
2081           TII.getNamedOperand(MI, AMDGPU::OpName::src2);
2082       assert(SrcC);
2083       if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg))
2084         return false;
2085 
2086       MFMA = &MI;
2087       return true;
2088     };
2089 
2090     MFMA = nullptr;
2091     int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn,
2092                                                 MaxWarWaitStates);
2093     if (!MFMA)
2094       continue;
2095 
2096     unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
2097     int NeedWaitStates = MaxWaitStates;
2098     switch (HazardDefLatency) {
2099     case 2:  NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
2100              break;
2101     case 4:  assert(ST.hasGFX940Insts());
2102              NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates;
2103              break;
2104     case 8:  NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates;
2105              break;
2106     case 16: LLVM_FALLTHROUGH;
2107     default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates;
2108              break;
2109     }
2110 
2111     int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse;
2112     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2113   }
2114 
2115   return WaitStatesNeeded;
2116 }
2117 
2118 bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) {
2119   if (!SU->isInstr())
2120     return false;
2121 
2122   const MachineInstr *MAI = nullptr;
2123 
2124   auto IsMFMAFn = [&MAI](const MachineInstr &MI) {
2125     MAI = nullptr;
2126     if (SIInstrInfo::isMFMA(MI))
2127       MAI = &MI;
2128     return MAI != nullptr;
2129   };
2130 
2131   MachineInstr *MI = SU->getInstr();
2132   if (IsMFMAFn(*MI)) {
2133     int W = getWaitStatesSince(IsMFMAFn, 16);
2134     if (MAI)
2135       return W < (int)TSchedModel.computeInstrLatency(MAI);
2136   }
2137 
2138   return false;
2139 }
2140