1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements hazard recognizers for scheduling on GCN processors. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "GCNHazardRecognizer.h" 14 #include "GCNSubtarget.h" 15 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 16 #include "SIMachineFunctionInfo.h" 17 #include "llvm/CodeGen/MachineFunction.h" 18 #include "llvm/CodeGen/ScheduleDAG.h" 19 #include "llvm/Support/TargetParser.h" 20 21 using namespace llvm; 22 23 namespace { 24 25 struct MFMAPaddingRatioParser : public cl::parser<unsigned> { 26 MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {} 27 28 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) { 29 if (Arg.getAsInteger(0, Value)) 30 return O.error("'" + Arg + "' value invalid for uint argument!"); 31 32 if (Value > 100) 33 return O.error("'" + Arg + "' value must be in the range [0, 100]!"); 34 35 return false; 36 } 37 }; 38 39 } // end anonymous namespace 40 41 static cl::opt<unsigned, false, MFMAPaddingRatioParser> 42 MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, 43 cl::desc("Fill a percentage of the latency between " 44 "neighboring MFMA with s_nops.")); 45 46 //===----------------------------------------------------------------------===// 47 // Hazard Recognizer Implementation 48 //===----------------------------------------------------------------------===// 49 50 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, 51 const GCNSubtarget &ST); 52 53 GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) : 54 IsHazardRecognizerMode(false), 55 CurrCycleInstr(nullptr), 56 MF(MF), 57 ST(MF.getSubtarget<GCNSubtarget>()), 58 TII(*ST.getInstrInfo()), 59 TRI(TII.getRegisterInfo()), 60 ClauseUses(TRI.getNumRegUnits()), 61 ClauseDefs(TRI.getNumRegUnits()) { 62 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5; 63 TSchedModel.init(&ST); 64 RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST); 65 } 66 67 void GCNHazardRecognizer::Reset() { 68 EmittedInstrs.clear(); 69 } 70 71 void GCNHazardRecognizer::EmitInstruction(SUnit *SU) { 72 EmitInstruction(SU->getInstr()); 73 } 74 75 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) { 76 CurrCycleInstr = MI; 77 } 78 79 static bool isDivFMas(unsigned Opcode) { 80 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64; 81 } 82 83 static bool isSGetReg(unsigned Opcode) { 84 return Opcode == AMDGPU::S_GETREG_B32; 85 } 86 87 static bool isSSetReg(unsigned Opcode) { 88 switch (Opcode) { 89 case AMDGPU::S_SETREG_B32: 90 case AMDGPU::S_SETREG_B32_mode: 91 case AMDGPU::S_SETREG_IMM32_B32: 92 case AMDGPU::S_SETREG_IMM32_B32_mode: 93 return true; 94 } 95 return false; 96 } 97 98 static bool isRWLane(unsigned Opcode) { 99 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32; 100 } 101 102 static bool isRFE(unsigned Opcode) { 103 return Opcode == AMDGPU::S_RFE_B64; 104 } 105 106 static bool isSMovRel(unsigned Opcode) { 107 switch (Opcode) { 108 case AMDGPU::S_MOVRELS_B32: 109 case AMDGPU::S_MOVRELS_B64: 110 case AMDGPU::S_MOVRELD_B32: 111 case AMDGPU::S_MOVRELD_B64: 112 return true; 113 default: 114 return false; 115 } 116 } 117 118 static bool isDGEMM(unsigned Opcode) { 119 return AMDGPU::getMAIIsDGEMM(Opcode); 120 } 121 122 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) { 123 unsigned Opcode = MI.getOpcode(); 124 125 if (!SIInstrInfo::isMAI(MI) || 126 isDGEMM(Opcode) || 127 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 || 128 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64) 129 return false; 130 131 if (!ST.hasGFX940Insts()) 132 return true; 133 134 return AMDGPU::getMAIIsGFX940XDL(Opcode); 135 } 136 137 static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, 138 const MachineInstr &MI) { 139 if (TII.isAlwaysGDS(MI.getOpcode())) 140 return true; 141 142 switch (MI.getOpcode()) { 143 case AMDGPU::S_SENDMSG: 144 case AMDGPU::S_SENDMSGHALT: 145 case AMDGPU::S_TTRACEDATA: 146 return true; 147 // These DS opcodes don't support GDS. 148 case AMDGPU::DS_NOP: 149 case AMDGPU::DS_PERMUTE_B32: 150 case AMDGPU::DS_BPERMUTE_B32: 151 return false; 152 default: 153 if (TII.isDS(MI.getOpcode())) { 154 int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 155 AMDGPU::OpName::gds); 156 if (MI.getOperand(GDS).getImm()) 157 return true; 158 } 159 return false; 160 } 161 } 162 163 static bool isPermlane(const MachineInstr &MI) { 164 unsigned Opcode = MI.getOpcode(); 165 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 || 166 Opcode == AMDGPU::V_PERMLANEX16_B32_e64; 167 } 168 169 static bool isLdsDma(const MachineInstr &MI) { 170 return SIInstrInfo::isVALU(MI) && 171 (SIInstrInfo::isMUBUF(MI) || SIInstrInfo::isFLAT(MI)); 172 } 173 174 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) { 175 const MachineOperand *RegOp = TII->getNamedOperand(RegInstr, 176 AMDGPU::OpName::simm16); 177 return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_; 178 } 179 180 ScheduleHazardRecognizer::HazardType 181 GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { 182 MachineInstr *MI = SU->getInstr(); 183 // If we are not in "HazardRecognizerMode" and therefore not being run from 184 // the scheduler, track possible stalls from hazards but don't insert noops. 185 auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard; 186 187 if (MI->isBundle()) 188 return NoHazard; 189 190 if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0) 191 return HazardType; 192 193 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0) 194 return HazardType; 195 196 if (checkFPAtomicToDenormModeHazard(MI) > 0) 197 return HazardType; 198 199 if (ST.hasNoDataDepHazard()) 200 return NoHazard; 201 202 // FIXME: Should flat be considered vmem? 203 if ((SIInstrInfo::isVMEM(*MI) || 204 SIInstrInfo::isFLAT(*MI)) 205 && checkVMEMHazards(MI) > 0) 206 return HazardType; 207 208 if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0) 209 return HazardType; 210 211 if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0) 212 return HazardType; 213 214 if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0) 215 return HazardType; 216 217 if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0) 218 return HazardType; 219 220 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || 221 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || 222 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) 223 return HazardType; 224 225 if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0) 226 return HazardType; 227 228 if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0) 229 return HazardType; 230 231 if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0) 232 return HazardType; 233 234 if (((ST.hasReadM0MovRelInterpHazard() && 235 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()))) || 236 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || 237 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || 238 (ST.hasReadM0LdsDirectHazard() && 239 MI->readsRegister(AMDGPU::LDS_DIRECT))) && 240 checkReadM0Hazards(MI) > 0) 241 return HazardType; 242 243 if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0) 244 return HazardType; 245 246 if ((SIInstrInfo::isVMEM(*MI) || 247 SIInstrInfo::isFLAT(*MI) || 248 SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0) 249 return HazardType; 250 251 if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0) 252 return HazardType; 253 254 return NoHazard; 255 } 256 257 static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, 258 unsigned Quantity) { 259 while (Quantity > 0) { 260 unsigned Arg = std::min(Quantity, 8u); 261 Quantity -= Arg; 262 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP)) 263 .addImm(Arg - 1); 264 } 265 } 266 267 unsigned 268 GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const { 269 const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&MI); 270 assert(TSchedModel.getWriteProcResBegin(SC) != 271 TSchedModel.getWriteProcResEnd(SC)); 272 return TSchedModel.getWriteProcResBegin(SC)->Cycles; 273 } 274 275 void GCNHazardRecognizer::processBundle() { 276 MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator()); 277 MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end(); 278 // Check bundled MachineInstr's for hazards. 279 for (; MI != E && MI->isInsideBundle(); ++MI) { 280 CurrCycleInstr = &*MI; 281 unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr); 282 283 if (IsHazardRecognizerMode) { 284 fixHazards(CurrCycleInstr); 285 286 insertNoopsInBundle(CurrCycleInstr, TII, WaitStates); 287 } 288 289 // It’s unnecessary to track more than MaxLookAhead instructions. Since we 290 // include the bundled MI directly after, only add a maximum of 291 // (MaxLookAhead - 1) noops to EmittedInstrs. 292 for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i) 293 EmittedInstrs.push_front(nullptr); 294 295 EmittedInstrs.push_front(CurrCycleInstr); 296 EmittedInstrs.resize(MaxLookAhead); 297 } 298 CurrCycleInstr = nullptr; 299 } 300 301 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) { 302 IsHazardRecognizerMode = true; 303 CurrCycleInstr = MI; 304 unsigned W = PreEmitNoopsCommon(MI); 305 fixHazards(MI); 306 CurrCycleInstr = nullptr; 307 return W; 308 } 309 310 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { 311 if (MI->isBundle()) 312 return 0; 313 314 int WaitStates = 0; 315 316 if (SIInstrInfo::isSMRD(*MI)) 317 return std::max(WaitStates, checkSMRDHazards(MI)); 318 319 if (ST.hasNSAtoVMEMBug()) 320 WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI)); 321 322 WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI)); 323 324 if (ST.hasNoDataDepHazard()) 325 return WaitStates; 326 327 if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI)) 328 WaitStates = std::max(WaitStates, checkVMEMHazards(MI)); 329 330 if (SIInstrInfo::isVALU(*MI)) 331 WaitStates = std::max(WaitStates, checkVALUHazards(MI)); 332 333 if (SIInstrInfo::isDPP(*MI)) 334 WaitStates = std::max(WaitStates, checkDPPHazards(MI)); 335 336 if (isDivFMas(MI->getOpcode())) 337 WaitStates = std::max(WaitStates, checkDivFMasHazards(MI)); 338 339 if (isRWLane(MI->getOpcode())) 340 WaitStates = std::max(WaitStates, checkRWLaneHazards(MI)); 341 342 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || 343 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || 344 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) 345 WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI)); 346 347 if (MI->isInlineAsm()) 348 return std::max(WaitStates, checkInlineAsmHazards(MI)); 349 350 if (isSGetReg(MI->getOpcode())) 351 return std::max(WaitStates, checkGetRegHazards(MI)); 352 353 if (isSSetReg(MI->getOpcode())) 354 return std::max(WaitStates, checkSetRegHazards(MI)); 355 356 if (isRFE(MI->getOpcode())) 357 return std::max(WaitStates, checkRFEHazards(MI)); 358 359 if ((ST.hasReadM0MovRelInterpHazard() && 360 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()))) || 361 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || 362 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || 363 (ST.hasReadM0LdsDirectHazard() && MI->readsRegister(AMDGPU::LDS_DIRECT))) 364 return std::max(WaitStates, checkReadM0Hazards(MI)); 365 366 if (SIInstrInfo::isMAI(*MI)) 367 return std::max(WaitStates, checkMAIHazards(MI)); 368 369 if (SIInstrInfo::isVMEM(*MI) || 370 SIInstrInfo::isFLAT(*MI) || 371 SIInstrInfo::isDS(*MI)) 372 return std::max(WaitStates, checkMAILdStHazards(MI)); 373 374 return WaitStates; 375 } 376 377 void GCNHazardRecognizer::EmitNoop() { 378 EmittedInstrs.push_front(nullptr); 379 } 380 381 void GCNHazardRecognizer::AdvanceCycle() { 382 // When the scheduler detects a stall, it will call AdvanceCycle() without 383 // emitting any instructions. 384 if (!CurrCycleInstr) { 385 EmittedInstrs.push_front(nullptr); 386 return; 387 } 388 389 if (CurrCycleInstr->isBundle()) { 390 processBundle(); 391 return; 392 } 393 394 unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr); 395 if (!NumWaitStates) { 396 CurrCycleInstr = nullptr; 397 return; 398 } 399 400 // Keep track of emitted instructions 401 EmittedInstrs.push_front(CurrCycleInstr); 402 403 // Add a nullptr for each additional wait state after the first. Make sure 404 // not to add more than getMaxLookAhead() items to the list, since we 405 // truncate the list to that size right after this loop. 406 for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead()); 407 i < e; ++i) { 408 EmittedInstrs.push_front(nullptr); 409 } 410 411 // getMaxLookahead() is the largest number of wait states we will ever need 412 // to insert, so there is no point in keeping track of more than that many 413 // wait states. 414 EmittedInstrs.resize(getMaxLookAhead()); 415 416 CurrCycleInstr = nullptr; 417 } 418 419 void GCNHazardRecognizer::RecedeCycle() { 420 llvm_unreachable("hazard recognizer does not support bottom-up scheduling."); 421 } 422 423 //===----------------------------------------------------------------------===// 424 // Helper Functions 425 //===----------------------------------------------------------------------===// 426 427 typedef function_ref<bool(const MachineInstr &, int WaitStates)> IsExpiredFn; 428 429 // Returns a minimum wait states since \p I walking all predecessors. 430 // Only scans until \p IsExpired does not return true. 431 // Can only be run in a hazard recognizer mode. 432 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, 433 const MachineBasicBlock *MBB, 434 MachineBasicBlock::const_reverse_instr_iterator I, 435 int WaitStates, IsExpiredFn IsExpired, 436 DenseSet<const MachineBasicBlock *> &Visited) { 437 for (auto E = MBB->instr_rend(); I != E; ++I) { 438 // Don't add WaitStates for parent BUNDLE instructions. 439 if (I->isBundle()) 440 continue; 441 442 if (IsHazard(*I)) 443 return WaitStates; 444 445 if (I->isInlineAsm()) 446 continue; 447 448 WaitStates += SIInstrInfo::getNumWaitStates(*I); 449 450 if (IsExpired(*I, WaitStates)) 451 return std::numeric_limits<int>::max(); 452 } 453 454 int MinWaitStates = std::numeric_limits<int>::max(); 455 for (MachineBasicBlock *Pred : MBB->predecessors()) { 456 if (!Visited.insert(Pred).second) 457 continue; 458 459 int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(), 460 WaitStates, IsExpired, Visited); 461 462 MinWaitStates = std::min(MinWaitStates, W); 463 } 464 465 return MinWaitStates; 466 } 467 468 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, 469 const MachineInstr *MI, IsExpiredFn IsExpired) { 470 DenseSet<const MachineBasicBlock *> Visited; 471 return getWaitStatesSince(IsHazard, MI->getParent(), 472 std::next(MI->getReverseIterator()), 473 0, IsExpired, Visited); 474 } 475 476 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) { 477 if (IsHazardRecognizerMode) { 478 auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) { 479 return WaitStates >= Limit; 480 }; 481 return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn); 482 } 483 484 int WaitStates = 0; 485 for (MachineInstr *MI : EmittedInstrs) { 486 if (MI) { 487 if (IsHazard(*MI)) 488 return WaitStates; 489 490 if (MI->isInlineAsm()) 491 continue; 492 } 493 ++WaitStates; 494 495 if (WaitStates >= Limit) 496 break; 497 } 498 return std::numeric_limits<int>::max(); 499 } 500 501 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg, 502 IsHazardFn IsHazardDef, 503 int Limit) { 504 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 505 506 auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) { 507 return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI); 508 }; 509 510 return getWaitStatesSince(IsHazardFn, Limit); 511 } 512 513 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard, 514 int Limit) { 515 auto IsHazardFn = [IsHazard](const MachineInstr &MI) { 516 return isSSetReg(MI.getOpcode()) && IsHazard(MI); 517 }; 518 519 return getWaitStatesSince(IsHazardFn, Limit); 520 } 521 522 //===----------------------------------------------------------------------===// 523 // No-op Hazard Detection 524 //===----------------------------------------------------------------------===// 525 526 static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, 527 MCRegister Reg) { 528 for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI) 529 BV.set(*RUI); 530 } 531 532 static void addRegsToSet(const SIRegisterInfo &TRI, 533 iterator_range<MachineInstr::const_mop_iterator> Ops, 534 BitVector &Set) { 535 for (const MachineOperand &Op : Ops) { 536 if (Op.isReg()) 537 addRegUnits(TRI, Set, Op.getReg().asMCReg()); 538 } 539 } 540 541 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) { 542 // XXX: Do we need to worry about implicit operands 543 addRegsToSet(TRI, MI.defs(), ClauseDefs); 544 addRegsToSet(TRI, MI.uses(), ClauseUses); 545 } 546 547 static bool breaksSMEMSoftClause(MachineInstr *MI) { 548 return !SIInstrInfo::isSMRD(*MI); 549 } 550 551 static bool breaksVMEMSoftClause(MachineInstr *MI) { 552 return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI); 553 } 554 555 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) { 556 // SMEM soft clause are only present on VI+, and only matter if xnack is 557 // enabled. 558 if (!ST.isXNACKEnabled()) 559 return 0; 560 561 bool IsSMRD = TII.isSMRD(*MEM); 562 563 resetClause(); 564 565 // A soft-clause is any group of consecutive SMEM instructions. The 566 // instructions in this group may return out of order and/or may be 567 // replayed (i.e. the same instruction issued more than once). 568 // 569 // In order to handle these situations correctly we need to make sure that 570 // when a clause has more than one instruction, no instruction in the clause 571 // writes to a register that is read by another instruction in the clause 572 // (including itself). If we encounter this situation, we need to break the 573 // clause by inserting a non SMEM instruction. 574 575 for (MachineInstr *MI : EmittedInstrs) { 576 // When we hit a non-SMEM instruction then we have passed the start of the 577 // clause and we can stop. 578 if (!MI) 579 break; 580 581 if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI)) 582 break; 583 584 addClauseInst(*MI); 585 } 586 587 if (ClauseDefs.none()) 588 return 0; 589 590 // We need to make sure not to put loads and stores in the same clause if they 591 // use the same address. For now, just start a new clause whenever we see a 592 // store. 593 if (MEM->mayStore()) 594 return 1; 595 596 addClauseInst(*MEM); 597 598 // If the set of defs and uses intersect then we cannot add this instruction 599 // to the clause, so we have a hazard. 600 return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0; 601 } 602 603 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) { 604 int WaitStatesNeeded = 0; 605 606 WaitStatesNeeded = checkSoftClauseHazards(SMRD); 607 608 // This SMRD hazard only affects SI. 609 if (!ST.hasSMRDReadVALUDefHazard()) 610 return WaitStatesNeeded; 611 612 // A read of an SGPR by SMRD instruction requires 4 wait states when the 613 // SGPR was written by a VALU instruction. 614 int SmrdSgprWaitStates = 4; 615 auto IsHazardDefFn = [this](const MachineInstr &MI) { 616 return TII.isVALU(MI); 617 }; 618 auto IsBufferHazardDefFn = [this](const MachineInstr &MI) { 619 return TII.isSALU(MI); 620 }; 621 622 bool IsBufferSMRD = TII.isBufferSMRD(*SMRD); 623 624 for (const MachineOperand &Use : SMRD->uses()) { 625 if (!Use.isReg()) 626 continue; 627 int WaitStatesNeededForUse = 628 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn, 629 SmrdSgprWaitStates); 630 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 631 632 // This fixes what appears to be undocumented hardware behavior in SI where 633 // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor 634 // needs some number of nops in between. We don't know how many we need, but 635 // let's use 4. This wasn't discovered before probably because the only 636 // case when this happens is when we expand a 64-bit pointer into a full 637 // descriptor and use s_buffer_load_dword instead of s_load_dword, which was 638 // probably never encountered in the closed-source land. 639 if (IsBufferSMRD) { 640 int WaitStatesNeededForUse = 641 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), 642 IsBufferHazardDefFn, 643 SmrdSgprWaitStates); 644 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 645 } 646 } 647 648 return WaitStatesNeeded; 649 } 650 651 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) { 652 if (!ST.hasVMEMReadSGPRVALUDefHazard()) 653 return 0; 654 655 int WaitStatesNeeded = checkSoftClauseHazards(VMEM); 656 657 // A read of an SGPR by a VMEM instruction requires 5 wait states when the 658 // SGPR was written by a VALU Instruction. 659 const int VmemSgprWaitStates = 5; 660 auto IsHazardDefFn = [this](const MachineInstr &MI) { 661 return TII.isVALU(MI); 662 }; 663 for (const MachineOperand &Use : VMEM->uses()) { 664 if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg())) 665 continue; 666 667 int WaitStatesNeededForUse = 668 VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn, 669 VmemSgprWaitStates); 670 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 671 } 672 return WaitStatesNeeded; 673 } 674 675 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) { 676 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 677 const SIInstrInfo *TII = ST.getInstrInfo(); 678 679 // Check for DPP VGPR read after VALU VGPR write and EXEC write. 680 int DppVgprWaitStates = 2; 681 int DppExecWaitStates = 5; 682 int WaitStatesNeeded = 0; 683 auto IsHazardDefFn = [TII](const MachineInstr &MI) { 684 return TII->isVALU(MI); 685 }; 686 687 for (const MachineOperand &Use : DPP->uses()) { 688 if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg())) 689 continue; 690 int WaitStatesNeededForUse = 691 DppVgprWaitStates - getWaitStatesSinceDef( 692 Use.getReg(), 693 [](const MachineInstr &) { return true; }, 694 DppVgprWaitStates); 695 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 696 } 697 698 WaitStatesNeeded = std::max( 699 WaitStatesNeeded, 700 DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn, 701 DppExecWaitStates)); 702 703 return WaitStatesNeeded; 704 } 705 706 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) { 707 const SIInstrInfo *TII = ST.getInstrInfo(); 708 709 // v_div_fmas requires 4 wait states after a write to vcc from a VALU 710 // instruction. 711 const int DivFMasWaitStates = 4; 712 auto IsHazardDefFn = [TII](const MachineInstr &MI) { 713 return TII->isVALU(MI); 714 }; 715 int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn, 716 DivFMasWaitStates); 717 718 return DivFMasWaitStates - WaitStatesNeeded; 719 } 720 721 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) { 722 const SIInstrInfo *TII = ST.getInstrInfo(); 723 unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr); 724 725 const int GetRegWaitStates = 2; 726 auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) { 727 return GetRegHWReg == getHWReg(TII, MI); 728 }; 729 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates); 730 731 return GetRegWaitStates - WaitStatesNeeded; 732 } 733 734 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) { 735 const SIInstrInfo *TII = ST.getInstrInfo(); 736 unsigned HWReg = getHWReg(TII, *SetRegInstr); 737 738 const int SetRegWaitStates = ST.getSetRegWaitStates(); 739 auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) { 740 return HWReg == getHWReg(TII, MI); 741 }; 742 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates); 743 return SetRegWaitStates - WaitStatesNeeded; 744 } 745 746 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) { 747 if (!MI.mayStore()) 748 return -1; 749 750 const SIInstrInfo *TII = ST.getInstrInfo(); 751 unsigned Opcode = MI.getOpcode(); 752 const MCInstrDesc &Desc = MI.getDesc(); 753 754 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); 755 int VDataRCID = -1; 756 if (VDataIdx != -1) 757 VDataRCID = Desc.OpInfo[VDataIdx].RegClass; 758 759 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) { 760 // There is no hazard if the instruction does not use vector regs 761 // (like wbinvl1) 762 if (VDataIdx == -1) 763 return -1; 764 // For MUBUF/MTBUF instructions this hazard only exists if the 765 // instruction is not using a register in the soffset field. 766 const MachineOperand *SOffset = 767 TII->getNamedOperand(MI, AMDGPU::OpName::soffset); 768 // If we have no soffset operand, then assume this field has been 769 // hardcoded to zero. 770 if (AMDGPU::getRegBitWidth(VDataRCID) > 64 && 771 (!SOffset || !SOffset->isReg())) 772 return VDataIdx; 773 } 774 775 // MIMG instructions create a hazard if they don't use a 256-bit T# and 776 // the store size is greater than 8 bytes and they have more than two bits 777 // of their dmask set. 778 // All our MIMG definitions use a 256-bit T#, so we can skip checking for them. 779 if (TII->isMIMG(MI)) { 780 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); 781 assert(SRsrcIdx != -1 && 782 AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256); 783 (void)SRsrcIdx; 784 } 785 786 if (TII->isFLAT(MI)) { 787 int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); 788 if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64) 789 return DataIdx; 790 } 791 792 return -1; 793 } 794 795 int 796 GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def, 797 const MachineRegisterInfo &MRI) { 798 // Helper to check for the hazard where VMEM instructions that store more than 799 // 8 bytes can have there store data over written by the next instruction. 800 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 801 802 const int VALUWaitStates = ST.hasGFX940Insts() ? 2 : 1; 803 int WaitStatesNeeded = 0; 804 805 if (!TRI->isVectorRegister(MRI, Def.getReg())) 806 return WaitStatesNeeded; 807 Register Reg = Def.getReg(); 808 auto IsHazardFn = [this, Reg, TRI](const MachineInstr &MI) { 809 int DataIdx = createsVALUHazard(MI); 810 return DataIdx >= 0 && 811 TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg); 812 }; 813 int WaitStatesNeededForDef = 814 VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates); 815 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 816 817 return WaitStatesNeeded; 818 } 819 820 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) { 821 int WaitStatesNeeded = 0; 822 823 if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(*VALU)) { 824 const int TransDefWaitstates = 1; 825 826 auto IsTransDefFn = [this, VALU](const MachineInstr &MI) { 827 if (!SIInstrInfo::isTRANS(MI)) 828 return false; 829 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 830 const SIInstrInfo *TII = ST.getInstrInfo(); 831 Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg(); 832 833 for (const MachineOperand &Use : VALU->explicit_uses()) { 834 if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg())) 835 return true; 836 } 837 838 return false; 839 }; 840 841 int WaitStatesNeededForDef = 842 TransDefWaitstates - 843 getWaitStatesSince(IsTransDefFn, TransDefWaitstates); 844 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 845 } 846 847 if (ST.hasDstSelForwardingHazard()) { 848 const int Shift16DefWaitstates = 1; 849 850 auto IsShift16BitDefFn = [this, VALU](const MachineInstr &MI) { 851 if (!SIInstrInfo::isVALU(MI)) 852 return false; 853 const SIInstrInfo *TII = ST.getInstrInfo(); 854 if (SIInstrInfo::isSDWA(MI)) { 855 if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel)) 856 if (DstSel->getImm() == AMDGPU::SDWA::DWORD) 857 return false; 858 } else { 859 if ((AMDGPU::getNamedOperandIdx(MI.getOpcode(), 860 AMDGPU::OpName::op_sel) == -1) || 861 !(TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers) 862 ->getImm() & 863 SISrcMods::DST_OP_SEL)) 864 return false; 865 } 866 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 867 if (auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) { 868 Register Def = Dst->getReg(); 869 870 for (const MachineOperand &Use : VALU->explicit_uses()) { 871 if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg())) 872 return true; 873 } 874 } 875 876 return false; 877 }; 878 879 int WaitStatesNeededForDef = 880 Shift16DefWaitstates - 881 getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates); 882 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 883 } 884 885 if (ST.hasVDecCoExecHazard()) { 886 const int VALUWriteSGPRVALUReadWaitstates = 2; 887 const int VALUWriteEXECRWLane = 4; 888 const int VALUWriteVGPRReadlaneRead = 1; 889 890 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 891 const MachineRegisterInfo &MRI = MF.getRegInfo(); 892 Register UseReg; 893 auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) { 894 if (!SIInstrInfo::isVALU(MI)) 895 return false; 896 return MI.modifiesRegister(UseReg, TRI); 897 }; 898 899 for (const MachineOperand &Use : VALU->explicit_uses()) { 900 if (!Use.isReg()) 901 continue; 902 903 UseReg = Use.getReg(); 904 if (TRI->isSGPRReg(MRI, UseReg)) { 905 int WaitStatesNeededForDef = 906 VALUWriteSGPRVALUReadWaitstates - 907 getWaitStatesSince(IsVALUDefSGPRFn, 908 VALUWriteSGPRVALUReadWaitstates); 909 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 910 } 911 } 912 913 if (VALU->readsRegister(AMDGPU::VCC, TRI)) { 914 UseReg = AMDGPU::VCC; 915 int WaitStatesNeededForDef = 916 VALUWriteSGPRVALUReadWaitstates - 917 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteSGPRVALUReadWaitstates); 918 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 919 } 920 921 switch (VALU->getOpcode()) { 922 case AMDGPU::V_READLANE_B32: 923 case AMDGPU::V_READFIRSTLANE_B32: { 924 MachineOperand *Src = TII.getNamedOperand(*VALU, AMDGPU::OpName::src0); 925 UseReg = Src->getReg(); 926 int WaitStatesNeededForDef = 927 VALUWriteVGPRReadlaneRead - 928 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteVGPRReadlaneRead); 929 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 930 } 931 LLVM_FALLTHROUGH; 932 case AMDGPU::V_WRITELANE_B32: { 933 UseReg = AMDGPU::EXEC; 934 int WaitStatesNeededForDef = 935 VALUWriteEXECRWLane - 936 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteEXECRWLane); 937 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 938 break; 939 } 940 default: 941 break; 942 } 943 } 944 945 // This checks for the hazard where VMEM instructions that store more than 946 // 8 bytes can have there store data over written by the next instruction. 947 if (!ST.has12DWordStoreHazard()) 948 return WaitStatesNeeded; 949 950 const MachineRegisterInfo &MRI = MF.getRegInfo(); 951 952 for (const MachineOperand &Def : VALU->defs()) { 953 WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI)); 954 } 955 956 return WaitStatesNeeded; 957 } 958 959 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) { 960 // This checks for hazards associated with inline asm statements. 961 // Since inline asms can contain just about anything, we use this 962 // to call/leverage other check*Hazard routines. Note that 963 // this function doesn't attempt to address all possible inline asm 964 // hazards (good luck), but is a collection of what has been 965 // problematic thus far. 966 967 // see checkVALUHazards() 968 if (!ST.has12DWordStoreHazard()) 969 return 0; 970 971 const MachineRegisterInfo &MRI = MF.getRegInfo(); 972 int WaitStatesNeeded = 0; 973 974 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands(); 975 I != E; ++I) { 976 const MachineOperand &Op = IA->getOperand(I); 977 if (Op.isReg() && Op.isDef()) { 978 WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI)); 979 } 980 } 981 982 return WaitStatesNeeded; 983 } 984 985 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) { 986 const SIInstrInfo *TII = ST.getInstrInfo(); 987 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 988 const MachineRegisterInfo &MRI = MF.getRegInfo(); 989 990 const MachineOperand *LaneSelectOp = 991 TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1); 992 993 if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg())) 994 return 0; 995 996 Register LaneSelectReg = LaneSelectOp->getReg(); 997 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); }; 998 999 const int RWLaneWaitStates = 4; 1000 int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn, 1001 RWLaneWaitStates); 1002 return RWLaneWaitStates - WaitStatesSince; 1003 } 1004 1005 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) { 1006 if (!ST.hasRFEHazards()) 1007 return 0; 1008 1009 const SIInstrInfo *TII = ST.getInstrInfo(); 1010 1011 const int RFEWaitStates = 1; 1012 1013 auto IsHazardFn = [TII](const MachineInstr &MI) { 1014 return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS; 1015 }; 1016 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates); 1017 return RFEWaitStates - WaitStatesNeeded; 1018 } 1019 1020 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) { 1021 const SIInstrInfo *TII = ST.getInstrInfo(); 1022 const int ReadM0WaitStates = 1; 1023 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); }; 1024 return ReadM0WaitStates - 1025 getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, ReadM0WaitStates); 1026 } 1027 1028 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) { 1029 fixVMEMtoScalarWriteHazards(MI); 1030 fixVcmpxPermlaneHazards(MI); 1031 fixSMEMtoVectorWriteHazards(MI); 1032 fixVcmpxExecWARHazard(MI); 1033 fixLdsBranchVmemWARHazard(MI); 1034 } 1035 1036 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) { 1037 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI)) 1038 return false; 1039 1040 const SIInstrInfo *TII = ST.getInstrInfo(); 1041 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1042 auto IsHazardFn = [TII, TRI](const MachineInstr &MI) { 1043 return (TII->isVOPC(MI) || 1044 ((TII->isVOP3(MI) || TII->isSDWA(MI)) && MI.isCompare())) && 1045 MI.modifiesRegister(AMDGPU::EXEC, TRI); 1046 }; 1047 1048 auto IsExpiredFn = [](const MachineInstr &MI, int) { 1049 unsigned Opc = MI.getOpcode(); 1050 return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 && 1051 Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa; 1052 }; 1053 1054 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1055 std::numeric_limits<int>::max()) 1056 return false; 1057 1058 // V_NOP will be discarded by SQ. 1059 // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE* 1060 // which is always a VGPR and available. 1061 auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0); 1062 Register Reg = Src0->getReg(); 1063 bool IsUndef = Src0->isUndef(); 1064 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1065 TII->get(AMDGPU::V_MOV_B32_e32)) 1066 .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0)) 1067 .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill); 1068 1069 return true; 1070 } 1071 1072 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) { 1073 if (!ST.hasVMEMtoScalarWriteHazard()) 1074 return false; 1075 1076 if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI)) 1077 return false; 1078 1079 if (MI->getNumDefs() == 0) 1080 return false; 1081 1082 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1083 1084 auto IsHazardFn = [TRI, MI](const MachineInstr &I) { 1085 if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I) && 1086 !SIInstrInfo::isFLAT(I)) 1087 return false; 1088 1089 for (const MachineOperand &Def : MI->defs()) { 1090 const MachineOperand *Op = 1091 I.findRegisterUseOperand(Def.getReg(), false, TRI); 1092 if (!Op) 1093 continue; 1094 return true; 1095 } 1096 return false; 1097 }; 1098 1099 auto IsExpiredFn = [](const MachineInstr &MI, int) { 1100 return SIInstrInfo::isVALU(MI) || 1101 (MI.getOpcode() == AMDGPU::S_WAITCNT && 1102 !MI.getOperand(0).getImm()) || 1103 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1104 MI.getOperand(0).getImm() == 0xffe3); 1105 }; 1106 1107 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1108 std::numeric_limits<int>::max()) 1109 return false; 1110 1111 const SIInstrInfo *TII = ST.getInstrInfo(); 1112 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1113 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) 1114 .addImm(0xffe3); 1115 return true; 1116 } 1117 1118 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) { 1119 if (!ST.hasSMEMtoVectorWriteHazard()) 1120 return false; 1121 1122 if (!SIInstrInfo::isVALU(*MI)) 1123 return false; 1124 1125 unsigned SDSTName; 1126 switch (MI->getOpcode()) { 1127 case AMDGPU::V_READLANE_B32: 1128 case AMDGPU::V_READFIRSTLANE_B32: 1129 SDSTName = AMDGPU::OpName::vdst; 1130 break; 1131 default: 1132 SDSTName = AMDGPU::OpName::sdst; 1133 break; 1134 } 1135 1136 const SIInstrInfo *TII = ST.getInstrInfo(); 1137 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1138 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU()); 1139 const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName); 1140 if (!SDST) { 1141 for (const auto &MO : MI->implicit_operands()) { 1142 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) { 1143 SDST = &MO; 1144 break; 1145 } 1146 } 1147 } 1148 1149 if (!SDST) 1150 return false; 1151 1152 const Register SDSTReg = SDST->getReg(); 1153 auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) { 1154 return SIInstrInfo::isSMRD(I) && I.readsRegister(SDSTReg, TRI); 1155 }; 1156 1157 auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) { 1158 if (TII->isSALU(MI)) { 1159 switch (MI.getOpcode()) { 1160 case AMDGPU::S_SETVSKIP: 1161 case AMDGPU::S_VERSION: 1162 case AMDGPU::S_WAITCNT_VSCNT: 1163 case AMDGPU::S_WAITCNT_VMCNT: 1164 case AMDGPU::S_WAITCNT_EXPCNT: 1165 // These instructions cannot not mitigate the hazard. 1166 return false; 1167 case AMDGPU::S_WAITCNT_LGKMCNT: 1168 // Reducing lgkmcnt count to 0 always mitigates the hazard. 1169 return (MI.getOperand(1).getImm() == 0) && 1170 (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL); 1171 case AMDGPU::S_WAITCNT: { 1172 const int64_t Imm = MI.getOperand(0).getImm(); 1173 AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm); 1174 return (Decoded.LgkmCnt == 0); 1175 } 1176 default: 1177 // SOPP instructions cannot mitigate the hazard. 1178 if (TII->isSOPP(MI)) 1179 return false; 1180 // At this point the SALU can be assumed to mitigate the hazard 1181 // because either: 1182 // (a) it is independent of the at risk SMEM (breaking chain), 1183 // or 1184 // (b) it is dependent on the SMEM, in which case an appropriate 1185 // s_waitcnt lgkmcnt _must_ exist between it and the at risk 1186 // SMEM instruction. 1187 return true; 1188 } 1189 } 1190 return false; 1191 }; 1192 1193 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1194 std::numeric_limits<int>::max()) 1195 return false; 1196 1197 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1198 TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL) 1199 .addImm(0); 1200 return true; 1201 } 1202 1203 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) { 1204 if (!ST.hasVcmpxExecWARHazard() || !SIInstrInfo::isVALU(*MI)) 1205 return false; 1206 1207 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1208 if (!MI->modifiesRegister(AMDGPU::EXEC, TRI)) 1209 return false; 1210 1211 auto IsHazardFn = [TRI](const MachineInstr &I) { 1212 if (SIInstrInfo::isVALU(I)) 1213 return false; 1214 return I.readsRegister(AMDGPU::EXEC, TRI); 1215 }; 1216 1217 const SIInstrInfo *TII = ST.getInstrInfo(); 1218 auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) { 1219 if (SIInstrInfo::isVALU(MI)) { 1220 if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) 1221 return true; 1222 for (auto MO : MI.implicit_operands()) 1223 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) 1224 return true; 1225 } 1226 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1227 (MI.getOperand(0).getImm() & 0xfffe) == 0xfffe) 1228 return true; 1229 return false; 1230 }; 1231 1232 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1233 std::numeric_limits<int>::max()) 1234 return false; 1235 1236 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1237 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) 1238 .addImm(0xfffe); 1239 return true; 1240 } 1241 1242 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, 1243 const GCNSubtarget &ST) { 1244 if (!ST.hasLdsBranchVmemWARHazard()) 1245 return false; 1246 1247 // Check if the necessary condition for the hazard is met: both LDS and VMEM 1248 // instructions need to appear in the same function. 1249 bool HasLds = false; 1250 bool HasVmem = false; 1251 for (auto &MBB : MF) { 1252 for (auto &MI : MBB) { 1253 HasLds |= SIInstrInfo::isDS(MI); 1254 HasVmem |= 1255 SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI); 1256 if (HasLds && HasVmem) 1257 return true; 1258 } 1259 } 1260 return false; 1261 } 1262 1263 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) { 1264 if (!RunLdsBranchVmemWARHazardFixup) 1265 return false; 1266 1267 assert(ST.hasLdsBranchVmemWARHazard()); 1268 1269 auto IsHazardInst = [](const MachineInstr &MI) { 1270 if (SIInstrInfo::isDS(MI)) 1271 return 1; 1272 if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI)) 1273 return 2; 1274 return 0; 1275 }; 1276 1277 auto InstType = IsHazardInst(*MI); 1278 if (!InstType) 1279 return false; 1280 1281 auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) { 1282 return IsHazardInst(I) || (I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT && 1283 I.getOperand(0).getReg() == AMDGPU::SGPR_NULL && 1284 !I.getOperand(1).getImm()); 1285 }; 1286 1287 auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) { 1288 if (!I.isBranch()) 1289 return false; 1290 1291 auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) { 1292 auto InstType2 = IsHazardInst(I); 1293 return InstType2 && InstType != InstType2; 1294 }; 1295 1296 auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) { 1297 auto InstType2 = IsHazardInst(I); 1298 if (InstType == InstType2) 1299 return true; 1300 1301 return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT && 1302 I.getOperand(0).getReg() == AMDGPU::SGPR_NULL && 1303 !I.getOperand(1).getImm(); 1304 }; 1305 1306 return ::getWaitStatesSince(IsHazardFn, &I, IsExpiredFn) != 1307 std::numeric_limits<int>::max(); 1308 }; 1309 1310 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1311 std::numeric_limits<int>::max()) 1312 return false; 1313 1314 const SIInstrInfo *TII = ST.getInstrInfo(); 1315 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1316 TII->get(AMDGPU::S_WAITCNT_VSCNT)) 1317 .addReg(AMDGPU::SGPR_NULL, RegState::Undef) 1318 .addImm(0); 1319 1320 return true; 1321 } 1322 1323 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) { 1324 int NSAtoVMEMWaitStates = 1; 1325 1326 if (!ST.hasNSAtoVMEMBug()) 1327 return 0; 1328 1329 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI)) 1330 return 0; 1331 1332 const SIInstrInfo *TII = ST.getInstrInfo(); 1333 const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset); 1334 if (!Offset || (Offset->getImm() & 6) == 0) 1335 return 0; 1336 1337 auto IsHazardFn = [TII](const MachineInstr &I) { 1338 if (!SIInstrInfo::isMIMG(I)) 1339 return false; 1340 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode()); 1341 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA && 1342 TII->getInstSizeInBytes(I) >= 16; 1343 }; 1344 1345 return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1); 1346 } 1347 1348 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) { 1349 int FPAtomicToDenormModeWaitStates = 3; 1350 1351 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE) 1352 return 0; 1353 1354 auto IsHazardFn = [](const MachineInstr &I) { 1355 if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I)) 1356 return false; 1357 return SIInstrInfo::isFPAtomic(I); 1358 }; 1359 1360 auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) { 1361 if (WaitStates >= 3 || SIInstrInfo::isVALU(MI)) 1362 return true; 1363 1364 switch (MI.getOpcode()) { 1365 case AMDGPU::S_WAITCNT: 1366 case AMDGPU::S_WAITCNT_VSCNT: 1367 case AMDGPU::S_WAITCNT_VMCNT: 1368 case AMDGPU::S_WAITCNT_EXPCNT: 1369 case AMDGPU::S_WAITCNT_LGKMCNT: 1370 case AMDGPU::S_WAIT_IDLE: 1371 return true; 1372 default: 1373 break; 1374 } 1375 1376 return false; 1377 }; 1378 1379 return FPAtomicToDenormModeWaitStates - 1380 ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn); 1381 } 1382 1383 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) { 1384 assert(SIInstrInfo::isMAI(*MI)); 1385 1386 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI); 1387 } 1388 1389 int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) { 1390 // Early exit if no padding is requested. 1391 if (MFMAPaddingRatio == 0) 1392 return 0; 1393 1394 auto IsMFMAFn = [](const MachineInstr &MI) { 1395 return SIInstrInfo::isMAI(MI) && 1396 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1397 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; 1398 }; 1399 1400 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 1401 if (!IsMFMAFn(*MI) || MFI->getOccupancy() < 2) 1402 return 0; 1403 1404 int NeighborMFMALatency = 0; 1405 auto IsNeighboringMFMA = [&IsMFMAFn, &NeighborMFMALatency, 1406 this](const MachineInstr &MI) { 1407 if (!IsMFMAFn(MI)) 1408 return false; 1409 1410 NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI); 1411 return true; 1412 }; 1413 1414 const int MaxMFMAPipelineWaitStates = 16; 1415 int WaitStatesSinceNeighborMFMA = 1416 getWaitStatesSince(IsNeighboringMFMA, MaxMFMAPipelineWaitStates); 1417 1418 int NeighborMFMAPaddingNeeded = 1419 (NeighborMFMALatency * MFMAPaddingRatio / 100) - 1420 WaitStatesSinceNeighborMFMA; 1421 1422 return std::max(0, NeighborMFMAPaddingNeeded); 1423 } 1424 1425 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) { 1426 int WaitStatesNeeded = 0; 1427 unsigned Opc = MI->getOpcode(); 1428 1429 auto IsVALUFn = [](const MachineInstr &MI) { 1430 return SIInstrInfo::isVALU(MI); 1431 }; 1432 1433 if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write 1434 const int LegacyVALUWritesVGPRWaitStates = 2; 1435 const int VALUWritesExecWaitStates = 4; 1436 const int MaxWaitStates = 4; 1437 1438 int WaitStatesNeededForUse = VALUWritesExecWaitStates - 1439 getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates); 1440 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1441 1442 if (WaitStatesNeeded < MaxWaitStates) { 1443 for (const MachineOperand &Use : MI->explicit_uses()) { 1444 const int MaxWaitStates = 2; 1445 1446 if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg())) 1447 continue; 1448 1449 int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates - 1450 getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates); 1451 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1452 1453 if (WaitStatesNeeded == MaxWaitStates) 1454 break; 1455 } 1456 } 1457 } 1458 1459 auto IsMFMAFn = [](const MachineInstr &MI) { 1460 return SIInstrInfo::isMAI(MI) && 1461 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1462 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; 1463 }; 1464 1465 for (const MachineOperand &Op : MI->explicit_operands()) { 1466 if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg())) 1467 continue; 1468 1469 if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 1470 continue; 1471 1472 const int MFMAWritesAGPROverlappedSrcABWaitStates = 4; 1473 const int MFMAWritesAGPROverlappedSrcCWaitStates = 2; 1474 const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4; 1475 const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10; 1476 const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18; 1477 const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1; 1478 const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7; 1479 const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15; 1480 const int MaxWaitStates = 18; 1481 Register Reg = Op.getReg(); 1482 unsigned HazardDefLatency = 0; 1483 1484 auto IsOverlappedMFMAFn = [Reg, &IsMFMAFn, &HazardDefLatency, 1485 this](const MachineInstr &MI) { 1486 if (!IsMFMAFn(MI)) 1487 return false; 1488 Register DstReg = MI.getOperand(0).getReg(); 1489 if (DstReg == Reg) 1490 return false; 1491 HazardDefLatency = 1492 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); 1493 return TRI.regsOverlap(DstReg, Reg); 1494 }; 1495 1496 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, 1497 MaxWaitStates); 1498 int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates; 1499 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 1500 int OpNo = MI->getOperandNo(&Op); 1501 if (OpNo == SrcCIdx) { 1502 NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates; 1503 } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) { 1504 switch (HazardDefLatency) { 1505 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates; 1506 break; 1507 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates; 1508 break; 1509 case 16: LLVM_FALLTHROUGH; 1510 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates; 1511 break; 1512 } 1513 } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { 1514 switch (HazardDefLatency) { 1515 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates; 1516 break; 1517 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates; 1518 break; 1519 case 16: LLVM_FALLTHROUGH; 1520 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates; 1521 break; 1522 } 1523 } 1524 1525 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1526 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1527 1528 if (WaitStatesNeeded == MaxWaitStates) 1529 return WaitStatesNeeded; // Early exit. 1530 1531 auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) { 1532 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 1533 return false; 1534 Register DstReg = MI.getOperand(0).getReg(); 1535 return TRI.regsOverlap(Reg, DstReg); 1536 }; 1537 1538 const int AccVGPRWriteMFMAReadSrcCWaitStates = 1; 1539 const int AccVGPRWriteMFMAReadSrcABWaitStates = 3; 1540 const int AccVGPRWriteAccVgprReadWaitStates = 3; 1541 NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates; 1542 if (OpNo == SrcCIdx) 1543 NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates; 1544 else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) 1545 NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates; 1546 1547 WaitStatesNeededForUse = NeedWaitStates - 1548 getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates); 1549 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1550 1551 if (WaitStatesNeeded == MaxWaitStates) 1552 return WaitStatesNeeded; // Early exit. 1553 } 1554 1555 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { 1556 const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0; 1557 const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5; 1558 const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13; 1559 const int MaxWaitStates = 13; 1560 Register DstReg = MI->getOperand(0).getReg(); 1561 unsigned HazardDefLatency = 0; 1562 1563 auto IsSrcCMFMAFn = [DstReg, &IsMFMAFn, &HazardDefLatency, 1564 this](const MachineInstr &MI) { 1565 if (!IsMFMAFn(MI)) 1566 return false; 1567 Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg(); 1568 HazardDefLatency = 1569 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); 1570 return TRI.regsOverlap(Reg, DstReg); 1571 }; 1572 1573 int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates); 1574 int NeedWaitStates; 1575 switch (HazardDefLatency) { 1576 case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates; 1577 break; 1578 case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates; 1579 break; 1580 case 16: LLVM_FALLTHROUGH; 1581 default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates; 1582 break; 1583 } 1584 1585 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince; 1586 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1587 } 1588 1589 // Pad neighboring MFMA with noops for better inter-wave performance. 1590 WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI)); 1591 1592 return WaitStatesNeeded; 1593 } 1594 1595 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { 1596 int WaitStatesNeeded = 0; 1597 unsigned Opc = MI->getOpcode(); 1598 1599 auto IsMFMAFn = [](const MachineInstr &MI) { 1600 return SIInstrInfo::isMAI(MI) && 1601 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1602 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; 1603 }; 1604 1605 auto IsLegacyVALUFn = [&IsMFMAFn](const MachineInstr &MI) { 1606 return SIInstrInfo::isVALU(MI) && !IsMFMAFn(MI); 1607 }; 1608 1609 auto IsLegacyVALUNotDotFn = [&IsMFMAFn](const MachineInstr &MI) { 1610 return SIInstrInfo::isVALU(MI) && !IsMFMAFn(MI) && !SIInstrInfo::isDOT(MI); 1611 }; 1612 1613 if (!IsMFMAFn(*MI)) 1614 return WaitStatesNeeded; 1615 1616 const int VALUWritesExecWaitStates = 4; 1617 int WaitStatesNeededForUse = VALUWritesExecWaitStates - 1618 getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn, 1619 VALUWritesExecWaitStates); 1620 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1621 1622 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 1623 1624 // Loop for both DGEMM and S/HGEMM 2nd instruction. 1625 for (const MachineOperand &Use : MI->explicit_uses()) { 1626 const int LegacyVALUNotDotWritesVGPRWaitStates = 2; 1627 const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2; 1628 const int GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates = 3; 1629 const int GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates = 5; 1630 const int GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates = 4; 1631 const int GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates = 9; 1632 const int GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates = 8; 1633 const int GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates = 17; 1634 const int GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates = 16; 1635 const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8; 1636 const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16; 1637 const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3; 1638 const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9; 1639 const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17; 1640 const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9; 1641 const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4; 1642 const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5; 1643 const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11; 1644 const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19; 1645 const int GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates = 4; 1646 const int GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates = 6; 1647 const int GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates = 10; 1648 const int GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates = 18; 1649 const int GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates = 5; 1650 const int GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates = 7; 1651 const int GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates = 11; 1652 const int GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates = 19; 1653 const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6; 1654 const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11; 1655 const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4; 1656 const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2; 1657 const int MaxWaitStates = 19; 1658 1659 if (!Use.isReg()) 1660 continue; 1661 Register Reg = Use.getReg(); 1662 bool FullReg; 1663 const MachineInstr *MI1; 1664 1665 auto IsOverlappedMFMAFn = [Reg, &IsMFMAFn, &FullReg, &MI1, 1666 this](const MachineInstr &MI) { 1667 if (!IsMFMAFn(MI)) 1668 return false; 1669 Register DstReg = MI.getOperand(0).getReg(); 1670 FullReg = (DstReg == Reg); 1671 MI1 = &MI; 1672 return TRI.regsOverlap(DstReg, Reg); 1673 }; 1674 1675 WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates - 1676 getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates); 1677 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1678 1679 int NumWaitStates = 1680 getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates); 1681 if (NumWaitStates == std::numeric_limits<int>::max()) 1682 continue; 1683 1684 int OpNo = MI->getOperandNo(&Use); 1685 unsigned Opc1 = MI1->getOpcode(); 1686 int NeedWaitStates = 0; 1687 if (OpNo == SrcCIdx) { 1688 if (!isDGEMM(Opc) && (!ST.hasGFX940Insts() && isDGEMM(Opc1))) { 1689 NeedWaitStates = 0; 1690 } else if (FullReg) { 1691 if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 1692 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) && 1693 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 1694 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64)) 1695 NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates; 1696 else if (ST.hasGFX940Insts() && 1697 TSchedModel.computeInstrLatency(MI1) == 2) 1698 NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates; 1699 } else { 1700 switch (Opc1) { 1701 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: 1702 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: 1703 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64: 1704 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: 1705 if (!isXDL(ST, *MI)) 1706 NeedWaitStates = DMFMA16x16WritesVGPROverlappedSrcCWaitStates; 1707 break; 1708 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: 1709 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: 1710 if (!isXDL(ST, *MI)) 1711 NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates; 1712 break; 1713 default: 1714 if (ST.hasGFX940Insts() && isXDL(ST, *MI) && !isXDL(ST, *MI1)) 1715 break; 1716 switch (TSchedModel.computeInstrLatency(MI1)) { 1717 case 2: 1718 NeedWaitStates = ST.hasGFX940Insts() 1719 ? isXDL(ST, *MI1) 1720 ? GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates 1721 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates 1722 : isDGEMM(Opc) 1723 ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates 1724 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates; 1725 break; 1726 case 4: 1727 assert(ST.hasGFX940Insts()); 1728 NeedWaitStates = isXDL(ST, *MI1) 1729 ? GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates 1730 : GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates; 1731 break; 1732 case 8: 1733 NeedWaitStates = ST.hasGFX940Insts() 1734 ? isXDL(ST, *MI1) 1735 ? GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates 1736 : GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates 1737 : isDGEMM(Opc) 1738 ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates 1739 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates; 1740 break; 1741 case 16: LLVM_FALLTHROUGH; 1742 default: 1743 NeedWaitStates = ST.hasGFX940Insts() 1744 ? isXDL(ST, *MI1) 1745 ? GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates 1746 : GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates 1747 : isDGEMM(Opc) 1748 ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates 1749 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates; 1750 } 1751 } 1752 } 1753 } else { 1754 switch (Opc1) { 1755 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: 1756 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: 1757 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64: 1758 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: 1759 NeedWaitStates = DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates; 1760 break; 1761 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: 1762 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: 1763 NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates; 1764 break; 1765 default: 1766 switch (TSchedModel.computeInstrLatency(MI1)) { 1767 case 2: 1768 NeedWaitStates = ST.hasGFX940Insts() 1769 ? isXDL(ST, *MI1) 1770 ? GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates 1771 : GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates 1772 : SMFMA4x4WritesVGPROverlappedSrcABWaitStates; 1773 break; 1774 case 4: 1775 assert(ST.hasGFX940Insts()); 1776 NeedWaitStates = isXDL(ST, *MI1) 1777 ? GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates 1778 : GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates; 1779 break; 1780 case 8: 1781 NeedWaitStates = ST.hasGFX940Insts() 1782 ? isXDL(ST, *MI1) 1783 ? GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates 1784 : GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates 1785 : SMFMA16x16WritesVGPROverlappedSrcABWaitStates; 1786 break; 1787 case 16: LLVM_FALLTHROUGH; 1788 default: 1789 NeedWaitStates = ST.hasGFX940Insts() 1790 ? isXDL(ST, *MI1) 1791 ? GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates 1792 : GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates 1793 : SMFMA32x32WritesVGPROverlappedSrcABWaitStates; 1794 } 1795 } 1796 } 1797 if (WaitStatesNeeded >= NeedWaitStates) 1798 continue; 1799 1800 WaitStatesNeededForUse = NeedWaitStates - NumWaitStates; 1801 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1802 1803 if (WaitStatesNeeded == MaxWaitStates) 1804 break; 1805 } 1806 1807 return WaitStatesNeeded; 1808 } 1809 1810 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) { 1811 // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards() 1812 if (!ST.hasMAIInsts() || ST.hasGFX90AInsts()) 1813 return 0; 1814 1815 int WaitStatesNeeded = 0; 1816 1817 auto IsAccVgprReadFn = [](const MachineInstr &MI) { 1818 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64; 1819 }; 1820 1821 for (const MachineOperand &Op : MI->explicit_uses()) { 1822 if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg())) 1823 continue; 1824 1825 Register Reg = Op.getReg(); 1826 1827 const int AccVgprReadLdStWaitStates = 2; 1828 const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1; 1829 const int MaxWaitStates = 2; 1830 1831 int WaitStatesNeededForUse = AccVgprReadLdStWaitStates - 1832 getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates); 1833 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1834 1835 if (WaitStatesNeeded == MaxWaitStates) 1836 return WaitStatesNeeded; // Early exit. 1837 1838 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) { 1839 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 && 1840 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 1841 return false; 1842 auto IsVALUFn = [](const MachineInstr &MI) { 1843 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMAI(MI); 1844 }; 1845 return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) < 1846 std::numeric_limits<int>::max(); 1847 }; 1848 1849 WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates - 1850 getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates); 1851 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1852 } 1853 1854 return WaitStatesNeeded; 1855 } 1856 1857 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { 1858 if (!ST.hasGFX90AInsts()) 1859 return 0; 1860 1861 auto IsMFMAFn = [](const MachineInstr &MI) -> bool { 1862 return SIInstrInfo::isMAI(MI) && 1863 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1864 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; 1865 }; 1866 1867 auto IsDGEMMFn = [](const MachineInstr &MI) -> bool { 1868 return isDGEMM(MI.getOpcode()); 1869 }; 1870 1871 // This is checked in checkMAIHazards90A() 1872 if (IsMFMAFn(*MI)) 1873 return 0; 1874 1875 int WaitStatesNeeded = 0; 1876 1877 bool IsMemOrExport = SIInstrInfo::isVMEM(*MI) || 1878 SIInstrInfo::isFLAT(*MI) || 1879 SIInstrInfo::isDS(*MI) || 1880 SIInstrInfo::isEXP(*MI); 1881 bool IsVALU = SIInstrInfo::isVALU(*MI); 1882 1883 const MachineInstr *MFMA = nullptr; 1884 unsigned Reg; 1885 auto IsMFMAWriteFn = [&Reg, &IsMFMAFn, &MFMA, this](const MachineInstr &MI) { 1886 if (!IsMFMAFn(MI) || !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) 1887 return false; 1888 MFMA = &MI; 1889 return true; 1890 }; 1891 1892 const MachineInstr *DOT = nullptr; 1893 auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) { 1894 if (!SIInstrInfo::isDOT(MI) || 1895 !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) 1896 return false; 1897 DOT = &MI; 1898 return true; 1899 }; 1900 1901 int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 1902 AMDGPU::OpName::src2); 1903 1904 if (IsMemOrExport || IsVALU) { 1905 const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5; 1906 const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11; 1907 const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19; 1908 const int GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates = 4; 1909 const int GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates = 6; 1910 const int GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates = 10; 1911 const int GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates = 18; 1912 const int GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates = 5; 1913 const int GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates = 7; 1914 const int GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates = 11; 1915 const int GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates = 19; 1916 const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9; 1917 const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18; 1918 const int DMFMA4x4WriteVgprVALUReadWaitStates = 6; 1919 const int DMFMA16x16WriteVgprVALUReadWaitStates = 11; 1920 const int DotWriteSameDotReadSrcAB = 3; 1921 const int DotWriteDifferentVALURead = 3; 1922 const int MaxWaitStates = 19; 1923 1924 for (const MachineOperand &Use : MI->explicit_uses()) { 1925 if (!Use.isReg()) 1926 continue; 1927 Reg = Use.getReg(); 1928 1929 DOT = nullptr; 1930 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn, 1931 MaxWaitStates); 1932 if (DOT) { 1933 int NeedWaitStates = 0; 1934 if (DOT->getOpcode() == MI->getOpcode()) { 1935 if (&Use - &MI->getOperand(0) != SrcCIdx) 1936 NeedWaitStates = DotWriteSameDotReadSrcAB; 1937 } else { 1938 NeedWaitStates = DotWriteDifferentVALURead; 1939 } 1940 1941 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1942 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1943 } 1944 1945 MFMA = nullptr; 1946 WaitStatesSinceDef = 1947 getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates); 1948 if (!MFMA) 1949 continue; 1950 1951 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); 1952 int NeedWaitStates = MaxWaitStates; 1953 switch (HazardDefLatency) { 1954 case 2: 1955 NeedWaitStates = 1956 ST.hasGFX940Insts() 1957 ? isXDL(ST, *MFMA) 1958 ? GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates 1959 : GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates 1960 : SMFMA4x4WriteVgprVALUMemExpReadWaitStates; 1961 break; 1962 case 4: 1963 assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts()); 1964 NeedWaitStates = 1965 isDGEMM(MFMA->getOpcode()) 1966 ? IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates 1967 : DMFMA4x4WriteVgprVALUReadWaitStates 1968 : isXDL(ST, *MFMA) 1969 ? GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates 1970 : GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates; 1971 break; 1972 case 8: 1973 NeedWaitStates = 1974 ST.hasGFX940Insts() 1975 ? isXDL(ST, *MFMA) 1976 ? GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates 1977 : GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates 1978 : SMFMA16x16WriteVgprVALUMemExpReadWaitStates; 1979 break; 1980 case 16: LLVM_FALLTHROUGH; 1981 default: 1982 NeedWaitStates = 1983 isDGEMM(MFMA->getOpcode()) 1984 ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates 1985 : DMFMA16x16WriteVgprVALUReadWaitStates 1986 : ST.hasGFX940Insts() 1987 ? isXDL(ST, *MFMA) 1988 ? GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates 1989 : GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates 1990 : SMFMA32x32WriteVgprVALUMemExpReadWaitStates; 1991 break; 1992 } 1993 1994 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1995 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1996 1997 if (WaitStatesNeeded == MaxWaitStates) 1998 break; 1999 } 2000 } 2001 2002 unsigned Opc = MI->getOpcode(); 2003 const int DMFMAToFMA64WaitStates = 2; 2004 if ((Opc == AMDGPU::V_FMA_F64_e64 || 2005 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 || 2006 Opc == AMDGPU::V_FMAC_F64_dpp) && 2007 WaitStatesNeeded < DMFMAToFMA64WaitStates) { 2008 int WaitStatesNeededForUse = DMFMAToFMA64WaitStates - 2009 getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates); 2010 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2011 } 2012 2013 if (!IsVALU && !IsMemOrExport) 2014 return WaitStatesNeeded; 2015 2016 for (const MachineOperand &Def : MI->defs()) { 2017 const int SMFMA4x4WriteVgprVALUWawWaitStates = 5; 2018 const int SMFMA16x16WriteVgprVALUWawWaitStates = 11; 2019 const int SMFMA32x32WriteVgprVALUWawWaitStates = 19; 2020 const int GFX940_SMFMA2PassWriteVgprVALUWawWaitStates = 4; 2021 const int GFX940_SMFMA4PassWriteVgprVALUWawWaitStates = 6; 2022 const int GFX940_SMFMA8PassWriteVgprVALUWawWaitStates = 10; 2023 const int GFX940_SMFMA16PassWriteVgprVALUWawWaitStates = 18; 2024 const int GFX940_XDL2PassWriteVgprVALUWawWaitStates = 5; 2025 const int GFX940_XDL4PassWriteVgprVALUWawWaitStates = 7; 2026 const int GFX940_XDL8PassWriteVgprVALUWawWaitStates = 11; 2027 const int GFX940_XDL16PassWriteVgprVALUWawWaitStates = 19; 2028 const int SMFMA4x4ReadVgprVALUWarWaitStates = 1; 2029 const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3; 2030 const int SMFMA16x16ReadVgprVALUWarWaitStates = 7; 2031 const int SMFMA32x32ReadVgprVALUWarWaitStates = 15; 2032 const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6; 2033 const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11; 2034 const int DotWriteDifferentVALUWrite = 3; 2035 const int MaxWaitStates = 19; 2036 const int MaxWarWaitStates = 15; 2037 2038 Reg = Def.getReg(); 2039 2040 DOT = nullptr; 2041 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn, 2042 MaxWaitStates); 2043 if (DOT && DOT->getOpcode() != MI->getOpcode()) 2044 WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite - 2045 WaitStatesSinceDef); 2046 2047 MFMA = nullptr; 2048 WaitStatesSinceDef = 2049 getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates); 2050 if (MFMA) { 2051 int NeedWaitStates = MaxWaitStates; 2052 switch (TSchedModel.computeInstrLatency(MFMA)) { 2053 case 2: 2054 NeedWaitStates = ST.hasGFX940Insts() 2055 ? isXDL(ST, *MFMA) 2056 ? GFX940_XDL2PassWriteVgprVALUWawWaitStates 2057 : GFX940_SMFMA2PassWriteVgprVALUWawWaitStates 2058 : SMFMA4x4WriteVgprVALUWawWaitStates; 2059 break; 2060 case 4: 2061 assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts()); 2062 NeedWaitStates = isDGEMM(MFMA->getOpcode()) 2063 ? DMFMA4x4WriteVgprVALUWriteWaitStates 2064 : isXDL(ST, *MFMA) 2065 ? GFX940_XDL4PassWriteVgprVALUWawWaitStates 2066 : GFX940_SMFMA4PassWriteVgprVALUWawWaitStates; 2067 break; 2068 case 8: 2069 NeedWaitStates = ST.hasGFX940Insts() 2070 ? isXDL(ST, *MFMA) 2071 ? GFX940_XDL8PassWriteVgprVALUWawWaitStates 2072 : GFX940_SMFMA8PassWriteVgprVALUWawWaitStates 2073 : SMFMA16x16WriteVgprVALUWawWaitStates; 2074 break; 2075 case 16: LLVM_FALLTHROUGH; 2076 default: 2077 NeedWaitStates = isDGEMM(MFMA->getOpcode()) 2078 ? DMFMA16x16WriteVgprVALUWriteWaitStates 2079 : ST.hasGFX940Insts() 2080 ? isXDL(ST, *MFMA) 2081 ? GFX940_XDL16PassWriteVgprVALUWawWaitStates 2082 : GFX940_SMFMA16PassWriteVgprVALUWawWaitStates 2083 : SMFMA32x32WriteVgprVALUWawWaitStates; 2084 break; 2085 } 2086 2087 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 2088 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2089 2090 if (WaitStatesNeeded == MaxWaitStates) 2091 break; 2092 } 2093 2094 auto IsSMFMAReadAsCFn = [&Reg, &IsMFMAFn, &MFMA, 2095 this](const MachineInstr &MI) { 2096 if (!IsMFMAFn(MI) || isDGEMM(MI.getOpcode()) || 2097 !MI.readsRegister(Reg, &TRI)) 2098 return false; 2099 2100 if (ST.hasGFX940Insts() && !isXDL(ST, MI)) 2101 return false; 2102 2103 const MachineOperand *SrcC = 2104 TII.getNamedOperand(MI, AMDGPU::OpName::src2); 2105 assert(SrcC); 2106 if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg)) 2107 return false; 2108 2109 MFMA = &MI; 2110 return true; 2111 }; 2112 2113 MFMA = nullptr; 2114 int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn, 2115 MaxWarWaitStates); 2116 if (!MFMA) 2117 continue; 2118 2119 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); 2120 int NeedWaitStates = MaxWaitStates; 2121 switch (HazardDefLatency) { 2122 case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates; 2123 break; 2124 case 4: assert(ST.hasGFX940Insts()); 2125 NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates; 2126 break; 2127 case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates; 2128 break; 2129 case 16: LLVM_FALLTHROUGH; 2130 default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates; 2131 break; 2132 } 2133 2134 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse; 2135 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2136 } 2137 2138 return WaitStatesNeeded; 2139 } 2140 2141 bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) { 2142 if (!SU->isInstr()) 2143 return false; 2144 2145 const MachineInstr *MAI = nullptr; 2146 auto IsMFMAFn = [&MAI](const MachineInstr &MI) { 2147 MAI = nullptr; 2148 if (SIInstrInfo::isMAI(MI) && 2149 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 2150 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64) 2151 MAI = &MI; 2152 return MAI != nullptr; 2153 }; 2154 2155 MachineInstr *MI = SU->getInstr(); 2156 if (IsMFMAFn(*MI)) { 2157 int W = getWaitStatesSince(IsMFMAFn, 16); 2158 if (MAI) 2159 return W < (int)TSchedModel.computeInstrLatency(MAI); 2160 } 2161 2162 return false; 2163 } 2164