1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements hazard recognizers for scheduling on GCN processors. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "GCNHazardRecognizer.h" 14 #include "GCNSubtarget.h" 15 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 16 #include "SIMachineFunctionInfo.h" 17 #include "llvm/CodeGen/MachineFunction.h" 18 #include "llvm/CodeGen/ScheduleDAG.h" 19 #include "llvm/TargetParser/TargetParser.h" 20 21 using namespace llvm; 22 23 namespace { 24 25 struct MFMAPaddingRatioParser : public cl::parser<unsigned> { 26 MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {} 27 28 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) { 29 if (Arg.getAsInteger(0, Value)) 30 return O.error("'" + Arg + "' value invalid for uint argument!"); 31 32 if (Value > 100) 33 return O.error("'" + Arg + "' value must be in the range [0, 100]!"); 34 35 return false; 36 } 37 }; 38 39 } // end anonymous namespace 40 41 static cl::opt<unsigned, false, MFMAPaddingRatioParser> 42 MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, 43 cl::desc("Fill a percentage of the latency between " 44 "neighboring MFMA with s_nops.")); 45 46 //===----------------------------------------------------------------------===// 47 // Hazard Recognizer Implementation 48 //===----------------------------------------------------------------------===// 49 50 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, 51 const GCNSubtarget &ST); 52 53 GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) : 54 IsHazardRecognizerMode(false), 55 CurrCycleInstr(nullptr), 56 MF(MF), 57 ST(MF.getSubtarget<GCNSubtarget>()), 58 TII(*ST.getInstrInfo()), 59 TRI(TII.getRegisterInfo()), 60 ClauseUses(TRI.getNumRegUnits()), 61 ClauseDefs(TRI.getNumRegUnits()) { 62 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5; 63 TSchedModel.init(&ST); 64 RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST); 65 } 66 67 void GCNHazardRecognizer::Reset() { 68 EmittedInstrs.clear(); 69 } 70 71 void GCNHazardRecognizer::EmitInstruction(SUnit *SU) { 72 EmitInstruction(SU->getInstr()); 73 } 74 75 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) { 76 CurrCycleInstr = MI; 77 } 78 79 static bool isDivFMas(unsigned Opcode) { 80 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64; 81 } 82 83 static bool isSGetReg(unsigned Opcode) { 84 return Opcode == AMDGPU::S_GETREG_B32; 85 } 86 87 static bool isSSetReg(unsigned Opcode) { 88 switch (Opcode) { 89 case AMDGPU::S_SETREG_B32: 90 case AMDGPU::S_SETREG_B32_mode: 91 case AMDGPU::S_SETREG_IMM32_B32: 92 case AMDGPU::S_SETREG_IMM32_B32_mode: 93 return true; 94 } 95 return false; 96 } 97 98 static bool isRWLane(unsigned Opcode) { 99 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32; 100 } 101 102 static bool isRFE(unsigned Opcode) { 103 return Opcode == AMDGPU::S_RFE_B64; 104 } 105 106 static bool isSMovRel(unsigned Opcode) { 107 switch (Opcode) { 108 case AMDGPU::S_MOVRELS_B32: 109 case AMDGPU::S_MOVRELS_B64: 110 case AMDGPU::S_MOVRELD_B32: 111 case AMDGPU::S_MOVRELD_B64: 112 return true; 113 default: 114 return false; 115 } 116 } 117 118 static bool isDGEMM(unsigned Opcode) { 119 return AMDGPU::getMAIIsDGEMM(Opcode); 120 } 121 122 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) { 123 unsigned Opcode = MI.getOpcode(); 124 125 if (!SIInstrInfo::isMAI(MI) || 126 isDGEMM(Opcode) || 127 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 || 128 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64) 129 return false; 130 131 if (!ST.hasGFX940Insts()) 132 return true; 133 134 return AMDGPU::getMAIIsGFX940XDL(Opcode); 135 } 136 137 static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, 138 const MachineInstr &MI) { 139 if (TII.isAlwaysGDS(MI.getOpcode())) 140 return true; 141 142 switch (MI.getOpcode()) { 143 case AMDGPU::S_SENDMSG: 144 case AMDGPU::S_SENDMSGHALT: 145 case AMDGPU::S_TTRACEDATA: 146 return true; 147 // These DS opcodes don't support GDS. 148 case AMDGPU::DS_NOP: 149 case AMDGPU::DS_PERMUTE_B32: 150 case AMDGPU::DS_BPERMUTE_B32: 151 return false; 152 default: 153 if (TII.isDS(MI.getOpcode())) { 154 int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 155 AMDGPU::OpName::gds); 156 if (MI.getOperand(GDS).getImm()) 157 return true; 158 } 159 return false; 160 } 161 } 162 163 static bool isPermlane(const MachineInstr &MI) { 164 unsigned Opcode = MI.getOpcode(); 165 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 || 166 Opcode == AMDGPU::V_PERMLANEX16_B32_e64 || 167 Opcode == AMDGPU::V_PERMLANE16_VAR_B32_e64 || 168 Opcode == AMDGPU::V_PERMLANEX16_VAR_B32_e64; 169 } 170 171 static bool isLdsDma(const MachineInstr &MI) { 172 return SIInstrInfo::isVALU(MI) && 173 (SIInstrInfo::isMUBUF(MI) || SIInstrInfo::isFLAT(MI)); 174 } 175 176 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) { 177 const MachineOperand *RegOp = TII->getNamedOperand(RegInstr, 178 AMDGPU::OpName::simm16); 179 return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_; 180 } 181 182 ScheduleHazardRecognizer::HazardType 183 GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { 184 MachineInstr *MI = SU->getInstr(); 185 // If we are not in "HazardRecognizerMode" and therefore not being run from 186 // the scheduler, track possible stalls from hazards but don't insert noops. 187 auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard; 188 189 if (MI->isBundle()) 190 return NoHazard; 191 192 if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0) 193 return HazardType; 194 195 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0) 196 return HazardType; 197 198 if (checkFPAtomicToDenormModeHazard(MI) > 0) 199 return HazardType; 200 201 if (ST.hasNoDataDepHazard()) 202 return NoHazard; 203 204 // FIXME: Should flat be considered vmem? 205 if ((SIInstrInfo::isVMEM(*MI) || 206 SIInstrInfo::isFLAT(*MI)) 207 && checkVMEMHazards(MI) > 0) 208 return HazardType; 209 210 if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0) 211 return HazardType; 212 213 if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0) 214 return HazardType; 215 216 if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0) 217 return HazardType; 218 219 if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0) 220 return HazardType; 221 222 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || 223 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || 224 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) 225 return HazardType; 226 227 if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0) 228 return HazardType; 229 230 if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0) 231 return HazardType; 232 233 if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0) 234 return HazardType; 235 236 if (((ST.hasReadM0MovRelInterpHazard() && 237 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) || 238 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 || 239 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) || 240 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || 241 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || 242 (ST.hasReadM0LdsDirectHazard() && 243 MI->readsRegister(AMDGPU::LDS_DIRECT))) && 244 checkReadM0Hazards(MI) > 0) 245 return HazardType; 246 247 if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0) 248 return HazardType; 249 250 if ((SIInstrInfo::isVMEM(*MI) || 251 SIInstrInfo::isFLAT(*MI) || 252 SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0) 253 return HazardType; 254 255 if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0) 256 return HazardType; 257 258 return NoHazard; 259 } 260 261 static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, 262 unsigned Quantity) { 263 while (Quantity > 0) { 264 unsigned Arg = std::min(Quantity, 8u); 265 Quantity -= Arg; 266 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP)) 267 .addImm(Arg - 1); 268 } 269 } 270 271 unsigned 272 GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const { 273 const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&MI); 274 assert(TSchedModel.getWriteProcResBegin(SC) != 275 TSchedModel.getWriteProcResEnd(SC)); 276 return TSchedModel.getWriteProcResBegin(SC)->ReleaseAtCycle; 277 } 278 279 void GCNHazardRecognizer::processBundle() { 280 MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator()); 281 MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end(); 282 // Check bundled MachineInstr's for hazards. 283 for (; MI != E && MI->isInsideBundle(); ++MI) { 284 CurrCycleInstr = &*MI; 285 unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr); 286 287 if (IsHazardRecognizerMode) { 288 fixHazards(CurrCycleInstr); 289 290 insertNoopsInBundle(CurrCycleInstr, TII, WaitStates); 291 } 292 293 // It’s unnecessary to track more than MaxLookAhead instructions. Since we 294 // include the bundled MI directly after, only add a maximum of 295 // (MaxLookAhead - 1) noops to EmittedInstrs. 296 for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i) 297 EmittedInstrs.push_front(nullptr); 298 299 EmittedInstrs.push_front(CurrCycleInstr); 300 EmittedInstrs.resize(MaxLookAhead); 301 } 302 CurrCycleInstr = nullptr; 303 } 304 305 void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) { 306 assert(IsHazardRecognizerMode); 307 308 unsigned NumPreNoops = PreEmitNoops(MI); 309 EmitNoops(NumPreNoops); 310 if (MI->isInsideBundle()) 311 insertNoopsInBundle(MI, TII, NumPreNoops); 312 else 313 TII.insertNoops(*MI->getParent(), MachineBasicBlock::iterator(MI), 314 NumPreNoops); 315 EmitInstruction(MI); 316 AdvanceCycle(); 317 } 318 319 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) { 320 IsHazardRecognizerMode = true; 321 CurrCycleInstr = MI; 322 unsigned W = PreEmitNoopsCommon(MI); 323 fixHazards(MI); 324 CurrCycleInstr = nullptr; 325 return W; 326 } 327 328 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { 329 if (MI->isBundle()) 330 return 0; 331 332 int WaitStates = 0; 333 334 if (SIInstrInfo::isSMRD(*MI)) 335 return std::max(WaitStates, checkSMRDHazards(MI)); 336 337 if (ST.hasNSAtoVMEMBug()) 338 WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI)); 339 340 WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI)); 341 342 if (ST.hasNoDataDepHazard()) 343 return WaitStates; 344 345 if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI)) 346 WaitStates = std::max(WaitStates, checkVMEMHazards(MI)); 347 348 if (SIInstrInfo::isVALU(*MI)) 349 WaitStates = std::max(WaitStates, checkVALUHazards(MI)); 350 351 if (SIInstrInfo::isDPP(*MI)) 352 WaitStates = std::max(WaitStates, checkDPPHazards(MI)); 353 354 if (isDivFMas(MI->getOpcode())) 355 WaitStates = std::max(WaitStates, checkDivFMasHazards(MI)); 356 357 if (isRWLane(MI->getOpcode())) 358 WaitStates = std::max(WaitStates, checkRWLaneHazards(MI)); 359 360 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || 361 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || 362 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) 363 WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI)); 364 365 if (MI->isInlineAsm()) 366 return std::max(WaitStates, checkInlineAsmHazards(MI)); 367 368 if (isSGetReg(MI->getOpcode())) 369 return std::max(WaitStates, checkGetRegHazards(MI)); 370 371 if (isSSetReg(MI->getOpcode())) 372 return std::max(WaitStates, checkSetRegHazards(MI)); 373 374 if (isRFE(MI->getOpcode())) 375 return std::max(WaitStates, checkRFEHazards(MI)); 376 377 if ((ST.hasReadM0MovRelInterpHazard() && 378 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) || 379 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 || 380 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) || 381 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || 382 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || 383 (ST.hasReadM0LdsDirectHazard() && MI->readsRegister(AMDGPU::LDS_DIRECT))) 384 return std::max(WaitStates, checkReadM0Hazards(MI)); 385 386 if (SIInstrInfo::isMAI(*MI)) 387 return std::max(WaitStates, checkMAIHazards(MI)); 388 389 if (SIInstrInfo::isVMEM(*MI) || 390 SIInstrInfo::isFLAT(*MI) || 391 SIInstrInfo::isDS(*MI)) 392 return std::max(WaitStates, checkMAILdStHazards(MI)); 393 394 return WaitStates; 395 } 396 397 void GCNHazardRecognizer::EmitNoop() { 398 EmittedInstrs.push_front(nullptr); 399 } 400 401 void GCNHazardRecognizer::AdvanceCycle() { 402 // When the scheduler detects a stall, it will call AdvanceCycle() without 403 // emitting any instructions. 404 if (!CurrCycleInstr) { 405 EmittedInstrs.push_front(nullptr); 406 return; 407 } 408 409 if (CurrCycleInstr->isBundle()) { 410 processBundle(); 411 return; 412 } 413 414 unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr); 415 if (!NumWaitStates) { 416 CurrCycleInstr = nullptr; 417 return; 418 } 419 420 // Keep track of emitted instructions 421 EmittedInstrs.push_front(CurrCycleInstr); 422 423 // Add a nullptr for each additional wait state after the first. Make sure 424 // not to add more than getMaxLookAhead() items to the list, since we 425 // truncate the list to that size right after this loop. 426 for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead()); 427 i < e; ++i) { 428 EmittedInstrs.push_front(nullptr); 429 } 430 431 // getMaxLookahead() is the largest number of wait states we will ever need 432 // to insert, so there is no point in keeping track of more than that many 433 // wait states. 434 EmittedInstrs.resize(getMaxLookAhead()); 435 436 CurrCycleInstr = nullptr; 437 } 438 439 void GCNHazardRecognizer::RecedeCycle() { 440 llvm_unreachable("hazard recognizer does not support bottom-up scheduling."); 441 } 442 443 //===----------------------------------------------------------------------===// 444 // Helper Functions 445 //===----------------------------------------------------------------------===// 446 447 typedef enum { HazardFound, HazardExpired, NoHazardFound } HazardFnResult; 448 449 typedef function_ref<bool(const MachineInstr &, int WaitStates)> IsExpiredFn; 450 typedef function_ref<unsigned int(const MachineInstr &)> GetNumWaitStatesFn; 451 452 // Search for a hazard in a block and its predecessors. 453 template <typename StateT> 454 static bool 455 hasHazard(StateT State, 456 function_ref<HazardFnResult(StateT &, const MachineInstr &)> IsHazard, 457 function_ref<void(StateT &, const MachineInstr &)> UpdateState, 458 const MachineBasicBlock *MBB, 459 MachineBasicBlock::const_reverse_instr_iterator I, 460 DenseSet<const MachineBasicBlock *> &Visited) { 461 for (auto E = MBB->instr_rend(); I != E; ++I) { 462 // No need to look at parent BUNDLE instructions. 463 if (I->isBundle()) 464 continue; 465 466 switch (IsHazard(State, *I)) { 467 case HazardFound: 468 return true; 469 case HazardExpired: 470 return false; 471 default: 472 // Continue search 473 break; 474 } 475 476 if (I->isInlineAsm() || I->isMetaInstruction()) 477 continue; 478 479 UpdateState(State, *I); 480 } 481 482 for (MachineBasicBlock *Pred : MBB->predecessors()) { 483 if (!Visited.insert(Pred).second) 484 continue; 485 486 if (hasHazard(State, IsHazard, UpdateState, Pred, Pred->instr_rbegin(), 487 Visited)) 488 return true; 489 } 490 491 return false; 492 } 493 494 // Returns a minimum wait states since \p I walking all predecessors. 495 // Only scans until \p IsExpired does not return true. 496 // Can only be run in a hazard recognizer mode. 497 static int getWaitStatesSince( 498 GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock *MBB, 499 MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates, 500 IsExpiredFn IsExpired, DenseSet<const MachineBasicBlock *> &Visited, 501 GetNumWaitStatesFn GetNumWaitStates = SIInstrInfo::getNumWaitStates) { 502 for (auto E = MBB->instr_rend(); I != E; ++I) { 503 // Don't add WaitStates for parent BUNDLE instructions. 504 if (I->isBundle()) 505 continue; 506 507 if (IsHazard(*I)) 508 return WaitStates; 509 510 if (I->isInlineAsm()) 511 continue; 512 513 WaitStates += GetNumWaitStates(*I); 514 515 if (IsExpired(*I, WaitStates)) 516 return std::numeric_limits<int>::max(); 517 } 518 519 int MinWaitStates = std::numeric_limits<int>::max(); 520 for (MachineBasicBlock *Pred : MBB->predecessors()) { 521 if (!Visited.insert(Pred).second) 522 continue; 523 524 int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(), WaitStates, 525 IsExpired, Visited, GetNumWaitStates); 526 527 MinWaitStates = std::min(MinWaitStates, W); 528 } 529 530 return MinWaitStates; 531 } 532 533 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, 534 const MachineInstr *MI, IsExpiredFn IsExpired) { 535 DenseSet<const MachineBasicBlock *> Visited; 536 return getWaitStatesSince(IsHazard, MI->getParent(), 537 std::next(MI->getReverseIterator()), 538 0, IsExpired, Visited); 539 } 540 541 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) { 542 if (IsHazardRecognizerMode) { 543 auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) { 544 return WaitStates >= Limit; 545 }; 546 return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn); 547 } 548 549 int WaitStates = 0; 550 for (MachineInstr *MI : EmittedInstrs) { 551 if (MI) { 552 if (IsHazard(*MI)) 553 return WaitStates; 554 555 if (MI->isInlineAsm()) 556 continue; 557 } 558 ++WaitStates; 559 560 if (WaitStates >= Limit) 561 break; 562 } 563 return std::numeric_limits<int>::max(); 564 } 565 566 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg, 567 IsHazardFn IsHazardDef, 568 int Limit) { 569 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 570 571 auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) { 572 return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI); 573 }; 574 575 return getWaitStatesSince(IsHazardFn, Limit); 576 } 577 578 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard, 579 int Limit) { 580 auto IsHazardFn = [IsHazard](const MachineInstr &MI) { 581 return isSSetReg(MI.getOpcode()) && IsHazard(MI); 582 }; 583 584 return getWaitStatesSince(IsHazardFn, Limit); 585 } 586 587 //===----------------------------------------------------------------------===// 588 // No-op Hazard Detection 589 //===----------------------------------------------------------------------===// 590 591 static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, 592 MCRegister Reg) { 593 for (MCRegUnit Unit : TRI.regunits(Reg)) 594 BV.set(Unit); 595 } 596 597 static void addRegsToSet(const SIRegisterInfo &TRI, 598 iterator_range<MachineInstr::const_mop_iterator> Ops, 599 BitVector &DefSet, BitVector &UseSet) { 600 for (const MachineOperand &Op : Ops) { 601 if (Op.isReg()) 602 addRegUnits(TRI, Op.isDef() ? DefSet : UseSet, Op.getReg().asMCReg()); 603 } 604 } 605 606 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) { 607 addRegsToSet(TRI, MI.operands(), ClauseDefs, ClauseUses); 608 } 609 610 static bool breaksSMEMSoftClause(MachineInstr *MI) { 611 return !SIInstrInfo::isSMRD(*MI); 612 } 613 614 static bool breaksVMEMSoftClause(MachineInstr *MI) { 615 return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI); 616 } 617 618 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) { 619 // SMEM soft clause are only present on VI+, and only matter if xnack is 620 // enabled. 621 if (!ST.isXNACKEnabled()) 622 return 0; 623 624 bool IsSMRD = TII.isSMRD(*MEM); 625 626 resetClause(); 627 628 // A soft-clause is any group of consecutive SMEM instructions. The 629 // instructions in this group may return out of order and/or may be 630 // replayed (i.e. the same instruction issued more than once). 631 // 632 // In order to handle these situations correctly we need to make sure that 633 // when a clause has more than one instruction, no instruction in the clause 634 // writes to a register that is read by another instruction in the clause 635 // (including itself). If we encounter this situation, we need to break the 636 // clause by inserting a non SMEM instruction. 637 638 for (MachineInstr *MI : EmittedInstrs) { 639 // When we hit a non-SMEM instruction then we have passed the start of the 640 // clause and we can stop. 641 if (!MI) 642 break; 643 644 if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI)) 645 break; 646 647 addClauseInst(*MI); 648 } 649 650 if (ClauseDefs.none()) 651 return 0; 652 653 // We need to make sure not to put loads and stores in the same clause if they 654 // use the same address. For now, just start a new clause whenever we see a 655 // store. 656 if (MEM->mayStore()) 657 return 1; 658 659 addClauseInst(*MEM); 660 661 // If the set of defs and uses intersect then we cannot add this instruction 662 // to the clause, so we have a hazard. 663 return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0; 664 } 665 666 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) { 667 int WaitStatesNeeded = 0; 668 669 WaitStatesNeeded = checkSoftClauseHazards(SMRD); 670 671 // This SMRD hazard only affects SI. 672 if (!ST.hasSMRDReadVALUDefHazard()) 673 return WaitStatesNeeded; 674 675 // A read of an SGPR by SMRD instruction requires 4 wait states when the 676 // SGPR was written by a VALU instruction. 677 int SmrdSgprWaitStates = 4; 678 auto IsHazardDefFn = [this](const MachineInstr &MI) { 679 return TII.isVALU(MI); 680 }; 681 auto IsBufferHazardDefFn = [this](const MachineInstr &MI) { 682 return TII.isSALU(MI); 683 }; 684 685 bool IsBufferSMRD = TII.isBufferSMRD(*SMRD); 686 687 for (const MachineOperand &Use : SMRD->uses()) { 688 if (!Use.isReg()) 689 continue; 690 int WaitStatesNeededForUse = 691 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn, 692 SmrdSgprWaitStates); 693 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 694 695 // This fixes what appears to be undocumented hardware behavior in SI where 696 // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor 697 // needs some number of nops in between. We don't know how many we need, but 698 // let's use 4. This wasn't discovered before probably because the only 699 // case when this happens is when we expand a 64-bit pointer into a full 700 // descriptor and use s_buffer_load_dword instead of s_load_dword, which was 701 // probably never encountered in the closed-source land. 702 if (IsBufferSMRD) { 703 int WaitStatesNeededForUse = 704 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), 705 IsBufferHazardDefFn, 706 SmrdSgprWaitStates); 707 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 708 } 709 } 710 711 return WaitStatesNeeded; 712 } 713 714 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) { 715 if (!ST.hasVMEMReadSGPRVALUDefHazard()) 716 return 0; 717 718 int WaitStatesNeeded = checkSoftClauseHazards(VMEM); 719 720 // A read of an SGPR by a VMEM instruction requires 5 wait states when the 721 // SGPR was written by a VALU Instruction. 722 const int VmemSgprWaitStates = 5; 723 auto IsHazardDefFn = [this](const MachineInstr &MI) { 724 return TII.isVALU(MI); 725 }; 726 for (const MachineOperand &Use : VMEM->uses()) { 727 if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg())) 728 continue; 729 730 int WaitStatesNeededForUse = 731 VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn, 732 VmemSgprWaitStates); 733 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 734 } 735 return WaitStatesNeeded; 736 } 737 738 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) { 739 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 740 const SIInstrInfo *TII = ST.getInstrInfo(); 741 742 // Check for DPP VGPR read after VALU VGPR write and EXEC write. 743 int DppVgprWaitStates = 2; 744 int DppExecWaitStates = 5; 745 int WaitStatesNeeded = 0; 746 auto IsHazardDefFn = [TII](const MachineInstr &MI) { 747 return TII->isVALU(MI); 748 }; 749 750 for (const MachineOperand &Use : DPP->uses()) { 751 if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg())) 752 continue; 753 int WaitStatesNeededForUse = 754 DppVgprWaitStates - getWaitStatesSinceDef( 755 Use.getReg(), 756 [](const MachineInstr &) { return true; }, 757 DppVgprWaitStates); 758 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 759 } 760 761 WaitStatesNeeded = std::max( 762 WaitStatesNeeded, 763 DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn, 764 DppExecWaitStates)); 765 766 return WaitStatesNeeded; 767 } 768 769 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) { 770 const SIInstrInfo *TII = ST.getInstrInfo(); 771 772 // v_div_fmas requires 4 wait states after a write to vcc from a VALU 773 // instruction. 774 const int DivFMasWaitStates = 4; 775 auto IsHazardDefFn = [TII](const MachineInstr &MI) { 776 return TII->isVALU(MI); 777 }; 778 int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn, 779 DivFMasWaitStates); 780 781 return DivFMasWaitStates - WaitStatesNeeded; 782 } 783 784 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) { 785 const SIInstrInfo *TII = ST.getInstrInfo(); 786 unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr); 787 788 const int GetRegWaitStates = 2; 789 auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) { 790 return GetRegHWReg == getHWReg(TII, MI); 791 }; 792 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates); 793 794 return GetRegWaitStates - WaitStatesNeeded; 795 } 796 797 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) { 798 const SIInstrInfo *TII = ST.getInstrInfo(); 799 unsigned HWReg = getHWReg(TII, *SetRegInstr); 800 801 const int SetRegWaitStates = ST.getSetRegWaitStates(); 802 auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) { 803 return HWReg == getHWReg(TII, MI); 804 }; 805 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates); 806 return SetRegWaitStates - WaitStatesNeeded; 807 } 808 809 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) { 810 if (!MI.mayStore()) 811 return -1; 812 813 const SIInstrInfo *TII = ST.getInstrInfo(); 814 unsigned Opcode = MI.getOpcode(); 815 const MCInstrDesc &Desc = MI.getDesc(); 816 817 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); 818 int VDataRCID = -1; 819 if (VDataIdx != -1) 820 VDataRCID = Desc.operands()[VDataIdx].RegClass; 821 822 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) { 823 // There is no hazard if the instruction does not use vector regs 824 // (like wbinvl1) 825 if (VDataIdx == -1) 826 return -1; 827 // For MUBUF/MTBUF instructions this hazard only exists if the 828 // instruction is not using a register in the soffset field. 829 const MachineOperand *SOffset = 830 TII->getNamedOperand(MI, AMDGPU::OpName::soffset); 831 // If we have no soffset operand, then assume this field has been 832 // hardcoded to zero. 833 if (AMDGPU::getRegBitWidth(VDataRCID) > 64 && 834 (!SOffset || !SOffset->isReg())) 835 return VDataIdx; 836 } 837 838 // MIMG instructions create a hazard if they don't use a 256-bit T# and 839 // the store size is greater than 8 bytes and they have more than two bits 840 // of their dmask set. 841 // All our MIMG definitions use a 256-bit T#, so we can skip checking for them. 842 if (TII->isMIMG(MI)) { 843 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); 844 assert(SRsrcIdx != -1 && 845 AMDGPU::getRegBitWidth(Desc.operands()[SRsrcIdx].RegClass) == 256); 846 (void)SRsrcIdx; 847 } 848 849 if (TII->isFLAT(MI)) { 850 int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); 851 if (AMDGPU::getRegBitWidth(Desc.operands()[DataIdx].RegClass) > 64) 852 return DataIdx; 853 } 854 855 return -1; 856 } 857 858 int 859 GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def, 860 const MachineRegisterInfo &MRI) { 861 // Helper to check for the hazard where VMEM instructions that store more than 862 // 8 bytes can have there store data over written by the next instruction. 863 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 864 865 const int VALUWaitStates = ST.hasGFX940Insts() ? 2 : 1; 866 int WaitStatesNeeded = 0; 867 868 if (!TRI->isVectorRegister(MRI, Def.getReg())) 869 return WaitStatesNeeded; 870 Register Reg = Def.getReg(); 871 auto IsHazardFn = [this, Reg, TRI](const MachineInstr &MI) { 872 int DataIdx = createsVALUHazard(MI); 873 return DataIdx >= 0 && 874 TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg); 875 }; 876 int WaitStatesNeededForDef = 877 VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates); 878 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 879 880 return WaitStatesNeeded; 881 } 882 883 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) { 884 int WaitStatesNeeded = 0; 885 886 if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(*VALU)) { 887 const int TransDefWaitstates = 1; 888 889 auto IsTransDefFn = [this, VALU](const MachineInstr &MI) { 890 if (!SIInstrInfo::isTRANS(MI)) 891 return false; 892 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 893 const SIInstrInfo *TII = ST.getInstrInfo(); 894 Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg(); 895 896 for (const MachineOperand &Use : VALU->explicit_uses()) { 897 if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg())) 898 return true; 899 } 900 901 return false; 902 }; 903 904 int WaitStatesNeededForDef = 905 TransDefWaitstates - 906 getWaitStatesSince(IsTransDefFn, TransDefWaitstates); 907 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 908 } 909 910 if (ST.hasDstSelForwardingHazard()) { 911 const int Shift16DefWaitstates = 1; 912 913 auto IsShift16BitDefFn = [this, VALU](const MachineInstr &MI) { 914 if (!SIInstrInfo::isVALU(MI)) 915 return false; 916 const SIInstrInfo *TII = ST.getInstrInfo(); 917 if (SIInstrInfo::isSDWA(MI)) { 918 if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel)) 919 if (DstSel->getImm() == AMDGPU::SDWA::DWORD) 920 return false; 921 } else { 922 if (!AMDGPU::hasNamedOperand(MI.getOpcode(), AMDGPU::OpName::op_sel) || 923 !(TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers) 924 ->getImm() & 925 SISrcMods::DST_OP_SEL)) 926 return false; 927 } 928 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 929 if (auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) { 930 Register Def = Dst->getReg(); 931 932 for (const MachineOperand &Use : VALU->explicit_uses()) { 933 if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg())) 934 return true; 935 } 936 } 937 938 return false; 939 }; 940 941 int WaitStatesNeededForDef = 942 Shift16DefWaitstates - 943 getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates); 944 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 945 } 946 947 if (ST.hasVDecCoExecHazard()) { 948 const int VALUWriteSGPRVALUReadWaitstates = 2; 949 const int VALUWriteEXECRWLane = 4; 950 const int VALUWriteVGPRReadlaneRead = 1; 951 952 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 953 const MachineRegisterInfo &MRI = MF.getRegInfo(); 954 Register UseReg; 955 auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) { 956 if (!SIInstrInfo::isVALU(MI)) 957 return false; 958 return MI.modifiesRegister(UseReg, TRI); 959 }; 960 961 for (const MachineOperand &Use : VALU->explicit_uses()) { 962 if (!Use.isReg()) 963 continue; 964 965 UseReg = Use.getReg(); 966 if (TRI->isSGPRReg(MRI, UseReg)) { 967 int WaitStatesNeededForDef = 968 VALUWriteSGPRVALUReadWaitstates - 969 getWaitStatesSince(IsVALUDefSGPRFn, 970 VALUWriteSGPRVALUReadWaitstates); 971 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 972 } 973 } 974 975 if (VALU->readsRegister(AMDGPU::VCC, TRI)) { 976 UseReg = AMDGPU::VCC; 977 int WaitStatesNeededForDef = 978 VALUWriteSGPRVALUReadWaitstates - 979 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteSGPRVALUReadWaitstates); 980 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 981 } 982 983 switch (VALU->getOpcode()) { 984 case AMDGPU::V_READLANE_B32: 985 case AMDGPU::V_READFIRSTLANE_B32: { 986 MachineOperand *Src = TII.getNamedOperand(*VALU, AMDGPU::OpName::src0); 987 UseReg = Src->getReg(); 988 int WaitStatesNeededForDef = 989 VALUWriteVGPRReadlaneRead - 990 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteVGPRReadlaneRead); 991 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 992 } 993 [[fallthrough]]; 994 case AMDGPU::V_WRITELANE_B32: { 995 UseReg = AMDGPU::EXEC; 996 int WaitStatesNeededForDef = 997 VALUWriteEXECRWLane - 998 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteEXECRWLane); 999 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 1000 break; 1001 } 1002 default: 1003 break; 1004 } 1005 } 1006 1007 // This checks for the hazard where VMEM instructions that store more than 1008 // 8 bytes can have there store data over written by the next instruction. 1009 if (!ST.has12DWordStoreHazard()) 1010 return WaitStatesNeeded; 1011 1012 const MachineRegisterInfo &MRI = MF.getRegInfo(); 1013 1014 for (const MachineOperand &Def : VALU->defs()) { 1015 WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI)); 1016 } 1017 1018 return WaitStatesNeeded; 1019 } 1020 1021 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) { 1022 // This checks for hazards associated with inline asm statements. 1023 // Since inline asms can contain just about anything, we use this 1024 // to call/leverage other check*Hazard routines. Note that 1025 // this function doesn't attempt to address all possible inline asm 1026 // hazards (good luck), but is a collection of what has been 1027 // problematic thus far. 1028 1029 // see checkVALUHazards() 1030 if (!ST.has12DWordStoreHazard()) 1031 return 0; 1032 1033 const MachineRegisterInfo &MRI = MF.getRegInfo(); 1034 int WaitStatesNeeded = 0; 1035 1036 for (const MachineOperand &Op : 1037 llvm::drop_begin(IA->operands(), InlineAsm::MIOp_FirstOperand)) { 1038 if (Op.isReg() && Op.isDef()) { 1039 WaitStatesNeeded = 1040 std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI)); 1041 } 1042 } 1043 1044 return WaitStatesNeeded; 1045 } 1046 1047 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) { 1048 const SIInstrInfo *TII = ST.getInstrInfo(); 1049 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1050 const MachineRegisterInfo &MRI = MF.getRegInfo(); 1051 1052 const MachineOperand *LaneSelectOp = 1053 TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1); 1054 1055 if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg())) 1056 return 0; 1057 1058 Register LaneSelectReg = LaneSelectOp->getReg(); 1059 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); }; 1060 1061 const int RWLaneWaitStates = 4; 1062 int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn, 1063 RWLaneWaitStates); 1064 return RWLaneWaitStates - WaitStatesSince; 1065 } 1066 1067 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) { 1068 if (!ST.hasRFEHazards()) 1069 return 0; 1070 1071 const SIInstrInfo *TII = ST.getInstrInfo(); 1072 1073 const int RFEWaitStates = 1; 1074 1075 auto IsHazardFn = [TII](const MachineInstr &MI) { 1076 return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS; 1077 }; 1078 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates); 1079 return RFEWaitStates - WaitStatesNeeded; 1080 } 1081 1082 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) { 1083 const SIInstrInfo *TII = ST.getInstrInfo(); 1084 const int ReadM0WaitStates = 1; 1085 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); }; 1086 return ReadM0WaitStates - 1087 getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, ReadM0WaitStates); 1088 } 1089 1090 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) { 1091 fixVMEMtoScalarWriteHazards(MI); 1092 fixVcmpxPermlaneHazards(MI); 1093 fixSMEMtoVectorWriteHazards(MI); 1094 fixVcmpxExecWARHazard(MI); 1095 fixLdsBranchVmemWARHazard(MI); 1096 if (ST.hasLdsDirect()) { 1097 fixLdsDirectVALUHazard(MI); 1098 fixLdsDirectVMEMHazard(MI); 1099 } 1100 fixVALUPartialForwardingHazard(MI); 1101 fixVALUTransUseHazard(MI); 1102 fixWMMAHazards(MI); 1103 fixShift64HighRegBug(MI); 1104 fixVALUMaskWriteHazard(MI); 1105 } 1106 1107 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) { 1108 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI)) 1109 return false; 1110 1111 const SIInstrInfo *TII = ST.getInstrInfo(); 1112 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1113 auto IsHazardFn = [TII, TRI](const MachineInstr &MI) { 1114 return (TII->isVOPC(MI) || 1115 ((TII->isVOP3(MI) || TII->isSDWA(MI)) && MI.isCompare())) && 1116 MI.modifiesRegister(AMDGPU::EXEC, TRI); 1117 }; 1118 1119 auto IsExpiredFn = [](const MachineInstr &MI, int) { 1120 unsigned Opc = MI.getOpcode(); 1121 return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 && 1122 Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa; 1123 }; 1124 1125 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1126 std::numeric_limits<int>::max()) 1127 return false; 1128 1129 // V_NOP will be discarded by SQ. 1130 // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE* 1131 // which is always a VGPR and available. 1132 auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0); 1133 Register Reg = Src0->getReg(); 1134 bool IsUndef = Src0->isUndef(); 1135 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1136 TII->get(AMDGPU::V_MOV_B32_e32)) 1137 .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0)) 1138 .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill); 1139 1140 return true; 1141 } 1142 1143 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) { 1144 if (!ST.hasVMEMtoScalarWriteHazard()) 1145 return false; 1146 1147 if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI)) 1148 return false; 1149 1150 if (MI->getNumDefs() == 0) 1151 return false; 1152 1153 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1154 1155 auto IsHazardFn = [TRI, MI](const MachineInstr &I) { 1156 if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I) && 1157 !SIInstrInfo::isFLAT(I)) 1158 return false; 1159 1160 for (const MachineOperand &Def : MI->defs()) { 1161 const MachineOperand *Op = 1162 I.findRegisterUseOperand(Def.getReg(), false, TRI); 1163 if (!Op) 1164 continue; 1165 return true; 1166 } 1167 return false; 1168 }; 1169 1170 auto IsExpiredFn = [](const MachineInstr &MI, int) { 1171 return SIInstrInfo::isVALU(MI) || 1172 (MI.getOpcode() == AMDGPU::S_WAITCNT && 1173 !MI.getOperand(0).getImm()) || 1174 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1175 AMDGPU::DepCtr::decodeFieldVmVsrc(MI.getOperand(0).getImm()) == 0); 1176 }; 1177 1178 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1179 std::numeric_limits<int>::max()) 1180 return false; 1181 1182 const SIInstrInfo *TII = ST.getInstrInfo(); 1183 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1184 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) 1185 .addImm(AMDGPU::DepCtr::encodeFieldVmVsrc(0)); 1186 return true; 1187 } 1188 1189 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) { 1190 if (!ST.hasSMEMtoVectorWriteHazard()) 1191 return false; 1192 1193 if (!SIInstrInfo::isVALU(*MI)) 1194 return false; 1195 1196 unsigned SDSTName; 1197 switch (MI->getOpcode()) { 1198 case AMDGPU::V_READLANE_B32: 1199 case AMDGPU::V_READFIRSTLANE_B32: 1200 SDSTName = AMDGPU::OpName::vdst; 1201 break; 1202 default: 1203 SDSTName = AMDGPU::OpName::sdst; 1204 break; 1205 } 1206 1207 const SIInstrInfo *TII = ST.getInstrInfo(); 1208 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1209 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU()); 1210 const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName); 1211 if (!SDST) { 1212 for (const auto &MO : MI->implicit_operands()) { 1213 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegBaseClass(MO.getReg()))) { 1214 SDST = &MO; 1215 break; 1216 } 1217 } 1218 } 1219 1220 if (!SDST) 1221 return false; 1222 1223 const Register SDSTReg = SDST->getReg(); 1224 auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) { 1225 return SIInstrInfo::isSMRD(I) && I.readsRegister(SDSTReg, TRI); 1226 }; 1227 1228 auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) { 1229 if (TII->isSALU(MI)) { 1230 switch (MI.getOpcode()) { 1231 case AMDGPU::S_SETVSKIP: 1232 case AMDGPU::S_VERSION: 1233 case AMDGPU::S_WAITCNT_VSCNT: 1234 case AMDGPU::S_WAITCNT_VMCNT: 1235 case AMDGPU::S_WAITCNT_EXPCNT: 1236 // These instructions cannot not mitigate the hazard. 1237 return false; 1238 case AMDGPU::S_WAITCNT_LGKMCNT: 1239 // Reducing lgkmcnt count to 0 always mitigates the hazard. 1240 return (MI.getOperand(1).getImm() == 0) && 1241 (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL); 1242 case AMDGPU::S_WAITCNT: { 1243 const int64_t Imm = MI.getOperand(0).getImm(); 1244 AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm); 1245 return (Decoded.LgkmCnt == 0); 1246 } 1247 default: 1248 // SOPP instructions cannot mitigate the hazard. 1249 if (TII->isSOPP(MI)) 1250 return false; 1251 // At this point the SALU can be assumed to mitigate the hazard 1252 // because either: 1253 // (a) it is independent of the at risk SMEM (breaking chain), 1254 // or 1255 // (b) it is dependent on the SMEM, in which case an appropriate 1256 // s_waitcnt lgkmcnt _must_ exist between it and the at risk 1257 // SMEM instruction. 1258 return true; 1259 } 1260 } 1261 return false; 1262 }; 1263 1264 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1265 std::numeric_limits<int>::max()) 1266 return false; 1267 1268 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1269 TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL) 1270 .addImm(0); 1271 return true; 1272 } 1273 1274 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) { 1275 if (!ST.hasVcmpxExecWARHazard() || !SIInstrInfo::isVALU(*MI)) 1276 return false; 1277 1278 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1279 if (!MI->modifiesRegister(AMDGPU::EXEC, TRI)) 1280 return false; 1281 1282 auto IsHazardFn = [TRI](const MachineInstr &I) { 1283 if (SIInstrInfo::isVALU(I)) 1284 return false; 1285 return I.readsRegister(AMDGPU::EXEC, TRI); 1286 }; 1287 1288 const SIInstrInfo *TII = ST.getInstrInfo(); 1289 auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) { 1290 if (SIInstrInfo::isVALU(MI)) { 1291 if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) 1292 return true; 1293 for (auto MO : MI.implicit_operands()) 1294 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegBaseClass(MO.getReg()))) 1295 return true; 1296 } 1297 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1298 AMDGPU::DepCtr::decodeFieldSaSdst(MI.getOperand(0).getImm()) == 0) 1299 return true; 1300 return false; 1301 }; 1302 1303 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1304 std::numeric_limits<int>::max()) 1305 return false; 1306 1307 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1308 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) 1309 .addImm(AMDGPU::DepCtr::encodeFieldSaSdst(0)); 1310 return true; 1311 } 1312 1313 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, 1314 const GCNSubtarget &ST) { 1315 if (!ST.hasLdsBranchVmemWARHazard()) 1316 return false; 1317 1318 // Check if the necessary condition for the hazard is met: both LDS and VMEM 1319 // instructions need to appear in the same function. 1320 bool HasLds = false; 1321 bool HasVmem = false; 1322 for (auto &MBB : MF) { 1323 for (auto &MI : MBB) { 1324 HasLds |= SIInstrInfo::isDS(MI); 1325 HasVmem |= 1326 SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI); 1327 if (HasLds && HasVmem) 1328 return true; 1329 } 1330 } 1331 return false; 1332 } 1333 1334 static bool isStoreCountWaitZero(const MachineInstr &I) { 1335 return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT && 1336 I.getOperand(0).getReg() == AMDGPU::SGPR_NULL && 1337 !I.getOperand(1).getImm(); 1338 } 1339 1340 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) { 1341 if (!RunLdsBranchVmemWARHazardFixup) 1342 return false; 1343 1344 assert(ST.hasLdsBranchVmemWARHazard()); 1345 1346 auto IsHazardInst = [](const MachineInstr &MI) { 1347 if (SIInstrInfo::isDS(MI)) 1348 return 1; 1349 if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI)) 1350 return 2; 1351 return 0; 1352 }; 1353 1354 auto InstType = IsHazardInst(*MI); 1355 if (!InstType) 1356 return false; 1357 1358 auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) { 1359 return IsHazardInst(I) || isStoreCountWaitZero(I); 1360 }; 1361 1362 auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) { 1363 if (!I.isBranch()) 1364 return false; 1365 1366 auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) { 1367 auto InstType2 = IsHazardInst(I); 1368 return InstType2 && InstType != InstType2; 1369 }; 1370 1371 auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) { 1372 auto InstType2 = IsHazardInst(I); 1373 if (InstType == InstType2) 1374 return true; 1375 1376 return isStoreCountWaitZero(I); 1377 }; 1378 1379 return ::getWaitStatesSince(IsHazardFn, &I, IsExpiredFn) != 1380 std::numeric_limits<int>::max(); 1381 }; 1382 1383 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1384 std::numeric_limits<int>::max()) 1385 return false; 1386 1387 const SIInstrInfo *TII = ST.getInstrInfo(); 1388 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1389 TII->get(AMDGPU::S_WAITCNT_VSCNT)) 1390 .addReg(AMDGPU::SGPR_NULL, RegState::Undef) 1391 .addImm(0); 1392 1393 return true; 1394 } 1395 1396 bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) { 1397 if (!SIInstrInfo::isLDSDIR(*MI)) 1398 return false; 1399 1400 const int NoHazardWaitStates = 15; 1401 const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst); 1402 const Register VDSTReg = VDST->getReg(); 1403 1404 bool VisitedTrans = false; 1405 auto IsHazardFn = [this, VDSTReg, &VisitedTrans](const MachineInstr &I) { 1406 if (!SIInstrInfo::isVALU(I)) 1407 return false; 1408 VisitedTrans = VisitedTrans || SIInstrInfo::isTRANS(I); 1409 // Cover both WAR and WAW 1410 return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI); 1411 }; 1412 auto IsExpiredFn = [&](const MachineInstr &I, int WaitStates) { 1413 if (WaitStates >= NoHazardWaitStates) 1414 return true; 1415 // Instructions which cause va_vdst==0 expire hazard 1416 return SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) || 1417 SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I); 1418 }; 1419 auto GetWaitStatesFn = [](const MachineInstr &MI) { 1420 return SIInstrInfo::isVALU(MI) ? 1 : 0; 1421 }; 1422 1423 DenseSet<const MachineBasicBlock *> Visited; 1424 auto Count = ::getWaitStatesSince(IsHazardFn, MI->getParent(), 1425 std::next(MI->getReverseIterator()), 0, 1426 IsExpiredFn, Visited, GetWaitStatesFn); 1427 1428 // Transcendentals can execute in parallel to other VALUs. 1429 // This makes va_vdst count unusable with a mixture of VALU and TRANS. 1430 if (VisitedTrans) 1431 Count = 0; 1432 1433 MachineOperand *WaitVdstOp = 1434 TII.getNamedOperand(*MI, AMDGPU::OpName::waitvdst); 1435 WaitVdstOp->setImm(std::min(Count, NoHazardWaitStates)); 1436 1437 return true; 1438 } 1439 1440 bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) { 1441 if (!SIInstrInfo::isLDSDIR(*MI)) 1442 return false; 1443 1444 const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst); 1445 const Register VDSTReg = VDST->getReg(); 1446 1447 auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) { 1448 if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I) && 1449 !SIInstrInfo::isDS(I)) 1450 return false; 1451 return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI); 1452 }; 1453 bool LdsdirCanWait = ST.hasLdsWaitVMSRC(); 1454 auto IsExpiredFn = [this, LdsdirCanWait](const MachineInstr &I, int) { 1455 return SIInstrInfo::isVALU(I) || SIInstrInfo::isEXP(I) || 1456 (I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(0).getImm()) || 1457 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1458 AMDGPU::DepCtr::decodeFieldVmVsrc(I.getOperand(0).getImm()) == 0) || 1459 (LdsdirCanWait && SIInstrInfo::isLDSDIR(I) && 1460 !TII.getNamedOperand(I, AMDGPU::OpName::waitvsrc)->getImm()); 1461 }; 1462 1463 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1464 std::numeric_limits<int>::max()) 1465 return false; 1466 1467 if (LdsdirCanWait) { 1468 TII.getNamedOperand(*MI, AMDGPU::OpName::waitvsrc)->setImm(0); 1469 } else { 1470 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1471 TII.get(AMDGPU::S_WAITCNT_DEPCTR)) 1472 .addImm(AMDGPU::DepCtr::encodeFieldVmVsrc(0)); 1473 } 1474 1475 return true; 1476 } 1477 1478 bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) { 1479 if (!ST.isWave64()) 1480 return false; 1481 if (!ST.hasVALUPartialForwardingHazard()) 1482 return false; 1483 if (!SIInstrInfo::isVALU(*MI)) 1484 return false; 1485 1486 SmallSetVector<Register, 4> SrcVGPRs; 1487 1488 for (const MachineOperand &Use : MI->explicit_uses()) { 1489 if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg())) 1490 SrcVGPRs.insert(Use.getReg()); 1491 } 1492 1493 // Only applies with >= 2 unique VGPR sources 1494 if (SrcVGPRs.size() <= 1) 1495 return false; 1496 1497 // Look for the following pattern: 1498 // Va <- VALU [PreExecPos] 1499 // intv1 1500 // Exec <- SALU [ExecPos] 1501 // intv2 1502 // Vb <- VALU [PostExecPos] 1503 // intv3 1504 // MI Va, Vb (WaitState = 0) 1505 // 1506 // Where: 1507 // intv1 + intv2 <= 2 VALUs 1508 // intv3 <= 4 VALUs 1509 // 1510 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI. 1511 1512 const int Intv1plus2MaxVALUs = 2; 1513 const int Intv3MaxVALUs = 4; 1514 const int IntvMaxVALUs = 6; 1515 const int NoHazardVALUWaitStates = IntvMaxVALUs + 2; 1516 1517 struct StateType { 1518 SmallDenseMap<Register, int, 4> DefPos; 1519 int ExecPos = std::numeric_limits<int>::max(); 1520 int VALUs = 0; 1521 }; 1522 1523 StateType State; 1524 1525 // This overloads expiry testing with all the hazard detection 1526 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) { 1527 // Too many VALU states have passed 1528 if (State.VALUs > NoHazardVALUWaitStates) 1529 return HazardExpired; 1530 1531 // Instructions which cause va_vdst==0 expire hazard 1532 if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) || 1533 SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I) || 1534 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1535 AMDGPU::DepCtr::decodeFieldVaVdst(I.getOperand(0).getImm()) == 0)) 1536 return HazardExpired; 1537 1538 // Track registers writes 1539 bool Changed = false; 1540 if (SIInstrInfo::isVALU(I)) { 1541 for (Register Src : SrcVGPRs) { 1542 if (!State.DefPos.count(Src) && I.modifiesRegister(Src, &TRI)) { 1543 State.DefPos[Src] = State.VALUs; 1544 Changed = true; 1545 } 1546 } 1547 } else if (SIInstrInfo::isSALU(I)) { 1548 if (State.ExecPos == std::numeric_limits<int>::max()) { 1549 if (!State.DefPos.empty() && I.modifiesRegister(AMDGPU::EXEC, &TRI)) { 1550 State.ExecPos = State.VALUs; 1551 Changed = true; 1552 } 1553 } 1554 } 1555 1556 // Early expiration: too many VALUs in intv3 1557 if (State.VALUs > Intv3MaxVALUs && State.DefPos.empty()) 1558 return HazardExpired; 1559 1560 // Only evaluate state if something changed 1561 if (!Changed) 1562 return NoHazardFound; 1563 1564 // Determine positions of VALUs pre/post exec change 1565 if (State.ExecPos == std::numeric_limits<int>::max()) 1566 return NoHazardFound; 1567 1568 int PreExecPos = std::numeric_limits<int>::max(); 1569 int PostExecPos = std::numeric_limits<int>::max(); 1570 1571 for (auto Entry : State.DefPos) { 1572 int DefVALUs = Entry.second; 1573 if (DefVALUs != std::numeric_limits<int>::max()) { 1574 if (DefVALUs >= State.ExecPos) 1575 PreExecPos = std::min(PreExecPos, DefVALUs); 1576 else if (DefVALUs < State.ExecPos) 1577 PostExecPos = std::min(PostExecPos, DefVALUs); 1578 } 1579 } 1580 1581 // Need a VALUs post exec change 1582 if (PostExecPos == std::numeric_limits<int>::max()) 1583 return NoHazardFound; 1584 1585 // Too many VALUs in intv3? 1586 int Intv3VALUs = PostExecPos; 1587 if (Intv3VALUs > Intv3MaxVALUs) 1588 return HazardExpired; 1589 1590 // Too many VALUs in intv2? 1591 int Intv2VALUs = (State.ExecPos - PostExecPos) - 1; 1592 if (Intv2VALUs > Intv1plus2MaxVALUs) 1593 return HazardExpired; 1594 1595 // Need a VALUs pre exec change 1596 if (PreExecPos == std::numeric_limits<int>::max()) 1597 return NoHazardFound; 1598 1599 // Too many VALUs in intv1? 1600 int Intv1VALUs = PreExecPos - State.ExecPos; 1601 if (Intv1VALUs > Intv1plus2MaxVALUs) 1602 return HazardExpired; 1603 1604 // Too many VALUs in intv1 + intv2 1605 if (Intv1VALUs + Intv2VALUs > Intv1plus2MaxVALUs) 1606 return HazardExpired; 1607 1608 return HazardFound; 1609 }; 1610 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) { 1611 if (SIInstrInfo::isVALU(MI)) 1612 State.VALUs += 1; 1613 }; 1614 1615 DenseSet<const MachineBasicBlock *> Visited; 1616 if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(), 1617 std::next(MI->getReverseIterator()), Visited)) 1618 return false; 1619 1620 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1621 TII.get(AMDGPU::S_WAITCNT_DEPCTR)) 1622 .addImm(0x0fff); 1623 1624 return true; 1625 } 1626 1627 bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) { 1628 if (!ST.hasVALUTransUseHazard()) 1629 return false; 1630 if (!SIInstrInfo::isVALU(*MI)) 1631 return false; 1632 1633 SmallSet<Register, 4> SrcVGPRs; 1634 1635 for (const MachineOperand &Use : MI->explicit_uses()) { 1636 if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg())) 1637 SrcVGPRs.insert(Use.getReg()); 1638 } 1639 1640 // Look for the following pattern: 1641 // Va <- TRANS VALU 1642 // intv 1643 // MI Va (WaitState = 0) 1644 // 1645 // Where: 1646 // intv <= 5 VALUs / 1 TRANS 1647 // 1648 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI. 1649 1650 const int IntvMaxVALUs = 5; 1651 const int IntvMaxTRANS = 1; 1652 1653 struct StateType { 1654 int VALUs = 0; 1655 int TRANS = 0; 1656 }; 1657 1658 StateType State; 1659 1660 // This overloads expiry testing with all the hazard detection 1661 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) { 1662 // Too many VALU states have passed 1663 if (State.VALUs > IntvMaxVALUs || State.TRANS > IntvMaxTRANS) 1664 return HazardExpired; 1665 1666 // Instructions which cause va_vdst==0 expire hazard 1667 if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) || 1668 SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I) || 1669 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1670 I.getOperand(0).getImm() == 0x0fff)) 1671 return HazardExpired; 1672 1673 // Track registers writes 1674 if (SIInstrInfo::isTRANS(I)) { 1675 for (Register Src : SrcVGPRs) { 1676 if (I.modifiesRegister(Src, &TRI)) { 1677 return HazardFound; 1678 } 1679 } 1680 } 1681 1682 return NoHazardFound; 1683 }; 1684 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) { 1685 if (SIInstrInfo::isVALU(MI)) 1686 State.VALUs += 1; 1687 if (SIInstrInfo::isTRANS(MI)) 1688 State.TRANS += 1; 1689 }; 1690 1691 DenseSet<const MachineBasicBlock *> Visited; 1692 if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(), 1693 std::next(MI->getReverseIterator()), Visited)) 1694 return false; 1695 1696 // Hazard is observed - insert a wait on va_dst counter to ensure hazard is 1697 // avoided. 1698 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1699 TII.get(AMDGPU::S_WAITCNT_DEPCTR)) 1700 .addImm(AMDGPU::DepCtr::encodeFieldVaVdst(0)); 1701 1702 return true; 1703 } 1704 1705 bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) { 1706 if (!SIInstrInfo::isWMMA(*MI)) 1707 return false; 1708 1709 const SIInstrInfo *TII = ST.getInstrInfo(); 1710 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1711 1712 auto IsHazardFn = [MI, TII, TRI](const MachineInstr &I) { 1713 if (!SIInstrInfo::isWMMA(I)) 1714 return false; 1715 1716 // Src0 or Src1 of the current wmma instruction overlaps with the dest of 1717 // the previous wmma. 1718 const Register CurSrc0Reg = 1719 TII->getNamedOperand(*MI, AMDGPU::OpName::src0)->getReg(); 1720 const Register CurSrc1Reg = 1721 TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg(); 1722 1723 const Register PrevDstReg = 1724 TII->getNamedOperand(I, AMDGPU::OpName::vdst)->getReg(); 1725 1726 if (TRI->regsOverlap(PrevDstReg, CurSrc0Reg) || 1727 TRI->regsOverlap(PrevDstReg, CurSrc1Reg)) { 1728 return true; 1729 } 1730 1731 // Src2 of the current wmma instruction overlaps with the dest of the 1732 // previous wmma. 1733 const MachineOperand *Src2 = 1734 TII->getNamedOperand(*MI, AMDGPU::OpName::src2); 1735 const Register CurSrc2Reg = Src2->isReg() ? Src2->getReg() : Register(); 1736 1737 if (CurSrc2Reg != AMDGPU::NoRegister && 1738 TRI->regsOverlap(PrevDstReg, CurSrc2Reg)) { 1739 1740 const MachineOperand *Src2Mods = 1741 TII->getNamedOperand(*MI, AMDGPU::OpName::src2_modifiers); 1742 const bool NoSrc2Mods = 1743 (Src2Mods->getImm() & (SISrcMods::NEG | SISrcMods::NEG_HI)) == 0; 1744 // Exception: there is no hazard if the wmma instructions are of the same 1745 // type and there is no input modifier on src2 of the current instruction. 1746 return !(NoSrc2Mods && (TII->pseudoToMCOpcode(I.getOpcode()) == 1747 TII->pseudoToMCOpcode(MI->getOpcode()))); 1748 } 1749 1750 return false; 1751 }; 1752 1753 auto IsExpiredFn = [](const MachineInstr &I, int) { 1754 return SIInstrInfo::isVALU(I); 1755 }; 1756 1757 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1758 std::numeric_limits<int>::max()) 1759 return false; 1760 1761 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32)); 1762 1763 return true; 1764 } 1765 1766 bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) { 1767 if (!ST.hasShift64HighRegBug()) 1768 return false; 1769 1770 switch (MI->getOpcode()) { 1771 default: 1772 return false; 1773 case AMDGPU::V_LSHLREV_B64_e64: 1774 case AMDGPU::V_LSHRREV_B64_e64: 1775 case AMDGPU::V_ASHRREV_I64_e64: 1776 break; 1777 } 1778 1779 MachineOperand *Amt = TII.getNamedOperand(*MI, AMDGPU::OpName::src0); 1780 if (!Amt->isReg()) 1781 return false; 1782 1783 Register AmtReg = Amt->getReg(); 1784 const MachineRegisterInfo &MRI = MF.getRegInfo(); 1785 // Check if this is a last VGPR in the allocation block. 1786 if (!TRI.isVGPR(MRI, AmtReg) || ((AmtReg - AMDGPU::VGPR0) & 7) != 7) 1787 return false; 1788 1789 if (AmtReg != AMDGPU::VGPR255 && MRI.isPhysRegUsed(AmtReg + 1)) 1790 return false; 1791 1792 MachineOperand *Src1 = TII.getNamedOperand(*MI, AMDGPU::OpName::src1); 1793 bool OverlappedSrc = Src1->isReg() && TRI.regsOverlap(Src1->getReg(), AmtReg); 1794 bool OverlappedDst = MI->modifiesRegister(AmtReg, &TRI); 1795 bool Overlapped = OverlappedSrc || OverlappedDst; 1796 1797 assert(!OverlappedDst || !OverlappedSrc || 1798 Src1->getReg() == MI->getOperand(0).getReg()); 1799 assert(ST.needsAlignedVGPRs()); 1800 static_assert(AMDGPU::VGPR0 + 1 == AMDGPU::VGPR1); 1801 1802 Register NewReg; 1803 for (MCRegister Reg : Overlapped ? AMDGPU::VReg_64_Align2RegClass 1804 : AMDGPU::VGPR_32RegClass) { 1805 if (!MI->modifiesRegister(Reg, &TRI) && !MI->readsRegister(Reg, &TRI)) { 1806 NewReg = Reg; 1807 break; 1808 } 1809 } 1810 1811 Register NewAmt = Overlapped ? (Register)TRI.getSubReg(NewReg, AMDGPU::sub1) 1812 : NewReg; 1813 Register NewAmtLo; 1814 1815 if (Overlapped) 1816 NewAmtLo = TRI.getSubReg(NewReg, AMDGPU::sub0); 1817 1818 DebugLoc DL = MI->getDebugLoc(); 1819 MachineBasicBlock *MBB = MI->getParent(); 1820 // Insert a full wait count because found register might be pending a wait. 1821 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_WAITCNT)) 1822 .addImm(0); 1823 1824 // Insert V_SWAP_B32 instruction(s) and run hazard recognizer on them. 1825 if (Overlapped) 1826 runOnInstruction( 1827 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmtLo) 1828 .addDef(AmtReg - 1) 1829 .addReg(AmtReg - 1, RegState::Undef) 1830 .addReg(NewAmtLo, RegState::Undef)); 1831 runOnInstruction(BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmt) 1832 .addDef(AmtReg) 1833 .addReg(AmtReg, RegState::Undef) 1834 .addReg(NewAmt, RegState::Undef)); 1835 1836 // Instructions emitted after the current instruction will be processed by the 1837 // parent loop of the hazard recognizer in a natural way. 1838 BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32), 1839 AmtReg) 1840 .addDef(NewAmt) 1841 .addReg(NewAmt) 1842 .addReg(AmtReg); 1843 if (Overlapped) 1844 BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32), 1845 AmtReg - 1) 1846 .addDef(NewAmtLo) 1847 .addReg(NewAmtLo) 1848 .addReg(AmtReg - 1); 1849 1850 // Re-running hazard recognizer on the modified instruction is not necessary, 1851 // inserted V_SWAP_B32 has already both read and write new registers so 1852 // hazards related to these register has already been handled. 1853 Amt->setReg(NewAmt); 1854 Amt->setIsKill(false); 1855 // We do not update liveness, so verifier may see it as undef. 1856 Amt->setIsUndef(); 1857 if (OverlappedDst) 1858 MI->getOperand(0).setReg(NewReg); 1859 if (OverlappedSrc) { 1860 Src1->setReg(NewReg); 1861 Src1->setIsKill(false); 1862 Src1->setIsUndef(); 1863 } 1864 1865 return true; 1866 } 1867 1868 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) { 1869 int NSAtoVMEMWaitStates = 1; 1870 1871 if (!ST.hasNSAtoVMEMBug()) 1872 return 0; 1873 1874 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI)) 1875 return 0; 1876 1877 const SIInstrInfo *TII = ST.getInstrInfo(); 1878 const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset); 1879 if (!Offset || (Offset->getImm() & 6) == 0) 1880 return 0; 1881 1882 auto IsHazardFn = [TII](const MachineInstr &I) { 1883 if (!SIInstrInfo::isMIMG(I)) 1884 return false; 1885 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode()); 1886 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA && 1887 TII->getInstSizeInBytes(I) >= 16; 1888 }; 1889 1890 return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1); 1891 } 1892 1893 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) { 1894 int FPAtomicToDenormModeWaitStates = 3; 1895 1896 if (!ST.hasFPAtomicToDenormModeHazard()) 1897 return 0; 1898 1899 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE) 1900 return 0; 1901 1902 auto IsHazardFn = [](const MachineInstr &I) { 1903 if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I)) 1904 return false; 1905 return SIInstrInfo::isFPAtomic(I); 1906 }; 1907 1908 auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) { 1909 if (WaitStates >= 3 || SIInstrInfo::isVALU(MI)) 1910 return true; 1911 1912 switch (MI.getOpcode()) { 1913 case AMDGPU::S_WAITCNT: 1914 case AMDGPU::S_WAITCNT_VSCNT: 1915 case AMDGPU::S_WAITCNT_VMCNT: 1916 case AMDGPU::S_WAITCNT_EXPCNT: 1917 case AMDGPU::S_WAITCNT_LGKMCNT: 1918 case AMDGPU::S_WAIT_IDLE: 1919 return true; 1920 default: 1921 break; 1922 } 1923 1924 return false; 1925 }; 1926 1927 return FPAtomicToDenormModeWaitStates - 1928 ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn); 1929 } 1930 1931 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) { 1932 assert(SIInstrInfo::isMAI(*MI)); 1933 1934 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI); 1935 } 1936 1937 int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) { 1938 // Early exit if no padding is requested. 1939 if (MFMAPaddingRatio == 0) 1940 return 0; 1941 1942 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 1943 if (!SIInstrInfo::isMFMA(*MI) || MFI->getOccupancy() < 2) 1944 return 0; 1945 1946 int NeighborMFMALatency = 0; 1947 auto IsNeighboringMFMA = [&NeighborMFMALatency, 1948 this](const MachineInstr &MI) { 1949 if (!SIInstrInfo::isMFMA(MI)) 1950 return false; 1951 1952 NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI); 1953 return true; 1954 }; 1955 1956 const int MaxMFMAPipelineWaitStates = 16; 1957 int WaitStatesSinceNeighborMFMA = 1958 getWaitStatesSince(IsNeighboringMFMA, MaxMFMAPipelineWaitStates); 1959 1960 int NeighborMFMAPaddingNeeded = 1961 (NeighborMFMALatency * MFMAPaddingRatio / 100) - 1962 WaitStatesSinceNeighborMFMA; 1963 1964 return std::max(0, NeighborMFMAPaddingNeeded); 1965 } 1966 1967 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) { 1968 int WaitStatesNeeded = 0; 1969 unsigned Opc = MI->getOpcode(); 1970 1971 auto IsVALUFn = [](const MachineInstr &MI) { 1972 return SIInstrInfo::isVALU(MI) || MI.isInlineAsm(); 1973 }; 1974 1975 if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write 1976 const int LegacyVALUWritesVGPRWaitStates = 2; 1977 const int VALUWritesExecWaitStates = 4; 1978 const int MaxWaitStates = 4; 1979 1980 int WaitStatesNeededForUse = VALUWritesExecWaitStates - 1981 getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates); 1982 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1983 1984 if (WaitStatesNeeded < MaxWaitStates) { 1985 for (const MachineOperand &Use : MI->explicit_uses()) { 1986 const int MaxWaitStates = 2; 1987 1988 if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg())) 1989 continue; 1990 1991 int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates - 1992 getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates); 1993 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1994 1995 if (WaitStatesNeeded == MaxWaitStates) 1996 break; 1997 } 1998 } 1999 } 2000 2001 for (const MachineOperand &Op : MI->explicit_operands()) { 2002 if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg())) 2003 continue; 2004 2005 if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 2006 continue; 2007 2008 const int MFMAWritesAGPROverlappedSrcABWaitStates = 4; 2009 const int MFMAWritesAGPROverlappedSrcCWaitStates = 2; 2010 const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4; 2011 const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10; 2012 const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18; 2013 const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1; 2014 const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7; 2015 const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15; 2016 const int MaxWaitStates = 18; 2017 Register Reg = Op.getReg(); 2018 unsigned HazardDefLatency = 0; 2019 2020 auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency, 2021 this](const MachineInstr &MI) { 2022 if (!SIInstrInfo::isMFMA(MI)) 2023 return false; 2024 Register DstReg = MI.getOperand(0).getReg(); 2025 if (DstReg == Reg) 2026 return false; 2027 HazardDefLatency = 2028 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); 2029 return TRI.regsOverlap(DstReg, Reg); 2030 }; 2031 2032 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, 2033 MaxWaitStates); 2034 int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates; 2035 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 2036 int OpNo = Op.getOperandNo(); 2037 if (OpNo == SrcCIdx) { 2038 NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates; 2039 } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) { 2040 switch (HazardDefLatency) { 2041 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates; 2042 break; 2043 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates; 2044 break; 2045 case 16: [[fallthrough]]; 2046 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates; 2047 break; 2048 } 2049 } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { 2050 switch (HazardDefLatency) { 2051 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates; 2052 break; 2053 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates; 2054 break; 2055 case 16: [[fallthrough]]; 2056 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates; 2057 break; 2058 } 2059 } 2060 2061 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 2062 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2063 2064 if (WaitStatesNeeded == MaxWaitStates) 2065 return WaitStatesNeeded; // Early exit. 2066 2067 auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) { 2068 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 2069 return false; 2070 Register DstReg = MI.getOperand(0).getReg(); 2071 return TRI.regsOverlap(Reg, DstReg); 2072 }; 2073 2074 const int AccVGPRWriteMFMAReadSrcCWaitStates = 1; 2075 const int AccVGPRWriteMFMAReadSrcABWaitStates = 3; 2076 const int AccVGPRWriteAccVgprReadWaitStates = 3; 2077 NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates; 2078 if (OpNo == SrcCIdx) 2079 NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates; 2080 else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) 2081 NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates; 2082 2083 WaitStatesNeededForUse = NeedWaitStates - 2084 getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates); 2085 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2086 2087 if (WaitStatesNeeded == MaxWaitStates) 2088 return WaitStatesNeeded; // Early exit. 2089 } 2090 2091 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { 2092 const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0; 2093 const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5; 2094 const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13; 2095 const int MaxWaitStates = 13; 2096 Register DstReg = MI->getOperand(0).getReg(); 2097 unsigned HazardDefLatency = 0; 2098 2099 auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency, 2100 this](const MachineInstr &MI) { 2101 if (!SIInstrInfo::isMFMA(MI)) 2102 return false; 2103 Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg(); 2104 HazardDefLatency = 2105 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); 2106 return TRI.regsOverlap(Reg, DstReg); 2107 }; 2108 2109 int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates); 2110 int NeedWaitStates; 2111 switch (HazardDefLatency) { 2112 case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates; 2113 break; 2114 case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates; 2115 break; 2116 case 16: [[fallthrough]]; 2117 default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates; 2118 break; 2119 } 2120 2121 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince; 2122 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2123 } 2124 2125 // Pad neighboring MFMA with noops for better inter-wave performance. 2126 WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI)); 2127 2128 return WaitStatesNeeded; 2129 } 2130 2131 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { 2132 int WaitStatesNeeded = 0; 2133 unsigned Opc = MI->getOpcode(); 2134 2135 auto IsLegacyVALUFn = [](const MachineInstr &MI) { 2136 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI); 2137 }; 2138 2139 auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) { 2140 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI) && 2141 !SIInstrInfo::isDOT(MI); 2142 }; 2143 2144 if (!SIInstrInfo::isMFMA(*MI)) 2145 return WaitStatesNeeded; 2146 2147 const int VALUWritesExecWaitStates = 4; 2148 int WaitStatesNeededForUse = VALUWritesExecWaitStates - 2149 getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn, 2150 VALUWritesExecWaitStates); 2151 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2152 2153 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 2154 2155 // Loop for both DGEMM and S/HGEMM 2nd instruction. 2156 for (const MachineOperand &Use : MI->explicit_uses()) { 2157 const int LegacyVALUNotDotWritesVGPRWaitStates = 2; 2158 const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2; 2159 const int GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates = 3; 2160 const int GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates = 5; 2161 const int GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates = 4; 2162 const int GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates = 9; 2163 const int GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates = 8; 2164 const int GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates = 17; 2165 const int GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates = 16; 2166 const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8; 2167 const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16; 2168 const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3; 2169 const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9; 2170 const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17; 2171 const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9; 2172 const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4; 2173 const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5; 2174 const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11; 2175 const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19; 2176 const int GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates = 4; 2177 const int GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates = 6; 2178 const int GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates = 10; 2179 const int GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates = 18; 2180 const int GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates = 5; 2181 const int GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates = 7; 2182 const int GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates = 11; 2183 const int GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates = 19; 2184 const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6; 2185 const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11; 2186 const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4; 2187 const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2; 2188 const int MaxWaitStates = 19; 2189 2190 if (!Use.isReg()) 2191 continue; 2192 Register Reg = Use.getReg(); 2193 bool FullReg; 2194 const MachineInstr *MI1; 2195 2196 auto IsOverlappedMFMAFn = [Reg, &FullReg, &MI1, 2197 this](const MachineInstr &MI) { 2198 if (!SIInstrInfo::isMFMA(MI)) 2199 return false; 2200 Register DstReg = MI.getOperand(0).getReg(); 2201 FullReg = (DstReg == Reg); 2202 MI1 = &MI; 2203 return TRI.regsOverlap(DstReg, Reg); 2204 }; 2205 2206 WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates - 2207 getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates); 2208 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2209 2210 int NumWaitStates = 2211 getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates); 2212 if (NumWaitStates == std::numeric_limits<int>::max()) 2213 continue; 2214 2215 int OpNo = Use.getOperandNo(); 2216 unsigned Opc1 = MI1->getOpcode(); 2217 int NeedWaitStates = 0; 2218 if (OpNo == SrcCIdx) { 2219 if (!isDGEMM(Opc) && (!ST.hasGFX940Insts() && isDGEMM(Opc1))) { 2220 NeedWaitStates = 0; 2221 } else if (FullReg) { 2222 if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 2223 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) && 2224 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 2225 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64)) 2226 NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates; 2227 else if (ST.hasGFX940Insts() && 2228 TSchedModel.computeInstrLatency(MI1) == 2) 2229 NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates; 2230 } else { 2231 switch (Opc1) { 2232 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: 2233 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: 2234 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64: 2235 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: 2236 if (!isXDL(ST, *MI)) 2237 NeedWaitStates = DMFMA16x16WritesVGPROverlappedSrcCWaitStates; 2238 break; 2239 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: 2240 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: 2241 if (!isXDL(ST, *MI)) 2242 NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates; 2243 break; 2244 default: 2245 if (ST.hasGFX940Insts() && isXDL(ST, *MI) && !isXDL(ST, *MI1)) 2246 break; 2247 switch (TSchedModel.computeInstrLatency(MI1)) { 2248 case 2: 2249 NeedWaitStates = ST.hasGFX940Insts() 2250 ? isXDL(ST, *MI1) 2251 ? GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates 2252 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates 2253 : isDGEMM(Opc) 2254 ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates 2255 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates; 2256 break; 2257 case 4: 2258 assert(ST.hasGFX940Insts()); 2259 NeedWaitStates = isXDL(ST, *MI1) 2260 ? GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates 2261 : GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates; 2262 break; 2263 case 8: 2264 NeedWaitStates = ST.hasGFX940Insts() 2265 ? isXDL(ST, *MI1) 2266 ? GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates 2267 : GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates 2268 : isDGEMM(Opc) 2269 ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates 2270 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates; 2271 break; 2272 case 16: [[fallthrough]]; 2273 default: 2274 NeedWaitStates = ST.hasGFX940Insts() 2275 ? isXDL(ST, *MI1) 2276 ? GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates 2277 : GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates 2278 : isDGEMM(Opc) 2279 ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates 2280 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates; 2281 } 2282 } 2283 } 2284 } else { 2285 switch (Opc1) { 2286 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: 2287 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: 2288 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64: 2289 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: 2290 NeedWaitStates = DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates; 2291 break; 2292 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: 2293 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: 2294 NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates; 2295 break; 2296 default: 2297 switch (TSchedModel.computeInstrLatency(MI1)) { 2298 case 2: 2299 NeedWaitStates = ST.hasGFX940Insts() 2300 ? isXDL(ST, *MI1) 2301 ? GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates 2302 : GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates 2303 : SMFMA4x4WritesVGPROverlappedSrcABWaitStates; 2304 break; 2305 case 4: 2306 assert(ST.hasGFX940Insts()); 2307 NeedWaitStates = isXDL(ST, *MI1) 2308 ? GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates 2309 : GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates; 2310 break; 2311 case 8: 2312 NeedWaitStates = ST.hasGFX940Insts() 2313 ? isXDL(ST, *MI1) 2314 ? GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates 2315 : GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates 2316 : SMFMA16x16WritesVGPROverlappedSrcABWaitStates; 2317 break; 2318 case 16: [[fallthrough]]; 2319 default: 2320 NeedWaitStates = ST.hasGFX940Insts() 2321 ? isXDL(ST, *MI1) 2322 ? GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates 2323 : GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates 2324 : SMFMA32x32WritesVGPROverlappedSrcABWaitStates; 2325 } 2326 } 2327 } 2328 if (WaitStatesNeeded >= NeedWaitStates) 2329 continue; 2330 2331 WaitStatesNeededForUse = NeedWaitStates - NumWaitStates; 2332 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2333 2334 if (WaitStatesNeeded == MaxWaitStates) 2335 break; 2336 } 2337 2338 return WaitStatesNeeded; 2339 } 2340 2341 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) { 2342 // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards() 2343 if (!ST.hasMAIInsts() || ST.hasGFX90AInsts()) 2344 return 0; 2345 2346 int WaitStatesNeeded = 0; 2347 2348 auto IsAccVgprReadFn = [](const MachineInstr &MI) { 2349 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64; 2350 }; 2351 2352 for (const MachineOperand &Op : MI->explicit_uses()) { 2353 if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg())) 2354 continue; 2355 2356 Register Reg = Op.getReg(); 2357 2358 const int AccVgprReadLdStWaitStates = 2; 2359 const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1; 2360 const int MaxWaitStates = 2; 2361 2362 int WaitStatesNeededForUse = AccVgprReadLdStWaitStates - 2363 getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates); 2364 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2365 2366 if (WaitStatesNeeded == MaxWaitStates) 2367 return WaitStatesNeeded; // Early exit. 2368 2369 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) { 2370 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 && 2371 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 2372 return false; 2373 auto IsVALUFn = [](const MachineInstr &MI) { 2374 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMAI(MI); 2375 }; 2376 return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) < 2377 std::numeric_limits<int>::max(); 2378 }; 2379 2380 WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates - 2381 getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates); 2382 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2383 } 2384 2385 return WaitStatesNeeded; 2386 } 2387 2388 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { 2389 if (!ST.hasGFX90AInsts()) 2390 return 0; 2391 2392 auto IsDGEMMFn = [](const MachineInstr &MI) -> bool { 2393 return isDGEMM(MI.getOpcode()); 2394 }; 2395 2396 // This is checked in checkMAIHazards90A() 2397 if (SIInstrInfo::isMFMA(*MI)) 2398 return 0; 2399 2400 const MachineRegisterInfo &MRI = MF.getRegInfo(); 2401 2402 int WaitStatesNeeded = 0; 2403 2404 bool IsMem = SIInstrInfo::isVMEM(*MI) || 2405 SIInstrInfo::isFLAT(*MI) || 2406 SIInstrInfo::isDS(*MI); 2407 bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(*MI); 2408 bool IsVALU = SIInstrInfo::isVALU(*MI); 2409 2410 const MachineInstr *MFMA = nullptr; 2411 unsigned Reg; 2412 auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) { 2413 if (!SIInstrInfo::isMFMA(MI) || 2414 !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) 2415 return false; 2416 MFMA = &MI; 2417 return true; 2418 }; 2419 2420 const MachineInstr *DOT = nullptr; 2421 auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) { 2422 if (!SIInstrInfo::isDOT(MI) || 2423 !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) 2424 return false; 2425 DOT = &MI; 2426 return true; 2427 }; 2428 2429 bool DGEMMAfterVALUWrite = false; 2430 auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) { 2431 // Found DGEMM on reverse traversal to def. 2432 if (isDGEMM(MI.getOpcode())) 2433 DGEMMAfterVALUWrite = true; 2434 2435 // Only hazard if register is defined by a VALU and a DGEMM is found after 2436 // after the def. 2437 if (!TII.isVALU(MI) || !DGEMMAfterVALUWrite) 2438 return false; 2439 2440 return true; 2441 }; 2442 2443 int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 2444 AMDGPU::OpName::src2); 2445 2446 if (IsMemOrExport || IsVALU) { 2447 const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5; 2448 const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11; 2449 const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19; 2450 const int GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates = 4; 2451 const int GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates = 6; 2452 const int GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates = 10; 2453 const int GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates = 18; 2454 const int GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates = 5; 2455 const int GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates = 7; 2456 const int GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates = 11; 2457 const int GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates = 19; 2458 const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9; 2459 const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18; 2460 const int DMFMA4x4WriteVgprVALUReadWaitStates = 6; 2461 const int DMFMA16x16WriteVgprVALUReadWaitStates = 11; 2462 const int DotWriteSameDotReadSrcAB = 3; 2463 const int DotWriteDifferentVALURead = 3; 2464 const int DMFMABetweenVALUWriteVMEMRead = 2; 2465 const int MaxWaitStates = 19; 2466 2467 for (const MachineOperand &Use : MI->explicit_uses()) { 2468 if (!Use.isReg()) 2469 continue; 2470 Reg = Use.getReg(); 2471 2472 DOT = nullptr; 2473 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn, 2474 MaxWaitStates); 2475 if (DOT) { 2476 int NeedWaitStates = 0; 2477 if (DOT->getOpcode() == MI->getOpcode()) { 2478 if (&Use - &MI->getOperand(0) != SrcCIdx) 2479 NeedWaitStates = DotWriteSameDotReadSrcAB; 2480 } else { 2481 NeedWaitStates = DotWriteDifferentVALURead; 2482 } 2483 2484 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 2485 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2486 } 2487 2488 // Workaround for HW data hazard bug observed only in GFX90A. When there 2489 // is a DGEMM instruction in-between a VALU and a VMEM instruction it 2490 // causes the SQ to incorrectly not insert two wait states between the two 2491 // instructions needed to avoid data hazard. 2492 if (IsMem && ST.hasGFX90AInsts() && !ST.hasGFX940Insts()) { 2493 DGEMMAfterVALUWrite = false; 2494 if (TRI.isVectorRegister(MRI, Reg)) { 2495 int WaitStatesNeededForUse = 2496 DMFMABetweenVALUWriteVMEMRead - 2497 getWaitStatesSinceDef(Reg, IsDGEMMHazard, 2498 DMFMABetweenVALUWriteVMEMRead); 2499 2500 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2501 } 2502 } 2503 2504 MFMA = nullptr; 2505 WaitStatesSinceDef = 2506 getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates); 2507 if (!MFMA) 2508 continue; 2509 2510 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); 2511 int NeedWaitStates = MaxWaitStates; 2512 switch (HazardDefLatency) { 2513 case 2: 2514 NeedWaitStates = 2515 ST.hasGFX940Insts() 2516 ? isXDL(ST, *MFMA) 2517 ? GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates 2518 : GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates 2519 : SMFMA4x4WriteVgprVALUMemExpReadWaitStates; 2520 break; 2521 case 4: 2522 assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts()); 2523 NeedWaitStates = 2524 isDGEMM(MFMA->getOpcode()) 2525 ? IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates 2526 : DMFMA4x4WriteVgprVALUReadWaitStates 2527 : isXDL(ST, *MFMA) 2528 ? GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates 2529 : GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates; 2530 break; 2531 case 8: 2532 NeedWaitStates = 2533 ST.hasGFX940Insts() 2534 ? isXDL(ST, *MFMA) 2535 ? GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates 2536 : GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates 2537 : SMFMA16x16WriteVgprVALUMemExpReadWaitStates; 2538 break; 2539 case 16: [[fallthrough]]; 2540 default: 2541 NeedWaitStates = 2542 isDGEMM(MFMA->getOpcode()) 2543 ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates 2544 : DMFMA16x16WriteVgprVALUReadWaitStates 2545 : ST.hasGFX940Insts() 2546 ? isXDL(ST, *MFMA) 2547 ? GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates 2548 : GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates 2549 : SMFMA32x32WriteVgprVALUMemExpReadWaitStates; 2550 break; 2551 } 2552 2553 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 2554 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2555 2556 if (WaitStatesNeeded == MaxWaitStates) 2557 break; 2558 } 2559 } 2560 2561 unsigned Opc = MI->getOpcode(); 2562 const int DMFMAToFMA64WaitStates = 2; 2563 if ((Opc == AMDGPU::V_FMA_F64_e64 || 2564 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 || 2565 Opc == AMDGPU::V_FMAC_F64_dpp) && 2566 WaitStatesNeeded < DMFMAToFMA64WaitStates) { 2567 int WaitStatesNeededForUse = DMFMAToFMA64WaitStates - 2568 getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates); 2569 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2570 } 2571 2572 if (!IsVALU && !IsMemOrExport) 2573 return WaitStatesNeeded; 2574 2575 for (const MachineOperand &Def : MI->defs()) { 2576 const int SMFMA4x4WriteVgprVALUWawWaitStates = 5; 2577 const int SMFMA16x16WriteVgprVALUWawWaitStates = 11; 2578 const int SMFMA32x32WriteVgprVALUWawWaitStates = 19; 2579 const int GFX940_SMFMA2PassWriteVgprVALUWawWaitStates = 4; 2580 const int GFX940_SMFMA4PassWriteVgprVALUWawWaitStates = 6; 2581 const int GFX940_SMFMA8PassWriteVgprVALUWawWaitStates = 10; 2582 const int GFX940_SMFMA16PassWriteVgprVALUWawWaitStates = 18; 2583 const int GFX940_XDL2PassWriteVgprVALUWawWaitStates = 5; 2584 const int GFX940_XDL4PassWriteVgprVALUWawWaitStates = 7; 2585 const int GFX940_XDL8PassWriteVgprVALUWawWaitStates = 11; 2586 const int GFX940_XDL16PassWriteVgprVALUWawWaitStates = 19; 2587 const int SMFMA4x4ReadVgprVALUWarWaitStates = 1; 2588 const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3; 2589 const int SMFMA16x16ReadVgprVALUWarWaitStates = 7; 2590 const int SMFMA32x32ReadVgprVALUWarWaitStates = 15; 2591 const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6; 2592 const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11; 2593 const int DotWriteDifferentVALUWrite = 3; 2594 const int MaxWaitStates = 19; 2595 const int MaxWarWaitStates = 15; 2596 2597 Reg = Def.getReg(); 2598 2599 DOT = nullptr; 2600 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn, 2601 MaxWaitStates); 2602 if (DOT && DOT->getOpcode() != MI->getOpcode()) 2603 WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite - 2604 WaitStatesSinceDef); 2605 2606 MFMA = nullptr; 2607 WaitStatesSinceDef = 2608 getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates); 2609 if (MFMA) { 2610 int NeedWaitStates = MaxWaitStates; 2611 switch (TSchedModel.computeInstrLatency(MFMA)) { 2612 case 2: 2613 NeedWaitStates = ST.hasGFX940Insts() 2614 ? isXDL(ST, *MFMA) 2615 ? GFX940_XDL2PassWriteVgprVALUWawWaitStates 2616 : GFX940_SMFMA2PassWriteVgprVALUWawWaitStates 2617 : SMFMA4x4WriteVgprVALUWawWaitStates; 2618 break; 2619 case 4: 2620 assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts()); 2621 NeedWaitStates = isDGEMM(MFMA->getOpcode()) 2622 ? DMFMA4x4WriteVgprVALUWriteWaitStates 2623 : isXDL(ST, *MFMA) 2624 ? GFX940_XDL4PassWriteVgprVALUWawWaitStates 2625 : GFX940_SMFMA4PassWriteVgprVALUWawWaitStates; 2626 break; 2627 case 8: 2628 NeedWaitStates = ST.hasGFX940Insts() 2629 ? isXDL(ST, *MFMA) 2630 ? GFX940_XDL8PassWriteVgprVALUWawWaitStates 2631 : GFX940_SMFMA8PassWriteVgprVALUWawWaitStates 2632 : SMFMA16x16WriteVgprVALUWawWaitStates; 2633 break; 2634 case 16: [[fallthrough]]; 2635 default: 2636 NeedWaitStates = isDGEMM(MFMA->getOpcode()) 2637 ? DMFMA16x16WriteVgprVALUWriteWaitStates 2638 : ST.hasGFX940Insts() 2639 ? isXDL(ST, *MFMA) 2640 ? GFX940_XDL16PassWriteVgprVALUWawWaitStates 2641 : GFX940_SMFMA16PassWriteVgprVALUWawWaitStates 2642 : SMFMA32x32WriteVgprVALUWawWaitStates; 2643 break; 2644 } 2645 2646 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 2647 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2648 2649 if (WaitStatesNeeded == MaxWaitStates) 2650 break; 2651 } 2652 2653 auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) { 2654 if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) || 2655 !MI.readsRegister(Reg, &TRI)) 2656 return false; 2657 2658 if (ST.hasGFX940Insts() && !isXDL(ST, MI)) 2659 return false; 2660 2661 const MachineOperand *SrcC = 2662 TII.getNamedOperand(MI, AMDGPU::OpName::src2); 2663 assert(SrcC); 2664 if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg)) 2665 return false; 2666 2667 MFMA = &MI; 2668 return true; 2669 }; 2670 2671 MFMA = nullptr; 2672 int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn, 2673 MaxWarWaitStates); 2674 if (!MFMA) 2675 continue; 2676 2677 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); 2678 int NeedWaitStates = MaxWaitStates; 2679 switch (HazardDefLatency) { 2680 case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates; 2681 break; 2682 case 4: assert(ST.hasGFX940Insts()); 2683 NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates; 2684 break; 2685 case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates; 2686 break; 2687 case 16: [[fallthrough]]; 2688 default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates; 2689 break; 2690 } 2691 2692 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse; 2693 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2694 } 2695 2696 return WaitStatesNeeded; 2697 } 2698 2699 bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) { 2700 if (!SU->isInstr()) 2701 return false; 2702 2703 const MachineInstr *MAI = nullptr; 2704 2705 auto IsMFMAFn = [&MAI](const MachineInstr &MI) { 2706 MAI = nullptr; 2707 if (SIInstrInfo::isMFMA(MI)) 2708 MAI = &MI; 2709 return MAI != nullptr; 2710 }; 2711 2712 MachineInstr *MI = SU->getInstr(); 2713 if (IsMFMAFn(*MI)) { 2714 int W = getWaitStatesSince(IsMFMAFn, 16); 2715 if (MAI) 2716 return W < (int)TSchedModel.computeInstrLatency(MAI); 2717 } 2718 2719 return false; 2720 } 2721 2722 bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) { 2723 if (!ST.isWave64()) 2724 return false; 2725 if (!ST.hasVALUMaskWriteHazard()) 2726 return false; 2727 if (!SIInstrInfo::isSALU(*MI)) 2728 return false; 2729 2730 // The hazard sequence is three instructions: 2731 // 1. VALU reads SGPR as mask 2732 // 2. SALU writes SGPR 2733 // 3. SALU reads SGPR 2734 // The hazard can expire if the distance between 2 and 3 is sufficient. 2735 // In practice this happens <10% of the time, hence this always assumes 2736 // the hazard exists if 1 and 2 are present to avoid searching. 2737 2738 const MachineOperand *SDSTOp = TII.getNamedOperand(*MI, AMDGPU::OpName::sdst); 2739 if (!SDSTOp || !SDSTOp->isReg()) 2740 return false; 2741 2742 const Register HazardReg = SDSTOp->getReg(); 2743 if (HazardReg == AMDGPU::EXEC || 2744 HazardReg == AMDGPU::EXEC_LO || 2745 HazardReg == AMDGPU::EXEC_HI || 2746 HazardReg == AMDGPU::M0) 2747 return false; 2748 2749 auto IsHazardFn = [HazardReg, this](const MachineInstr &I) { 2750 switch (I.getOpcode()) { 2751 case AMDGPU::V_ADDC_U32_e32: 2752 case AMDGPU::V_ADDC_U32_dpp: 2753 case AMDGPU::V_CNDMASK_B16_e32: 2754 case AMDGPU::V_CNDMASK_B16_dpp: 2755 case AMDGPU::V_CNDMASK_B32_e32: 2756 case AMDGPU::V_CNDMASK_B32_dpp: 2757 case AMDGPU::V_DIV_FMAS_F32_e64: 2758 case AMDGPU::V_DIV_FMAS_F64_e64: 2759 case AMDGPU::V_SUBB_U32_e32: 2760 case AMDGPU::V_SUBB_U32_dpp: 2761 case AMDGPU::V_SUBBREV_U32_e32: 2762 case AMDGPU::V_SUBBREV_U32_dpp: 2763 // These implicitly read VCC as mask source. 2764 return HazardReg == AMDGPU::VCC || 2765 HazardReg == AMDGPU::VCC_LO || 2766 HazardReg == AMDGPU::VCC_HI; 2767 case AMDGPU::V_ADDC_U32_e64: 2768 case AMDGPU::V_ADDC_U32_e64_dpp: 2769 case AMDGPU::V_CNDMASK_B16_e64: 2770 case AMDGPU::V_CNDMASK_B16_e64_dpp: 2771 case AMDGPU::V_CNDMASK_B32_e64: 2772 case AMDGPU::V_CNDMASK_B32_e64_dpp: 2773 case AMDGPU::V_SUBB_U32_e64: 2774 case AMDGPU::V_SUBB_U32_e64_dpp: 2775 case AMDGPU::V_SUBBREV_U32_e64: 2776 case AMDGPU::V_SUBBREV_U32_e64_dpp: { 2777 // Only check mask register overlaps. 2778 const MachineOperand *SSRCOp = TII.getNamedOperand(I, AMDGPU::OpName::src2); 2779 assert(SSRCOp); 2780 return TRI.regsOverlap(SSRCOp->getReg(), HazardReg); 2781 } 2782 default: 2783 return false; 2784 } 2785 }; 2786 2787 const MachineRegisterInfo &MRI = MF.getRegInfo(); 2788 auto IsExpiredFn = [&MRI, this](const MachineInstr &I, int) { 2789 // s_waitcnt_depctr sa_sdst(0) mitigates hazard. 2790 if (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 2791 AMDGPU::DepCtr::decodeFieldSaSdst(I.getOperand(0).getImm()) == 0) 2792 return true; 2793 2794 // VALU access to any SGPR or literal constant other than HazardReg 2795 // mitigates hazard. No need to check HazardReg here as this will 2796 // only be called when !IsHazardFn. 2797 if (!SIInstrInfo::isVALU(I)) 2798 return false; 2799 for (int OpNo = 0, End = I.getNumOperands(); OpNo < End; ++OpNo) { 2800 const MachineOperand &Op = I.getOperand(OpNo); 2801 if (Op.isReg()) { 2802 Register OpReg = Op.getReg(); 2803 // Only consider uses 2804 if (!Op.isUse()) 2805 continue; 2806 // Ignore EXEC 2807 if (OpReg == AMDGPU::EXEC || 2808 OpReg == AMDGPU::EXEC_LO || 2809 OpReg == AMDGPU::EXEC_HI) 2810 continue; 2811 // Ignore all implicit uses except VCC 2812 if (Op.isImplicit()) { 2813 if (OpReg == AMDGPU::VCC || 2814 OpReg == AMDGPU::VCC_LO || 2815 OpReg == AMDGPU::VCC_HI) 2816 return true; 2817 continue; 2818 } 2819 if (TRI.isSGPRReg(MRI, OpReg)) 2820 return true; 2821 } else { 2822 const MCInstrDesc &InstDesc = I.getDesc(); 2823 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo]; 2824 if (!TII.isInlineConstant(Op, OpInfo)) 2825 return true; 2826 } 2827 } 2828 return false; 2829 }; 2830 2831 // Check for hazard 2832 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 2833 std::numeric_limits<int>::max()) 2834 return false; 2835 2836 auto NextMI = std::next(MI->getIterator()); 2837 2838 // Add s_waitcnt_depctr sa_sdst(0) after SALU write. 2839 BuildMI(*MI->getParent(), NextMI, MI->getDebugLoc(), 2840 TII.get(AMDGPU::S_WAITCNT_DEPCTR)) 2841 .addImm(AMDGPU::DepCtr::encodeFieldSaSdst(0)); 2842 2843 // SALU write may be s_getpc in a bundle. 2844 if (MI->getOpcode() == AMDGPU::S_GETPC_B64) { 2845 // Update offsets of any references in the bundle. 2846 while (NextMI != MI->getParent()->end() && 2847 NextMI->isBundledWithPred()) { 2848 for (auto &Operand : NextMI->operands()) { 2849 if (Operand.isGlobal()) 2850 Operand.setOffset(Operand.getOffset() + 4); 2851 } 2852 NextMI++; 2853 } 2854 } 2855 2856 return true; 2857 } 2858