1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements hazard recognizers for scheduling on GCN processors. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "GCNHazardRecognizer.h" 14 #include "GCNSubtarget.h" 15 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 16 #include "llvm/CodeGen/MachineFunction.h" 17 #include "llvm/CodeGen/ScheduleDAG.h" 18 #include "llvm/Support/TargetParser.h" 19 20 using namespace llvm; 21 22 //===----------------------------------------------------------------------===// 23 // Hazard Recoginizer Implementation 24 //===----------------------------------------------------------------------===// 25 26 GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) : 27 IsHazardRecognizerMode(false), 28 CurrCycleInstr(nullptr), 29 MF(MF), 30 ST(MF.getSubtarget<GCNSubtarget>()), 31 TII(*ST.getInstrInfo()), 32 TRI(TII.getRegisterInfo()), 33 ClauseUses(TRI.getNumRegUnits()), 34 ClauseDefs(TRI.getNumRegUnits()) { 35 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5; 36 TSchedModel.init(&ST); 37 } 38 39 void GCNHazardRecognizer::Reset() { 40 EmittedInstrs.clear(); 41 } 42 43 void GCNHazardRecognizer::EmitInstruction(SUnit *SU) { 44 EmitInstruction(SU->getInstr()); 45 } 46 47 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) { 48 CurrCycleInstr = MI; 49 } 50 51 static bool isDivFMas(unsigned Opcode) { 52 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64; 53 } 54 55 static bool isSGetReg(unsigned Opcode) { 56 return Opcode == AMDGPU::S_GETREG_B32; 57 } 58 59 static bool isSSetReg(unsigned Opcode) { 60 switch (Opcode) { 61 case AMDGPU::S_SETREG_B32: 62 case AMDGPU::S_SETREG_B32_mode: 63 case AMDGPU::S_SETREG_IMM32_B32: 64 case AMDGPU::S_SETREG_IMM32_B32_mode: 65 return true; 66 } 67 return false; 68 } 69 70 static bool isRWLane(unsigned Opcode) { 71 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32; 72 } 73 74 static bool isRFE(unsigned Opcode) { 75 return Opcode == AMDGPU::S_RFE_B64; 76 } 77 78 static bool isSMovRel(unsigned Opcode) { 79 switch (Opcode) { 80 case AMDGPU::S_MOVRELS_B32: 81 case AMDGPU::S_MOVRELS_B64: 82 case AMDGPU::S_MOVRELD_B32: 83 case AMDGPU::S_MOVRELD_B64: 84 return true; 85 default: 86 return false; 87 } 88 } 89 90 static bool isDGEMM(unsigned Opcode) { 91 return Opcode == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 92 Opcode == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64 || 93 Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_e64 || 94 Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64; 95 } 96 97 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) { 98 unsigned Opcode = MI.getOpcode(); 99 100 if (!SIInstrInfo::isMAI(MI) || 101 isDGEMM(Opcode) || 102 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 || 103 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64) 104 return false; 105 106 return true; 107 } 108 109 static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, 110 const MachineInstr &MI) { 111 if (TII.isAlwaysGDS(MI.getOpcode())) 112 return true; 113 114 switch (MI.getOpcode()) { 115 case AMDGPU::S_SENDMSG: 116 case AMDGPU::S_SENDMSGHALT: 117 case AMDGPU::S_TTRACEDATA: 118 return true; 119 // These DS opcodes don't support GDS. 120 case AMDGPU::DS_NOP: 121 case AMDGPU::DS_PERMUTE_B32: 122 case AMDGPU::DS_BPERMUTE_B32: 123 return false; 124 default: 125 if (TII.isDS(MI.getOpcode())) { 126 int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 127 AMDGPU::OpName::gds); 128 if (MI.getOperand(GDS).getImm()) 129 return true; 130 } 131 return false; 132 } 133 } 134 135 static bool isPermlane(const MachineInstr &MI) { 136 unsigned Opcode = MI.getOpcode(); 137 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 || 138 Opcode == AMDGPU::V_PERMLANEX16_B32_e64; 139 } 140 141 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) { 142 const MachineOperand *RegOp = TII->getNamedOperand(RegInstr, 143 AMDGPU::OpName::simm16); 144 return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_; 145 } 146 147 ScheduleHazardRecognizer::HazardType 148 GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { 149 MachineInstr *MI = SU->getInstr(); 150 // If we are not in "HazardRecognizerMode" and therefore not being run from 151 // the scheduler, track possible stalls from hazards but don't insert noops. 152 auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard; 153 154 if (MI->isBundle()) 155 return NoHazard; 156 157 if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0) 158 return HazardType; 159 160 // FIXME: Should flat be considered vmem? 161 if ((SIInstrInfo::isVMEM(*MI) || 162 SIInstrInfo::isFLAT(*MI)) 163 && checkVMEMHazards(MI) > 0) 164 return HazardType; 165 166 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0) 167 return HazardType; 168 169 if (checkFPAtomicToDenormModeHazard(MI) > 0) 170 return HazardType; 171 172 if (ST.hasNoDataDepHazard()) 173 return NoHazard; 174 175 if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0) 176 return HazardType; 177 178 if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0) 179 return HazardType; 180 181 if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0) 182 return HazardType; 183 184 if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0) 185 return HazardType; 186 187 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || 188 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || 189 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) 190 return HazardType; 191 192 if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0) 193 return HazardType; 194 195 if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0) 196 return HazardType; 197 198 if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0) 199 return HazardType; 200 201 if (ST.hasReadM0MovRelInterpHazard() && 202 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode())) && 203 checkReadM0Hazards(MI) > 0) 204 return HazardType; 205 206 if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI) && 207 checkReadM0Hazards(MI) > 0) 208 return HazardType; 209 210 if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0) 211 return HazardType; 212 213 if ((SIInstrInfo::isVMEM(*MI) || 214 SIInstrInfo::isFLAT(*MI) || 215 SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0) 216 return HazardType; 217 218 if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0) 219 return HazardType; 220 221 return NoHazard; 222 } 223 224 static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, 225 unsigned Quantity) { 226 while (Quantity > 0) { 227 unsigned Arg = std::min(Quantity, 8u); 228 Quantity -= Arg; 229 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP)) 230 .addImm(Arg - 1); 231 } 232 } 233 234 void GCNHazardRecognizer::processBundle() { 235 MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator()); 236 MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end(); 237 // Check bundled MachineInstr's for hazards. 238 for (; MI != E && MI->isInsideBundle(); ++MI) { 239 CurrCycleInstr = &*MI; 240 unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr); 241 242 if (IsHazardRecognizerMode) { 243 fixHazards(CurrCycleInstr); 244 245 insertNoopsInBundle(CurrCycleInstr, TII, WaitStates); 246 } 247 248 // It’s unnecessary to track more than MaxLookAhead instructions. Since we 249 // include the bundled MI directly after, only add a maximum of 250 // (MaxLookAhead - 1) noops to EmittedInstrs. 251 for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i) 252 EmittedInstrs.push_front(nullptr); 253 254 EmittedInstrs.push_front(CurrCycleInstr); 255 EmittedInstrs.resize(MaxLookAhead); 256 } 257 CurrCycleInstr = nullptr; 258 } 259 260 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) { 261 IsHazardRecognizerMode = true; 262 CurrCycleInstr = MI; 263 unsigned W = PreEmitNoopsCommon(MI); 264 fixHazards(MI); 265 CurrCycleInstr = nullptr; 266 return W; 267 } 268 269 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { 270 if (MI->isBundle()) 271 return 0; 272 273 int WaitStates = 0; 274 275 if (SIInstrInfo::isSMRD(*MI)) 276 return std::max(WaitStates, checkSMRDHazards(MI)); 277 278 if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI)) 279 WaitStates = std::max(WaitStates, checkVMEMHazards(MI)); 280 281 if (ST.hasNSAtoVMEMBug()) 282 WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI)); 283 284 WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI)); 285 286 if (ST.hasNoDataDepHazard()) 287 return WaitStates; 288 289 if (SIInstrInfo::isVALU(*MI)) 290 WaitStates = std::max(WaitStates, checkVALUHazards(MI)); 291 292 if (SIInstrInfo::isDPP(*MI)) 293 WaitStates = std::max(WaitStates, checkDPPHazards(MI)); 294 295 if (isDivFMas(MI->getOpcode())) 296 WaitStates = std::max(WaitStates, checkDivFMasHazards(MI)); 297 298 if (isRWLane(MI->getOpcode())) 299 WaitStates = std::max(WaitStates, checkRWLaneHazards(MI)); 300 301 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || 302 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || 303 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) 304 WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI)); 305 306 if (MI->isInlineAsm()) 307 return std::max(WaitStates, checkInlineAsmHazards(MI)); 308 309 if (isSGetReg(MI->getOpcode())) 310 return std::max(WaitStates, checkGetRegHazards(MI)); 311 312 if (isSSetReg(MI->getOpcode())) 313 return std::max(WaitStates, checkSetRegHazards(MI)); 314 315 if (isRFE(MI->getOpcode())) 316 return std::max(WaitStates, checkRFEHazards(MI)); 317 318 if (ST.hasReadM0MovRelInterpHazard() && (TII.isVINTRP(*MI) || 319 isSMovRel(MI->getOpcode()))) 320 return std::max(WaitStates, checkReadM0Hazards(MI)); 321 322 if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) 323 return std::max(WaitStates, checkReadM0Hazards(MI)); 324 325 if (SIInstrInfo::isMAI(*MI)) 326 return std::max(WaitStates, checkMAIHazards(MI)); 327 328 if (SIInstrInfo::isVMEM(*MI) || 329 SIInstrInfo::isFLAT(*MI) || 330 SIInstrInfo::isDS(*MI)) 331 return std::max(WaitStates, checkMAILdStHazards(MI)); 332 333 return WaitStates; 334 } 335 336 void GCNHazardRecognizer::EmitNoop() { 337 EmittedInstrs.push_front(nullptr); 338 } 339 340 void GCNHazardRecognizer::AdvanceCycle() { 341 // When the scheduler detects a stall, it will call AdvanceCycle() without 342 // emitting any instructions. 343 if (!CurrCycleInstr) { 344 EmittedInstrs.push_front(nullptr); 345 return; 346 } 347 348 // Do not track non-instructions which do not affect the wait states. 349 // If included, these instructions can lead to buffer overflow such that 350 // detectable hazards are missed. 351 if (CurrCycleInstr->isImplicitDef() || CurrCycleInstr->isDebugInstr() || 352 CurrCycleInstr->isKill()) { 353 CurrCycleInstr = nullptr; 354 return; 355 } 356 357 if (CurrCycleInstr->isBundle()) { 358 processBundle(); 359 return; 360 } 361 362 unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr); 363 364 // Keep track of emitted instructions 365 EmittedInstrs.push_front(CurrCycleInstr); 366 367 // Add a nullptr for each additional wait state after the first. Make sure 368 // not to add more than getMaxLookAhead() items to the list, since we 369 // truncate the list to that size right after this loop. 370 for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead()); 371 i < e; ++i) { 372 EmittedInstrs.push_front(nullptr); 373 } 374 375 // getMaxLookahead() is the largest number of wait states we will ever need 376 // to insert, so there is no point in keeping track of more than that many 377 // wait states. 378 EmittedInstrs.resize(getMaxLookAhead()); 379 380 CurrCycleInstr = nullptr; 381 } 382 383 void GCNHazardRecognizer::RecedeCycle() { 384 llvm_unreachable("hazard recognizer does not support bottom-up scheduling."); 385 } 386 387 //===----------------------------------------------------------------------===// 388 // Helper Functions 389 //===----------------------------------------------------------------------===// 390 391 typedef function_ref<bool(MachineInstr *, int WaitStates)> IsExpiredFn; 392 393 // Returns a minimum wait states since \p I walking all predecessors. 394 // Only scans until \p IsExpired does not return true. 395 // Can only be run in a hazard recognizer mode. 396 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, 397 MachineBasicBlock *MBB, 398 MachineBasicBlock::reverse_instr_iterator I, 399 int WaitStates, 400 IsExpiredFn IsExpired, 401 DenseSet<const MachineBasicBlock *> &Visited) { 402 for (auto E = MBB->instr_rend(); I != E; ++I) { 403 // Don't add WaitStates for parent BUNDLE instructions. 404 if (I->isBundle()) 405 continue; 406 407 if (IsHazard(&*I)) 408 return WaitStates; 409 410 if (I->isInlineAsm() || I->isMetaInstruction()) 411 continue; 412 413 WaitStates += SIInstrInfo::getNumWaitStates(*I); 414 415 if (IsExpired(&*I, WaitStates)) 416 return std::numeric_limits<int>::max(); 417 } 418 419 int MinWaitStates = WaitStates; 420 bool Found = false; 421 for (MachineBasicBlock *Pred : MBB->predecessors()) { 422 if (!Visited.insert(Pred).second) 423 continue; 424 425 int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(), 426 WaitStates, IsExpired, Visited); 427 428 if (W == std::numeric_limits<int>::max()) 429 continue; 430 431 MinWaitStates = Found ? std::min(MinWaitStates, W) : W; 432 if (IsExpired(nullptr, MinWaitStates)) 433 return MinWaitStates; 434 435 Found = true; 436 } 437 438 if (Found) 439 return MinWaitStates; 440 441 return std::numeric_limits<int>::max(); 442 } 443 444 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, 445 MachineInstr *MI, 446 IsExpiredFn IsExpired) { 447 DenseSet<const MachineBasicBlock *> Visited; 448 return getWaitStatesSince(IsHazard, MI->getParent(), 449 std::next(MI->getReverseIterator()), 450 0, IsExpired, Visited); 451 } 452 453 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) { 454 if (IsHazardRecognizerMode) { 455 auto IsExpiredFn = [Limit] (MachineInstr *, int WaitStates) { 456 return WaitStates >= Limit; 457 }; 458 return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn); 459 } 460 461 int WaitStates = 0; 462 for (MachineInstr *MI : EmittedInstrs) { 463 if (MI) { 464 if (IsHazard(MI)) 465 return WaitStates; 466 467 if (MI->isInlineAsm()) 468 continue; 469 } 470 ++WaitStates; 471 472 if (WaitStates >= Limit) 473 break; 474 } 475 return std::numeric_limits<int>::max(); 476 } 477 478 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg, 479 IsHazardFn IsHazardDef, 480 int Limit) { 481 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 482 483 auto IsHazardFn = [IsHazardDef, TRI, Reg] (MachineInstr *MI) { 484 return IsHazardDef(MI) && MI->modifiesRegister(Reg, TRI); 485 }; 486 487 return getWaitStatesSince(IsHazardFn, Limit); 488 } 489 490 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard, 491 int Limit) { 492 auto IsHazardFn = [IsHazard] (MachineInstr *MI) { 493 return isSSetReg(MI->getOpcode()) && IsHazard(MI); 494 }; 495 496 return getWaitStatesSince(IsHazardFn, Limit); 497 } 498 499 //===----------------------------------------------------------------------===// 500 // No-op Hazard Detection 501 //===----------------------------------------------------------------------===// 502 503 static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, 504 MCRegister Reg) { 505 for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI) 506 BV.set(*RUI); 507 } 508 509 static void addRegsToSet(const SIRegisterInfo &TRI, 510 iterator_range<MachineInstr::const_mop_iterator> Ops, 511 BitVector &Set) { 512 for (const MachineOperand &Op : Ops) { 513 if (Op.isReg()) 514 addRegUnits(TRI, Set, Op.getReg().asMCReg()); 515 } 516 } 517 518 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) { 519 // XXX: Do we need to worry about implicit operands 520 addRegsToSet(TRI, MI.defs(), ClauseDefs); 521 addRegsToSet(TRI, MI.uses(), ClauseUses); 522 } 523 524 static bool breaksSMEMSoftClause(MachineInstr *MI) { 525 return !SIInstrInfo::isSMRD(*MI); 526 } 527 528 static bool breaksVMEMSoftClause(MachineInstr *MI) { 529 return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI); 530 } 531 532 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) { 533 // SMEM soft clause are only present on VI+, and only matter if xnack is 534 // enabled. 535 if (!ST.isXNACKEnabled()) 536 return 0; 537 538 bool IsSMRD = TII.isSMRD(*MEM); 539 540 resetClause(); 541 542 // A soft-clause is any group of consecutive SMEM instructions. The 543 // instructions in this group may return out of order and/or may be 544 // replayed (i.e. the same instruction issued more than once). 545 // 546 // In order to handle these situations correctly we need to make sure that 547 // when a clause has more than one instruction, no instruction in the clause 548 // writes to a register that is read by another instruction in the clause 549 // (including itself). If we encounter this situaion, we need to break the 550 // clause by inserting a non SMEM instruction. 551 552 for (MachineInstr *MI : EmittedInstrs) { 553 // When we hit a non-SMEM instruction then we have passed the start of the 554 // clause and we can stop. 555 if (!MI) 556 break; 557 558 if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI)) 559 break; 560 561 addClauseInst(*MI); 562 } 563 564 if (ClauseDefs.none()) 565 return 0; 566 567 // We need to make sure not to put loads and stores in the same clause if they 568 // use the same address. For now, just start a new clause whenever we see a 569 // store. 570 if (MEM->mayStore()) 571 return 1; 572 573 addClauseInst(*MEM); 574 575 // If the set of defs and uses intersect then we cannot add this instruction 576 // to the clause, so we have a hazard. 577 return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0; 578 } 579 580 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) { 581 int WaitStatesNeeded = 0; 582 583 WaitStatesNeeded = checkSoftClauseHazards(SMRD); 584 585 // This SMRD hazard only affects SI. 586 if (!ST.hasSMRDReadVALUDefHazard()) 587 return WaitStatesNeeded; 588 589 // A read of an SGPR by SMRD instruction requires 4 wait states when the 590 // SGPR was written by a VALU instruction. 591 int SmrdSgprWaitStates = 4; 592 auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); }; 593 auto IsBufferHazardDefFn = [this] (MachineInstr *MI) { return TII.isSALU(*MI); }; 594 595 bool IsBufferSMRD = TII.isBufferSMRD(*SMRD); 596 597 for (const MachineOperand &Use : SMRD->uses()) { 598 if (!Use.isReg()) 599 continue; 600 int WaitStatesNeededForUse = 601 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn, 602 SmrdSgprWaitStates); 603 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 604 605 // This fixes what appears to be undocumented hardware behavior in SI where 606 // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor 607 // needs some number of nops in between. We don't know how many we need, but 608 // let's use 4. This wasn't discovered before probably because the only 609 // case when this happens is when we expand a 64-bit pointer into a full 610 // descriptor and use s_buffer_load_dword instead of s_load_dword, which was 611 // probably never encountered in the closed-source land. 612 if (IsBufferSMRD) { 613 int WaitStatesNeededForUse = 614 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), 615 IsBufferHazardDefFn, 616 SmrdSgprWaitStates); 617 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 618 } 619 } 620 621 return WaitStatesNeeded; 622 } 623 624 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) { 625 if (!ST.hasVMEMReadSGPRVALUDefHazard()) 626 return 0; 627 628 int WaitStatesNeeded = checkSoftClauseHazards(VMEM); 629 630 // A read of an SGPR by a VMEM instruction requires 5 wait states when the 631 // SGPR was written by a VALU Instruction. 632 const int VmemSgprWaitStates = 5; 633 auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); }; 634 for (const MachineOperand &Use : VMEM->uses()) { 635 if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg())) 636 continue; 637 638 int WaitStatesNeededForUse = 639 VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn, 640 VmemSgprWaitStates); 641 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 642 } 643 return WaitStatesNeeded; 644 } 645 646 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) { 647 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 648 const SIInstrInfo *TII = ST.getInstrInfo(); 649 650 // Check for DPP VGPR read after VALU VGPR write and EXEC write. 651 int DppVgprWaitStates = 2; 652 int DppExecWaitStates = 5; 653 int WaitStatesNeeded = 0; 654 auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); }; 655 656 for (const MachineOperand &Use : DPP->uses()) { 657 if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg())) 658 continue; 659 int WaitStatesNeededForUse = 660 DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg(), 661 [](MachineInstr *) { return true; }, 662 DppVgprWaitStates); 663 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 664 } 665 666 WaitStatesNeeded = std::max( 667 WaitStatesNeeded, 668 DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn, 669 DppExecWaitStates)); 670 671 return WaitStatesNeeded; 672 } 673 674 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) { 675 const SIInstrInfo *TII = ST.getInstrInfo(); 676 677 // v_div_fmas requires 4 wait states after a write to vcc from a VALU 678 // instruction. 679 const int DivFMasWaitStates = 4; 680 auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); }; 681 int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn, 682 DivFMasWaitStates); 683 684 return DivFMasWaitStates - WaitStatesNeeded; 685 } 686 687 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) { 688 const SIInstrInfo *TII = ST.getInstrInfo(); 689 unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr); 690 691 const int GetRegWaitStates = 2; 692 auto IsHazardFn = [TII, GetRegHWReg] (MachineInstr *MI) { 693 return GetRegHWReg == getHWReg(TII, *MI); 694 }; 695 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates); 696 697 return GetRegWaitStates - WaitStatesNeeded; 698 } 699 700 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) { 701 const SIInstrInfo *TII = ST.getInstrInfo(); 702 unsigned HWReg = getHWReg(TII, *SetRegInstr); 703 704 const int SetRegWaitStates = ST.getSetRegWaitStates(); 705 auto IsHazardFn = [TII, HWReg] (MachineInstr *MI) { 706 return HWReg == getHWReg(TII, *MI); 707 }; 708 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates); 709 return SetRegWaitStates - WaitStatesNeeded; 710 } 711 712 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) { 713 if (!MI.mayStore()) 714 return -1; 715 716 const SIInstrInfo *TII = ST.getInstrInfo(); 717 unsigned Opcode = MI.getOpcode(); 718 const MCInstrDesc &Desc = MI.getDesc(); 719 720 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); 721 int VDataRCID = -1; 722 if (VDataIdx != -1) 723 VDataRCID = Desc.OpInfo[VDataIdx].RegClass; 724 725 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) { 726 // There is no hazard if the instruction does not use vector regs 727 // (like wbinvl1) 728 if (VDataIdx == -1) 729 return -1; 730 // For MUBUF/MTBUF instructions this hazard only exists if the 731 // instruction is not using a register in the soffset field. 732 const MachineOperand *SOffset = 733 TII->getNamedOperand(MI, AMDGPU::OpName::soffset); 734 // If we have no soffset operand, then assume this field has been 735 // hardcoded to zero. 736 if (AMDGPU::getRegBitWidth(VDataRCID) > 64 && 737 (!SOffset || !SOffset->isReg())) 738 return VDataIdx; 739 } 740 741 // MIMG instructions create a hazard if they don't use a 256-bit T# and 742 // the store size is greater than 8 bytes and they have more than two bits 743 // of their dmask set. 744 // All our MIMG definitions use a 256-bit T#, so we can skip checking for them. 745 if (TII->isMIMG(MI)) { 746 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); 747 assert(SRsrcIdx != -1 && 748 AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256); 749 (void)SRsrcIdx; 750 } 751 752 if (TII->isFLAT(MI)) { 753 int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); 754 if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64) 755 return DataIdx; 756 } 757 758 return -1; 759 } 760 761 int 762 GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def, 763 const MachineRegisterInfo &MRI) { 764 // Helper to check for the hazard where VMEM instructions that store more than 765 // 8 bytes can have there store data over written by the next instruction. 766 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 767 768 const int VALUWaitStates = 1; 769 int WaitStatesNeeded = 0; 770 771 if (!TRI->isVectorRegister(MRI, Def.getReg())) 772 return WaitStatesNeeded; 773 Register Reg = Def.getReg(); 774 auto IsHazardFn = [this, Reg, TRI] (MachineInstr *MI) { 775 int DataIdx = createsVALUHazard(*MI); 776 return DataIdx >= 0 && 777 TRI->regsOverlap(MI->getOperand(DataIdx).getReg(), Reg); 778 }; 779 int WaitStatesNeededForDef = 780 VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates); 781 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 782 783 return WaitStatesNeeded; 784 } 785 786 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) { 787 // This checks for the hazard where VMEM instructions that store more than 788 // 8 bytes can have there store data over written by the next instruction. 789 if (!ST.has12DWordStoreHazard()) 790 return 0; 791 792 const MachineRegisterInfo &MRI = MF.getRegInfo(); 793 int WaitStatesNeeded = 0; 794 795 for (const MachineOperand &Def : VALU->defs()) { 796 WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI)); 797 } 798 799 return WaitStatesNeeded; 800 } 801 802 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) { 803 // This checks for hazards associated with inline asm statements. 804 // Since inline asms can contain just about anything, we use this 805 // to call/leverage other check*Hazard routines. Note that 806 // this function doesn't attempt to address all possible inline asm 807 // hazards (good luck), but is a collection of what has been 808 // problematic thus far. 809 810 // see checkVALUHazards() 811 if (!ST.has12DWordStoreHazard()) 812 return 0; 813 814 const MachineRegisterInfo &MRI = MF.getRegInfo(); 815 int WaitStatesNeeded = 0; 816 817 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands(); 818 I != E; ++I) { 819 const MachineOperand &Op = IA->getOperand(I); 820 if (Op.isReg() && Op.isDef()) { 821 WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI)); 822 } 823 } 824 825 return WaitStatesNeeded; 826 } 827 828 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) { 829 const SIInstrInfo *TII = ST.getInstrInfo(); 830 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 831 const MachineRegisterInfo &MRI = MF.getRegInfo(); 832 833 const MachineOperand *LaneSelectOp = 834 TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1); 835 836 if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg())) 837 return 0; 838 839 Register LaneSelectReg = LaneSelectOp->getReg(); 840 auto IsHazardFn = [TII] (MachineInstr *MI) { 841 return TII->isVALU(*MI); 842 }; 843 844 const int RWLaneWaitStates = 4; 845 int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn, 846 RWLaneWaitStates); 847 return RWLaneWaitStates - WaitStatesSince; 848 } 849 850 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) { 851 if (!ST.hasRFEHazards()) 852 return 0; 853 854 const SIInstrInfo *TII = ST.getInstrInfo(); 855 856 const int RFEWaitStates = 1; 857 858 auto IsHazardFn = [TII] (MachineInstr *MI) { 859 return getHWReg(TII, *MI) == AMDGPU::Hwreg::ID_TRAPSTS; 860 }; 861 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates); 862 return RFEWaitStates - WaitStatesNeeded; 863 } 864 865 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) { 866 const SIInstrInfo *TII = ST.getInstrInfo(); 867 const int SMovRelWaitStates = 1; 868 auto IsHazardFn = [TII] (MachineInstr *MI) { 869 return TII->isSALU(*MI); 870 }; 871 return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, 872 SMovRelWaitStates); 873 } 874 875 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) { 876 fixVMEMtoScalarWriteHazards(MI); 877 fixVcmpxPermlaneHazards(MI); 878 fixSMEMtoVectorWriteHazards(MI); 879 fixVcmpxExecWARHazard(MI); 880 fixLdsBranchVmemWARHazard(MI); 881 } 882 883 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) { 884 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI)) 885 return false; 886 887 const SIInstrInfo *TII = ST.getInstrInfo(); 888 auto IsHazardFn = [TII] (MachineInstr *MI) { 889 return TII->isVOPC(*MI); 890 }; 891 892 auto IsExpiredFn = [] (MachineInstr *MI, int) { 893 if (!MI) 894 return false; 895 unsigned Opc = MI->getOpcode(); 896 return SIInstrInfo::isVALU(*MI) && 897 Opc != AMDGPU::V_NOP_e32 && 898 Opc != AMDGPU::V_NOP_e64 && 899 Opc != AMDGPU::V_NOP_sdwa; 900 }; 901 902 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 903 std::numeric_limits<int>::max()) 904 return false; 905 906 // V_NOP will be discarded by SQ. 907 // Use V_MOB_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE* 908 // which is always a VGPR and available. 909 auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0); 910 Register Reg = Src0->getReg(); 911 bool IsUndef = Src0->isUndef(); 912 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 913 TII->get(AMDGPU::V_MOV_B32_e32)) 914 .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0)) 915 .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill); 916 917 return true; 918 } 919 920 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) { 921 if (!ST.hasVMEMtoScalarWriteHazard()) 922 return false; 923 924 if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI)) 925 return false; 926 927 if (MI->getNumDefs() == 0) 928 return false; 929 930 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 931 932 auto IsHazardFn = [TRI, MI] (MachineInstr *I) { 933 if (!SIInstrInfo::isVMEM(*I) && !SIInstrInfo::isDS(*I) && 934 !SIInstrInfo::isFLAT(*I)) 935 return false; 936 937 for (const MachineOperand &Def : MI->defs()) { 938 MachineOperand *Op = I->findRegisterUseOperand(Def.getReg(), false, TRI); 939 if (!Op) 940 continue; 941 return true; 942 } 943 return false; 944 }; 945 946 auto IsExpiredFn = [](MachineInstr *MI, int) { 947 return MI && (SIInstrInfo::isVALU(*MI) || 948 (MI->getOpcode() == AMDGPU::S_WAITCNT && 949 !MI->getOperand(0).getImm()) || 950 (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 951 MI->getOperand(0).getImm() == 0xffe3)); 952 }; 953 954 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 955 std::numeric_limits<int>::max()) 956 return false; 957 958 const SIInstrInfo *TII = ST.getInstrInfo(); 959 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 960 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) 961 .addImm(0xffe3); 962 return true; 963 } 964 965 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) { 966 if (!ST.hasSMEMtoVectorWriteHazard()) 967 return false; 968 969 if (!SIInstrInfo::isVALU(*MI)) 970 return false; 971 972 unsigned SDSTName; 973 switch (MI->getOpcode()) { 974 case AMDGPU::V_READLANE_B32: 975 case AMDGPU::V_READFIRSTLANE_B32: 976 SDSTName = AMDGPU::OpName::vdst; 977 break; 978 default: 979 SDSTName = AMDGPU::OpName::sdst; 980 break; 981 } 982 983 const SIInstrInfo *TII = ST.getInstrInfo(); 984 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 985 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU()); 986 const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName); 987 if (!SDST) { 988 for (const auto &MO : MI->implicit_operands()) { 989 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) { 990 SDST = &MO; 991 break; 992 } 993 } 994 } 995 996 if (!SDST) 997 return false; 998 999 const Register SDSTReg = SDST->getReg(); 1000 auto IsHazardFn = [SDSTReg, TRI] (MachineInstr *I) { 1001 return SIInstrInfo::isSMRD(*I) && I->readsRegister(SDSTReg, TRI); 1002 }; 1003 1004 auto IsExpiredFn = [TII, IV] (MachineInstr *MI, int) { 1005 if (MI) { 1006 if (TII->isSALU(*MI)) { 1007 switch (MI->getOpcode()) { 1008 case AMDGPU::S_SETVSKIP: 1009 case AMDGPU::S_VERSION: 1010 case AMDGPU::S_WAITCNT_VSCNT: 1011 case AMDGPU::S_WAITCNT_VMCNT: 1012 case AMDGPU::S_WAITCNT_EXPCNT: 1013 // These instructions cannot not mitigate the hazard. 1014 return false; 1015 case AMDGPU::S_WAITCNT_LGKMCNT: 1016 // Reducing lgkmcnt count to 0 always mitigates the hazard. 1017 return (MI->getOperand(1).getImm() == 0) && 1018 (MI->getOperand(0).getReg() == AMDGPU::SGPR_NULL); 1019 case AMDGPU::S_WAITCNT: { 1020 const int64_t Imm = MI->getOperand(0).getImm(); 1021 AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm); 1022 return (Decoded.LgkmCnt == 0); 1023 } 1024 default: 1025 // SOPP instructions cannot mitigate the hazard. 1026 if (TII->isSOPP(*MI)) 1027 return false; 1028 // At this point the SALU can be assumed to mitigate the hazard 1029 // because either: 1030 // (a) it is independent of the at risk SMEM (breaking chain), 1031 // or 1032 // (b) it is dependent on the SMEM, in which case an appropriate 1033 // s_waitcnt lgkmcnt _must_ exist between it and the at risk 1034 // SMEM instruction. 1035 return true; 1036 } 1037 } 1038 } 1039 return false; 1040 }; 1041 1042 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1043 std::numeric_limits<int>::max()) 1044 return false; 1045 1046 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1047 TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL) 1048 .addImm(0); 1049 return true; 1050 } 1051 1052 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) { 1053 if (!ST.hasVcmpxExecWARHazard() || !SIInstrInfo::isVALU(*MI)) 1054 return false; 1055 1056 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1057 if (!MI->modifiesRegister(AMDGPU::EXEC, TRI)) 1058 return false; 1059 1060 auto IsHazardFn = [TRI] (MachineInstr *I) { 1061 if (SIInstrInfo::isVALU(*I)) 1062 return false; 1063 return I->readsRegister(AMDGPU::EXEC, TRI); 1064 }; 1065 1066 const SIInstrInfo *TII = ST.getInstrInfo(); 1067 auto IsExpiredFn = [TII, TRI] (MachineInstr *MI, int) { 1068 if (!MI) 1069 return false; 1070 if (SIInstrInfo::isVALU(*MI)) { 1071 if (TII->getNamedOperand(*MI, AMDGPU::OpName::sdst)) 1072 return true; 1073 for (auto MO : MI->implicit_operands()) 1074 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) 1075 return true; 1076 } 1077 if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1078 (MI->getOperand(0).getImm() & 0xfffe) == 0xfffe) 1079 return true; 1080 return false; 1081 }; 1082 1083 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1084 std::numeric_limits<int>::max()) 1085 return false; 1086 1087 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1088 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) 1089 .addImm(0xfffe); 1090 return true; 1091 } 1092 1093 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) { 1094 if (!ST.hasLdsBranchVmemWARHazard()) 1095 return false; 1096 1097 auto IsHazardInst = [] (const MachineInstr *MI) { 1098 if (SIInstrInfo::isDS(*MI)) 1099 return 1; 1100 if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isSegmentSpecificFLAT(*MI)) 1101 return 2; 1102 return 0; 1103 }; 1104 1105 auto InstType = IsHazardInst(MI); 1106 if (!InstType) 1107 return false; 1108 1109 auto IsExpiredFn = [&IsHazardInst] (MachineInstr *I, int) { 1110 return I && (IsHazardInst(I) || 1111 (I->getOpcode() == AMDGPU::S_WAITCNT_VSCNT && 1112 I->getOperand(0).getReg() == AMDGPU::SGPR_NULL && 1113 !I->getOperand(1).getImm())); 1114 }; 1115 1116 auto IsHazardFn = [InstType, &IsHazardInst] (MachineInstr *I) { 1117 if (!I->isBranch()) 1118 return false; 1119 1120 auto IsHazardFn = [InstType, IsHazardInst] (MachineInstr *I) { 1121 auto InstType2 = IsHazardInst(I); 1122 return InstType2 && InstType != InstType2; 1123 }; 1124 1125 auto IsExpiredFn = [InstType, &IsHazardInst] (MachineInstr *I, int) { 1126 if (!I) 1127 return false; 1128 1129 auto InstType2 = IsHazardInst(I); 1130 if (InstType == InstType2) 1131 return true; 1132 1133 return I->getOpcode() == AMDGPU::S_WAITCNT_VSCNT && 1134 I->getOperand(0).getReg() == AMDGPU::SGPR_NULL && 1135 !I->getOperand(1).getImm(); 1136 }; 1137 1138 return ::getWaitStatesSince(IsHazardFn, I, IsExpiredFn) != 1139 std::numeric_limits<int>::max(); 1140 }; 1141 1142 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1143 std::numeric_limits<int>::max()) 1144 return false; 1145 1146 const SIInstrInfo *TII = ST.getInstrInfo(); 1147 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1148 TII->get(AMDGPU::S_WAITCNT_VSCNT)) 1149 .addReg(AMDGPU::SGPR_NULL, RegState::Undef) 1150 .addImm(0); 1151 1152 return true; 1153 } 1154 1155 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) { 1156 int NSAtoVMEMWaitStates = 1; 1157 1158 if (!ST.hasNSAtoVMEMBug()) 1159 return 0; 1160 1161 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI)) 1162 return 0; 1163 1164 const SIInstrInfo *TII = ST.getInstrInfo(); 1165 const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset); 1166 if (!Offset || (Offset->getImm() & 6) == 0) 1167 return 0; 1168 1169 auto IsHazardFn = [TII] (MachineInstr *I) { 1170 if (!SIInstrInfo::isMIMG(*I)) 1171 return false; 1172 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I->getOpcode()); 1173 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA && 1174 TII->getInstSizeInBytes(*I) >= 16; 1175 }; 1176 1177 return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1); 1178 } 1179 1180 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) { 1181 int FPAtomicToDenormModeWaitStates = 3; 1182 1183 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE) 1184 return 0; 1185 1186 auto IsHazardFn = [] (MachineInstr *I) { 1187 if (!SIInstrInfo::isVMEM(*I) && !SIInstrInfo::isFLAT(*I)) 1188 return false; 1189 return SIInstrInfo::isFPAtomic(*I); 1190 }; 1191 1192 auto IsExpiredFn = [] (MachineInstr *MI, int WaitStates) { 1193 if (WaitStates >= 3 || SIInstrInfo::isVALU(*MI)) 1194 return true; 1195 1196 switch (MI->getOpcode()) { 1197 case AMDGPU::S_WAITCNT: 1198 case AMDGPU::S_WAITCNT_VSCNT: 1199 case AMDGPU::S_WAITCNT_VMCNT: 1200 case AMDGPU::S_WAITCNT_EXPCNT: 1201 case AMDGPU::S_WAITCNT_LGKMCNT: 1202 case AMDGPU::S_WAIT_IDLE: 1203 return true; 1204 default: 1205 break; 1206 } 1207 1208 return false; 1209 }; 1210 1211 1212 return FPAtomicToDenormModeWaitStates - 1213 ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn); 1214 } 1215 1216 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) { 1217 assert(SIInstrInfo::isMAI(*MI)); 1218 1219 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI); 1220 } 1221 1222 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) { 1223 int WaitStatesNeeded = 0; 1224 unsigned Opc = MI->getOpcode(); 1225 1226 auto IsVALUFn = [] (MachineInstr *MI) { 1227 return SIInstrInfo::isVALU(*MI); 1228 }; 1229 1230 if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write 1231 const int LegacyVALUWritesVGPRWaitStates = 2; 1232 const int VALUWritesExecWaitStates = 4; 1233 const int MaxWaitStates = 4; 1234 1235 int WaitStatesNeededForUse = VALUWritesExecWaitStates - 1236 getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates); 1237 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1238 1239 if (WaitStatesNeeded < MaxWaitStates) { 1240 for (const MachineOperand &Use : MI->explicit_uses()) { 1241 const int MaxWaitStates = 2; 1242 1243 if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg())) 1244 continue; 1245 1246 int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates - 1247 getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates); 1248 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1249 1250 if (WaitStatesNeeded == MaxWaitStates) 1251 break; 1252 } 1253 } 1254 } 1255 1256 auto IsMFMAFn = [] (MachineInstr *MI) { 1257 return SIInstrInfo::isMAI(*MI) && 1258 MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1259 MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; 1260 }; 1261 1262 for (const MachineOperand &Op : MI->explicit_operands()) { 1263 if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg())) 1264 continue; 1265 1266 if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 1267 continue; 1268 1269 const int MFMAWritesAGPROverlappedSrcABWaitStates = 4; 1270 const int MFMAWritesAGPROverlappedSrcCWaitStates = 2; 1271 const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4; 1272 const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10; 1273 const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18; 1274 const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1; 1275 const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7; 1276 const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15; 1277 const int MaxWaitStates = 18; 1278 Register Reg = Op.getReg(); 1279 unsigned HazardDefLatency = 0; 1280 1281 auto IsOverlappedMFMAFn = [Reg, &IsMFMAFn, &HazardDefLatency, this] 1282 (MachineInstr *MI) { 1283 if (!IsMFMAFn(MI)) 1284 return false; 1285 Register DstReg = MI->getOperand(0).getReg(); 1286 if (DstReg == Reg) 1287 return false; 1288 HazardDefLatency = std::max(HazardDefLatency, 1289 TSchedModel.computeInstrLatency(MI)); 1290 return TRI.regsOverlap(DstReg, Reg); 1291 }; 1292 1293 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, 1294 MaxWaitStates); 1295 int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates; 1296 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 1297 int OpNo = MI->getOperandNo(&Op); 1298 if (OpNo == SrcCIdx) { 1299 NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates; 1300 } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) { 1301 switch (HazardDefLatency) { 1302 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates; 1303 break; 1304 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates; 1305 break; 1306 case 16: LLVM_FALLTHROUGH; 1307 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates; 1308 break; 1309 } 1310 } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { 1311 switch (HazardDefLatency) { 1312 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates; 1313 break; 1314 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates; 1315 break; 1316 case 16: LLVM_FALLTHROUGH; 1317 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates; 1318 break; 1319 } 1320 } 1321 1322 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1323 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1324 1325 if (WaitStatesNeeded == MaxWaitStates) 1326 return WaitStatesNeeded; // Early exit. 1327 1328 auto IsAccVgprWriteFn = [Reg, this] (MachineInstr *MI) { 1329 if (MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 1330 return false; 1331 Register DstReg = MI->getOperand(0).getReg(); 1332 return TRI.regsOverlap(Reg, DstReg); 1333 }; 1334 1335 const int AccVGPRWriteMFMAReadSrcCWaitStates = 1; 1336 const int AccVGPRWriteMFMAReadSrcABWaitStates = 3; 1337 const int AccVGPRWriteAccVgprReadWaitStates = 3; 1338 NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates; 1339 if (OpNo == SrcCIdx) 1340 NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates; 1341 else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) 1342 NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates; 1343 1344 WaitStatesNeededForUse = NeedWaitStates - 1345 getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates); 1346 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1347 1348 if (WaitStatesNeeded == MaxWaitStates) 1349 return WaitStatesNeeded; // Early exit. 1350 } 1351 1352 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { 1353 const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0; 1354 const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5; 1355 const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13; 1356 const int MaxWaitStates = 13; 1357 Register DstReg = MI->getOperand(0).getReg(); 1358 unsigned HazardDefLatency = 0; 1359 1360 auto IsSrcCMFMAFn = [DstReg, &IsMFMAFn, &HazardDefLatency, this] 1361 (MachineInstr *MI) { 1362 if (!IsMFMAFn(MI)) 1363 return false; 1364 Register Reg = TII.getNamedOperand(*MI, AMDGPU::OpName::src2)->getReg(); 1365 HazardDefLatency = std::max(HazardDefLatency, 1366 TSchedModel.computeInstrLatency(MI)); 1367 return TRI.regsOverlap(Reg, DstReg); 1368 }; 1369 1370 int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates); 1371 int NeedWaitStates; 1372 switch (HazardDefLatency) { 1373 case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates; 1374 break; 1375 case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates; 1376 break; 1377 case 16: LLVM_FALLTHROUGH; 1378 default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates; 1379 break; 1380 } 1381 1382 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince; 1383 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1384 } 1385 1386 return WaitStatesNeeded; 1387 } 1388 1389 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { 1390 int WaitStatesNeeded = 0; 1391 unsigned Opc = MI->getOpcode(); 1392 1393 auto IsMFMAFn = [] (MachineInstr *MI) { 1394 return SIInstrInfo::isMAI(*MI) && 1395 MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1396 MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; 1397 }; 1398 1399 auto IsLegacyVALUFn = [&IsMFMAFn] (MachineInstr *MI) { 1400 return SIInstrInfo::isVALU(*MI) && !IsMFMAFn(MI); 1401 }; 1402 1403 auto IsLegacyVALUNotDotFn = [&IsMFMAFn] (MachineInstr *MI) { 1404 return SIInstrInfo::isVALU(*MI) && 1405 !IsMFMAFn(MI) && !SIInstrInfo::isDOT(*MI); 1406 }; 1407 1408 if (!IsMFMAFn(MI)) 1409 return WaitStatesNeeded; 1410 1411 const int VALUWritesExecWaitStates = 4; 1412 int WaitStatesNeededForUse = VALUWritesExecWaitStates - 1413 getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn, 1414 VALUWritesExecWaitStates); 1415 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1416 1417 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 1418 1419 // Loop for both DGEMM and S/HGEMM 2nd instruction. 1420 for (const MachineOperand &Use : MI->explicit_uses()) { 1421 const int LegacyVALUNotDotWritesVGPRWaitStates = 2; 1422 const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2; 1423 const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8; 1424 const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16; 1425 const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3; 1426 const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9; 1427 const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17; 1428 const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9; 1429 const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4; 1430 const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5; 1431 const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11; 1432 const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19; 1433 const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6; 1434 const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11; 1435 const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4; 1436 const int MaxWaitStates = 19; 1437 1438 if (!Use.isReg()) 1439 continue; 1440 unsigned Reg = Use.getReg(); 1441 bool FullReg; 1442 MachineInstr *MI1; 1443 1444 auto IsOverlappedDGEMMorXDLFn = [Reg, &IsMFMAFn, &FullReg, &MI1, this] 1445 (MachineInstr *MI) { 1446 if (!IsMFMAFn(MI)) 1447 return false; 1448 if (!isDGEMM(MI->getOpcode()) && !isXDL(ST, *MI)) 1449 return false; 1450 Register DstReg = MI->getOperand(0).getReg(); 1451 FullReg = (DstReg == Reg); 1452 MI1 = MI; 1453 return TRI.regsOverlap(DstReg, Reg); 1454 }; 1455 1456 WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates - 1457 getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates); 1458 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1459 1460 int NumWaitStates = getWaitStatesSinceDef(Reg, IsOverlappedDGEMMorXDLFn, 1461 MaxWaitStates); 1462 if (NumWaitStates == std::numeric_limits<int>::max()) 1463 continue; 1464 1465 int OpNo = MI->getOperandNo(&Use); 1466 unsigned Opc1 = MI1->getOpcode(); 1467 int NeedWaitStates = 0; 1468 if (OpNo == SrcCIdx) { 1469 if (!isDGEMM(Opc) && isDGEMM(Opc1)) { 1470 NeedWaitStates = 0; 1471 } else if (FullReg) { 1472 if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 1473 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) && 1474 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 1475 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64)) 1476 NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates; 1477 } else { 1478 switch (Opc1) { 1479 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: 1480 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: 1481 if (!isXDL(ST, *MI)) 1482 NeedWaitStates = DMFMA16x16WritesVGPROverlappedSrcCWaitStates; 1483 break; 1484 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: 1485 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: 1486 if (!isXDL(ST, *MI)) 1487 NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates; 1488 break; 1489 default: 1490 switch (TSchedModel.computeInstrLatency(MI1)) { 1491 case 2: 1492 NeedWaitStates = isDGEMM(Opc) 1493 ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates 1494 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates; 1495 break; 1496 case 8: 1497 NeedWaitStates = isDGEMM(Opc) 1498 ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates 1499 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates; 1500 break; 1501 case 16: LLVM_FALLTHROUGH; 1502 default: 1503 NeedWaitStates = isDGEMM(Opc) 1504 ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates 1505 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates; 1506 } 1507 } 1508 } 1509 } else { 1510 switch (Opc1) { 1511 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: 1512 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: 1513 NeedWaitStates = DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates; 1514 break; 1515 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: 1516 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: 1517 NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates; 1518 break; 1519 default: 1520 switch (TSchedModel.computeInstrLatency(MI1)) { 1521 case 2: 1522 NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates; 1523 break; 1524 case 8: 1525 NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates; 1526 break; 1527 case 16: LLVM_FALLTHROUGH; 1528 default: 1529 NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates; 1530 } 1531 } 1532 } 1533 if (WaitStatesNeeded >= NeedWaitStates) 1534 continue; 1535 1536 WaitStatesNeededForUse = NeedWaitStates - NumWaitStates; 1537 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1538 1539 if (WaitStatesNeeded == MaxWaitStates) 1540 break; 1541 } 1542 1543 return WaitStatesNeeded; 1544 } 1545 1546 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) { 1547 // On gfx90a+ releveant hazards are checked in checkMAIVALUHazards() 1548 if (!ST.hasMAIInsts() || ST.hasGFX90AInsts()) 1549 return 0; 1550 1551 int WaitStatesNeeded = 0; 1552 1553 auto IsAccVgprReadFn = [] (MachineInstr *MI) { 1554 return MI->getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64; 1555 }; 1556 1557 for (const MachineOperand &Op : MI->explicit_uses()) { 1558 if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg())) 1559 continue; 1560 1561 Register Reg = Op.getReg(); 1562 1563 const int AccVgprReadLdStWaitStates = 2; 1564 const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1; 1565 const int MaxWaitStates = 2; 1566 1567 int WaitStatesNeededForUse = AccVgprReadLdStWaitStates - 1568 getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates); 1569 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1570 1571 if (WaitStatesNeeded == MaxWaitStates) 1572 return WaitStatesNeeded; // Early exit. 1573 1574 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](MachineInstr *MI) { 1575 if (MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 && 1576 MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 1577 return false; 1578 auto IsVALUFn = [] (MachineInstr *MI) { 1579 return SIInstrInfo::isVALU(*MI) && !SIInstrInfo::isMAI(*MI); 1580 }; 1581 return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) < 1582 std::numeric_limits<int>::max(); 1583 }; 1584 1585 WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates - 1586 getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates); 1587 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1588 } 1589 1590 return WaitStatesNeeded; 1591 } 1592 1593 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { 1594 if (!ST.hasGFX90AInsts()) 1595 return 0; 1596 1597 auto IsMFMAFn = [] (MachineInstr *MI) -> bool { 1598 return SIInstrInfo::isMAI(*MI) && 1599 MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1600 MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; 1601 }; 1602 1603 auto IsDGEMMFn = [] (MachineInstr *MI) -> bool { 1604 return isDGEMM(MI->getOpcode()); 1605 }; 1606 1607 // This is checked in checkMAIHazards90A() 1608 if (IsMFMAFn(MI)) 1609 return 0; 1610 1611 int WaitStatesNeeded = 0; 1612 1613 bool IsMemOrExport = SIInstrInfo::isVMEM(*MI) || 1614 SIInstrInfo::isFLAT(*MI) || 1615 SIInstrInfo::isDS(*MI) || 1616 SIInstrInfo::isEXP(*MI); 1617 bool IsVALU = SIInstrInfo::isVALU(*MI); 1618 1619 MachineInstr *MFMA = nullptr; 1620 unsigned Reg; 1621 auto IsDGEMMorXDLWriteFn = [&Reg, &IsMFMAFn, &MFMA, this] (MachineInstr *MI) { 1622 if (!IsMFMAFn(MI) || !TRI.regsOverlap(MI->getOperand(0).getReg(), Reg)) 1623 return false; 1624 if (!isDGEMM(MI->getOpcode()) && !isXDL(ST, *MI)) 1625 return false; 1626 MFMA = MI; 1627 return true; 1628 }; 1629 1630 MachineInstr *DOT = nullptr; 1631 auto IsDotWriteFn = [&Reg, &DOT, this] (MachineInstr *MI) { 1632 if (!SIInstrInfo::isDOT(*MI) || 1633 !TRI.regsOverlap(MI->getOperand(0).getReg(), Reg)) 1634 return false; 1635 DOT = MI; 1636 return true; 1637 }; 1638 1639 int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 1640 AMDGPU::OpName::src2); 1641 1642 if (IsMemOrExport || IsVALU) { 1643 const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5; 1644 const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11; 1645 const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19; 1646 const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9; 1647 const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18; 1648 const int DMFMA4x4WriteVgprVALUReadWaitStates = 6; 1649 const int DMFMA16x16WriteVgprVALUReadWaitStates = 11; 1650 const int DotWriteSameDotReadSrcAB = 3; 1651 const int DotWriteDifferentVALURead = 3; 1652 const int MaxWaitStates = 19; 1653 1654 for (const MachineOperand &Use : MI->explicit_uses()) { 1655 if (!Use.isReg()) 1656 continue; 1657 Reg = Use.getReg(); 1658 1659 DOT = nullptr; 1660 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn, 1661 MaxWaitStates); 1662 if (DOT) { 1663 int NeedWaitStates = 0; 1664 if (DOT->getOpcode() == MI->getOpcode()) { 1665 if (&Use - &MI->getOperand(0) != SrcCIdx) 1666 NeedWaitStates = DotWriteSameDotReadSrcAB; 1667 } else { 1668 NeedWaitStates = DotWriteDifferentVALURead; 1669 } 1670 1671 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1672 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1673 } 1674 1675 MFMA = nullptr; 1676 WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDGEMMorXDLWriteFn, 1677 MaxWaitStates); 1678 if (!MFMA) 1679 continue; 1680 1681 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); 1682 int NeedWaitStates = MaxWaitStates; 1683 switch (HazardDefLatency) { 1684 case 2: 1685 NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates; 1686 break; 1687 case 4: 1688 assert(isDGEMM(MFMA->getOpcode())); 1689 NeedWaitStates = 1690 IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates 1691 : DMFMA4x4WriteVgprVALUReadWaitStates; 1692 break; 1693 case 8: 1694 NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates; 1695 break; 1696 case 16: LLVM_FALLTHROUGH; 1697 default: 1698 NeedWaitStates = 1699 isDGEMM(MFMA->getOpcode()) 1700 ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates 1701 : DMFMA16x16WriteVgprVALUReadWaitStates 1702 : SMFMA32x32WriteVgprVALUMemExpReadWaitStates; 1703 break; 1704 } 1705 1706 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1707 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1708 1709 if (WaitStatesNeeded == MaxWaitStates) 1710 break; 1711 } 1712 } 1713 1714 unsigned Opc = MI->getOpcode(); 1715 const int DMFMAToFMA64WaitStates = 2; 1716 if ((Opc == AMDGPU::V_FMA_F64_e64 || 1717 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 || 1718 Opc == AMDGPU::V_FMAC_F64_dpp) && 1719 WaitStatesNeeded < DMFMAToFMA64WaitStates) { 1720 int WaitStatesNeededForUse = DMFMAToFMA64WaitStates - 1721 getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates); 1722 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1723 } 1724 1725 if (!IsVALU && !IsMemOrExport) 1726 return WaitStatesNeeded; 1727 1728 for (const MachineOperand &Def : MI->defs()) { 1729 const int SMFMA4x4WriteVgprVALUWawWaitStates = 5; 1730 const int SMFMA16x16WriteVgprVALUWawWaitStates = 11; 1731 const int SMFMA32x32WriteVgprVALUWawWaitStates = 19; 1732 const int SMFMA4x4ReadVgprVALUWarWaitStates = 1; 1733 const int SMFMA16x16ReadVgprVALUWarWaitStates = 7; 1734 const int SMFMA32x32ReadVgprVALUWarWaitStates = 15; 1735 const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6; 1736 const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11; 1737 const int DotWriteDifferentVALUWrite = 3; 1738 const int MaxWaitStates = 19; 1739 const int MaxWarWaitStates = 15; 1740 1741 Reg = Def.getReg(); 1742 1743 DOT = nullptr; 1744 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn, 1745 MaxWaitStates); 1746 if (DOT && DOT->getOpcode() != MI->getOpcode()) 1747 WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite - 1748 WaitStatesSinceDef); 1749 1750 MFMA = nullptr; 1751 WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDGEMMorXDLWriteFn, 1752 MaxWaitStates); 1753 if (MFMA) { 1754 int NeedWaitStates = MaxWaitStates; 1755 switch (TSchedModel.computeInstrLatency(MFMA)) { 1756 case 2: 1757 NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates; 1758 break; 1759 case 4: 1760 assert(isDGEMM(MFMA->getOpcode())); 1761 NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates; 1762 break; 1763 case 8: 1764 NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates; 1765 break; 1766 case 16: LLVM_FALLTHROUGH; 1767 default: 1768 NeedWaitStates = isDGEMM(MFMA->getOpcode()) 1769 ? DMFMA16x16WriteVgprVALUWriteWaitStates 1770 : SMFMA32x32WriteVgprVALUWawWaitStates; 1771 break; 1772 } 1773 1774 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1775 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1776 1777 if (WaitStatesNeeded == MaxWaitStates) 1778 break; 1779 } 1780 1781 auto IsSMFMAReadAsCFn = [&Reg, &IsMFMAFn, &MFMA, this] 1782 (MachineInstr *MI) { 1783 if (!IsMFMAFn(MI) || isDGEMM(MI->getOpcode()) || 1784 !MI->readsRegister(Reg, &TRI)) 1785 return false; 1786 1787 MachineOperand *SrcC = TII.getNamedOperand(*MI, AMDGPU::OpName::src2); 1788 assert(SrcC); 1789 if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg)) 1790 return false; 1791 1792 MFMA = MI; 1793 return true; 1794 }; 1795 1796 MFMA = nullptr; 1797 int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn, 1798 MaxWarWaitStates); 1799 if (!MFMA) 1800 continue; 1801 1802 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); 1803 int NeedWaitStates = MaxWaitStates; 1804 switch (HazardDefLatency) { 1805 case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates; 1806 break; 1807 case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates; 1808 break; 1809 case 16: LLVM_FALLTHROUGH; 1810 default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates; 1811 break; 1812 } 1813 1814 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse; 1815 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1816 } 1817 1818 return WaitStatesNeeded; 1819 } 1820 1821 bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) { 1822 if (!SU->isInstr()) 1823 return false; 1824 1825 MachineInstr *MAI = nullptr; 1826 auto IsMFMAFn = [&MAI] (MachineInstr *MI) { 1827 MAI = nullptr; 1828 if (SIInstrInfo::isMAI(*MI) && 1829 MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1830 MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64) 1831 MAI = MI; 1832 return MAI != nullptr; 1833 }; 1834 1835 MachineInstr *MI = SU->getInstr(); 1836 if (IsMFMAFn(MI)) { 1837 int W = getWaitStatesSince(IsMFMAFn, 16); 1838 if (MAI) 1839 return W < (int)TSchedModel.computeInstrLatency(MAI); 1840 } 1841 1842 return false; 1843 } 1844