1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements hazard recognizers for scheduling on GCN processors. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "GCNHazardRecognizer.h" 14 #include "GCNSubtarget.h" 15 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 16 #include "llvm/CodeGen/MachineFunction.h" 17 #include "llvm/CodeGen/ScheduleDAG.h" 18 #include "llvm/Support/TargetParser.h" 19 20 using namespace llvm; 21 22 //===----------------------------------------------------------------------===// 23 // Hazard Recoginizer Implementation 24 //===----------------------------------------------------------------------===// 25 26 GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) : 27 IsHazardRecognizerMode(false), 28 CurrCycleInstr(nullptr), 29 MF(MF), 30 ST(MF.getSubtarget<GCNSubtarget>()), 31 TII(*ST.getInstrInfo()), 32 TRI(TII.getRegisterInfo()), 33 ClauseUses(TRI.getNumRegUnits()), 34 ClauseDefs(TRI.getNumRegUnits()) { 35 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5; 36 TSchedModel.init(&ST); 37 } 38 39 void GCNHazardRecognizer::Reset() { 40 EmittedInstrs.clear(); 41 } 42 43 void GCNHazardRecognizer::EmitInstruction(SUnit *SU) { 44 EmitInstruction(SU->getInstr()); 45 } 46 47 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) { 48 CurrCycleInstr = MI; 49 } 50 51 static bool isDivFMas(unsigned Opcode) { 52 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64; 53 } 54 55 static bool isSGetReg(unsigned Opcode) { 56 return Opcode == AMDGPU::S_GETREG_B32; 57 } 58 59 static bool isSSetReg(unsigned Opcode) { 60 switch (Opcode) { 61 case AMDGPU::S_SETREG_B32: 62 case AMDGPU::S_SETREG_B32_mode: 63 case AMDGPU::S_SETREG_IMM32_B32: 64 case AMDGPU::S_SETREG_IMM32_B32_mode: 65 return true; 66 } 67 return false; 68 } 69 70 static bool isRWLane(unsigned Opcode) { 71 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32; 72 } 73 74 static bool isRFE(unsigned Opcode) { 75 return Opcode == AMDGPU::S_RFE_B64; 76 } 77 78 static bool isSMovRel(unsigned Opcode) { 79 switch (Opcode) { 80 case AMDGPU::S_MOVRELS_B32: 81 case AMDGPU::S_MOVRELS_B64: 82 case AMDGPU::S_MOVRELD_B32: 83 case AMDGPU::S_MOVRELD_B64: 84 return true; 85 default: 86 return false; 87 } 88 } 89 90 static bool isDGEMM(unsigned Opcode) { 91 return Opcode == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 92 Opcode == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64 || 93 Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_e64 || 94 Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64; 95 } 96 97 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) { 98 unsigned Opcode = MI.getOpcode(); 99 100 if (!SIInstrInfo::isMAI(MI) || 101 isDGEMM(Opcode) || 102 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 || 103 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64) 104 return false; 105 106 return true; 107 } 108 109 static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, 110 const MachineInstr &MI) { 111 if (TII.isAlwaysGDS(MI.getOpcode())) 112 return true; 113 114 switch (MI.getOpcode()) { 115 case AMDGPU::S_SENDMSG: 116 case AMDGPU::S_SENDMSGHALT: 117 case AMDGPU::S_TTRACEDATA: 118 return true; 119 // These DS opcodes don't support GDS. 120 case AMDGPU::DS_NOP: 121 case AMDGPU::DS_PERMUTE_B32: 122 case AMDGPU::DS_BPERMUTE_B32: 123 return false; 124 default: 125 if (TII.isDS(MI.getOpcode())) { 126 int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 127 AMDGPU::OpName::gds); 128 if (MI.getOperand(GDS).getImm()) 129 return true; 130 } 131 return false; 132 } 133 } 134 135 static bool isPermlane(const MachineInstr &MI) { 136 unsigned Opcode = MI.getOpcode(); 137 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 || 138 Opcode == AMDGPU::V_PERMLANEX16_B32_e64; 139 } 140 141 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) { 142 const MachineOperand *RegOp = TII->getNamedOperand(RegInstr, 143 AMDGPU::OpName::simm16); 144 return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_; 145 } 146 147 ScheduleHazardRecognizer::HazardType 148 GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { 149 MachineInstr *MI = SU->getInstr(); 150 // If we are not in "HazardRecognizerMode" and therefore not being run from 151 // the scheduler, track possible stalls from hazards but don't insert noops. 152 auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard; 153 154 if (MI->isBundle()) 155 return NoHazard; 156 157 if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0) 158 return HazardType; 159 160 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0) 161 return HazardType; 162 163 if (checkFPAtomicToDenormModeHazard(MI) > 0) 164 return HazardType; 165 166 if (ST.hasNoDataDepHazard()) 167 return NoHazard; 168 169 // FIXME: Should flat be considered vmem? 170 if ((SIInstrInfo::isVMEM(*MI) || 171 SIInstrInfo::isFLAT(*MI)) 172 && checkVMEMHazards(MI) > 0) 173 return HazardType; 174 175 if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0) 176 return HazardType; 177 178 if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0) 179 return HazardType; 180 181 if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0) 182 return HazardType; 183 184 if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0) 185 return HazardType; 186 187 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || 188 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || 189 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) 190 return HazardType; 191 192 if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0) 193 return HazardType; 194 195 if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0) 196 return HazardType; 197 198 if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0) 199 return HazardType; 200 201 if (ST.hasReadM0MovRelInterpHazard() && 202 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode())) && 203 checkReadM0Hazards(MI) > 0) 204 return HazardType; 205 206 if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI) && 207 checkReadM0Hazards(MI) > 0) 208 return HazardType; 209 210 if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0) 211 return HazardType; 212 213 if ((SIInstrInfo::isVMEM(*MI) || 214 SIInstrInfo::isFLAT(*MI) || 215 SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0) 216 return HazardType; 217 218 if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0) 219 return HazardType; 220 221 return NoHazard; 222 } 223 224 static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, 225 unsigned Quantity) { 226 while (Quantity > 0) { 227 unsigned Arg = std::min(Quantity, 8u); 228 Quantity -= Arg; 229 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP)) 230 .addImm(Arg - 1); 231 } 232 } 233 234 void GCNHazardRecognizer::processBundle() { 235 MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator()); 236 MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end(); 237 // Check bundled MachineInstr's for hazards. 238 for (; MI != E && MI->isInsideBundle(); ++MI) { 239 CurrCycleInstr = &*MI; 240 unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr); 241 242 if (IsHazardRecognizerMode) { 243 fixHazards(CurrCycleInstr); 244 245 insertNoopsInBundle(CurrCycleInstr, TII, WaitStates); 246 } 247 248 // It’s unnecessary to track more than MaxLookAhead instructions. Since we 249 // include the bundled MI directly after, only add a maximum of 250 // (MaxLookAhead - 1) noops to EmittedInstrs. 251 for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i) 252 EmittedInstrs.push_front(nullptr); 253 254 EmittedInstrs.push_front(CurrCycleInstr); 255 EmittedInstrs.resize(MaxLookAhead); 256 } 257 CurrCycleInstr = nullptr; 258 } 259 260 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) { 261 IsHazardRecognizerMode = true; 262 CurrCycleInstr = MI; 263 unsigned W = PreEmitNoopsCommon(MI); 264 fixHazards(MI); 265 CurrCycleInstr = nullptr; 266 return W; 267 } 268 269 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { 270 if (MI->isBundle()) 271 return 0; 272 273 int WaitStates = 0; 274 275 if (SIInstrInfo::isSMRD(*MI)) 276 return std::max(WaitStates, checkSMRDHazards(MI)); 277 278 if (ST.hasNSAtoVMEMBug()) 279 WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI)); 280 281 WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI)); 282 283 if (ST.hasNoDataDepHazard()) 284 return WaitStates; 285 286 if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI)) 287 WaitStates = std::max(WaitStates, checkVMEMHazards(MI)); 288 289 if (SIInstrInfo::isVALU(*MI)) 290 WaitStates = std::max(WaitStates, checkVALUHazards(MI)); 291 292 if (SIInstrInfo::isDPP(*MI)) 293 WaitStates = std::max(WaitStates, checkDPPHazards(MI)); 294 295 if (isDivFMas(MI->getOpcode())) 296 WaitStates = std::max(WaitStates, checkDivFMasHazards(MI)); 297 298 if (isRWLane(MI->getOpcode())) 299 WaitStates = std::max(WaitStates, checkRWLaneHazards(MI)); 300 301 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || 302 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || 303 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) 304 WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI)); 305 306 if (MI->isInlineAsm()) 307 return std::max(WaitStates, checkInlineAsmHazards(MI)); 308 309 if (isSGetReg(MI->getOpcode())) 310 return std::max(WaitStates, checkGetRegHazards(MI)); 311 312 if (isSSetReg(MI->getOpcode())) 313 return std::max(WaitStates, checkSetRegHazards(MI)); 314 315 if (isRFE(MI->getOpcode())) 316 return std::max(WaitStates, checkRFEHazards(MI)); 317 318 if (ST.hasReadM0MovRelInterpHazard() && (TII.isVINTRP(*MI) || 319 isSMovRel(MI->getOpcode()))) 320 return std::max(WaitStates, checkReadM0Hazards(MI)); 321 322 if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) 323 return std::max(WaitStates, checkReadM0Hazards(MI)); 324 325 if (SIInstrInfo::isMAI(*MI)) 326 return std::max(WaitStates, checkMAIHazards(MI)); 327 328 if (SIInstrInfo::isVMEM(*MI) || 329 SIInstrInfo::isFLAT(*MI) || 330 SIInstrInfo::isDS(*MI)) 331 return std::max(WaitStates, checkMAILdStHazards(MI)); 332 333 return WaitStates; 334 } 335 336 void GCNHazardRecognizer::EmitNoop() { 337 EmittedInstrs.push_front(nullptr); 338 } 339 340 void GCNHazardRecognizer::AdvanceCycle() { 341 // When the scheduler detects a stall, it will call AdvanceCycle() without 342 // emitting any instructions. 343 if (!CurrCycleInstr) { 344 EmittedInstrs.push_front(nullptr); 345 return; 346 } 347 348 // Do not track non-instructions which do not affect the wait states. 349 // If included, these instructions can lead to buffer overflow such that 350 // detectable hazards are missed. 351 if (CurrCycleInstr->isMetaInstruction()) { 352 CurrCycleInstr = nullptr; 353 return; 354 } 355 356 if (CurrCycleInstr->isBundle()) { 357 processBundle(); 358 return; 359 } 360 361 unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr); 362 363 // Keep track of emitted instructions 364 EmittedInstrs.push_front(CurrCycleInstr); 365 366 // Add a nullptr for each additional wait state after the first. Make sure 367 // not to add more than getMaxLookAhead() items to the list, since we 368 // truncate the list to that size right after this loop. 369 for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead()); 370 i < e; ++i) { 371 EmittedInstrs.push_front(nullptr); 372 } 373 374 // getMaxLookahead() is the largest number of wait states we will ever need 375 // to insert, so there is no point in keeping track of more than that many 376 // wait states. 377 EmittedInstrs.resize(getMaxLookAhead()); 378 379 CurrCycleInstr = nullptr; 380 } 381 382 void GCNHazardRecognizer::RecedeCycle() { 383 llvm_unreachable("hazard recognizer does not support bottom-up scheduling."); 384 } 385 386 //===----------------------------------------------------------------------===// 387 // Helper Functions 388 //===----------------------------------------------------------------------===// 389 390 typedef function_ref<bool(MachineInstr *, int WaitStates)> IsExpiredFn; 391 392 // Returns a minimum wait states since \p I walking all predecessors. 393 // Only scans until \p IsExpired does not return true. 394 // Can only be run in a hazard recognizer mode. 395 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, 396 MachineBasicBlock *MBB, 397 MachineBasicBlock::reverse_instr_iterator I, 398 int WaitStates, 399 IsExpiredFn IsExpired, 400 DenseSet<const MachineBasicBlock *> &Visited) { 401 for (auto E = MBB->instr_rend(); I != E; ++I) { 402 // Don't add WaitStates for parent BUNDLE instructions. 403 if (I->isBundle()) 404 continue; 405 406 if (IsHazard(&*I)) 407 return WaitStates; 408 409 if (I->isInlineAsm() || I->isMetaInstruction()) 410 continue; 411 412 WaitStates += SIInstrInfo::getNumWaitStates(*I); 413 414 if (IsExpired(&*I, WaitStates)) 415 return std::numeric_limits<int>::max(); 416 } 417 418 int MinWaitStates = WaitStates; 419 bool Found = false; 420 for (MachineBasicBlock *Pred : MBB->predecessors()) { 421 if (!Visited.insert(Pred).second) 422 continue; 423 424 int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(), 425 WaitStates, IsExpired, Visited); 426 427 if (W == std::numeric_limits<int>::max()) 428 continue; 429 430 MinWaitStates = Found ? std::min(MinWaitStates, W) : W; 431 Found = true; 432 } 433 434 if (Found) 435 return MinWaitStates; 436 437 return std::numeric_limits<int>::max(); 438 } 439 440 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, 441 MachineInstr *MI, 442 IsExpiredFn IsExpired) { 443 DenseSet<const MachineBasicBlock *> Visited; 444 return getWaitStatesSince(IsHazard, MI->getParent(), 445 std::next(MI->getReverseIterator()), 446 0, IsExpired, Visited); 447 } 448 449 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) { 450 if (IsHazardRecognizerMode) { 451 auto IsExpiredFn = [Limit] (MachineInstr *, int WaitStates) { 452 return WaitStates >= Limit; 453 }; 454 return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn); 455 } 456 457 int WaitStates = 0; 458 for (MachineInstr *MI : EmittedInstrs) { 459 if (MI) { 460 if (IsHazard(MI)) 461 return WaitStates; 462 463 if (MI->isInlineAsm()) 464 continue; 465 } 466 ++WaitStates; 467 468 if (WaitStates >= Limit) 469 break; 470 } 471 return std::numeric_limits<int>::max(); 472 } 473 474 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg, 475 IsHazardFn IsHazardDef, 476 int Limit) { 477 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 478 479 auto IsHazardFn = [IsHazardDef, TRI, Reg] (MachineInstr *MI) { 480 return IsHazardDef(MI) && MI->modifiesRegister(Reg, TRI); 481 }; 482 483 return getWaitStatesSince(IsHazardFn, Limit); 484 } 485 486 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard, 487 int Limit) { 488 auto IsHazardFn = [IsHazard] (MachineInstr *MI) { 489 return isSSetReg(MI->getOpcode()) && IsHazard(MI); 490 }; 491 492 return getWaitStatesSince(IsHazardFn, Limit); 493 } 494 495 //===----------------------------------------------------------------------===// 496 // No-op Hazard Detection 497 //===----------------------------------------------------------------------===// 498 499 static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, 500 MCRegister Reg) { 501 for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI) 502 BV.set(*RUI); 503 } 504 505 static void addRegsToSet(const SIRegisterInfo &TRI, 506 iterator_range<MachineInstr::const_mop_iterator> Ops, 507 BitVector &Set) { 508 for (const MachineOperand &Op : Ops) { 509 if (Op.isReg()) 510 addRegUnits(TRI, Set, Op.getReg().asMCReg()); 511 } 512 } 513 514 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) { 515 // XXX: Do we need to worry about implicit operands 516 addRegsToSet(TRI, MI.defs(), ClauseDefs); 517 addRegsToSet(TRI, MI.uses(), ClauseUses); 518 } 519 520 static bool breaksSMEMSoftClause(MachineInstr *MI) { 521 return !SIInstrInfo::isSMRD(*MI); 522 } 523 524 static bool breaksVMEMSoftClause(MachineInstr *MI) { 525 return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI); 526 } 527 528 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) { 529 // SMEM soft clause are only present on VI+, and only matter if xnack is 530 // enabled. 531 if (!ST.isXNACKEnabled()) 532 return 0; 533 534 bool IsSMRD = TII.isSMRD(*MEM); 535 536 resetClause(); 537 538 // A soft-clause is any group of consecutive SMEM instructions. The 539 // instructions in this group may return out of order and/or may be 540 // replayed (i.e. the same instruction issued more than once). 541 // 542 // In order to handle these situations correctly we need to make sure that 543 // when a clause has more than one instruction, no instruction in the clause 544 // writes to a register that is read by another instruction in the clause 545 // (including itself). If we encounter this situaion, we need to break the 546 // clause by inserting a non SMEM instruction. 547 548 for (MachineInstr *MI : EmittedInstrs) { 549 // When we hit a non-SMEM instruction then we have passed the start of the 550 // clause and we can stop. 551 if (!MI) 552 break; 553 554 if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI)) 555 break; 556 557 addClauseInst(*MI); 558 } 559 560 if (ClauseDefs.none()) 561 return 0; 562 563 // We need to make sure not to put loads and stores in the same clause if they 564 // use the same address. For now, just start a new clause whenever we see a 565 // store. 566 if (MEM->mayStore()) 567 return 1; 568 569 addClauseInst(*MEM); 570 571 // If the set of defs and uses intersect then we cannot add this instruction 572 // to the clause, so we have a hazard. 573 return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0; 574 } 575 576 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) { 577 int WaitStatesNeeded = 0; 578 579 WaitStatesNeeded = checkSoftClauseHazards(SMRD); 580 581 // This SMRD hazard only affects SI. 582 if (!ST.hasSMRDReadVALUDefHazard()) 583 return WaitStatesNeeded; 584 585 // A read of an SGPR by SMRD instruction requires 4 wait states when the 586 // SGPR was written by a VALU instruction. 587 int SmrdSgprWaitStates = 4; 588 auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); }; 589 auto IsBufferHazardDefFn = [this] (MachineInstr *MI) { return TII.isSALU(*MI); }; 590 591 bool IsBufferSMRD = TII.isBufferSMRD(*SMRD); 592 593 for (const MachineOperand &Use : SMRD->uses()) { 594 if (!Use.isReg()) 595 continue; 596 int WaitStatesNeededForUse = 597 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn, 598 SmrdSgprWaitStates); 599 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 600 601 // This fixes what appears to be undocumented hardware behavior in SI where 602 // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor 603 // needs some number of nops in between. We don't know how many we need, but 604 // let's use 4. This wasn't discovered before probably because the only 605 // case when this happens is when we expand a 64-bit pointer into a full 606 // descriptor and use s_buffer_load_dword instead of s_load_dword, which was 607 // probably never encountered in the closed-source land. 608 if (IsBufferSMRD) { 609 int WaitStatesNeededForUse = 610 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), 611 IsBufferHazardDefFn, 612 SmrdSgprWaitStates); 613 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 614 } 615 } 616 617 return WaitStatesNeeded; 618 } 619 620 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) { 621 if (!ST.hasVMEMReadSGPRVALUDefHazard()) 622 return 0; 623 624 int WaitStatesNeeded = checkSoftClauseHazards(VMEM); 625 626 // A read of an SGPR by a VMEM instruction requires 5 wait states when the 627 // SGPR was written by a VALU Instruction. 628 const int VmemSgprWaitStates = 5; 629 auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); }; 630 for (const MachineOperand &Use : VMEM->uses()) { 631 if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg())) 632 continue; 633 634 int WaitStatesNeededForUse = 635 VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn, 636 VmemSgprWaitStates); 637 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 638 } 639 return WaitStatesNeeded; 640 } 641 642 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) { 643 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 644 const SIInstrInfo *TII = ST.getInstrInfo(); 645 646 // Check for DPP VGPR read after VALU VGPR write and EXEC write. 647 int DppVgprWaitStates = 2; 648 int DppExecWaitStates = 5; 649 int WaitStatesNeeded = 0; 650 auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); }; 651 652 for (const MachineOperand &Use : DPP->uses()) { 653 if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg())) 654 continue; 655 int WaitStatesNeededForUse = 656 DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg(), 657 [](MachineInstr *) { return true; }, 658 DppVgprWaitStates); 659 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 660 } 661 662 WaitStatesNeeded = std::max( 663 WaitStatesNeeded, 664 DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn, 665 DppExecWaitStates)); 666 667 return WaitStatesNeeded; 668 } 669 670 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) { 671 const SIInstrInfo *TII = ST.getInstrInfo(); 672 673 // v_div_fmas requires 4 wait states after a write to vcc from a VALU 674 // instruction. 675 const int DivFMasWaitStates = 4; 676 auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); }; 677 int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn, 678 DivFMasWaitStates); 679 680 return DivFMasWaitStates - WaitStatesNeeded; 681 } 682 683 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) { 684 const SIInstrInfo *TII = ST.getInstrInfo(); 685 unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr); 686 687 const int GetRegWaitStates = 2; 688 auto IsHazardFn = [TII, GetRegHWReg] (MachineInstr *MI) { 689 return GetRegHWReg == getHWReg(TII, *MI); 690 }; 691 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates); 692 693 return GetRegWaitStates - WaitStatesNeeded; 694 } 695 696 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) { 697 const SIInstrInfo *TII = ST.getInstrInfo(); 698 unsigned HWReg = getHWReg(TII, *SetRegInstr); 699 700 const int SetRegWaitStates = ST.getSetRegWaitStates(); 701 auto IsHazardFn = [TII, HWReg] (MachineInstr *MI) { 702 return HWReg == getHWReg(TII, *MI); 703 }; 704 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates); 705 return SetRegWaitStates - WaitStatesNeeded; 706 } 707 708 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) { 709 if (!MI.mayStore()) 710 return -1; 711 712 const SIInstrInfo *TII = ST.getInstrInfo(); 713 unsigned Opcode = MI.getOpcode(); 714 const MCInstrDesc &Desc = MI.getDesc(); 715 716 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); 717 int VDataRCID = -1; 718 if (VDataIdx != -1) 719 VDataRCID = Desc.OpInfo[VDataIdx].RegClass; 720 721 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) { 722 // There is no hazard if the instruction does not use vector regs 723 // (like wbinvl1) 724 if (VDataIdx == -1) 725 return -1; 726 // For MUBUF/MTBUF instructions this hazard only exists if the 727 // instruction is not using a register in the soffset field. 728 const MachineOperand *SOffset = 729 TII->getNamedOperand(MI, AMDGPU::OpName::soffset); 730 // If we have no soffset operand, then assume this field has been 731 // hardcoded to zero. 732 if (AMDGPU::getRegBitWidth(VDataRCID) > 64 && 733 (!SOffset || !SOffset->isReg())) 734 return VDataIdx; 735 } 736 737 // MIMG instructions create a hazard if they don't use a 256-bit T# and 738 // the store size is greater than 8 bytes and they have more than two bits 739 // of their dmask set. 740 // All our MIMG definitions use a 256-bit T#, so we can skip checking for them. 741 if (TII->isMIMG(MI)) { 742 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); 743 assert(SRsrcIdx != -1 && 744 AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256); 745 (void)SRsrcIdx; 746 } 747 748 if (TII->isFLAT(MI)) { 749 int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); 750 if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64) 751 return DataIdx; 752 } 753 754 return -1; 755 } 756 757 int 758 GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def, 759 const MachineRegisterInfo &MRI) { 760 // Helper to check for the hazard where VMEM instructions that store more than 761 // 8 bytes can have there store data over written by the next instruction. 762 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 763 764 const int VALUWaitStates = 1; 765 int WaitStatesNeeded = 0; 766 767 if (!TRI->isVectorRegister(MRI, Def.getReg())) 768 return WaitStatesNeeded; 769 Register Reg = Def.getReg(); 770 auto IsHazardFn = [this, Reg, TRI] (MachineInstr *MI) { 771 int DataIdx = createsVALUHazard(*MI); 772 return DataIdx >= 0 && 773 TRI->regsOverlap(MI->getOperand(DataIdx).getReg(), Reg); 774 }; 775 int WaitStatesNeededForDef = 776 VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates); 777 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 778 779 return WaitStatesNeeded; 780 } 781 782 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) { 783 // This checks for the hazard where VMEM instructions that store more than 784 // 8 bytes can have there store data over written by the next instruction. 785 if (!ST.has12DWordStoreHazard()) 786 return 0; 787 788 const MachineRegisterInfo &MRI = MF.getRegInfo(); 789 int WaitStatesNeeded = 0; 790 791 for (const MachineOperand &Def : VALU->defs()) { 792 WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI)); 793 } 794 795 return WaitStatesNeeded; 796 } 797 798 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) { 799 // This checks for hazards associated with inline asm statements. 800 // Since inline asms can contain just about anything, we use this 801 // to call/leverage other check*Hazard routines. Note that 802 // this function doesn't attempt to address all possible inline asm 803 // hazards (good luck), but is a collection of what has been 804 // problematic thus far. 805 806 // see checkVALUHazards() 807 if (!ST.has12DWordStoreHazard()) 808 return 0; 809 810 const MachineRegisterInfo &MRI = MF.getRegInfo(); 811 int WaitStatesNeeded = 0; 812 813 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands(); 814 I != E; ++I) { 815 const MachineOperand &Op = IA->getOperand(I); 816 if (Op.isReg() && Op.isDef()) { 817 WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI)); 818 } 819 } 820 821 return WaitStatesNeeded; 822 } 823 824 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) { 825 const SIInstrInfo *TII = ST.getInstrInfo(); 826 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 827 const MachineRegisterInfo &MRI = MF.getRegInfo(); 828 829 const MachineOperand *LaneSelectOp = 830 TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1); 831 832 if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg())) 833 return 0; 834 835 Register LaneSelectReg = LaneSelectOp->getReg(); 836 auto IsHazardFn = [TII] (MachineInstr *MI) { 837 return TII->isVALU(*MI); 838 }; 839 840 const int RWLaneWaitStates = 4; 841 int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn, 842 RWLaneWaitStates); 843 return RWLaneWaitStates - WaitStatesSince; 844 } 845 846 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) { 847 if (!ST.hasRFEHazards()) 848 return 0; 849 850 const SIInstrInfo *TII = ST.getInstrInfo(); 851 852 const int RFEWaitStates = 1; 853 854 auto IsHazardFn = [TII] (MachineInstr *MI) { 855 return getHWReg(TII, *MI) == AMDGPU::Hwreg::ID_TRAPSTS; 856 }; 857 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates); 858 return RFEWaitStates - WaitStatesNeeded; 859 } 860 861 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) { 862 const SIInstrInfo *TII = ST.getInstrInfo(); 863 const int SMovRelWaitStates = 1; 864 auto IsHazardFn = [TII] (MachineInstr *MI) { 865 return TII->isSALU(*MI); 866 }; 867 return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, 868 SMovRelWaitStates); 869 } 870 871 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) { 872 fixVMEMtoScalarWriteHazards(MI); 873 fixVcmpxPermlaneHazards(MI); 874 fixSMEMtoVectorWriteHazards(MI); 875 fixVcmpxExecWARHazard(MI); 876 fixLdsBranchVmemWARHazard(MI); 877 } 878 879 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) { 880 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI)) 881 return false; 882 883 const SIInstrInfo *TII = ST.getInstrInfo(); 884 auto IsHazardFn = [TII] (MachineInstr *MI) { 885 return TII->isVOPC(*MI); 886 }; 887 888 auto IsExpiredFn = [] (MachineInstr *MI, int) { 889 if (!MI) 890 return false; 891 unsigned Opc = MI->getOpcode(); 892 return SIInstrInfo::isVALU(*MI) && 893 Opc != AMDGPU::V_NOP_e32 && 894 Opc != AMDGPU::V_NOP_e64 && 895 Opc != AMDGPU::V_NOP_sdwa; 896 }; 897 898 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 899 std::numeric_limits<int>::max()) 900 return false; 901 902 // V_NOP will be discarded by SQ. 903 // Use V_MOB_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE* 904 // which is always a VGPR and available. 905 auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0); 906 Register Reg = Src0->getReg(); 907 bool IsUndef = Src0->isUndef(); 908 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 909 TII->get(AMDGPU::V_MOV_B32_e32)) 910 .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0)) 911 .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill); 912 913 return true; 914 } 915 916 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) { 917 if (!ST.hasVMEMtoScalarWriteHazard()) 918 return false; 919 920 if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI)) 921 return false; 922 923 if (MI->getNumDefs() == 0) 924 return false; 925 926 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 927 928 auto IsHazardFn = [TRI, MI] (MachineInstr *I) { 929 if (!SIInstrInfo::isVMEM(*I) && !SIInstrInfo::isDS(*I) && 930 !SIInstrInfo::isFLAT(*I)) 931 return false; 932 933 for (const MachineOperand &Def : MI->defs()) { 934 MachineOperand *Op = I->findRegisterUseOperand(Def.getReg(), false, TRI); 935 if (!Op) 936 continue; 937 return true; 938 } 939 return false; 940 }; 941 942 auto IsExpiredFn = [](MachineInstr *MI, int) { 943 return MI && (SIInstrInfo::isVALU(*MI) || 944 (MI->getOpcode() == AMDGPU::S_WAITCNT && 945 !MI->getOperand(0).getImm()) || 946 (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 947 MI->getOperand(0).getImm() == 0xffe3)); 948 }; 949 950 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 951 std::numeric_limits<int>::max()) 952 return false; 953 954 const SIInstrInfo *TII = ST.getInstrInfo(); 955 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 956 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) 957 .addImm(0xffe3); 958 return true; 959 } 960 961 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) { 962 if (!ST.hasSMEMtoVectorWriteHazard()) 963 return false; 964 965 if (!SIInstrInfo::isVALU(*MI)) 966 return false; 967 968 unsigned SDSTName; 969 switch (MI->getOpcode()) { 970 case AMDGPU::V_READLANE_B32: 971 case AMDGPU::V_READFIRSTLANE_B32: 972 SDSTName = AMDGPU::OpName::vdst; 973 break; 974 default: 975 SDSTName = AMDGPU::OpName::sdst; 976 break; 977 } 978 979 const SIInstrInfo *TII = ST.getInstrInfo(); 980 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 981 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU()); 982 const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName); 983 if (!SDST) { 984 for (const auto &MO : MI->implicit_operands()) { 985 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) { 986 SDST = &MO; 987 break; 988 } 989 } 990 } 991 992 if (!SDST) 993 return false; 994 995 const Register SDSTReg = SDST->getReg(); 996 auto IsHazardFn = [SDSTReg, TRI] (MachineInstr *I) { 997 return SIInstrInfo::isSMRD(*I) && I->readsRegister(SDSTReg, TRI); 998 }; 999 1000 auto IsExpiredFn = [TII, IV] (MachineInstr *MI, int) { 1001 if (MI) { 1002 if (TII->isSALU(*MI)) { 1003 switch (MI->getOpcode()) { 1004 case AMDGPU::S_SETVSKIP: 1005 case AMDGPU::S_VERSION: 1006 case AMDGPU::S_WAITCNT_VSCNT: 1007 case AMDGPU::S_WAITCNT_VMCNT: 1008 case AMDGPU::S_WAITCNT_EXPCNT: 1009 // These instructions cannot not mitigate the hazard. 1010 return false; 1011 case AMDGPU::S_WAITCNT_LGKMCNT: 1012 // Reducing lgkmcnt count to 0 always mitigates the hazard. 1013 return (MI->getOperand(1).getImm() == 0) && 1014 (MI->getOperand(0).getReg() == AMDGPU::SGPR_NULL); 1015 case AMDGPU::S_WAITCNT: { 1016 const int64_t Imm = MI->getOperand(0).getImm(); 1017 AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm); 1018 return (Decoded.LgkmCnt == 0); 1019 } 1020 default: 1021 // SOPP instructions cannot mitigate the hazard. 1022 if (TII->isSOPP(*MI)) 1023 return false; 1024 // At this point the SALU can be assumed to mitigate the hazard 1025 // because either: 1026 // (a) it is independent of the at risk SMEM (breaking chain), 1027 // or 1028 // (b) it is dependent on the SMEM, in which case an appropriate 1029 // s_waitcnt lgkmcnt _must_ exist between it and the at risk 1030 // SMEM instruction. 1031 return true; 1032 } 1033 } 1034 } 1035 return false; 1036 }; 1037 1038 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1039 std::numeric_limits<int>::max()) 1040 return false; 1041 1042 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1043 TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL) 1044 .addImm(0); 1045 return true; 1046 } 1047 1048 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) { 1049 if (!ST.hasVcmpxExecWARHazard() || !SIInstrInfo::isVALU(*MI)) 1050 return false; 1051 1052 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1053 if (!MI->modifiesRegister(AMDGPU::EXEC, TRI)) 1054 return false; 1055 1056 auto IsHazardFn = [TRI] (MachineInstr *I) { 1057 if (SIInstrInfo::isVALU(*I)) 1058 return false; 1059 return I->readsRegister(AMDGPU::EXEC, TRI); 1060 }; 1061 1062 const SIInstrInfo *TII = ST.getInstrInfo(); 1063 auto IsExpiredFn = [TII, TRI] (MachineInstr *MI, int) { 1064 if (!MI) 1065 return false; 1066 if (SIInstrInfo::isVALU(*MI)) { 1067 if (TII->getNamedOperand(*MI, AMDGPU::OpName::sdst)) 1068 return true; 1069 for (auto MO : MI->implicit_operands()) 1070 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) 1071 return true; 1072 } 1073 if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1074 (MI->getOperand(0).getImm() & 0xfffe) == 0xfffe) 1075 return true; 1076 return false; 1077 }; 1078 1079 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1080 std::numeric_limits<int>::max()) 1081 return false; 1082 1083 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1084 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) 1085 .addImm(0xfffe); 1086 return true; 1087 } 1088 1089 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) { 1090 if (!ST.hasLdsBranchVmemWARHazard()) 1091 return false; 1092 1093 auto IsHazardInst = [] (const MachineInstr *MI) { 1094 if (SIInstrInfo::isDS(*MI)) 1095 return 1; 1096 if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isSegmentSpecificFLAT(*MI)) 1097 return 2; 1098 return 0; 1099 }; 1100 1101 auto InstType = IsHazardInst(MI); 1102 if (!InstType) 1103 return false; 1104 1105 auto IsExpiredFn = [&IsHazardInst] (MachineInstr *I, int) { 1106 return I && (IsHazardInst(I) || 1107 (I->getOpcode() == AMDGPU::S_WAITCNT_VSCNT && 1108 I->getOperand(0).getReg() == AMDGPU::SGPR_NULL && 1109 !I->getOperand(1).getImm())); 1110 }; 1111 1112 auto IsHazardFn = [InstType, &IsHazardInst] (MachineInstr *I) { 1113 if (!I->isBranch()) 1114 return false; 1115 1116 auto IsHazardFn = [InstType, IsHazardInst] (MachineInstr *I) { 1117 auto InstType2 = IsHazardInst(I); 1118 return InstType2 && InstType != InstType2; 1119 }; 1120 1121 auto IsExpiredFn = [InstType, &IsHazardInst] (MachineInstr *I, int) { 1122 if (!I) 1123 return false; 1124 1125 auto InstType2 = IsHazardInst(I); 1126 if (InstType == InstType2) 1127 return true; 1128 1129 return I->getOpcode() == AMDGPU::S_WAITCNT_VSCNT && 1130 I->getOperand(0).getReg() == AMDGPU::SGPR_NULL && 1131 !I->getOperand(1).getImm(); 1132 }; 1133 1134 return ::getWaitStatesSince(IsHazardFn, I, IsExpiredFn) != 1135 std::numeric_limits<int>::max(); 1136 }; 1137 1138 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1139 std::numeric_limits<int>::max()) 1140 return false; 1141 1142 const SIInstrInfo *TII = ST.getInstrInfo(); 1143 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1144 TII->get(AMDGPU::S_WAITCNT_VSCNT)) 1145 .addReg(AMDGPU::SGPR_NULL, RegState::Undef) 1146 .addImm(0); 1147 1148 return true; 1149 } 1150 1151 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) { 1152 int NSAtoVMEMWaitStates = 1; 1153 1154 if (!ST.hasNSAtoVMEMBug()) 1155 return 0; 1156 1157 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI)) 1158 return 0; 1159 1160 const SIInstrInfo *TII = ST.getInstrInfo(); 1161 const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset); 1162 if (!Offset || (Offset->getImm() & 6) == 0) 1163 return 0; 1164 1165 auto IsHazardFn = [TII] (MachineInstr *I) { 1166 if (!SIInstrInfo::isMIMG(*I)) 1167 return false; 1168 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I->getOpcode()); 1169 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA && 1170 TII->getInstSizeInBytes(*I) >= 16; 1171 }; 1172 1173 return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1); 1174 } 1175 1176 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) { 1177 int FPAtomicToDenormModeWaitStates = 3; 1178 1179 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE) 1180 return 0; 1181 1182 auto IsHazardFn = [] (MachineInstr *I) { 1183 if (!SIInstrInfo::isVMEM(*I) && !SIInstrInfo::isFLAT(*I)) 1184 return false; 1185 return SIInstrInfo::isFPAtomic(*I); 1186 }; 1187 1188 auto IsExpiredFn = [] (MachineInstr *MI, int WaitStates) { 1189 if (WaitStates >= 3 || SIInstrInfo::isVALU(*MI)) 1190 return true; 1191 1192 switch (MI->getOpcode()) { 1193 case AMDGPU::S_WAITCNT: 1194 case AMDGPU::S_WAITCNT_VSCNT: 1195 case AMDGPU::S_WAITCNT_VMCNT: 1196 case AMDGPU::S_WAITCNT_EXPCNT: 1197 case AMDGPU::S_WAITCNT_LGKMCNT: 1198 case AMDGPU::S_WAIT_IDLE: 1199 return true; 1200 default: 1201 break; 1202 } 1203 1204 return false; 1205 }; 1206 1207 1208 return FPAtomicToDenormModeWaitStates - 1209 ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn); 1210 } 1211 1212 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) { 1213 assert(SIInstrInfo::isMAI(*MI)); 1214 1215 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI); 1216 } 1217 1218 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) { 1219 int WaitStatesNeeded = 0; 1220 unsigned Opc = MI->getOpcode(); 1221 1222 auto IsVALUFn = [] (MachineInstr *MI) { 1223 return SIInstrInfo::isVALU(*MI); 1224 }; 1225 1226 if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write 1227 const int LegacyVALUWritesVGPRWaitStates = 2; 1228 const int VALUWritesExecWaitStates = 4; 1229 const int MaxWaitStates = 4; 1230 1231 int WaitStatesNeededForUse = VALUWritesExecWaitStates - 1232 getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates); 1233 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1234 1235 if (WaitStatesNeeded < MaxWaitStates) { 1236 for (const MachineOperand &Use : MI->explicit_uses()) { 1237 const int MaxWaitStates = 2; 1238 1239 if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg())) 1240 continue; 1241 1242 int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates - 1243 getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates); 1244 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1245 1246 if (WaitStatesNeeded == MaxWaitStates) 1247 break; 1248 } 1249 } 1250 } 1251 1252 auto IsMFMAFn = [] (MachineInstr *MI) { 1253 return SIInstrInfo::isMAI(*MI) && 1254 MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1255 MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; 1256 }; 1257 1258 for (const MachineOperand &Op : MI->explicit_operands()) { 1259 if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg())) 1260 continue; 1261 1262 if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 1263 continue; 1264 1265 const int MFMAWritesAGPROverlappedSrcABWaitStates = 4; 1266 const int MFMAWritesAGPROverlappedSrcCWaitStates = 2; 1267 const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4; 1268 const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10; 1269 const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18; 1270 const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1; 1271 const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7; 1272 const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15; 1273 const int MaxWaitStates = 18; 1274 Register Reg = Op.getReg(); 1275 unsigned HazardDefLatency = 0; 1276 1277 auto IsOverlappedMFMAFn = [Reg, &IsMFMAFn, &HazardDefLatency, this] 1278 (MachineInstr *MI) { 1279 if (!IsMFMAFn(MI)) 1280 return false; 1281 Register DstReg = MI->getOperand(0).getReg(); 1282 if (DstReg == Reg) 1283 return false; 1284 HazardDefLatency = std::max(HazardDefLatency, 1285 TSchedModel.computeInstrLatency(MI)); 1286 return TRI.regsOverlap(DstReg, Reg); 1287 }; 1288 1289 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, 1290 MaxWaitStates); 1291 int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates; 1292 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 1293 int OpNo = MI->getOperandNo(&Op); 1294 if (OpNo == SrcCIdx) { 1295 NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates; 1296 } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) { 1297 switch (HazardDefLatency) { 1298 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates; 1299 break; 1300 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates; 1301 break; 1302 case 16: LLVM_FALLTHROUGH; 1303 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates; 1304 break; 1305 } 1306 } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { 1307 switch (HazardDefLatency) { 1308 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates; 1309 break; 1310 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates; 1311 break; 1312 case 16: LLVM_FALLTHROUGH; 1313 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates; 1314 break; 1315 } 1316 } 1317 1318 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1319 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1320 1321 if (WaitStatesNeeded == MaxWaitStates) 1322 return WaitStatesNeeded; // Early exit. 1323 1324 auto IsAccVgprWriteFn = [Reg, this] (MachineInstr *MI) { 1325 if (MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 1326 return false; 1327 Register DstReg = MI->getOperand(0).getReg(); 1328 return TRI.regsOverlap(Reg, DstReg); 1329 }; 1330 1331 const int AccVGPRWriteMFMAReadSrcCWaitStates = 1; 1332 const int AccVGPRWriteMFMAReadSrcABWaitStates = 3; 1333 const int AccVGPRWriteAccVgprReadWaitStates = 3; 1334 NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates; 1335 if (OpNo == SrcCIdx) 1336 NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates; 1337 else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) 1338 NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates; 1339 1340 WaitStatesNeededForUse = NeedWaitStates - 1341 getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates); 1342 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1343 1344 if (WaitStatesNeeded == MaxWaitStates) 1345 return WaitStatesNeeded; // Early exit. 1346 } 1347 1348 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { 1349 const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0; 1350 const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5; 1351 const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13; 1352 const int MaxWaitStates = 13; 1353 Register DstReg = MI->getOperand(0).getReg(); 1354 unsigned HazardDefLatency = 0; 1355 1356 auto IsSrcCMFMAFn = [DstReg, &IsMFMAFn, &HazardDefLatency, this] 1357 (MachineInstr *MI) { 1358 if (!IsMFMAFn(MI)) 1359 return false; 1360 Register Reg = TII.getNamedOperand(*MI, AMDGPU::OpName::src2)->getReg(); 1361 HazardDefLatency = std::max(HazardDefLatency, 1362 TSchedModel.computeInstrLatency(MI)); 1363 return TRI.regsOverlap(Reg, DstReg); 1364 }; 1365 1366 int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates); 1367 int NeedWaitStates; 1368 switch (HazardDefLatency) { 1369 case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates; 1370 break; 1371 case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates; 1372 break; 1373 case 16: LLVM_FALLTHROUGH; 1374 default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates; 1375 break; 1376 } 1377 1378 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince; 1379 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1380 } 1381 1382 return WaitStatesNeeded; 1383 } 1384 1385 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { 1386 int WaitStatesNeeded = 0; 1387 unsigned Opc = MI->getOpcode(); 1388 1389 auto IsMFMAFn = [] (MachineInstr *MI) { 1390 return SIInstrInfo::isMAI(*MI) && 1391 MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1392 MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; 1393 }; 1394 1395 auto IsLegacyVALUFn = [&IsMFMAFn] (MachineInstr *MI) { 1396 return SIInstrInfo::isVALU(*MI) && !IsMFMAFn(MI); 1397 }; 1398 1399 auto IsLegacyVALUNotDotFn = [&IsMFMAFn] (MachineInstr *MI) { 1400 return SIInstrInfo::isVALU(*MI) && 1401 !IsMFMAFn(MI) && !SIInstrInfo::isDOT(*MI); 1402 }; 1403 1404 if (!IsMFMAFn(MI)) 1405 return WaitStatesNeeded; 1406 1407 const int VALUWritesExecWaitStates = 4; 1408 int WaitStatesNeededForUse = VALUWritesExecWaitStates - 1409 getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn, 1410 VALUWritesExecWaitStates); 1411 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1412 1413 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 1414 1415 // Loop for both DGEMM and S/HGEMM 2nd instruction. 1416 for (const MachineOperand &Use : MI->explicit_uses()) { 1417 const int LegacyVALUNotDotWritesVGPRWaitStates = 2; 1418 const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2; 1419 const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8; 1420 const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16; 1421 const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3; 1422 const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9; 1423 const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17; 1424 const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9; 1425 const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4; 1426 const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5; 1427 const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11; 1428 const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19; 1429 const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6; 1430 const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11; 1431 const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4; 1432 const int MaxWaitStates = 19; 1433 1434 if (!Use.isReg()) 1435 continue; 1436 unsigned Reg = Use.getReg(); 1437 bool FullReg; 1438 MachineInstr *MI1; 1439 1440 auto IsOverlappedDGEMMorXDLFn = [Reg, &IsMFMAFn, &FullReg, &MI1, this] 1441 (MachineInstr *MI) { 1442 if (!IsMFMAFn(MI)) 1443 return false; 1444 if (!isDGEMM(MI->getOpcode()) && !isXDL(ST, *MI)) 1445 return false; 1446 Register DstReg = MI->getOperand(0).getReg(); 1447 FullReg = (DstReg == Reg); 1448 MI1 = MI; 1449 return TRI.regsOverlap(DstReg, Reg); 1450 }; 1451 1452 WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates - 1453 getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates); 1454 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1455 1456 int NumWaitStates = getWaitStatesSinceDef(Reg, IsOverlappedDGEMMorXDLFn, 1457 MaxWaitStates); 1458 if (NumWaitStates == std::numeric_limits<int>::max()) 1459 continue; 1460 1461 int OpNo = MI->getOperandNo(&Use); 1462 unsigned Opc1 = MI1->getOpcode(); 1463 int NeedWaitStates = 0; 1464 if (OpNo == SrcCIdx) { 1465 if (!isDGEMM(Opc) && isDGEMM(Opc1)) { 1466 NeedWaitStates = 0; 1467 } else if (FullReg) { 1468 if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 1469 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) && 1470 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 1471 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64)) 1472 NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates; 1473 } else { 1474 switch (Opc1) { 1475 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: 1476 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: 1477 if (!isXDL(ST, *MI)) 1478 NeedWaitStates = DMFMA16x16WritesVGPROverlappedSrcCWaitStates; 1479 break; 1480 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: 1481 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: 1482 if (!isXDL(ST, *MI)) 1483 NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates; 1484 break; 1485 default: 1486 switch (TSchedModel.computeInstrLatency(MI1)) { 1487 case 2: 1488 NeedWaitStates = isDGEMM(Opc) 1489 ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates 1490 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates; 1491 break; 1492 case 8: 1493 NeedWaitStates = isDGEMM(Opc) 1494 ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates 1495 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates; 1496 break; 1497 case 16: LLVM_FALLTHROUGH; 1498 default: 1499 NeedWaitStates = isDGEMM(Opc) 1500 ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates 1501 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates; 1502 } 1503 } 1504 } 1505 } else { 1506 switch (Opc1) { 1507 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: 1508 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: 1509 NeedWaitStates = DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates; 1510 break; 1511 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: 1512 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: 1513 NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates; 1514 break; 1515 default: 1516 switch (TSchedModel.computeInstrLatency(MI1)) { 1517 case 2: 1518 NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates; 1519 break; 1520 case 8: 1521 NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates; 1522 break; 1523 case 16: LLVM_FALLTHROUGH; 1524 default: 1525 NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates; 1526 } 1527 } 1528 } 1529 if (WaitStatesNeeded >= NeedWaitStates) 1530 continue; 1531 1532 WaitStatesNeededForUse = NeedWaitStates - NumWaitStates; 1533 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1534 1535 if (WaitStatesNeeded == MaxWaitStates) 1536 break; 1537 } 1538 1539 return WaitStatesNeeded; 1540 } 1541 1542 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) { 1543 // On gfx90a+ releveant hazards are checked in checkMAIVALUHazards() 1544 if (!ST.hasMAIInsts() || ST.hasGFX90AInsts()) 1545 return 0; 1546 1547 int WaitStatesNeeded = 0; 1548 1549 auto IsAccVgprReadFn = [] (MachineInstr *MI) { 1550 return MI->getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64; 1551 }; 1552 1553 for (const MachineOperand &Op : MI->explicit_uses()) { 1554 if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg())) 1555 continue; 1556 1557 Register Reg = Op.getReg(); 1558 1559 const int AccVgprReadLdStWaitStates = 2; 1560 const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1; 1561 const int MaxWaitStates = 2; 1562 1563 int WaitStatesNeededForUse = AccVgprReadLdStWaitStates - 1564 getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates); 1565 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1566 1567 if (WaitStatesNeeded == MaxWaitStates) 1568 return WaitStatesNeeded; // Early exit. 1569 1570 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](MachineInstr *MI) { 1571 if (MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 && 1572 MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 1573 return false; 1574 auto IsVALUFn = [] (MachineInstr *MI) { 1575 return SIInstrInfo::isVALU(*MI) && !SIInstrInfo::isMAI(*MI); 1576 }; 1577 return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) < 1578 std::numeric_limits<int>::max(); 1579 }; 1580 1581 WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates - 1582 getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates); 1583 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1584 } 1585 1586 return WaitStatesNeeded; 1587 } 1588 1589 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { 1590 if (!ST.hasGFX90AInsts()) 1591 return 0; 1592 1593 auto IsMFMAFn = [] (MachineInstr *MI) -> bool { 1594 return SIInstrInfo::isMAI(*MI) && 1595 MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1596 MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; 1597 }; 1598 1599 auto IsDGEMMFn = [] (MachineInstr *MI) -> bool { 1600 return isDGEMM(MI->getOpcode()); 1601 }; 1602 1603 // This is checked in checkMAIHazards90A() 1604 if (IsMFMAFn(MI)) 1605 return 0; 1606 1607 int WaitStatesNeeded = 0; 1608 1609 bool IsMemOrExport = SIInstrInfo::isVMEM(*MI) || 1610 SIInstrInfo::isFLAT(*MI) || 1611 SIInstrInfo::isDS(*MI) || 1612 SIInstrInfo::isEXP(*MI); 1613 bool IsVALU = SIInstrInfo::isVALU(*MI); 1614 1615 MachineInstr *MFMA = nullptr; 1616 unsigned Reg; 1617 auto IsDGEMMorXDLWriteFn = [&Reg, &IsMFMAFn, &MFMA, this] (MachineInstr *MI) { 1618 if (!IsMFMAFn(MI) || !TRI.regsOverlap(MI->getOperand(0).getReg(), Reg)) 1619 return false; 1620 if (!isDGEMM(MI->getOpcode()) && !isXDL(ST, *MI)) 1621 return false; 1622 MFMA = MI; 1623 return true; 1624 }; 1625 1626 MachineInstr *DOT = nullptr; 1627 auto IsDotWriteFn = [&Reg, &DOT, this] (MachineInstr *MI) { 1628 if (!SIInstrInfo::isDOT(*MI) || 1629 !TRI.regsOverlap(MI->getOperand(0).getReg(), Reg)) 1630 return false; 1631 DOT = MI; 1632 return true; 1633 }; 1634 1635 int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 1636 AMDGPU::OpName::src2); 1637 1638 if (IsMemOrExport || IsVALU) { 1639 const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5; 1640 const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11; 1641 const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19; 1642 const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9; 1643 const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18; 1644 const int DMFMA4x4WriteVgprVALUReadWaitStates = 6; 1645 const int DMFMA16x16WriteVgprVALUReadWaitStates = 11; 1646 const int DotWriteSameDotReadSrcAB = 3; 1647 const int DotWriteDifferentVALURead = 3; 1648 const int MaxWaitStates = 19; 1649 1650 for (const MachineOperand &Use : MI->explicit_uses()) { 1651 if (!Use.isReg()) 1652 continue; 1653 Reg = Use.getReg(); 1654 1655 DOT = nullptr; 1656 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn, 1657 MaxWaitStates); 1658 if (DOT) { 1659 int NeedWaitStates = 0; 1660 if (DOT->getOpcode() == MI->getOpcode()) { 1661 if (&Use - &MI->getOperand(0) != SrcCIdx) 1662 NeedWaitStates = DotWriteSameDotReadSrcAB; 1663 } else { 1664 NeedWaitStates = DotWriteDifferentVALURead; 1665 } 1666 1667 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1668 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1669 } 1670 1671 MFMA = nullptr; 1672 WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDGEMMorXDLWriteFn, 1673 MaxWaitStates); 1674 if (!MFMA) 1675 continue; 1676 1677 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); 1678 int NeedWaitStates = MaxWaitStates; 1679 switch (HazardDefLatency) { 1680 case 2: 1681 NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates; 1682 break; 1683 case 4: 1684 assert(isDGEMM(MFMA->getOpcode())); 1685 NeedWaitStates = 1686 IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates 1687 : DMFMA4x4WriteVgprVALUReadWaitStates; 1688 break; 1689 case 8: 1690 NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates; 1691 break; 1692 case 16: LLVM_FALLTHROUGH; 1693 default: 1694 NeedWaitStates = 1695 isDGEMM(MFMA->getOpcode()) 1696 ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates 1697 : DMFMA16x16WriteVgprVALUReadWaitStates 1698 : SMFMA32x32WriteVgprVALUMemExpReadWaitStates; 1699 break; 1700 } 1701 1702 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1703 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1704 1705 if (WaitStatesNeeded == MaxWaitStates) 1706 break; 1707 } 1708 } 1709 1710 unsigned Opc = MI->getOpcode(); 1711 const int DMFMAToFMA64WaitStates = 2; 1712 if ((Opc == AMDGPU::V_FMA_F64_e64 || 1713 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 || 1714 Opc == AMDGPU::V_FMAC_F64_dpp) && 1715 WaitStatesNeeded < DMFMAToFMA64WaitStates) { 1716 int WaitStatesNeededForUse = DMFMAToFMA64WaitStates - 1717 getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates); 1718 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1719 } 1720 1721 if (!IsVALU && !IsMemOrExport) 1722 return WaitStatesNeeded; 1723 1724 for (const MachineOperand &Def : MI->defs()) { 1725 const int SMFMA4x4WriteVgprVALUWawWaitStates = 5; 1726 const int SMFMA16x16WriteVgprVALUWawWaitStates = 11; 1727 const int SMFMA32x32WriteVgprVALUWawWaitStates = 19; 1728 const int SMFMA4x4ReadVgprVALUWarWaitStates = 1; 1729 const int SMFMA16x16ReadVgprVALUWarWaitStates = 7; 1730 const int SMFMA32x32ReadVgprVALUWarWaitStates = 15; 1731 const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6; 1732 const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11; 1733 const int DotWriteDifferentVALUWrite = 3; 1734 const int MaxWaitStates = 19; 1735 const int MaxWarWaitStates = 15; 1736 1737 Reg = Def.getReg(); 1738 1739 DOT = nullptr; 1740 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn, 1741 MaxWaitStates); 1742 if (DOT && DOT->getOpcode() != MI->getOpcode()) 1743 WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite - 1744 WaitStatesSinceDef); 1745 1746 MFMA = nullptr; 1747 WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDGEMMorXDLWriteFn, 1748 MaxWaitStates); 1749 if (MFMA) { 1750 int NeedWaitStates = MaxWaitStates; 1751 switch (TSchedModel.computeInstrLatency(MFMA)) { 1752 case 2: 1753 NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates; 1754 break; 1755 case 4: 1756 assert(isDGEMM(MFMA->getOpcode())); 1757 NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates; 1758 break; 1759 case 8: 1760 NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates; 1761 break; 1762 case 16: LLVM_FALLTHROUGH; 1763 default: 1764 NeedWaitStates = isDGEMM(MFMA->getOpcode()) 1765 ? DMFMA16x16WriteVgprVALUWriteWaitStates 1766 : SMFMA32x32WriteVgprVALUWawWaitStates; 1767 break; 1768 } 1769 1770 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1771 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1772 1773 if (WaitStatesNeeded == MaxWaitStates) 1774 break; 1775 } 1776 1777 auto IsSMFMAReadAsCFn = [&Reg, &IsMFMAFn, &MFMA, this] 1778 (MachineInstr *MI) { 1779 if (!IsMFMAFn(MI) || isDGEMM(MI->getOpcode()) || 1780 !MI->readsRegister(Reg, &TRI)) 1781 return false; 1782 1783 MachineOperand *SrcC = TII.getNamedOperand(*MI, AMDGPU::OpName::src2); 1784 assert(SrcC); 1785 if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg)) 1786 return false; 1787 1788 MFMA = MI; 1789 return true; 1790 }; 1791 1792 MFMA = nullptr; 1793 int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn, 1794 MaxWarWaitStates); 1795 if (!MFMA) 1796 continue; 1797 1798 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); 1799 int NeedWaitStates = MaxWaitStates; 1800 switch (HazardDefLatency) { 1801 case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates; 1802 break; 1803 case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates; 1804 break; 1805 case 16: LLVM_FALLTHROUGH; 1806 default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates; 1807 break; 1808 } 1809 1810 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse; 1811 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1812 } 1813 1814 return WaitStatesNeeded; 1815 } 1816 1817 bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) { 1818 if (!SU->isInstr()) 1819 return false; 1820 1821 MachineInstr *MAI = nullptr; 1822 auto IsMFMAFn = [&MAI] (MachineInstr *MI) { 1823 MAI = nullptr; 1824 if (SIInstrInfo::isMAI(*MI) && 1825 MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1826 MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64) 1827 MAI = MI; 1828 return MAI != nullptr; 1829 }; 1830 1831 MachineInstr *MI = SU->getInstr(); 1832 if (IsMFMAFn(MI)) { 1833 int W = getWaitStatesSince(IsMFMAFn, 16); 1834 if (MAI) 1835 return W < (int)TSchedModel.computeInstrLatency(MAI); 1836 } 1837 1838 return false; 1839 } 1840