1 //===-- GCNHazardRecognizers.h - GCN Hazard Recognizers ---------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines hazard recognizers for scheduling on GCN processors. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #ifndef LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H 14 #define LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H 15 16 #include "llvm/ADT/BitVector.h" 17 #include "llvm/ADT/STLExtras.h" 18 #include "llvm/CodeGen/ScheduleHazardRecognizer.h" 19 #include "llvm/CodeGen/TargetSchedule.h" 20 #include <list> 21 22 namespace llvm { 23 24 class MachineFunction; 25 class MachineInstr; 26 class MachineOperand; 27 class MachineRegisterInfo; 28 class SIInstrInfo; 29 class SIRegisterInfo; 30 class GCNSubtarget; 31 32 class GCNHazardRecognizer final : public ScheduleHazardRecognizer { 33 public: 34 typedef function_ref<bool(const MachineInstr &)> IsHazardFn; 35 36 private: 37 // Distinguish if we are called from scheduler or hazard recognizer 38 bool IsHazardRecognizerMode; 39 40 // This variable stores the instruction that has been emitted this cycle. It 41 // will be added to EmittedInstrs, when AdvanceCycle() or RecedeCycle() is 42 // called. 43 MachineInstr *CurrCycleInstr; 44 std::list<MachineInstr*> EmittedInstrs; 45 const MachineFunction &MF; 46 const GCNSubtarget &ST; 47 const SIInstrInfo &TII; 48 const SIRegisterInfo &TRI; 49 const TargetSchedModel &TSchedModel; 50 bool RunLdsBranchVmemWARHazardFixup; 51 BitVector VALUReadHazardSGPRs; 52 bool UseVALUReadHazardExhaustiveSearch; 53 54 /// RegUnits of uses in the current soft memory clause. 55 BitVector ClauseUses; 56 57 /// RegUnits of defs in the current soft memory clause. 58 BitVector ClauseDefs; 59 60 void resetClause() { 61 ClauseUses.reset(); 62 ClauseDefs.reset(); 63 } 64 65 void addClauseInst(const MachineInstr &MI); 66 67 /// \returns the number of wait states before another MFMA instruction can be 68 /// issued after \p MI. 69 unsigned getMFMAPipelineWaitStates(const MachineInstr &MI) const; 70 71 // Advance over a MachineInstr bundle. Look for hazards in the bundled 72 // instructions. 73 void processBundle(); 74 75 // Run on an individual instruction in hazard recognizer mode. This can be 76 // used on a newly inserted instruction before returning from PreEmitNoops. 77 void runOnInstruction(MachineInstr *MI); 78 79 int getWaitStatesSince(IsHazardFn IsHazard, int Limit); 80 int getWaitStatesSinceDef(unsigned Reg, IsHazardFn IsHazardDef, int Limit); 81 int getWaitStatesSinceSetReg(IsHazardFn IsHazard, int Limit); 82 83 int checkSoftClauseHazards(MachineInstr *SMEM); 84 int checkSMRDHazards(MachineInstr *SMRD); 85 int checkVMEMHazards(MachineInstr* VMEM); 86 int checkDPPHazards(MachineInstr *DPP); 87 int checkDivFMasHazards(MachineInstr *DivFMas); 88 int checkGetRegHazards(MachineInstr *GetRegInstr); 89 int checkSetRegHazards(MachineInstr *SetRegInstr); 90 int createsVALUHazard(const MachineInstr &MI); 91 int checkVALUHazards(MachineInstr *VALU); 92 int checkVALUHazardsHelper(const MachineOperand &Def, const MachineRegisterInfo &MRI); 93 int checkRWLaneHazards(MachineInstr *RWLane); 94 int checkRFEHazards(MachineInstr *RFE); 95 int checkInlineAsmHazards(MachineInstr *IA); 96 int checkReadM0Hazards(MachineInstr *SMovRel); 97 int checkNSAtoVMEMHazard(MachineInstr *MI); 98 int checkFPAtomicToDenormModeHazard(MachineInstr *MI); 99 void fixHazards(MachineInstr *MI); 100 bool fixVcmpxPermlaneHazards(MachineInstr *MI); 101 bool fixVMEMtoScalarWriteHazards(MachineInstr *MI); 102 bool fixSMEMtoVectorWriteHazards(MachineInstr *MI); 103 bool fixVcmpxExecWARHazard(MachineInstr *MI); 104 bool fixLdsBranchVmemWARHazard(MachineInstr *MI); 105 bool fixLdsDirectVALUHazard(MachineInstr *MI); 106 bool fixLdsDirectVMEMHazard(MachineInstr *MI); 107 bool fixVALUPartialForwardingHazard(MachineInstr *MI); 108 bool fixVALUTransUseHazard(MachineInstr *MI); 109 bool fixWMMAHazards(MachineInstr *MI); 110 bool fixShift64HighRegBug(MachineInstr *MI); 111 bool fixVALUMaskWriteHazard(MachineInstr *MI); 112 void computeVALUHazardSGPRs(MachineFunction *MMF); 113 bool fixVALUReadSGPRHazard(MachineInstr *MI); 114 bool fixRequiredExportPriority(MachineInstr *MI); 115 116 int checkMAIHazards(MachineInstr *MI); 117 int checkMAIHazards908(MachineInstr *MI); 118 int checkMAIHazards90A(MachineInstr *MI); 119 /// Pad the latency between neighboring MFMA instructions with s_nops. The 120 /// percentage of wait states to fill with s_nops is specified by the command 121 /// line option '-amdgpu-mfma-padding-ratio'. 122 /// 123 /// For example, with '-amdgpu-mfma-padding-ratio=100': 124 /// 125 /// 2 pass MFMA instructions have a latency of 2 wait states. Therefore, a 126 /// 'S_NOP 1' will be added between sequential MFMA instructions. 127 /// 128 /// V_MFMA_F32_4X4X1F32 129 /// V_MFMA_F32_4X4X1F32 130 ///--> 131 /// V_MFMA_F32_4X4X1F32 132 /// S_NOP 1 133 /// V_MFMA_F32_4X4X1F32 134 int checkMFMAPadding(MachineInstr *MI); 135 int checkMAIVALUHazards(MachineInstr *MI); 136 int checkMAILdStHazards(MachineInstr *MI); 137 int checkPermlaneHazards(MachineInstr *MI); 138 139 public: 140 GCNHazardRecognizer(const MachineFunction &MF); 141 // We can only issue one instruction per cycle. 142 bool atIssueLimit() const override { return true; } 143 void EmitInstruction(SUnit *SU) override; 144 void EmitInstruction(MachineInstr *MI) override; 145 HazardType getHazardType(SUnit *SU, int Stalls) override; 146 void EmitNoop() override; 147 unsigned PreEmitNoops(MachineInstr *) override; 148 unsigned PreEmitNoopsCommon(MachineInstr *); 149 void AdvanceCycle() override; 150 void RecedeCycle() override; 151 bool ShouldPreferAnother(SUnit *SU) override; 152 void Reset() override; 153 }; 154 155 } // end namespace llvm 156 157 #endif //LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H 158