Lines Matching defs:TII
91 const SIInstrInfo *TII;
104 InstructionRule(const SIInstrInfo *TII, unsigned SGID,
106 : TII(TII), SGID(SGID) {
155 const SIInstrInfo *TII;
231 ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
232 : SGMask(SGMask), MaxSize(MaxSize), DAG(DAG), TII(TII) {
237 ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
238 : SGMask(SGMask), MaxSize(MaxSize), SyncID(SyncID), DAG(DAG), TII(TII) {
844 const SIInstrInfo *TII;
859 IGLPStrategy(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
860 : DAG(DAG), TII(TII) {}
878 MFMASmallGemmOpt(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
879 : IGLPStrategy(DAG, TII) {
891 if (TII->isMFMAorWMMA(I))
898 SchedGroupMask::DS, 2, PipelineSyncID, DAG, TII);
902 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
936 bool analyzeDAG(const SIInstrInfo *TII);
951 if (TII->isMFMAorWMMA(*I->getInstr()))
965 IsPipeExp(const SIInstrInfo *TII, unsigned SGID, bool NeedsCache = false)
966 : InstructionRule(TII, SGID, NeedsCache) {}
988 if (FoundTrans && TII->isMFMAorWMMA(*I->getInstr())) {
995 if (!FoundTrans && TII->isTRANS(I->getInstr()->getOpcode()))
1005 EnablesNthMFMA(unsigned Number, const SIInstrInfo *TII, unsigned SGID,
1007 : InstructionRule(TII, SGID, NeedsCache), Number(Number) {}
1022 if (!SU || !TII->isMFMAorWMMA(*ChainSeed->getInstr()))
1032 if (TII->isMFMAorWMMA(*Succ.getSUnit()->getInstr())) {
1052 const SIInstrInfo *TII, unsigned SGID,
1054 : InstructionRule(TII, SGID, NeedsCache), Number(Number),
1092 LessThanNSuccs(unsigned Size, const SIInstrInfo *TII, unsigned SGID,
1094 : InstructionRule(TII, SGID, NeedsCache), Size(Size),
1133 GreaterThanOrEqualToNSuccs(unsigned Size, const SIInstrInfo *TII,
1136 : InstructionRule(TII, SGID, NeedsCache), Size(Size),
1149 IsCvt(const SIInstrInfo *TII, unsigned SGID, bool NeedsCache = false)
1150 : InstructionRule(TII, SGID, NeedsCache) {}
1161 IsFMA(const SIInstrInfo *TII, unsigned SGID, bool NeedsCache = false)
1162 : InstructionRule(TII, SGID, NeedsCache) {}
1172 IsPipeAdd(const SIInstrInfo *TII, unsigned SGID, bool NeedsCache = false)
1173 : InstructionRule(TII, SGID, NeedsCache) {}
1208 IsSuccOfPrevNthGroup(unsigned Distance, const SIInstrInfo *TII,
1210 : InstructionRule(TII, SGID, NeedsCache), Distance(Distance) {}
1244 IsReachableFromPrevNthGroup(unsigned Distance, const SIInstrInfo *TII,
1246 : InstructionRule(TII, SGID, NeedsCache), Distance(Distance) {}
1260 OccursAtOrAfterNode(unsigned Number, const SIInstrInfo *TII, unsigned SGID,
1262 : InstructionRule(TII, SGID, NeedsCache), Number(Number) {}
1275 if (!SU || !TII->isMFMAorWMMA(*ChainSeed->getInstr()))
1285 if (TII->isMFMAorWMMA(*Succ.getSUnit()->getInstr())) {
1304 IsExactMFMA(unsigned Number, SUnit *ChainSeed, const SIInstrInfo *TII,
1306 : InstructionRule(TII, SGID, NeedsCache), Number(Number),
1322 if (TII->isTRANS(SU.getInstr()->getOpcode())) {
1333 OccursAfterExp(const SIInstrInfo *TII, unsigned SGID,
1335 : InstructionRule(TII, SGID, NeedsCache) {}
1347 MFMAExpInterleaveOpt(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
1348 : IGLPStrategy(DAG, TII) {
1364 bool MFMAExpInterleaveOpt::analyzeDAG(const SIInstrInfo *TII) {
1384 if (TII->isTRANS(Opc)) {
1395 if (TII->isMFMAorWMMA(*SU.getInstr()))
1484 [&TII](SDep &Succ) {
1485 return TII->isMFMAorWMMA(*Succ.getSUnit()->getInstr());
1496 if (TII->isDS(Pred.getSUnit()->getInstr()->getOpcode()) &&
1532 PackPred->getSUnit()->Succs.end(), [&TII](SDep &Succ) {
1533 return TII->isMFMAorWMMA(*Succ.getSUnit()->getInstr());
1553 const SIInstrInfo *TII = ST.getInstrInfo();
1557 if (Phase != AMDGPU::SchedulingPhase::PostRA && !analyzeDAG(TII))
1577 const SIInstrInfo *TII = ST.getInstrInfo();
1626 SchedGroupMask::VALU, ExpRequirement, PipelineSyncID, DAG, TII);
1629 PositionInChain, MFMAChainSeeds[MFMAChain], TII, SG->getSGID(),
1633 std::make_shared<EnablesNthMFMA>(1, TII, SG->getSGID(), true));
1634 SG->addRule(std::make_shared<IsFMA>(TII, SG->getSGID()));
1639 SchedGroupMask::VALU, ExpRequirement, PipelineSyncID, DAG, TII);
1643 MFMAChainSeeds[getNextTransMFMAChain()], TII, SG->getSGID(), true));
1645 SG->addRule(std::make_shared<EnablesNthMFMA>(MFMAEnablement + 1, TII,
1647 SG->addRule(std::make_shared<IsFMA>(TII, SG->getSGID()));
1653 SchedGroupMask::DS_READ, 2, PipelineSyncID, DAG, TII);
1654 SG->addRule(std::make_shared<OccursAtOrAfterNode>(*FirstPipeDSR, TII,
1661 SchedGroupMask::TRANS, ExpRequirement, PipelineSyncID, DAG, TII);
1664 PositionInChain, MFMAChainSeeds[MFMAChain], TII, SG->getSGID(), true));
1666 SG->addRule(std::make_shared<EnablesNthMFMA>(1, TII, SG->getSGID(), true));
1667 SG->addRule(std::make_shared<IsPipeExp>(TII, SG->getSGID(), true));
1668 SG->addRule(std::make_shared<LessThanNSuccs>(8, TII, SG->getSGID(),
1679 SchedGroupMask::VALU, 1, PipelineSyncID, DAG, TII);
1680 SG->addRule(std::make_shared<IsCvt>(TII, SG->getSGID()));
1683 1 + (2 + UsesFMA) * I, TII, SG->getSGID()));
1686 1 + (2 + UsesFMA) * I, TII, SG->getSGID()));
1693 SchedGroupMask::VALU, 1, PipelineSyncID, DAG, TII);
1697 MFMAChainSeeds[getNextTransMFMAChain()], TII, SG->getSGID(), true));
1700 TII, SG->getSGID(), true));
1701 SG->addRule(std::make_shared<IsFMA>(TII, SG->getSGID()));
1707 SchedGroupMask::TRANS, 1, PipelineSyncID, DAG, TII);
1710 PositionInChain, MFMAChainSeeds[MFMAChain], TII, SG->getSGID(),
1713 SG->addRule(std::make_shared<EnablesNthMFMA>(MFMAEnablement + 1, TII,
1715 SG->addRule(std::make_shared<IsPipeExp>(TII, SG->getSGID(), true));
1716 SG->addRule(std::make_shared<LessThanNSuccs>(8, TII, SG->getSGID(),
1724 SchedGroupMask::TRANS, 1, PipelineSyncID, DAG, TII);
1725 SG->addRule(std::make_shared<IsPipeExp>(TII, SG->getSGID(), true));
1727 8, TII, SG->getSGID(), HasChainBetweenCvt));
1758 SchedGroupMask::MFMA, MFMARatio, PipelineSyncID, DAG, TII);
1761 PositionInChainForMFMA, MFMAChainSeeds[MFMAChainForMFMA], TII,
1764 SG->addRule(std::make_shared<OccursAfterExp>(TII, SG->getSGID(), true));
1770 SchedGroupMask::VALU, VALUOps, PipelineSyncID, DAG, TII);
1771 SG->addRule(std::make_shared<IsPipeAdd>(TII, SG->getSGID()));
1777 SchedGroupMask::DS_READ, 2, PipelineSyncID, DAG, TII);
1778 SG->addRule(std::make_shared<OccursAtOrAfterNode>(*FirstPipeDSR, TII,
1792 SchedGroupMask::VALU, 1, PipelineSyncID, DAG, TII);
1793 SG->addRule(std::make_shared<IsCvt>(TII, SG->getSGID()));
1804 CurrentOffset, TII, SG->getSGID()));
1806 SG->addRule(std::make_shared<IsSuccOfPrevNthGroup>(CurrentOffset, TII,
1814 SchedGroupMask::VALU, 1, PipelineSyncID, DAG, TII);
1818 MFMAChainSeeds[getNextTransMFMAChain()], TII, SG->getSGID(),
1823 TII, SG->getSGID(), true));
1824 SG->addRule(std::make_shared<IsFMA>(TII, SG->getSGID()));
1830 SchedGroupMask::TRANS, 1, PipelineSyncID, DAG, TII);
1833 PositionInChain, MFMAChainSeeds[MFMAChain], TII, SG->getSGID(),
1838 TII, SG->getSGID(), true));
1839 SG->addRule(std::make_shared<IsPipeExp>(TII, SG->getSGID(), true));
1840 SG->addRule(std::make_shared<LessThanNSuccs>(8, TII, SG->getSGID(),
1848 SchedGroupMask::MFMA, MFMAEnablement * 2, PipelineSyncID, DAG, TII);
1849 SG->addRule(std::make_shared<OccursAfterExp>(TII, SG->getSGID(), true));
1866 if (TII->isMFMAorWMMA(*Elt.getInstr())) {
1884 EnablesInitialMFMA(const SIInstrInfo *TII, unsigned SGID,
1886 : InstructionRule(TII, SGID, NeedsCache) {}
1903 if (TII->isDS(*SuccUnit->getInstr()) &&
1923 IsPermForDSW(const SIInstrInfo *TII, unsigned SGID, bool NeedsCache = false)
1924 : InstructionRule(TII, SGID, NeedsCache) {}
1954 IsSuccOfPrevGroup(const SIInstrInfo *TII, unsigned SGID,
1956 : InstructionRule(TII, SGID, NeedsCache) {}
1972 auto TRI = TII->getRegisterInfo();
1982 assert(TII->isVMEM(*MI) && MI->mayLoad());
1992 VMEMSize(const SIInstrInfo *TII, unsigned SGID, bool NeedsCache = false)
1993 : InstructionRule(TII, SGID, NeedsCache) {}
2042 SharesPredWithPrevNthGroup(unsigned Distance, const SIInstrInfo *TII,
2044 : InstructionRule(TII, SGID, NeedsCache), Distance(Distance) {}
2058 MFMASmallGemmSingleWaveOpt(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
2059 : IGLPStrategy(DAG, TII) {
2083 if (TII->isMFMAorWMMA(*I))
2085 else if (TII->isDS(*I)) {
2126 if (!TII->isVMEM(*MI) || !MI->mayLoad())
2163 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2167 SchedGroupMask::VALU, 2, PipelineSyncID, DAG, TII);
2179 SchedGroupMask::DS_READ, 4, PipelineSyncID, DAG, TII);
2180 SG->addRule(std::make_shared<EnablesInitialMFMA>(TII, SG->getSGID(), true));
2184 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2190 SchedGroupMask::DS_READ, 1, PipelineSyncID, DAG, TII);
2194 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2203 SchedGroupMask::VALU, 4, PipelineSyncID, DAG, TII);
2204 SG->addRule(std::make_shared<IsPermForDSW>(TII, SG->getSGID(), true));
2208 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG, TII);
2209 SG->addRule(std::make_shared<IsSuccOfPrevGroup>(TII, SG->getSGID()));
2213 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2215 1, TII, SG->getSGID(), true));
2216 SG->addRule(std::make_shared<VMEMSize>(TII, SG->getSGID()));
2220 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2224 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2226 3, TII, SG->getSGID(), true));
2227 SG->addRule(std::make_shared<VMEMSize>(TII, SG->getSGID()));
2231 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2240 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG, TII);
2244 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2245 SG->addRule(std::make_shared<VMEMSize>(TII, SG->getSGID()));
2249 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2260 SchedGroupMask::VALU, 4, PipelineSyncID, DAG, TII);
2261 SG->addRule(std::make_shared<IsPermForDSW>(TII, SG->getSGID(), true));
2265 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG, TII);
2266 SG->addRule(std::make_shared<IsSuccOfPrevGroup>(TII, SG->getSGID()));
2270 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2274 SchedGroupMask::VALU, 4, PipelineSyncID, DAG, TII);
2275 SG->addRule(std::make_shared<IsPermForDSW>(TII, SG->getSGID(), true));
2279 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG, TII);
2280 SG->addRule(std::make_shared<IsSuccOfPrevGroup>(TII, SG->getSGID()));
2284 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2288 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2290 2, TII, SG->getSGID(), true));
2291 SG->addRule(std::make_shared<VMEMSize>(TII, SG->getSGID()));
2295 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2299 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2301 4, TII, SG->getSGID(), true));
2302 SG->addRule(std::make_shared<VMEMSize>(TII, SG->getSGID()));
2306 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2315 const SIInstrInfo *TII) {
2318 return std::make_unique<MFMASmallGemmOpt>(DAG, TII);
2320 return std::make_unique<MFMASmallGemmSingleWaveOpt>(DAG, TII);
2322 return std::make_unique<MFMAExpInterleaveOpt>(DAG, TII);
2330 const SIInstrInfo *TII;
2395 (TII->isVALU(MI) || TII->isMFMAorWMMA(MI) || TII->isSALU(MI) ||
2396 TII->isTRANS(MI)))
2400 TII->isVALU(MI) && !TII->isMFMAorWMMA(MI) && !TII->isTRANS(MI))
2404 TII->isSALU(MI))
2408 TII->isMFMAorWMMA(MI))
2412 (TII->isVMEM(MI) || (TII->isFLAT(MI) && !TII->isDS(MI))))
2417 (TII->isVMEM(MI) || (TII->isFLAT(MI) && !TII->isDS(MI))))
2422 (TII->isVMEM(MI) || (TII->isFLAT(MI) && !TII->isDS(MI))))
2426 TII->isDS(MI))
2430 MI.mayLoad() && TII->isDS(MI))
2434 MI.mayStore() && TII->isDS(MI))
2438 TII->isTRANS(MI))
2562 TII = ST.getInstrInfo();
2606 SchedGroup SG(InvertedMask, std::nullopt, DAG, TII);
2667 Size, SyncID, DAG, TII);
2675 auto S = createIGLPStrategy(StrategyID, DAG, TII);