Lines Matching defs:TII
90 const SIInstrInfo *TII;
103 InstructionRule(const SIInstrInfo *TII, unsigned SGID,
105 : TII(TII), SGID(SGID) {
154 const SIInstrInfo *TII;
230 ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
231 : SGMask(SGMask), MaxSize(MaxSize), DAG(DAG), TII(TII) {
236 ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
237 : SGMask(SGMask), MaxSize(MaxSize), SyncID(SyncID), DAG(DAG), TII(TII) {
826 const SIInstrInfo *TII;
841 IGLPStrategy(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
842 : DAG(DAG), TII(TII) {}
860 MFMASmallGemmOpt(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
861 : IGLPStrategy(DAG, TII) {
873 if (TII->isMFMAorWMMA(I))
880 SchedGroupMask::DS, 2, PipelineSyncID, DAG, TII);
884 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
918 bool analyzeDAG(const SIInstrInfo *TII);
933 if (TII->isMFMAorWMMA(*I->getInstr()))
946 IsPipeExp(const SIInstrInfo *TII, unsigned SGID, bool NeedsCache = false)
947 : InstructionRule(TII, SGID, NeedsCache) {}
969 if (FoundTrans && TII->isMFMAorWMMA(*I->getInstr())) {
976 if (!FoundTrans && TII->isTRANS(I->getInstr()->getOpcode()))
986 EnablesNthMFMA(unsigned Number, const SIInstrInfo *TII, unsigned SGID,
988 : InstructionRule(TII, SGID, NeedsCache), Number(Number) {}
1003 if (!SU || !TII->isMFMAorWMMA(*ChainSeed->getInstr()))
1013 if (TII->isMFMAorWMMA(*Succ.getSUnit()->getInstr())) {
1033 const SIInstrInfo *TII, unsigned SGID,
1035 : InstructionRule(TII, SGID, NeedsCache), Number(Number),
1073 LessThanNSuccs(unsigned Size, const SIInstrInfo *TII, unsigned SGID,
1075 : InstructionRule(TII, SGID, NeedsCache), Size(Size),
1114 GreaterThanOrEqualToNSuccs(unsigned Size, const SIInstrInfo *TII,
1117 : InstructionRule(TII, SGID, NeedsCache), Size(Size),
1130 IsCvt(const SIInstrInfo *TII, unsigned SGID, bool NeedsCache = false)
1131 : InstructionRule(TII, SGID, NeedsCache) {}
1142 IsFMA(const SIInstrInfo *TII, unsigned SGID, bool NeedsCache = false)
1143 : InstructionRule(TII, SGID, NeedsCache) {}
1153 IsPipeAdd(const SIInstrInfo *TII, unsigned SGID, bool NeedsCache = false)
1154 : InstructionRule(TII, SGID, NeedsCache) {}
1189 IsSuccOfPrevNthGroup(unsigned Distance, const SIInstrInfo *TII,
1191 : InstructionRule(TII, SGID, NeedsCache), Distance(Distance) {}
1225 IsReachableFromPrevNthGroup(unsigned Distance, const SIInstrInfo *TII,
1227 : InstructionRule(TII, SGID, NeedsCache), Distance(Distance) {}
1241 OccursAtOrAfterNode(unsigned Number, const SIInstrInfo *TII, unsigned SGID,
1243 : InstructionRule(TII, SGID, NeedsCache), Number(Number) {}
1256 if (!SU || !TII->isMFMAorWMMA(*ChainSeed->getInstr()))
1266 if (TII->isMFMAorWMMA(*Succ.getSUnit()->getInstr())) {
1285 IsExactMFMA(unsigned Number, SUnit *ChainSeed, const SIInstrInfo *TII,
1287 : InstructionRule(TII, SGID, NeedsCache), Number(Number),
1303 if (TII->isTRANS(SU.getInstr()->getOpcode())) {
1314 OccursAfterExp(const SIInstrInfo *TII, unsigned SGID,
1316 : InstructionRule(TII, SGID, NeedsCache) {}
1328 MFMAExpInterleaveOpt(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
1329 : IGLPStrategy(DAG, TII) {
1345 bool MFMAExpInterleaveOpt::analyzeDAG(const SIInstrInfo *TII) {
1365 if (TII->isTRANS(Opc)) {
1376 if (TII->isMFMAorWMMA(*SU.getInstr()))
1461 if (none_of(MFMAPipeSU->Preds, [&TII](SDep &Succ) {
1462 return TII->isMFMAorWMMA(*Succ.getSUnit()->getInstr());
1473 if (TII->isDS(Pred.getSUnit()->getInstr()->getOpcode()) &&
1509 PackPred->getSUnit()->Succs.end(), [&TII](SDep &Succ) {
1510 return TII->isMFMAorWMMA(*Succ.getSUnit()->getInstr());
1530 const SIInstrInfo *TII = ST.getInstrInfo();
1534 if (Phase != AMDGPU::SchedulingPhase::PostRA && !analyzeDAG(TII))
1554 const SIInstrInfo *TII = ST.getInstrInfo();
1603 SchedGroupMask::VALU, ExpRequirement, PipelineSyncID, DAG, TII);
1606 PositionInChain, MFMAChainSeeds[MFMAChain], TII, SG->getSGID(),
1610 std::make_shared<EnablesNthMFMA>(1, TII, SG->getSGID(), true));
1611 SG->addRule(std::make_shared<IsFMA>(TII, SG->getSGID()));
1616 SchedGroupMask::VALU, ExpRequirement, PipelineSyncID, DAG, TII);
1620 MFMAChainSeeds[getNextTransMFMAChain()], TII, SG->getSGID(), true));
1622 SG->addRule(std::make_shared<EnablesNthMFMA>(MFMAEnablement + 1, TII,
1624 SG->addRule(std::make_shared<IsFMA>(TII, SG->getSGID()));
1630 SchedGroupMask::DS_READ, 2, PipelineSyncID, DAG, TII);
1631 SG->addRule(std::make_shared<OccursAtOrAfterNode>(*FirstPipeDSR, TII,
1638 SchedGroupMask::TRANS, ExpRequirement, PipelineSyncID, DAG, TII);
1641 PositionInChain, MFMAChainSeeds[MFMAChain], TII, SG->getSGID(), true));
1643 SG->addRule(std::make_shared<EnablesNthMFMA>(1, TII, SG->getSGID(), true));
1644 SG->addRule(std::make_shared<IsPipeExp>(TII, SG->getSGID(), true));
1645 SG->addRule(std::make_shared<LessThanNSuccs>(8, TII, SG->getSGID(),
1656 SchedGroupMask::VALU, 1, PipelineSyncID, DAG, TII);
1657 SG->addRule(std::make_shared<IsCvt>(TII, SG->getSGID()));
1660 1 + (2 + UsesFMA) * I, TII, SG->getSGID()));
1663 1 + (2 + UsesFMA) * I, TII, SG->getSGID()));
1670 SchedGroupMask::VALU, 1, PipelineSyncID, DAG, TII);
1674 MFMAChainSeeds[getNextTransMFMAChain()], TII, SG->getSGID(), true));
1677 TII, SG->getSGID(), true));
1678 SG->addRule(std::make_shared<IsFMA>(TII, SG->getSGID()));
1684 SchedGroupMask::TRANS, 1, PipelineSyncID, DAG, TII);
1687 PositionInChain, MFMAChainSeeds[MFMAChain], TII, SG->getSGID(),
1690 SG->addRule(std::make_shared<EnablesNthMFMA>(MFMAEnablement + 1, TII,
1692 SG->addRule(std::make_shared<IsPipeExp>(TII, SG->getSGID(), true));
1693 SG->addRule(std::make_shared<LessThanNSuccs>(8, TII, SG->getSGID(),
1701 SchedGroupMask::TRANS, 1, PipelineSyncID, DAG, TII);
1702 SG->addRule(std::make_shared<IsPipeExp>(TII, SG->getSGID(), true));
1704 8, TII, SG->getSGID(), HasChainBetweenCvt));
1735 SchedGroupMask::MFMA, MFMARatio, PipelineSyncID, DAG, TII);
1738 PositionInChainForMFMA, MFMAChainSeeds[MFMAChainForMFMA], TII,
1741 SG->addRule(std::make_shared<OccursAfterExp>(TII, SG->getSGID(), true));
1747 SchedGroupMask::VALU, VALUOps, PipelineSyncID, DAG, TII);
1748 SG->addRule(std::make_shared<IsPipeAdd>(TII, SG->getSGID()));
1754 SchedGroupMask::DS_READ, 2, PipelineSyncID, DAG, TII);
1755 SG->addRule(std::make_shared<OccursAtOrAfterNode>(*FirstPipeDSR, TII,
1769 SchedGroupMask::VALU, 1, PipelineSyncID, DAG, TII);
1770 SG->addRule(std::make_shared<IsCvt>(TII, SG->getSGID()));
1781 CurrentOffset, TII, SG->getSGID()));
1783 SG->addRule(std::make_shared<IsSuccOfPrevNthGroup>(CurrentOffset, TII,
1791 SchedGroupMask::VALU, 1, PipelineSyncID, DAG, TII);
1795 MFMAChainSeeds[getNextTransMFMAChain()], TII, SG->getSGID(),
1800 TII, SG->getSGID(), true));
1801 SG->addRule(std::make_shared<IsFMA>(TII, SG->getSGID()));
1807 SchedGroupMask::TRANS, 1, PipelineSyncID, DAG, TII);
1810 PositionInChain, MFMAChainSeeds[MFMAChain], TII, SG->getSGID(),
1815 TII, SG->getSGID(), true));
1816 SG->addRule(std::make_shared<IsPipeExp>(TII, SG->getSGID(), true));
1817 SG->addRule(std::make_shared<LessThanNSuccs>(8, TII, SG->getSGID(),
1825 SchedGroupMask::MFMA, MFMAEnablement * 2, PipelineSyncID, DAG, TII);
1826 SG->addRule(std::make_shared<OccursAfterExp>(TII, SG->getSGID(), true));
1843 MFMAExpSimpleInterleaveOpt(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
1844 : IGLPStrategy(DAG, TII) {
1856 if (TII->isMFMAorWMMA(I))
1862 SchedGroupMask::TRANS, 1, PipelineSyncID, DAG, TII);
1866 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
1885 if (TII->isMFMAorWMMA(*Elt.getInstr())) {
1903 EnablesInitialMFMA(const SIInstrInfo *TII, unsigned SGID,
1905 : InstructionRule(TII, SGID, NeedsCache) {}
1922 if (TII->isDS(*SuccUnit->getInstr()) &&
1942 IsPermForDSW(const SIInstrInfo *TII, unsigned SGID, bool NeedsCache = false)
1943 : InstructionRule(TII, SGID, NeedsCache) {}
1969 IsSuccOfPrevGroup(const SIInstrInfo *TII, unsigned SGID,
1971 : InstructionRule(TII, SGID, NeedsCache) {}
1987 auto TRI = TII->getRegisterInfo();
1997 assert(TII->isVMEM(*MI) && MI->mayLoad());
2007 VMEMSize(const SIInstrInfo *TII, unsigned SGID, bool NeedsCache = false)
2008 : InstructionRule(TII, SGID, NeedsCache) {}
2057 SharesPredWithPrevNthGroup(unsigned Distance, const SIInstrInfo *TII,
2059 : InstructionRule(TII, SGID, NeedsCache), Distance(Distance) {}
2073 MFMASmallGemmSingleWaveOpt(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
2074 : IGLPStrategy(DAG, TII) {
2098 if (TII->isMFMAorWMMA(*I))
2100 else if (TII->isDS(*I)) {
2141 if (!TII->isVMEM(*MI) || !MI->mayLoad())
2178 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2182 SchedGroupMask::VALU, 2, PipelineSyncID, DAG, TII);
2194 SchedGroupMask::DS_READ, 4, PipelineSyncID, DAG, TII);
2195 SG->addRule(std::make_shared<EnablesInitialMFMA>(TII, SG->getSGID(), true));
2199 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2205 SchedGroupMask::DS_READ, 1, PipelineSyncID, DAG, TII);
2209 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2218 SchedGroupMask::VALU, 4, PipelineSyncID, DAG, TII);
2219 SG->addRule(std::make_shared<IsPermForDSW>(TII, SG->getSGID(), true));
2223 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG, TII);
2224 SG->addRule(std::make_shared<IsSuccOfPrevGroup>(TII, SG->getSGID()));
2228 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2230 1, TII, SG->getSGID(), true));
2231 SG->addRule(std::make_shared<VMEMSize>(TII, SG->getSGID()));
2235 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2239 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2241 3, TII, SG->getSGID(), true));
2242 SG->addRule(std::make_shared<VMEMSize>(TII, SG->getSGID()));
2246 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2255 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG, TII);
2259 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2260 SG->addRule(std::make_shared<VMEMSize>(TII, SG->getSGID()));
2264 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2275 SchedGroupMask::VALU, 4, PipelineSyncID, DAG, TII);
2276 SG->addRule(std::make_shared<IsPermForDSW>(TII, SG->getSGID(), true));
2280 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG, TII);
2281 SG->addRule(std::make_shared<IsSuccOfPrevGroup>(TII, SG->getSGID()));
2285 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2289 SchedGroupMask::VALU, 4, PipelineSyncID, DAG, TII);
2290 SG->addRule(std::make_shared<IsPermForDSW>(TII, SG->getSGID(), true));
2294 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG, TII);
2295 SG->addRule(std::make_shared<IsSuccOfPrevGroup>(TII, SG->getSGID()));
2299 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2303 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2305 2, TII, SG->getSGID(), true));
2306 SG->addRule(std::make_shared<VMEMSize>(TII, SG->getSGID()));
2310 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2314 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2316 4, TII, SG->getSGID(), true));
2317 SG->addRule(std::make_shared<VMEMSize>(TII, SG->getSGID()));
2321 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2330 const SIInstrInfo *TII) {
2333 return std::make_unique<MFMASmallGemmOpt>(DAG, TII);
2335 return std::make_unique<MFMASmallGemmSingleWaveOpt>(DAG, TII);
2337 return std::make_unique<MFMAExpInterleaveOpt>(DAG, TII);
2339 return std::make_unique<MFMAExpSimpleInterleaveOpt>(DAG, TII);
2347 const SIInstrInfo *TII;
2412 (TII->isVALU(MI) || TII->isMFMAorWMMA(MI) || TII->isSALU(MI) ||
2413 TII->isTRANS(MI)))
2417 TII->isVALU(MI) && !TII->isMFMAorWMMA(MI) && !TII->isTRANS(MI))
2421 TII->isSALU(MI))
2425 TII->isMFMAorWMMA(MI))
2429 (TII->isVMEM(MI) || (TII->isFLAT(MI) && !TII->isDS(MI))))
2434 (TII->isVMEM(MI) || (TII->isFLAT(MI) && !TII->isDS(MI))))
2439 (TII->isVMEM(MI) || (TII->isFLAT(MI) && !TII->isDS(MI))))
2443 TII->isDS(MI))
2447 MI.mayLoad() && TII->isDS(MI))
2451 MI.mayStore() && TII->isDS(MI))
2455 TII->isTRANS(MI))
2579 TII = ST.getInstrInfo();
2621 SchedGroup SG(InvertedMask, std::nullopt, DAG, TII);
2681 Size, SyncID, DAG, TII);
2689 auto S = createIGLPStrategy(StrategyID, DAG, TII);