Lines Matching defs:DAG
9 // \file This file defines a set of schedule DAG mutations that can be used to
153 ScheduleDAGInstrs *DAG;
159 // Add DAG dependencies from all SUnits in this SchedGroup and this SU. If
164 // Add DAG dependencies and track which edges are added, and the count of
169 // Add DAG dependencies from all SUnits in this SchedGroup and this SU.
174 // Add DAG dependencies such that SUnits in this group shall be ordered
210 // Identify and add all relevant SUs from the DAG to this SchedGroup.
230 ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
231 : SGMask(SGMask), MaxSize(MaxSize), DAG(DAG), TII(TII) {
236 ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
237 : SGMask(SGMask), MaxSize(MaxSize), SyncID(SyncID), DAG(DAG), TII(TII) {
248 // in the DAG, then we will have an instruction that can not be trivially
255 [[maybe_unused]] ScheduleDAGMI *DAG;
346 ScheduleDAGMI *DAG, bool IsBottomUp = true)
347 : DAG(DAG), SyncedInstrs(SyncedInstrs),
712 // traversed the DAG from top down, parse over the groups from last to
793 LLVM_DEBUG(DAG->dump());
811 LLVM_DEBUG(DAG->dump());
824 ScheduleDAGInstrs *DAG;
836 virtual bool shouldApplyStrategy(ScheduleDAGInstrs *DAG,
841 IGLPStrategy(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
842 : DAG(DAG), TII(TII) {}
855 bool shouldApplyStrategy(ScheduleDAGInstrs *DAG,
860 MFMASmallGemmOpt(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
861 : IGLPStrategy(DAG, TII) {
872 for (const MachineInstr &I : *DAG)
880 SchedGroupMask::DS, 2, PipelineSyncID, DAG, TII);
884 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
916 // Compute the heuristics for the pipeline, returning whether or not the DAG
927 auto *DAG = SyncPipe[0].DAG;
930 auto I = DAG->SUnits.rbegin();
931 auto E = DAG->SUnits.rend();
940 auto Reaches = any_of(*Cache, [&SU, &DAG](SUnit *TargetSU) {
941 return DAG->IsReachable(TargetSU, const_cast<SUnit *>(SU));
961 auto *DAG = SyncPipe[0].DAG;
966 auto I = DAG->SUnits.begin();
967 auto E = DAG->SUnits.end();
983 return DAG->IsReachable((*Cache)[0], const_cast<SUnit *>(SU));
1001 auto *DAG = SyncPipe[0].DAG;
1029 return DAG->IsReachable((*Cache)[0], const_cast<SUnit *>(SU));
1217 auto *DAG = SyncPipe[0].DAG;
1220 if (DAG->IsReachable(const_cast<SUnit *>(SU), OtherEle))
1300 auto *DAG = SyncPipe[0].DAG;
1302 for (auto &SU : DAG->SUnits)
1325 bool shouldApplyStrategy(ScheduleDAGInstrs *DAG,
1328 MFMAExpInterleaveOpt(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
1329 : IGLPStrategy(DAG, TII) {
1363 for (SUnit &SU : DAG->SUnits) {
1399 if (DAG->IsReachable(SuccSU, PredSU)) {
1427 if (DAG->IsReachable(SuccSU, PredSU)) {
1441 if (DAG->IsReachable(SuccSU, *TempExp)) {
1450 if (DAG->IsReachable(SuccSU, *TempCvt)) {
1483 return DAG->IsReachable(VPack, *TempExp);
1520 return DAG->IsReachable(PackPred->getSUnit(), ExpBase);
1527 bool MFMAExpInterleaveOpt::shouldApplyStrategy(ScheduleDAGInstrs *DAG,
1529 const GCNSubtarget &ST = DAG->MF.getSubtarget<GCNSubtarget>();
1553 const GCNSubtarget &ST = DAG->MF.getSubtarget<GCNSubtarget>();
1603 SchedGroupMask::VALU, ExpRequirement, PipelineSyncID, DAG, TII);
1616 SchedGroupMask::VALU, ExpRequirement, PipelineSyncID, DAG, TII);
1630 SchedGroupMask::DS_READ, 2, PipelineSyncID, DAG, TII);
1638 SchedGroupMask::TRANS, ExpRequirement, PipelineSyncID, DAG, TII);
1656 SchedGroupMask::VALU, 1, PipelineSyncID, DAG, TII);
1670 SchedGroupMask::VALU, 1, PipelineSyncID, DAG, TII);
1684 SchedGroupMask::TRANS, 1, PipelineSyncID, DAG, TII);
1701 SchedGroupMask::TRANS, 1, PipelineSyncID, DAG, TII);
1735 SchedGroupMask::MFMA, MFMARatio, PipelineSyncID, DAG, TII);
1747 SchedGroupMask::VALU, VALUOps, PipelineSyncID, DAG, TII);
1754 SchedGroupMask::DS_READ, 2, PipelineSyncID, DAG, TII);
1769 SchedGroupMask::VALU, 1, PipelineSyncID, DAG, TII);
1791 SchedGroupMask::VALU, 1, PipelineSyncID, DAG, TII);
1807 SchedGroupMask::TRANS, 1, PipelineSyncID, DAG, TII);
1825 SchedGroupMask::MFMA, MFMAEnablement * 2, PipelineSyncID, DAG, TII);
1838 bool shouldApplyStrategy(ScheduleDAGInstrs *DAG,
1843 MFMAExpSimpleInterleaveOpt(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
1844 : IGLPStrategy(DAG, TII) {
1855 for (const MachineInstr &I : *DAG)
1862 SchedGroupMask::TRANS, 1, PipelineSyncID, DAG, TII);
1866 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
1884 for (auto &Elt : SyncPipe[0].DAG->SUnits) {
1895 auto *DAG = SyncPipe[0].DAG;
1897 if (DAG->IsReachable(Elt, const_cast<SUnit *>(SU)))
2050 auto *DAG = SyncPipe[0].DAG;
2053 return llvm::any_of(*Cache, [&SU, &DAG](SUnit *Elt) {
2054 return DAG->IsReachable(const_cast<SUnit *>(SU), Elt);
2068 bool shouldApplyStrategy(ScheduleDAGInstrs *DAG,
2073 MFMASmallGemmSingleWaveOpt(ScheduleDAGInstrs *DAG, const SIInstrInfo *TII)
2074 : IGLPStrategy(DAG, TII) {
2096 for (auto &SU : DAG->SUnits) {
2178 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2182 SchedGroupMask::VALU, 2, PipelineSyncID, DAG, TII);
2194 SchedGroupMask::DS_READ, 4, PipelineSyncID, DAG, TII);
2199 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2205 SchedGroupMask::DS_READ, 1, PipelineSyncID, DAG, TII);
2209 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2218 SchedGroupMask::VALU, 4, PipelineSyncID, DAG, TII);
2223 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG, TII);
2228 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2235 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2239 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2246 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2255 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG, TII);
2259 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2264 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2275 SchedGroupMask::VALU, 4, PipelineSyncID, DAG, TII);
2280 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG, TII);
2285 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2289 SchedGroupMask::VALU, 4, PipelineSyncID, DAG, TII);
2294 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG, TII);
2299 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2303 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2310 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2314 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG, TII);
2321 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG, TII);
2329 createIGLPStrategy(IGLPStrategyID ID, ScheduleDAGInstrs *DAG,
2333 return std::make_unique<MFMASmallGemmOpt>(DAG, TII);
2335 return std::make_unique<MFMASmallGemmSingleWaveOpt>(DAG, TII);
2337 return std::make_unique<MFMAExpInterleaveOpt>(DAG, TII);
2339 return std::make_unique<MFMAExpSimpleInterleaveOpt>(DAG, TII);
2349 ScheduleDAGMI *DAG;
2359 // Add DAG edges that enforce SCHED_BARRIER ordering.
2399 if (A != B && DAG->canAddEdge(B, A)) {
2400 DAG->addEdge(B, SDep(A, SDep::Artificial));
2475 if (DAG->IsReachable(B, A))
2534 for (auto &SU : DAG->SUnits) {
2546 for (auto E = DAG->SUnits.rend(); RIter != E; ++RIter) {
2561 auto I = DAG->SUnits.rbegin();
2562 auto E = DAG->SUnits.rend();
2580 DAG = static_cast<ScheduleDAGMI *>(DAGInstrs);
2586 for (auto R = DAG->SUnits.rbegin(), E = DAG->SUnits.rend(); R != E; ++R) {
2604 PipelineSolver PS(SyncedSchedGroups, SyncedInstrs, DAG, IsBottomUp);
2621 SchedGroup SG(InvertedMask, std::nullopt, DAG, TII);
2681 Size, SyncID, DAG, TII);
2689 auto S = createIGLPStrategy(StrategyID, DAG, TII);
2690 if (!S->shouldApplyStrategy(DAG, Phase))