109467b48Spatrick //===- ScheduleDAG.cpp - Implement the ScheduleDAG class ------------------===//
209467b48Spatrick //
309467b48Spatrick // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
409467b48Spatrick // See https://llvm.org/LICENSE.txt for license information.
509467b48Spatrick // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
609467b48Spatrick //
709467b48Spatrick //===----------------------------------------------------------------------===//
809467b48Spatrick //
909467b48Spatrick /// \file Implements the ScheduleDAG class, which is a base class used by
1009467b48Spatrick /// scheduling implementation classes.
1109467b48Spatrick //
1209467b48Spatrick //===----------------------------------------------------------------------===//
1309467b48Spatrick
1409467b48Spatrick #include "llvm/CodeGen/ScheduleDAG.h"
1509467b48Spatrick #include "llvm/ADT/STLExtras.h"
1609467b48Spatrick #include "llvm/ADT/SmallVector.h"
1709467b48Spatrick #include "llvm/ADT/Statistic.h"
1809467b48Spatrick #include "llvm/ADT/iterator_range.h"
1909467b48Spatrick #include "llvm/CodeGen/MachineFunction.h"
2009467b48Spatrick #include "llvm/CodeGen/ScheduleHazardRecognizer.h"
2109467b48Spatrick #include "llvm/CodeGen/SelectionDAGNodes.h"
2209467b48Spatrick #include "llvm/CodeGen/TargetInstrInfo.h"
2309467b48Spatrick #include "llvm/CodeGen/TargetRegisterInfo.h"
2409467b48Spatrick #include "llvm/CodeGen/TargetSubtargetInfo.h"
2509467b48Spatrick #include "llvm/Config/llvm-config.h"
2609467b48Spatrick #include "llvm/Support/CommandLine.h"
2709467b48Spatrick #include "llvm/Support/Compiler.h"
2809467b48Spatrick #include "llvm/Support/Debug.h"
2909467b48Spatrick #include "llvm/Support/raw_ostream.h"
3009467b48Spatrick #include <algorithm>
3109467b48Spatrick #include <cassert>
3209467b48Spatrick #include <iterator>
3309467b48Spatrick #include <limits>
3409467b48Spatrick #include <utility>
3509467b48Spatrick #include <vector>
3609467b48Spatrick
3709467b48Spatrick using namespace llvm;
3809467b48Spatrick
3909467b48Spatrick #define DEBUG_TYPE "pre-RA-sched"
4009467b48Spatrick
4109467b48Spatrick STATISTIC(NumNewPredsAdded, "Number of times a single predecessor was added");
4209467b48Spatrick STATISTIC(NumTopoInits,
4309467b48Spatrick "Number of times the topological order has been recomputed");
4409467b48Spatrick
4509467b48Spatrick #ifndef NDEBUG
4609467b48Spatrick static cl::opt<bool> StressSchedOpt(
4709467b48Spatrick "stress-sched", cl::Hidden, cl::init(false),
4809467b48Spatrick cl::desc("Stress test instruction scheduling"));
4909467b48Spatrick #endif
5009467b48Spatrick
anchor()5109467b48Spatrick void SchedulingPriorityQueue::anchor() {}
5209467b48Spatrick
ScheduleDAG(MachineFunction & mf)5309467b48Spatrick ScheduleDAG::ScheduleDAG(MachineFunction &mf)
5409467b48Spatrick : TM(mf.getTarget()), TII(mf.getSubtarget().getInstrInfo()),
5509467b48Spatrick TRI(mf.getSubtarget().getRegisterInfo()), MF(mf),
5609467b48Spatrick MRI(mf.getRegInfo()) {
5709467b48Spatrick #ifndef NDEBUG
5809467b48Spatrick StressSched = StressSchedOpt;
5909467b48Spatrick #endif
6009467b48Spatrick }
6109467b48Spatrick
6209467b48Spatrick ScheduleDAG::~ScheduleDAG() = default;
6309467b48Spatrick
clearDAG()6409467b48Spatrick void ScheduleDAG::clearDAG() {
6509467b48Spatrick SUnits.clear();
6609467b48Spatrick EntrySU = SUnit();
6709467b48Spatrick ExitSU = SUnit();
6809467b48Spatrick }
6909467b48Spatrick
getNodeDesc(const SDNode * Node) const7009467b48Spatrick const MCInstrDesc *ScheduleDAG::getNodeDesc(const SDNode *Node) const {
7109467b48Spatrick if (!Node || !Node->isMachineOpcode()) return nullptr;
7209467b48Spatrick return &TII->get(Node->getMachineOpcode());
7309467b48Spatrick }
7409467b48Spatrick
dump(const TargetRegisterInfo * TRI) const7509467b48Spatrick LLVM_DUMP_METHOD void SDep::dump(const TargetRegisterInfo *TRI) const {
7609467b48Spatrick switch (getKind()) {
7709467b48Spatrick case Data: dbgs() << "Data"; break;
7809467b48Spatrick case Anti: dbgs() << "Anti"; break;
7909467b48Spatrick case Output: dbgs() << "Out "; break;
8009467b48Spatrick case Order: dbgs() << "Ord "; break;
8109467b48Spatrick }
8209467b48Spatrick
8309467b48Spatrick switch (getKind()) {
8409467b48Spatrick case Data:
8509467b48Spatrick dbgs() << " Latency=" << getLatency();
8609467b48Spatrick if (TRI && isAssignedRegDep())
8709467b48Spatrick dbgs() << " Reg=" << printReg(getReg(), TRI);
8809467b48Spatrick break;
8909467b48Spatrick case Anti:
9009467b48Spatrick case Output:
9109467b48Spatrick dbgs() << " Latency=" << getLatency();
9209467b48Spatrick break;
9309467b48Spatrick case Order:
9409467b48Spatrick dbgs() << " Latency=" << getLatency();
9509467b48Spatrick switch(Contents.OrdKind) {
9609467b48Spatrick case Barrier: dbgs() << " Barrier"; break;
9709467b48Spatrick case MayAliasMem:
9809467b48Spatrick case MustAliasMem: dbgs() << " Memory"; break;
9909467b48Spatrick case Artificial: dbgs() << " Artificial"; break;
10009467b48Spatrick case Weak: dbgs() << " Weak"; break;
10109467b48Spatrick case Cluster: dbgs() << " Cluster"; break;
10209467b48Spatrick }
10309467b48Spatrick break;
10409467b48Spatrick }
10509467b48Spatrick }
10609467b48Spatrick
addPred(const SDep & D,bool Required)10709467b48Spatrick bool SUnit::addPred(const SDep &D, bool Required) {
10809467b48Spatrick // If this node already has this dependence, don't add a redundant one.
10909467b48Spatrick for (SDep &PredDep : Preds) {
11009467b48Spatrick // Zero-latency weak edges may be added purely for heuristic ordering. Don't
11109467b48Spatrick // add them if another kind of edge already exists.
11209467b48Spatrick if (!Required && PredDep.getSUnit() == D.getSUnit())
11309467b48Spatrick return false;
11409467b48Spatrick if (PredDep.overlaps(D)) {
11509467b48Spatrick // Extend the latency if needed. Equivalent to
11609467b48Spatrick // removePred(PredDep) + addPred(D).
11709467b48Spatrick if (PredDep.getLatency() < D.getLatency()) {
11809467b48Spatrick SUnit *PredSU = PredDep.getSUnit();
11909467b48Spatrick // Find the corresponding successor in N.
12009467b48Spatrick SDep ForwardD = PredDep;
12109467b48Spatrick ForwardD.setSUnit(this);
12209467b48Spatrick for (SDep &SuccDep : PredSU->Succs) {
12309467b48Spatrick if (SuccDep == ForwardD) {
12409467b48Spatrick SuccDep.setLatency(D.getLatency());
12509467b48Spatrick break;
12609467b48Spatrick }
12709467b48Spatrick }
12809467b48Spatrick PredDep.setLatency(D.getLatency());
12909467b48Spatrick }
13009467b48Spatrick return false;
13109467b48Spatrick }
13209467b48Spatrick }
13309467b48Spatrick // Now add a corresponding succ to N.
13409467b48Spatrick SDep P = D;
13509467b48Spatrick P.setSUnit(this);
13609467b48Spatrick SUnit *N = D.getSUnit();
13709467b48Spatrick // Update the bookkeeping.
13809467b48Spatrick if (D.getKind() == SDep::Data) {
13909467b48Spatrick assert(NumPreds < std::numeric_limits<unsigned>::max() &&
14009467b48Spatrick "NumPreds will overflow!");
14109467b48Spatrick assert(N->NumSuccs < std::numeric_limits<unsigned>::max() &&
14209467b48Spatrick "NumSuccs will overflow!");
14309467b48Spatrick ++NumPreds;
14409467b48Spatrick ++N->NumSuccs;
14509467b48Spatrick }
14609467b48Spatrick if (!N->isScheduled) {
14709467b48Spatrick if (D.isWeak()) {
14809467b48Spatrick ++WeakPredsLeft;
14909467b48Spatrick }
15009467b48Spatrick else {
15109467b48Spatrick assert(NumPredsLeft < std::numeric_limits<unsigned>::max() &&
15209467b48Spatrick "NumPredsLeft will overflow!");
15309467b48Spatrick ++NumPredsLeft;
15409467b48Spatrick }
15509467b48Spatrick }
15609467b48Spatrick if (!isScheduled) {
15709467b48Spatrick if (D.isWeak()) {
15809467b48Spatrick ++N->WeakSuccsLeft;
15909467b48Spatrick }
16009467b48Spatrick else {
16109467b48Spatrick assert(N->NumSuccsLeft < std::numeric_limits<unsigned>::max() &&
16209467b48Spatrick "NumSuccsLeft will overflow!");
16309467b48Spatrick ++N->NumSuccsLeft;
16409467b48Spatrick }
16509467b48Spatrick }
16609467b48Spatrick Preds.push_back(D);
16709467b48Spatrick N->Succs.push_back(P);
16809467b48Spatrick if (P.getLatency() != 0) {
16909467b48Spatrick this->setDepthDirty();
17009467b48Spatrick N->setHeightDirty();
17109467b48Spatrick }
17209467b48Spatrick return true;
17309467b48Spatrick }
17409467b48Spatrick
removePred(const SDep & D)17509467b48Spatrick void SUnit::removePred(const SDep &D) {
17609467b48Spatrick // Find the matching predecessor.
17709467b48Spatrick SmallVectorImpl<SDep>::iterator I = llvm::find(Preds, D);
17809467b48Spatrick if (I == Preds.end())
17909467b48Spatrick return;
18009467b48Spatrick // Find the corresponding successor in N.
18109467b48Spatrick SDep P = D;
18209467b48Spatrick P.setSUnit(this);
18309467b48Spatrick SUnit *N = D.getSUnit();
18409467b48Spatrick SmallVectorImpl<SDep>::iterator Succ = llvm::find(N->Succs, P);
18509467b48Spatrick assert(Succ != N->Succs.end() && "Mismatching preds / succs lists!");
18609467b48Spatrick N->Succs.erase(Succ);
18709467b48Spatrick Preds.erase(I);
18809467b48Spatrick // Update the bookkeeping.
18909467b48Spatrick if (P.getKind() == SDep::Data) {
19009467b48Spatrick assert(NumPreds > 0 && "NumPreds will underflow!");
19109467b48Spatrick assert(N->NumSuccs > 0 && "NumSuccs will underflow!");
19209467b48Spatrick --NumPreds;
19309467b48Spatrick --N->NumSuccs;
19409467b48Spatrick }
19509467b48Spatrick if (!N->isScheduled) {
19609467b48Spatrick if (D.isWeak())
19709467b48Spatrick --WeakPredsLeft;
19809467b48Spatrick else {
19909467b48Spatrick assert(NumPredsLeft > 0 && "NumPredsLeft will underflow!");
20009467b48Spatrick --NumPredsLeft;
20109467b48Spatrick }
20209467b48Spatrick }
20309467b48Spatrick if (!isScheduled) {
20409467b48Spatrick if (D.isWeak())
20509467b48Spatrick --N->WeakSuccsLeft;
20609467b48Spatrick else {
20709467b48Spatrick assert(N->NumSuccsLeft > 0 && "NumSuccsLeft will underflow!");
20809467b48Spatrick --N->NumSuccsLeft;
20909467b48Spatrick }
21009467b48Spatrick }
21109467b48Spatrick if (P.getLatency() != 0) {
21209467b48Spatrick this->setDepthDirty();
21309467b48Spatrick N->setHeightDirty();
21409467b48Spatrick }
21509467b48Spatrick }
21609467b48Spatrick
setDepthDirty()21709467b48Spatrick void SUnit::setDepthDirty() {
21809467b48Spatrick if (!isDepthCurrent) return;
21909467b48Spatrick SmallVector<SUnit*, 8> WorkList;
22009467b48Spatrick WorkList.push_back(this);
22109467b48Spatrick do {
22209467b48Spatrick SUnit *SU = WorkList.pop_back_val();
22309467b48Spatrick SU->isDepthCurrent = false;
22409467b48Spatrick for (SDep &SuccDep : SU->Succs) {
22509467b48Spatrick SUnit *SuccSU = SuccDep.getSUnit();
22609467b48Spatrick if (SuccSU->isDepthCurrent)
22709467b48Spatrick WorkList.push_back(SuccSU);
22809467b48Spatrick }
22909467b48Spatrick } while (!WorkList.empty());
23009467b48Spatrick }
23109467b48Spatrick
setHeightDirty()23209467b48Spatrick void SUnit::setHeightDirty() {
23309467b48Spatrick if (!isHeightCurrent) return;
23409467b48Spatrick SmallVector<SUnit*, 8> WorkList;
23509467b48Spatrick WorkList.push_back(this);
23609467b48Spatrick do {
23709467b48Spatrick SUnit *SU = WorkList.pop_back_val();
23809467b48Spatrick SU->isHeightCurrent = false;
23909467b48Spatrick for (SDep &PredDep : SU->Preds) {
24009467b48Spatrick SUnit *PredSU = PredDep.getSUnit();
24109467b48Spatrick if (PredSU->isHeightCurrent)
24209467b48Spatrick WorkList.push_back(PredSU);
24309467b48Spatrick }
24409467b48Spatrick } while (!WorkList.empty());
24509467b48Spatrick }
24609467b48Spatrick
setDepthToAtLeast(unsigned NewDepth)24709467b48Spatrick void SUnit::setDepthToAtLeast(unsigned NewDepth) {
24809467b48Spatrick if (NewDepth <= getDepth())
24909467b48Spatrick return;
25009467b48Spatrick setDepthDirty();
25109467b48Spatrick Depth = NewDepth;
25209467b48Spatrick isDepthCurrent = true;
25309467b48Spatrick }
25409467b48Spatrick
setHeightToAtLeast(unsigned NewHeight)25509467b48Spatrick void SUnit::setHeightToAtLeast(unsigned NewHeight) {
25609467b48Spatrick if (NewHeight <= getHeight())
25709467b48Spatrick return;
25809467b48Spatrick setHeightDirty();
25909467b48Spatrick Height = NewHeight;
26009467b48Spatrick isHeightCurrent = true;
26109467b48Spatrick }
26209467b48Spatrick
26309467b48Spatrick /// Calculates the maximal path from the node to the exit.
ComputeDepth()26409467b48Spatrick void SUnit::ComputeDepth() {
26509467b48Spatrick SmallVector<SUnit*, 8> WorkList;
26609467b48Spatrick WorkList.push_back(this);
26709467b48Spatrick do {
26809467b48Spatrick SUnit *Cur = WorkList.back();
26909467b48Spatrick
27009467b48Spatrick bool Done = true;
27109467b48Spatrick unsigned MaxPredDepth = 0;
27209467b48Spatrick for (const SDep &PredDep : Cur->Preds) {
27309467b48Spatrick SUnit *PredSU = PredDep.getSUnit();
27409467b48Spatrick if (PredSU->isDepthCurrent)
27509467b48Spatrick MaxPredDepth = std::max(MaxPredDepth,
27609467b48Spatrick PredSU->Depth + PredDep.getLatency());
27709467b48Spatrick else {
27809467b48Spatrick Done = false;
27909467b48Spatrick WorkList.push_back(PredSU);
28009467b48Spatrick }
28109467b48Spatrick }
28209467b48Spatrick
28309467b48Spatrick if (Done) {
28409467b48Spatrick WorkList.pop_back();
28509467b48Spatrick if (MaxPredDepth != Cur->Depth) {
28609467b48Spatrick Cur->setDepthDirty();
28709467b48Spatrick Cur->Depth = MaxPredDepth;
28809467b48Spatrick }
28909467b48Spatrick Cur->isDepthCurrent = true;
29009467b48Spatrick }
29109467b48Spatrick } while (!WorkList.empty());
29209467b48Spatrick }
29309467b48Spatrick
29409467b48Spatrick /// Calculates the maximal path from the node to the entry.
ComputeHeight()29509467b48Spatrick void SUnit::ComputeHeight() {
29609467b48Spatrick SmallVector<SUnit*, 8> WorkList;
29709467b48Spatrick WorkList.push_back(this);
29809467b48Spatrick do {
29909467b48Spatrick SUnit *Cur = WorkList.back();
30009467b48Spatrick
30109467b48Spatrick bool Done = true;
30209467b48Spatrick unsigned MaxSuccHeight = 0;
30309467b48Spatrick for (const SDep &SuccDep : Cur->Succs) {
30409467b48Spatrick SUnit *SuccSU = SuccDep.getSUnit();
30509467b48Spatrick if (SuccSU->isHeightCurrent)
30609467b48Spatrick MaxSuccHeight = std::max(MaxSuccHeight,
30709467b48Spatrick SuccSU->Height + SuccDep.getLatency());
30809467b48Spatrick else {
30909467b48Spatrick Done = false;
31009467b48Spatrick WorkList.push_back(SuccSU);
31109467b48Spatrick }
31209467b48Spatrick }
31309467b48Spatrick
31409467b48Spatrick if (Done) {
31509467b48Spatrick WorkList.pop_back();
31609467b48Spatrick if (MaxSuccHeight != Cur->Height) {
31709467b48Spatrick Cur->setHeightDirty();
31809467b48Spatrick Cur->Height = MaxSuccHeight;
31909467b48Spatrick }
32009467b48Spatrick Cur->isHeightCurrent = true;
32109467b48Spatrick }
32209467b48Spatrick } while (!WorkList.empty());
32309467b48Spatrick }
32409467b48Spatrick
biasCriticalPath()32509467b48Spatrick void SUnit::biasCriticalPath() {
32609467b48Spatrick if (NumPreds < 2)
32709467b48Spatrick return;
32809467b48Spatrick
32909467b48Spatrick SUnit::pred_iterator BestI = Preds.begin();
33009467b48Spatrick unsigned MaxDepth = BestI->getSUnit()->getDepth();
33109467b48Spatrick for (SUnit::pred_iterator I = std::next(BestI), E = Preds.end(); I != E;
33209467b48Spatrick ++I) {
33309467b48Spatrick if (I->getKind() == SDep::Data && I->getSUnit()->getDepth() > MaxDepth)
33409467b48Spatrick BestI = I;
33509467b48Spatrick }
33609467b48Spatrick if (BestI != Preds.begin())
33709467b48Spatrick std::swap(*Preds.begin(), *BestI);
33809467b48Spatrick }
33909467b48Spatrick
34009467b48Spatrick #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
dumpAttributes() const34109467b48Spatrick LLVM_DUMP_METHOD void SUnit::dumpAttributes() const {
34209467b48Spatrick dbgs() << " # preds left : " << NumPredsLeft << "\n";
34309467b48Spatrick dbgs() << " # succs left : " << NumSuccsLeft << "\n";
34409467b48Spatrick if (WeakPredsLeft)
34509467b48Spatrick dbgs() << " # weak preds left : " << WeakPredsLeft << "\n";
34609467b48Spatrick if (WeakSuccsLeft)
34709467b48Spatrick dbgs() << " # weak succs left : " << WeakSuccsLeft << "\n";
34809467b48Spatrick dbgs() << " # rdefs left : " << NumRegDefsLeft << "\n";
34909467b48Spatrick dbgs() << " Latency : " << Latency << "\n";
35009467b48Spatrick dbgs() << " Depth : " << getDepth() << "\n";
35109467b48Spatrick dbgs() << " Height : " << getHeight() << "\n";
35209467b48Spatrick }
35309467b48Spatrick
dumpNodeName(const SUnit & SU) const35409467b48Spatrick LLVM_DUMP_METHOD void ScheduleDAG::dumpNodeName(const SUnit &SU) const {
35509467b48Spatrick if (&SU == &EntrySU)
35609467b48Spatrick dbgs() << "EntrySU";
35709467b48Spatrick else if (&SU == &ExitSU)
35809467b48Spatrick dbgs() << "ExitSU";
35909467b48Spatrick else
36009467b48Spatrick dbgs() << "SU(" << SU.NodeNum << ")";
36109467b48Spatrick }
36209467b48Spatrick
dumpNodeAll(const SUnit & SU) const36309467b48Spatrick LLVM_DUMP_METHOD void ScheduleDAG::dumpNodeAll(const SUnit &SU) const {
36409467b48Spatrick dumpNode(SU);
36509467b48Spatrick SU.dumpAttributes();
36609467b48Spatrick if (SU.Preds.size() > 0) {
36709467b48Spatrick dbgs() << " Predecessors:\n";
36809467b48Spatrick for (const SDep &Dep : SU.Preds) {
36909467b48Spatrick dbgs() << " ";
37009467b48Spatrick dumpNodeName(*Dep.getSUnit());
37109467b48Spatrick dbgs() << ": ";
37209467b48Spatrick Dep.dump(TRI);
37309467b48Spatrick dbgs() << '\n';
37409467b48Spatrick }
37509467b48Spatrick }
37609467b48Spatrick if (SU.Succs.size() > 0) {
37709467b48Spatrick dbgs() << " Successors:\n";
37809467b48Spatrick for (const SDep &Dep : SU.Succs) {
37909467b48Spatrick dbgs() << " ";
38009467b48Spatrick dumpNodeName(*Dep.getSUnit());
38109467b48Spatrick dbgs() << ": ";
38209467b48Spatrick Dep.dump(TRI);
38309467b48Spatrick dbgs() << '\n';
38409467b48Spatrick }
38509467b48Spatrick }
38609467b48Spatrick }
38709467b48Spatrick #endif
38809467b48Spatrick
38909467b48Spatrick #ifndef NDEBUG
VerifyScheduledDAG(bool isBottomUp)39009467b48Spatrick unsigned ScheduleDAG::VerifyScheduledDAG(bool isBottomUp) {
39109467b48Spatrick bool AnyNotSched = false;
39209467b48Spatrick unsigned DeadNodes = 0;
39309467b48Spatrick for (const SUnit &SUnit : SUnits) {
39409467b48Spatrick if (!SUnit.isScheduled) {
39509467b48Spatrick if (SUnit.NumPreds == 0 && SUnit.NumSuccs == 0) {
39609467b48Spatrick ++DeadNodes;
39709467b48Spatrick continue;
39809467b48Spatrick }
39909467b48Spatrick if (!AnyNotSched)
40009467b48Spatrick dbgs() << "*** Scheduling failed! ***\n";
40109467b48Spatrick dumpNode(SUnit);
40209467b48Spatrick dbgs() << "has not been scheduled!\n";
40309467b48Spatrick AnyNotSched = true;
40409467b48Spatrick }
40509467b48Spatrick if (SUnit.isScheduled &&
40609467b48Spatrick (isBottomUp ? SUnit.getHeight() : SUnit.getDepth()) >
40709467b48Spatrick unsigned(std::numeric_limits<int>::max())) {
40809467b48Spatrick if (!AnyNotSched)
40909467b48Spatrick dbgs() << "*** Scheduling failed! ***\n";
41009467b48Spatrick dumpNode(SUnit);
41109467b48Spatrick dbgs() << "has an unexpected "
41209467b48Spatrick << (isBottomUp ? "Height" : "Depth") << " value!\n";
41309467b48Spatrick AnyNotSched = true;
41409467b48Spatrick }
41509467b48Spatrick if (isBottomUp) {
41609467b48Spatrick if (SUnit.NumSuccsLeft != 0) {
41709467b48Spatrick if (!AnyNotSched)
41809467b48Spatrick dbgs() << "*** Scheduling failed! ***\n";
41909467b48Spatrick dumpNode(SUnit);
42009467b48Spatrick dbgs() << "has successors left!\n";
42109467b48Spatrick AnyNotSched = true;
42209467b48Spatrick }
42309467b48Spatrick } else {
42409467b48Spatrick if (SUnit.NumPredsLeft != 0) {
42509467b48Spatrick if (!AnyNotSched)
42609467b48Spatrick dbgs() << "*** Scheduling failed! ***\n";
42709467b48Spatrick dumpNode(SUnit);
42809467b48Spatrick dbgs() << "has predecessors left!\n";
42909467b48Spatrick AnyNotSched = true;
43009467b48Spatrick }
43109467b48Spatrick }
43209467b48Spatrick }
43309467b48Spatrick assert(!AnyNotSched);
43409467b48Spatrick return SUnits.size() - DeadNodes;
43509467b48Spatrick }
43609467b48Spatrick #endif
43709467b48Spatrick
InitDAGTopologicalSorting()43809467b48Spatrick void ScheduleDAGTopologicalSort::InitDAGTopologicalSorting() {
43909467b48Spatrick // The idea of the algorithm is taken from
44009467b48Spatrick // "Online algorithms for managing the topological order of
44109467b48Spatrick // a directed acyclic graph" by David J. Pearce and Paul H.J. Kelly
44209467b48Spatrick // This is the MNR algorithm, which was first introduced by
44309467b48Spatrick // A. Marchetti-Spaccamela, U. Nanni and H. Rohnert in
44409467b48Spatrick // "Maintaining a topological order under edge insertions".
44509467b48Spatrick //
44609467b48Spatrick // Short description of the algorithm:
44709467b48Spatrick //
44809467b48Spatrick // Topological ordering, ord, of a DAG maps each node to a topological
44909467b48Spatrick // index so that for all edges X->Y it is the case that ord(X) < ord(Y).
45009467b48Spatrick //
45109467b48Spatrick // This means that if there is a path from the node X to the node Z,
45209467b48Spatrick // then ord(X) < ord(Z).
45309467b48Spatrick //
45409467b48Spatrick // This property can be used to check for reachability of nodes:
45509467b48Spatrick // if Z is reachable from X, then an insertion of the edge Z->X would
45609467b48Spatrick // create a cycle.
45709467b48Spatrick //
45809467b48Spatrick // The algorithm first computes a topological ordering for the DAG by
45909467b48Spatrick // initializing the Index2Node and Node2Index arrays and then tries to keep
46009467b48Spatrick // the ordering up-to-date after edge insertions by reordering the DAG.
46109467b48Spatrick //
46209467b48Spatrick // On insertion of the edge X->Y, the algorithm first marks by calling DFS
46309467b48Spatrick // the nodes reachable from Y, and then shifts them using Shift to lie
46409467b48Spatrick // immediately after X in Index2Node.
46509467b48Spatrick
46609467b48Spatrick // Cancel pending updates, mark as valid.
46709467b48Spatrick Dirty = false;
46809467b48Spatrick Updates.clear();
46909467b48Spatrick
47009467b48Spatrick unsigned DAGSize = SUnits.size();
47109467b48Spatrick std::vector<SUnit*> WorkList;
47209467b48Spatrick WorkList.reserve(DAGSize);
47309467b48Spatrick
47409467b48Spatrick Index2Node.resize(DAGSize);
47509467b48Spatrick Node2Index.resize(DAGSize);
47609467b48Spatrick
47709467b48Spatrick // Initialize the data structures.
47809467b48Spatrick if (ExitSU)
47909467b48Spatrick WorkList.push_back(ExitSU);
48009467b48Spatrick for (SUnit &SU : SUnits) {
48109467b48Spatrick int NodeNum = SU.NodeNum;
48209467b48Spatrick unsigned Degree = SU.Succs.size();
48309467b48Spatrick // Temporarily use the Node2Index array as scratch space for degree counts.
48409467b48Spatrick Node2Index[NodeNum] = Degree;
48509467b48Spatrick
48609467b48Spatrick // Is it a node without dependencies?
48709467b48Spatrick if (Degree == 0) {
48809467b48Spatrick assert(SU.Succs.empty() && "SUnit should have no successors");
48909467b48Spatrick // Collect leaf nodes.
49009467b48Spatrick WorkList.push_back(&SU);
49109467b48Spatrick }
49209467b48Spatrick }
49309467b48Spatrick
49409467b48Spatrick int Id = DAGSize;
49509467b48Spatrick while (!WorkList.empty()) {
49609467b48Spatrick SUnit *SU = WorkList.back();
49709467b48Spatrick WorkList.pop_back();
49809467b48Spatrick if (SU->NodeNum < DAGSize)
49909467b48Spatrick Allocate(SU->NodeNum, --Id);
50009467b48Spatrick for (const SDep &PredDep : SU->Preds) {
50109467b48Spatrick SUnit *SU = PredDep.getSUnit();
50209467b48Spatrick if (SU->NodeNum < DAGSize && !--Node2Index[SU->NodeNum])
50309467b48Spatrick // If all dependencies of the node are processed already,
50409467b48Spatrick // then the node can be computed now.
50509467b48Spatrick WorkList.push_back(SU);
50609467b48Spatrick }
50709467b48Spatrick }
50809467b48Spatrick
50909467b48Spatrick Visited.resize(DAGSize);
51009467b48Spatrick NumTopoInits++;
51109467b48Spatrick
51209467b48Spatrick #ifndef NDEBUG
51309467b48Spatrick // Check correctness of the ordering
51409467b48Spatrick for (SUnit &SU : SUnits) {
51509467b48Spatrick for (const SDep &PD : SU.Preds) {
51609467b48Spatrick assert(Node2Index[SU.NodeNum] > Node2Index[PD.getSUnit()->NodeNum] &&
51709467b48Spatrick "Wrong topological sorting");
51809467b48Spatrick }
51909467b48Spatrick }
52009467b48Spatrick #endif
52109467b48Spatrick }
52209467b48Spatrick
FixOrder()52309467b48Spatrick void ScheduleDAGTopologicalSort::FixOrder() {
52409467b48Spatrick // Recompute from scratch after new nodes have been added.
52509467b48Spatrick if (Dirty) {
52609467b48Spatrick InitDAGTopologicalSorting();
52709467b48Spatrick return;
52809467b48Spatrick }
52909467b48Spatrick
53009467b48Spatrick // Otherwise apply updates one-by-one.
53109467b48Spatrick for (auto &U : Updates)
53209467b48Spatrick AddPred(U.first, U.second);
53309467b48Spatrick Updates.clear();
53409467b48Spatrick }
53509467b48Spatrick
AddPredQueued(SUnit * Y,SUnit * X)53609467b48Spatrick void ScheduleDAGTopologicalSort::AddPredQueued(SUnit *Y, SUnit *X) {
53709467b48Spatrick // Recomputing the order from scratch is likely more efficient than applying
53809467b48Spatrick // updates one-by-one for too many updates. The current cut-off is arbitrarily
53909467b48Spatrick // chosen.
54009467b48Spatrick Dirty = Dirty || Updates.size() > 10;
54109467b48Spatrick
54209467b48Spatrick if (Dirty)
54309467b48Spatrick return;
54409467b48Spatrick
54509467b48Spatrick Updates.emplace_back(Y, X);
54609467b48Spatrick }
54709467b48Spatrick
AddPred(SUnit * Y,SUnit * X)54809467b48Spatrick void ScheduleDAGTopologicalSort::AddPred(SUnit *Y, SUnit *X) {
54909467b48Spatrick int UpperBound, LowerBound;
55009467b48Spatrick LowerBound = Node2Index[Y->NodeNum];
55109467b48Spatrick UpperBound = Node2Index[X->NodeNum];
55209467b48Spatrick bool HasLoop = false;
55309467b48Spatrick // Is Ord(X) < Ord(Y) ?
55409467b48Spatrick if (LowerBound < UpperBound) {
55509467b48Spatrick // Update the topological order.
55609467b48Spatrick Visited.reset();
55709467b48Spatrick DFS(Y, UpperBound, HasLoop);
55809467b48Spatrick assert(!HasLoop && "Inserted edge creates a loop!");
55909467b48Spatrick // Recompute topological indexes.
56009467b48Spatrick Shift(Visited, LowerBound, UpperBound);
56109467b48Spatrick }
56209467b48Spatrick
56309467b48Spatrick NumNewPredsAdded++;
56409467b48Spatrick }
56509467b48Spatrick
RemovePred(SUnit * M,SUnit * N)56609467b48Spatrick void ScheduleDAGTopologicalSort::RemovePred(SUnit *M, SUnit *N) {
56709467b48Spatrick // InitDAGTopologicalSorting();
56809467b48Spatrick }
56909467b48Spatrick
DFS(const SUnit * SU,int UpperBound,bool & HasLoop)57009467b48Spatrick void ScheduleDAGTopologicalSort::DFS(const SUnit *SU, int UpperBound,
57109467b48Spatrick bool &HasLoop) {
57209467b48Spatrick std::vector<const SUnit*> WorkList;
57309467b48Spatrick WorkList.reserve(SUnits.size());
57409467b48Spatrick
57509467b48Spatrick WorkList.push_back(SU);
57609467b48Spatrick do {
57709467b48Spatrick SU = WorkList.back();
57809467b48Spatrick WorkList.pop_back();
57909467b48Spatrick Visited.set(SU->NodeNum);
580*d415bd75Srobert for (const SDep &SuccDep : llvm::reverse(SU->Succs)) {
58109467b48Spatrick unsigned s = SuccDep.getSUnit()->NodeNum;
58209467b48Spatrick // Edges to non-SUnits are allowed but ignored (e.g. ExitSU).
58309467b48Spatrick if (s >= Node2Index.size())
58409467b48Spatrick continue;
58509467b48Spatrick if (Node2Index[s] == UpperBound) {
58609467b48Spatrick HasLoop = true;
58709467b48Spatrick return;
58809467b48Spatrick }
58909467b48Spatrick // Visit successors if not already and in affected region.
59009467b48Spatrick if (!Visited.test(s) && Node2Index[s] < UpperBound) {
59109467b48Spatrick WorkList.push_back(SuccDep.getSUnit());
59209467b48Spatrick }
59309467b48Spatrick }
59409467b48Spatrick } while (!WorkList.empty());
59509467b48Spatrick }
59609467b48Spatrick
GetSubGraph(const SUnit & StartSU,const SUnit & TargetSU,bool & Success)59709467b48Spatrick std::vector<int> ScheduleDAGTopologicalSort::GetSubGraph(const SUnit &StartSU,
59809467b48Spatrick const SUnit &TargetSU,
59909467b48Spatrick bool &Success) {
60009467b48Spatrick std::vector<const SUnit*> WorkList;
60109467b48Spatrick int LowerBound = Node2Index[StartSU.NodeNum];
60209467b48Spatrick int UpperBound = Node2Index[TargetSU.NodeNum];
60309467b48Spatrick bool Found = false;
60409467b48Spatrick BitVector VisitedBack;
60509467b48Spatrick std::vector<int> Nodes;
60609467b48Spatrick
60709467b48Spatrick if (LowerBound > UpperBound) {
60809467b48Spatrick Success = false;
60909467b48Spatrick return Nodes;
61009467b48Spatrick }
61109467b48Spatrick
61209467b48Spatrick WorkList.reserve(SUnits.size());
61309467b48Spatrick Visited.reset();
61409467b48Spatrick
61509467b48Spatrick // Starting from StartSU, visit all successors up
61609467b48Spatrick // to UpperBound.
61709467b48Spatrick WorkList.push_back(&StartSU);
61809467b48Spatrick do {
61909467b48Spatrick const SUnit *SU = WorkList.back();
62009467b48Spatrick WorkList.pop_back();
621*d415bd75Srobert for (const SDep &SD : llvm::reverse(SU->Succs)) {
622*d415bd75Srobert const SUnit *Succ = SD.getSUnit();
62309467b48Spatrick unsigned s = Succ->NodeNum;
62409467b48Spatrick // Edges to non-SUnits are allowed but ignored (e.g. ExitSU).
62509467b48Spatrick if (Succ->isBoundaryNode())
62609467b48Spatrick continue;
62709467b48Spatrick if (Node2Index[s] == UpperBound) {
62809467b48Spatrick Found = true;
62909467b48Spatrick continue;
63009467b48Spatrick }
63109467b48Spatrick // Visit successors if not already and in affected region.
63209467b48Spatrick if (!Visited.test(s) && Node2Index[s] < UpperBound) {
63309467b48Spatrick Visited.set(s);
63409467b48Spatrick WorkList.push_back(Succ);
63509467b48Spatrick }
63609467b48Spatrick }
63709467b48Spatrick } while (!WorkList.empty());
63809467b48Spatrick
63909467b48Spatrick if (!Found) {
64009467b48Spatrick Success = false;
64109467b48Spatrick return Nodes;
64209467b48Spatrick }
64309467b48Spatrick
64409467b48Spatrick WorkList.clear();
64509467b48Spatrick VisitedBack.resize(SUnits.size());
64609467b48Spatrick Found = false;
64709467b48Spatrick
64809467b48Spatrick // Starting from TargetSU, visit all predecessors up
64909467b48Spatrick // to LowerBound. SUs that are visited by the two
65009467b48Spatrick // passes are added to Nodes.
65109467b48Spatrick WorkList.push_back(&TargetSU);
65209467b48Spatrick do {
65309467b48Spatrick const SUnit *SU = WorkList.back();
65409467b48Spatrick WorkList.pop_back();
655*d415bd75Srobert for (const SDep &SD : llvm::reverse(SU->Preds)) {
656*d415bd75Srobert const SUnit *Pred = SD.getSUnit();
65709467b48Spatrick unsigned s = Pred->NodeNum;
65809467b48Spatrick // Edges to non-SUnits are allowed but ignored (e.g. EntrySU).
65909467b48Spatrick if (Pred->isBoundaryNode())
66009467b48Spatrick continue;
66109467b48Spatrick if (Node2Index[s] == LowerBound) {
66209467b48Spatrick Found = true;
66309467b48Spatrick continue;
66409467b48Spatrick }
66509467b48Spatrick if (!VisitedBack.test(s) && Visited.test(s)) {
66609467b48Spatrick VisitedBack.set(s);
66709467b48Spatrick WorkList.push_back(Pred);
66809467b48Spatrick Nodes.push_back(s);
66909467b48Spatrick }
67009467b48Spatrick }
67109467b48Spatrick } while (!WorkList.empty());
67209467b48Spatrick
67309467b48Spatrick assert(Found && "Error in SUnit Graph!");
67409467b48Spatrick Success = true;
67509467b48Spatrick return Nodes;
67609467b48Spatrick }
67709467b48Spatrick
Shift(BitVector & Visited,int LowerBound,int UpperBound)67809467b48Spatrick void ScheduleDAGTopologicalSort::Shift(BitVector& Visited, int LowerBound,
67909467b48Spatrick int UpperBound) {
68009467b48Spatrick std::vector<int> L;
68109467b48Spatrick int shift = 0;
68209467b48Spatrick int i;
68309467b48Spatrick
68409467b48Spatrick for (i = LowerBound; i <= UpperBound; ++i) {
68509467b48Spatrick // w is node at topological index i.
68609467b48Spatrick int w = Index2Node[i];
68709467b48Spatrick if (Visited.test(w)) {
68809467b48Spatrick // Unmark.
68909467b48Spatrick Visited.reset(w);
69009467b48Spatrick L.push_back(w);
69109467b48Spatrick shift = shift + 1;
69209467b48Spatrick } else {
69309467b48Spatrick Allocate(w, i - shift);
69409467b48Spatrick }
69509467b48Spatrick }
69609467b48Spatrick
69709467b48Spatrick for (unsigned LI : L) {
69809467b48Spatrick Allocate(LI, i - shift);
69909467b48Spatrick i = i + 1;
70009467b48Spatrick }
70109467b48Spatrick }
70209467b48Spatrick
WillCreateCycle(SUnit * TargetSU,SUnit * SU)70309467b48Spatrick bool ScheduleDAGTopologicalSort::WillCreateCycle(SUnit *TargetSU, SUnit *SU) {
70409467b48Spatrick FixOrder();
70509467b48Spatrick // Is SU reachable from TargetSU via successor edges?
70609467b48Spatrick if (IsReachable(SU, TargetSU))
70709467b48Spatrick return true;
70809467b48Spatrick for (const SDep &PredDep : TargetSU->Preds)
70909467b48Spatrick if (PredDep.isAssignedRegDep() &&
71009467b48Spatrick IsReachable(SU, PredDep.getSUnit()))
71109467b48Spatrick return true;
71209467b48Spatrick return false;
71309467b48Spatrick }
71409467b48Spatrick
AddSUnitWithoutPredecessors(const SUnit * SU)715097a140dSpatrick void ScheduleDAGTopologicalSort::AddSUnitWithoutPredecessors(const SUnit *SU) {
716097a140dSpatrick assert(SU->NodeNum == Index2Node.size() && "Node cannot be added at the end");
717097a140dSpatrick assert(SU->NumPreds == 0 && "Can only add SU's with no predecessors");
718097a140dSpatrick Node2Index.push_back(Index2Node.size());
719097a140dSpatrick Index2Node.push_back(SU->NodeNum);
720097a140dSpatrick Visited.resize(Node2Index.size());
721097a140dSpatrick }
722097a140dSpatrick
IsReachable(const SUnit * SU,const SUnit * TargetSU)72309467b48Spatrick bool ScheduleDAGTopologicalSort::IsReachable(const SUnit *SU,
72409467b48Spatrick const SUnit *TargetSU) {
72509467b48Spatrick FixOrder();
72609467b48Spatrick // If insertion of the edge SU->TargetSU would create a cycle
72709467b48Spatrick // then there is a path from TargetSU to SU.
72809467b48Spatrick int UpperBound, LowerBound;
72909467b48Spatrick LowerBound = Node2Index[TargetSU->NodeNum];
73009467b48Spatrick UpperBound = Node2Index[SU->NodeNum];
73109467b48Spatrick bool HasLoop = false;
73209467b48Spatrick // Is Ord(TargetSU) < Ord(SU) ?
73309467b48Spatrick if (LowerBound < UpperBound) {
73409467b48Spatrick Visited.reset();
73509467b48Spatrick // There may be a path from TargetSU to SU. Check for it.
73609467b48Spatrick DFS(TargetSU, UpperBound, HasLoop);
73709467b48Spatrick }
73809467b48Spatrick return HasLoop;
73909467b48Spatrick }
74009467b48Spatrick
Allocate(int n,int index)74109467b48Spatrick void ScheduleDAGTopologicalSort::Allocate(int n, int index) {
74209467b48Spatrick Node2Index[n] = index;
74309467b48Spatrick Index2Node[index] = n;
74409467b48Spatrick }
74509467b48Spatrick
74609467b48Spatrick ScheduleDAGTopologicalSort::
ScheduleDAGTopologicalSort(std::vector<SUnit> & sunits,SUnit * exitsu)74709467b48Spatrick ScheduleDAGTopologicalSort(std::vector<SUnit> &sunits, SUnit *exitsu)
74809467b48Spatrick : SUnits(sunits), ExitSU(exitsu) {}
74909467b48Spatrick
75009467b48Spatrick ScheduleHazardRecognizer::~ScheduleHazardRecognizer() = default;
751