Lines Matching defs:SU
687 for (const SUnit *SU : Queue)
688 dbgs() << SU->NodeNum << " ";
706 void ScheduleDAGMI::releaseSucc(SUnit *SU, SDep *SuccEdge) {
723 // SU->TopReadyCycle was set to CurrCycle when it was scheduled. However,
725 if (SuccSU->TopReadyCycle < SU->TopReadyCycle + SuccEdge->getLatency())
726 SuccSU->TopReadyCycle = SU->TopReadyCycle + SuccEdge->getLatency();
733 /// releaseSuccessors - Call releaseSucc on each of SU's successors.
734 void ScheduleDAGMI::releaseSuccessors(SUnit *SU) {
735 for (SDep &Succ : SU->Succs)
736 releaseSucc(SU, &Succ);
743 void ScheduleDAGMI::releasePred(SUnit *SU, SDep *PredEdge) {
760 // SU->BotReadyCycle was set to CurrCycle when it was scheduled. However,
762 if (PredSU->BotReadyCycle < SU->BotReadyCycle + PredEdge->getLatency())
763 PredSU->BotReadyCycle = SU->BotReadyCycle + PredEdge->getLatency();
770 /// releasePredecessors - Call releasePred on each of SU's predecessors.
771 void ScheduleDAGMI::releasePredecessors(SUnit *SU) {
772 for (SDep &Pred : SU->Preds)
773 releasePred(SU, &Pred);
861 SUnit *SU = SchedImpl->pickNode(IsTopNode);
862 if (!SU) break;
864 assert(!SU->isScheduled && "Node already scheduled");
868 MachineInstr *MI = SU->getInstr();
870 assert(SU->isTopReady() && "node still has unscheduled dependencies");
876 assert(SU->isBottomReady() && "node still has unscheduled dependencies");
892 SchedImpl->schedNode(SU, IsTopNode);
894 updateQueues(SU, IsTopNode);
917 for (SUnit &SU : SUnits) {
918 assert(!SU.isBoundaryNode() && "Boundary node should not be in SUnits");
921 SU.biasCriticalPath();
924 if (!SU.NumPredsLeft)
925 TopRoots.push_back(&SU);
927 if (!SU.NumSuccsLeft)
928 BotRoots.push_back(&SU);
943 for (SUnit *SU : TopRoots)
944 SchedImpl->releaseTopNode(SU);
964 void ScheduleDAGMI::updateQueues(SUnit *SU, bool IsTopNode) {
967 releaseSuccessors(SU);
969 releasePredecessors(SU);
971 SU->isScheduled = true;
1012 SUnit *SU = getSUnit(&MI);
1013 if (!SU)
1015 const MCSchedClassDesc *SC = getSchedClass(SU);
1019 if (SU->TopReadyCycle + PI->ReleaseAtCycle - 1 > LastCycle)
1020 LastCycle = SU->TopReadyCycle + PI->ReleaseAtCycle - 1;
1030 SUnit *SU = getSUnit(&MI);
1031 if (!SU) {
1035 std::string NodeName("SU(");
1036 NodeName += std::to_string(SU->NodeNum) + ")";
1040 if (C == SU->TopReadyCycle)
1046 const MCSchedClassDesc *SC = getSchedClass(SU);
1065 for (; C < SU->TopReadyCycle + PI.AcquireAtCycle; ++C) {
1094 SUnit *SU = getSUnit(&MI);
1095 if (!SU)
1097 const MCSchedClassDesc *SC = getSchedClass(SU);
1101 if ((int)SU->BotReadyCycle - PI->ReleaseAtCycle + 1 < LastCycle)
1102 LastCycle = (int)SU->BotReadyCycle - PI->ReleaseAtCycle + 1;
1112 SUnit *SU = getSUnit(&MI);
1113 if (!SU) {
1117 std::string NodeName("SU(");
1118 NodeName += std::to_string(SU->NodeNum) + ")";
1122 if (C == (int)SU->BotReadyCycle)
1128 const MCSchedClassDesc *SC = getSchedClass(SU);
1146 for (; C > ((int)SU->BotReadyCycle - (int)PI.AcquireAtCycle); --C) {
1176 if (SUnit *SU = getSUnit(&MI))
1177 dumpNode(*SU);
1193 void ScheduleDAGMILive::collectVRegUses(SUnit &SU) {
1194 const MachineInstr &MI = *SU.getInstr();
1223 if (UI->SU == &SU)
1227 VRegUses.insert(VReg2SUnit(Reg, LaneBitmask::getNone(), &SU));
1260 for (SUnit &SU : SUnits)
1261 collectVRegUses(SU);
1332 updateScheduledPressure(const SUnit *SU,
1334 const PressureDiff &PDiff = getPressureDiff(SU);
1377 SUnit &SU = *V2SU.SU;
1378 if (SU.isScheduled || &SU == &ExitSU)
1381 PressureDiff &PDiff = getPressureDiff(&SU);
1383 LLVM_DEBUG(dbgs() << " UpdateRegP: SU(" << SU.NodeNum << ") "
1385 << PrintLaneMask(P.LaneMask) << ' ' << *SU.getInstr();
1409 SUnit *SU = V2SU.SU;
1412 if (!SU->isScheduled && SU != &ExitSU) {
1414 LI.Query(LIS->getInstructionIndex(*SU->getInstr()));
1416 PressureDiff &PDiff = getPressureDiff(SU);
1418 LLVM_DEBUG(dbgs() << " UpdateRegP: SU(" << SU->NodeNum << ") "
1419 << *SU->getInstr();
1432 for (const SUnit &SU : SUnits) {
1433 dumpNodeAll(SU);
1436 getPressureDiff(&SU).dump(*TRI);
1439 if (SchedModel.mustBeginGroup(SU.getInstr()) &&
1440 SchedModel.mustEndGroup(SU.getInstr()))
1485 SUnit *SU = SchedImpl->pickNode(IsTopNode);
1486 if (!SU) break;
1488 assert(!SU->isScheduled && "Node already scheduled");
1492 scheduleMI(SU, IsTopNode);
1495 unsigned SubtreeID = DFSResult->getSubtreeID(SU);
1504 SchedImpl->schedNode(SU, IsTopNode);
1506 updateQueues(SU, IsTopNode);
1606 SUnit *SU = V2SU.SU;
1607 if (SU == &ExitSU)
1611 LiveQueryResult LRQ = LI.Query(LIS->getInstructionIndex(*SU->getInstr()));
1619 if (LiveOutDepth > SU->getDepth())
1620 CyclicLatency = LiveOutDepth - SU->getDepth();
1622 unsigned LiveInHeight = SU->getHeight() + DefSU->Latency;
1629 LLVM_DEBUG(dbgs() << "Cyclic Path: SU(" << DefSU->NodeNum << ") -> SU("
1630 << SU->NodeNum << ") = " << CyclicLatency << "c\n");
1651 void ScheduleDAGMILive::scheduleMI(SUnit *SU, bool IsTopNode) {
1653 MachineInstr *MI = SU->getInstr();
1656 assert(SU->isTopReady() && "node still has unscheduled dependencies");
1683 updateScheduledPressure(SU, TopRPTracker.getPressure().MaxSetPressure);
1686 assert(SU->isBottomReady() && "node still has unscheduled dependencies");
1721 updateScheduledPressure(SU, BotRPTracker.getPressure().MaxSetPressure);
1737 SUnit *SU;
1743 MemOpInfo(SUnit *SU, ArrayRef<const MachineOperand *> BaseOps,
1745 : SU(SU), BaseOps(BaseOps.begin(), BaseOps.end()), Offset(Offset),
1779 return SU->NodeNum < RHS.SU->NodeNum;
1867 if (!SUnit2ClusterInfo.count(MemOpRecords[NextIdx].SU->NodeNum) &&
1869 (!DAG->IsReachable(MemOpRecords[NextIdx].SU, MemOpa.SU) &&
1870 !DAG->IsReachable(MemOpa.SU, MemOpRecords[NextIdx].SU))))
1879 if (SUnit2ClusterInfo.count(MemOpa.SU->NodeNum)) {
1880 ClusterLength = SUnit2ClusterInfo[MemOpa.SU->NodeNum].first + 1;
1881 CurrentClusterBytes = SUnit2ClusterInfo[MemOpa.SU->NodeNum].second +
1891 SUnit *SUa = MemOpa.SU;
1892 SUnit *SUb = MemOpb.SU;
1900 LLVM_DEBUG(dbgs() << "Cluster ld/st SU(" << SUa->NodeNum << ") - SU("
1912 LLVM_DEBUG(dbgs() << " Copy Succ SU(" << Succ.getSUnit()->NodeNum
1926 LLVM_DEBUG(dbgs() << " Copy Pred SU(" << Pred.getSUnit()->NodeNum
1932 SUnit2ClusterInfo[MemOpb.SU->NodeNum] = {ClusterLength,
1943 for (auto &SU : SUnits) {
1944 if ((IsLoad && !SU.getInstr()->mayLoad()) ||
1945 (!IsLoad && !SU.getInstr()->mayStore()))
1948 const MachineInstr &MI = *SU.getInstr();
1956 MemOpInfo(&SU, BaseOps, Offset, OffsetIsScalable, Width));
1979 for (const SDep &Pred : MemOp.SU->Preds) {
2189 LLVM_DEBUG(dbgs() << "Constraining copy SU(" << CopySU->NodeNum << ")\n");
2192 LLVM_DEBUG(dbgs() << " Local use SU(" << LU->NodeNum << ") -> SU("
2197 LLVM_DEBUG(dbgs() << " Global use SU(" << GU->NodeNum << ") -> SU("
2216 for (SUnit &SU : DAG->SUnits) {
2217 if (!SU.getInstr()->isCopy())
2220 constrainLocalCopy(&SU, static_cast<ScheduleDAGMILive*>(DAG));
2287 for (SUnit &SU : DAG->SUnits) {
2288 const MCSchedClassDesc *SC = DAG->getSchedClass(&SU);
2289 RemIssueCount += SchedModel->getNumMicroOps(SU.getInstr(), SC)
2338 unsigned SchedBoundary::getLatencyStallCycles(SUnit *SU) {
2339 if (!SU->isUnbuffered)
2342 unsigned ReadyCycle = (isTop() ? SU->TopReadyCycle : SU->BotReadyCycle);
2444 /// Does this SU have a hazard within the current instruction group.
2456 /// TODO: Also check whether the SU must start a new group.
2457 bool SchedBoundary::checkHazard(SUnit *SU) {
2459 && HazardRec->getHazardType(SU) != ScheduleHazardRecognizer::NoHazard) {
2463 unsigned uops = SchedModel->getNumMicroOps(SU->getInstr());
2465 LLVM_DEBUG(dbgs() << " SU(" << SU->NodeNum << ") uops="
2466 << SchedModel->getNumMicroOps(SU->getInstr()) << '\n');
2471 ((isTop() && SchedModel->mustBeginGroup(SU->getInstr())) ||
2472 (!isTop() && SchedModel->mustEndGroup(SU->getInstr())))) {
2473 LLVM_DEBUG(dbgs() << " hazard: SU(" << SU->NodeNum << ") must "
2478 if (SchedModel->hasInstrSchedModel() && SU->hasReservedResource) {
2479 const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
2493 LLVM_DEBUG(dbgs() << " SU(" << SU->NodeNum << ") "
2509 for (SUnit *SU : ReadySUs) {
2510 unsigned L = getUnscheduledLatency(SU);
2513 LateSU = SU;
2517 LLVM_DEBUG(dbgs() << Available.getName() << " RemLatency SU("
2553 void SchedBoundary::releaseNode(SUnit *SU, unsigned ReadyCycle, bool InPQueue,
2555 assert(SU->getInstr() && "Scheduled SUnit must have instr");
2572 checkHazard(SU) || (Available.size() >= ReadyListLimit);
2575 Available.push(SU);
2583 Pending.push(SU);
2678 void SchedBoundary::bumpNode(SUnit *SU) {
2681 if (!isTop() && SU->isCall) {
2686 HazardRec->EmitInstruction(SU);
2692 const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
2693 unsigned IncMOps = SchedModel->getNumMicroOps(SU->getInstr());
2698 unsigned ReadyCycle = (isTop() ? SU->TopReadyCycle : SU->BotReadyCycle);
2717 if (SU->isUnbuffered && ReadyCycle > NextCycle)
2752 if (SU->hasReservedResource) {
2796 if (SU->getDepth() > TopLatency) {
2797 TopLatency = SU->getDepth();
2798 LLVM_DEBUG(dbgs() << " " << Available.getName() << " TopLatency SU("
2799 << SU->NodeNum << ") " << TopLatency << "c\n");
2801 if (SU->getHeight() > BotLatency) {
2802 BotLatency = SU->getHeight();
2803 LLVM_DEBUG(dbgs() << " " << Available.getName() << " BotLatency SU("
2804 << SU->NodeNum << ") " << BotLatency << "c\n");
2826 if ((isTop() && SchedModel->mustEndGroup(SU->getInstr())) ||
2827 (!isTop() && SchedModel->mustBeginGroup(SU->getInstr()))) {
2851 SUnit *SU = *(Pending.begin() + I);
2852 unsigned ReadyCycle = isTop() ? SU->TopReadyCycle : SU->BotReadyCycle;
2860 releaseNode(SU, ReadyCycle, true, I);
2869 /// Remove SU from the ready set for this boundary.
2870 void SchedBoundary::removeReady(SUnit *SU) {
2871 if (Available.isInQueue(SU))
2872 Available.remove(Available.find(SU));
2874 assert(Pending.isInQueue(SU) && "bad ready count");
2875 Pending.remove(Pending.find(SU));
2978 const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
3139 Latency = Cand.SU->getDepth();
3142 Latency = Cand.SU->getHeight();
3145 Latency = Cand.SU->getHeight();
3148 Latency = Cand.SU->getDepth();
3151 dbgs() << " Cand SU(" << Cand.SU->NodeNum << ") " << getReasonStr(Cand.Reason);
3212 if (std::max(TryCand.SU->getDepth(), Cand.SU->getDepth()) >
3214 if (tryLess(TryCand.SU->getDepth(), Cand.SU->getDepth(),
3218 if (tryGreater(TryCand.SU->getHeight(), Cand.SU->getHeight(),
3225 if (std::max(TryCand.SU->getHeight(), Cand.SU->getHeight()) >
3227 if (tryLess(TryCand.SU->getHeight(), Cand.SU->getHeight(),
3231 if (tryGreater(TryCand.SU->getDepth(), Cand.SU->getDepth(),
3273 TopCand.SU = nullptr;
3274 BotCand.SU = nullptr;
3380 for (const SUnit *SU : Bot.Available) {
3381 if (SU->getDepth() > Rem.CriticalPath)
3382 Rem.CriticalPath = SU->getDepth();
3435 unsigned getWeakLeft(const SUnit *SU, bool isTop) {
3436 return (isTop) ? SU->WeakPredsLeft : SU->WeakSuccsLeft;
3446 int biasPhysReg(const SUnit *SU, bool isTop) {
3447 const MachineInstr *MI = SU->getInstr();
3458 bool AtBoundary = isTop ? !SU->NumSuccsLeft : !SU->NumPredsLeft;
3483 void GenericScheduler::initCandidate(SchedCandidate &Cand, SUnit *SU,
3487 Cand.SU = SU;
3492 Cand.SU->getInstr(),
3499 Cand.SU->getInstr(),
3500 &DAG->getPressureDiff(Cand.SU),
3506 Cand.SU->getInstr(),
3507 DAG->getPressureDiff(Cand.SU),
3515 << " Try SU(" << Cand.SU->NodeNum << ") "
3541 if (tryGreater(biasPhysReg(TryCand.SU, TryCand.AtTop),
3542 biasPhysReg(Cand.SU, Cand.AtTop), TryCand, Cand, PhysReg))
3574 if (tryLess(Zone->getLatencyStallCycles(TryCand.SU),
3575 Zone->getLatencyStallCycles(Cand.SU), TryCand, Cand, Stall))
3589 if (tryGreater(TryCand.SU == TryCandNextClusterSU,
3590 Cand.SU == CandNextClusterSU,
3596 if (tryLess(getWeakLeft(TryCand.SU, TryCand.AtTop),
3597 getWeakLeft(Cand.SU, Cand.AtTop),
3627 if ((Zone->isTop() && TryCand.SU->NodeNum < Cand.SU->NodeNum)
3628 || (!Zone->isTop() && TryCand.SU->NodeNum > Cand.SU->NodeNum)) {
3650 for (SUnit *SU : Q) {
3653 initCandidate(TryCand, SU, Zone.isTop(), RPTracker, TempTracker);
3670 if (SUnit *SU = Bot.pickOnlyChoice()) {
3673 return SU;
3675 if (SUnit *SU = Top.pickOnlyChoice()) {
3678 return SU;
3691 if (!BotCand.isValid() || BotCand.SU->isScheduled ||
3703 assert(TCand.SU == BotCand.SU &&
3711 if (!TopCand.isValid() || TopCand.SU->isScheduled ||
3723 assert(TCand.SU == TopCand.SU &&
3741 return Cand.SU;
3751 SUnit *SU;
3754 SU = Top.pickOnlyChoice();
3755 if (!SU) {
3761 SU = TopCand.SU;
3765 SU = Bot.pickOnlyChoice();
3766 if (!SU) {
3772 SU = BotCand.SU;
3776 SU = pickNodeBidirectional(IsTopNode);
3778 } while (SU->isScheduled);
3780 // If IsTopNode, then SU is in Top.Available and must be removed. Otherwise,
3781 // if isTopReady(), then SU is in either Top.Available or Top.Pending.
3782 // If !IsTopNode, then SU is in Bot.Available and must be removed. Otherwise,
3783 // if isBottomReady(), then SU is in either Bot.Available or Bot.Pending.
3786 // isBottomReady. That is, it didn't factor into the decision to choose SU
3790 // picking a node. So if SU isTopReady on a OnlyBottomUp pick, then it was
3791 // because we schduled everything but the top roots. Conversley, if SU
3795 if (SU->isTopReady())
3796 Top.removeReady(SU);
3797 if (SU->isBottomReady())
3798 Bot.removeReady(SU);
3800 LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "
3801 << *SU->getInstr());
3802 return SU;
3805 void GenericScheduler::reschedulePhysReg(SUnit *SU, bool isTop) {
3806 MachineBasicBlock::iterator InsertPos = SU->getInstr();
3809 SmallVectorImpl<SDep> &Deps = isTop ? SU->Preds : SU->Succs;
3836 void GenericScheduler::schedNode(SUnit *SU, bool IsTopNode) {
3838 SU->TopReadyCycle = std::max(SU->TopReadyCycle, Top.getCurrCycle());
3839 Top.bumpNode(SU);
3840 if (SU->hasPhysRegUses)
3841 reschedulePhysReg(SU, true);
3843 SU->BotReadyCycle = std::max(SU->BotReadyCycle, Bot.getCurrCycle());
3844 Bot.bumpNode(SU);
3845 if (SU->hasPhysRegDefs)
3846 reschedulePhysReg(SU, false);
3921 for (const SUnit *SU : Bot.Available) {
3922 if (SU->getDepth() > Rem.CriticalPath)
3923 Rem.CriticalPath = SU->getDepth();
3945 if (tryLess(Top.getLatencyStallCycles(TryCand.SU),
3946 Top.getLatencyStallCycles(Cand.SU), TryCand, Cand, Stall))
3950 if (tryGreater(TryCand.SU == DAG->getNextClusterSucc(),
3951 Cand.SU == DAG->getNextClusterSucc(),
3970 if (TryCand.SU->NodeNum < Cand.SU->NodeNum) {
3981 for (SUnit *SU : Q) {
3983 TryCand.SU = SU;
4000 if (SUnit *SU = Bot.pickOnlyChoice()) {
4003 return SU;
4005 if (SUnit *SU = Top.pickOnlyChoice()) {
4008 return SU;
4021 if (!BotCand.isValid() || BotCand.SU->isScheduled ||
4033 assert(TCand.SU == BotCand.SU &&
4041 if (!TopCand.isValid() || TopCand.SU->isScheduled ||
4053 assert(TCand.SU == TopCand.SU &&
4071 return Cand.SU;
4081 SUnit *SU;
4084 SU = Bot.pickOnlyChoice();
4085 if (SU) {
4096 SU = BotCand.SU;
4100 SU = Top.pickOnlyChoice();
4101 if (SU) {
4112 SU = TopCand.SU;
4116 SU = pickNodeBidirectional(IsTopNode);
4118 } while (SU->isScheduled);
4120 if (SU->isTopReady())
4121 Top.removeReady(SU);
4122 if (SU->isBottomReady())
4123 Bot.removeReady(SU);
4125 LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "
4126 << *SU->getInstr());
4127 return SU;
4132 void PostGenericScheduler::schedNode(SUnit *SU, bool IsTopNode) {
4134 SU->TopReadyCycle = std::max(SU->TopReadyCycle, Top.getCurrCycle());
4135 Top.bumpNode(SU);
4137 SU->BotReadyCycle = std::max(SU->BotReadyCycle, Bot.getCurrCycle());
4138 Bot.bumpNode(SU);
4224 SUnit *SU = ReadyQ.back();
4228 << "SU(" << SU->NodeNum << ") "
4229 << " ILP: " << DAG->getDFSResult()->getILP(SU)
4230 << " Tree: " << DAG->getDFSResult()->getSubtreeID(SU)
4233 DAG->getDFSResult()->getSubtreeID(SU))
4235 << "Scheduling " << *SU->getInstr());
4236 return SU;
4246 void schedNode(SUnit *SU, bool IsTopNode) override {
4252 void releaseBottomNode(SUnit *SU) override {
4253 ReadyQ.push_back(SU);
4319 SUnit *SU;
4323 SU = TopQ.top();
4325 } while (SU->isScheduled);
4330 SU = BottomQ.top();
4332 } while (SU->isScheduled);
4337 return SU;
4340 void schedNode(SUnit *SU, bool IsTopNode) override {}
4342 void releaseTopNode(SUnit *SU) override {
4343 TopQ.push(SU);
4345 void releaseBottomNode(SUnit *SU) override {
4346 BottomQ.push(SU);
4407 static std::string getNodeLabel(const SUnit *SU, const ScheduleDAG *G) {
4413 SS << "SU:" << SU->NodeNum;
4415 SS << " I:" << DFS->getNumInstrs(SU);
4419 static std::string getNodeDescription(const SUnit *SU, const ScheduleDAG *G) {
4420 return G->getGraphNodeLabel(SU);