Lines Matching defs:SU
672 for (const SUnit *SU : Queue)
673 dbgs() << SU->NodeNum << " ";
691 void ScheduleDAGMI::releaseSucc(SUnit *SU, SDep *SuccEdge) {
708 // SU->TopReadyCycle was set to CurrCycle when it was scheduled. However,
710 if (SuccSU->TopReadyCycle < SU->TopReadyCycle + SuccEdge->getLatency())
711 SuccSU->TopReadyCycle = SU->TopReadyCycle + SuccEdge->getLatency();
718 /// releaseSuccessors - Call releaseSucc on each of SU's successors.
719 void ScheduleDAGMI::releaseSuccessors(SUnit *SU) {
720 for (SDep &Succ : SU->Succs)
721 releaseSucc(SU, &Succ);
728 void ScheduleDAGMI::releasePred(SUnit *SU, SDep *PredEdge) {
745 // SU->BotReadyCycle was set to CurrCycle when it was scheduled. However,
747 if (PredSU->BotReadyCycle < SU->BotReadyCycle + PredEdge->getLatency())
748 PredSU->BotReadyCycle = SU->BotReadyCycle + PredEdge->getLatency();
755 /// releasePredecessors - Call releasePred on each of SU's predecessors.
756 void ScheduleDAGMI::releasePredecessors(SUnit *SU) {
757 for (SDep &Pred : SU->Preds)
758 releasePred(SU, &Pred);
856 SUnit *SU = SchedImpl->pickNode(IsTopNode);
857 if (!SU) break;
859 assert(!SU->isScheduled && "Node already scheduled");
863 MachineInstr *MI = SU->getInstr();
865 assert(SU->isTopReady() && "node still has unscheduled dependencies");
871 assert(SU->isBottomReady() && "node still has unscheduled dependencies");
887 SchedImpl->schedNode(SU, IsTopNode);
889 updateQueues(SU, IsTopNode);
912 for (SUnit &SU : SUnits) {
913 assert(!SU.isBoundaryNode() && "Boundary node should not be in SUnits");
916 SU.biasCriticalPath();
919 if (!SU.NumPredsLeft)
920 TopRoots.push_back(&SU);
922 if (!SU.NumSuccsLeft)
923 BotRoots.push_back(&SU);
938 for (SUnit *SU : TopRoots)
939 SchedImpl->releaseTopNode(SU);
959 void ScheduleDAGMI::updateQueues(SUnit *SU, bool IsTopNode) {
962 releaseSuccessors(SU);
964 releasePredecessors(SU);
966 SU->isScheduled = true;
1007 SUnit *SU = getSUnit(&MI);
1008 if (!SU)
1010 const MCSchedClassDesc *SC = getSchedClass(SU);
1014 if (SU->TopReadyCycle + PI->ReleaseAtCycle - 1 > LastCycle)
1015 LastCycle = SU->TopReadyCycle + PI->ReleaseAtCycle - 1;
1025 SUnit *SU = getSUnit(&MI);
1026 if (!SU) {
1030 std::string NodeName("SU(");
1031 NodeName += std::to_string(SU->NodeNum) + ")";
1035 if (C == SU->TopReadyCycle)
1041 const MCSchedClassDesc *SC = getSchedClass(SU);
1060 for (; C < SU->TopReadyCycle + PI.AcquireAtCycle; ++C) {
1089 SUnit *SU = getSUnit(&MI);
1090 if (!SU)
1092 const MCSchedClassDesc *SC = getSchedClass(SU);
1096 if ((int)SU->BotReadyCycle - PI->ReleaseAtCycle + 1 < LastCycle)
1097 LastCycle = (int)SU->BotReadyCycle - PI->ReleaseAtCycle + 1;
1107 SUnit *SU = getSUnit(&MI);
1108 if (!SU) {
1112 std::string NodeName("SU(");
1113 NodeName += std::to_string(SU->NodeNum) + ")";
1117 if (C == (int)SU->BotReadyCycle)
1123 const MCSchedClassDesc *SC = getSchedClass(SU);
1141 for (; C > ((int)SU->BotReadyCycle - (int)PI.AcquireAtCycle); --C) {
1171 if (SUnit *SU = getSUnit(&MI))
1172 dumpNode(*SU);
1188 void ScheduleDAGMILive::collectVRegUses(SUnit &SU) {
1189 const MachineInstr &MI = *SU.getInstr();
1218 if (UI->SU == &SU)
1222 VRegUses.insert(VReg2SUnit(Reg, LaneBitmask::getNone(), &SU));
1255 for (SUnit &SU : SUnits)
1256 collectVRegUses(SU);
1327 updateScheduledPressure(const SUnit *SU,
1329 const PressureDiff &PDiff = getPressureDiff(SU);
1371 SUnit &SU = *V2SU.SU;
1372 if (SU.isScheduled || &SU == &ExitSU)
1375 PressureDiff &PDiff = getPressureDiff(&SU);
1377 LLVM_DEBUG(dbgs() << " UpdateRegP: SU(" << SU.NodeNum << ") "
1379 << PrintLaneMask(P.LaneMask) << ' ' << *SU.getInstr();
1403 SUnit *SU = V2SU.SU;
1406 if (!SU->isScheduled && SU != &ExitSU) {
1408 LI.Query(LIS->getInstructionIndex(*SU->getInstr()));
1410 PressureDiff &PDiff = getPressureDiff(SU);
1412 LLVM_DEBUG(dbgs() << " UpdateRegP: SU(" << SU->NodeNum << ") "
1413 << *SU->getInstr();
1426 for (const SUnit &SU : SUnits) {
1427 dumpNodeAll(SU);
1430 getPressureDiff(&SU).dump(*TRI);
1433 if (SchedModel.mustBeginGroup(SU.getInstr()) &&
1434 SchedModel.mustEndGroup(SU.getInstr()))
1479 SUnit *SU = SchedImpl->pickNode(IsTopNode);
1480 if (!SU) break;
1482 assert(!SU->isScheduled && "Node already scheduled");
1486 scheduleMI(SU, IsTopNode);
1489 unsigned SubtreeID = DFSResult->getSubtreeID(SU);
1498 SchedImpl->schedNode(SU, IsTopNode);
1500 updateQueues(SU, IsTopNode);
1600 SUnit *SU = V2SU.SU;
1601 if (SU == &ExitSU)
1605 LiveQueryResult LRQ = LI.Query(LIS->getInstructionIndex(*SU->getInstr()));
1613 if (LiveOutDepth > SU->getDepth())
1614 CyclicLatency = LiveOutDepth - SU->getDepth();
1616 unsigned LiveInHeight = SU->getHeight() + DefSU->Latency;
1623 LLVM_DEBUG(dbgs() << "Cyclic Path: SU(" << DefSU->NodeNum << ") -> SU("
1624 << SU->NodeNum << ") = " << CyclicLatency << "c\n");
1645 void ScheduleDAGMILive::scheduleMI(SUnit *SU, bool IsTopNode) {
1647 MachineInstr *MI = SU->getInstr();
1650 assert(SU->isTopReady() && "node still has unscheduled dependencies");
1677 updateScheduledPressure(SU, TopRPTracker.getPressure().MaxSetPressure);
1680 assert(SU->isBottomReady() && "node still has unscheduled dependencies");
1715 updateScheduledPressure(SU, BotRPTracker.getPressure().MaxSetPressure);
1731 SUnit *SU;
1737 MemOpInfo(SUnit *SU, ArrayRef<const MachineOperand *> BaseOps,
1739 : SU(SU), BaseOps(BaseOps), Offset(Offset), Width(Width),
1773 return SU->NodeNum < RHS.SU->NodeNum;
1861 if (!SUnit2ClusterInfo.count(MemOpRecords[NextIdx].SU->NodeNum) &&
1863 (!DAG->IsReachable(MemOpRecords[NextIdx].SU, MemOpa.SU) &&
1864 !DAG->IsReachable(MemOpa.SU, MemOpRecords[NextIdx].SU))))
1873 if (SUnit2ClusterInfo.count(MemOpa.SU->NodeNum)) {
1874 ClusterLength = SUnit2ClusterInfo[MemOpa.SU->NodeNum].first + 1;
1875 CurrentClusterBytes = SUnit2ClusterInfo[MemOpa.SU->NodeNum].second +
1885 SUnit *SUa = MemOpa.SU;
1886 SUnit *SUb = MemOpb.SU;
1894 LLVM_DEBUG(dbgs() << "Cluster ld/st SU(" << SUa->NodeNum << ") - SU("
1906 LLVM_DEBUG(dbgs() << " Copy Succ SU(" << Succ.getSUnit()->NodeNum
1920 LLVM_DEBUG(dbgs() << " Copy Pred SU(" << Pred.getSUnit()->NodeNum
1926 SUnit2ClusterInfo[MemOpb.SU->NodeNum] = {ClusterLength,
1937 for (auto &SU : SUnits) {
1938 if ((IsLoad && !SU.getInstr()->mayLoad()) ||
1939 (!IsLoad && !SU.getInstr()->mayStore()))
1942 const MachineInstr &MI = *SU.getInstr();
1953 MemOpInfo(&SU, BaseOps, Offset, OffsetIsScalable, Width));
1976 for (const SDep &Pred : MemOp.SU->Preds) {
2186 LLVM_DEBUG(dbgs() << "Constraining copy SU(" << CopySU->NodeNum << ")\n");
2189 LLVM_DEBUG(dbgs() << " Local use SU(" << LU->NodeNum << ") -> SU("
2194 LLVM_DEBUG(dbgs() << " Global use SU(" << GU->NodeNum << ") -> SU("
2213 for (SUnit &SU : DAG->SUnits) {
2214 if (!SU.getInstr()->isCopy())
2217 constrainLocalCopy(&SU, static_cast<ScheduleDAGMILive*>(DAG));
2284 for (SUnit &SU : DAG->SUnits) {
2285 const MCSchedClassDesc *SC = DAG->getSchedClass(&SU);
2286 RemIssueCount += SchedModel->getNumMicroOps(SU.getInstr(), SC)
2335 unsigned SchedBoundary::getLatencyStallCycles(SUnit *SU) {
2336 if (!SU->isUnbuffered)
2339 unsigned ReadyCycle = (isTop() ? SU->TopReadyCycle : SU->BotReadyCycle);
2441 /// Does this SU have a hazard within the current instruction group.
2453 /// TODO: Also check whether the SU must start a new group.
2454 bool SchedBoundary::checkHazard(SUnit *SU) {
2456 && HazardRec->getHazardType(SU) != ScheduleHazardRecognizer::NoHazard) {
2460 unsigned uops = SchedModel->getNumMicroOps(SU->getInstr());
2462 LLVM_DEBUG(dbgs() << " SU(" << SU->NodeNum << ") uops="
2463 << SchedModel->getNumMicroOps(SU->getInstr()) << '\n');
2468 ((isTop() && SchedModel->mustBeginGroup(SU->getInstr())) ||
2469 (!isTop() && SchedModel->mustEndGroup(SU->getInstr())))) {
2470 LLVM_DEBUG(dbgs() << " hazard: SU(" << SU->NodeNum << ") must "
2475 if (SchedModel->hasInstrSchedModel() && SU->hasReservedResource) {
2476 const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
2490 LLVM_DEBUG(dbgs() << " SU(" << SU->NodeNum << ") "
2506 for (SUnit *SU : ReadySUs) {
2507 unsigned L = getUnscheduledLatency(SU);
2510 LateSU = SU;
2514 LLVM_DEBUG(dbgs() << Available.getName() << " RemLatency SU("
2550 void SchedBoundary::releaseNode(SUnit *SU, unsigned ReadyCycle, bool InPQueue,
2552 assert(SU->getInstr() && "Scheduled SUnit must have instr");
2569 checkHazard(SU) || (Available.size() >= ReadyListLimit);
2572 Available.push(SU);
2580 Pending.push(SU);
2675 void SchedBoundary::bumpNode(SUnit *SU) {
2678 if (!isTop() && SU->isCall) {
2683 HazardRec->EmitInstruction(SU);
2689 const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
2690 unsigned IncMOps = SchedModel->getNumMicroOps(SU->getInstr());
2695 unsigned ReadyCycle = (isTop() ? SU->TopReadyCycle : SU->BotReadyCycle);
2714 if (SU->isUnbuffered && ReadyCycle > NextCycle)
2749 if (SU->hasReservedResource) {
2793 if (SU->getDepth() > TopLatency) {
2794 TopLatency = SU->getDepth();
2795 LLVM_DEBUG(dbgs() << " " << Available.getName() << " TopLatency SU("
2796 << SU->NodeNum << ") " << TopLatency << "c\n");
2798 if (SU->getHeight() > BotLatency) {
2799 BotLatency = SU->getHeight();
2800 LLVM_DEBUG(dbgs() << " " << Available.getName() << " BotLatency SU("
2801 << SU->NodeNum << ") " << BotLatency << "c\n");
2823 if ((isTop() && SchedModel->mustEndGroup(SU->getInstr())) ||
2824 (!isTop() && SchedModel->mustBeginGroup(SU->getInstr()))) {
2848 SUnit *SU = *(Pending.begin() + I);
2849 unsigned ReadyCycle = isTop() ? SU->TopReadyCycle : SU->BotReadyCycle;
2857 releaseNode(SU, ReadyCycle, true, I);
2866 /// Remove SU from the ready set for this boundary.
2867 void SchedBoundary::removeReady(SUnit *SU) {
2868 if (Available.isInQueue(SU))
2869 Available.remove(Available.find(SU));
2871 assert(Pending.isInQueue(SU) && "bad ready count");
2872 Pending.remove(Pending.find(SU));
2975 const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
3136 Latency = Cand.SU->getDepth();
3139 Latency = Cand.SU->getHeight();
3142 Latency = Cand.SU->getHeight();
3145 Latency = Cand.SU->getDepth();
3148 dbgs() << " Cand SU(" << Cand.SU->NodeNum << ") " << getReasonStr(Cand.Reason);
3209 if (std::max(TryCand.SU->getDepth(), Cand.SU->getDepth()) >
3211 if (tryLess(TryCand.SU->getDepth(), Cand.SU->getDepth(),
3215 if (tryGreater(TryCand.SU->getHeight(), Cand.SU->getHeight(),
3222 if (std::max(TryCand.SU->getHeight(), Cand.SU->getHeight()) >
3224 if (tryLess(TryCand.SU->getHeight(), Cand.SU->getHeight(),
3228 if (tryGreater(TryCand.SU->getDepth(), Cand.SU->getDepth(),
3270 TopCand.SU = nullptr;
3271 BotCand.SU = nullptr;
3373 for (const SUnit *SU : Bot.Available) {
3374 if (SU->getDepth() > Rem.CriticalPath)
3375 Rem.CriticalPath = SU->getDepth();
3428 unsigned getWeakLeft(const SUnit *SU, bool isTop) {
3429 return (isTop) ? SU->WeakPredsLeft : SU->WeakSuccsLeft;
3439 int biasPhysReg(const SUnit *SU, bool isTop) {
3440 const MachineInstr *MI = SU->getInstr();
3451 bool AtBoundary = isTop ? !SU->NumSuccsLeft : !SU->NumPredsLeft;
3476 void GenericScheduler::initCandidate(SchedCandidate &Cand, SUnit *SU,
3480 Cand.SU = SU;
3485 Cand.SU->getInstr(),
3492 Cand.SU->getInstr(),
3493 &DAG->getPressureDiff(Cand.SU),
3499 Cand.SU->getInstr(),
3500 DAG->getPressureDiff(Cand.SU),
3508 << " Try SU(" << Cand.SU->NodeNum << ") "
3534 if (tryGreater(biasPhysReg(TryCand.SU, TryCand.AtTop),
3535 biasPhysReg(Cand.SU, Cand.AtTop), TryCand, Cand, PhysReg))
3567 if (tryLess(Zone->getLatencyStallCycles(TryCand.SU),
3568 Zone->getLatencyStallCycles(Cand.SU), TryCand, Cand, Stall))
3582 if (tryGreater(TryCand.SU == TryCandNextClusterSU,
3583 Cand.SU == CandNextClusterSU,
3589 if (tryLess(getWeakLeft(TryCand.SU, TryCand.AtTop),
3590 getWeakLeft(Cand.SU, Cand.AtTop),
3620 if ((Zone->isTop() && TryCand.SU->NodeNum < Cand.SU->NodeNum)
3621 || (!Zone->isTop() && TryCand.SU->NodeNum > Cand.SU->NodeNum)) {
3643 for (SUnit *SU : Q) {
3646 initCandidate(TryCand, SU, Zone.isTop(), RPTracker, TempTracker);
3663 if (SUnit *SU = Bot.pickOnlyChoice()) {
3666 return SU;
3668 if (SUnit *SU = Top.pickOnlyChoice()) {
3671 return SU;
3684 if (!BotCand.isValid() || BotCand.SU->isScheduled ||
3696 assert(TCand.SU == BotCand.SU &&
3704 if (!TopCand.isValid() || TopCand.SU->isScheduled ||
3716 assert(TCand.SU == TopCand.SU &&
3734 return Cand.SU;
3744 SUnit *SU;
3747 SU = Top.pickOnlyChoice();
3748 if (!SU) {
3754 SU = TopCand.SU;
3758 SU = Bot.pickOnlyChoice();
3759 if (!SU) {
3765 SU = BotCand.SU;
3769 SU = pickNodeBidirectional(IsTopNode);
3771 } while (SU->isScheduled);
3773 // If IsTopNode, then SU is in Top.Available and must be removed. Otherwise,
3774 // if isTopReady(), then SU is in either Top.Available or Top.Pending.
3775 // If !IsTopNode, then SU is in Bot.Available and must be removed. Otherwise,
3776 // if isBottomReady(), then SU is in either Bot.Available or Bot.Pending.
3779 // isBottomReady. That is, it didn't factor into the decision to choose SU
3783 // picking a node. So if SU isTopReady on a OnlyBottomUp pick, then it was
3784 // because we schduled everything but the top roots. Conversley, if SU
3788 if (SU->isTopReady())
3789 Top.removeReady(SU);
3790 if (SU->isBottomReady())
3791 Bot.removeReady(SU);
3793 LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "
3794 << *SU->getInstr());
3795 return SU;
3798 void GenericScheduler::reschedulePhysReg(SUnit *SU, bool isTop) {
3799 MachineBasicBlock::iterator InsertPos = SU->getInstr();
3802 SmallVectorImpl<SDep> &Deps = isTop ? SU->Preds : SU->Succs;
3829 void GenericScheduler::schedNode(SUnit *SU, bool IsTopNode) {
3831 SU->TopReadyCycle = std::max(SU->TopReadyCycle, Top.getCurrCycle());
3832 Top.bumpNode(SU);
3833 if (SU->hasPhysRegUses)
3834 reschedulePhysReg(SU, true);
3836 SU->BotReadyCycle = std::max(SU->BotReadyCycle, Bot.getCurrCycle());
3837 Bot.bumpNode(SU);
3838 if (SU->hasPhysRegDefs)
3839 reschedulePhysReg(SU, false);
3925 for (const SUnit *SU : Bot.Available) {
3926 if (SU->getDepth() > Rem.CriticalPath)
3927 Rem.CriticalPath = SU->getDepth();
3949 if (tryLess(Top.getLatencyStallCycles(TryCand.SU),
3950 Top.getLatencyStallCycles(Cand.SU), TryCand, Cand, Stall))
3958 if (tryGreater(TryCand.SU == TryCandNextClusterSU,
3959 Cand.SU == CandNextClusterSU, TryCand, Cand, Cluster))
3981 if (TryCand.SU->NodeNum < Cand.SU->NodeNum) {
3992 for (SUnit *SU : Q) {
3994 TryCand.SU = SU;
4011 if (SUnit *SU = Bot.pickOnlyChoice()) {
4014 return SU;
4016 if (SUnit *SU = Top.pickOnlyChoice()) {
4019 return SU;
4032 if (!BotCand.isValid() || BotCand.SU->isScheduled ||
4044 assert(TCand.SU == BotCand.SU &&
4052 if (!TopCand.isValid() || TopCand.SU->isScheduled ||
4064 assert(TCand.SU == TopCand.SU &&
4082 return Cand.SU;
4092 SUnit *SU;
4095 SU = Bot.pickOnlyChoice();
4096 if (SU) {
4107 SU = BotCand.SU;
4111 SU = Top.pickOnlyChoice();
4112 if (SU) {
4123 SU = TopCand.SU;
4127 SU = pickNodeBidirectional(IsTopNode);
4129 } while (SU->isScheduled);
4131 if (SU->isTopReady())
4132 Top.removeReady(SU);
4133 if (SU->isBottomReady())
4134 Bot.removeReady(SU);
4136 LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "
4137 << *SU->getInstr());
4138 return SU;
4143 void PostGenericScheduler::schedNode(SUnit *SU, bool IsTopNode) {
4145 SU->TopReadyCycle = std::max(SU->TopReadyCycle, Top.getCurrCycle());
4146 Top.bumpNode(SU);
4148 SU->BotReadyCycle = std::max(SU->BotReadyCycle, Bot.getCurrCycle());
4149 Bot.bumpNode(SU);
4235 SUnit *SU = ReadyQ.back();
4239 << "SU(" << SU->NodeNum << ") "
4240 << " ILP: " << DAG->getDFSResult()->getILP(SU)
4241 << " Tree: " << DAG->getDFSResult()->getSubtreeID(SU)
4244 DAG->getDFSResult()->getSubtreeID(SU))
4246 << "Scheduling " << *SU->getInstr());
4247 return SU;
4257 void schedNode(SUnit *SU, bool IsTopNode) override {
4263 void releaseBottomNode(SUnit *SU) override {
4264 ReadyQ.push_back(SU);
4330 SUnit *SU;
4334 SU = TopQ.top();
4336 } while (SU->isScheduled);
4341 SU = BottomQ.top();
4343 } while (SU->isScheduled);
4348 return SU;
4351 void schedNode(SUnit *SU, bool IsTopNode) override {}
4353 void releaseTopNode(SUnit *SU) override {
4354 TopQ.push(SU);
4356 void releaseBottomNode(SUnit *SU) override {
4357 BottomQ.push(SU);
4417 static std::string getNodeLabel(const SUnit *SU, const ScheduleDAG *G) {
4423 SS << "SU:" << SU->NodeNum;
4425 SS << " I:" << DFS->getNumInstrs(SU);
4429 static std::string getNodeDescription(const SUnit *SU, const ScheduleDAG *G) {
4430 return G->getGraphNodeLabel(SU);