Lines Matching defs:SU
210 /// IsReachable - Checks if SU is reachable from TargetSU.
211 bool IsReachable(const SUnit *SU, const SUnit *TargetSU) {
212 return Topo.IsReachable(SU, TargetSU);
215 /// WillCreateCycle - Returns true if adding an edge from SU to TargetSU will
217 bool WillCreateCycle(SUnit *SU, SUnit *TargetSU) {
218 return Topo.WillCreateCycle(SU, TargetSU);
221 /// AddPredQueued - Queues and update to add a predecessor edge to SUnit SU.
224 void AddPredQueued(SUnit *SU, const SDep &D) {
225 Topo.AddPredQueued(SU, D.getSUnit());
226 SU->addPred(D);
229 /// AddPred - adds a predecessor edge to SUnit SU.
232 void AddPred(SUnit *SU, const SDep &D) {
233 Topo.AddPred(SU, D.getSUnit());
234 SU->addPred(D);
237 /// RemovePred - removes a predecessor edge from SUnit SU.
240 void RemovePred(SUnit *SU, const SDep &D) {
241 Topo.RemovePred(SU, D.getSUnit());
242 SU->removePred(D);
246 bool isReady(SUnit *SU) {
248 AvailableQueue->isReady(SU);
251 void ReleasePred(SUnit *SU, const SDep *PredEdge);
252 void ReleasePredecessors(SUnit *SU);
255 void AdvancePastStalls(SUnit *SU);
256 void EmitNode(SUnit *SU);
400 void ScheduleDAGRRList::ReleasePred(SUnit *SU, const SDep *PredEdge) {
416 PredSU->setHeightToAtLeast(SU->getHeight() + PredEdge->getLatency());
540 /// Always update LiveRegDefs for a register dependence even if the current SU
556 void ScheduleDAGRRList::ReleasePredecessors(SUnit *SU) {
558 for (SDep &Pred : SU->Preds) {
559 ReleasePred(SU, &Pred);
566 assert((!RegDef || RegDef == SU || RegDef == Pred.getSUnit()) &&
571 LiveRegGens[Pred.getReg()] = SU;
581 for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode())
590 CallSeqEndForStart[Def] = SU;
594 LiveRegGens[CallResource] = SU;
653 void ScheduleDAGRRList::AdvancePastStalls(SUnit *SU) {
662 // if (!SU->getNode() || !SU->getNode()->isMachineOpcode()) return;
664 unsigned ReadyCycle = SU->getHeight();
675 if (SU->isCall)
683 HazardRec->getHazardType(SU, -Stalls);
695 void ScheduleDAGRRList::EmitNode(SUnit *SU) {
700 if (!SU->getNode())
703 switch (SU->getNode()->getOpcode()) {
705 assert(SU->getNode()->isMachineOpcode() &&
724 if (SU->isCall) {
730 HazardRec->EmitInstruction(SU);
733 static void resetVRegCycle(SUnit *SU);
738 void ScheduleDAGRRList::ScheduleNodeBottomUp(SUnit *SU) {
740 LLVM_DEBUG(dumpNode(*SU));
743 if (CurCycle < SU->getHeight())
744 LLVM_DEBUG(dbgs() << " Height [" << SU->getHeight()
752 SU->setHeightToAtLeast(CurCycle);
755 EmitNode(SU);
757 Sequence.push_back(SU);
759 AvailableQueue->scheduledNode(SU);
769 ReleasePredecessors(SU);
772 for (SDep &Succ : SU->Succs) {
773 // LiveRegDegs[Succ.getReg()] != SU when SU is a two-address node.
774 if (Succ.isAssignedRegDep() && LiveRegDefs[Succ.getReg()] == SU) {
785 if (LiveRegDefs[CallResource] == SU)
786 for (const SDNode *SUNode = SU->getNode(); SUNode;
798 resetVRegCycle(SU);
800 SU->isScheduled = true;
811 if (SU->getNode() && SU->getNode()->isMachineOpcode())
819 /// CapturePred - This does the opposite of ReleasePred. Since SU is being
837 void ScheduleDAGRRList::UnscheduleNodeBottomUp(SUnit *SU) {
838 LLVM_DEBUG(dbgs() << "*** Unscheduling [" << SU->getHeight() << "]: ");
839 LLVM_DEBUG(dumpNode(*SU));
841 for (SDep &Pred : SU->Preds) {
843 if (Pred.isAssignedRegDep() && SU == LiveRegGens[Pred.getReg()]){
857 for (const SDNode *SUNode = SU->getNode(); SUNode;
861 SUnit *SeqEnd = CallSeqEndForStart[SU];
866 LiveRegDefs[CallResource] = SU;
873 if (LiveRegGens[CallResource] == SU)
874 for (const SDNode *SUNode = SU->getNode(); SUNode;
888 for (auto &Succ : SU->Succs) {
895 LiveRegDefs[Reg] = SU;
902 for (auto &Succ2 : SU->Succs) {
910 if (SU->getHeight() < MinAvailableCycle)
911 MinAvailableCycle = SU->getHeight();
913 SU->setHeightDirty();
914 SU->isScheduled = false;
915 SU->isAvailable = true;
918 SU->isPending = true;
919 PendingQueue.push_back(SU);
922 AvailableQueue->push(SU);
924 AvailableQueue->unscheduledNode(SU);
940 SUnit *SU = *I;
941 for (; SU->getHeight() > HazardCycle; ++HazardCycle) {
944 EmitNode(SU);
950 void ScheduleDAGRRList::BacktrackBottomUp(SUnit *SU, SUnit *BtSU) {
963 assert(!SU->isSucc(OldSU) && "Something is wrong!");
972 static bool isOperandOf(const SUnit *SU, SDNode *N) {
973 for (const SDNode *SUNode = SU->getNode(); SUNode;
982 SUnit *ScheduleDAGRRList::TryUnfoldSU(SUnit *SU) {
983 SDNode *N = SU->getNode();
994 unsigned OldNumVals = SU->getNode()->getNumValues();
1004 // this would negate the benefit to unfolding so just return SU.
1006 return SU;
1022 // negates the benefit to unfolding so just return SU.
1024 return SU;
1045 LLVM_DEBUG(dbgs() << "Unfolding SU #" << SU->NodeNum << "\n");
1049 DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), i), SDValue(N, i));
1050 DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), OldNumVals - 1),
1053 // Record all the edges to and from the old SU, by category.
1059 for (SDep &Pred : SU->Preds) {
1067 for (SDep &Succ : SU->Succs) {
1076 RemovePred(SU, Pred);
1081 RemovePred(SU, Pred);
1086 RemovePred(SU, Pred);
1091 D.setSUnit(SU);
1102 D.setSUnit(SU);
1131 SUnit *ScheduleDAGRRList::CopyAndMoveSuccessors(SUnit *SU) {
1132 SDNode *N = SU->getNode();
1136 LLVM_DEBUG(dbgs() << "Considering duplicating the SU\n");
1137 LLVM_DEBUG(dumpNode(*SU));
1170 SUnit *UnfoldSU = TryUnfoldSU(SU);
1173 SU = UnfoldSU;
1174 N = SU->getNode();
1176 if (SU->NumSuccsLeft == 0)
1177 return SU;
1180 LLVM_DEBUG(dbgs() << " Duplicating SU #" << SU->NodeNum << "\n");
1181 NewSU = CreateClone(SU);
1184 for (SDep &Pred : SU->Preds)
1190 AddPredQueued(NewSU, SDep(SU, SDep::Artificial));
1195 for (SDep &Succ : SU->Succs) {
1203 D.setSUnit(SU);
1210 AvailableQueue->updateNode(SU);
1219 void ScheduleDAGRRList::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg,
1234 for (SDep &Succ : SU->Succs) {
1254 SDep FromDep(SU, SDep::Data, Reg);
1255 FromDep.setLatency(SU->Latency);
1261 AvailableQueue->updateNode(SU);
1295 static void CheckForLiveRegDef(SUnit *SU, unsigned Reg, SUnit **LiveRegDefs,
1306 if (LiveRegDefs[*AliasI] == SU) continue;
1321 static void CheckForLiveRegDefMasked(SUnit *SU, const uint32_t *RegMask,
1328 if (LiveRegDefs[i] == SU) continue;
1348 DelayForLiveRegsBottomUp(SUnit *SU, SmallVectorImpl<unsigned> &LRegs) {
1355 // If SU is the currently live definition of the same register that it uses,
1357 for (SDep &Pred : SU->Preds) {
1358 if (Pred.isAssignedRegDep() && LiveRegDefs[Pred.getReg()] != SU)
1363 for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) {
1383 CheckForLiveRegDef(SU, Reg, LiveRegDefs.get(), RegAdded, LRegs, TRI);
1395 CheckForLiveRegDef(SU, Reg, LiveRegDefs.get(), RegAdded, LRegs, TRI,
1418 CheckForLiveRegDefMasked(SU, RegMask,
1432 CheckForLiveRegDef(SU, Reg, LiveRegDefs.get(), RegAdded, LRegs, TRI);
1436 CheckForLiveRegDef(SU, Reg, LiveRegDefs.get(), RegAdded, LRegs, TRI);
1445 SUnit *SU = Interferences[i-1];
1446 LRegsMapT::iterator LRegsPos = LRegsMap.find(SU);
1452 SU->isPending = false;
1456 if (SU->isAvailable && !SU->NodeQueueId) {
1457 LLVM_DEBUG(dbgs() << " Repushing SU #" << SU->NodeNum << '\n');
1458 AvailableQueue->push(SU);
1481 dbgs() << " SU #" << CurSU->NodeNum << '\n');
1484 CurSU->isPending = true; // This SU is not in AvailableQueue right now.
1531 LLVM_DEBUG(dbgs() << "ARTIFICIAL edge from SU(" << BtSU->NodeNum
1532 << ") to SU(" << TrySU->NodeNum << ")\n");
1585 LLVM_DEBUG(dbgs() << " Adding an edge from SU #" << TrySU->NodeNum
1586 << " to SU #" << Copies.front()->NodeNum << "\n");
1591 LLVM_DEBUG(dbgs() << " Adding an edge from SU #" << NewDef->NodeNum
1592 << " to SU #" << TrySU->NodeNum << "\n");
1625 SUnit *SU = PickNodeToScheduleBottomUp();
1627 AdvancePastStalls(SU);
1629 ScheduleNodeBottomUp(SU);
1632 // Advance the cycle to free resources. Skip ahead to the next ready SU.
1652 bool isReady(SUnit* SU, unsigned CurCycle) const { return true; }
1710 bool isReady(SUnit *SU, unsigned CurCycle) const;
1727 bool isReady(SUnit *SU, unsigned CurCycle) const;
1789 void addNode(const SUnit *SU) override;
1791 void updateNode(const SUnit *SU) override;
1799 unsigned getNodePriority(const SUnit *SU) const;
1801 unsigned getNodeOrdering(const SUnit *SU) const {
1802 if (!SU->getNode()) return 0;
1804 return SU->getNode()->getIROrder();
1815 void remove(SUnit *SU) override {
1817 assert(SU->NodeQueueId != 0 && "Not in queue!");
1818 std::vector<SUnit *>::iterator I = llvm::find(Queue, SU);
1822 SU->NodeQueueId = 0;
1829 bool HighRegPressure(const SUnit *SU) const;
1831 bool MayReduceRegPressure(SUnit *SU) const;
1833 int RegPressureDiff(SUnit *SU, unsigned &LiveUses) const;
1835 void scheduledNode(SUnit *SU) override;
1837 void unscheduledNode(SUnit *SU) override;
1840 bool canClobber(const SUnit *SU, const SUnit *Op);
1916 SUnit *SU = popFromQueue(DumpQueue, DumpPicker, scheduleDAG);
1917 dbgs() << "Height " << SU->getHeight() << ": ";
1918 DAG->dumpNode(*SU);
1952 CalcNodeSethiUllmanNumber(const SUnit *SU, std::vector<unsigned> &SUNumbers) {
1953 if (SUNumbers[SU->NodeNum] != 0)
1954 return SUNumbers[SU->NodeNum];
1958 WorkState(const SUnit *SU) : SU(SU) {}
1959 const SUnit *SU;
1964 WorkList.push_back(SU);
1967 auto *TempSU = Temp.SU;
1978 assert(It.SU != PredSU && "Trying to push an element twice?");
2013 assert(SUNumbers[SU->NodeNum] > 0 && "SethiUllman should never be zero!");
2014 return SUNumbers[SU->NodeNum];
2022 for (const SUnit &SU : *SUnits)
2023 CalcNodeSethiUllmanNumber(&SU, SethiUllmanNumbers);
2026 void RegReductionPQBase::addNode(const SUnit *SU) {
2030 CalcNodeSethiUllmanNumber(SU, SethiUllmanNumbers);
2033 void RegReductionPQBase::updateNode(const SUnit *SU) {
2034 SethiUllmanNumbers[SU->NodeNum] = 0;
2035 CalcNodeSethiUllmanNumber(SU, SethiUllmanNumbers);
2040 unsigned RegReductionPQBase::getNodePriority(const SUnit *SU) const {
2041 assert(SU->NodeNum < SethiUllmanNumbers.size());
2042 unsigned Opc = SU->getNode() ? SU->getNode()->getOpcode() : 0;
2053 if (SU->NumSuccs == 0 && SU->NumPreds != 0)
2054 // If SU does not have a register use, i.e. it doesn't produce a value
2060 if (SU->NumPreds == 0 && SU->NumSuccs != 0)
2061 // If SU does not have a register def, schedule it close to its uses
2065 return SethiUllmanNumbers[SU->NodeNum];
2067 unsigned Priority = SethiUllmanNumbers[SU->NodeNum];
2068 if (SU->isCallOp) {
2070 int NP = (int)Priority - SU->getNode()->getNumValues();
2093 bool RegReductionPQBase::HighRegPressure(const SUnit *SU) const {
2097 for (const SDep &Pred : SU->Preds) {
2118 bool RegReductionPQBase::MayReduceRegPressure(SUnit *SU) const {
2119 const SDNode *N = SU->getNode();
2121 if (!N->isMachineOpcode() || !SU->NumSuccs)
2143 int RegReductionPQBase::RegPressureDiff(SUnit *SU, unsigned &LiveUses) const {
2146 for (const SDep &Pred : SU->Preds) {
2165 const SDNode *N = SU->getNode();
2167 if (!N || !N->isMachineOpcode() || !SU->NumSuccs)
2182 void RegReductionPQBase::scheduledNode(SUnit *SU) {
2186 if (!SU->getNode())
2189 for (const SDep &Pred : SU->Preds) {
2209 // here with the reduction further below. Note that this SU may use multiple
2229 //assert(SU->NumRegDefsLeft == 0 && "not all regdefs have scheduled uses");
2230 int SkipRegDefs = (int)SU->NumRegDefsLeft;
2231 for (ScheduleDAGSDNodes::RegDefIter RegDefPos(SU, scheduleDAG);
2240 LLVM_DEBUG(dbgs() << " SU(" << SU->NodeNum
2251 void RegReductionPQBase::unscheduledNode(SUnit *SU) {
2255 const SDNode *N = SU->getNode();
2271 for (const SDep &Pred : SU->Preds) {
2324 if (SU->NumSuccs && N->isMachineOpcode()) {
2346 static unsigned closestSucc(const SUnit *SU) {
2348 for (const SDep &Succ : SU->Succs) {
2364 static unsigned calcMaxScratches(const SUnit *SU) {
2366 for (const SDep &Pred : SU->Preds) {
2373 /// hasOnlyLiveInOpers - Return true if SU has only value predecessors that are
2375 static bool hasOnlyLiveInOpers(const SUnit *SU) {
2377 for (const SDep &Pred : SU->Preds) {
2394 /// hasOnlyLiveOutUses - Return true if SU has only value successors that are
2395 /// CopyToReg to a virtual register. This SU def is probably a liveout and
2397 static bool hasOnlyLiveOutUses(const SUnit *SU) {
2399 for (const SDep &Succ : SU->Succs) {
2425 static void initVRegCycle(SUnit *SU) {
2429 if (!hasOnlyLiveInOpers(SU) || !hasOnlyLiveOutUses(SU))
2432 LLVM_DEBUG(dbgs() << "VRegCycle: SU(" << SU->NodeNum << ")\n");
2434 SU->isVRegCycle = true;
2436 for (const SDep &Pred : SU->Preds) {
2444 static void resetVRegCycle(SUnit *SU) {
2445 if (!SU->isVRegCycle)
2448 for (const SDep &Pred : SU->Preds) {
2461 static bool hasVRegCycleUse(const SUnit *SU) {
2462 // If this SU also defines the VReg, don't hoist it as a "use".
2463 if (SU->isVRegCycle)
2466 for (const SDep &Pred : SU->Preds) {
2470 LLVM_DEBUG(dbgs() << " VReg cycle use: SU (" << SU->NodeNum << ")\n");
2479 // Note: The ScheduleHazardRecognizer interface requires a non-const SU.
2480 static bool BUHasStall(SUnit *SU, int Height, RegReductionPQBase *SPQ) {
2482 if (SPQ->getHazardRec()->getHazardType(SU, 0)
2530 LLVM_DEBUG(dbgs() << " Comparing latency of SU (" << left->NodeNum
2531 << ") depth " << LDepth << " vs SU (" << right->NodeNum
2554 LLVM_DEBUG(dbgs() << " SU (" << left->NodeNum << ") "
2555 << PhysRegMsg[LHasPhysReg] << " SU(" << right->NodeNum
2672 bool hybrid_ls_rr_sort::isReady(SUnit *SU, unsigned CurCycle) const {
2675 if (SPQ->MayReduceRegPressure(SU)) return true;
2677 if (SU->getHeight() > (CurCycle + ReadyDelay)) return false;
2679 if (SPQ->getHazardRec()->getHazardType(SU, -ReadyDelay)
2700 LLVM_DEBUG(dbgs() << " pressure SU(" << left->NodeNum << ") > SU("
2705 LLVM_DEBUG(dbgs() << " pressure SU(" << right->NodeNum << ") > SU("
2719 bool ilp_ls_rr_sort::isReady(SUnit *SU, unsigned CurCycle) const {
2720 if (SU->getHeight() > CurCycle) return false;
2722 if (SPQ->getHazardRec()->getHazardType(SU, 0)
2729 static bool canEnableCoalescing(SUnit *SU) {
2730 unsigned Opc = SU->getNode() ? SU->getNode()->getOpcode() : 0;
2743 if (SU->NumPreds == 0 && SU->NumSuccs != 0)
2744 // If SU does not have a register def, schedule it close to its uses
2768 LLVM_DEBUG(dbgs() << "RegPressureDiff SU(" << left->NodeNum
2769 << "): " << LPDiff << " != SU(" << right->NodeNum
2782 LLVM_DEBUG(dbgs() << "Live uses SU(" << left->NodeNum << "): " << LLiveUses
2783 << " != SU(" << right->NodeNum << "): " << RLiveUses
2798 LLVM_DEBUG(dbgs() << "Depth of SU(" << left->NodeNum << "): "
2799 << left->getDepth() << " != SU(" << right->NodeNum
2827 for (SUnit &SU : sunits)
2828 initVRegCycle(&SU);
2835 bool RegReductionPQBase::canClobber(const SUnit *SU, const SUnit *Op) {
2836 if (SU->isTwoAddress) {
2837 unsigned Opc = SU->getNode()->getMachineOpcode();
2843 SDNode *DU = SU->getNode()->getOperand(i).getNode();
2853 /// canClobberReachingPhysRegUse - True if SU would clobber one of it's
2855 /// i.e. DepSU should not be scheduled above SU.
2856 static bool canClobberReachingPhysRegUse(const SUnit *DepSU, const SUnit *SU,
2861 TII->get(SU->getNode()->getMachineOpcode()).implicit_defs();
2862 const uint32_t *RegMask = getNodeRegMask(SU->getNode());
2866 for (const SDep &Succ : SU->Succs) {
2878 // Return true if SU clobbers this physical register use and the
2890 /// canClobberPhysRegDefs - True if SU would clobber one of SuccSU's
2892 static bool canClobberPhysRegDefs(const SUnit *SuccSU, const SUnit *SU,
2899 for (const SDNode *SUNode = SU->getNode(); SUNode;
2958 for (SUnit &SU : *SUnits) {
2962 if (SU.NumSuccs != 0)
2965 if (SU.NumPreds != 1)
2969 if (SDNode *N = SU.getNode())
2975 for (const SDep &Pred : SU.Preds)
2999 for (const SDep &Pred : SU.Preds)
3010 // Short-circuit the case where SU is PredSU's only data successor.
3015 if (SDNode *N = SU.getNode())
3023 if (PredSuccSU == &SU) continue;
3029 if (SU.hasPhysRegClobbers && PredSuccSU->hasPhysRegDefs)
3030 if (canClobberPhysRegDefs(PredSuccSU, &SU, TII, TRI))
3033 if (scheduleDAG->IsReachable(&SU, PredSuccSU))
3040 dbgs() << " Prescheduling SU #" << SU.NodeNum << " next to PredSU #"
3047 if (SuccSU != &SU) {
3050 scheduleDAG->AddPredQueued(&SU, Edge);
3051 Edge.setSUnit(&SU);
3068 for (SUnit &SU : *SUnits) {
3069 if (!SU.isTwoAddress)
3072 SDNode *Node = SU.getNode();
3073 if (!Node || !Node->isMachineOpcode() || SU.getNode()->getGluedNode())
3076 bool isLiveOut = hasOnlyLiveOutUses(&SU);
3084 SDNode *DU = SU.getNode()->getOperand(j).getNode();
3094 if (SuccSU == &SU)
3098 if (SuccSU->getHeight() < SU.getHeight() &&
3099 (SU.getHeight() - SuccSU->getHeight()) > 1)
3115 if (SuccSU->hasPhysRegDefs && SU.hasPhysRegClobbers) {
3116 if (canClobberPhysRegDefs(SuccSU, &SU, TII, TRI))
3126 if (!canClobberReachingPhysRegUse(SuccSU, &SU, scheduleDAG, TII, TRI) &&
3129 (!SU.isCommutable && SuccSU->isCommutable)) &&
3130 !scheduleDAG->IsReachable(SuccSU, &SU)) {
3132 << " Adding a pseudo-two-addr edge from SU #"
3133 << SU.NodeNum << " to SU #" << SuccSU->NodeNum << "\n");
3134 scheduleDAG->AddPredQueued(&SU, SDep(SuccSU, SDep::Artificial));