Index: include/llvm/CodeGen/MachineScheduler.h =================================================================== --- include/llvm/CodeGen/MachineScheduler.h +++ include/llvm/CodeGen/MachineScheduler.h @@ -578,15 +578,15 @@ ScheduleHazardRecognizer *HazardRec; + // For heuristics, keep a list of the nodes that immediately depend on the + // most recently scheduled node. + SmallPtrSet NextSUs; + private: /// True if the pending Q should be checked/updated before scheduling another /// instruction. bool CheckPending; - // For heuristics, keep a list of the nodes that immediately depend on the - // most recently scheduled node. - SmallPtrSet NextSUs; - /// Number of cycles it takes to issue the instructions scheduled in this /// zone. It is defined as: scheduled-micro-ops / issue-width + stalls. /// See getStalls(). @@ -731,6 +731,7 @@ unsigned countResource(unsigned PIdx, unsigned Cycles, unsigned ReadyCycle); void bumpNode(SUnit *SU); + void updateLatencyCounters(SUnit *SU); void releasePending(); @@ -843,6 +844,21 @@ void setPolicy(CandPolicy &Policy, bool IsPostRA, SchedBoundary &CurrZone, SchedBoundary *OtherZone); +protected: + + bool tryLess(int TryVal, int CandVal, SchedCandidate &TryCand, + SchedCandidate &Cand, CandReason Reason); + bool tryGreater(int TryVal, int CandVal, SchedCandidate &TryCand, + SchedCandidate &Cand, CandReason Reason); + bool tryLatency(SchedCandidate &TryCand, SchedCandidate &Cand, + SchedBoundary &Zone); + void tracePick(const GenericSchedulerBase::SchedCandidate &Cand, + bool IsTop); + int biasPhysRegCopy(const SUnit *SU, bool isTop); + bool tryPressure(const PressureChange &TryP, const PressureChange &CandP, + SchedCandidate &TryCand, SchedCandidate &Cand, + CandReason Reason, const TargetRegisterInfo *TRI, + const MachineFunction &MF); #ifndef NDEBUG void traceCandidate(const SchedCandidate &Cand); @@ -852,6 +868,7 @@ /// GenericScheduler shrinks the unscheduled zone using heuristics to balance /// the schedule. class GenericScheduler : public GenericSchedulerBase { +protected: ScheduleDAGMILive *DAG; // State of the top and bottom scheduled instruction boundaries. @@ -893,11 +910,15 @@ protected: void checkAcyclicLatency(); - void tryCandidate(SchedCandidate &Cand, - SchedCandidate &TryCand, - SchedBoundary &Zone, - const RegPressureTracker &RPTracker, - RegPressureTracker &TempTracker); + virtual void tryCandidate(SchedCandidate &Cand, + SchedCandidate &TryCand, + SchedBoundary &Zone, + const RegPressureTracker &RPTracker, + RegPressureTracker &TempTracker); + + void initTryCandRPDelta(SchedCandidate &TryCand, SchedBoundary &Zone, + const RegPressureTracker &RPTracker, + RegPressureTracker &TempTracker); SUnit *pickNodeBidirectional(bool &IsTopNode); Index: lib/CodeGen/MachineScheduler.cpp =================================================================== --- lib/CodeGen/MachineScheduler.cpp +++ lib/CodeGen/MachineScheduler.cpp @@ -2052,19 +2052,9 @@ } } } - // Update ExpectedLatency and DependentLatency. - unsigned &TopLatency = isTop() ? ExpectedLatency : DependentLatency; - unsigned &BotLatency = isTop() ? DependentLatency : ExpectedLatency; - if (SU->getDepth() > TopLatency) { - TopLatency = SU->getDepth(); - DEBUG(dbgs() << " " << Available.getName() - << " TopLatency SU(" << SU->NodeNum << ") " << TopLatency << "c\n"); - } - if (SU->getHeight() > BotLatency) { - BotLatency = SU->getHeight(); - DEBUG(dbgs() << " " << Available.getName() - << " BotLatency SU(" << SU->NodeNum << ") " << BotLatency << "c\n"); - } + + updateLatencyCounters(SU); + // If we stall for any reason, bump the cycle. if (NextCycle > CurrCycle) { bumpCycle(NextCycle); @@ -2090,6 +2080,22 @@ DEBUG(dumpScheduledState()); } +/// Update ExpectedLatency and DependentLatency. +void SchedBoundary::updateLatencyCounters(SUnit *SU) { + unsigned &TopLatency = isTop() ? ExpectedLatency : DependentLatency; + unsigned &BotLatency = isTop() ? DependentLatency : ExpectedLatency; + if (SU->getDepth() > TopLatency) { + TopLatency = SU->getDepth(); + DEBUG(dbgs() << " " << Available.getName() + << " TopLatency SU(" << SU->NodeNum << ") " << TopLatency << "c\n"); + } + if (SU->getHeight() > BotLatency) { + BotLatency = SU->getHeight(); + DEBUG(dbgs() << " " << Available.getName() + << " BotLatency SU(" << SU->NodeNum << ") " << BotLatency << "c\n"); + } +} + /// Release pending ready nodes in to the available queue. This makes them /// visible to heuristics. void SchedBoundary::releasePending() { @@ -2362,10 +2368,10 @@ #endif /// Return true if this heuristic determines order. -static bool tryLess(int TryVal, int CandVal, - GenericSchedulerBase::SchedCandidate &TryCand, - GenericSchedulerBase::SchedCandidate &Cand, - GenericSchedulerBase::CandReason Reason) { +bool GenericSchedulerBase::tryLess(int TryVal, int CandVal, + SchedCandidate &TryCand, + SchedCandidate &Cand, + CandReason Reason) { if (TryVal < CandVal) { TryCand.Reason = Reason; return true; @@ -2379,10 +2385,10 @@ return false; } -static bool tryGreater(int TryVal, int CandVal, - GenericSchedulerBase::SchedCandidate &TryCand, - GenericSchedulerBase::SchedCandidate &Cand, - GenericSchedulerBase::CandReason Reason) { +bool GenericSchedulerBase::tryGreater(int TryVal, int CandVal, + SchedCandidate &TryCand, + SchedCandidate &Cand, + CandReason Reason) { if (TryVal > CandVal) { TryCand.Reason = Reason; return true; @@ -2396,9 +2402,9 @@ return false; } -static bool tryLatency(GenericSchedulerBase::SchedCandidate &TryCand, - GenericSchedulerBase::SchedCandidate &Cand, - SchedBoundary &Zone) { +bool GenericSchedulerBase::tryLatency(SchedCandidate &TryCand, + SchedCandidate &Cand, + SchedBoundary &Zone) { if (Zone.isTop()) { if (Cand.SU->getDepth() > Zone.getScheduledLatency()) { if (tryLess(TryCand.SU->getDepth(), Cand.SU->getDepth(), @@ -2422,8 +2428,8 @@ return false; } -static void tracePick(const GenericSchedulerBase::SchedCandidate &Cand, - bool IsTop) { +void GenericSchedulerBase:: +tracePick(const GenericSchedulerBase::SchedCandidate &Cand, bool IsTop) { DEBUG(dbgs() << "Pick " << (IsTop ? "Top " : "Bot ") << GenericSchedulerBase::getReasonStr(Cand.Reason) << '\n'); } @@ -2569,13 +2575,13 @@ } } -static bool tryPressure(const PressureChange &TryP, - const PressureChange &CandP, - GenericSchedulerBase::SchedCandidate &TryCand, - GenericSchedulerBase::SchedCandidate &Cand, - GenericSchedulerBase::CandReason Reason, - const TargetRegisterInfo *TRI, - const MachineFunction &MF) { +bool GenericSchedulerBase::tryPressure(const PressureChange &TryP, + const PressureChange &CandP, + SchedCandidate &TryCand, + SchedCandidate &Cand, + CandReason Reason, + const TargetRegisterInfo *TRI, + const MachineFunction &MF) { unsigned TryPSet = TryP.getPSetOrMax(); unsigned CandPSet = CandP.getPSetOrMax(); // If both candidates affect the same set, go with the smallest increase. @@ -2613,7 +2619,7 @@ /// copies which can be prescheduled. The rest (e.g. x86 MUL) could be bundled /// with the operation that produces or consumes the physreg. We'll do this when /// regalloc has support for parallel copies. -static int biasPhysRegCopy(const SUnit *SU, bool isTop) { +int GenericSchedulerBase::biasPhysRegCopy(const SUnit *SU, bool isTop) { const MachineInstr *MI = SU->getInstr(); if (!MI->isCopy()) return 0; @@ -2634,23 +2640,10 @@ return 0; } -/// Apply a set of heursitics to a new candidate. Heuristics are currently -/// hierarchical. This may be more efficient than a graduated cost model because -/// we don't need to evaluate all aspects of the model for each node in the -/// queue. But it's really done to make the heuristics easier to debug and -/// statistically analyze. -/// -/// \param Cand provides the policy and current best candidate. -/// \param TryCand refers to the next SUnit candidate, otherwise uninitialized. -/// \param Zone describes the scheduled zone that we are extending. -/// \param RPTracker describes reg pressure within the scheduled zone. -/// \param TempTracker is a scratch pressure tracker to reuse in queries. -void GenericScheduler::tryCandidate(SchedCandidate &Cand, - SchedCandidate &TryCand, - SchedBoundary &Zone, - const RegPressureTracker &RPTracker, - RegPressureTracker &TempTracker) { - +void GenericScheduler::initTryCandRPDelta(SchedCandidate &TryCand, + SchedBoundary &Zone, + const RegPressureTracker &RPTracker, + RegPressureTracker &TempTracker) { if (DAG->isTrackingPressure()) { // Always initialize TryCand's RPDelta. if (Zone.isTop()) { @@ -2683,6 +2676,26 @@ dbgs() << " Try SU(" << TryCand.SU->NodeNum << ") " << TRI->getRegPressureSetName(TryCand.RPDelta.Excess.getPSet()) << ":" << TryCand.RPDelta.Excess.getUnitInc() << "\n"); +} + +/// Apply a set of heursitics to a new candidate. Heuristics are currently +/// hierarchical. This may be more efficient than a graduated cost model because +/// we don't need to evaluate all aspects of the model for each node in the +/// queue. But it's really done to make the heuristics easier to debug and +/// statistically analyze. +/// +/// \param Cand provides the policy and current best candidate. +/// \param TryCand refers to the next SUnit candidate, otherwise uninitialized. +/// \param Zone describes the scheduled zone that we are extending. +/// \param RPTracker describes reg pressure within the scheduled zone. +/// \param TempTracker is a scratch pressure tracker to reuse in queries. +void GenericScheduler::tryCandidate(SchedCandidate &Cand, + SchedCandidate &TryCand, + SchedBoundary &Zone, + const RegPressureTracker &RPTracker, + RegPressureTracker &TempTracker) { + + initTryCandRPDelta(TryCand, Zone, RPTracker, TempTracker); // Initialize the candidate if needed. if (!Cand.isValid()) {