Index: llvm/include/llvm/CodeGen/MachineScheduler.h =================================================================== --- llvm/include/llvm/CodeGen/MachineScheduler.h +++ llvm/include/llvm/CodeGen/MachineScheduler.h @@ -756,7 +756,8 @@ unsigned getOtherResourceCount(unsigned &OtherCritIdx); - void releaseNode(SUnit *SU, unsigned ReadyCycle); + template + void releaseNode(SUnit *SU, unsigned ReadyCycle, unsigned Idx = 0); void bumpCycle(unsigned NextCycle); @@ -954,7 +955,7 @@ if (SU->isScheduled) return; - Top.releaseNode(SU, SU->TopReadyCycle); + Top.releaseNode(SU, SU->TopReadyCycle); TopCand.SU = nullptr; } @@ -962,7 +963,7 @@ if (SU->isScheduled) return; - Bot.releaseNode(SU, SU->BotReadyCycle); + Bot.releaseNode(SU, SU->BotReadyCycle); BotCand.SU = nullptr; } @@ -1042,7 +1043,7 @@ void releaseTopNode(SUnit *SU) override { if (SU->isScheduled) return; - Top.releaseNode(SU, SU->TopReadyCycle); + Top.releaseNode(SU, SU->TopReadyCycle); } // Only called for roots. Index: llvm/lib/CodeGen/MachineScheduler.cpp =================================================================== --- llvm/lib/CodeGen/MachineScheduler.cpp +++ llvm/lib/CodeGen/MachineScheduler.cpp @@ -2083,7 +2083,8 @@ return OtherCritCount; } -void SchedBoundary::releaseNode(SUnit *SU, unsigned ReadyCycle) { +template +void SchedBoundary::releaseNode(SUnit *SU, unsigned ReadyCycle, unsigned Idx) { assert(SU->getInstr() && "Scheduled SUnit must have instr"); #ifndef NDEBUG @@ -2094,17 +2095,30 @@ MaxObservedStall = std::max(ReadyCycle - CurrCycle, MaxObservedStall); #endif - if (ReadyCycle < MinReadyCycle) - MinReadyCycle = ReadyCycle; - // Check for interlocks first. For the purpose of other heuristics, an // instruction that cannot issue appears as if it's not in the ReadyQueue. bool IsBuffered = SchedModel->getMicroOpBufferSize() != 0; - if ((!IsBuffered && ReadyCycle > CurrCycle) || checkHazard(SU) || - Available.size() >= ReadyListLimit) - Pending.push(SU); - else + bool HazardDetected = ((!IsBuffered && ReadyCycle > CurrCycle) || + checkHazard(SU)); + + // Check on Available's size already performed, if SU is in Pending queue. + // See 'SchedBoundary::releasePending'. + if (!InPQueue) + HazardDetected = HazardDetected || Available.size() >= ReadyListLimit; + + if (!HazardDetected) { Available.push(SU); + + if (ReadyCycle < MinReadyCycle) + MinReadyCycle = ReadyCycle; + + if (InPQueue) + Pending.remove(Pending.begin() + Idx); + return; + } + + if (!InPQueue) + Pending.push(SU); } /// Move the boundary of scheduled code by one cycle. @@ -2344,26 +2358,14 @@ // Check to see if any of the pending instructions are ready to issue. If // so, add them to the available queue. - bool IsBuffered = SchedModel->getMicroOpBufferSize() != 0; - for (unsigned i = 0, e = Pending.size(); i != e; ++i) { - SUnit *SU = *(Pending.begin()+i); - unsigned ReadyCycle = isTop() ? SU->TopReadyCycle : SU->BotReadyCycle; - - if (ReadyCycle < MinReadyCycle) - MinReadyCycle = ReadyCycle; - - if (!IsBuffered && ReadyCycle > CurrCycle) - continue; - - if (checkHazard(SU)) - continue; - + for (signed E = Pending.size() - 1; E >= 0; --E) { if (Available.size() >= ReadyListLimit) break; - Available.push(SU); - Pending.remove(Pending.begin()+i); - --i; --e; + SUnit *SU = *(Pending.begin() + E); + unsigned ReadyCycle = isTop() ? SU->TopReadyCycle : SU->BotReadyCycle; + + releaseNode(SU, ReadyCycle, E); } CheckPending = false; }