Index: include/llvm/MCA/HardwareUnits/Scheduler.h =================================================================== --- include/llvm/MCA/HardwareUnits/Scheduler.h +++ include/llvm/MCA/HardwareUnits/Scheduler.h @@ -93,6 +93,7 @@ std::unique_ptr Resources; std::vector WaitSet; + std::vector PendingSet; std::vector ReadySet; std::vector IssuedSet; @@ -118,9 +119,13 @@ // vector 'Executed'. void updateIssuedSet(SmallVectorImpl &Executed); - // Try to promote instructions from WaitSet to ReadySet. + // Try to promote instructions from PendingSet to ReadySet. // Add promoted instructions to the 'Ready' vector in input. - void promoteToReadySet(SmallVectorImpl &Ready); + bool promoteToReadySet(SmallVectorImpl &Ready); + + // Try to promote instructions from WaitSet to PendingSet. + // Returns true if at least one instruction was promoted. + bool promoteToPendingSet(); public: Scheduler(const MCSchedModel &Model, LSUnit &Lsu) Index: include/llvm/MCA/Instruction.h =================================================================== --- include/llvm/MCA/Instruction.h +++ include/llvm/MCA/Instruction.h @@ -168,6 +168,14 @@ bool clearsSuperRegisters() const { return ClearsSuperRegs; } bool isWriteZero() const { return WritesZero; } bool isEliminated() const { return IsEliminated; } + + bool isReady() const { + if (getDependentWrite()) + return false; + unsigned CyclesLeft = getDependentWriteCyclesLeft(); + return !CyclesLeft || CyclesLeft < getLatency(); + } + bool isExecuted() const { return CyclesLeft != UNKNOWN_CYCLES && CyclesLeft <= 0; } @@ -239,6 +247,7 @@ unsigned getRegisterID() const { return RegisterID; } unsigned getRegisterFileID() const { return PRFID; } + bool isWaiting() const { return !IndependentFromDef && CyclesLeft > 0; } bool isReady() const { return IsReady; } bool isImplicitRead() const { return RD->isImplicitRead(); } @@ -411,6 +420,7 @@ enum InstrStage { IS_INVALID, // Instruction in an invalid state. IS_DISPATCHED, // Instruction dispatched but operands are not ready. + IS_WAITING, // Instruction is not ready, but operand latency is known. IS_READY, // Instruction dispatched and operands ready. IS_EXECUTING, // Instruction issued. IS_EXECUTED, // Instruction executed. Values are written back. @@ -444,15 +454,18 @@ // all the definitions. void execute(); - // Force a transition from the IS_DISPATCHED state to the IS_READY state if - // input operands are all ready. State transitions normally occur at the - // beginning of a new cycle (see method cycleEvent()). However, the scheduler - // may decide to promote instructions from the wait queue to the ready queue - // as the result of another issue event. This method is called every time the - // instruction might have changed in state. + // Force a transition from the IS_DISPATCHED state to the IS_READY or + // IS_WAITING state. State transitions normally occur either at the beginning + // of a new cycle (see method cycleEvent()), or as a result of another issue + // event. This method is called every time the instruction might have changed + // in state. It internally delegates to method updateDispatched() and + // updateWaiting(). void update(); + bool updateDispatched(); + bool updateWaiting(); bool isDispatched() const { return Stage == IS_DISPATCHED; } + bool isWaiting() const { return Stage == IS_WAITING; } bool isReady() const { return Stage == IS_READY; } bool isExecuting() const { return Stage == IS_EXECUTING; } bool isExecuted() const { return Stage == IS_EXECUTED; } Index: lib/MCA/HardwareUnits/Scheduler.cpp =================================================================== --- lib/MCA/HardwareUnits/Scheduler.cpp +++ lib/MCA/HardwareUnits/Scheduler.cpp @@ -97,14 +97,15 @@ // this same cycle if operands have ReadAdvance entries. Promote those // instructions to the ReadySet and notify the caller that those are ready. if (HasDependentUsers) - promoteToReadySet(ReadyInstructions); + if (promoteToPendingSet()) + promoteToReadySet(ReadyInstructions); } -void Scheduler::promoteToReadySet(SmallVectorImpl &Ready) { +bool Scheduler::promoteToReadySet(SmallVectorImpl &Ready) { // Scan the set of waiting instructions and promote them to the - // ready queue if operands are all ready. + // ready set if operands are all ready. unsigned RemovedElements = 0; - for (auto I = WaitSet.begin(), E = WaitSet.end(); I != E;) { + for (auto I = PendingSet.begin(), E = PendingSet.end(); I != E;) { InstRef &IR = *I; if (!IR) break; @@ -113,7 +114,7 @@ // a transition in state using method 'update()'. Instruction &IS = *IR.getInstruction(); if (!IS.isReady()) - IS.update(); + IS.updateWaiting(); // Check if there are still unsolved data dependencies. if (!isReady(IR)) { @@ -121,6 +122,9 @@ continue; } + LLVM_DEBUG(dbgs() << "[SCHEDULER]: Instruction #" << IR + << " promoted to the READY set.\n"); + Ready.emplace_back(IR); ReadySet.emplace_back(IR); @@ -129,7 +133,38 @@ std::iter_swap(I, E - RemovedElements); } + PendingSet.resize(PendingSet.size() - RemovedElements); + return RemovedElements; +} + +bool Scheduler::promoteToPendingSet() { + // Scan the set of waiting instructions and promote them to the + // pending set if operands are all ready. + unsigned RemovedElements = 0; + for (auto I = WaitSet.begin(), E = WaitSet.end(); I != E;) { + InstRef &IR = *I; + if (!IR) + break; + + // Check if this instruction is now ready. In case, force + // a transition in state using method 'update()'. + Instruction &IS = *IR.getInstruction(); + if (IS.isDispatched() && !IS.updateDispatched()) { + ++I; + continue; + } + LLVM_DEBUG(dbgs() << "[SCHEDULER]: Instruction #" << IR + << " promoted to the PENDING set.\n"); + + PendingSet.emplace_back(IR); + + IR.invalidate(); + ++RemovedElements; + std::iter_swap(I, E - RemovedElements); + } + WaitSet.resize(WaitSet.size() - RemovedElements); + return RemovedElements; } InstRef Scheduler::select() { @@ -193,9 +228,13 @@ updateIssuedSet(Executed); + for (InstRef &IR : PendingSet) + IR.getInstruction()->cycleEvent(); + for (InstRef &IR : WaitSet) IR.getInstruction()->cycleEvent(); + promoteToPendingSet(); promoteToReadySet(Ready); BusyResourceUnits = 0; @@ -220,6 +259,13 @@ if (IsMemOp) LSU.dispatch(IR); + if (IR.getInstruction()->isWaiting()) { + LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR + << " to the PendingSet\n"); + PendingSet.push_back(IR); + return; + } + if (!isReady(IR)) { LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR << " to the WaitSet\n"); WaitSet.push_back(IR); Index: lib/MCA/Instruction.cpp =================================================================== --- lib/MCA/Instruction.cpp +++ lib/MCA/Instruction.cpp @@ -151,30 +151,54 @@ Stage = IS_EXECUTED; } -void Instruction::update() { - assert(isDispatched() && "Unexpected instruction stage found!"); +bool Instruction::updateWaiting() { + assert(isWaiting() && "Unexpected instruction stage found!"); if (!all_of(getUses(), [](const ReadState &Use) { return Use.isReady(); })) - return; + return false; // A partial register write cannot complete before a dependent write. - auto IsDefReady = [&](const WriteState &Def) { - if (!Def.getDependentWrite()) { - unsigned CyclesLeft = Def.getDependentWriteCyclesLeft(); - return !CyclesLeft || CyclesLeft < getLatency(); - } + auto IsDefReady = [&](const WriteState &Def) { return Def.isReady(); }; + + if (!all_of(getDefs(), IsDefReady)) + return false; + + Stage = IS_READY; + return true; +} + +bool Instruction::updateDispatched() { + assert(isDispatched() && "Unexpected instruction stage found!"); + + if (!all_of(getUses(), [](const ReadState &Use) { + return Use.isWaiting() || Use.isReady(); + })) return false; + + // A partial register write cannot complete before a dependent write. + auto IsDefWaitingOrReady = [&](const WriteState &Def) { + return !Def.getDependentWrite(); }; - if (all_of(getDefs(), IsDefReady)) - Stage = IS_READY; + if (!all_of(getDefs(), IsDefWaitingOrReady)) + return false; + + Stage = IS_WAITING; + return true; +} + +void Instruction::update() { + if (isDispatched()) + updateDispatched(); + if (isWaiting()) + updateWaiting(); } void Instruction::cycleEvent() { if (isReady()) return; - if (isDispatched()) { + if (isDispatched() || isWaiting()) { for (ReadState &Use : getUses()) Use.cycleEvent();