Index: include/llvm/CodeGen/RegisterPressure.h =================================================================== --- include/llvm/CodeGen/RegisterPressure.h +++ include/llvm/CodeGen/RegisterPressure.h @@ -141,6 +141,21 @@ LLVM_DUMP_METHOD void dump(const TargetRegisterInfo &TRI) const; }; +/// List of register defined and used by a machine instruction. +class RegisterOperands { +public: + SmallVector Uses; + SmallVector Defs; + SmallVector DeadDefs; + + void collect(const MachineInstr &MI, const TargetRegisterInfo &TRI, + const MachineRegisterInfo &MRI, bool IgnoreDead = false); + + /// Use liveness information to find dead defs not marked with a dead flag + /// and move them to the DeadDefs vector. + void detectDeadDefs(const MachineInstr &MI, const LiveIntervals &LIS); +}; + /// Array of PressureDiffs. class PressureDiffs { PressureDiff *PDiffArray; @@ -161,6 +176,10 @@ const PressureDiff &operator[](unsigned Idx) const { return const_cast(this)->operator[](Idx); } + /// \brief Record pressure difference induced by the given operand list to + /// node with index \p Idx. + void addInstruction(unsigned Idx, const RegisterOperands &RegOpers, + const MachineRegisterInfo &MRI); }; /// Store the effects of a change in pressure on things that MI scheduler cares @@ -329,8 +348,17 @@ void setPos(MachineBasicBlock::const_iterator Pos) { CurrPos = Pos; } /// Recede across the previous instruction. - void recede(SmallVectorImpl *LiveUses = nullptr, - PressureDiff *PDiff = nullptr); + void recede(SmallVectorImpl *LiveUses = nullptr); + + /// Recede across the previous instruction. + /// This "low-level" variant assumes that recedeSkipDebugValues() was + /// called previously and takes precomputed RegisterOperands for the + /// instruction. + void recede(const RegisterOperands &RegOpers, + SmallVectorImpl *LiveUses = nullptr); + + /// Recede until we find an instruction which is not a DebugValue. + void recedeSkipDebugValues(); /// Advance across the current instruction. void advance(); Index: lib/CodeGen/RegisterPressure.cpp =================================================================== --- lib/CodeGen/RegisterPressure.cpp +++ lib/CodeGen/RegisterPressure.cpp @@ -314,21 +314,6 @@ namespace { -/// List of register defined and used by a machine instruction. -class RegisterOperands { -public: - SmallVector Uses; - SmallVector Defs; - SmallVector DeadDefs; - - void collect(const MachineInstr &MI, const TargetRegisterInfo &TRI, - const MachineRegisterInfo &MRI, bool IgnoreDead = false); - - /// Use liveness information to find dead defs not marked with a dead flag - /// and move them to the DeadDefs vector. - void detectDeadDefs(const MachineInstr &MI, const LiveIntervals &LIS); -}; - /// Collect this instruction's unique uses and defs into SmallVectors for /// processing defs and uses in order. /// @@ -386,9 +371,11 @@ } } - friend class RegisterOperands; + friend class llvm::RegisterOperands; }; +} // namespace + void RegisterOperands::collect(const MachineInstr &MI, const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI, @@ -418,8 +405,6 @@ } } -} // namespace - /// Initialize an array of N PressureDiffs. void PressureDiffs::init(unsigned N) { Size = N; @@ -432,6 +417,18 @@ PDiffArray = reinterpret_cast(calloc(N, sizeof(PressureDiff))); } +void PressureDiffs::addInstruction(unsigned Idx, + const RegisterOperands &RegOpers, + const MachineRegisterInfo &MRI) { + PressureDiff &PDiff = (*this)[Idx]; + assert(!PDiff.begin()->isValid() && "stale PDiff"); + for (unsigned Reg : RegOpers.Defs) + PDiff.addPressureChange(Reg, true, &MRI); + + for (unsigned Reg : RegOpers.Uses) + PDiff.addPressureChange(Reg, false, &MRI); +} + /// Add a change in pressure to the pressure diff of a given instruction. void PressureDiff::addPressureChange(unsigned RegUnit, bool IsDec, const MachineRegisterInfo *MRI) { @@ -468,18 +465,6 @@ } } -/// Record the pressure difference induced by the given operand list. -static void collectPDiff(PressureDiff &PDiff, RegisterOperands &RegOpers, - const MachineRegisterInfo *MRI) { - assert(!PDiff.begin()->isValid() && "stale PDiff"); - - for (unsigned Reg : RegOpers.Defs) - PDiff.addPressureChange(Reg, true, MRI); - - for (unsigned Reg : RegOpers.Uses) - PDiff.addPressureChange(Reg, false, MRI); -} - /// Force liveness of registers. void RegPressureTracker::addLiveRegs(ArrayRef Regs) { for (unsigned Reg : Regs) { @@ -515,39 +500,10 @@ /// registers that are both defined and used by the instruction. If a pressure /// difference pointer is provided record the changes is pressure caused by this /// instruction independent of liveness. -void RegPressureTracker::recede(SmallVectorImpl *LiveUses, - PressureDiff *PDiff) { - assert(CurrPos != MBB->begin()); - if (!isBottomClosed()) - closeBottom(); - - // Open the top of the region using block iterators. - if (!RequireIntervals && isTopClosed()) - static_cast(P).openTop(CurrPos); - - // Find the previous instruction. - do - --CurrPos; - while (CurrPos != MBB->begin() && CurrPos->isDebugValue()); +void RegPressureTracker::recede(const RegisterOperands &RegOpers, + SmallVectorImpl *LiveUses) { assert(!CurrPos->isDebugValue()); - SlotIndex SlotIdx; - if (RequireIntervals) - SlotIdx = LIS->getInstructionIndex(CurrPos).getRegSlot(); - - // Open the top of the region using slot indexes. - if (RequireIntervals && isTopClosed()) - static_cast(P).openTop(SlotIdx); - - const MachineInstr &MI = *CurrPos; - RegisterOperands RegOpers; - RegOpers.collect(MI, *TRI, *MRI); - if (RequireIntervals) - RegOpers.detectDeadDefs(MI, *LIS); - - if (PDiff) - collectPDiff(*PDiff, RegOpers, MRI); - // Boost pressure for all dead defs together. increaseRegPressure(RegOpers.DeadDefs); decreaseRegPressure(RegOpers.DeadDefs); @@ -561,6 +517,10 @@ discoverLiveOut(Reg); } + SlotIndex SlotIdx; + if (RequireIntervals) + SlotIdx = LIS->getInstructionIndex(CurrPos).getRegSlot(); + // Generate liveness for uses. for (unsigned Reg : RegOpers.Uses) { if (!LiveRegs.contains(Reg)) { @@ -587,6 +547,41 @@ } } +void RegPressureTracker::recedeSkipDebugValues() { + assert(CurrPos != MBB->begin()); + if (!isBottomClosed()) + closeBottom(); + + // Open the top of the region using block iterators. + if (!RequireIntervals && isTopClosed()) + static_cast(P).openTop(CurrPos); + + // Find the previous instruction. + do + --CurrPos; + while (CurrPos != MBB->begin() && CurrPos->isDebugValue()); + + SlotIndex SlotIdx; + if (RequireIntervals) + SlotIdx = LIS->getInstructionIndex(CurrPos).getRegSlot(); + + // Open the top of the region using slot indexes. + if (RequireIntervals && isTopClosed()) + static_cast(P).openTop(SlotIdx); +} + +void RegPressureTracker::recede(SmallVectorImpl *LiveUses) { + recedeSkipDebugValues(); + + const MachineInstr &MI = *CurrPos; + RegisterOperands RegOpers; + RegOpers.collect(MI, *TRI, *MRI); + if (RequireIntervals) + RegOpers.detectDeadDefs(MI, *LIS); + + recede(RegOpers, LiveUses); +} + /// Advance across the current instruction. void RegPressureTracker::advance() { assert(!TrackUntiedDefs && "unsupported mode"); Index: lib/CodeGen/ScheduleDAGInstrs.cpp =================================================================== --- lib/CodeGen/ScheduleDAGInstrs.cpp +++ lib/CodeGen/ScheduleDAGInstrs.cpp @@ -896,11 +896,16 @@ assert(SU && "No SUnit mapped to this MI"); if (RPTracker) { - PressureDiff *PDiff = PDiffs ? &(*PDiffs)[SU->NodeNum] : nullptr; - RPTracker->recede(/*LiveUses=*/nullptr, PDiff); - assert(RPTracker->getPos() == std::prev(MII) && - "RPTracker can't find MI"); collectVRegUses(SU); + + RegisterOperands RegOpers; + RegOpers.collect(*MI, *TRI, MRI); + if (PDiffs != nullptr) + PDiffs->addInstruction(SU->NodeNum, RegOpers, MRI); + + RPTracker->recedeSkipDebugValues(); + assert(&*RPTracker->getPos() == MI && "RPTracker in sync"); + RPTracker->recede(RegOpers); } assert(