Index: lib/Target/AMDGPU/GCNRegPressure.h =================================================================== --- lib/Target/AMDGPU/GCNRegPressure.h +++ lib/Target/AMDGPU/GCNRegPressure.h @@ -98,8 +98,6 @@ const MachineInstr *LastTrackedMI = nullptr; mutable const MachineRegisterInfo *MRI = nullptr; GCNRPTracker(const LiveIntervals &LIS_) : LIS(LIS_) {} - LaneBitmask getDefRegMask(const MachineOperand &MO) const; - LaneBitmask getUsedRegMask(const MachineOperand &MO) const; public: // live regs for the current state const decltype(LiveRegs) &getLiveRegs() const { return LiveRegs; } Index: lib/Target/AMDGPU/GCNRegPressure.cpp =================================================================== --- lib/Target/AMDGPU/GCNRegPressure.cpp +++ lib/Target/AMDGPU/GCNRegPressure.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "GCNRegPressure.h" +#include "llvm/CodeGen/RegisterPressure.h" using namespace llvm; @@ -63,15 +64,6 @@ return true; } -static GCNRPTracker::LiveRegSet -stripEmpty(const GCNRPTracker::LiveRegSet &LR) { - GCNRPTracker::LiveRegSet Res; - for (const auto &P : LR) { - if (P.second.any()) - Res.insert(P); - } - return Res; -} #endif /////////////////////////////////////////////////////////////////////////////// @@ -185,6 +177,64 @@ } #endif + +static LaneBitmask getDefRegMask(const MachineOperand &MO, + const MachineRegisterInfo &MRI) { + assert(MO.isDef() && MO.isReg() && + TargetRegisterInfo::isVirtualRegister(MO.getReg())); + + // We don't rely on read-undef flag because in case of tentative schedule + // tracking it isn't set correctly yet. This works correctly however since + // use mask has been tracked before using LIS. + return MO.getSubReg() == 0 ? + MRI.getMaxLaneMaskForVReg(MO.getReg()) : + MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(MO.getSubReg()); +} + +static LaneBitmask getUsedRegMask(const MachineOperand &MO, + const MachineRegisterInfo &MRI, + const LiveIntervals &LIS) { + assert(MO.isUse() && MO.isReg() && + TargetRegisterInfo::isVirtualRegister(MO.getReg())); + + if (auto SubReg = MO.getSubReg()) + return MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(SubReg); + + auto MaxMask = MRI.getMaxLaneMaskForVReg(MO.getReg()); + if (MaxMask.getAsInteger() == 1) // cannot have subregs + return MaxMask; + + // For a tentative schedule LIS isn't updated yet but livemask should remain + // the same on any schedule. Subreg defs can be reordered but they all must + // dominate uses anyway. + auto SI = LIS.getInstructionIndex(*MO.getParent()).getBaseIndex(); + return getLiveLaneMask(MO.getReg(), SI, LIS, MRI); +} + +SmallVector collectVirtualRegUses(const MachineInstr &MI, + const LiveIntervals &LIS, + const MachineRegisterInfo &MRI) { + SmallVector Res; + for (const auto &MO : MI.operands()) { + if (!MO.isReg() || !TargetRegisterInfo::isVirtualRegister(MO.getReg())) + continue; + if (!MO.isUse() || !MO.readsReg()) + continue; + + auto const UsedMask = getUsedRegMask(MO, MRI, LIS); + + auto Reg = MO.getReg(); + auto I = std::find_if(Res.begin(), Res.end(), [Reg](const RegisterMaskPair &RM) { + return RM.RegUnit == Reg; + }); + if (I != Res.end()) + I->LaneMask |= UsedMask; + else + Res.push_back(RegisterMaskPair(Reg, UsedMask)); + } + return Res; +} + /////////////////////////////////////////////////////////////////////////////// // GCNRPTracker @@ -222,36 +272,6 @@ return LiveRegs; } -LaneBitmask GCNRPTracker::getDefRegMask(const MachineOperand &MO) const { - assert(MO.isDef() && MO.isReg() && - TargetRegisterInfo::isVirtualRegister(MO.getReg())); - - // We don't rely on read-undef flag because in case of tentative schedule - // tracking it isn't set correctly yet. This works correctly however since - // use mask has been tracked before using LIS. - return MO.getSubReg() == 0 ? - MRI->getMaxLaneMaskForVReg(MO.getReg()) : - MRI->getTargetRegisterInfo()->getSubRegIndexLaneMask(MO.getSubReg()); -} - -LaneBitmask GCNRPTracker::getUsedRegMask(const MachineOperand &MO) const { - assert(MO.isUse() && MO.isReg() && - TargetRegisterInfo::isVirtualRegister(MO.getReg())); - - if (auto SubReg = MO.getSubReg()) - return MRI->getTargetRegisterInfo()->getSubRegIndexLaneMask(SubReg); - - auto MaxMask = MRI->getMaxLaneMaskForVReg(MO.getReg()); - if (MaxMask.getAsInteger() == 1) // cannot have subregs - return MaxMask; - - // For a tentative schedule LIS isn't updated yet but livemask should remain - // the same on any schedule. Subreg defs can be reordered but they all must - // dominate uses anyway. - auto SI = LIS.getInstructionIndex(*MO.getParent()).getBaseIndex(); - return getLiveLaneMask(MO.getReg(), SI, LIS, *MRI); -} - void GCNUpwardRPTracker::reset(const MachineInstr &MI, const LiveRegSet *LiveRegsCopy) { MRI = &MI.getParent()->getParent()->getRegInfo(); @@ -272,34 +292,39 @@ if (MI.isDebugValue()) return; - // process all defs first to ensure early clobbers are handled correctly - // iterating over operands() to catch implicit defs - for (const auto &MO : MI.operands()) { - if (!MO.isReg() || !MO.isDef() || - !TargetRegisterInfo::isVirtualRegister(MO.getReg())) - continue; + auto const RegUses = collectVirtualRegUses(MI, LIS, *MRI); - auto Reg = MO.getReg(); - auto &LiveMask = LiveRegs[Reg]; - auto PrevMask = LiveMask; - LiveMask &= ~getDefRegMask(MO); - CurPressure.inc(Reg, PrevMask, LiveMask, *MRI); + // calc pressure at the MI (defs + uses) + auto AtMIPressure = CurPressure; + for (const auto &U : RegUses) { + auto LiveMask = LiveRegs[U.RegUnit]; + AtMIPressure.inc(U.RegUnit, LiveMask, LiveMask | U.LaneMask, *MRI); } + // update max pressure + MaxPressure = max(AtMIPressure, MaxPressure); - // then all uses - for (const auto &MO : MI.uses()) { - if (!MO.isReg() || !MO.readsReg() || - !TargetRegisterInfo::isVirtualRegister(MO.getReg())) + for (const auto &MO : MI.defs()) { + if (!MO.isReg() || !TargetRegisterInfo::isVirtualRegister(MO.getReg()) || + MO.isDead()) continue; auto Reg = MO.getReg(); - auto &LiveMask = LiveRegs[Reg]; + auto I = LiveRegs.find(Reg); + assert(I != LiveRegs.end()); + auto &LiveMask = I->second; auto PrevMask = LiveMask; - LiveMask |= getUsedRegMask(MO); + LiveMask &= ~ getDefRegMask(MO, *MRI); CurPressure.inc(Reg, PrevMask, LiveMask, *MRI); + if (LiveMask.none()) + LiveRegs.erase(I); } - - MaxPressure = max(MaxPressure, CurPressure); + for (const auto &U : RegUses) { + auto &LiveMask = LiveRegs[U.RegUnit]; + auto PrevMask = LiveMask; + LiveMask |= U.LaneMask; + CurPressure.inc(U.RegUnit, PrevMask, LiveMask, *MRI); + } + assert(CurPressure == getRegPressure(*MRI, LiveRegs)); } bool GCNDownwardRPTracker::reset(const MachineInstr &MI, @@ -368,7 +393,7 @@ continue; auto &LiveMask = LiveRegs[Reg]; auto PrevMask = LiveMask; - LiveMask |= getDefRegMask(MO); + LiveMask |= getDefRegMask(MO, *MRI); CurPressure.inc(Reg, PrevMask, LiveMask, *MRI); } @@ -430,7 +455,7 @@ bool GCNUpwardRPTracker::isValid() const { const auto &SI = LIS.getInstructionIndex(*LastTrackedMI).getBaseIndex(); const auto LISLR = llvm::getLiveRegs(SI, LIS, *MRI); - const auto TrackedLR = stripEmpty(LiveRegs); + const auto &TrackedLR = LiveRegs; if (!isEqual(LISLR, TrackedLR)) { dbgs() << "\nGCNUpwardRPTracker error: Tracked and"