diff --git a/llvm/include/llvm/CodeGen/LiveRegUnits.h b/llvm/include/llvm/CodeGen/LiveRegUnits.h --- a/llvm/include/llvm/CodeGen/LiveRegUnits.h +++ b/llvm/include/llvm/CodeGen/LiveRegUnits.h @@ -112,14 +112,12 @@ /// The regmask has the same format as the one in the RegMask machine operand. void addRegsInMask(const uint32_t *RegMask); - /// Returns true if no part of physical register \p Reg is live. - bool available(MCPhysReg Reg) const { - for (MCRegUnit Unit : TRI->regunits(Reg)) { - if (Units.test(Unit)) - return false; - } - return true; - } + /// Returns true if no part of non-reserved physical register \p Reg is live + /// (overloaded) + bool available(const MachineRegisterInfo &MRI, MCPhysReg Reg) const; + + /// Returns true if no part of physical register \p Reg is live. (overloaded) + bool available(MCPhysReg Reg) const; /// Updates liveness when stepping backwards over the instruction \p MI. /// This removes all register units defined or clobbered in \p MI and then diff --git a/llvm/lib/CodeGen/LiveRegUnits.cpp b/llvm/lib/CodeGen/LiveRegUnits.cpp --- a/llvm/lib/CodeGen/LiveRegUnits.cpp +++ b/llvm/lib/CodeGen/LiveRegUnits.cpp @@ -84,6 +84,25 @@ } } +bool LiveRegUnits::available(const MachineRegisterInfo &MRI, + MCPhysReg Reg) const { + for (MCRegUnit Unit : TRI->regunits(Reg)) { + if (Units.test(Unit)) + return false; + else if (MRI.isReserved(Reg)) + return false; + } + return true; +} + +bool LiveRegUnits::available(MCPhysReg Reg) const { + for (MCRegUnit Unit : TRI->regunits(Reg)) { + if (Units.test(Unit)) + return false; + } + return true; +} + /// Add live-in registers of basic block \p MBB to \p LiveUnits. static void addBlockLiveIns(LiveRegUnits &LiveUnits, const MachineBasicBlock &MBB) { diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.h b/llvm/lib/Target/AMDGPU/SIFrameLowering.h --- a/llvm/lib/Target/AMDGPU/SIFrameLowering.h +++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.h @@ -38,11 +38,11 @@ bool NeedExecCopyReservedReg) const; void emitCSRSpillStores(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL, - LivePhysRegs &LiveRegs, Register FrameReg, + LiveRegUnits &LiveUnits, Register FrameReg, Register FramePtrRegScratchCopy) const; void emitCSRSpillRestores(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL, - LivePhysRegs &LiveRegs, Register FrameReg, + LiveRegUnits &LiveUnits, Register FrameReg, Register FramePtrRegScratchCopy) const; bool assignCalleeSavedSpillSlots(MachineFunction &MF, diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp --- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp @@ -11,7 +11,7 @@ #include "GCNSubtarget.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIMachineFunctionInfo.h" -#include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/RegisterScavenging.h" #include "llvm/Target/TargetMachine.h" @@ -26,13 +26,13 @@ cl::ReallyHidden, cl::init(true)); -// Find a register matching \p RC from \p LiveRegs which is unused and available -// throughout the function. On failure, returns AMDGPU::NoRegister. +// Find a register matching \p RC from \p LiveUnits which is unused and +// available throughout the function. On failure, returns AMDGPU::NoRegister. static MCRegister findUnusedRegister(MachineRegisterInfo &MRI, - const LivePhysRegs &LiveRegs, + const LiveRegUnits &LiveUnits, const TargetRegisterClass &RC) { for (MCRegister Reg : RC) { - if (!MRI.isPhysRegUsed(Reg) && LiveRegs.available(MRI, Reg)) + if (!MRI.isPhysRegUsed(Reg) && LiveUnits.available(MRI, Reg)) return Reg; } return MCRegister(); @@ -42,22 +42,21 @@ // callee-save registers since they may appear to be free when this is called // from canUseAsPrologue (during shrink wrapping), but then no longer be free // when this is called from emitPrologue. -static MCRegister findScratchNonCalleeSaveRegister(MachineRegisterInfo &MRI, - LivePhysRegs &LiveRegs, - const TargetRegisterClass &RC, - bool Unused = false) { +static MCRegister findScratchNonCalleeSaveRegister( + MachineRegisterInfo &MRI, LiveRegUnits &LiveUnits, + const TargetRegisterClass &RC, bool Unused = false) { // Mark callee saved registers as used so we will not choose them. const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs(); for (unsigned i = 0; CSRegs[i]; ++i) - LiveRegs.addReg(CSRegs[i]); + LiveUnits.addReg(CSRegs[i]); // We are looking for a register that can be used throughout the entire // function, so any use is unacceptable. if (Unused) - return findUnusedRegister(MRI, LiveRegs, RC); + return findUnusedRegister(MRI, LiveUnits, RC); for (MCRegister Reg : RC) { - if (LiveRegs.available(MRI, Reg)) + if (LiveUnits.available(MRI, Reg)) return Reg; } @@ -65,9 +64,9 @@ } /// Query target location for spilling SGPRs -/// \p IncludeScratchCopy : Also look for free scratch SGPRs +/// \p IncludeScratchCopy : Also look for free scratch SGPRs static void getVGPRSpillLaneOrTempRegister( - MachineFunction &MF, LivePhysRegs &LiveRegs, Register SGPR, + MachineFunction &MF, LiveRegUnits &LiveUnits, Register SGPR, const TargetRegisterClass &RC = AMDGPU::SReg_32_XM0_XEXECRegClass, bool IncludeScratchCopy = true) { SIMachineFunctionInfo *MFI = MF.getInfo(); @@ -81,11 +80,11 @@ // We need to save and restore the given SGPR. Register ScratchSGPR; - // 1: Try to save the given register into an unused scratch SGPR. The LiveRegs - // should have all the callee saved registers marked as used. For certain - // cases we skip copy to scratch SGPR. + // 1: Try to save the given register into an unused scratch SGPR. The + // LiveUnits should have all the callee saved registers marked as used. For + // certain cases we skip copy to scratch SGPR. if (IncludeScratchCopy) - ScratchSGPR = findUnusedRegister(MF.getRegInfo(), LiveRegs, RC); + ScratchSGPR = findUnusedRegister(MF.getRegInfo(), LiveUnits, RC); if (!ScratchSGPR) { int FI = FrameInfo.CreateStackObject(Size, Alignment, true, nullptr, @@ -118,7 +117,7 @@ MFI->addToPrologEpilogSGPRSpills( SGPR, PrologEpilogSGPRSaveRestoreInfo( SGPRSaveKind::COPY_TO_SCRATCH_SGPR, ScratchSGPR)); - LiveRegs.addReg(ScratchSGPR); + LiveUnits.addReg(ScratchSGPR); LLVM_DEBUG(dbgs() << "Saving " << printReg(SGPR, TRI) << " with copy to " << printReg(ScratchSGPR, TRI) << '\n'); } @@ -129,7 +128,7 @@ // use. static void buildPrologSpill(const GCNSubtarget &ST, const SIRegisterInfo &TRI, const SIMachineFunctionInfo &FuncInfo, - LivePhysRegs &LiveRegs, MachineFunction &MF, + LiveRegUnits &LiveUnits, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register SpillReg, int FI, Register FrameReg, @@ -142,18 +141,18 @@ MachineMemOperand *MMO = MF.getMachineMemOperand( PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FI), FrameInfo.getObjectAlign(FI)); - LiveRegs.addReg(SpillReg); + LiveUnits.addReg(SpillReg); bool IsKill = !MBB.isLiveIn(SpillReg); TRI.buildSpillLoadStore(MBB, I, DL, Opc, FI, SpillReg, IsKill, FrameReg, - DwordOff, MMO, nullptr, &LiveRegs); + DwordOff, MMO, nullptr, &LiveUnits); if (IsKill) - LiveRegs.removeReg(SpillReg); + LiveUnits.removeReg(SpillReg); } static void buildEpilogRestore(const GCNSubtarget &ST, const SIRegisterInfo &TRI, const SIMachineFunctionInfo &FuncInfo, - LivePhysRegs &LiveRegs, MachineFunction &MF, + LiveRegUnits &LiveUnits, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register SpillReg, int FI, @@ -167,7 +166,7 @@ PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FI), FrameInfo.getObjectAlign(FI)); TRI.buildSpillLoadStore(MBB, I, DL, Opc, FI, SpillReg, false, FrameReg, - DwordOff, MMO, nullptr, &LiveRegs); + DwordOff, MMO, nullptr, &LiveUnits); } static void buildGitPtr(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, @@ -195,18 +194,18 @@ .addReg(GitPtrLo); } -static void initLiveRegs(LivePhysRegs &LiveRegs, const SIRegisterInfo &TRI, - const SIMachineFunctionInfo *FuncInfo, - MachineFunction &MF, MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, bool IsProlog) { - if (LiveRegs.empty()) { - LiveRegs.init(TRI); +static void initLiveUnits(LiveRegUnits &LiveUnits, const SIRegisterInfo &TRI, + const SIMachineFunctionInfo *FuncInfo, + MachineFunction &MF, MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, bool IsProlog) { + if (LiveUnits.empty()) { + LiveUnits.init(TRI); if (IsProlog) { - LiveRegs.addLiveIns(MBB); + LiveUnits.addLiveIns(MBB); } else { // In epilog. - LiveRegs.addLiveOuts(MBB); - LiveRegs.stepBackward(*MBBI); + LiveUnits.addLiveOuts(MBB); + LiveUnits.stepBackward(*MBBI); } } } @@ -228,7 +227,7 @@ const SIRegisterInfo &TRI; Register SuperReg; const PrologEpilogSGPRSaveRestoreInfo SI; - LivePhysRegs &LiveRegs; + LiveRegUnits &LiveUnits; const DebugLoc &DL; Register FrameReg; ArrayRef SplitParts; @@ -239,10 +238,10 @@ MachineRegisterInfo &MRI = MF.getRegInfo(); assert(!MFI.isDeadObjectIndex(FI)); - initLiveRegs(LiveRegs, TRI, FuncInfo, MF, MBB, MI, /*IsProlog*/ true); + initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MI, /*IsProlog*/ true); MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister( - MRI, LiveRegs, AMDGPU::VGPR_32RegClass); + MRI, LiveUnits, AMDGPU::VGPR_32RegClass); if (!TmpVGPR) report_fatal_error("failed to find free scratch register"); @@ -253,7 +252,7 @@ BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR) .addReg(SubReg); - buildPrologSpill(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MI, DL, TmpVGPR, + buildPrologSpill(ST, TRI, *FuncInfo, LiveUnits, MF, MBB, MI, DL, TmpVGPR, FI, FrameReg, DwordOff); DwordOff += 4; } @@ -287,9 +286,9 @@ void restoreFromMemory(const int FI) { MachineRegisterInfo &MRI = MF.getRegInfo(); - initLiveRegs(LiveRegs, TRI, FuncInfo, MF, MBB, MI, /*IsProlog*/ false); + initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MI, /*IsProlog*/ false); MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister( - MRI, LiveRegs, AMDGPU::VGPR_32RegClass); + MRI, LiveUnits, AMDGPU::VGPR_32RegClass); if (!TmpVGPR) report_fatal_error("failed to find free scratch register"); @@ -298,8 +297,8 @@ ? SuperReg : Register(TRI.getSubReg(SuperReg, SplitParts[I])); - buildEpilogRestore(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MI, DL, TmpVGPR, - FI, FrameReg, DwordOff); + buildEpilogRestore(ST, TRI, *FuncInfo, LiveUnits, MF, MBB, MI, DL, + TmpVGPR, FI, FrameReg, DwordOff); BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), SubReg) .addReg(TmpVGPR, RegState::Kill); DwordOff += 4; @@ -335,11 +334,12 @@ MachineBasicBlock::iterator MI, const DebugLoc &DL, const SIInstrInfo *TII, const SIRegisterInfo &TRI, - LivePhysRegs &LiveRegs, Register FrameReg) + LiveRegUnits &LiveUnits, Register FrameReg) : MI(MI), MBB(MBB), MF(*MBB.getParent()), ST(MF.getSubtarget()), MFI(MF.getFrameInfo()), FuncInfo(MF.getInfo()), TII(TII), TRI(TRI), - SuperReg(Reg), SI(SI), LiveRegs(LiveRegs), DL(DL), FrameReg(FrameReg) { + SuperReg(Reg), SI(SI), LiveUnits(LiveUnits), DL(DL), + FrameReg(FrameReg) { const TargetRegisterClass *RC = TRI.getPhysRegBaseClass(SuperReg); SplitParts = TRI.getRegSplitParts(RC, EltSize); NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size(); @@ -396,9 +396,9 @@ if (ST.isAmdPalOS()) { // Extract the scratch offset from the descriptor in the GIT - LivePhysRegs LiveRegs; - LiveRegs.init(*TRI); - LiveRegs.addLiveIns(MBB); + LiveRegUnits LiveUnits; + LiveUnits.init(*TRI); + LiveUnits.addLiveIns(MBB); // Find unused reg to load flat scratch init into MachineRegisterInfo &MRI = MF.getRegInfo(); @@ -409,7 +409,7 @@ std::min(static_cast(AllSGPR64s.size()), NumPreloaded)); Register GITPtrLoReg = MFI->getGITPtrLoReg(MF); for (MCPhysReg Reg : AllSGPR64s) { - if (LiveRegs.available(MRI, Reg) && MRI.isAllocatable(Reg) && + if (LiveUnits.available(MRI, Reg) && MRI.isAllocatable(Reg) && !TRI->isSubRegisterEq(Reg, GITPtrLoReg)) { FlatScrInit = Reg; break; @@ -873,7 +873,7 @@ // Activate only the inactive lanes when \p EnableInactiveLanes is true. // Otherwise, activate all lanes. It returns the saved exec. -static Register buildScratchExecCopy(LivePhysRegs &LiveRegs, +static Register buildScratchExecCopy(LiveRegUnits &LiveUnits, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, @@ -886,14 +886,14 @@ const SIRegisterInfo &TRI = TII->getRegisterInfo(); SIMachineFunctionInfo *FuncInfo = MF.getInfo(); - initLiveRegs(LiveRegs, TRI, FuncInfo, MF, MBB, MBBI, IsProlog); + initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI, IsProlog); ScratchExecCopy = findScratchNonCalleeSaveRegister( - MRI, LiveRegs, *TRI.getWaveMaskRegClass()); + MRI, LiveUnits, *TRI.getWaveMaskRegClass()); if (!ScratchExecCopy) report_fatal_error("failed to find free scratch register"); - LiveRegs.addReg(ScratchExecCopy); + LiveUnits.addReg(ScratchExecCopy); const unsigned SaveExecOpc = ST.isWave32() ? (EnableInactiveLanes ? AMDGPU::S_XOR_SAVEEXEC_B32 @@ -909,7 +909,7 @@ void SIFrameLowering::emitCSRSpillStores( MachineFunction &MF, MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, DebugLoc &DL, LivePhysRegs &LiveRegs, + MachineBasicBlock::iterator MBBI, DebugLoc &DL, LiveRegUnits &LiveUnits, Register FrameReg, Register FramePtrRegScratchCopy) const { SIMachineFunctionInfo *FuncInfo = MF.getInfo(); const GCNSubtarget &ST = MF.getSubtarget(); @@ -924,7 +924,7 @@ FuncInfo->splitWWMSpillRegisters(MF, WWMCalleeSavedRegs, WWMScratchRegs); if (!WWMScratchRegs.empty()) ScratchExecCopy = - buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, DL, + buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL, /*IsProlog*/ true, /*EnableInactiveLanes*/ true); auto StoreWWMRegisters = @@ -932,7 +932,7 @@ for (const auto &Reg : WWMRegs) { Register VGPR = Reg.first; int FI = Reg.second; - buildPrologSpill(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MBBI, DL, + buildPrologSpill(ST, TRI, *FuncInfo, LiveUnits, MF, MBB, MBBI, DL, VGPR, FI, FrameReg); } }; @@ -943,7 +943,7 @@ unsigned MovOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; BuildMI(MBB, MBBI, DL, TII->get(MovOpc), TRI.getExec()).addImm(-1); } else { - ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, DL, + ScratchExecCopy = buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL, /*IsProlog*/ true, /*EnableInactiveLanes*/ false); } @@ -955,7 +955,7 @@ unsigned ExecMov = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; BuildMI(MBB, MBBI, DL, TII->get(ExecMov), TRI.getExec()) .addReg(ScratchExecCopy, RegState::Kill); - LiveRegs.addReg(ScratchExecCopy); + LiveUnits.addReg(ScratchExecCopy); } Register FramePtrReg = FuncInfo->getFrameOffsetReg(); @@ -971,7 +971,7 @@ continue; PrologEpilogSGPRSpillBuilder SB(Reg, Spill.second, MBB, MBBI, DL, TII, TRI, - LiveRegs, FrameReg); + LiveUnits, FrameReg); SB.save(); } @@ -986,16 +986,16 @@ MBB.sortUniqueLiveIns(); } - if (!LiveRegs.empty()) { + if (!LiveUnits.empty()) { for (MCPhysReg Reg : ScratchSGPRs) - LiveRegs.addReg(Reg); + LiveUnits.addReg(Reg); } } } void SIFrameLowering::emitCSRSpillRestores( MachineFunction &MF, MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, DebugLoc &DL, LivePhysRegs &LiveRegs, + MachineBasicBlock::iterator MBBI, DebugLoc &DL, LiveRegUnits &LiveUnits, Register FrameReg, Register FramePtrRegScratchCopy) const { const SIMachineFunctionInfo *FuncInfo = MF.getInfo(); const GCNSubtarget &ST = MF.getSubtarget(); @@ -1015,7 +1015,7 @@ continue; PrologEpilogSGPRSpillBuilder SB(Reg, Spill.second, MBB, MBBI, DL, TII, TRI, - LiveRegs, FrameReg); + LiveUnits, FrameReg); SB.restore(); } @@ -1027,7 +1027,7 @@ FuncInfo->splitWWMSpillRegisters(MF, WWMCalleeSavedRegs, WWMScratchRegs); if (!WWMScratchRegs.empty()) ScratchExecCopy = - buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, DL, + buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL, /*IsProlog*/ false, /*EnableInactiveLanes*/ true); auto RestoreWWMRegisters = @@ -1035,7 +1035,7 @@ for (const auto &Reg : WWMRegs) { Register VGPR = Reg.first; int FI = Reg.second; - buildEpilogRestore(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MBBI, DL, + buildEpilogRestore(ST, TRI, *FuncInfo, LiveUnits, MF, MBB, MBBI, DL, VGPR, FI, FrameReg); } }; @@ -1046,7 +1046,7 @@ unsigned MovOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; BuildMI(MBB, MBBI, DL, TII->get(MovOpc), TRI.getExec()).addImm(-1); } else { - ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, DL, + ScratchExecCopy = buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL, /*IsProlog*/ false, /*EnableInactiveLanes*/ false); } @@ -1079,7 +1079,7 @@ Register FramePtrReg = FuncInfo->getFrameOffsetReg(); Register BasePtrReg = TRI.hasBasePointer(MF) ? TRI.getBaseRegister() : Register(); - LivePhysRegs LiveRegs; + LiveRegUnits LiveUnits; MachineBasicBlock::iterator MBBI = MBB.begin(); // DebugLoc must be unknown since the first instruction with DebugLoc is used @@ -1097,14 +1097,14 @@ Register FramePtrRegScratchCopy; if (!HasFP && !hasFP(MF)) { // Emit the CSR spill stores with SP base register. - emitCSRSpillStores(MF, MBB, MBBI, DL, LiveRegs, StackPtrReg, + emitCSRSpillStores(MF, MBB, MBBI, DL, LiveUnits, StackPtrReg, FramePtrRegScratchCopy); } else { // CSR spill stores will use FP as base register. Register SGPRForFPSaveRestoreCopy = FuncInfo->getScratchSGPRCopyDstReg(FramePtrReg); - initLiveRegs(LiveRegs, TRI, FuncInfo, MF, MBB, MBBI, /*IsProlog*/ true); + initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI, /*IsProlog*/ true); if (SGPRForFPSaveRestoreCopy) { // Copy FP to the scratch register now and emit the CFI entry. It avoids // the extra FP copy needed in the other two cases when FP is spilled to @@ -1112,18 +1112,18 @@ PrologEpilogSGPRSpillBuilder SB( FramePtrReg, FuncInfo->getPrologEpilogSGPRSaveRestoreInfo(FramePtrReg), MBB, MBBI, - DL, TII, TRI, LiveRegs, FramePtrReg); + DL, TII, TRI, LiveUnits, FramePtrReg); SB.save(); - LiveRegs.addReg(SGPRForFPSaveRestoreCopy); + LiveUnits.addReg(SGPRForFPSaveRestoreCopy); } else { // Copy FP into a new scratch register so that its previous value can be // spilled after setting up the new frame. FramePtrRegScratchCopy = findScratchNonCalleeSaveRegister( - MRI, LiveRegs, AMDGPU::SReg_32_XM0_XEXECRegClass); + MRI, LiveUnits, AMDGPU::SReg_32_XM0_XEXECRegClass); if (!FramePtrRegScratchCopy) report_fatal_error("failed to find free scratch register"); - LiveRegs.addReg(FramePtrRegScratchCopy); + LiveUnits.addReg(FramePtrRegScratchCopy); BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), FramePtrRegScratchCopy) .addReg(FramePtrReg); } @@ -1133,9 +1133,9 @@ const unsigned Alignment = MFI.getMaxAlign().value(); RoundedSize += Alignment; - if (LiveRegs.empty()) { - LiveRegs.init(TRI); - LiveRegs.addLiveIns(MBB); + if (LiveUnits.empty()) { + LiveUnits.init(TRI); + LiveUnits.addLiveIns(MBB); } // s_add_i32 s33, s32, NumBytes @@ -1158,10 +1158,10 @@ // If FP is used, emit the CSR spills with FP base register. if (HasFP) { - emitCSRSpillStores(MF, MBB, MBBI, DL, LiveRegs, FramePtrReg, + emitCSRSpillStores(MF, MBB, MBBI, DL, LiveUnits, FramePtrReg, FramePtrRegScratchCopy); if (FramePtrRegScratchCopy) - LiveRegs.removeReg(FramePtrRegScratchCopy); + LiveUnits.removeReg(FramePtrRegScratchCopy); } // If we need a base pointer, set it up here. It's whatever the value of @@ -1210,7 +1210,7 @@ const SIInstrInfo *TII = ST.getInstrInfo(); const SIRegisterInfo &TRI = TII->getRegisterInfo(); MachineRegisterInfo &MRI = MF.getRegInfo(); - LivePhysRegs LiveRegs; + LiveRegUnits LiveUnits; // Get the insert location for the epilogue. If there were no terminators in // the block, get the last instruction. MachineBasicBlock::iterator MBBI = MBB.end(); @@ -1240,19 +1240,19 @@ // SGPRForFPSaveRestoreCopy is not true, restore the previous value of FP // into a new scratch register and copy to FP later when other registers are // restored from the current stack frame. - initLiveRegs(LiveRegs, TRI, FuncInfo, MF, MBB, MBBI, /*IsProlog*/ false); + initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI, /*IsProlog*/ false); if (SGPRForFPSaveRestoreCopy) { - LiveRegs.addReg(SGPRForFPSaveRestoreCopy); + LiveUnits.addReg(SGPRForFPSaveRestoreCopy); } else { FramePtrRegScratchCopy = findScratchNonCalleeSaveRegister( - MRI, LiveRegs, AMDGPU::SReg_32_XM0_XEXECRegClass); + MRI, LiveUnits, AMDGPU::SReg_32_XM0_XEXECRegClass); if (!FramePtrRegScratchCopy) report_fatal_error("failed to find free scratch register"); - LiveRegs.addReg(FramePtrRegScratchCopy); + LiveUnits.addReg(FramePtrRegScratchCopy); } - emitCSRSpillRestores(MF, MBB, MBBI, DL, LiveRegs, FramePtrReg, + emitCSRSpillRestores(MF, MBB, MBBI, DL, LiveUnits, FramePtrReg, FramePtrRegScratchCopy); } @@ -1275,7 +1275,7 @@ MIB.setMIFlag(MachineInstr::FrameDestroy); } else { // Insert the CSR spill restores with SP as the base register. - emitCSRSpillRestores(MF, MBB, MBBI, DL, LiveRegs, StackPtrReg, + emitCSRSpillRestores(MF, MBB, MBBI, DL, LiveUnits, StackPtrReg, FramePtrRegScratchCopy); } } @@ -1472,30 +1472,30 @@ SIMachineFunctionInfo *MFI = MF.getInfo(); const GCNSubtarget &ST = MF.getSubtarget(); const SIRegisterInfo *TRI = ST.getRegisterInfo(); - LivePhysRegs LiveRegs; - LiveRegs.init(*TRI); + LiveRegUnits LiveUnits; + LiveUnits.init(*TRI); // Initially mark callee saved registers as used so we will not choose them // while looking for scratch SGPRs. const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs(); for (unsigned I = 0; CSRegs[I]; ++I) - LiveRegs.addReg(CSRegs[I]); + LiveUnits.addReg(CSRegs[I]); const TargetRegisterClass &RC = *TRI->getWaveMaskRegClass(); if (NeedExecCopyReservedReg) { Register ReservedReg = MFI->getSGPRForEXECCopy(); assert(ReservedReg && "Should have reserved an SGPR for EXEC copy."); - Register UnusedScratchReg = findUnusedRegister(MRI, LiveRegs, RC); + Register UnusedScratchReg = findUnusedRegister(MRI, LiveUnits, RC); if (UnusedScratchReg) { // If found any unused scratch SGPR, reserve the register itself for Exec // copy and there is no need for any spills in that case. MFI->setSGPRForEXECCopy(UnusedScratchReg); - LiveRegs.addReg(UnusedScratchReg); + LiveUnits.addReg(UnusedScratchReg); } else { // Needs spill. assert(!MFI->hasPrologEpilogSGPRSpillEntry(ReservedReg) && "Re-reserving spill slot for EXEC copy register"); - getVGPRSpillLaneOrTempRegister(MF, LiveRegs, ReservedReg, RC, + getVGPRSpillLaneOrTempRegister(MF, LiveUnits, ReservedReg, RC, /*IncludeScratchCopy=*/false); } } @@ -1516,14 +1516,14 @@ Register FramePtrReg = MFI->getFrameOffsetReg(); assert(!MFI->hasPrologEpilogSGPRSpillEntry(FramePtrReg) && "Re-reserving spill slot for FP"); - getVGPRSpillLaneOrTempRegister(MF, LiveRegs, FramePtrReg); + getVGPRSpillLaneOrTempRegister(MF, LiveUnits, FramePtrReg); } if (TRI->hasBasePointer(MF)) { Register BasePtrReg = TRI->getBaseRegister(); assert(!MFI->hasPrologEpilogSGPRSpillEntry(BasePtrReg) && "Re-reserving spill slot for BP"); - getVGPRSpillLaneOrTempRegister(MF, LiveRegs, BasePtrReg); + getVGPRSpillLaneOrTempRegister(MF, LiveUnits, BasePtrReg); } } diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h @@ -23,7 +23,7 @@ class GCNSubtarget; class LiveIntervals; -class LivePhysRegs; +class LiveRegUnits; class RegisterBank; struct SGPRSpillBuilder; @@ -415,14 +415,14 @@ // Insert spill or restore instructions. // When lowering spill pseudos, the RegScavenger should be set. // For creating spill instructions during frame lowering, where no scavenger - // is available, LiveRegs can be used. + // is available, LiveUnits can be used. void buildSpillLoadStore(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, unsigned LoadStoreOp, int Index, Register ValueReg, bool ValueIsKill, MCRegister ScratchOffsetReg, int64_t InstrOffset, MachineMemOperand *MMO, RegScavenger *RS, - LivePhysRegs *LiveRegs = nullptr) const; + LiveRegUnits *LiveUnits = nullptr) const; // Return alignment in register file of first register in a register tuple. unsigned getRegClassAlignmentNumBits(const TargetRegisterClass *RC) const { diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp @@ -19,7 +19,7 @@ #include "SIMachineFunctionInfo.h" #include "SIRegisterInfo.h" #include "llvm/CodeGen/LiveIntervals.h" -#include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/RegisterScavenging.h" @@ -1307,8 +1307,8 @@ MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, unsigned LoadStoreOp, int Index, Register ValueReg, bool IsKill, MCRegister ScratchOffsetReg, int64_t InstOffset, MachineMemOperand *MMO, - RegScavenger *RS, LivePhysRegs *LiveRegs) const { - assert((!RS || !LiveRegs) && "Only RS or LiveRegs can be set but not both"); + RegScavenger *RS, LiveRegUnits *LiveUnits) const { + assert((!RS || !LiveUnits) && "Only RS or LiveUnits can be set but not both"); MachineFunction *MF = MBB.getParent(); const SIInstrInfo *TII = ST.getInstrInfo(); @@ -1396,7 +1396,7 @@ SOffset = MCRegister(); // We don't have access to the register scavenger if this function is called - // during PEI::scavengeFrameVirtualRegs() so use LiveRegs in this case. + // during PEI::scavengeFrameVirtualRegs() so use LiveUnits in this case. // TODO: Clobbering SCC is not necessary for scratch instructions in the // entry. if (RS) { @@ -1404,10 +1404,10 @@ // Piggy back on the liveness scan we just did see if SCC is dead. CanClobberSCC = !RS->isRegUsed(AMDGPU::SCC); - } else if (LiveRegs) { - CanClobberSCC = !LiveRegs->contains(AMDGPU::SCC); + } else if (LiveUnits) { + CanClobberSCC = LiveUnits->available(AMDGPU::SCC); for (MCRegister Reg : AMDGPU::SGPR_32RegClass) { - if (LiveRegs->available(MF->getRegInfo(), Reg)) { + if (LiveUnits->available(MF->getRegInfo(), Reg)) { SOffset = Reg; break; } @@ -1423,9 +1423,9 @@ if (RS) { TmpOffsetVGPR = RS->scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI, false, 0); } else { - assert(LiveRegs); + assert(LiveUnits); for (MCRegister Reg : AMDGPU::VGPR_32RegClass) { - if (LiveRegs->available(MF->getRegInfo(), Reg)) { + if (LiveUnits->available(MF->getRegInfo(), Reg)) { TmpOffsetVGPR = Reg; break; }