diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp --- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp @@ -778,7 +778,7 @@ std::optional FPSaveIndex = FuncInfo->FramePointerSaveIndex; std::optional BPSaveIndex = FuncInfo->BasePointerSaveIndex; - // VGPRs used for SGPR->VGPR spills + // Spill Whole-Wave Mode VGPRs. for (const auto &Reg : FuncInfo->getWWMSpills()) { Register VGPR = Reg.first; int FI = Reg.second; @@ -789,15 +789,6 @@ buildPrologSpill(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MBBI, DL, VGPR, FI); } - for (auto ReservedWWM : FuncInfo->wwmAllocation()) { - if (!ScratchExecCopy) - ScratchExecCopy = - buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, DL, /*IsProlog*/ true); - - buildPrologSpill(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MBBI, DL, - std::get<0>(ReservedWWM), std::get<1>(ReservedWWM)); - } - if (ScratchExecCopy) { // FIXME: Split block and make terminator. unsigned ExecMov = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; @@ -1064,15 +1055,6 @@ FI); } - for (auto ReservedWWM : FuncInfo->wwmAllocation()) { - if (!ScratchExecCopy) - ScratchExecCopy = - buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, DL, /*IsProlog*/ false); - - buildEpilogRestore(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MBBI, DL, - std::get<0>(ReservedWWM), std::get<1>(ReservedWWM)); - } - if (ScratchExecCopy) { // FIXME: Split block and make terminator. unsigned ExecMov = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; @@ -1120,9 +1102,13 @@ MachineRegisterInfo &MRI = MF.getRegInfo(); SIMachineFunctionInfo *FuncInfo = MF.getInfo(); + // Allocate spill slots for WWM reserved VGPRs. if (!FuncInfo->isEntryFunction()) { - // Spill VGPRs used for Whole Wave Mode - FuncInfo->allocateWWMReservedSpillSlots(MFI, *TRI); + for (Register Reg : FuncInfo->getWWMReservedRegs()) { + const TargetRegisterClass *RC = TRI->getPhysRegClass(Reg); + FuncInfo->allocateWWMSpill(MF, Reg, TRI->getSpillSize(*RC), + TRI->getSpillAlign(*RC)); + } } const bool SpillVGPRToAGPR = ST.hasMAIInsts() && FuncInfo->hasSpilledVGPRs() @@ -1306,8 +1292,8 @@ FrameInfo.hasCalls() && (SavedVGPRs.any() || !allStackObjectsAreDead(FrameInfo)); - // VGPRs used for SGPR spilling need to be specially inserted in the prolog, - // so don't allow the default insertion to handle them. + // The Whole-Wave VGPRs need to be specially inserted in the prolog, so don't + // allow the default insertion to handle them. for (auto &Reg : MFI->getWWMSpills()) SavedVGPRs.reset(Reg.first); diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h @@ -437,24 +437,6 @@ bool IsDead = false; }; - // Track VGPRs reserved for WWM. - SmallSetVector WWMReservedRegs; - - /// Track stack slots used for save/restore of reserved WWM VGPRs in the - /// prolog/epilog. - - /// FIXME: This is temporary state only needed in PrologEpilogInserter, and - /// doesn't really belong here. It does not require serialization - SmallVector WWMReservedFrameIndexes; - - void allocateWWMReservedSpillSlots(MachineFrameInfo &MFI, - const SIRegisterInfo &TRI); - - auto wwmAllocation() const { - assert(WWMReservedRegs.size() == WWMReservedFrameIndexes.size()); - return zip(WWMReservedRegs, WWMReservedFrameIndexes); - } - private: // Track VGPR + wave index for each subregister of the SGPR spilled to // frameindex key. @@ -470,6 +452,13 @@ // the VGPR and its stack slot index. WWMSpillsMap WWMSpills; + using ReservedRegSet = SmallSetVector; + // To track the VGPRs reserved for WWM instructions. They get stack slots + // later during PrologEpilogInserter and get added into the superset WWMSpills + // for actual spilling. A separate set makes the register reserved part and + // the serialization easier. + ReservedRegSet WWMReservedRegs; + DenseMap VGPRToAGPRSpills; // AGPRs used for VGPR spills. @@ -521,9 +510,7 @@ PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange); - void reserveWWMRegister(Register Reg) { - WWMReservedRegs.insert(Reg); - } + void reserveWWMRegister(Register Reg) { WWMReservedRegs.insert(Reg); } AMDGPU::SIModeRegisterDefaults getMode() const { return Mode; @@ -539,6 +526,7 @@ ArrayRef getSGPRSpillVGPRs() const { return SpillVGPRs; } const WWMSpillsMap &getWWMSpills() const { return WWMSpills; } + const ReservedRegSet &getWWMReservedRegs() const { return WWMReservedRegs; } void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size = 4, Align Alignment = Align(4)); diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp @@ -473,20 +473,6 @@ return HaveSGPRToMemory; } -void SIMachineFunctionInfo::allocateWWMReservedSpillSlots( - MachineFrameInfo &MFI, const SIRegisterInfo &TRI) { - assert(WWMReservedFrameIndexes.empty()); - - WWMReservedFrameIndexes.resize(WWMReservedRegs.size()); - - int I = 0; - for (Register VGPR : WWMReservedRegs) { - const TargetRegisterClass *RC = TRI.getPhysRegClass(VGPR); - WWMReservedFrameIndexes[I++] = MFI.CreateSpillStackObject( - TRI.getSpillSize(*RC), TRI.getSpillAlign(*RC)); - } -} - int SIMachineFunctionInfo::getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI) { if (ScavengeFI) @@ -613,7 +599,7 @@ BytesInStackArgArea(MFI.getBytesInStackArgArea()), ReturnsVoid(MFI.returnsVoid()), ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)), Mode(MFI.getMode()) { - for (Register Reg : MFI.WWMReservedRegs) + for (Register Reg : MFI.getWWMReservedRegs()) WWMReservedRegs.push_back(regToString(Reg, TRI)); if (MFI.getVGPRForAGPRCopy()) diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp @@ -701,7 +701,7 @@ reserveRegisterTuples(Reserved, MFI->getVGPRForAGPRCopy()); } - for (Register Reg : MFI->WWMReservedRegs) + for (Register Reg : MFI->getWWMReservedRegs()) reserveRegisterTuples(Reserved, Reg); // FIXME: Stop using reserved registers for this.