Index: llvm/lib/Target/AMDGPU/GCNSchedStrategy.h =================================================================== --- llvm/lib/Target/AMDGPU/GCNSchedStrategy.h +++ llvm/lib/Target/AMDGPU/GCNSchedStrategy.h @@ -75,6 +75,10 @@ // track register pressure for actual scheduling heuristics. bool HasHighPressure; + // Schedule known to have excess register pressure. Be more conservative in + // increasing ILP and preserving VGPRs. + bool KnownExcessRP = false; + // An error margin is necessary because of poor performance of the generic RP // tracker and can be adjusted up for tuning heuristics to try and more // aggressively reduce register pressure. @@ -296,6 +300,11 @@ // Returns true if scheduling should be reverted. virtual bool shouldRevertScheduling(unsigned WavesAfter); + // Returns true if current region has known excess pressure. + bool isRegionWithExcessRP() const { + return DAG.RegionsWithExcessRP[RegionIdx]; + } + // Returns true if the new schedule may result in more spilling. bool mayCauseSpilling(unsigned WavesAfter); Index: llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp =================================================================== --- llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp +++ llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp @@ -70,8 +70,20 @@ TargetOccupancy = MFI.getOccupancy(); SGPRCriticalLimit = std::min(ST.getMaxNumSGPRs(TargetOccupancy, true), SGPRExcessLimit); - VGPRCriticalLimit = - std::min(ST.getMaxNumVGPRs(TargetOccupancy), VGPRExcessLimit); + + if (!KnownExcessRP) { + VGPRCriticalLimit = + std::min(ST.getMaxNumVGPRs(TargetOccupancy), VGPRExcessLimit); + } else { + // This is similar to ST.getMaxNumVGPRs(TargetOccupancy) result except + // returns a reasonably small number for targets with lots of VGPRs, such + // as GFX10 and GFX11. + unsigned Granule = AMDGPU::IsaInfo::getVGPRAllocGranule(&ST); + unsigned Addressable = AMDGPU::IsaInfo::getAddressableNumVGPRs(&ST); + unsigned VGPRBudget = alignDown(Addressable / TargetOccupancy, Granule); + VGPRBudget = std::max(VGPRBudget, Granule); + VGPRCriticalLimit = std::min(VGPRBudget, VGPRExcessLimit); + } // Subtract error margin from register limits and avoid overflow. SGPRCriticalLimit = @@ -603,6 +615,8 @@ for (auto Region : Regions) { RegionBegin = Region.first; RegionEnd = Region.second; + S.KnownExcessRP = Stage->isRegionWithExcessRP(); + // Setup for scheduling the region and check whether it should be skipped. if (!Stage->initGCNRegion()) { Stage->advanceRegion(); @@ -1135,7 +1149,7 @@ bool GCNSchedStage::mayCauseSpilling(unsigned WavesAfter) { if (WavesAfter <= MFI.getMinWavesPerEU() && !PressureAfter.less(ST, PressureBefore) && - DAG.RegionsWithExcessRP[RegionIdx]) { + isRegionWithExcessRP()) { LLVM_DEBUG(dbgs() << "New pressure will result in more spilling.\n"); return true; }