AMDGPU/SI: xnack_mask is always reserved on VI

[nhaehnle]

Summary

Somehow, I first interpreted the docs as saying space for xnack_mask is only
reserved when XNACK is enabled via SH_MEM_CONFIG. I felt uneasy about this and
went back to actually test what is happening, and it turns out that xnack_mask
is always reserved at least on Tonga and Carrizo, in the sense that flat_scr
is always fixed below the SGPRs that are used to implement xnack_mask, whether
or not they are actually used.

I confirmed this by writing a shader using inline assembly to tease out the
aliasing between flat_scratch and regular SGPRs. For example, on Tonga, where
we fix the number of SGPRs to 80, s[74:75] aliases flat_scratch (so
xnack_mask is s[76:77] and vcc is s[78:79]).

This patch changes both the calculation of the total number of SGPRs and the
various register reservations to account for this. In particular, when the
XNACK feature isn't used, the gap between flat_scr and vcc can be used. This
seems to be in line with the hardware documentation.

Note that previously, even before my earlier change in r256794, the SGPRs that
alias to xnack_mask could end up being used as well when flat_scr was unused
and the total number of SGPRs happened to fall on the right alignment
(e.g. highest regular SGPR being used s29 and VCC used would lead to number
of SGPRs being 32, where s28 and s29 alias with xnack_mask). So if there
were some conflict due to such aliasing, we should have noticed that already.

[nhaehnle]

Sorry for the churn. We cannot leave a gap when the xnack feature is unused
because that breaks with the accounting done for the number of SGPRs.

[• tstellarAMD]

I had to stare at the extra register computation for a while to convince myself it was correct. Thanks for fixing this.

LGTM.

lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp

for (const MachineInstr &MI : MBB) {
           MaxSGPR = maxUsed > MaxSGPR ? maxUsed : MaxSGPR;
         } else {
           MaxVGPR = maxUsed > MaxVGPR ? maxUsed : MaxVGPR;
         }
       }
     }
   }
 
-  if (VCCUsed || FlatUsed || STM.isXNACKEnabled()) {
-    MaxSGPR += 2;
+  unsigned ExtraSGPRs = 0;
 
-    if (FlatUsed)
-      MaxSGPR += 2;
+  if (VCCUsed)
+    ExtraSGPRs = 2;
 
+  if (STM.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS) {
+    if (FlatUsed)
+      ExtraSGPRs = 4;
+  } else {
     if (STM.isXNACKEnabled())
-      MaxSGPR += 2;
+      ExtraSGPRs = 4;
+
+    if (FlatUsed)
+      ExtraSGPRs = 6;
   }
 
+  MaxSGPR += ExtraSGPRs;
+
   // We found the maximum register index. They start at 0, so add one to get the
   // number of registers.
   ProgInfo.NumVGPR = MaxVGPR + 1;
   ProgInfo.NumSGPR = MaxSGPR + 1;
 
   if (STM.hasSGPRInitBug()) {
     if (ProgInfo.NumSGPR > AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG) {
       LLVMContext &Ctx = MF.getFunction()->getContext();

lib/Target/AMDGPU/SIRegisterInfo.cpp

void SIRegisterInfo::reserveRegisterTuples(BitVector &Reserved, unsigned Reg) const {
   for (; R.isValid(); ++R)
     Reserved.set(*R);
 }
 
 unsigned SIRegisterInfo::reservedPrivateSegmentBufferReg(
   const MachineFunction &MF) const {
   const AMDGPUSubtarget &ST = MF.getSubtarget<AMDGPUSubtarget>();
   if (ST.hasSGPRInitBug()) {
-    unsigned BaseIdx = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG - 4 - 4;
-    if (ST.isXNACKEnabled())
-      BaseIdx -= 4;
-
+    // Leave space for flat_scr, xnack_mask, vcc, and alignment
+    unsigned BaseIdx = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG - 8 - 4;
     unsigned BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx));
     return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SReg_128RegClass);
   }
 
   if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
-    // 98/99 need to be reserved for flat_scr or 96/97 for flat_scr and
-    // 98/99 for xnack_mask, and 100/101 for vcc. This is the next sgpr128 down
-    // either way.
+    // 96/97 need to be reserved for flat_scr, 98/99 for xnack_mask, and
+    // 100/101 for vcc. This is the next sgpr128 down.
     return AMDGPU::SGPR92_SGPR93_SGPR94_SGPR95;
   }
 
   return AMDGPU::SGPR96_SGPR97_SGPR98_SGPR99;
 }
 
 unsigned SIRegisterInfo::reservedPrivateSegmentWaveByteOffsetReg(
   const MachineFunction &MF) const {
   const AMDGPUSubtarget &ST = MF.getSubtarget<AMDGPUSubtarget>();
   if (ST.hasSGPRInitBug()) {
     unsigned Idx;
 
-    if (!ST.isXNACKEnabled())
-      Idx = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG - 4 - 5;
-    else
+    if (ST.isXNACKEnabled()) {
       Idx = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG - 6 - 1;
+    } else {
+      // Use the gap that is left by the unused xnack_mask.
+      Idx = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG - 2 - 1;
+    }
 
     return AMDGPU::SGPR_32RegClass.getRegister(Idx);
   }
 
   if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
     if (!ST.isXNACKEnabled()) {
-      // Next register before reservations for flat_scr and vcc.
-      return AMDGPU::SGPR97;
+      // Use the gap that is left by the unused xnack_mask.
+      return AMDGPU::SGPR99;
     } else {
       // Next register before reservations for flat_scr, xnack_mask, vcc,
       // and scratch resource.
       return AMDGPU::SGPR91;
     }
   }
 
   return AMDGPU::SGPR95;
BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   // will physically contain VCC.
   reserveRegisterTuples(Reserved, AMDGPU::SGPR102_SGPR103);
 
   const AMDGPUSubtarget &ST = MF.getSubtarget<AMDGPUSubtarget>();
 
   if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
     // SI/CI have 104 SGPRs. VI has 102. We need to shift down the reservation
     // for VCC/FLAT_SCR.
+    reserveRegisterTuples(Reserved, AMDGPU::SGPR96_SGPR97);
+    if (ST.isXNACKEnabled())
       reserveRegisterTuples(Reserved, AMDGPU::SGPR98_SGPR99);
     reserveRegisterTuples(Reserved, AMDGPU::SGPR100_SGPR101);
-
-    if (ST.isXNACKEnabled())
-      reserveRegisterTuples(Reserved, AMDGPU::SGPR96_SGPR97);
   }
 
   // Tonga and Iceland can only allocate a fixed number of SGPRs due
   // to a hw bug.
   if (ST.hasSGPRInitBug()) {
     unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
-    // Reserve some SGPRs for FLAT_SCRATCH and VCC (4 SGPRs).
-    unsigned Limit = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG - 4;
-
-    if (ST.isXNACKEnabled())
-      Limit -= 2;
+    unsigned Limit = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG;
 
     for (unsigned i = Limit; i < NumSGPRs; ++i) {
       unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i);
       reserveRegisterTuples(Reserved, Reg);
     }
+
+    // Reserve some SGPRs for FLAT_SCRATCH, XNACK_MASK, and VCC (6 SGPRs).
+    reserveRegisterTuples(Reserved, AMDGPU::SGPR74_SGPR75);
+    if (ST.isXNACKEnabled())
+      reserveRegisterTuples(Reserved, AMDGPU::SGPR76_SGPR77);
+    reserveRegisterTuples(Reserved, AMDGPU::SGPR78_SGPR79);
   }
 
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
 
   unsigned ScratchWaveOffsetReg = MFI->getScratchWaveOffsetReg();
   if (ScratchWaveOffsetReg != AMDGPU::NoRegister) {
     // Reserve 1 SGPR for scratch wave offset in case we need to spill.
     reserveRegisterTuples(Reserved, ScratchWaveOffsetReg);

test/CodeGen/AMDGPU/flat-scratch-reg.ll

 ; RUN: llc < %s -march=amdgcn -mcpu=kaveri -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=CI --check-prefix=NO-XNACK
 ; RUN: llc < %s -march=amdgcn -mcpu=fiji -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=VI --check-prefix=NO-XNACK
-; RUN: llc < %s -march=amdgcn -mcpu=carrizo -mattr=+xnack -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=XNACK
+; RUN: llc < %s -march=amdgcn -mcpu=carrizo -mattr=+xnack -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=VI --check-prefix=XNACK
 
 ; GCN-LABEL: {{^}}no_vcc_no_flat:
 ; NO-XNACK: ; NumSgprs: 8
 ; XNACK: ; NumSgprs: 12
 define void @no_vcc_no_flat() {
 entry:
   call void asm sideeffect "", "~{SGPR7}"()
   ret void
 }
 
 ; GCN-LABEL: {{^}}vcc_no_flat:
 ; NO-XNACK: ; NumSgprs: 10
 ; XNACK: ; NumSgprs: 12
 define void @vcc_no_flat() {
 entry:
   call void asm sideeffect "", "~{SGPR7},~{VCC}"()
   ret void
 }
 
 ; GCN-LABEL: {{^}}no_vcc_flat:
 ; CI: ; NumSgprs: 12
-; VI: ; NumSgprs: 12
-; XNACK: ; NumSgprs: 14
+; VI: ; NumSgprs: 14
 define void @no_vcc_flat() {
 entry:
   call void asm sideeffect "", "~{SGPR7},~{FLAT_SCR}"()
   ret void
 }
 
 ; GCN-LABEL: {{^}}vcc_flat:
 ; CI: ; NumSgprs: 12
-; VI: ; NumSgprs: 12
-; XNACK: ; NumSgprs: 14
+; VI: ; NumSgprs: 14
 define void @vcc_flat() {
 entry:
   call void asm sideeffect "", "~{SGPR7},~{VCC},~{FLAT_SCR}"()
   ret void
 }