AMDGPU: add +xnack feature

[nhaehnle]

Summary

Enabling this feature will account for the two SGPRs used by the hardware
to store the XNACK_MASK physically.

The hardware only requires this reservation when the XNACK feature is
explicitly enabled. At some point, HSA will probably want to do that, but
it does increase SGPR register pressure, so leave it disabled by default
for now (but do add a small test).

[arsenm]

LGTM.

Is there some bit that should be set if it is enabled in the binary?

[nhaehnle]

I've added a hunk that sets the code_properties bit so it does not get forgotten. According to Tom, this enum isn't uptodate anyway.

llvm/trunk/lib/Target/AMDGPU/AMDGPU.td

def FeatureFlatForGlobal : SubtargetFeature<"flat-for-global",
         "true",
         "Force to generate flat instruction for global">;
 
 def FeatureFlatAddressSpace : SubtargetFeature<"flat-address-space",
         "FlatAddressSpace",
         "true",
         "Support flat address space">;
 
+def FeatureXNACK : SubtargetFeature<"xnack",
+        "EnableXNACK",
+        "true",
+        "Enable XNACK support">;
+
 def FeatureVGPRSpilling : SubtargetFeature<"vgpr-spilling",
         "EnableVGPRSpilling",
         "true",
         "Enable spilling of VGPRs to scratch memory">;
 
 def FeatureSGPRInitBug : SubtargetFeature<"sgpr-init-bug",
         "SGPRInitBug",
         "true",

llvm/trunk/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp

for (const MachineInstr &MI : MBB) {
           MaxSGPR = maxUsed > MaxSGPR ? maxUsed : MaxSGPR;
         } else {
           MaxVGPR = maxUsed > MaxVGPR ? maxUsed : MaxVGPR;
         }
       }
     }
   }
 
-  if (VCCUsed || FlatUsed)
+  if (VCCUsed || FlatUsed || STM.isXNACKEnabled()) {
     MaxSGPR += 2;
 
-  if (FlatUsed) {
+    if (FlatUsed)
       MaxSGPR += 2;
-    // 2 additional for VI+.
-    if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
+
+    if (STM.isXNACKEnabled())
       MaxSGPR += 2;
   }
 
   // We found the maximum register index. They start at 0, so add one to get the
   // number of registers.
   ProgInfo.NumVGPR = MaxVGPR + 1;
   ProgInfo.NumSGPR = MaxSGPR + 1;
 
void AMDGPUAsmPrinter::EmitAmdKernelCodeT(const MachineFunction &MF,
   if (MFI->hasGridWorkgroupCountZ()) {
     header.code_properties |=
       AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_Z;
   }
 
   if (MFI->hasDispatchPtr())
     header.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR;
 
+  if (STM.isXNACKEnabled())
+    header.code_properties |= AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED;
+
   header.kernarg_segment_byte_size = MFI->ABIArgOffset;
   header.wavefront_sgpr_count = KernelInfo.NumSGPR;
   header.workitem_vgpr_count = KernelInfo.NumVGPR;
   header.workitem_private_segment_byte_size = KernelInfo.ScratchSize;
   header.workgroup_group_segment_byte_size = KernelInfo.LDSSize;
 
   AMDGPUTargetStreamer *TS =
       static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer());

llvm/trunk/lib/Target/AMDGPU/AMDGPUSubtarget.h

private:
   bool CaymanISA;
   bool FlatAddressSpace;
   bool FlatForGlobal;
   bool EnableIRStructurizer;
   bool EnablePromoteAlloca;
   bool EnableIfCvt;
   bool EnableLoadStoreOpt;
   bool EnableUnsafeDSOffsetFolding;
+  bool EnableXNACK;
   unsigned WavefrontSize;
   bool CFALUBug;
   int LocalMemorySize;
   bool EnableVGPRSpilling;
   bool SGPRInitBug;
   bool IsGCN;
   bool GCN1Encoding;
   bool GCN3Encoding;
public:
   bool r600ALUEncoding() const {
     return R600ALUInst;
   }
   bool isAmdHsaOS() const {
     return TargetTriple.getOS() == Triple::AMDHSA;
   }
   bool isVGPRSpillingEnabled(const SIMachineFunctionInfo *MFI) const;
 
+  bool isXNACKEnabled() const {
+    return EnableXNACK;
+  }
+
   unsigned getMaxWavesPerCU() const {
     if (getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS)
       return 10;
 
     // FIXME: Not sure what this is for other subtagets.
     llvm_unreachable("do not know max waves per CU for this subtarget.");
   }
 

llvm/trunk/lib/Target/AMDGPU/AMDGPUSubtarget.cpp

AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
                                  TargetMachine &TM)
     : AMDGPUGenSubtargetInfo(TT, GPU, FS), DevName(GPU), Is64bit(false),
       DumpCode(false), R600ALUInst(false), HasVertexCache(false),
       TexVTXClauseSize(0), Gen(AMDGPUSubtarget::R600), FP64(false),
       FP64Denormals(false), FP32Denormals(false), FastFMAF32(false),
       CaymanISA(false), FlatAddressSpace(false), FlatForGlobal(false),
       EnableIRStructurizer(true), EnablePromoteAlloca(false), EnableIfCvt(true),
       EnableLoadStoreOpt(false), EnableUnsafeDSOffsetFolding(false),
+      EnableXNACK(false),
       WavefrontSize(0), CFALUBug(false), LocalMemorySize(0),
       EnableVGPRSpilling(false), SGPRInitBug(false), IsGCN(false),
       GCN1Encoding(false), GCN3Encoding(false), CIInsts(false), LDSBankCount(0),
       IsaVersion(ISAVersion0_0_0), EnableHugeScratchBuffer(false),
       FrameLowering(nullptr),
       InstrItins(getInstrItineraryForCPU(GPU)), TargetTriple(TT) {
 
   initializeSubtargetDependencies(TT, GPU, FS);

llvm/trunk/lib/Target/AMDGPU/SIRegisterInfo.cpp

for (; R.isValid(); ++R)
     Reserved.set(*R);
 }
 
 unsigned SIRegisterInfo::reservedPrivateSegmentBufferReg(
   const MachineFunction &MF) const {
   const AMDGPUSubtarget &ST = MF.getSubtarget<AMDGPUSubtarget>();
   if (ST.hasSGPRInitBug()) {
     unsigned BaseIdx = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG - 4 - 4;
+    if (ST.isXNACKEnabled())
+      BaseIdx -= 4;
+
     unsigned BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx));
     return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SReg_128RegClass);
   }
 
   if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
-    // 98/99 need to be reserved for flat_scr, and 100/101 for vcc. This is the
-    // next sgpr128 down.
+    // 98/99 need to be reserved for flat_scr or 96/97 for flat_scr and
+    // 98/99 for xnack_mask, and 100/101 for vcc. This is the next sgpr128 down
+    // either way.
     return AMDGPU::SGPR92_SGPR93_SGPR94_SGPR95;
   }
 
   return AMDGPU::SGPR96_SGPR97_SGPR98_SGPR99;
 }
 
 unsigned SIRegisterInfo::reservedPrivateSegmentWaveByteOffsetReg(
   const MachineFunction &MF) const {
   const AMDGPUSubtarget &ST = MF.getSubtarget<AMDGPUSubtarget>();
   if (ST.hasSGPRInitBug()) {
-    unsigned Idx = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG - 4 - 5;
+    unsigned Idx;
+
+    if (!ST.isXNACKEnabled())
+      Idx = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG - 4 - 5;
+    else
+      Idx = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG - 6 - 1;
+
     return AMDGPU::SGPR_32RegClass.getRegister(Idx);
   }
 
   if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
+    if (!ST.isXNACKEnabled()) {
       // Next register before reservations for flat_scr and vcc.
       return AMDGPU::SGPR97;
+    } else {
+      // Next register before reservations for flat_scr, xnack_mask, vcc,
+      // and scratch resource.
+      return AMDGPU::SGPR91;
+    }
   }
 
   return AMDGPU::SGPR95;
 }
 
 BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   BitVector Reserved(getNumRegs());
   Reserved.set(AMDGPU::INDIRECT_BASE_ADDR);
BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
 
   const AMDGPUSubtarget &ST = MF.getSubtarget<AMDGPUSubtarget>();
 
   if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
     // SI/CI have 104 SGPRs. VI has 102. We need to shift down the reservation
     // for VCC/FLAT_SCR.
     reserveRegisterTuples(Reserved, AMDGPU::SGPR98_SGPR99);
     reserveRegisterTuples(Reserved, AMDGPU::SGPR100_SGPR101);
+
+    if (ST.isXNACKEnabled())
+      reserveRegisterTuples(Reserved, AMDGPU::SGPR96_SGPR97);
   }
 
   // Tonga and Iceland can only allocate a fixed number of SGPRs due
   // to a hw bug.
   if (ST.hasSGPRInitBug()) {
     unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
     // Reserve some SGPRs for FLAT_SCRATCH and VCC (4 SGPRs).
-    // Assume XNACK_MASK is unused.
     unsigned Limit = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG - 4;
 
+    if (ST.isXNACKEnabled())
+      Limit -= 2;
+
     for (unsigned i = Limit; i < NumSGPRs; ++i) {
       unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i);
       reserveRegisterTuples(Reserved, Reg);
     }
   }
 
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
 

llvm/trunk/test/CodeGen/AMDGPU/flat-scratch-reg.ll

-; RUN: llc < %s -march=amdgcn -mcpu=kaveri -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=CI
-; RUN: llc < %s -march=amdgcn -mcpu=fiji -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=VI
+; RUN: llc < %s -march=amdgcn -mcpu=kaveri -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=CI --check-prefix=NO-XNACK
+; RUN: llc < %s -march=amdgcn -mcpu=fiji -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=VI --check-prefix=NO-XNACK
+; RUN: llc < %s -march=amdgcn -mcpu=carrizo -mattr=+xnack -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=XNACK
 
 ; GCN-LABEL: {{^}}no_vcc_no_flat:
-; GCN: ; NumSgprs: 8
+; NO-XNACK: ; NumSgprs: 8
+; XNACK: ; NumSgprs: 12
 define void @no_vcc_no_flat() {
 entry:
   call void asm sideeffect "", "~{SGPR7}"()
   ret void
 }
 
 ; GCN-LABEL: {{^}}vcc_no_flat:
-; GCN: ; NumSgprs: 10
+; NO-XNACK: ; NumSgprs: 10
+; XNACK: ; NumSgprs: 12
 define void @vcc_no_flat() {
 entry:
   call void asm sideeffect "", "~{SGPR7},~{VCC}"()
   ret void
 }
 
 ; GCN-LABEL: {{^}}no_vcc_flat:
 ; CI: ; NumSgprs: 12
-; VI: ; NumSgprs: 14
+; VI: ; NumSgprs: 12
+; XNACK: ; NumSgprs: 14
 define void @no_vcc_flat() {
 entry:
   call void asm sideeffect "", "~{SGPR7},~{FLAT_SCR}"()
   ret void
 }
 
 ; GCN-LABEL: {{^}}vcc_flat:
 ; CI: ; NumSgprs: 12
-; VI: ; NumSgprs: 14
+; VI: ; NumSgprs: 12
+; XNACK: ; NumSgprs: 14
 define void @vcc_flat() {
 entry:
   call void asm sideeffect "", "~{SGPR7},~{VCC},~{FLAT_SCR}"()
   ret void
 }