AMDGPU: Add options to override SGPR and VGPR blocks

[kzhuravl]

Summary

This is needed for experimentation

[t-tye]

llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

4815

Should this set to max of calculated and overridden value? We do not want to override to force fewer registers than the code actually uses. Same comment for SGPR.

[kzhuravl]

llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

4815

Instead of setting max, can we return error saying "cannot override because isa is using more registers"?

[t-tye]

llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

4815

That works for me.

[kzhuravl]

Address review feedback.

[kzhuravl]

ping

[kzhuravl]

ping

[arsenm]

Is this still relevant? Would this be better served by making the lower bound of amdgpu-waves-per-eu artificially increase the register usage?

[kzhuravl]

In D108322#3932212, @arsenm wrote:

Is this still relevant? Would this be better served by making the lower bound of amdgpu-waves-per-eu artificially increase the register usage?

No longer relevant.

llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp

if (MFI->getLDSSize() > static_cast<unsigned>(STM.getLocalMemorySize())) {
     LLVMContext &Ctx = MF.getFunction().getContext();
     DiagnosticInfoResourceLimit Diag(MF.getFunction(), "local memory",
                                      MFI->getLDSSize(), DS_Error);
     Ctx.diagnose(Diag);
   }
 
   ProgInfo.SGPRBlocks = IsaInfo::getNumSGPRBlocks(
       &STM, ProgInfo.NumSGPRsForWavesPerEU);
+  if (auto OverrideSGPRBlocks = AMDGPU::overrideSGPRBlocks()) {
+    if (!isUInt<amdhsa::COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_WIDTH>(
+            *OverrideSGPRBlocks)) {
+      OutContext.reportError({}, "requested override of sgpr blocks (" +
+                                 Twine(*OverrideSGPRBlocks) + ") is out " \
+                                 "of range");
+    } else if (ProgInfo.SGPRBlocks > *OverrideSGPRBlocks) {
+      OutContext.reportError({}, "requested override of sgpr blocks (" +
+                                 Twine(*OverrideSGPRBlocks) + ") is smaller " \
+                                 "than the required number of sgpr blocks (" +
+                                 Twine(ProgInfo.SGPRBlocks) + ")");
+    } else {
+      ProgInfo.SGPRBlocks = *OverrideSGPRBlocks;
+    }
+  }
+
   ProgInfo.VGPRBlocks = IsaInfo::getNumVGPRBlocks(
       &STM, ProgInfo.NumVGPRsForWavesPerEU);
+  if (auto OverrideVGPRBlocks = AMDGPU::overrideVGPRBlocks()) {
+    if (!isUInt<amdhsa::COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_WIDTH>(
+            *OverrideVGPRBlocks)) {
+      OutContext.reportError({}, "requested override of vgpr blocks (" +
+                                 Twine(*OverrideVGPRBlocks) + ") is out " \
+                                 "of range");
+    } else if (ProgInfo.VGPRBlocks > *OverrideVGPRBlocks) {
+      OutContext.reportError({}, "requested override of vgpr blocks (" +
+                                 Twine(*OverrideVGPRBlocks) + ") is smaller " \
+                                 "than the required number of vgpr blocks (" +
+                                 Twine(ProgInfo.VGPRBlocks) + ")");
+    } else {
+      ProgInfo.VGPRBlocks = *OverrideVGPRBlocks;
+    }
+  }
 
   const SIModeRegisterDefaults Mode = MFI->getMode();
 
   // Set the value to initialize FP_ROUND and FP_DENORM parts of the mode
   // register.
   ProgInfo.FloatMode = getFPMode(Mode);
 
   ProgInfo.IEEEMode = Mode.IEEE;

llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

#undef PARSE_BITS_ENTRY
   unsigned SGPRBlocks;
   if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
                          getTargetStreamer().getTargetID()->isXnackOnOrAny(),
                          EnableWavefrontSize32, NextFreeVGPR,
                          VGPRRange, NextFreeSGPR, SGPRRange, VGPRBlocks,
                          SGPRBlocks))
     return true;
 
+  if (auto OverrideVGPRBlocks = AMDGPU::overrideVGPRBlocks()) {
+    if (VGPRBlocks > *OverrideVGPRBlocks) {
+      return TokError("requested override of vgpr blocks (" +
+                      Twine(*OverrideVGPRBlocks) + ") is smaller " \
+                      "than the required number of vgpr blocks (" +
+                      Twine(VGPRBlocks) + ")");
+    }
+    VGPRBlocks = *OverrideVGPRBlocks;
+  }
+  if (auto OverrideSGPRBlocks = AMDGPU::overrideSGPRBlocks()) {
+    if (SGPRBlocks > *OverrideSGPRBlocks) {
+      return TokError("requested override of sgpr blocks (" +
+                      Twine(*OverrideSGPRBlocks) + ") is smaller " \
+                      "than the required number of sgpr blocks (" +
+                      Twine(SGPRBlocks) + ")");
+    }
+    SGPRBlocks = *OverrideSGPRBlocks;
+  }
+
   if (!isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_WIDTH>(
           VGPRBlocks))
     return OutOfRangeError(VGPRRange);
   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
                   COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, VGPRBlocks);
 
   if (!isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_WIDTH>(
           SGPRBlocks))

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h

 bool isHsaAbiVersion3(const MCSubtargetInfo *STI);
 /// \returns True if HSA OS ABI Version identification is 4,
 /// false otherwise.
 bool isHsaAbiVersion4(const MCSubtargetInfo *STI);
 /// \returns True if HSA OS ABI Version identification is 3 or 4,
 /// false otherwise.
 bool isHsaAbiVersion3Or4(const MCSubtargetInfo *STI);
 
+/// \returns Requested override of VGPR blocks, which can be directly written
+/// into COMPUTE_PGM_RSRC1.VGPRS. None if override is not requested.
+Optional<unsigned> overrideVGPRBlocks();
+
+/// \returns Requested override of SGPR blocks, which can be directly written
+/// into COMPUTE_PGM_RSRC1.SGPRS. None if override is not requested.
+Optional<unsigned> overrideSGPRBlocks();
+
 struct GcnBufferFormatInfo {
   unsigned Format;
   unsigned BitsPerComp;
   unsigned NumComponents;
   unsigned NumFormat;
   unsigned DataFormat;
 };
 

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

 #define GET_INSTRMAP_INFO
 #include "AMDGPUGenInstrInfo.inc"
 
 static llvm::cl::opt<unsigned> AmdhsaCodeObjectVersion(
   "amdhsa-code-object-version", llvm::cl::Hidden,
   llvm::cl::desc("AMDHSA Code Object Version"), llvm::cl::init(4),
   llvm::cl::ZeroOrMore);
 
+static llvm::cl::opt<unsigned> AmdgpuOverrideVGPRBlocks(
+  "amdgpu-override-vgpr-blocks", llvm::cl::Hidden,
+  llvm::cl::desc("Override VGPR blocks in COMPUTE_PGM_RSRC1.VGPRS"),
+  llvm::cl::init(std::numeric_limits<unsigned>::max()), llvm::cl::ZeroOrMore);
+
+static llvm::cl::opt<unsigned> AmdgpuOverrideSGPRBlocks(
+  "amdgpu-override-sgpr-blocks", llvm::cl::Hidden,
+  llvm::cl::desc("Override SGPR blocks in COMPUTE_PGM_RSRC1.SGPRS"),
+  llvm::cl::init(std::numeric_limits<unsigned>::max()), llvm::cl::ZeroOrMore);
+
 namespace {
 
 /// \returns Bit mask for given bit \p Shift and bit \p Width.
 unsigned getBitMask(unsigned Shift, unsigned Width) {
   return ((1 << Width) - 1) << Shift;
 }
 
 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI))
     return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V4;
   return false;
 }
 
 bool isHsaAbiVersion3Or4(const MCSubtargetInfo *STI) {
   return isHsaAbiVersion3(STI) || isHsaAbiVersion4(STI);
 }
 
+Optional<unsigned> overrideVGPRBlocks() {
+  unsigned VGPRBlocks = AmdgpuOverrideVGPRBlocks;
+  if (AmdgpuOverrideVGPRBlocks.getNumOccurrences())
+    return VGPRBlocks;
+  return None;
+}
+
+Optional<unsigned> overrideSGPRBlocks() {
+  unsigned SGPRBlocks = AmdgpuOverrideSGPRBlocks;
+  if (AmdgpuOverrideSGPRBlocks.getNumOccurrences())
+    return SGPRBlocks;
+  return None;
+}
+
 #define GET_MIMGBaseOpcodesTable_IMPL
 #define GET_MIMGDimInfoTable_IMPL
 #define GET_MIMGInfoTable_IMPL
 #define GET_MIMGLZMappingTable_IMPL
 #define GET_MIMGMIPMappingTable_IMPL
 #define GET_MIMGG16MappingTable_IMPL
 #include "AMDGPUGenSearchableTables.inc"
 

llvm/test/CodeGen/AMDGPU/override-svgpr-blocks-err.ll

+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-override-sgpr-blocks=0 -amdgpu-override-vgpr-blocks=0 < %s 2>&1 | FileCheck --check-prefix=ERR %s
+
+; ERR: error: requested override of sgpr blocks (0) is smaller than the required number of sgpr blocks (1)
+; ERR: error: requested override of vgpr blocks (0) is smaller than the required number of vgpr blocks (1)
+
+@var = addrspace(1) global float 0.0
+
+define amdgpu_kernel void @test1() {
+  ; 1 SGPR Block.
+  %one = load volatile i32, i32 addrspace(4)* undef
+  %two = load volatile i32, i32 addrspace(4)* undef
+  %three = load volatile i32, i32 addrspace(4)* undef
+  %four = load volatile i32, i32 addrspace(4)* undef
+  %five = load volatile i32, i32 addrspace(4)* undef
+  %six = load volatile i32, i32 addrspace(4)* undef
+  %seven = load volatile i32, i32 addrspace(4)* undef
+  %eight = load volatile i32, i32 addrspace(4)* undef
+  %nine = load volatile i32, i32 addrspace(4)* undef
+  %ten = load volatile i32, i32 addrspace(4)* undef
+  %eleven = load volatile i32, i32 addrspace(4)* undef
+  call void asm sideeffect "", "s,s,s,s,s,s,s,s,s,s"(i32 %one, i32 %two, i32 %three, i32 %four, i32 %five, i32 %six, i32 %seven, i32 %eight, i32 %nine, i32 %ten)
+  store volatile i32 %one, i32 addrspace(1)* undef
+  store volatile i32 %two, i32 addrspace(1)* undef
+  store volatile i32 %three, i32 addrspace(1)* undef
+  store volatile i32 %four, i32 addrspace(1)* undef
+  store volatile i32 %five, i32 addrspace(1)* undef
+  store volatile i32 %six, i32 addrspace(1)* undef
+  store volatile i32 %seven, i32 addrspace(1)* undef
+  store volatile i32 %eight, i32 addrspace(1)* undef
+  store volatile i32 %nine, i32 addrspace(1)* undef
+  store volatile i32 %ten, i32 addrspace(1)* undef
+  store volatile i32 %eleven, i32 addrspace(1)* undef
+
+  ; 1 VGPR Block.
+  %val0 = load volatile float, float addrspace(1)* @var
+  %val1 = load volatile float, float addrspace(1)* @var
+  %val2 = load volatile float, float addrspace(1)* @var
+  %val3 = load volatile float, float addrspace(1)* @var
+  %val4 = load volatile float, float addrspace(1)* @var
+  %val5 = load volatile float, float addrspace(1)* @var
+  %val6 = load volatile float, float addrspace(1)* @var
+  store volatile float %val0, float addrspace(1)* @var
+  store volatile float %val1, float addrspace(1)* @var
+  store volatile float %val2, float addrspace(1)* @var
+  store volatile float %val3, float addrspace(1)* @var
+  store volatile float %val4, float addrspace(1)* @var
+  store volatile float %val5, float addrspace(1)* @var
+  store volatile float %val6, float addrspace(1)* @var
+
+  ret void
+}

llvm/test/CodeGen/AMDGPU/override-svgpr-blocks.ll

+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=GCN -check-prefix=NO-OVERRIDE-ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj < %s | llvm-objdump -s -j .rodata - | FileCheck -check-prefix=NO-OVERRIDE-OBJ %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-override-sgpr-blocks=1 -amdgpu-override-vgpr-blocks=3 < %s | FileCheck -check-prefix=GCN -check-prefix=OVERRIDE-ASM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -amdgpu-override-sgpr-blocks=1 -amdgpu-override-vgpr-blocks=3 < %s | llvm-objdump -s -j .rodata - | FileCheck -check-prefix=OVERRIDE-OBJ %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-override-sgpr-blocks=16 < %s 2>&1 | FileCheck --check-prefix=ERR-SGPRS-OUT-OF-RANGE %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-override-vgpr-blocks=64 < %s 2>&1 | FileCheck --check-prefix=ERR-VGPRS-OUT-OF-RANGE %s
+
+; ERR-SGPRS-OUT-OF-RANGE: error: requested override of sgpr blocks (16) is out of range
+; ERR-VGPRS-OUT-OF-RANGE: error: requested override of vgpr blocks (64) is out of range
+
+; NO-OVERRIDE-OBJ: 0030 0000af00 88000000 01000000 00000000
+; NO-OVERRIDE-OBJ: 0070 0000af00 8c000000 09000000 00000000
+; OVERRIDE-OBJ: 0030 4300af00 88000000 01000000 00000000
+; OVERRIDE-OBJ: 0070 4300af00 8c000000 09000000 00000000
+
+; GCN-LABEL: {{^}}test1
+; GCN: NumSgprs: 0
+; GCN: NumVgprs: 0
+; NO-OVERRIDE-ASM: SGPRBlocks: 0
+; NO-OVERRIDE-ASM: VGPRBlocks: 0
+; OVERRIDE-ASM: SGPRBlocks: 1
+; OVERRIDE-ASM: VGPRBlocks: 3
+define amdgpu_kernel void @test1() {
+entry:
+  ret void
+}
+
+; GCN-LABEL: {{^}}test2
+; GCN: NumSgprs: 8
+; GCN: NumVgprs: 3
+; NO-OVERRIDE-ASM: SGPRBlocks: 0
+; NO-OVERRIDE-ASM: VGPRBlocks: 0
+; OVERRIDE-ASM: SGPRBlocks: 1
+; OVERRIDE-ASM: VGPRBlocks: 3
+define amdgpu_kernel void @test2(
+    half addrspace(1)* %r,
+    half addrspace(1)* %a,
+    half addrspace(1)* %b) {
+entry:
+  %a.val = load volatile half, half addrspace(1)* %a
+  %b.val = load volatile half, half addrspace(1)* %b
+  %r.val = fadd half %a.val, %b.val
+  store half %r.val, half addrspace(1)* %r
+  ret void
+}

llvm/test/MC/AMDGPU/override-svgpr-blocks-err.s

+// RUN: not llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -amdgpu-override-sgpr-blocks=1 -amdgpu-override-vgpr-blocks=1 < %s 2>&1 | FileCheck -check-prefix=ERR %s
+
+// ERR: error: requested override of vgpr blocks (1) is smaller than the required number of vgpr blocks (3)
+// ERR: error: requested override of sgpr blocks (1) is smaller than the required number of sgpr blocks (2)
+
+.text
+.p2align 8
+.type test1,@function
+test1:
+  s_endpgm
+
+.p2align 8
+.type test2,@function
+test1:
+  s_endpgm
+
+.rodata
+
+.p2align 6
+.amdhsa_kernel test1
+  .amdhsa_next_free_vgpr 16
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+.p2align 6
+.amdhsa_kernel test2
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 16
+.end_amdhsa_kernel

llvm/test/MC/AMDGPU/override-svgpr-blocks.s

+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj < %s | llvm-objdump -s -j .rodata - | FileCheck -check-prefix=NO-OVERRIDE %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -amdgpu-override-sgpr-blocks=5 -amdgpu-override-vgpr-blocks=7 < %s | llvm-objdump -s -j .rodata - | FileCheck -check-prefix=OVERRIDE %s
+
+// NO-OVERRIDE: 0030 0000ac00 80000000 00000000 00000000
+// NO-OVERRIDE: 0070 4000ac00 80000000 00000000 00000000
+// OVERRIDE: 0030 4701ac00 80000000 00000000 00000000
+// OVERRIDE: 0070 4701ac00 80000000 00000000 00000000
+
+.text
+.p2align 8
+.type test1,@function
+test1:
+  s_endpgm
+
+.p2align 8
+.type test2,@function
+test2:
+  s_endpgm
+
+.rodata
+
+.p2align 6
+.amdhsa_kernel test1
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+.p2align 6
+.amdhsa_kernel test2
+  .amdhsa_next_free_vgpr 1
+  .amdhsa_next_free_sgpr 3
+.end_amdhsa_kernel