AMDGPU: Add gfx11 feature to force initializing 16 input SGPRs

[arsenm]

Summary

The total user+system SGPR count needs to be padded out to 16 if fewer
inputs are enabled.

[rampitec]

llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp

908

I would not call scratch_en a wave offset. With architected flat scratch there is no wave offset.

[arsenm]

Rename variable

[rampitec]

LGTM

[foad]

llvm/lib/Target/AMDGPU/AMDGPU.td

1271

Should apply to gfx1100 but not gfx1101.

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

2102

Should only do this in wave32 mode.

[arsenm]

llvm/lib/Target/AMDGPU/AMDGPU.td

1271

D130216

llvm/lib/Target/AMDGPU/AMDGPU.td

 >;
 
 def FeatureSGPRInitBug : SubtargetFeature<"sgpr-init-bug",
   "SGPRInitBug",
   "true",
   "VI SGPR initialization bug requiring a fixed SGPR allocation size"
 >;
 
+def FeatureUserSGPRInit16Bug : SubtargetFeature<"user-sgpr-init16-bug",
+  "UserSGPRInit16Bug",
+  "true",
+  "Bug requiring at least 16 user+system SGPRs to be enabled"
+>;
+
 def FeatureLdsMisalignedBug : SubtargetFeature<"lds-misaligned-bug",
   "LDSMisalignedBug",
   "true",
   "Some GFX10 bug with multi-dword LDS and flat access that is not naturally aligned in WGP mode"
 >;
 
 def FeatureMFMAInlineLiteralBug : SubtargetFeature<"mfma-inline-literal-bug",
   "HasMFMAInlineLiteralBug",
def FeatureISAVersion11_Common : FeatureSet<
    FeatureAtomicFaddNoRtnInsts,
    FeatureImageInsts,
    FeaturePackedTID,
    FeatureVcmpxPermlaneHazard]>;
 
 // Features for GFX 11.0.0 and 11.0.1
 def FeatureISAVersion11_0 : FeatureSet<
   !listconcat(FeatureISAVersion11_Common.Features,
-    [])>;
+    [FeatureUserSGPRInit16Bug])>;
 
 def FeatureISAVersion11_0_2 : FeatureSet<
   !listconcat(FeatureISAVersion11_Common.Features,
-    [])>;
+    [FeatureUserSGPRInit16Bug])>;
 
 //===----------------------------------------------------------------------===//
 
 def AMDGPUInstrInfo : InstrInfo {
   let guessInstructionProperties = 1;
   let noNamedPositionallyEncodedOperands = 1;
 }
 

llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp

void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
 
   // 0 = X, 1 = XY, 2 = XYZ
   unsigned TIDIGCompCnt = 0;
   if (MFI->hasWorkItemIDZ())
     TIDIGCompCnt = 2;
   else if (MFI->hasWorkItemIDY())
     TIDIGCompCnt = 1;
 
+  // The private segment wave byte offset is the last of the system SGPRs. We
+  // initially assumed it was allocated, and may have used it. It shouldn't harm
+  // anything to disable it if we know the stack isn't used here. We may still
+  // have emitted code reading it to initialize scratch, but if that's unused
+  // reading garbage should be OK.
+  const bool EnablePrivateSegment = ProgInfo.ScratchBlocks > 0;
   ProgInfo.ComputePGMRSrc2 =
-      S_00B84C_SCRATCH_EN(ProgInfo.ScratchBlocks > 0) |
+      S_00B84C_SCRATCH_EN(EnablePrivateSegment) |
       S_00B84C_USER_SGPR(MFI->getNumUserSGPRs()) |
       // For AMDHSA, TRAP_HANDLER must be zero, as it is populated by the CP.
       S_00B84C_TRAP_HANDLER(STM.isAmdHsaOS() ? 0 : STM.isTrapHandlerEnabled()) |
       S_00B84C_TGID_X_EN(MFI->hasWorkGroupIDX()) |
       S_00B84C_TGID_Y_EN(MFI->hasWorkGroupIDY()) |
       S_00B84C_TGID_Z_EN(MFI->hasWorkGroupIDZ()) |
       S_00B84C_TG_SIZE_EN(MFI->hasWorkGroupInfo()) |
       S_00B84C_TIDIG_COMP_CNT(TIDIGCompCnt) |

llvm/lib/Target/AMDGPU/GCNSubtarget.h

protected:
   bool GFX9Insts = false;
   bool GFX90AInsts = false;
   bool GFX940Insts = false;
   bool GFX10Insts = false;
   bool GFX11Insts = false;
   bool GFX10_3Insts = false;
   bool GFX7GFX8GFX9Insts = false;
   bool SGPRInitBug = false;
+  bool UserSGPRInit16Bug = false;
   bool NegativeScratchOffsetBug = false;
   bool NegativeUnalignedScratchOffsetBug = false;
   bool HasSMemRealTime = false;
   bool HasIntClamp = false;
   bool HasFmaMixInsts = false;
   bool HasMovrel = false;
   bool HasVGPRIndexMode = false;
   bool HasScalarStores = false;
public:
   bool loadStoreOptEnabled() const {
     return EnableLoadStoreOpt;
   }
 
   bool hasSGPRInitBug() const {
     return SGPRInitBug;
   }
 
+  bool hasUserSGPRInit16Bug() const {
+    return UserSGPRInit16Bug;
+  }
+
   bool hasNegativeScratchOffsetBug() const { return NegativeScratchOffsetBug; }
 
   bool hasNegativeUnalignedScratchOffsetBug() const {
     return NegativeUnalignedScratchOffsetBug;
   }
 
   bool hasMFMAInlineLiteralBug() const {
     return HasMFMAInlineLiteralBug;

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

 }
 
 // Allocate special input registers that are initialized per-wave.
 void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo,
                                            MachineFunction &MF,
                                            SIMachineFunctionInfo &Info,
                                            CallingConv::ID CallConv,
                                            bool IsShader) const {
+  if (Subtarget->hasUserSGPRInit16Bug()) {
+    // Pad up the used user SGPRs with dead inputs.
+    unsigned CurrentUserSGPRs = Info.getNumUserSGPRs();
+
+    // Note we do not count the PrivateSegmentWaveByteOffset. We do not want to
+    // rely on it to reach 16 since if we end up having no stack usage, it will
+    // not really be added.
+    unsigned NumRequiredSystemSGPRs = Info.hasWorkGroupIDX() +
+                                      Info.hasWorkGroupIDY() +
+                                      Info.hasWorkGroupIDZ() +
+                                      Info.hasWorkGroupInfo();
+    for (unsigned i = NumRequiredSystemSGPRs + CurrentUserSGPRs; i < 16; ++i) {
+      Register Reg = Info.addReservedUserSGPR();
+      MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
+      CCInfo.AllocateReg(Reg);
+    }
+  }
+
   if (Info.hasWorkGroupIDX()) {
     Register Reg = Info.addWorkGroupIDX();
     MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
     CCInfo.AllocateReg(Reg);
   }
 
   if (Info.hasWorkGroupIDY()) {
     Register Reg = Info.addWorkGroupIDY();
if (IsShader) {
         Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
       }
     } else
       PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
 
     MF.addLiveIn(PrivateSegmentWaveByteOffsetReg, &AMDGPU::SGPR_32RegClass);
     CCInfo.AllocateReg(PrivateSegmentWaveByteOffsetReg);
   }
+
+  assert(!Subtarget->hasUserSGPRInit16Bug() || Info.getNumPreloadedSGPRs() >= 16);
 }
 
 static void reservePrivateMemoryRegs(const TargetMachine &TM,
                                      MachineFunction &MF,
                                      const SIRegisterInfo &TRI,
                                      SIMachineFunctionInfo &Info) {
   // Now that we've figured out where the scratch register inputs are, see if
   // should reserve the arguments and use them directly.

llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h

public:
   Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI);
   Register addDispatchPtr(const SIRegisterInfo &TRI);
   Register addQueuePtr(const SIRegisterInfo &TRI);
   Register addKernargSegmentPtr(const SIRegisterInfo &TRI);
   Register addDispatchID(const SIRegisterInfo &TRI);
   Register addFlatScratchInit(const SIRegisterInfo &TRI);
   Register addImplicitBufferPtr(const SIRegisterInfo &TRI);
 
+  /// Increment user SGPRs used for padding the argument list only.
+  Register addReservedUserSGPR() {
+    Register Next = getNextUserSGPR();
+    ++NumUserSGPRs;
+    return Next;
+  }
+
   // Add system SGPRs.
   Register addWorkGroupIDX() {
     ArgInfo.WorkGroupIDX = ArgDescriptor::createRegister(getNextSystemSGPR());
     NumSystemSGPRs += 1;
     return ArgInfo.WorkGroupIDX.getRegister();
   }
 
   Register addWorkGroupIDY() {

llvm/test/CodeGen/AMDGPU/gfx11-user-sgpr-init16-bug.ll

+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1102 < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1102 < %s | FileCheck -check-prefix=GCN %s
+
+; There aren't any stack objects, but we still enable the
+; private_segment_wavefront_offset to get to 16, and the workgroup ID
+; is in s14.
+
+; private_segment_buffer + workgroup_id_x = 5, + 11 padding
+
+; GCN-LABEL: {{^}}minimal_kernel_inputs:
+; GCN: v_mov_b32_e32 [[V:v[0-9]+]], s15
+; GCN-NEXT: global_store_b32 v{{\[[0-9]+:[0-9]+\]}}, [[V]], off
+
+; GCN: .amdhsa_kernel minimal_kernel_inputs
+; GCN: .amdhsa_user_sgpr_count 15
+; GCN-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+; GCN-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+; GCN-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+; GCN-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+; GCN-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; GCN-NEXT: .amdhsa_wavefront_size32 1
+; GCN-NEXT: .amdhsa_enable_private_segment 0
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_info 0
+; GCN-NEXT: .amdhsa_system_vgpr_workitem_id 0
+; GCN: ; COMPUTE_PGM_RSRC2:USER_SGPR: 15
+define amdgpu_kernel void @minimal_kernel_inputs() {
+  %id = call i32 @llvm.amdgcn.workgroup.id.x()
+  store volatile i32 %id, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}minimal_kernel_inputs_with_stack:
+; GCN: v_mov_b32_e32 [[V:v[0-9]+]], s15
+; GCN: global_store_b32 v{{\[[0-9]+:[0-9]+\]}}, [[V]], off
+
+; GCN: .amdhsa_kernel minimal_kernel_inputs
+; GCN: .amdhsa_user_sgpr_count 15
+; GCN-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+; GCN-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+; GCN-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+; GCN-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+; GCN-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; GCN-NEXT: .amdhsa_wavefront_size32 1
+; GCN-NEXT: .amdhsa_enable_private_segment 1
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_info 0
+; GCN-NEXT: .amdhsa_system_vgpr_workitem_id 0
+; GCN: ; COMPUTE_PGM_RSRC2:USER_SGPR: 15
+define amdgpu_kernel void @minimal_kernel_inputs_with_stack() {
+  %alloca = alloca i32, addrspace(5)
+  %id = call i32 @llvm.amdgcn.workgroup.id.x()
+  store volatile i32 %id, i32 addrspace(1)* undef
+  store volatile i32 0, i32 addrspace(5)* %alloca
+  ret void
+}
+
+; GCN-LABEL: {{^}}queue_ptr:
+; GCN: global_load_u8 v{{[0-9]+}}, v{{[0-9]+}}, s[0:1]
+
+; GCN: v_mov_b32_e32 [[V:v[0-9]+]], s15
+; GCN-NEXT: global_store_b32 v{{\[[0-9]+:[0-9]+\]}}, [[V]], off
+
+; GCN: .amdhsa_kernel queue_ptr
+; GCN: .amdhsa_user_sgpr_count 15
+; GCN-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+; GCN-NEXT: .amdhsa_user_sgpr_queue_ptr 1
+; GCN-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+; GCN-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+; GCN-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; GCN-NEXT: .amdhsa_wavefront_size32 1
+; GCN-NEXT: .amdhsa_enable_private_segment 0
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_info 0
+; GCN-NEXT: .amdhsa_system_vgpr_workitem_id 0
+; GCN: ; COMPUTE_PGM_RSRC2:USER_SGPR: 15
+define amdgpu_kernel void @queue_ptr() {
+  %queue.ptr = call noalias i8 addrspace(4)* @llvm.amdgcn.queue.ptr() #0
+  %load = load volatile i8, i8 addrspace(4)* %queue.ptr
+  %id = call i32 @llvm.amdgcn.workgroup.id.x()
+  store volatile i32 %id, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}all_inputs:
+; GCN: v_mov_b32_e32 [[V_X:v[0-9]+]], s13
+; GCN: v_mov_b32_e32 [[V_Y:v[0-9]+]], s14
+; GCN: v_mov_b32_e32 [[V_Z:v[0-9]+]], s15
+
+; GCN: global_load_u8 v{{[0-9]+}}, v{{[0-9]+}}, s[0:1]
+; GCN: global_load_u8 v{{[0-9]+}}, v{{[0-9]+}}, s[2:3]
+; GCN: global_load_u8 v{{[0-9]+}}, v{{[0-9]+}}, s[4:5]
+
+; GCN-DAG: v_mov_b32_e32 v[[DISPATCH_LO:[0-9]+]], s6
+; GCN-DAG: v_mov_b32_e32 v[[DISPATCH_HI:[0-9]+]], s7
+
+; GCN: global_store_b32 v{{\[[0-9]+:[0-9]+\]}}, [[V_X]], off
+; GCN: global_store_b32 v{{\[[0-9]+:[0-9]+\]}}, [[V_Y]], off
+; GCN: global_store_b32 v{{\[[0-9]+:[0-9]+\]}}, [[V_Z]], off
+; GCN: global_store_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[DISPATCH_LO]]:[[DISPATCH_HI]]{{\]}}, off
+
+; GCN: .amdhsa_kernel all_inputs
+; GCN: .amdhsa_user_sgpr_count 13
+; GCN-NEXT: .amdhsa_user_sgpr_dispatch_ptr 1
+; GCN-NEXT: .amdhsa_user_sgpr_queue_ptr 1
+; GCN-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 1
+; GCN-NEXT: .amdhsa_user_sgpr_dispatch_id 1
+; GCN-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; GCN-NEXT: .amdhsa_wavefront_size32 1
+; GCN-NEXT: .amdhsa_enable_private_segment 1
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
+; GCN-NEXT: .amdhsa_system_sgpr_workgroup_info 0
+; GCN-NEXT: .amdhsa_system_vgpr_workitem_id 0
+; GCN: ; COMPUTE_PGM_RSRC2:USER_SGPR: 13
+define amdgpu_kernel void @all_inputs() {
+  %alloca = alloca i32, addrspace(5)
+  store volatile i32 0, i32 addrspace(5)* %alloca
+
+  %dispatch.ptr = call noalias i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+  %load.dispatch = load volatile i8, i8 addrspace(4)* %dispatch.ptr
+
+  %queue.ptr = call noalias i8 addrspace(4)* @llvm.amdgcn.queue.ptr()
+  %load.queue = load volatile i8, i8 addrspace(4)* %queue.ptr
+
+  %implicitarg.ptr = call noalias i8 addrspace(4)* @llvm.amdgcn.implicitarg.ptr()
+  %load.implicitarg = load volatile i8, i8 addrspace(4)* %implicitarg.ptr
+
+  %id.x = call i32 @llvm.amdgcn.workgroup.id.x()
+  store volatile i32 %id.x, i32 addrspace(1)* undef
+
+  %id.y = call i32 @llvm.amdgcn.workgroup.id.y()
+  store volatile i32 %id.y, i32 addrspace(1)* undef
+
+  %id.z = call i32 @llvm.amdgcn.workgroup.id.z()
+  store volatile i32 %id.z, i32 addrspace(1)* undef
+
+  %dispatch.id = call i64 @llvm.amdgcn.dispatch.id()
+  store volatile i64 %dispatch.id, i64 addrspace(1)* undef
+
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workgroup.id.x() #0
+declare i32 @llvm.amdgcn.workgroup.id.y() #0
+declare i32 @llvm.amdgcn.workgroup.id.z() #0
+declare align 4 i8 addrspace(4)* @llvm.amdgcn.implicitarg.ptr() #0
+declare align 4 i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() #0
+declare align 4 i8 addrspace(4)* @llvm.amdgcn.queue.ptr() #0
+declare align 4 i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr() #0
+declare i64 @llvm.amdgcn.dispatch.id() #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }