[AMDGPU] Fixed occupancy calculation for gfx10

[arsenm]

lib/Target/AMDGPU/AMDGPUSubtarget.cpp

595 ↗

(On Diff #210881)

s/Granula/Granule

lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

258–260 ↗

(On Diff #210881)

return isGFX10 ? 20 : 10

llvm/trunk/lib/Target/AMDGPU/AMDGPUSubtarget.h

public:
   /// subtarget and limited by given \p FlatWorkGroupSize.
   unsigned getMaxWavesPerCU(unsigned FlatWorkGroupSize) const {
     return AMDGPU::IsaInfo::getMaxWavesPerCU(this, FlatWorkGroupSize);
   }
 
   /// \returns Maximum number of waves per execution unit supported by the
   /// subtarget without any kind of limitation.
   unsigned getMaxWavesPerEU() const {
-    return AMDGPU::IsaInfo::getMaxWavesPerEU();
+    return AMDGPU::IsaInfo::getMaxWavesPerEU(this);
   }
 
   /// \returns Number of waves per work group supported by the subtarget and
   /// limited by given \p FlatWorkGroupSize.
   unsigned getWavesPerWorkGroup(unsigned FlatWorkGroupSize) const {
     return AMDGPU::IsaInfo::getWavesPerWorkGroup(this, FlatWorkGroupSize);
   }
 

llvm/trunk/lib/Target/AMDGPU/AMDGPUSubtarget.cpp

unsigned GCNSubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const {
   if (SGPRs <= 72)
     return 7;
   if (SGPRs <= 80)
     return 6;
   return 5;
 }
 
 unsigned GCNSubtarget::getOccupancyWithNumVGPRs(unsigned VGPRs) const {
-  if (VGPRs <= 24)
-    return 10;
-  if (VGPRs <= 28)
-    return 9;
-  if (VGPRs <= 32)
-    return 8;
-  if (VGPRs <= 36)
-    return 7;
-  if (VGPRs <= 40)
-    return 6;
-  if (VGPRs <= 48)
-    return 5;
-  if (VGPRs <= 64)
-    return 4;
-  if (VGPRs <= 84)
-    return 3;
-  if (VGPRs <= 128)
-    return 2;
-  return 1;
+  unsigned MaxWaves = getMaxWavesPerEU();
+  unsigned Granule = getVGPRAllocGranule();
+  if (VGPRs < Granule)
+    return MaxWaves;
+  unsigned RoundedRegs = ((VGPRs + Granule - 1) / Granule) * Granule;
+  return std::min(getTotalNumVGPRs() / RoundedRegs, MaxWaves);
 }
 
 unsigned GCNSubtarget::getReservedNumSGPRs(const MachineFunction &MF) const {
   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
   if (getGeneration() >= AMDGPUSubtarget::GFX10)
     return 2; // VCC. FLAT_SCRATCH and XNACK are no longer in SGPRs.
 
   if (MFI.hasFlatScratchInit()) {

llvm/trunk/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h

unsigned getMaxWavesPerCU(const MCSubtargetInfo *STI,
                           unsigned FlatWorkGroupSize);
 
 /// \returns Minimum number of waves per execution unit for given subtarget \p
 /// STI.
 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI);
 
 /// \returns Maximum number of waves per execution unit for given subtarget \p
 /// STI without any kind of limitation.
-unsigned getMaxWavesPerEU();
+unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI);
 
 /// \returns Maximum number of waves per execution unit for given subtarget \p
 /// STI and limited by given \p FlatWorkGroupSize.
 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI,
                           unsigned FlatWorkGroupSize);
 
 /// \returns Minimum flat work group size for given subtarget \p STI.
 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI);

llvm/trunk/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI,
   unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
   if (N == 1)
     return 40;
   N = 40 / N;
   return std::min(N, 16u);
 }
 
 unsigned getMaxWavesPerCU(const MCSubtargetInfo *STI) {
-  return getMaxWavesPerEU() * getEUsPerCU(STI);
+  return getMaxWavesPerEU(STI) * getEUsPerCU(STI);
 }
 
 unsigned getMaxWavesPerCU(const MCSubtargetInfo *STI,
                           unsigned FlatWorkGroupSize) {
   return getWavesPerWorkGroup(STI, FlatWorkGroupSize);
 }
 
 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) {
   return 1;
 }
 
-unsigned getMaxWavesPerEU() {
+unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) {
   // FIXME: Need to take scratch memory into account.
+  if (!isGFX10(*STI))
     return 10;
+  return 20;
 }
 
 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI,
                           unsigned FlatWorkGroupSize) {
   return alignTo(getMaxWavesPerCU(STI, FlatWorkGroupSize),
                  getEUsPerCU(STI)) / getEUsPerCU(STI);
 }
 

 
 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
   assert(WavesPerEU != 0);
 
   IsaVersion Version = getIsaVersion(STI->getCPU());
   if (Version.Major >= 10)
     return 0;
 
-  if (WavesPerEU >= getMaxWavesPerEU())
+  if (WavesPerEU >= getMaxWavesPerEU(STI))
     return 0;
 
   unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1);
   if (STI->getFeatureBits().test(FeatureTrapHandler))
     MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
   MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1;
   return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI));
 }

 }
 
 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI,
                                 Optional<bool> EnableWavefrontSize32) {
   return getVGPRAllocGranule(STI, EnableWavefrontSize32);
 }
 
 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) {
+  if (!isGFX10(*STI))
     return 256;
+  return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512;
 }
 
 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) {
-  return getTotalNumVGPRs(STI);
+  return 256;
 }
 
 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
   assert(WavesPerEU != 0);
 
-  if (WavesPerEU >= getMaxWavesPerEU())
+  if (WavesPerEU >= getMaxWavesPerEU(STI))
     return 0;
   unsigned MinNumVGPRs =
       alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1),
                 getVGPRAllocGranule(STI)) + 1;
   return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI));
 }
 
 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {

llvm/trunk/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props-v3.ll

entry:
   store i32 %ind, i32 addrspace(1)* %outd
   store i32 %ine, i32 addrspace(1)* %oute
   store i32 %inf, i32 addrspace(1)* %outf
   ret void
 }
 
 ; CHECK:   .name:       num_spilled_vgprs
 ; CHECK:   .symbol:     num_spilled_vgprs.kd
-; CHECK:   .vgpr_spill_count: 14
+; GFX700:   .vgpr_spill_count: 14
+; GFX803:   .vgpr_spill_count: 14
+; GFX900:   .vgpr_spill_count: 14
+; GFX1010:  .vgpr_spill_count: 0
 define amdgpu_kernel void @num_spilled_vgprs() #1 {
   %val0 = load volatile float, float addrspace(1)* @var
   %val1 = load volatile float, float addrspace(1)* @var
   %val2 = load volatile float, float addrspace(1)* @var
   %val3 = load volatile float, float addrspace(1)* @var
   %val4 = load volatile float, float addrspace(1)* @var
   %val5 = load volatile float, float addrspace(1)* @var
   %val6 = load volatile float, float addrspace(1)* @var

llvm/trunk/test/CodeGen/AMDGPU/idot8s.ll

 ; GFX10-DL-NEXT:    v_and_b32_e32 v22, v7, v2
 ; GFX10-DL-NEXT:    v_and_b32_e32 v11, v11, v2
 ; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v5, v5, v8
 ; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s4
 ; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v4, v4, v9
 ; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s6
 ; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v7, v22, v11
 ; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v15, 12, s0
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v20, 12, s8
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v31, 12, s8
 ; GFX10-DL-NEXT:    v_mul_lo_u16_e64 v23, v23, v10
-; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v21, 12, s1
+; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v27, 12, s1
 ; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v16, 12, s5
 ; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v17, 12, s9
 ; GFX10-DL-NEXT:    v_lshlrev_b16_e64 v19, 12, s7
 ; GFX10-DL-NEXT:    v_and_b32_e32 v8, v12, v2
 ; GFX10-DL-NEXT:    v_and_b32_e32 v9, v13, v2
 ; GFX10-DL-NEXT:    v_and_b32_e32 v11, v15, v2
 ; GFX10-DL-NEXT:    v_and_b32_e32 v12, v16, v2
 ; GFX10-DL-NEXT:    v_and_b32_e32 v13, v17, v2
 ; GFX10-DL-NEXT:    v_and_b32_e32 v15, v19, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v10, v21, v2
-; GFX10-DL-NEXT:    v_and_b32_e32 v14, v20, v2
+; GFX10-DL-NEXT:    v_and_b32_e32 v10, v27, v2
+; GFX10-DL-NEXT:    v_and_b32_e32 v14, v31, v2
 ; GFX10-DL-NEXT:    v_and_b32_sdwa v6, v23, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-DL-NEXT:    v_and_b32_sdwa v7, v7, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX10-DL-NEXT:    v_and_b32_sdwa v4, v4, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX10-DL-NEXT:    v_and_b32_sdwa v5, v5, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v12, 12, v12
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v13, 12, v13
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v8, 12, v8
 ; GFX10-DL-NEXT:    v_ashrrev_i16_e64 v9, 12, v9

llvm/trunk/test/CodeGen/AMDGPU/nsa-reassign.ll

main_body:
   %v1 = call float @llvm.amdgcn.image.sample.c.l.3d.f32.f32(i32 1, float %zcompare.1, float %s1.1, float %t1.1, float %r1.1, float %lod.1, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   %v2 = call float @llvm.amdgcn.image.sample.3d.f32.f32(i32 1, float %s2.1, float %t2.1, float %r2.1, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   %r.0 = insertelement <2 x float> undef, float %v1, i32 0
   %r = insertelement <2 x float> %r.0, float %v2, i32 1
   ret <2 x float> %r
 }
 
 ; GCN-LABEL: {{^}}sample_contig_nsa_10vgprs:
-; GCN-DAG: image_sample_c_l v{{[0-9]+}}, [{{v[0-9]+, v[0-9]+, v[0-9]+, v[0-9]+, v[0-9]+}}],
-; GCN-DAG: image_sample v{{[0-9]+}}, [{{v[0-9]+, v[0-9]+, v[0-9]+}}],
+; GCN-DAG: image_sample_c_l v{{[0-9]+}}, v[{{[0-9:]+}}],
+; GCN-DAG: image_sample v{{[0-9]+}}, v[{{[0-9:]+}}],
 define amdgpu_ps <2 x float> @sample_contig_nsa_10vgprs(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s1, float %t1, float %r1, float %lod, float %r2, float %s2, float %t2) #0 {
 main_body:
   %zcompare.1 = fadd float %zcompare, 1.0
   %s1.1 = fadd float %s1, 1.0
   %t1.1 = fadd float %t1, 1.0
   %r1.1 = fadd float %r1, 1.0
   %s2.1 = fadd float %s2, 1.0
   %t2.1 = fadd float %t2, 1.0

llvm/trunk/test/CodeGen/AMDGPU/regbank-reassign.mir

registers:
   - { id: 2, class: vgpr_32, preferred-register: '$vgpr1' }
   - { id: 3, class: vreg_64, preferred-register: '$vgpr2_vgpr3' }
   - { id: 4, class: vgpr_32, preferred-register: '$vgpr5' }
   - { id: 5, class: vreg_64, preferred-register: '$vgpr6_vgpr7' }
   - { id: 6, class: vreg_128, preferred-register: '$vgpr8_vgpr9_vgpr10_vgpr11' }
   - { id: 7, class: vreg_128, preferred-register: '$vgpr12_vgpr13_vgpr14_vgpr15' }
   - { id: 8, class: vreg_128, preferred-register: '$vgpr16_vgpr17_vgpr18_vgpr19' }
   - { id: 9, class: vreg_128, preferred-register: '$vgpr20_vgpr21_vgpr22_vgpr23' }
+  - { id: 10, class: vreg_128, preferred-register: '$vgpr24_vgpr25_vgpr26_vgpr27' }
+  - { id: 11, class: vreg_128, preferred-register: '$vgpr28_vgpr29_vgpr30_vgpr31' }
+  - { id: 12, class: vreg_128, preferred-register: '$vgpr32_vgpr33_vgpr34_vgpr35' }
+  - { id: 13, class: vreg_128, preferred-register: '$vgpr36_vgpr37_vgpr38_vgpr39' }
+  - { id: 14, class: vreg_128, preferred-register: '$vgpr40_vgpr41_vgpr42_vgpr43' }
+  - { id: 15, class: vreg_128, preferred-register: '$vgpr44_vgpr45_vgpr46_vgpr47' }
 body: |
   bb.0:
     %0 = IMPLICIT_DEF
     %1 = IMPLICIT_DEF
     %3 = IMPLICIT_DEF
     %4 = IMPLICIT_DEF
     %5 = IMPLICIT_DEF
     %6 = IMPLICIT_DEF
     %7 = IMPLICIT_DEF
     %8 = IMPLICIT_DEF
     %9 = IMPLICIT_DEF
+    %10 = IMPLICIT_DEF
+    %11 = IMPLICIT_DEF
+    %12 = IMPLICIT_DEF
+    %13 = IMPLICIT_DEF
+    %14 = IMPLICIT_DEF
+    %15 = IMPLICIT_DEF
     %2 = V_AND_B32_e32 %1, %0, implicit $exec
     GLOBAL_STORE_DWORD %3, %0, 0, 0, 0, 0, implicit $exec
     GLOBAL_STORE_DWORD %3, %1, 0, 0, 0, 0, implicit $exec
     GLOBAL_STORE_DWORD %3, %2, 0, 0, 0, 0, implicit $exec
     GLOBAL_STORE_DWORD %3, %4, 0, 0, 0, 0, implicit $exec
     GLOBAL_STORE_DWORDX2 %3, %5, 0, 0, 0, 0, implicit $exec
     GLOBAL_STORE_DWORDX4 %3, %6, 0, 0, 0, 0, implicit $exec
     GLOBAL_STORE_DWORDX4 %3, %7, 0, 0, 0, 0, implicit $exec

llvm/trunk/test/CodeGen/AMDGPU/wave32.ll

main_body:
   %s.8 = fadd float %s.7, %j
   %s.9 = fadd float %s.8, %k
   %s.10 = fadd float %s.9, %l
   ret float %s.10
 }
 
 ; GCN-LABEL: {{^}}test_vgprblocks_w64_attr:
 ; Test that the wave size can be overridden in function attributes and that the block size is correct as a result
-; GFX10DEFWAVE: ; VGPRBlocks: 2
+; GFX10DEFWAVE: ; VGPRBlocks: 11
 define amdgpu_gs float @test_vgprblocks_w64_attr(float %a, float %b, float %c, float %d, float %e,
                                         float %f, float %g, float %h, float %i, float %j, float %k, float %l) #4 {
 main_body:
   %s = fadd float %a, %b
   %s.1 = fadd float %s, %c
   %s.2 = fadd float %s.1, %d
   %s.3 = fadd float %s.2, %e
   %s.4 = fadd float %s.3, %f