[AMDGPU] Fix GCNSubtarget::getMinNumVGPRs, add unit test to check consistency between GCNSubtarget's getMinNumVGPRs, getMaxNumVGPRs and getOccupancyWithNumVGPRs.

[vpykhtin]

Summary

/// \returns Minimum number of VGPRs that meets given number of waves per
/// execution unit requirement supported by the subtarget.
unsigned getMinNumVGPRs(unsigned WavesPerEU) const;

/// \returns Maximum number of VGPRs that meets given number of waves per
/// execution unit requirement supported by the subtarget.
unsigned getMaxNumVGPRs(unsigned WavesPerEU) const;

/// Return the maximum number of waves per SIMD for kernels using \p VGPRs
/// VGPRs
unsigned getOccupancyWithNumVGPRs(unsigned VGPRs) const;

While working on RP tracking issues I noticed that getMinNumVGPRs return incorrect
values: the problem is large VGPR granule sizes on GFX10+ architectures. Some of the
occupancies aren't reachable because require the same amount of VGPR granules as others.
For example 19 waves occupancy on gfx1010 require the same amount of granules as 20 waves
so the resultng occupancy would be 20.

SGPRs have the same issue and even have inconsistency between getMaxNumSGPRs and getOccupancyWithNumSGPRs.
It will be addressed in the next patch.

Legend:

1. MinVGPR and MaxVGPR are values returned by getMinNumVGPRs and getMaxNumVGPRs for a given Occ.
2. (ONumber) is the value returned by getOccupancyWithNumVGPRs for a given MinVGPR or MaxVGPR.
3. R means range problem: MinVGPR should be less than MaxVGPR and both should refer to the same occupancy.

Unit test output without the fix:

./build/unittests/Target/AMDGPU/AMDGPUTests --gtest_filter=AMDGPU.TestVGPRLimitsPerOccupancy --print-cpu-reg-limits

 gfx90a gfx940:
Occ    MinVGPR        MaxVGPR
  8        0 (O8)     64  (O8)
  7       65 (O7)     72  (O7)
  6       73 (O6)     80  (O6)
  5       81 (O5)     96  (O5)
  4       97 (O4)     128 (O4)
  3      129 (O3)     168 (O3)
  2      169 (O2)     256 (O2)
  1      257 (O1)     512 (O1)

 gfx600 gfx600 gfx601 gfx601 gfx601 gfx602 gfx602 gfx602 gfx700 gfx700 gfx701 gfx701 gfx702 gfx703 gfx703 gfx703 gfx704 gfx704 gfx705 gfx801 gfx801 gfx802 gfx802 gfx802 gfx803 gfx803 gfx803 gfx803 gfx805 gfx805 gfx810 gfx810 gfx900 gfx902 gfx904 gfx906 gfx908 gfx909 gfx90c:
Occ    MinVGPR        MaxVGPR
 10        0 (O10)    24  (O10)
  9       25 (O9)     28  (O9)
  8       29 (O8)     32  (O8)
  7       33 (O7)     36  (O7)
  6       37 (O6)     40  (O6)
  5       41 (O5)     48  (O5)
  4       49 (O4)     64  (O4)
  3       65 (O3)     84  (O3)
  2       85 (O2)     128 (O2)
  1      129 (O1)     256 (O1)

 gfx1030w64 gfx1031w64 gfx1032w64 gfx1033w64 gfx1034w64 gfx1035w64 gfx1036w64 gfx1102w64 gfx1103w64:
Occ    MinVGPR        MaxVGPR
 16        0 (O16)    32  (O16)
 15       33 (O12) R  32  (O16)
 14       33 (O12) R  32  (O16)
 13       33 (O12) R  32  (O16)
 12       33 (O12)    40  (O12)
 11       41 (O10) R  40  (O12)
 10       41 (O10)    48  (O10)
  9       49 (O9)     56  (O9)
  8       57 (O8)     64  (O8)
  7       65 (O7)     72  (O7)
  6       73 (O6)     80  (O6)
  5       81 (O5)     96  (O5)
  4       97 (O4)     128 (O4)
  3      129 (O3)     168 (O3)
  2      169 (O2)     256 (O2)
  1      256 (O2) R   256 (O2)

 gfx1100w64 gfx1101w64:
Occ    MinVGPR        MaxVGPR
 16        0 (O16)    48  (O16)
 15       49 (O12) R  48  (O16)
 14       49 (O12) R  48  (O16)
 13       49 (O12) R  48  (O16)
 12       49 (O12)    60  (O12)
 11       61 (O10) R  60  (O12)
 10       61 (O10)    72  (O10)
  9       73 (O9)     84  (O9)
  8       85 (O8)     96  (O8)
  7       97 (O7)     108 (O7)
  6      109 (O6)     120 (O6)
  5      121 (O5)     144 (O5)
  4      145 (O4)     192 (O4)
  3      193 (O3)     252 (O3)
  2      253 (O2)     256 (O2)
  1      256 (O2) R   256 (O2)

 gfx1030w32 gfx1031w32 gfx1032w32 gfx1033w32 gfx1034w32 gfx1035w32 gfx1036w32 gfx1102w32 gfx1103w32:
Occ    MinVGPR        MaxVGPR
 16        0 (O16)    64  (O16)
 15       65 (O12) R  64  (O16)
 14       65 (O12) R  64  (O16)
 13       65 (O12) R  64  (O16)
 12       65 (O12)    80  (O12)
 11       81 (O10) R  80  (O12)
 10       81 (O10)    96  (O10)
  9       97 (O9)     112 (O9)
  8      113 (O8)     128 (O8)
  7      129 (O7)     144 (O7)
  6      145 (O6)     160 (O6)
  5      161 (O5)     192 (O5)
  4      193 (O4)     256 (O4)
  3      256 (O4) R   256 (O4)
  2      256 (O4) R   256 (O4)
  1      256 (O4) R   256 (O4)

 gfx1100w32 gfx1101w32:
Occ    MinVGPR        MaxVGPR
 16        0 (O16)    96  (O16)
 15       97 (O12) R  96  (O16)
 14       97 (O12) R  96  (O16)
 13       97 (O12) R  96  (O16)
 12       97 (O12)    120 (O12)
 11      121 (O10) R  120 (O12)
 10      121 (O10)    144 (O10)
  9      145 (O9)     168 (O9)
  8      169 (O8)     192 (O8)
  7      193 (O7)     216 (O7)
  6      217 (O6)     240 (O6)
  5      241 (O5)     256 (O5)
  4      256 (O5) R   256 (O5)
  3      256 (O5) R   256 (O5)
  2      256 (O5) R   256 (O5)
  1      256 (O5) R   256 (O5)

 gfx1010w64 gfx1011w64 gfx1012w64 gfx1013w64:
Occ    MinVGPR        MaxVGPR
 20        0 (O20)    24  (O20)
 19       25 (O18) R  24  (O20)
 18       25 (O18)    28  (O18)
 17       29 (O16) R  28  (O18)
 16       29 (O16)    32  (O16)
 15       33 (O14) R  32  (O16)
 14       33 (O14)    36  (O14)
 13       37 (O12) R  36  (O14)
 12       37 (O12)    40  (O12)
 11       41 (O11)    44  (O11)
 10       45 (O10)    48  (O10)
  9       49 (O9)     56  (O9)
  8       57 (O8)     64  (O8)
  7       65 (O7)     72  (O7)
  6       73 (O6)     84  (O6)
  5       85 (O5)     100 (O5)
  4      101 (O4)     128 (O4)
  3      129 (O3)     168 (O3)
  2      169 (O2)     256 (O2)
  1      256 (O2) R   256 (O2)

 gfx1010w32 gfx1011w32 gfx1012w32 gfx1013w32:
Occ    MinVGPR        MaxVGPR
 20        0 (O20)    48  (O20)
 19       49 (O18) R  48  (O20)
 18       49 (O18)    56  (O18)
 17       57 (O16) R  56  (O18)
 16       57 (O16)    64  (O16)
 15       65 (O14) R  64  (O16)
 14       65 (O14)    72  (O14)
 13       73 (O12) R  72  (O14)
 12       73 (O12)    80  (O12)
 11       81 (O11)    88  (O11)
 10       89 (O10)    96  (O10)
  9       97 (O9)     112 (O9)
  8      113 (O8)     128 (O8)
  7      129 (O7)     144 (O7)
  6      145 (O6)     168 (O6)
  5      169 (O5)     200 (O5)
  4      201 (O4)     256 (O4)
  3      256 (O4) R   256 (O4)
  2      256 (O4) R   256 (O4)
  1      256 (O4) R   256 (O4)

After the fix:

 gfx90a gfx940:
Occ    MinVGPR        MaxVGPR
  8        0 (O8)     64  (O8)
  7       65 (O7)     72  (O7)
  6       73 (O6)     80  (O6)
  5       81 (O5)     96  (O5)
  4       97 (O4)     128 (O4)
  3      129 (O3)     168 (O3)
  2      169 (O2)     256 (O2)
  1      257 (O1)     512 (O1)

 gfx600 gfx600 gfx601 gfx601 gfx601 gfx602 gfx602 gfx602 gfx700 gfx700 gfx701 gfx701 gfx702 gfx703 gfx703 gfx703 gfx704 gfx704 gfx705 gfx801 gfx801 gfx802 gfx802 gfx802 gfx803 gfx803 gfx803 gfx803 gfx805 gfx805 gfx810 gfx810 gfx900 gfx902 gfx904 gfx906 gfx908 gfx909 gfx90c:
Occ    MinVGPR        MaxVGPR
 10        0 (O10)    24  (O10)
  9       25 (O9)     28  (O9)
  8       29 (O8)     32  (O8)
  7       33 (O7)     36  (O7)
  6       37 (O6)     40  (O6)
  5       41 (O5)     48  (O5)
  4       49 (O4)     64  (O4)
  3       65 (O3)     84  (O3)
  2       85 (O2)     128 (O2)
  1      129 (O1)     256 (O1)

 gfx1030w64 gfx1031w64 gfx1032w64 gfx1033w64 gfx1034w64 gfx1035w64 gfx1036w64 gfx1102w64 gfx1103w64:
Occ    MinVGPR        MaxVGPR
 16        0 (O16)    32  (O16)
 15        0 (O16)    32  (O16)
 14        0 (O16)    32  (O16)
 13        0 (O16)    32  (O16)
 12       33 (O12)    40  (O12)
 11       33 (O12)    40  (O12)
 10       41 (O10)    48  (O10)
  9       49 (O9)     56  (O9)
  8       57 (O8)     64  (O8)
  7       65 (O7)     72  (O7)
  6       73 (O6)     80  (O6)
  5       81 (O5)     96  (O5)
  4       97 (O4)     128 (O4)
  3      129 (O3)     168 (O3)
  2      169 (O2)     256 (O2)
  1      169 (O2)     256 (O2)

 gfx1100w64 gfx1101w64:
Occ    MinVGPR        MaxVGPR
 16        0 (O16)    48  (O16)
 15        0 (O16)    48  (O16)
 14        0 (O16)    48  (O16)
 13        0 (O16)    48  (O16)
 12       49 (O12)    60  (O12)
 11       49 (O12)    60  (O12)
 10       61 (O10)    72  (O10)
  9       73 (O9)     84  (O9)
  8       85 (O8)     96  (O8)
  7       97 (O7)     108 (O7)
  6      109 (O6)     120 (O6)
  5      121 (O5)     144 (O5)
  4      145 (O4)     192 (O4)
  3      193 (O3)     252 (O3)
  2      253 (O2)     256 (O2)
  1      253 (O2)     256 (O2)

 gfx1030w32 gfx1031w32 gfx1032w32 gfx1033w32 gfx1034w32 gfx1035w32 gfx1036w32 gfx1102w32 gfx1103w32:
Occ    MinVGPR        MaxVGPR
 16        0 (O16)    64  (O16)
 15        0 (O16)    64  (O16)
 14        0 (O16)    64  (O16)
 13        0 (O16)    64  (O16)
 12       65 (O12)    80  (O12)
 11       65 (O12)    80  (O12)
 10       81 (O10)    96  (O10)
  9       97 (O9)     112 (O9)
  8      113 (O8)     128 (O8)
  7      129 (O7)     144 (O7)
  6      145 (O6)     160 (O6)
  5      161 (O5)     192 (O5)
  4      193 (O4)     256 (O4)
  3      193 (O4)     256 (O4)
  2      193 (O4)     256 (O4)
  1      193 (O4)     256 (O4)

 gfx1100w32 gfx1101w32:
Occ    MinVGPR        MaxVGPR
 16        0 (O16)    96  (O16)
 15        0 (O16)    96  (O16)
 14        0 (O16)    96  (O16)
 13        0 (O16)    96  (O16)
 12       97 (O12)    120 (O12)
 11       97 (O12)    120 (O12)
 10      121 (O10)    144 (O10)
  9      145 (O9)     168 (O9)
  8      169 (O8)     192 (O8)
  7      193 (O7)     216 (O7)
  6      217 (O6)     240 (O6)
  5      241 (O5)     256 (O5)
  4      241 (O5)     256 (O5)
  3      241 (O5)     256 (O5)
  2      241 (O5)     256 (O5)
  1      241 (O5)     256 (O5)

 gfx1010w64 gfx1011w64 gfx1012w64 gfx1013w64:
Occ    MinVGPR        MaxVGPR
 20        0 (O20)    24  (O20)
 19        0 (O20)    24  (O20)
 18       25 (O18)    28  (O18)
 17       25 (O18)    28  (O18)
 16       29 (O16)    32  (O16)
 15       29 (O16)    32  (O16)
 14       33 (O14)    36  (O14)
 13       33 (O14)    36  (O14)
 12       37 (O12)    40  (O12)
 11       41 (O11)    44  (O11)
 10       45 (O10)    48  (O10)
  9       49 (O9)     56  (O9)
  8       57 (O8)     64  (O8)
  7       65 (O7)     72  (O7)
  6       73 (O6)     84  (O6)
  5       85 (O5)     100 (O5)
  4      101 (O4)     128 (O4)
  3      129 (O3)     168 (O3)
  2      169 (O2)     256 (O2)
  1      169 (O2)     256 (O2)

 gfx1010w32 gfx1011w32 gfx1012w32 gfx1013w32:
Occ    MinVGPR        MaxVGPR
 20        0 (O20)    48  (O20)
 19        0 (O20)    48  (O20)
 18       49 (O18)    56  (O18)
 17       49 (O18)    56  (O18)
 16       57 (O16)    64  (O16)
 15       57 (O16)    64  (O16)
 14       65 (O14)    72  (O14)
 13       65 (O14)    72  (O14)
 12       73 (O12)    80  (O12)
 11       81 (O11)    88  (O11)
 10       89 (O10)    96  (O10)
  9       97 (O9)     112 (O9)
  8      113 (O8)     128 (O8)
  7      129 (O7)     144 (O7)
  6      145 (O6)     168 (O6)
  5      169 (O5)     200 (O5)
  4      201 (O4)     256 (O4)
  3      201 (O4)     256 (O4)
  2      201 (O4)     256 (O4)
  1      201 (O4)     256 (O4)

[arsenm]

I thought I noticed funny behavior of the min occupancy while working on D115559

llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp

20

Not sure why you need this

28–30

This is confusing. Probably should move this somewhere else and have a proper declaration for it. Also the name is too generic

32

static

61

raw_string_ostream or raw_svector_ostream

71

static

72–73

Can you go just off the features? This is going to be annoying to maintain (I thought we already had a feature to indicate it supports wave32 vs. wave64 separate from the mode itself, but I don't see it)

76–78

static const std::array?

80

static

86

DenseMap

98

Same as above

110

I'm not sure why you're printing everything to a string before printing it. Why not just directly print if printing is enabled?

121

Can you also check SGPRs and LDS?

[vpykhtin]

llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp

28–30

I just reuse the function from the llvm/unittests/Target/AMDGPU/DwarfRegMappings.cpp, but this can be changed.

61

I have to work with std::string and std::ostream because of the unittest framework.

72–73

The problem is that I cannot know the feature until I have GCNSubtarget ready, but then I have to recreate the machine. Maybe I can test if the GPU has w32 mode and then run it again with w64 mode.

86

DenseMap doesn't work with std::string as a key.

110

This is how unittest framework works. It's multitreaded and uses std::string and std::ostream everywhere. DenseMap doesn't work with std::string.

The actual output happens here as an indication of error:

118  EXPECT_TRUE(ErrStr.empty()) << ErrStr;

When the test pass it shouldn't print anything. I added "--print-cpu-reg-limits" command line option just to be able to print entire results if needed.

121

This is planned with subsequent patches.

[vpykhtin]

Per review issues:

• Don't rely on arch name, use feature bits for wave32.
• Refactored common code for unit tests, unified test namings.

[foad]

/// \returns Minimum number of VGPRs that meets given number of waves per
/// execution unit requirement supported by the subtarget.
unsigned getMinNumVGPRs(unsigned WavesPerEU) const;

/// \returns Maximum number of VGPRs that meets given number of waves per
/// execution unit requirement supported by the subtarget.
unsigned getMaxNumVGPRs(unsigned WavesPerEU) const;

I think your patch shows that this documentation needs improving here. My understanding is:

getMaxNumVGPRs returns the maximum number of VGPRs that you can use and still achieve at least the specified occupancy.
getMinNumVGPRs returns the minimum number of VGPRs that will prevent you from achieving more than the specified occupancy.

llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp

629

Why change terminology from "Occupancy" to "NumWaves"? AMDGPUBaseInfo seems to use "WavesPerEU" a lot instead of just "NumWaves".

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

995

This is divideCeil(NumVGPRs, Granule) * Granule. But how about adding uint64_t alignUp(uint64_t Value, uint64_t Align) to MathExtras.h (like alignDown but no need for the Skew parameter) and using it here?

[vpykhtin]

I think your patch shows that this documentation needs improving here. My understanding is:
getMaxNumVGPRs returns the maximum number of VGPRs that you can use and still achieve at least the specified occupancy.
getMinNumVGPRs returns the minimum number of VGPRs that will prevent you from achieving more than the specified occupancy.

Right, your description is precise, I'll change the documentation.

llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp

629

Do you think it would be more consistent to use WavesPerEU in AMDGPUBaseInfo? Like getNumWavesPerEUWithNumVGPRs?

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

995

I just didn't want to change the function body since I moved it but if I changed the name the body can be also imporved :)
MathExtras.h already has alignTo, I'll use it here.

inline uint64_t alignTo(uint64_t Value, uint64_t Align) {
  assert(Align != 0u && "Align can't be 0.");
  return (Value + Align - 1) / Align * Align;
}

[vpykhtin]

• Updated description for getMinNumVGPRs, getMaxNumVGPRs
• Used alignTo in getNumWavesWithNumVGPRs
• Rebased

[arsenm]

llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp

629

Yes

[vpykhtin]

• Changed function name to getNumWavesPerEUWithNumVGPRs.
• Rebased.

[arsenm]

LGTM with nits

llvm/lib/Target/AMDGPU/GCNSubtarget.h

1219 ↗

(On Diff #478597)

Remove "you from"

1225 ↗

(On Diff #478597)

s/"you can use"/"can be used"

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

995

I'd somewhat prefer non-moving changes to be separate, but doesn't really matter

llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp

101

StringMap?

[vpykhtin]

Fixed documentation.

[vpykhtin]

llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp

101

I need std::string as a key, StringMap uses StringRef but it doesn't own string data. Its not a big deal here because its only used on test fail or when printing.

[arsenm]

llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp

101

DenseMap

[vpykhtin]

llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp

101

DenseMap doesn't work with std::string, it's not specialized for it.

[foad]

I have no objection to committing this.

The only thing I'd like to try cleaning up in future is that getMinNumVGPRs relies on calling getNumWavesPerEUWithNumVGPRs with a NumVGPRs value greater than getAddressableNumVGPRs and still getting sensible answers, which seems to me like a case that getNumWavesPerEUWithNumVGPRs should not have to handle. I.e. I think we should be able to do this:

--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -1020,6 +1020,7 @@ unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) {
 
 unsigned getNumWavesWithNumVGPRs(const MCSubtargetInfo *STI,
                                  unsigned NumVGPRs) {
+  assert(NumVGPRs <= getAddressableNumVGPRs(STI));
   NumVGPRs = std::min(std::max(NumVGPRs, 1u), getAddressableNumVGPRs(STI));
   return std::min(getTotalNumVGPRs(STI) /
                       unsigned(alignTo(NumVGPRs, getVGPRAllocGranule(STI))),

but currently we can't.

llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp

101

StringMap<std:string>?

[jmmartinez]

llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp

33

Maybe replace std::string by StringRef ?

101

I need std::string as a key, StringMap uses StringRef but it doesn't own string data. Its not a big deal here because its only used on test fail or when printing.

@vpykhtin What do you mean that it doesn't own the string data? When creating the StringMapEntry (see StringMapEntryBase::allocateWithKey) it performs a copy of the input StringRef into its own internal buffer.

The awkward side is that you'll have to use raw_string_ostream for building the ErrStr as shown below (notice the call to .first() instead of .first):

std::string ErrStr;
raw_string_ostream OS(ErrStr);
for (auto &P : TablePerCPUs) {
  for (auto &CPUName : P.second)
    OS << ' ' << CPUName;
  OS << ":\nOcc    Min" << RegName << "        Max" << RegName << '\n'
     << P.first() << '\n';
}
EXPECT_TRUE(ErrStr.empty()) << ErrStr;

[jmmartinez]

llvm/unittests/Target/AMDGPU/CMakeLists.txt

6–9

I think AMDGPUUtils is missing from the list. Otherwise I get an undefined reference to llvm::AMDGPU::IsaInfo::getMaxNumVGPRs(llvm::MCSubtargetInfo const*, unsigned int) on my local build.

[vpykhtin]

• Replaced call to getNumWavesPerEUWithNumVGPRs with NumVGPRs above addressable with better code.
• Assert cannot be added right now at getNumWavesPerEUWithNumVGPRs because it fail other tests.
• Better naming.

[vpykhtin]

The only thing I'd like to try cleaning up in future is that getMinNumVGPRs relies on calling getNumWavesPerEUWithNumVGPRs with a NumVGPRs value greater than getAddressableNumVGPRs and still getting sensible answers, which seems to me like a case that getNumWavesPerEUWithNumVGPRs should not have to handle.

@foad I replaced call to getNumWavesPerEUWithNumVGPRs with a NumVGPRs value greater than getAddressableNumVGPRs with:

if (MaxNumVGPRs == alignDown(TotNumVGPRs / MaxWavesPerEU, Granule))
  return 0;

but I cannot add assert now because it fails other tests at the moment.

llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp

101

This way you specify Value type as std::string but not the key.

101

<< P.first() << '\n';

I don't see () in my code.

101

When creating the StringMapEntry (see StringMapEntryBase::allocateWithKey) it performs a copy of the input StringRef into its own internal buffer

Ok, but here is the final argument: strings are sorted at the map and this gives nice picture at output :) This map is only used on error or printing, it's not used during usual llvm-check routine so there is no penalty.

llvm/unittests/Target/AMDGPU/CMakeLists.txt

6–9

not sure, my build is ok, I build AMDGPUTests executable. Probably should go to another patch

[jmmartinez]

llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp

101

Sorry, I might have not been clear. I meant the .first() in my code snippet.

101

Pretty-printing as an argument sounds good to me.

llvm/unittests/Target/AMDGPU/CMakeLists.txt

6–9

Are you building with -DBUILD_SHARED_LIBS=OFF ? If that's the case please consier reproducing the problem with -DBUILD_SHARED_LIBS=ON and adding the modification I suggested, to avoid breaking other peoples's builds.

[vpykhtin]

• Rebased.
• Added AMDGPUUtils reference to CMakelists.txt

[vpykhtin]

llvm/unittests/Target/AMDGPU/CMakeLists.txt

6–9

Done, thank you!

llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp

if (SGPRs <= 64)
     return 8;
   if (SGPRs <= 72)
     return 7;
   if (SGPRs <= 80)
     return 6;
   return 5;
 }
 
-unsigned GCNSubtarget::getOccupancyWithNumVGPRs(unsigned VGPRs) const {
-  unsigned MaxWaves = getMaxWavesPerEU();
-  unsigned Granule = getVGPRAllocGranule();
-  if (VGPRs < Granule)
-    return MaxWaves;
-  unsigned RoundedRegs = ((VGPRs + Granule - 1) / Granule) * Granule;
-  return std::min(std::max(getTotalNumVGPRs() / RoundedRegs, 1u), MaxWaves);
+unsigned GCNSubtarget::getOccupancyWithNumVGPRs(unsigned NumVGPRs) const {
+  return AMDGPU::IsaInfo::getNumWavesWithNumVGPRs(this, NumVGPRs);
 }
 
 unsigned
 GCNSubtarget::getBaseReservedNumSGPRs(const bool HasFlatScratch) const {
   if (getGeneration() >= AMDGPUSubtarget::GFX10)
     return 2; // VCC. FLAT_SCRATCH and XNACK are no longer in SGPRs.
 
   if (HasFlatScratch || HasArchitectedFlatScratch) {

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h

 /// \returns Minimum number of VGPRs that meets given number of waves per
 /// execution unit requirement for given subtarget \p STI.
 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU);
 
 /// \returns Maximum number of VGPRs that meets given number of waves per
 /// execution unit requirement for given subtarget \p STI.
 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU);
 
+/// \returns Number of waves reachable for a given \p NumVGPRs usage for given
+/// subtarget \p STI.
+unsigned getNumWavesWithNumVGPRs(const MCSubtargetInfo *STI, unsigned NumVGPRs);
+
 /// \returns Number of VGPR blocks needed for given subtarget \p STI when
 /// \p NumVGPRs are used.
 ///
 /// For subtargets which support it, \p EnableWavefrontSize32 should match the
 /// ENABLE_WAVEFRONT_SIZE32 kernel descriptor field.
 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs,
                           Optional<bool> EnableWavefrontSize32 = None);
 

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

 }
 
 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) {
   if (STI->getFeatureBits().test(FeatureGFX90AInsts))
     return 512;
   return 256;
 }
 
+unsigned getNumWavesWithNumVGPRs(const MCSubtargetInfo *STI,
+                                 unsigned NumVGPRs) {
+  unsigned MaxWaves = getMaxWavesPerEU(STI);
+  unsigned Granule = getVGPRAllocGranule(STI);
+  if (NumVGPRs < Granule)
+    return MaxWaves;
+  unsigned RoundedRegs = ((NumVGPRs + Granule - 1) / Granule) * Granule;
+  return std::min(std::max(getTotalNumVGPRs(STI) / RoundedRegs, 1u), MaxWaves);
+}
+
 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
   assert(WavesPerEU != 0);
 
-  if (WavesPerEU >= getMaxWavesPerEU(STI))
+  unsigned MaxNumWaves = getMaxWavesPerEU(STI);
+  if (WavesPerEU >= MaxNumWaves)
+    return 0;
+
+  unsigned NumVGPRs = getTotalNumVGPRs(STI);
+  unsigned Granule = getVGPRAllocGranule(STI);
+  unsigned MaxNumVGPRs = alignDown(NumVGPRs / WavesPerEU, Granule);
+
+  unsigned NumWaves = getNumWavesWithNumVGPRs(STI, MaxNumVGPRs);
+  if (NumWaves >= MaxNumWaves)
     return 0;
-  unsigned MinNumVGPRs =
-      alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1),
-                getVGPRAllocGranule(STI)) + 1;
-  return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI));
+
+  unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI);
+  unsigned MinNumWaves = getNumWavesWithNumVGPRs(STI, AddressableNumVGPRs);
+  if (NumWaves < MinNumWaves)
+    return getMinNumVGPRs(STI, MinNumWaves);
+
+  unsigned MaxNumVGPRsNext = alignDown(NumVGPRs / (WavesPerEU + 1), Granule);
+  unsigned MinNumVGPRs = 1 + std::min(MaxNumVGPRs - Granule, MaxNumVGPRsNext);
+  return std::min(MinNumVGPRs, AddressableNumVGPRs);
 }
 
 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
   assert(WavesPerEU != 0);
 
   unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU,
                                    getVGPRAllocGranule(STI));
   unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI);

llvm/test/CodeGen/AMDGPU/occupancy-levels.ll

 ; GFX1030:    ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
 define amdgpu_kernel void @limited_occupancy_18() #1 {
   ret void
 }
 
 ; GCN-LABEL: {{^}}limited_occupancy_19:
 ; GFX9:       ; Occupancy: 10
-; GFX1010:    ; Occupancy: 18
+; GFX1010:    ; Occupancy: 20
 ; GFX1030:    ; Occupancy: 16
 ; GFX1100:    ; Occupancy: 16
 define amdgpu_kernel void @limited_occupancy_19() #2 {
   ret void
 }
 
 ; GCN-LABEL: {{^}}used_24_vgprs:
 ; GFX9:       ; Occupancy: 10

llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp

+//===- llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUTargetMachine.h"
+#include "GCNSubtarget.h"
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/MC/MCTargetOptions.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/TargetParser.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Target/TargetMachine.h"
+#include "gtest/gtest.h"
+#include <thread>
+
+using namespace llvm;
+
+static cl::opt<bool> PrintCpuRegLimits(
+    "print-cpu-reg-limits", cl::NotHidden, cl::init(false),
+    cl::desc("force printing per AMDGPU CPU register limits"));
+
+// implementation is in the llvm/unittests/Target/AMDGPU/DwarfRegMappings.cpp
+std::unique_ptr<const GCNTargetMachine>
+createTargetMachine(std::string TStr, StringRef CPU, StringRef FS);
+
+bool checkMinMax(std::stringstream &OS, unsigned Occ, unsigned MinOcc,
+                 unsigned MaxOcc, std::function<unsigned(unsigned)> GetOcc,
+                 std::function<unsigned(unsigned)> GetMinGPRs,
+                 std::function<unsigned(unsigned)> GetMaxGPRs) {
+  bool MinValid = true, MaxValid = true, RangeValid = true;
+  unsigned MinGPRs = GetMinGPRs(Occ);
+  unsigned MaxGPRs = GetMaxGPRs(Occ);
+  unsigned RealOcc;
+
+  if (MinGPRs >= MaxGPRs)
+    RangeValid = false;
+  else {
+    RealOcc = GetOcc(MinGPRs);
+    for (unsigned NumRegs = MinGPRs + 1; NumRegs <= MaxGPRs; ++NumRegs) {
+      if (RealOcc != GetOcc(NumRegs)) {
+        RangeValid = false;
+        break;
+      }
+    }
+  }
+
+  if (RangeValid && RealOcc > MinOcc && RealOcc <= MaxOcc) {
+    if (MinGPRs > 0 && GetOcc(MinGPRs - 1) <= RealOcc)
+      MinValid = false;
+
+    if (GetOcc(MaxGPRs + 1) >= RealOcc)
+      MaxValid = false;
+  }
+
+  std::stringstream MinStr;
+  MinStr << (MinValid ? ' ' : '<') << ' ' << std::setw(3) << MinGPRs << " (O"
+         << GetOcc(MinGPRs) << ") " << (RangeValid ? ' ' : 'R');
+
+  OS << std::left << std::setw(15) << MinStr.str() << std::setw(3) << MaxGPRs
+     << " (O" << GetOcc(MaxGPRs) << ')' << (MaxValid ? "" : " >");
+
+  return MinValid && MaxValid && RangeValid;
+}
+
+bool hasW32(StringRef CanonicCPUName) {
+  return CanonicCPUName.starts_with("gfx10") ||
+         CanonicCPUName.starts_with("gfx11");
+}
+
+static std::vector<std::pair<StringRef, StringRef>>
+    EmptySet = {{"", ""}},
+    W32W64 = {{"+wavefrontsize32", "w32"}, {"+wavefrontsize64", "w64"}};
+
+void testGPRLimits(
+    const char *RegName, bool TestW32W64,
+    std::function<bool(std::stringstream &, unsigned, GCNSubtarget &)> test) {
+  SmallVector<StringRef> CPUs;
+  AMDGPU::fillValidArchListAMDGCN(CPUs);
+
+  std::map<std::string, SmallVector<std::string>> TablePerCPUs;
+  for (auto CPUName : CPUs) {
+    auto CanonCPUName =
+        AMDGPU::getArchNameAMDGCN(AMDGPU::parseArchAMDGCN(CPUName));
+
+    auto *AttrSet = (TestW32W64 && hasW32(CanonCPUName)) ? &W32W64 : &EmptySet;
+    for (auto &P : *AttrSet) {
+      auto TM = createTargetMachine("amdgcn-amd-", CPUName, P.first);
+      if (!TM)
+        continue;
+      GCNSubtarget ST(TM->getTargetTriple(), std::string(TM->getTargetCPU()),
+                      std::string(TM->getTargetFeatureString()), *TM);
+      std::stringstream Table;
+      bool Success = true;
+      unsigned MaxOcc = ST.getMaxWavesPerEU();
+      for (unsigned Occ = MaxOcc; Occ > 0; --Occ) {
+        Table << std::right << std::setw(3) << Occ << "    ";
+        Success = test(Table, Occ, ST) && Success;
+        Table << '\n';
+      }
+      if (!Success || PrintCpuRegLimits)
+        TablePerCPUs[Table.str()].push_back((CanonCPUName + P.second).str());
+    }
+  }
+  std::stringstream OS;
+  for (auto &P : TablePerCPUs) {
+    for (auto &CPUName : P.second)
+      OS << ' ' << CPUName;
+    OS << ":\nOcc    Min" << RegName << "        Max" << RegName << '\n'
+       << P.first << '\n';
+  }
+  auto ErrStr = OS.str();
+  EXPECT_TRUE(ErrStr.empty()) << ErrStr;
+}
+
+TEST(AMDGPU, TestVGPRLimitsPerOccupancy) {
+  testGPRLimits("VGPR", true, [](std::stringstream &OS, unsigned Occ,
+                                 GCNSubtarget &ST) {
+    unsigned MaxVGPRNum = ST.getAddressableNumVGPRs();
+    return checkMinMax(
+        OS, Occ, ST.getOccupancyWithNumVGPRs(MaxVGPRNum), ST.getMaxWavesPerEU(),
+        [&](unsigned NumGPRs) { return ST.getOccupancyWithNumVGPRs(NumGPRs); },
+        [&](unsigned Occ) { return ST.getMinNumVGPRs(Occ); },
+        [&](unsigned Occ) { return ST.getMaxNumVGPRs(Occ); });
+  });
+}

llvm/unittests/Target/AMDGPU/CMakeLists.txt

 include_directories(
   ${PROJECT_SOURCE_DIR}/lib/Target/AMDGPU
   ${PROJECT_BINARY_DIR}/lib/Target/AMDGPU
   )
 
 set(LLVM_LINK_COMPONENTS
   AMDGPUCodeGen
   AMDGPUDesc
   AMDGPUInfo
   CodeGen
   Core
   MC
   Support
   )
 
 add_llvm_target_unittest(AMDGPUTests
+  AMDGPUUnitTests.cpp
   DwarfRegMappings.cpp
   ExecMayBeModifiedBeforeAnyUse.cpp
   )
 
 set_property(TARGET AMDGPUTests PROPERTY FOLDER "Tests/UnitTests/TargetTests")