AMDGPU/SI: Set INDEX_STRIDE for scratch coalescing

[mareko]

Summary

Mesa and other users must set this to enable coalescing:

• STRIDE = 0
• SWIZZLE_ENABLE = 1

DiRT Showdown - apitrace results:

Before: 17.73 FPS
After:  19.77 FPS

[arsenm]

Can you add a test that shows the bit change in the resource setup? I would expect the tests would be breaking on this already

lib/Target/AMDGPU/SIInstrInfo.cpp

3098 ↗

(On Diff #60033)

C++ style comments

[mareko]

Why would it be breaking? This is the only place that clears DATA_FORMAT and sets ADD_TID_ENABLE, both of which are required for scratch to even work. Or did you mean something else?

[arsenm]

In D21136#452923, @mareko wrote:

Why would it be breaking? This is the only place that clears DATA_FORMAT and sets ADD_TID_ENABLE, both of which are required for scratch to even work. Or did you mean something else?

The tests check the complete 32-bit value moved into the register

[mareko]

Fixed the tests and the comment.

[qcolombet]

+llvm-commit

[arsenm]

LGTM

lib/Target/AMDGPU/SIInstrInfo.cpp

3098 ↗

(On Diff #60215)

Ditto

lib/Target/AMDGPU/SIInstrInfo.h

553 ↗

(On Diff #60215)

UINT64_C() instead of LL

llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.h

namespace AMDGPU {
 
   LLVM_READONLY
   int getAtomicRetOp(uint16_t Opcode);
 
   LLVM_READONLY
   int getAtomicNoRetOp(uint16_t Opcode);
 
   const uint64_t RSRC_DATA_FORMAT = 0xf00000000000LL;
-  const uint64_t RSRC_TID_ENABLE = 1LL << 55;
-  const uint64_t RSRC_ELEMENT_SIZE_SHIFT = 51;
+  const uint64_t RSRC_ELEMENT_SIZE_SHIFT = (32 + 19);
+  const uint64_t RSRC_INDEX_STRIDE_SHIFT = (32 + 21);
+  const uint64_t RSRC_TID_ENABLE = UINT64_C(1) << (32 + 23);
 } // End namespace AMDGPU
 
 namespace SI {
 namespace KernelInputOffsets {
 
 /// Offsets in bytes from the start of the input buffer
 enum Offsets {
   NGROUPS_X = 0,

llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.cpp

 
 uint64_t SIInstrInfo::getScratchRsrcWords23() const {
   uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
                     AMDGPU::RSRC_TID_ENABLE |
                     0xffffffff; // Size;
 
   uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize()) - 1;
 
-  Rsrc23 |= (EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT);
+  Rsrc23 |= (EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT) |
+            // IndexStride = 64
+            (UINT64_C(3) << AMDGPU::RSRC_INDEX_STRIDE_SHIFT);
 
   // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
   // Clear them unless we want a huge stride.
   if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
     Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
 
   return Rsrc23;
 }

llvm/trunk/test/CodeGen/AMDGPU/large-alloca-compute.ll

 ; RUN: llc -march=amdgcn -mcpu=bonaire -show-mc-encoding < %s | FileCheck -check-prefix=GCN -check-prefix=CI -check-prefix=ALL %s
 ; RUN: llc -march=amdgcn -mcpu=carrizo --show-mc-encoding < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=ALL %s
 ; RUN: llc -march=amdgcn -mcpu=bonaire -mtriple=amdgcn-unknown-amdhsa < %s -mattr=-flat-for-global | FileCheck -check-prefix=GCNHSA -check-prefix=CIHSA -check-prefix=ALL %s
 ; RUN: llc -march=amdgcn -mcpu=carrizo -mtriple=amdgcn-unknown-amdhsa -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCNHSA -check-prefix=VIHSA -check-prefix=ALL %s
 
 ; FIXME: align on alloca seems to be ignored for private_segment_alignment
 
 ; ALL-LABEL: {{^}}large_alloca_compute_shader:
 
 ; GCN-DAG: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0
 ; GCN-DAG: ; fixup A - offset: 4, value: SCRATCH_RSRC_DWORD0, kind: FK_Data_4
 ; GCN-DAG: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1
 ; GCN-DAG: ; fixup A - offset: 4, value: SCRATCH_RSRC_DWORD1, kind: FK_Data_4
 ; GCN-DAG: s_mov_b32 s{{[0-9]+}}, -1
-; CI-DAG: s_mov_b32 s{{[0-9]+}}, 0x88f000
-; VI-DAG: s_mov_b32 s{{[0-9]+}}, 0x880000
+; CI-DAG: s_mov_b32 s{{[0-9]+}}, 0xe8f000
+; VI-DAG: s_mov_b32 s{{[0-9]+}}, 0xe80000
 
 
 ; GCNHSA: .amd_kernel_code_t
 
 ; GCNHSA: compute_pgm_rsrc2_scratch_en = 1
 ; GCNHSA: compute_pgm_rsrc2_user_sgpr = 8
 ; GCNHSA: compute_pgm_rsrc2_tgid_x_en = 1
 ; GCNHSA: compute_pgm_rsrc2_tgid_y_en = 0

llvm/trunk/test/CodeGen/AMDGPU/large-alloca-graphics.ll

 ; RUN: llc -march=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefix=GCN -check-prefix=CI -check-prefix=ALL %s
 ; RUN: llc -march=amdgcn -mcpu=carrizo < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=ALL %s
 
 ; ALL-LABEL: {{^}}large_alloca_pixel_shader:
 ; GCN-DAG: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
 ; GCN-DAG: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
 ; GCN-DAG: s_mov_b32 s10, -1
-; CI-DAG: s_mov_b32 s11, 0x88f000
-; VI-DAG: s_mov_b32 s11, 0x880000
+; CI-DAG: s_mov_b32 s11, 0xe8f000
+; VI-DAG: s_mov_b32 s11, 0xe80000
 
 ; GCN: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, s[8:11], s0 offen
 ; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, s[8:11], s0 offen
 
 ; ALL: ; ScratchSize: 32772
 define amdgpu_ps void @large_alloca_pixel_shader(i32 %x, i32 %y) #0 {
   %large = alloca [8192 x i32], align 4
   %gep = getelementptr [8192 x i32], [8192 x i32]* %large, i32 0, i32 8191
   store volatile i32 %x, i32* %gep
   %gep1 = getelementptr [8192 x i32], [8192 x i32]* %large, i32 0, i32 %y
   %val = load volatile i32, i32* %gep1
   store volatile i32 %val, i32 addrspace(1)* undef
   ret void
 }
 
 ; ALL-LABEL: {{^}}large_alloca_pixel_shader_inreg:
 ; GCN-DAG: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
 ; GCN-DAG: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
 ; GCN-DAG: s_mov_b32 s10, -1
-; CI-DAG: s_mov_b32 s11, 0x88f000
-; VI-DAG: s_mov_b32 s11, 0x880000
+; CI-DAG: s_mov_b32 s11, 0xe8f000
+; VI-DAG: s_mov_b32 s11, 0xe80000
 
 ; GCN: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, s[8:11], s2 offen
 ; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, s[8:11], s2 offen
 
 ; ALL: ; ScratchSize: 32772
 define amdgpu_ps void @large_alloca_pixel_shader_inreg(i32 inreg %x, i32 inreg %y) #0 {
   %large = alloca [8192 x i32], align 4
   %gep = getelementptr [8192 x i32], [8192 x i32]* %large, i32 0, i32 8191
   store volatile i32 %x, i32* %gep
   %gep1 = getelementptr [8192 x i32], [8192 x i32]* %large, i32 0, i32 %y
   %val = load volatile i32, i32* %gep1
   store volatile i32 %val, i32 addrspace(1)* undef
   ret void
 }
 
 attributes #0 = { nounwind  }

llvm/trunk/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll

 ; HSA: workitem_private_segment_byte_size = 1024
 
 ; GCN-NOT: flat_scr
 
 ; GCNMESA-DAG: s_mov_b32 s16, s3
 ; GCNMESA-DAG: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GCNMESA--DAG: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GCNMESA-DAG: s_mov_b32 s14, -1
-; SIMESA-DAG: s_mov_b32 s15, 0x88f000
-; VIMESA-DAG: s_mov_b32 s15, 0x880000
+; SIMESA-DAG: s_mov_b32 s15, 0xe8f000
+; VIMESA-DAG: s_mov_b32 s15, 0xe80000
 
 
 ; GCN: buffer_store_dword {{v[0-9]+}}, off, s[12:15], s16 offset:{{[0-9]+}} ; 4-byte Folded Spill
 
 ; GCN: buffer_store_dword {{v[0-9]}}, {{v[0-9]+}}, s[12:15], s16 offen offset:{{[0-9]+}}
 ; GCN: buffer_store_dword {{v[0-9]}}, {{v[0-9]+}}, s[12:15], s16 offen offset:{{[0-9]+}}
 ; GCN: buffer_store_dword {{v[0-9]}}, {{v[0-9]+}}, s[12:15], s16 offen offset:{{[0-9]+}}
 ; GCN: buffer_store_dword {{v[0-9]}}, {{v[0-9]+}}, s[12:15], s16 offen offset:{{[0-9]+}}

llvm/trunk/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll

 ; FIXME: The unused arguments are removed
 
 ; GCN-LABEL: {{^}}main:
 
 ; GCN-DAG: s_mov_b32 s13, s12
 ; GCN-DAG: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
 ; GCN-DAG: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
 ; GCN-DAG: s_mov_b32 s18, -1
-; SI-DAG: s_mov_b32 s19, 0x88f000
-; VI-DAG: s_mov_b32 s19, 0x880000
+; SI-DAG: s_mov_b32 s19, 0xe8f000
+; VI-DAG: s_mov_b32 s19, 0xe80000
 
 ; s13 is offset system SGPR
 ; GCN: buffer_store_dword {{v[0-9]+}}, off, s[16:19], s13 offset:{{[0-9]+}} ; 16-byte Folded Spill
 ; GCN: buffer_load_dword v{{[0-9]+}}, off, s[16:19], s13 offset:{{[0-9]+}} ; 16-byte Folded Reload
 
 ; GCN: NumVgprs: 256
 ; GCN: ScratchSize: 1024