[AMDGPU][GFX8][GFX9] Allow LDS direct reads for DWORDX2/X3/X4

[dp]

Summary

See bug 37653: https://bugs.llvm.org/show_bug.cgi?id=37653

[artem.tamazov]

LGTM

[rampitec]

I do not understand how may this work. The LDS address has the component TID * 4. I.e. neighboring lanes shall overwrite each other. I can only imagine a scenario with a very specific EXEC mask where it might be different.

lib/Target/AMDGPU/BUFInstructions.td

defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <
   "buffer_load_dwordx2", VReg_64, v2i32, mubuf_load
 >;
 defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads <
   "buffer_load_dwordx3", VReg_96, untyped, mubuf_load
 >;
 defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads <
   "buffer_load_dwordx4", VReg_128, v4i32, mubuf_load
 >;
+
+// This is not described in AMD documentation,
+// but 'lds' versions of these opcodes are available
+// in at least GFX8+ chips. See Bug 37653.
+let SubtargetPredicate = isVI in {
+defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads <
+  "buffer_load_dwordx2", VReg_64, v2i32, null_frag, 0, 1
+>;
+defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads <
+  "buffer_load_dwordx3", VReg_96, untyped, null_frag, 0, 1
+>;
+defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads <
+  "buffer_load_dwordx4", VReg_128, v4i32, null_frag, 0, 1
+>;
+}
+
 defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <
   "buffer_store_byte", VGPR_32, i32, truncstorei8_global
 >;
 defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <
   "buffer_store_short", VGPR_32, i32, truncstorei16_global
 >;
 defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
   "buffer_store_dword", VGPR_32, i32, store_global
let SubtargetPredicate = HasPackedD16VMem in {
   defm BUFFER_STORE_FORMAT_D16_XYZ    : MUBUF_Real_AllAddr_vi <0x0e>;
   defm BUFFER_STORE_FORMAT_D16_XYZW   : MUBUF_Real_AllAddr_vi <0x0f>;
 } // End HasPackedD16VMem.
 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x10>;
 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x11>;
 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_Lds_vi <0x12>;
 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_Lds_vi <0x13>;
 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_Lds_vi <0x14>;
-defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_vi <0x15>;
-defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_vi <0x16>;
-defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_vi <0x17>;
+defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_Lds_vi <0x15>;
+defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_Lds_vi <0x16>;
+defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_Lds_vi <0x17>;
 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_vi <0x18>;
 defm BUFFER_STORE_BYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x19>;
 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_vi <0x1a>;
 defm BUFFER_STORE_SHORT_D16_HI  : MUBUF_Real_AllAddr_vi <0x1b>;
 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_vi <0x1c>;
 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_vi <0x1d>;
 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_vi <0x1e>;
 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_vi <0x1f>;

test/MC/AMDGPU/mubuf.s

 buffer_store_lds_dword s[4:7], s0 offset:4095 lds
 // NOSICI: error: not a valid operand.
 // VI: buffer_store_lds_dword s[4:7], s0 offset:4095 lds ; encoding: [0xff,0x0f,0xf5,0xe0,0x00,0x00,0x01,0x00]
 
 buffer_store_lds_dword s[4:7], s8 offset:4 lds glc slc
 // NOSICI: error: not a valid operand.
 // VI: buffer_store_lds_dword s[4:7], s8 offset:4 lds glc slc ; encoding: [0x04,0x40,0xf7,0xe0,0x00,0x00,0x01,0x08]
 
+buffer_load_dwordx2 v[1:2], off, s[4:7], s1 lds
+// NOSICI: error: instruction not supported on this GPU
+// VI:   buffer_load_dwordx2 v[1:2], off, s[4:7], s1 lds ; encoding: [0x00,0x00,0x55,0xe0,0x00,0x01,0x01,0x01]
+
+buffer_load_dwordx3 v[0:2], off, s[4:7], s0 offset:4095 lds
+// NOSICI: error: instruction not supported on this GPU
+// VI:   buffer_load_dwordx3 v[0:2], off, s[4:7], s0 offset:4095 lds ; encoding: [0xff,0x0f,0x59,0xe0,0x00,0x00,0x01,0x00]
+
+buffer_load_dwordx4 v[1:4], off, s[4:7], s1 lds
+// NOSICI: error: instruction not supported on this GPU
+// VI:   buffer_load_dwordx4 v[1:4], off, s[4:7], s1 lds ; encoding: [0x00,0x00,0x5d,0xe0,0x00,0x01,0x01,0x01]
+
 //===----------------------------------------------------------------------===//
 // Errors handling
 //===----------------------------------------------------------------------===//
 
 buffer_load_sbyte v5, off, s[8:11], s3 lds tfe
 // NOSICIVI: error: invalid operand for instruction
 
 buffer_load_dword v5, off, s[8:11], s3 tfe lds