[AMDGPU][GFX9] Support base+soffset+offset SMEM stores.

[dp]

LGTM.

llvm/lib/Target/AMDGPU/SMInstructions.td

602

This is a matter of taste, but I'd have renamed SMEM_Real_Store_Probe_vi to SMEM_Real_Store_Base_vi.

[kosarev]

llvm/lib/Target/AMDGPU/SMInstructions.td

602

Sounds a good idea. Went to rGb4dbcba3b7b7.

llvm/lib/Target/AMDGPU/SMInstructions.td

let SOffsetEn = 1, Offset = ?, Subtarget = SIEncodingFamily.GFX9,
         AsmVariantName = "NonParsable" in
     def _SGPR_alt_gfx9 : SMEM_Real_Load_vi <op, ps#"_SGPR",
                                             (ins SReg_32:$soffset)>;
     def _SGPR_IMM_gfx9 : SMEM_Real_Load_vi <
       op, ps#"_SGPR_IMM", (ins SReg_32:$soffset, smem_offset_mod:$offset)>;
   }
 }
 
-class SMEM_Real_Store_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> {
+class SMEM_Real_Store_Probe_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> {
   // encoding
   bits<7> sdata;
 
   let sdst = ?;
   let Inst{12-6}  = !if(ps.has_sdst, sdata{6-0}, ?);
 }
 
-multiclass SM_Real_Stores_vi<bits<8> op, string ps,
-                            SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM),
-                            SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> {
-  def _IMM_vi : SMEM_Real_Store_vi <op, immPs> {
-    let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol);
-  }
+class SMEM_Real_Store_vi <bits<8> op, string ps, dag offsets>
+    : SMEM_Real_Store_Probe_vi <op, !cast<SM_Pseudo>(ps)> {
+  RegisterClass SrcClass = !cast<SM_Store_Pseudo>(ps).SrcClass;
+  RegisterClass BaseClass = !cast<SM_Store_Pseudo>(ps).BaseClass;
+  let InOperandList = !con((ins SrcClass:$sdata, BaseClass:$sbase),
+                           offsets, (ins CPol:$cpol));
 
-  def _SGPR_vi : SMEM_Real_Store_vi <op, sgprPs> {
-    let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol);
+multiclass SM_Real_Stores_vi<bits<8> op, string ps> {
+  def _IMM_vi : SMEM_Real_Store_vi <op, ps#_IMM, (ins smem_offset:$offset)>;
+  def _SGPR_vi : SMEM_Real_Store_vi <op, ps#_SGPR, (ins SReg_32:$soffset)>;
+  let IsGFX9Specific = true in {
+    // The alternative GFX9 SGPR encoding using soffset to encode the
+    // offset register. Not available in assembler and goes to the GFX9
+    // encoding family to avoid conflicts with the primary SGPR variant.
+    let SOffsetEn = 1, Offset = ?, Subtarget = SIEncodingFamily.GFX9,
+        AsmVariantName = "NonParsable" in
+    def _SGPR_alt_gfx9 : SMEM_Real_Store_vi <op, ps#"_SGPR",
+                                             (ins SReg_32:$soffset)>;
+    def _SGPR_IMM_gfx9 : SMEM_Real_Store_vi <
+      op, ps#"_SGPR_IMM", (ins SReg_32:$soffset, smem_offset_mod:$offset)>;
   }
 }
 
 multiclass SM_Real_Probe_vi<bits<8> op, string ps> {
-  def _IMM_vi  : SMEM_Real_Store_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>;
-  def _SGPR_vi : SMEM_Real_Store_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>;
+  def _IMM_vi  : SMEM_Real_Store_Probe_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>;
+  def _SGPR_vi : SMEM_Real_Store_Probe_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>;
 }
 
 defm S_LOAD_DWORD           : SM_Real_Loads_vi <0x00, "S_LOAD_DWORD">;
 defm S_LOAD_DWORDX2         : SM_Real_Loads_vi <0x01, "S_LOAD_DWORDX2">;
 defm S_LOAD_DWORDX4         : SM_Real_Loads_vi <0x02, "S_LOAD_DWORDX4">;
 defm S_LOAD_DWORDX8         : SM_Real_Loads_vi <0x03, "S_LOAD_DWORDX8">;
 defm S_LOAD_DWORDX16        : SM_Real_Loads_vi <0x04, "S_LOAD_DWORDX16">;
 defm S_BUFFER_LOAD_DWORD    : SM_Real_Loads_vi <0x08, "S_BUFFER_LOAD_DWORD">;

llvm/test/MC/AMDGPU/gfx9_asm_smem.s

 // CHECK: [0x42,0x00,0x40,0xc0,0x7b,0x00,0x00,0x00]
 
 s_store_dword s1, s[4:5], m0
 // CHECK: [0x42,0x00,0x40,0xc0,0x7c,0x00,0x00,0x00]
 
 s_store_dword s1, s[4:5], 0x0
 // CHECK: [0x42,0x00,0x42,0xc0,0x00,0x00,0x00,0x00]
 
+s_store_dword s1, s[4:5], s7 offset:0x12345
+// CHECK: [0x42,0x40,0x42,0xc0,0x45,0x23,0x01,0x0e]
+
 s_store_dword s1, s[4:5], s0 glc
 // CHECK: [0x42,0x00,0x41,0xc0,0x00,0x00,0x00,0x00]
 
 s_store_dwordx2 s[2:3], s[4:5], s0
 // CHECK: [0x82,0x00,0x44,0xc0,0x00,0x00,0x00,0x00]
 
 s_store_dwordx2 s[4:5], s[4:5], s0
 // CHECK: [0x02,0x01,0x44,0xc0,0x00,0x00,0x00,0x00]

llvm/test/MC/Disassembler/AMDGPU/gfx8_dasm_all.txt

 0x02,0x05,0x32,0xc0,0xff,0xff,0x07,0x00
 
 # CHECK: s_buffer_load_dwordx16 s[20:35], s[4:7], s2 glc ; encoding: [0x02,0x05,0x31,0xc0,0x02,0x00,0x00,0x00]
 0x02,0x05,0x31,0xc0,0x02,0x00,0x00,0x00
 
 # CHECK: s_store_dword s1, s[4:5], m0            ; encoding: [0x42,0x00,0x40,0xc0,0x7c,0x00,0x00,0x00]
 0x42,0x00,0x40,0xc0,0x7c,0x00,0x00,0x00
 
+# Make sure that raising the GFX9 soffset_en bit doesn't affect GFX8
+# decoding.
+# CHECK: s_store_dword s1, s[4:5], m0            ; encoding: [0x42,0x00,0x40,0xc0,0x7c,0x00,0x00,0x00]
+0x42,0x40,0x40,0xc0,0x7c,0x00,0x00,0x00
+
 # CHECK: s_store_dword s101, s[4:5], m0          ; encoding: [0x42,0x19,0x40,0xc0,0x7c,0x00,0x00,0x00]
 0x42,0x19,0x40,0xc0,0x7c,0x00,0x00,0x00
 
 # CHECK: s_store_dword flat_scratch_lo, s[4:5], m0 ; encoding: [0x82,0x19,0x40,0xc0,0x7c,0x00,0x00,0x00]
 0x82,0x19,0x40,0xc0,0x7c,0x00,0x00,0x00
 
 # CHECK: s_store_dword flat_scratch_hi, s[4:5], m0 ; encoding: [0xc2,0x19,0x40,0xc0,0x7c,0x00,0x00,0x00]
 0xc2,0x19,0x40,0xc0,0x7c,0x00,0x00,0x00

 0x77,0x00,0x40,0xc0,0x7c,0x00,0x00,0x00
 
 # CHECK: s_store_dword s1, ttmp[10:11], m0       ; encoding: [0x7d,0x00,0x40,0xc0,0x7c,0x00,0x00,0x00]
 0x7d,0x00,0x40,0xc0,0x7c,0x00,0x00,0x00
 
 # CHECK: s_store_dword s1, s[4:5], 0x7ffff       ; encoding: [0x42,0x00,0x42,0xc0,0xff,0xff,0x07,0x00]
 0x42,0x00,0x42,0xc0,0xff,0xff,0x07,0x00
 
+# Make sure that raising the GFX9 soffset_en bit doesn't affect GFX8
+# decoding.
+# CHECK: s_store_dword s1, s[4:5], 0x7ffff       ; encoding: [0x42,0x00,0x42,0xc0,0xff,0xff,0x07,0x00]
+0x42,0x40,0x42,0xc0,0xff,0xff,0x07,0x00
+
 # CHECK: s_store_dword s1, s[4:5], m0 glc        ; encoding: [0x42,0x00,0x41,0xc0,0x7c,0x00,0x00,0x00]
 0x42,0x00,0x41,0xc0,0x7c,0x00,0x00,0x00
 
 # CHECK: s_store_dwordx2 s[2:3], s[4:5], m0      ; encoding: [0x82,0x00,0x44,0xc0,0x7c,0x00,0x00,0x00]
 0x82,0x00,0x44,0xc0,0x7c,0x00,0x00,0x00
 
 # CHECK: s_store_dwordx2 s[4:5], s[4:5], m0      ; encoding: [0x02,0x01,0x44,0xc0,0x7c,0x00,0x00,0x00]
 0x02,0x01,0x44,0xc0,0x7c,0x00,0x00,0x00

llvm/test/MC/Disassembler/AMDGPU/gfx9_dasm_all.txt

 0x73,0x00,0x40,0xc0,0x00,0x00,0x00,0x00
 
 # CHECK: s_store_dword s1, vcc, s0               ; encoding: [0x75,0x00,0x40,0xc0,0x00,0x00,0x00,0x00]
 0x75,0x00,0x40,0xc0,0x00,0x00,0x00,0x00
 
 # CHECK: s_store_dword s1, s[4:5], s101          ; encoding: [0x42,0x00,0x40,0xc0,0x65,0x00,0x00,0x00]
 0x42,0x00,0x40,0xc0,0x65,0x00,0x00,0x00
 
+# The SGRP variants can alternatively be encoded with imm=0, soffset_en=1
+# and the offset register encoded in the soffset field with the offset
+# field being disregarded.
+# CHECK: s_store_dword s5, s[2:3], s64           ; encoding: [0x41,0x41,0x40,0xc0,0x00,0x00,0x00,0x80]
+0x41,0x41,0x40,0xc0,0x65,0x00,0x00,0x80
+
 # CHECK: s_store_dword s1, s[4:5], flat_scratch_lo ; encoding: [0x42,0x00,0x40,0xc0,0x66,0x00,0x00,0x00]
 0x42,0x00,0x40,0xc0,0x66,0x00,0x00,0x00
 
 # CHECK: s_store_dword s1, s[4:5], flat_scratch_hi ; encoding: [0x42,0x00,0x40,0xc0,0x67,0x00,0x00,0x00]
 0x42,0x00,0x40,0xc0,0x67,0x00,0x00,0x00
 
 # CHECK: s_store_dword s1, s[4:5], vcc_lo        ; encoding: [0x42,0x00,0x40,0xc0,0x6a,0x00,0x00,0x00]
 0x42,0x00,0x40,0xc0,0x6a,0x00,0x00,0x00
 
 # CHECK: s_store_dword s1, s[4:5], vcc_hi        ; encoding: [0x42,0x00,0x40,0xc0,0x6b,0x00,0x00,0x00]
 0x42,0x00,0x40,0xc0,0x6b,0x00,0x00,0x00
 
 # CHECK: s_store_dword s1, s[4:5], m0            ; encoding: [0x42,0x00,0x40,0xc0,0x7c,0x00,0x00,0x00]
 0x42,0x00,0x40,0xc0,0x7c,0x00,0x00,0x00
 
 # CHECK: s_store_dword s1, s[4:5], 0x0           ; encoding: [0x42,0x00,0x42,0xc0,0x00,0x00,0x00,0x00]
 0x42,0x00,0x42,0xc0,0x00,0x00,0x00,0x00
 
+# CHECK: s_store_dword s1, s[4:5], s7 offset:0x12345 ; encoding: [0x42,0x40,0x42,0xc0,0x45,0x23,0x01,0x0e]
+0x42,0x40,0x42,0xc0,0x45,0x23,0x01,0x0e
+
+# SP3 prefers to decode instructions with imm=1 and soffset_en=1 to the
+# form with the 'offset:' modifier, even if the offset is 0.
+# CHECK: s_store_dword s5, s[2:3], s0 offset:0x0 ; encoding: [0x41,0x41,0x42,0xc0,0x00,0x00,0x00,0x00]
+0x41,0x41,0x42,0xc0,0x00,0x00,0x00,0x00
+
+
 # CHECK: s_store_dword s1, s[4:5], s0 glc        ; encoding: [0x42,0x00,0x41,0xc0,0x00,0x00,0x00,0x00]
 0x42,0x00,0x41,0xc0,0x00,0x00,0x00,0x00
 
 # CHECK: s_store_dwordx2 s[2:3], s[4:5], s0      ; encoding: [0x82,0x00,0x44,0xc0,0x00,0x00,0x00,0x00]
 0x82,0x00,0x44,0xc0,0x00,0x00,0x00,0x00
 
 # CHECK: s_store_dwordx2 s[4:5], s[4:5], s0      ; encoding: [0x02,0x01,0x44,0xc0,0x00,0x00,0x00,0x00]
 0x02,0x01,0x44,0xc0,0x00,0x00,0x00,0x00