[AMDGPU][MC][GFX6][GFX7][GFX10] Added instructions buffer_atomic_[fcmpswap/fmin/fmax]*

[dp]

Summary

Bug 28232: https://bugs.llvm.org/show_bug.cgi?id=28232

[rampitec]

lib/Target/AMDGPU/BUFInstructions.td

2022 ↗

(On Diff #224329)

Looks like this comment is no longer irrelevant?

[dp]

lib/Target/AMDGPU/BUFInstructions.td

2022 ↗

(On Diff #224329)

I'm not sure if it is still relevant - I do not understand why FCMPSWAP_X2 is special.

Doc states that "If the buffer instruction uses an SGPR for its offset, there is no data hazard." This condition is checked by GCNHazardRecognizer, but it affects all BUF opcodes, not only FCMPSWAP_X2.

[rampitec]

LGTM

llvm/lib/Target/AMDGPU/BUFInstructions.td

 
 let SubtargetPredicate = isGFX8GFX9 in {
 def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">;
 }
 
 let SubtargetPredicate = isGFX6 in { // isn't on CI & VI
 /*
 defm BUFFER_ATOMIC_RSUB        : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;
-defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">;
-defm BUFFER_ATOMIC_FMIN        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">;
-defm BUFFER_ATOMIC_FMAX        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">;
 defm BUFFER_ATOMIC_RSUB_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;
-defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">;
-defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">;
-defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">;
 */
 
 def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
                                           int_amdgcn_buffer_wbinvl1_sc>;
 }
 
+let SubtargetPredicate = isGFX6GFX7GFX10 in {
+
+defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <
+  "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag
+>;
+defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <
+  "buffer_atomic_fmin", VGPR_32, f32, null_frag
+>;
+defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <
+  "buffer_atomic_fmax", VGPR_32, f32, null_frag
+>;
+defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <
+  "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag
+>;
+defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics <
+  "buffer_atomic_fmin_x2", VReg_64, f64, null_frag
+>;
+defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics <
+  "buffer_atomic_fmax_x2", VReg_64, f64, null_frag
+>;
+
+}
+
 let SubtargetPredicate = HasD16LoadStore in {
 
 defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads <
   "buffer_load_ubyte_d16", i32, null_frag, 1
 >;
 
 defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads <
   "buffer_load_ubyte_d16_hi", i32, null_frag, 1

 defm BUFFER_ATOMIC_UMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>;
 defm BUFFER_ATOMIC_SMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>;
 defm BUFFER_ATOMIC_UMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>;
 defm BUFFER_ATOMIC_AND         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>;
 defm BUFFER_ATOMIC_OR          : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>;
 defm BUFFER_ATOMIC_XOR         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>;
 defm BUFFER_ATOMIC_INC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>;
 defm BUFFER_ATOMIC_DEC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>;
-// FIXME-GFX6-GFX7-GFX10: Add following instructions:
-//defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>;
-//defm BUFFER_ATOMIC_FMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>;
-//defm BUFFER_ATOMIC_FMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>;
+defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>;
+defm BUFFER_ATOMIC_FMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>;
+defm BUFFER_ATOMIC_FMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>;
 defm BUFFER_ATOMIC_SWAP_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>;
 defm BUFFER_ATOMIC_CMPSWAP_X2  : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>;
 defm BUFFER_ATOMIC_ADD_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>;
 defm BUFFER_ATOMIC_SUB_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>;
 defm BUFFER_ATOMIC_SMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>;
 defm BUFFER_ATOMIC_UMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>;
 defm BUFFER_ATOMIC_SMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>;
 defm BUFFER_ATOMIC_UMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>;
 defm BUFFER_ATOMIC_AND_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>;
 defm BUFFER_ATOMIC_OR_X2       : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>;
 defm BUFFER_ATOMIC_XOR_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>;
 defm BUFFER_ATOMIC_INC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>;
 defm BUFFER_ATOMIC_DEC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>;
 // FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7.
-// FIXME-GFX6-GFX7-GFX10: Add following instructions:
-//defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>;
-//defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>;
-//defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>;
+defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>;
+defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>;
+defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>;
 
 defm BUFFER_WBINVL1_SC        : MUBUF_Real_gfx6<0x070>;
 defm BUFFER_WBINVL1_VOL       : MUBUF_Real_gfx7<0x070>;
 def  BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>;
 
 //===----------------------------------------------------------------------===//
 // Base ENC_MTBUF for GFX6, GFX7, GFX10.
 //===----------------------------------------------------------------------===//

llvm/test/MC/AMDGPU/mubuf-gfx10.s

 // RUN: llvm-mc -arch=amdgcn -mcpu=gfx1010 -show-encoding %s | FileCheck -check-prefix=GFX10 %s
 
 buffer_load_sbyte v5, off, s[8:11], s3 glc slc lds
 // GFX10: buffer_load_sbyte v5, off, s[8:11], s3 glc slc lds ; encoding: [0x00,0x40,0x25,0xe0,0x00,0x05,0x42,0x03]
 
 buffer_load_sbyte v5, off, s[8:11], s3 glc slc lds dlc
 // GFX10: buffer_load_sbyte v5, off, s[8:11], s3 glc slc lds dlc ; encoding: [0x00,0xc0,0x25,0xe0,0x00,0x05,0x42,0x03]
 
 buffer_load_sbyte v5, off, s[8:11], s3 glc slc dlc
 // GFX10: buffer_load_sbyte v5, off, s[8:11], s3 glc slc dlc ; encoding: [0x00,0xc0,0x24,0xe0,0x00,0x05,0x42,0x03]
+
+buffer_atomic_fcmpswap v[0:1], off, s[0:3], s0 offset:4095
+// GFX10: buffer_atomic_fcmpswap v[0:1], off, s[0:3], s0 offset:4095 ; encoding: [0xff,0x0f,0xf8,0xe0,0x00,0x00,0x00,0x00]
+
+buffer_atomic_fcmpswap_x2 v[0:3], off, s[0:3], s0 offset:4095
+// GFX10: buffer_atomic_fcmpswap_x2 v[0:3], off, s[0:3], s0 offset:4095 ; encoding: [0xff,0x0f,0x78,0xe1,0x00,0x00,0x00,0x00]
+
+buffer_atomic_fcmpswap_x2 v[0:3], v0, s[0:3], s0 idxen offset:4095
+// GFX10: buffer_atomic_fcmpswap_x2 v[0:3], v0, s[0:3], s0 idxen offset:4095 ; encoding: [0xff,0x2f,0x78,0xe1,0x00,0x00,0x00,0x00]
+
+buffer_atomic_fmax v1, off, s[0:3], s0 offset:4095
+// GFX10: buffer_atomic_fmax v1, off, s[0:3], s0 offset:4095 ; encoding: [0xff,0x0f,0x00,0xe1,0x00,0x01,0x00,0x00]
+
+buffer_atomic_fmax v0, off, s[0:3], s0 offset:7
+// GFX10: buffer_atomic_fmax v0, off, s[0:3], s0 offset:7 ; encoding: [0x07,0x00,0x00,0xe1,0x00,0x00,0x00,0x00]
+
+buffer_atomic_fmax v0, off, s[0:3], s0 offset:4095 glc
+// GFX10: buffer_atomic_fmax v0, off, s[0:3], s0 offset:4095 glc ; encoding: [0xff,0x4f,0x00,0xe1,0x00,0x00,0x00,0x00]
+
+buffer_atomic_fmax_x2 v[5:6], off, s[0:3], s0 offset:4095
+// GFX10: buffer_atomic_fmax_x2 v[5:6], off, s[0:3], s0 offset:4095 ; encoding: [0xff,0x0f,0x80,0xe1,0x00,0x05,0x00,0x00]
+
+buffer_atomic_fmax_x2 v[0:1], v0, s[0:3], s0 idxen offset:4095
+// GFX10: buffer_atomic_fmax_x2 v[0:1], v0, s[0:3], s0 idxen offset:4095 ; encoding: [0xff,0x2f,0x80,0xe1,0x00,0x00,0x00,0x00]
+
+buffer_atomic_fmin v0, off, s[0:3], s0
+// GFX10: buffer_atomic_fmin v0, off, s[0:3], s0 ; encoding: [0x00,0x00,0xfc,0xe0,0x00,0x00,0x00,0x00]
+
+buffer_atomic_fmin v0, off, s[0:3], s0 offset:0
+// GFX10: buffer_atomic_fmin v0, off, s[0:3], s0 ; encoding: [0x00,0x00,0xfc,0xe0,0x00,0x00,0x00,0x00]
+
+buffer_atomic_fmin_x2 v[0:1], off, s[0:3], s0 offset:4095 slc
+// GFX10: buffer_atomic_fmin_x2 v[0:1], off, s[0:3], s0 offset:4095 slc ; encoding: [0xff,0x0f,0x7c,0xe1,0x00,0x00,0x40,0x00]
+
+buffer_atomic_fmin_x2 v[0:1], v0, s[0:3], s0 idxen offset:4095
+// GFX10: buffer_atomic_fmin_x2 v[0:1], v0, s[0:3], s0 idxen offset:4095 ; encoding: [0xff,0x2f,0x7c,0xe1,0x00,0x00,0x00,0x00]

llvm/test/MC/AMDGPU/mubuf.s

 buffer_atomic_add v5, off, s[8:11], 0.5 offset:4095 glc
 // SICI: buffer_atomic_add v5, off, s[8:11], 0.5 offset:4095 glc ; encoding: [0xff,0x4f,0xc8,0xe0,0x00,0x05,0x02,0xf0]
 // VI:   buffer_atomic_add v5, off, s[8:11], 0.5 offset:4095 glc ; encoding: [0xff,0x4f,0x08,0xe1,0x00,0x05,0x02,0xf0]
 
 buffer_atomic_add v5, off, s[8:11], 0.15915494 offset:4095 glc
 // NOSICI: error: invalid operand for instruction
 // VI:   buffer_atomic_add v5, off, s[8:11], 0.15915494 offset:4095 glc ; encoding: [0xff,0x4f,0x08,0xe1,0x00,0x05,0x02,0xf8]
 
+buffer_atomic_fcmpswap v[0:1], off, s[0:3], s0 offset:4095
+// SICI: buffer_atomic_fcmpswap v[0:1], off, s[0:3], s0 offset:4095 ; encoding: [0xff,0x0f,0xf8,0xe0,0x00,0x00,0x00,0x00]
+// NOVI: error: not a valid operand.
+
+buffer_atomic_fcmpswap v[0:1], v[0:1], s[0:3], s0 addr64 offset:4095
+// SICI: buffer_atomic_fcmpswap v[0:1], v[0:1], s[0:3], s0 addr64 offset:4095 ; encoding: [0xff,0x8f,0xf8,0xe0,0x00,0x00,0x00,0x00]
+// NOVI: error: not a valid operand.
+
+buffer_atomic_fcmpswap_x2 v[0:3], off, s[0:3], s0 offset:4095
+// SICI: buffer_atomic_fcmpswap_x2 v[0:3], off, s[0:3], s0 offset:4095 ; encoding: [0xff,0x0f,0x78,0xe1,0x00,0x00,0x00,0x00]
+// NOVI: error: not a valid operand.
+
+buffer_atomic_fcmpswap_x2 v[0:3], v0, s[0:3], s0 idxen offset:4095
+// SICI: buffer_atomic_fcmpswap_x2 v[0:3], v0, s[0:3], s0 idxen offset:4095 ; encoding: [0xff,0x2f,0x78,0xe1,0x00,0x00,0x00,0x00]
+// NOVI: error: not a valid operand.
+
+buffer_atomic_fmax v1, off, s[0:3], s0 offset:4095
+// SICI: buffer_atomic_fmax v1, off, s[0:3], s0 offset:4095 ; encoding: [0xff,0x0f,0x00,0xe1,0x00,0x01,0x00,0x00]
+// NOVI: error: not a valid operand.
+
+buffer_atomic_fmax v0, off, s[0:3], s0 offset:7
+// SICI: buffer_atomic_fmax v0, off, s[0:3], s0 offset:7 ; encoding: [0x07,0x00,0x00,0xe1,0x00,0x00,0x00,0x00]
+// NOVI: error: not a valid operand.
+
+buffer_atomic_fmax v0, off, s[0:3], s0 offset:4095 glc
+// SICI: buffer_atomic_fmax v0, off, s[0:3], s0 offset:4095 glc ; encoding: [0xff,0x4f,0x00,0xe1,0x00,0x00,0x00,0x00]
+// NOVI: error: not a valid operand.
+
+buffer_atomic_fmax_x2 v[5:6], off, s[0:3], s0 offset:4095
+// SICI: buffer_atomic_fmax_x2 v[5:6], off, s[0:3], s0 offset:4095 ; encoding: [0xff,0x0f,0x80,0xe1,0x00,0x05,0x00,0x00]
+// NOVI: error: not a valid operand.
+
+buffer_atomic_fmax_x2 v[0:1], v0, s[0:3], s0 idxen offset:4095
+// SICI: buffer_atomic_fmax_x2 v[0:1], v0, s[0:3], s0 idxen offset:4095 ; encoding: [0xff,0x2f,0x80,0xe1,0x00,0x00,0x00,0x00]
+// NOVI: error: not a valid operand.
+
+buffer_atomic_fmin v0, v[0:1], s[0:3], s0 addr64 offset:4095
+// SICI: buffer_atomic_fmin v0, v[0:1], s[0:3], s0 addr64 offset:4095 ; encoding: [0xff,0x8f,0xfc,0xe0,0x00,0x00,0x00,0x00]
+// NOVI: error: not a valid operand.
+
+buffer_atomic_fmin v0, off, s[0:3], s0
+// SICI: buffer_atomic_fmin v0, off, s[0:3], s0 ; encoding: [0x00,0x00,0xfc,0xe0,0x00,0x00,0x00,0x00]
+// NOVI: error: instruction not supported on this GPU
+
+buffer_atomic_fmin v0, off, s[0:3], s0 offset:0
+// SICI: buffer_atomic_fmin v0, off, s[0:3], s0 ; encoding: [0x00,0x00,0xfc,0xe0,0x00,0x00,0x00,0x00]
+// NOVI: error: not a valid operand.
+
+buffer_atomic_fmin_x2 v[0:1], off, s[0:3], s0 offset:4095 slc
+// SICI: buffer_atomic_fmin_x2 v[0:1], off, s[0:3], s0 offset:4095 slc ; encoding: [0xff,0x0f,0x7c,0xe1,0x00,0x00,0x40,0x00]
+// NOVI: error: not a valid operand.
+
+buffer_atomic_fmin_x2 v[0:1], v0, s[0:3], s0 idxen offset:4095
+// SICI: buffer_atomic_fmin_x2 v[0:1], v0, s[0:3], s0 idxen offset:4095 ; encoding: [0xff,0x2f,0x7c,0xe1,0x00,0x00,0x00,0x00]
+// NOVI: error: not a valid operand.
+
 //===----------------------------------------------------------------------===//
 // Lds support
 //===----------------------------------------------------------------------===//
 
 buffer_load_sbyte v5, off, s[8:11], s3 lds
 // SICI: buffer_load_sbyte v5, off, s[8:11], s3 lds ; encoding: [0x00,0x00,0x25,0xe0,0x00,0x05,0x02,0x03]
 // VI:   buffer_load_sbyte v5, off, s[8:11], s3 lds ; encoding: [0x00,0x00,0x45,0xe0,0x00,0x05,0x02,0x03]
 

llvm/test/MC/Disassembler/AMDGPU/mubuf_gfx10.txt

+# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1010 -disassemble -show-encoding < %s | FileCheck %s
+
+# CHECK: buffer_atomic_fcmpswap v[5:6], off, s[8:11], s3 offset:4095 ; encoding: [0xff,0x0f,0xf8,0xe0,0x00,0x05,0x02,0x03]
+0xff,0x0f,0xf8,0xe0,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_fcmpswap v[254:255], off, s[8:11], s3 offset:4095 ; encoding: [0xff,0x0f,0xf8,0xe0,0x00,0xfe,0x02,0x03]
+0xff,0x0f,0xf8,0xe0,0x00,0xfe,0x02,0x03
+
+# CHECK: buffer_atomic_fcmpswap_x2 v[5:8], off, s[8:11], s3 offset:7 ; encoding: [0x07,0x00,0x78,0xe1,0x00,0x05,0x02,0x03]
+0x07,0x00,0x78,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_fcmpswap_x2 v[5:8], off, s[8:11], s3 offset:4095 glc ; encoding: [0xff,0x4f,0x78,0xe1,0x00,0x05,0x02,0x03]
+0xff,0x4f,0x78,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_fmax v5, v0, s[8:11], s3 idxen offset:4095 ; encoding: [0xff,0x2f,0x00,0xe1,0x00,0x05,0x02,0x03]
+0xff,0x2f,0x00,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_fmax_x2 v[5:6], off, s[8:11], s3 offset:4095 glc ; encoding: [0xff,0x4f,0x80,0xe1,0x00,0x05,0x02,0x03]
+0xff,0x4f,0x80,0xe1,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_fmax_x2 v[5:6], off, s[8:11], s3 offset:4095 slc ; encoding: [0xff,0x0f,0x80,0xe1,0x00,0x05,0x42,0x03]
+0xff,0x0f,0x80,0xe1,0x00,0x05,0x42,0x03
+
+# CHECK: buffer_atomic_fmin v5, off, s[8:11], s3 ; encoding: [0x00,0x00,0xfc,0xe0,0x00,0x05,0x02,0x03]
+0x00,0x00,0xfc,0xe0,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_fmin v5, off, s[8:11], s3 offset:7 ; encoding: [0x07,0x00,0xfc,0xe0,0x00,0x05,0x02,0x03]
+0x07,0x00,0xfc,0xe0,0x00,0x05,0x02,0x03
+
+# CHECK: buffer_atomic_fmin_x2 v[5:6], off, ttmp[12:15], s3 offset:4095 ; encoding: [0xff,0x0f,0x7c,0xe1,0x00,0x05,0x1e,0x03]
+0xff,0x0f,0x7c,0xe1,0x00,0x05,0x1e,0x03