[AMDGPU][MC][GFX9] Added s_atomic_* and s_buffer_atomic_* instructions

[dp]

Summary

See bug 36837: https://bugs.llvm.org/show_bug.cgi?id=36837

[arsenm]

LGTM

[thakis]

Looks like this made tablegen crash: http://lab.llvm.org:8011/builders/lldb-x86_64-ubuntu-14.04-buildserver/builds/21791/steps/ninja%20build%20local/logs/stdio

[thakis]

Reverted in r328978 to green up the bots.

[dp]

Sorry about that. I was unable to reproduce the issue locally.

Thanks,
Dmitry

[dp]

Bug fixed, change integrated. Thanks.

lib/Target/AMDGPU/AMDGPU.td

 >;
 
 def FeatureScalarStores : SubtargetFeature<"scalar-stores",
   "HasScalarStores",
   "true",
   "Has store scalar memory instructions"
 >;
 
+def FeatureScalarAtomics : SubtargetFeature<"scalar-atomics",
+  "HasScalarAtomics",
+  "true",
+  "Has atomic scalar memory instructions"
+>;
+
 def FeatureSDWA : SubtargetFeature<"sdwa",
   "HasSDWA",
   "true",
   "Support SDWA (Sub-DWORD Addressing) extension"
 >;
 
 def FeatureSDWAOmod : SubtargetFeature<"sdwa-omod",
   "HasSDWAOmod",
def FeatureGFX9 : SubtargetFeatureGeneration<"GFX9",
   [FeatureFP64, FeatureLocalMemorySize65536,
    FeatureWavefrontSize64, FeatureFlatAddressSpace, FeatureGCN,
    FeatureGCN3Encoding, FeatureCIInsts, Feature16BitInsts,
    FeatureSMemRealTime, FeatureScalarStores, FeatureInv2PiInlineImm,
    FeatureApertureRegs, FeatureGFX9Insts, FeatureVOP3P, FeatureVGPRIndexMode,
    FeatureFastFMAF32, FeatureDPP, FeatureIntClamp,
    FeatureSDWA, FeatureSDWAOmod, FeatureSDWAScalar, FeatureSDWASdst,
    FeatureFlatInstOffsets, FeatureFlatGlobalInsts, FeatureFlatScratchInsts,
-   FeatureAddNoCarryInsts
+   FeatureAddNoCarryInsts, FeatureScalarAtomics
   ]
 >;
 
 class SubtargetFeatureISAVersion <int Major, int Minor, int Stepping,
                                   list<SubtargetFeature> Implies>
                                  : SubtargetFeature <
   "isaver"#Major#"."#Minor#"."#Stepping,
   "IsaVersion",
def HasDPP : Predicate<"Subtarget->hasDPP()">,
   AssemblerPredicate<"FeatureDPP">;
 
 def HasIntClamp : Predicate<"Subtarget->hasIntClamp()">,
   AssemblerPredicate<"FeatureIntClamp">;
 
 def HasMadMixInsts : Predicate<"Subtarget->hasMadMixInsts()">,
   AssemblerPredicate<"FeatureMadMixInsts">;
 
+def HasScalarAtomics : Predicate<"Subtarget->hasScalarAtomics()">,
+  AssemblerPredicate<"FeatureScalarAtomics">;
+
 def EnableLateCFGStructurize : Predicate<
   "EnableLateStructurizeCFG">;
 
 // Exists to help track down where SubtargetPredicate isn't set rather
 // than letting tablegen crash with an unhelpful error.
 def InvalidPred : Predicate<"predicate not set on instruction or pattern">;
 
 class PredicateControl {

lib/Target/AMDGPU/AMDGPUSubtarget.h

protected:
   bool HasSMemRealTime;
   bool Has16BitInsts;
   bool HasIntClamp;
   bool HasVOP3PInsts;
   bool HasMadMixInsts;
   bool HasMovrel;
   bool HasVGPRIndexMode;
   bool HasScalarStores;
+  bool HasScalarAtomics;
   bool HasInv2PiInlineImm;
   bool HasSDWA;
   bool HasSDWAOmod;
   bool HasSDWAScalar;
   bool HasSDWASdst;
   bool HasSDWAMac;
   bool HasSDWAOutModsVOPC;
   bool HasDPP;
public:
   bool hasScalarCompareEq64() const {
     return getGeneration() >= VOLCANIC_ISLANDS;
   }
 
   bool hasScalarStores() const {
     return HasScalarStores;
   }
 
+  bool hasScalarAtomics() const {
+    return HasScalarAtomics;
+  }
+
   bool hasInv2PiInlineImm() const {
     return HasInv2PiInlineImm;
   }
 
   bool hasDPP() const {
     return HasDPP;
   }
 

lib/Target/AMDGPU/AMDGPUSubtarget.cpp

: AMDGPUGenSubtargetInfo(TT, GPU, FS),
     HasSMemRealTime(false),
     Has16BitInsts(false),
     HasIntClamp(false),
     HasVOP3PInsts(false),
     HasMadMixInsts(false),
     HasMovrel(false),
     HasVGPRIndexMode(false),
     HasScalarStores(false),
+    HasScalarAtomics(false),
     HasInv2PiInlineImm(false),
     HasSDWA(false),
     HasSDWAOmod(false),
     HasSDWAScalar(false),
     HasSDWASdst(false),
     HasSDWAMac(false),
     HasSDWAOutModsVOPC(false),
     HasDPP(false),

lib/Target/AMDGPU/SMInstructions.td

class SM_Inval_Pseudo <string opName, SDPatternOperator node> : SM_Pseudo<
   opName, (outs), (ins), "", [(node)]> {
   let hasSideEffects = 1;
   let mayStore = 1;
   let has_sdst = 0;
   let has_sbase = 0;
   let has_offset = 0;
 }
 
+//===----------------------------------------------------------------------===//
+// Scalar Atomic Memory Classes
+//===----------------------------------------------------------------------===//
+
+class SM_Atomic_Pseudo <string opName,
+                        dag outs, dag ins, string asmOps, bit isRet>
+  : SM_Pseudo<opName, outs, ins, asmOps, []> {
+
+  bit glc = isRet;
+
+  let mayLoad = 1;
+  let mayStore = 1;
+  let has_glc = 1;
+
+  // Should these be set?
+  let ScalarStore = 1;
+  let hasSideEffects = 1;
+  let maybeAtomic = 1;
+}
+
+class SM_Pseudo_Atomic<string opName,
+                       RegisterClass baseClass,
+                       RegisterClass dataClass,
+                       bit isImm,
+                       bit isRet> :
+  SM_Atomic_Pseudo<opName,
+                   !if(isRet, (outs dataClass:$sdst), (outs)),
+                   !if(isImm,
+                       (ins dataClass:$sdata, baseClass:$sbase, smrd_offset_20:$offset),
+                       (ins dataClass:$sdata, baseClass:$sbase, SReg_32:$offset)),
+                   !if(isRet, " $sdst", " $sdata") # ", $sbase, $offset" # !if(isRet, " glc", ""),
+                   isRet> {
+  let offset_is_imm = isImm;
+  let PseudoInstr = opName # !if(isImm,
+                                 !if(isRet, "_IMM_RTN", "_IMM"),
+                                 !if(isRet, "_SGPR_RTN", "_SGPR"));
+
+  let Constraints = !if(isRet, "$sdst = $sdata", "");
+  let DisableEncoding = !if(isRet, "$sdata", "");
+}
+
+multiclass SM_Pseudo_Atomics<string opName,
+                             RegisterClass baseClass,
+                             RegisterClass dataClass> {
+  def _IMM      : SM_Pseudo_Atomic <opName, baseClass, dataClass, 1, 0>;
+  def _SGPR     : SM_Pseudo_Atomic <opName, baseClass, dataClass, 0, 0>;
+  def _IMM_RTN  : SM_Pseudo_Atomic <opName, baseClass, dataClass, 1, 1>;
+  def _SGPR_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, 0, 1>;
+}
 
 //===----------------------------------------------------------------------===//
 // Scalar Memory Instructions
 //===----------------------------------------------------------------------===//
 
 // We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit
 // SMRD instructions, because the SReg_32_XM0 register class does not include M0
 // and writing to M0 from an SMRD instruction will hang the GPU.

 defm S_SCRATCH_LOAD_DWORDX2  : SM_Pseudo_Loads <"s_scratch_load_dwordx2", SReg_64, SReg_64_XEXEC>;
 defm S_SCRATCH_LOAD_DWORDX4  : SM_Pseudo_Loads <"s_scratch_load_dwordx4", SReg_64, SReg_128>;
 
 defm S_SCRATCH_STORE_DWORD   : SM_Pseudo_Stores <"s_scratch_store_dword",   SReg_64, SReg_32_XM0_XEXEC>;
 defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <"s_scratch_store_dwordx2", SReg_64, SReg_64_XEXEC>;
 defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <"s_scratch_store_dwordx4", SReg_64, SReg_128>;
 } // SubtargetPredicate = HasFlatScratchInsts
 
+let SubtargetPredicate = HasScalarAtomics in {
+
+defm S_BUFFER_ATOMIC_SWAP         : SM_Pseudo_Atomics <"s_buffer_atomic_swap", SReg_128, SReg_32_XM0_XEXEC>;
+defm S_BUFFER_ATOMIC_CMPSWAP      : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap", SReg_128, SReg_64_XEXEC>;
+defm S_BUFFER_ATOMIC_ADD          : SM_Pseudo_Atomics <"s_buffer_atomic_add", SReg_128, SReg_32_XM0_XEXEC>;
+defm S_BUFFER_ATOMIC_SUB          : SM_Pseudo_Atomics <"s_buffer_atomic_sub", SReg_128, SReg_32_XM0_XEXEC>;
+defm S_BUFFER_ATOMIC_SMIN         : SM_Pseudo_Atomics <"s_buffer_atomic_smin", SReg_128, SReg_32_XM0_XEXEC>;
+defm S_BUFFER_ATOMIC_UMIN         : SM_Pseudo_Atomics <"s_buffer_atomic_umin", SReg_128, SReg_32_XM0_XEXEC>;
+defm S_BUFFER_ATOMIC_SMAX         : SM_Pseudo_Atomics <"s_buffer_atomic_smax", SReg_128, SReg_32_XM0_XEXEC>;
+defm S_BUFFER_ATOMIC_UMAX         : SM_Pseudo_Atomics <"s_buffer_atomic_umax", SReg_128, SReg_32_XM0_XEXEC>;
+defm S_BUFFER_ATOMIC_AND          : SM_Pseudo_Atomics <"s_buffer_atomic_and", SReg_128, SReg_32_XM0_XEXEC>;
+defm S_BUFFER_ATOMIC_OR           : SM_Pseudo_Atomics <"s_buffer_atomic_or", SReg_128, SReg_32_XM0_XEXEC>;
+defm S_BUFFER_ATOMIC_XOR          : SM_Pseudo_Atomics <"s_buffer_atomic_xor", SReg_128, SReg_32_XM0_XEXEC>;
+defm S_BUFFER_ATOMIC_INC          : SM_Pseudo_Atomics <"s_buffer_atomic_inc", SReg_128, SReg_32_XM0_XEXEC>;
+defm S_BUFFER_ATOMIC_DEC          : SM_Pseudo_Atomics <"s_buffer_atomic_dec", SReg_128, SReg_32_XM0_XEXEC>;
+
+defm S_BUFFER_ATOMIC_SWAP_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_swap_x2", SReg_128, SReg_64_XEXEC>;
+defm S_BUFFER_ATOMIC_CMPSWAP_X2   : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap_x2", SReg_128, SReg_128>;
+defm S_BUFFER_ATOMIC_ADD_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_add_x2", SReg_128, SReg_64_XEXEC>;
+defm S_BUFFER_ATOMIC_SUB_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_sub_x2", SReg_128, SReg_64_XEXEC>;
+defm S_BUFFER_ATOMIC_SMIN_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_smin_x2", SReg_128, SReg_64_XEXEC>;
+defm S_BUFFER_ATOMIC_UMIN_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_umin_x2", SReg_128, SReg_64_XEXEC>;
+defm S_BUFFER_ATOMIC_SMAX_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_smax_x2", SReg_128, SReg_64_XEXEC>;
+defm S_BUFFER_ATOMIC_UMAX_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_umax_x2", SReg_128, SReg_64_XEXEC>;
+defm S_BUFFER_ATOMIC_AND_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_and_x2", SReg_128, SReg_64_XEXEC>;
+defm S_BUFFER_ATOMIC_OR_X2        : SM_Pseudo_Atomics <"s_buffer_atomic_or_x2", SReg_128, SReg_64_XEXEC>;
+defm S_BUFFER_ATOMIC_XOR_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_xor_x2", SReg_128, SReg_64_XEXEC>;
+defm S_BUFFER_ATOMIC_INC_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_inc_x2", SReg_128, SReg_64_XEXEC>;
+defm S_BUFFER_ATOMIC_DEC_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_dec_x2", SReg_128, SReg_64_XEXEC>;
+
+defm S_ATOMIC_SWAP                : SM_Pseudo_Atomics <"s_atomic_swap", SReg_64, SReg_32_XM0_XEXEC>;
+defm S_ATOMIC_CMPSWAP             : SM_Pseudo_Atomics <"s_atomic_cmpswap", SReg_64, SReg_64_XEXEC>;
+defm S_ATOMIC_ADD                 : SM_Pseudo_Atomics <"s_atomic_add", SReg_64, SReg_32_XM0_XEXEC>;
+defm S_ATOMIC_SUB                 : SM_Pseudo_Atomics <"s_atomic_sub", SReg_64, SReg_32_XM0_XEXEC>;
+defm S_ATOMIC_SMIN                : SM_Pseudo_Atomics <"s_atomic_smin", SReg_64, SReg_32_XM0_XEXEC>;
+defm S_ATOMIC_UMIN                : SM_Pseudo_Atomics <"s_atomic_umin", SReg_64, SReg_32_XM0_XEXEC>;
+defm S_ATOMIC_SMAX                : SM_Pseudo_Atomics <"s_atomic_smax", SReg_64, SReg_32_XM0_XEXEC>;
+defm S_ATOMIC_UMAX                : SM_Pseudo_Atomics <"s_atomic_umax", SReg_64, SReg_32_XM0_XEXEC>;
+defm S_ATOMIC_AND                 : SM_Pseudo_Atomics <"s_atomic_and", SReg_64, SReg_32_XM0_XEXEC>;
+defm S_ATOMIC_OR                  : SM_Pseudo_Atomics <"s_atomic_or", SReg_64, SReg_32_XM0_XEXEC>;
+defm S_ATOMIC_XOR                 : SM_Pseudo_Atomics <"s_atomic_xor", SReg_64, SReg_32_XM0_XEXEC>;
+defm S_ATOMIC_INC                 : SM_Pseudo_Atomics <"s_atomic_inc", SReg_64, SReg_32_XM0_XEXEC>;
+defm S_ATOMIC_DEC                 : SM_Pseudo_Atomics <"s_atomic_dec", SReg_64, SReg_32_XM0_XEXEC>;
+
+defm S_ATOMIC_SWAP_X2             : SM_Pseudo_Atomics <"s_atomic_swap_x2", SReg_64, SReg_64_XEXEC>;
+defm S_ATOMIC_CMPSWAP_X2          : SM_Pseudo_Atomics <"s_atomic_cmpswap_x2", SReg_64, SReg_128>;
+defm S_ATOMIC_ADD_X2              : SM_Pseudo_Atomics <"s_atomic_add_x2", SReg_64, SReg_64_XEXEC>;
+defm S_ATOMIC_SUB_X2              : SM_Pseudo_Atomics <"s_atomic_sub_x2", SReg_64, SReg_64_XEXEC>;
+defm S_ATOMIC_SMIN_X2             : SM_Pseudo_Atomics <"s_atomic_smin_x2", SReg_64, SReg_64_XEXEC>;
+defm S_ATOMIC_UMIN_X2             : SM_Pseudo_Atomics <"s_atomic_umin_x2", SReg_64, SReg_64_XEXEC>;
+defm S_ATOMIC_SMAX_X2             : SM_Pseudo_Atomics <"s_atomic_smax_x2", SReg_64, SReg_64_XEXEC>;
+defm S_ATOMIC_UMAX_X2             : SM_Pseudo_Atomics <"s_atomic_umax_x2", SReg_64, SReg_64_XEXEC>;
+defm S_ATOMIC_AND_X2              : SM_Pseudo_Atomics <"s_atomic_and_x2", SReg_64, SReg_64_XEXEC>;
+defm S_ATOMIC_OR_X2               : SM_Pseudo_Atomics <"s_atomic_or_x2", SReg_64, SReg_64_XEXEC>;
+defm S_ATOMIC_XOR_X2              : SM_Pseudo_Atomics <"s_atomic_xor_x2", SReg_64, SReg_64_XEXEC>;
+defm S_ATOMIC_INC_X2              : SM_Pseudo_Atomics <"s_atomic_inc_x2", SReg_64, SReg_64_XEXEC>;
+defm S_ATOMIC_DEC_X2              : SM_Pseudo_Atomics <"s_atomic_dec_x2", SReg_64, SReg_64_XEXEC>;
+
+} // let SubtargetPredicate = HasScalarAtomics
+
 //===----------------------------------------------------------------------===//
 // Scalar Memory Patterns
 //===----------------------------------------------------------------------===//
 
 
 def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{
   auto Ld = cast<LoadSDNode>(N);
   return Ld->getAlignment() >= 4  &&

 defm S_SCRATCH_LOAD_DWORDX2  : SM_Real_Loads_vi <0x06, "S_SCRATCH_LOAD_DWORDX2">;
 defm S_SCRATCH_LOAD_DWORDX4  : SM_Real_Loads_vi <0x07, "S_SCRATCH_LOAD_DWORDX4">;
 
 defm S_SCRATCH_STORE_DWORD   : SM_Real_Stores_vi <0x15, "S_SCRATCH_STORE_DWORD">;
 defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_vi <0x16, "S_SCRATCH_STORE_DWORDX2">;
 defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_vi <0x17, "S_SCRATCH_STORE_DWORDX4">;
 
 //===----------------------------------------------------------------------===//
+// GFX9
+//===----------------------------------------------------------------------===//
+
+class SMEM_Atomic_Real_vi <bits<8> op, SM_Atomic_Pseudo ps>
+  : SMEM_Real_vi <op, ps> {
+
+  bits<7> sdata;
+
+  let Constraints = ps.Constraints;
+  let DisableEncoding = ps.DisableEncoding;
+
+  let glc = ps.glc;
+  let Inst{12-6} = sdata{6-0};
+}
+
+multiclass SM_Real_Atomics_vi<bits<8> op, string ps> {
+  def _IMM_vi       : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>;
+  def _SGPR_vi      : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>;
+  def _IMM_RTN_vi   : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>;
+  def _SGPR_RTN_vi  : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>;
+}
+
+defm S_BUFFER_ATOMIC_SWAP         : SM_Real_Atomics_vi <0x40, "S_BUFFER_ATOMIC_SWAP">;
+defm S_BUFFER_ATOMIC_CMPSWAP      : SM_Real_Atomics_vi <0x41, "S_BUFFER_ATOMIC_CMPSWAP">;
+defm S_BUFFER_ATOMIC_ADD          : SM_Real_Atomics_vi <0x42, "S_BUFFER_ATOMIC_ADD">;
+defm S_BUFFER_ATOMIC_SUB          : SM_Real_Atomics_vi <0x43, "S_BUFFER_ATOMIC_SUB">;
+defm S_BUFFER_ATOMIC_SMIN         : SM_Real_Atomics_vi <0x44, "S_BUFFER_ATOMIC_SMIN">;
+defm S_BUFFER_ATOMIC_UMIN         : SM_Real_Atomics_vi <0x45, "S_BUFFER_ATOMIC_UMIN">;
+defm S_BUFFER_ATOMIC_SMAX         : SM_Real_Atomics_vi <0x46, "S_BUFFER_ATOMIC_SMAX">;
+defm S_BUFFER_ATOMIC_UMAX         : SM_Real_Atomics_vi <0x47, "S_BUFFER_ATOMIC_UMAX">;
+defm S_BUFFER_ATOMIC_AND          : SM_Real_Atomics_vi <0x48, "S_BUFFER_ATOMIC_AND">;
+defm S_BUFFER_ATOMIC_OR           : SM_Real_Atomics_vi <0x49, "S_BUFFER_ATOMIC_OR">;
+defm S_BUFFER_ATOMIC_XOR          : SM_Real_Atomics_vi <0x4a, "S_BUFFER_ATOMIC_XOR">;
+defm S_BUFFER_ATOMIC_INC          : SM_Real_Atomics_vi <0x4b, "S_BUFFER_ATOMIC_INC">;
+defm S_BUFFER_ATOMIC_DEC          : SM_Real_Atomics_vi <0x4c, "S_BUFFER_ATOMIC_DEC">;
+
+defm S_BUFFER_ATOMIC_SWAP_X2      : SM_Real_Atomics_vi <0x60, "S_BUFFER_ATOMIC_SWAP_X2">;
+defm S_BUFFER_ATOMIC_CMPSWAP_X2   : SM_Real_Atomics_vi <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">;
+defm S_BUFFER_ATOMIC_ADD_X2       : SM_Real_Atomics_vi <0x62, "S_BUFFER_ATOMIC_ADD_X2">;
+defm S_BUFFER_ATOMIC_SUB_X2       : SM_Real_Atomics_vi <0x63, "S_BUFFER_ATOMIC_SUB_X2">;
+defm S_BUFFER_ATOMIC_SMIN_X2      : SM_Real_Atomics_vi <0x64, "S_BUFFER_ATOMIC_SMIN_X2">;
+defm S_BUFFER_ATOMIC_UMIN_X2      : SM_Real_Atomics_vi <0x65, "S_BUFFER_ATOMIC_UMIN_X2">;
+defm S_BUFFER_ATOMIC_SMAX_X2      : SM_Real_Atomics_vi <0x66, "S_BUFFER_ATOMIC_SMAX_X2">;
+defm S_BUFFER_ATOMIC_UMAX_X2      : SM_Real_Atomics_vi <0x67, "S_BUFFER_ATOMIC_UMAX_X2">;
+defm S_BUFFER_ATOMIC_AND_X2       : SM_Real_Atomics_vi <0x68, "S_BUFFER_ATOMIC_AND_X2">;
+defm S_BUFFER_ATOMIC_OR_X2        : SM_Real_Atomics_vi <0x69, "S_BUFFER_ATOMIC_OR_X2">;
+defm S_BUFFER_ATOMIC_XOR_X2       : SM_Real_Atomics_vi <0x6a, "S_BUFFER_ATOMIC_XOR_X2">;
+defm S_BUFFER_ATOMIC_INC_X2       : SM_Real_Atomics_vi <0x6b, "S_BUFFER_ATOMIC_INC_X2">;
+defm S_BUFFER_ATOMIC_DEC_X2       : SM_Real_Atomics_vi <0x6c, "S_BUFFER_ATOMIC_DEC_X2">;
+
+defm S_ATOMIC_SWAP                : SM_Real_Atomics_vi <0x80, "S_ATOMIC_SWAP">;
+defm S_ATOMIC_CMPSWAP             : SM_Real_Atomics_vi <0x81, "S_ATOMIC_CMPSWAP">;
+defm S_ATOMIC_ADD                 : SM_Real_Atomics_vi <0x82, "S_ATOMIC_ADD">;
+defm S_ATOMIC_SUB                 : SM_Real_Atomics_vi <0x83, "S_ATOMIC_SUB">;
+defm S_ATOMIC_SMIN                : SM_Real_Atomics_vi <0x84, "S_ATOMIC_SMIN">;
+defm S_ATOMIC_UMIN                : SM_Real_Atomics_vi <0x85, "S_ATOMIC_UMIN">;
+defm S_ATOMIC_SMAX                : SM_Real_Atomics_vi <0x86, "S_ATOMIC_SMAX">;
+defm S_ATOMIC_UMAX                : SM_Real_Atomics_vi <0x87, "S_ATOMIC_UMAX">;
+defm S_ATOMIC_AND                 : SM_Real_Atomics_vi <0x88, "S_ATOMIC_AND">;
+defm S_ATOMIC_OR                  : SM_Real_Atomics_vi <0x89, "S_ATOMIC_OR">;
+defm S_ATOMIC_XOR                 : SM_Real_Atomics_vi <0x8a, "S_ATOMIC_XOR">;
+defm S_ATOMIC_INC                 : SM_Real_Atomics_vi <0x8b, "S_ATOMIC_INC">;
+defm S_ATOMIC_DEC                 : SM_Real_Atomics_vi <0x8c, "S_ATOMIC_DEC">;
+
+defm S_ATOMIC_SWAP_X2             : SM_Real_Atomics_vi <0xa0, "S_ATOMIC_SWAP_X2">;
+defm S_ATOMIC_CMPSWAP_X2          : SM_Real_Atomics_vi <0xa1, "S_ATOMIC_CMPSWAP_X2">;
+defm S_ATOMIC_ADD_X2              : SM_Real_Atomics_vi <0xa2, "S_ATOMIC_ADD_X2">;
+defm S_ATOMIC_SUB_X2              : SM_Real_Atomics_vi <0xa3, "S_ATOMIC_SUB_X2">;
+defm S_ATOMIC_SMIN_X2             : SM_Real_Atomics_vi <0xa4, "S_ATOMIC_SMIN_X2">;
+defm S_ATOMIC_UMIN_X2             : SM_Real_Atomics_vi <0xa5, "S_ATOMIC_UMIN_X2">;
+defm S_ATOMIC_SMAX_X2             : SM_Real_Atomics_vi <0xa6, "S_ATOMIC_SMAX_X2">;
+defm S_ATOMIC_UMAX_X2             : SM_Real_Atomics_vi <0xa7, "S_ATOMIC_UMAX_X2">;
+defm S_ATOMIC_AND_X2              : SM_Real_Atomics_vi <0xa8, "S_ATOMIC_AND_X2">;
+defm S_ATOMIC_OR_X2               : SM_Real_Atomics_vi <0xa9, "S_ATOMIC_OR_X2">;
+defm S_ATOMIC_XOR_X2              : SM_Real_Atomics_vi <0xaa, "S_ATOMIC_XOR_X2">;
+defm S_ATOMIC_INC_X2              : SM_Real_Atomics_vi <0xab, "S_ATOMIC_INC_X2">;
+defm S_ATOMIC_DEC_X2              : SM_Real_Atomics_vi <0xac, "S_ATOMIC_DEC_X2">;
+
+//===----------------------------------------------------------------------===//
 // CI
 //===----------------------------------------------------------------------===//
 
 def smrd_literal_offset : NamedOperandU32<"SMRDLiteralOffset",
                                           NamedMatchClass<"SMRDLiteralOffset">> {
   let OperandType = "OPERAND_IMMEDIATE";
 }
 

test/MC/AMDGPU/smem.s

 
 s_scratch_store_dwordx2 s[2:3], s[4:5], s101 glc
 // GFX9: s_scratch_store_dwordx2 s[2:3], s[4:5], s101 glc ; encoding: [0x82,0x00,0x59,0xc0,0x65,0x00,0x00,0x00]
 // NOSICIVI: error
 
 s_scratch_store_dwordx4 s[4:7], s[4:5], s0 glc
 // GFX9: s_scratch_store_dwordx4 s[4:7], s[4:5], s0 glc ; encoding: [0x02,0x01,0x5d,0xc0,0x00,0x00,0x00,0x00]
 // NOSICIVI: error
+
+//===----------------------------------------------------------------------===//
+// s_atomic instructions
+//===----------------------------------------------------------------------===//
+
+s_atomic_add s5, s[2:3], s101
+// GFX9:     s_atomic_add s5, s[2:3], s101 ; encoding: [0x41,0x01,0x08,0xc2,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_add s5, s[2:3], 0x0
+// GFX9:     s_atomic_add s5, s[2:3], 0x0 ; encoding: [0x41,0x01,0x0a,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_add s5, s[2:3], s0 glc
+// GFX9:     s_atomic_add s5, s[2:3], s0 glc ; encoding: [0x41,0x01,0x09,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_add_x2 s[10:11], s[2:3], s101
+// GFX9:     s_atomic_add_x2 s[10:11], s[2:3], s101 ; encoding: [0x81,0x02,0x88,0xc2,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_and s5, s[2:3], s101
+// GFX9:     s_atomic_and s5, s[2:3], s101 ; encoding: [0x41,0x01,0x20,0xc2,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_and_x2 s[10:11], s[2:3], 0x0
+// GFX9:     s_atomic_and_x2 s[10:11], s[2:3], 0x0 ; encoding: [0x81,0x02,0xa2,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_cmpswap s[10:11], s[2:3], s101
+// GFX9:     s_atomic_cmpswap s[10:11], s[2:3], s101 ; encoding: [0x81,0x02,0x04,0xc2,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_cmpswap s[10:11], s[2:3], 0x0
+// GFX9:     s_atomic_cmpswap s[10:11], s[2:3], 0x0 ; encoding: [0x81,0x02,0x06,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_cmpswap s[10:11], s[2:3], s0 glc
+// GFX9:     s_atomic_cmpswap s[10:11], s[2:3], s0 glc ; encoding: [0x81,0x02,0x05,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_cmpswap_x2 s[20:23], s[2:3], s101
+// GFX9:     s_atomic_cmpswap_x2 s[20:23], s[2:3], s101 ; encoding: [0x01,0x05,0x84,0xc2,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_cmpswap_x2 s[20:23], s[2:3], 0x0
+// GFX9:     s_atomic_cmpswap_x2 s[20:23], s[2:3], 0x0 ; encoding: [0x01,0x05,0x86,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_cmpswap_x2 s[20:23], s[2:3], s0 glc
+// GFX9:     s_atomic_cmpswap_x2 s[20:23], s[2:3], s0 glc ; encoding: [0x01,0x05,0x85,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_dec s5, s[2:3], s0 glc
+// GFX9:     s_atomic_dec s5, s[2:3], s0 glc ; encoding: [0x41,0x01,0x31,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_dec_x2 s[10:11], s[2:3], s101
+// GFX9:     s_atomic_dec_x2 s[10:11], s[2:3], s101 ; encoding: [0x81,0x02,0xb0,0xc2,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_inc s5, s[2:3], s0 glc
+// GFX9:     s_atomic_inc s5, s[2:3], s0 glc ; encoding: [0x41,0x01,0x2d,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_inc_x2 s[10:11], s[2:3], s101
+// GFX9:     s_atomic_inc_x2 s[10:11], s[2:3], s101 ; encoding: [0x81,0x02,0xac,0xc2,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_or s5, s[2:3], 0x0
+// GFX9:     s_atomic_or s5, s[2:3], 0x0 ; encoding: [0x41,0x01,0x26,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_or_x2 s[10:11], s[2:3], s0 glc
+// GFX9:     s_atomic_or_x2 s[10:11], s[2:3], s0 glc ; encoding: [0x81,0x02,0xa5,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_smax s5, s[2:3], s101
+// GFX9:     s_atomic_smax s5, s[2:3], s101 ; encoding: [0x41,0x01,0x18,0xc2,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_smax_x2 s[10:11], s[2:3], s0 glc
+// GFX9:     s_atomic_smax_x2 s[10:11], s[2:3], s0 glc ; encoding: [0x81,0x02,0x99,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_smin s5, s[2:3], s101
+// GFX9:     s_atomic_smin s5, s[2:3], s101 ; encoding: [0x41,0x01,0x10,0xc2,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_smin_x2 s[10:11], s[2:3], s0 glc
+// GFX9:     s_atomic_smin_x2 s[10:11], s[2:3], s0 glc ; encoding: [0x81,0x02,0x91,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_sub s5, s[2:3], s101
+// GFX9:     s_atomic_sub s5, s[2:3], s101 ; encoding: [0x41,0x01,0x0c,0xc2,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_sub_x2 s[10:11], s[2:3], s0 glc
+// GFX9:     s_atomic_sub_x2 s[10:11], s[2:3], s0 glc ; encoding: [0x81,0x02,0x8d,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_swap s5, s[2:3], s101
+// GFX9:     s_atomic_swap s5, s[2:3], s101 ; encoding: [0x41,0x01,0x00,0xc2,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_swap_x2 s[10:11], s[2:3], s0 glc
+// GFX9:     s_atomic_swap_x2 s[10:11], s[2:3], s0 glc ; encoding: [0x81,0x02,0x81,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_umax s5, s[2:3], s0 glc
+// GFX9:     s_atomic_umax s5, s[2:3], s0 glc ; encoding: [0x41,0x01,0x1d,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_umax_x2 s[10:11], s[2:3], s101
+// GFX9:     s_atomic_umax_x2 s[10:11], s[2:3], s101 ; encoding: [0x81,0x02,0x9c,0xc2,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_umin s5, s[2:3], s101
+// GFX9:     s_atomic_umin s5, s[2:3], s101 ; encoding: [0x41,0x01,0x14,0xc2,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_umin_x2 s[10:11], s[2:3], s0 glc
+// GFX9:     s_atomic_umin_x2 s[10:11], s[2:3], s0 glc ; encoding: [0x81,0x02,0x95,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_xor s5, s[2:3], s101
+// GFX9:     s_atomic_xor s5, s[2:3], s101 ; encoding: [0x41,0x01,0x28,0xc2,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_atomic_xor_x2 s[10:11], s[2:3], s0 glc
+// GFX9:     s_atomic_xor_x2 s[10:11], s[2:3], s0 glc ; encoding: [0x81,0x02,0xa9,0xc2,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+//===----------------------------------------------------------------------===//
+// s_buffer_atomic instructions
+//===----------------------------------------------------------------------===//
+
+s_buffer_atomic_add s5, s[4:7], s101
+// GFX9:     s_buffer_atomic_add s5, s[4:7], s101 ; encoding: [0x42,0x01,0x08,0xc1,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_add s5, s[4:7], 0x0
+// GFX9:     s_buffer_atomic_add s5, s[4:7], 0x0 ; encoding: [0x42,0x01,0x0a,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_add s5, s[4:7], s0 glc
+// GFX9:     s_buffer_atomic_add s5, s[4:7], s0 glc ; encoding: [0x42,0x01,0x09,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_add_x2 s[10:11], s[4:7], s0
+// GFX9:     s_buffer_atomic_add_x2 s[10:11], s[4:7], s0 ; encoding: [0x82,0x02,0x88,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_and s101, s[4:7], s0
+// GFX9:     s_buffer_atomic_and s101, s[4:7], s0 ; encoding: [0x42,0x19,0x20,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_and_x2 s[10:11], s[8:11], s0
+// GFX9:     s_buffer_atomic_and_x2 s[10:11], s[8:11], s0 ; encoding: [0x84,0x02,0xa0,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_cmpswap s[10:11], s[4:7], s0
+// GFX9:     s_buffer_atomic_cmpswap s[10:11], s[4:7], s0 ; encoding: [0x82,0x02,0x04,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_cmpswap s[10:11], s[4:7], 0x0
+// GFX9:     s_buffer_atomic_cmpswap s[10:11], s[4:7], 0x0 ; encoding: [0x82,0x02,0x06,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_cmpswap s[10:11], s[4:7], s0 glc
+// GFX9:     s_buffer_atomic_cmpswap s[10:11], s[4:7], s0 glc ; encoding: [0x82,0x02,0x05,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_cmpswap_x2 s[20:23], s[4:7], s101
+// GFX9:     s_buffer_atomic_cmpswap_x2 s[20:23], s[4:7], s101 ; encoding: [0x02,0x05,0x84,0xc1,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_cmpswap_x2 s[20:23], s[4:7], 0x0
+// GFX9:     s_buffer_atomic_cmpswap_x2 s[20:23], s[4:7], 0x0 ; encoding: [0x02,0x05,0x86,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_cmpswap_x2 s[20:23], s[4:7], s0 glc
+// GFX9:     s_buffer_atomic_cmpswap_x2 s[20:23], s[4:7], s0 glc ; encoding: [0x02,0x05,0x85,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_dec s5, s[4:7], s0
+// GFX9:     s_buffer_atomic_dec s5, s[4:7], s0 ; encoding: [0x42,0x01,0x30,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_dec_x2 s[10:11], s[4:7], s0 glc
+// GFX9:     s_buffer_atomic_dec_x2 s[10:11], s[4:7], s0 glc ; encoding: [0x82,0x02,0xb1,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_inc s101, s[4:7], s0
+// GFX9:     s_buffer_atomic_inc s101, s[4:7], s0 ; encoding: [0x42,0x19,0x2c,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_inc_x2 s[10:11], s[4:7], 0x0
+// GFX9:     s_buffer_atomic_inc_x2 s[10:11], s[4:7], 0x0 ; encoding: [0x82,0x02,0xae,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_or s5, s[8:11], s0
+// GFX9:     s_buffer_atomic_or s5, s[8:11], s0 ; encoding: [0x44,0x01,0x24,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_or_x2 s[10:11], s[96:99], s0
+// GFX9:     s_buffer_atomic_or_x2 s[10:11], s[96:99], s0 ; encoding: [0xb0,0x02,0xa4,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_smax s5, s[4:7], s101
+// GFX9:     s_buffer_atomic_smax s5, s[4:7], s101 ; encoding: [0x42,0x01,0x18,0xc1,0x65,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_smax_x2 s[100:101], s[4:7], s0
+// GFX9:     s_buffer_atomic_smax_x2 s[100:101], s[4:7], s0 ; encoding: [0x02,0x19,0x98,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_smin s5, s[4:7], 0x0
+// GFX9:     s_buffer_atomic_smin s5, s[4:7], 0x0 ; encoding: [0x42,0x01,0x12,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_smin_x2 s[12:13], s[4:7], s0
+// GFX9:     s_buffer_atomic_smin_x2 s[12:13], s[4:7], s0 ; encoding: [0x02,0x03,0x90,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_sub s5, s[4:7], s0 glc
+// GFX9:     s_buffer_atomic_sub s5, s[4:7], s0 glc ; encoding: [0x42,0x01,0x0d,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_sub_x2 s[10:11], s[4:7], s0
+// GFX9:     s_buffer_atomic_sub_x2 s[10:11], s[4:7], s0 ; encoding: [0x82,0x02,0x8c,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_swap s5, s[4:7], s0
+// GFX9:     s_buffer_atomic_swap s5, s[4:7], s0 ; encoding: [0x42,0x01,0x00,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_swap_x2 s[10:11], s[4:7], s0 glc
+// GFX9:     s_buffer_atomic_swap_x2 s[10:11], s[4:7], s0 glc ; encoding: [0x82,0x02,0x81,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_umax s5, s[4:7], s0
+// GFX9:     s_buffer_atomic_umax s5, s[4:7], s0 ; encoding: [0x42,0x01,0x1c,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_umax_x2 s[10:11], s[4:7], s0 glc
+// GFX9:     s_buffer_atomic_umax_x2 s[10:11], s[4:7], s0 glc ; encoding: [0x82,0x02,0x9d,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_umin s5, s[4:7], s0
+// GFX9:     s_buffer_atomic_umin s5, s[4:7], s0 ; encoding: [0x42,0x01,0x14,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_umin_x2 s[10:11], s[4:7], s0 glc
+// GFX9:     s_buffer_atomic_umin_x2 s[10:11], s[4:7], s0 glc ; encoding: [0x82,0x02,0x95,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_xor s5, s[4:7], s0
+// GFX9:     s_buffer_atomic_xor s5, s[4:7], s0 ; encoding: [0x42,0x01,0x28,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:
+
+s_buffer_atomic_xor_x2 s[10:11], s[4:7], s0 glc
+// GFX9:     s_buffer_atomic_xor_x2 s[10:11], s[4:7], s0 glc ; encoding: [0x82,0x02,0xa9,0xc1,0x00,0x00,0x00,0x00]
+// NOSICIVI: error:

test/MC/Disassembler/AMDGPU/smem_gfx9.txt

 # RUN: llvm-mc -arch=amdgcn -mcpu=gfx900 -disassemble -show-encoding < %s | FileCheck %s -check-prefix=GFX9
 
+#===------------------------------------------------------------------------===#
+# s_scratch
+#===------------------------------------------------------------------------===#
+
 # GFX9: s_scratch_load_dword s5, s[2:3], s101 ; encoding: [0x41,0x01,0x14,0xc0,0x65,0x00,0x00,0x00]
 0x41,0x01,0x14,0xc0,0x65,0x00,0x00,0x00
 
 # GFX9: s_scratch_load_dword s5, s[2:3], s0 glc ; encoding: [0x41,0x01,0x15,0xc0,0x00,0x00,0x00,0x00]
 0x41,0x01,0x15,0xc0,0x00,0x00,0x00,0x00
 
 # GFX9: s_scratch_load_dwordx2 s[100:101], s[2:3], s0 ; encoding: [0x01,0x19,0x18,0xc0,0x00,0x00,0x00,0x00]
 0x01,0x19,0x18,0xc0,0x00,0x00,0x00,0x00

 # GFX9: s_scratch_store_dword s1, s[4:5], 0x123 glc ; encoding: [0x42,0x00,0x57,0xc0,0x23,0x01,0x00,0x00]
 0x42,0x00,0x57,0xc0,0x23,0x01,0x00,0x00
 
 # GFX9: s_scratch_store_dwordx2 s[2:3], s[4:5], s101 glc ; encoding: [0x82,0x00,0x59,0xc0,0x65,0x00,0x00,0x00]
 0x82,0x00,0x59,0xc0,0x65,0x00,0x00,0x00
 
 # GFX9: s_scratch_store_dwordx4 s[4:7], s[4:5], s0 glc ; encoding: [0x02,0x01,0x5d,0xc0,0x00,0x00,0x00,0x00]
 0x02,0x01,0x5d,0xc0,0x00,0x00,0x00,0x00
+
+#===------------------------------------------------------------------------===#
+# s_atomic
+#===------------------------------------------------------------------------===#
+
+# GFX9: s_atomic_add s5, s[2:3], s101    ; encoding: [0x41,0x01,0x08,0xc2,0x65,0x00,0x00,0x00]
+0x41,0x01,0x08,0xc2,0x65,0x00,0x00,0x00
+
+# GFX9: s_atomic_add_x2 s[10:11], s[2:3], s101    ; encoding: [0x81,0x02,0x88,0xc2,0x65,0x00,0x00,0x00]
+0x81,0x02,0x88,0xc2,0x65,0x00,0x00,0x00
+
+# GFX9: s_atomic_and s101, s[2:3], s0    ; encoding: [0x41,0x19,0x20,0xc2,0x00,0x00,0x00,0x00]
+0x41,0x19,0x20,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_and_x2 s[12:13], s[2:3], s0    ; encoding: [0x01,0x03,0xa0,0xc2,0x00,0x00,0x00,0x00]
+0x01,0x03,0xa0,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_cmpswap s[10:11], s[2:3], s101    ; encoding: [0x81,0x02,0x04,0xc2,0x65,0x00,0x00,0x00]
+0x81,0x02,0x04,0xc2,0x65,0x00,0x00,0x00
+
+# GFX9: s_atomic_cmpswap s[10:11], s[2:3], 0x0    ; encoding: [0x81,0x02,0x06,0xc2,0x00,0x00,0x00,0x00]
+0x81,0x02,0x06,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_cmpswap s[10:11], s[2:3], s0 glc    ; encoding: [0x81,0x02,0x05,0xc2,0x00,0x00,0x00,0x00]
+0x81,0x02,0x05,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_cmpswap_x2 s[20:23], s[2:3], s101    ; encoding: [0x01,0x05,0x84,0xc2,0x65,0x00,0x00,0x00]
+0x01,0x05,0x84,0xc2,0x65,0x00,0x00,0x00
+
+# GFX9: s_atomic_cmpswap_x2 s[20:23], s[2:3], 0x0    ; encoding: [0x01,0x05,0x86,0xc2,0x00,0x00,0x00,0x00]
+0x01,0x05,0x86,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_cmpswap_x2 s[20:23], s[2:3], s0 glc    ; encoding: [0x01,0x05,0x85,0xc2,0x00,0x00,0x00,0x00]
+0x01,0x05,0x85,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_dec s5, s[4:5], s0    ; encoding: [0x42,0x01,0x30,0xc2,0x00,0x00,0x00,0x00]
+0x42,0x01,0x30,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_dec_x2 s[100:101], s[2:3], s0    ; encoding: [0x01,0x19,0xb0,0xc2,0x00,0x00,0x00,0x00]
+0x01,0x19,0xb0,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_inc s5, s[100:101], s0    ; encoding: [0x72,0x01,0x2c,0xc2,0x00,0x00,0x00,0x00]
+0x72,0x01,0x2c,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_inc_x2 s[10:11], s[4:5], s0    ; encoding: [0x82,0x02,0xac,0xc2,0x00,0x00,0x00,0x00]
+0x82,0x02,0xac,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_or s5, s[2:3], s101    ; encoding: [0x41,0x01,0x24,0xc2,0x65,0x00,0x00,0x00]
+0x41,0x01,0x24,0xc2,0x65,0x00,0x00,0x00
+
+# GFX9: s_atomic_or_x2 s[10:11], s[2:3], s101    ; encoding: [0x81,0x02,0xa4,0xc2,0x65,0x00,0x00,0x00]
+0x81,0x02,0xa4,0xc2,0x65,0x00,0x00,0x00
+
+# GFX9: s_atomic_smax s5, s[2:3], 0x0    ; encoding: [0x41,0x01,0x1a,0xc2,0x00,0x00,0x00,0x00]
+0x41,0x01,0x1a,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_smax_x2 s[10:11], s[2:3], 0x0    ; encoding: [0x81,0x02,0x9a,0xc2,0x00,0x00,0x00,0x00]
+0x81,0x02,0x9a,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_smin s5, s[2:3], s0 glc    ; encoding: [0x41,0x01,0x11,0xc2,0x00,0x00,0x00,0x00]
+0x41,0x01,0x11,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_smin_x2 s[10:11], s[2:3], s0 glc    ; encoding: [0x81,0x02,0x91,0xc2,0x00,0x00,0x00,0x00]
+0x81,0x02,0x91,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_sub s5, s[2:3], s0    ; encoding: [0x41,0x01,0x0c,0xc2,0x00,0x00,0x00,0x00]
+0x41,0x01,0x0c,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_sub_x2 s[10:11], s[2:3], s0    ; encoding: [0x81,0x02,0x8c,0xc2,0x00,0x00,0x00,0x00]
+0x81,0x02,0x8c,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_swap s101, s[2:3], s0    ; encoding: [0x41,0x19,0x00,0xc2,0x00,0x00,0x00,0x00]
+0x41,0x19,0x00,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_swap_x2 s[12:13], s[2:3], s0    ; encoding: [0x01,0x03,0x80,0xc2,0x00,0x00,0x00,0x00]
+0x01,0x03,0x80,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_umax s5, s[4:5], s0    ; encoding: [0x42,0x01,0x1c,0xc2,0x00,0x00,0x00,0x00]
+0x42,0x01,0x1c,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_umax_x2 s[100:101], s[2:3], s0    ; encoding: [0x01,0x19,0x9c,0xc2,0x00,0x00,0x00,0x00]
+0x01,0x19,0x9c,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_umin s5, s[100:101], s0    ; encoding: [0x72,0x01,0x14,0xc2,0x00,0x00,0x00,0x00]
+0x72,0x01,0x14,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_umin_x2 s[10:11], s[4:5], s0    ; encoding: [0x82,0x02,0x94,0xc2,0x00,0x00,0x00,0x00]
+0x82,0x02,0x94,0xc2,0x00,0x00,0x00,0x00
+
+# GFX9: s_atomic_xor s5, s[2:3], s101    ; encoding: [0x41,0x01,0x28,0xc2,0x65,0x00,0x00,0x00]
+0x41,0x01,0x28,0xc2,0x65,0x00,0x00,0x00
+
+# GFX9: s_atomic_xor_x2 s[10:11], s[2:3], s101    ; encoding: [0x81,0x02,0xa8,0xc2,0x65,0x00,0x00,0x00]
+0x81,0x02,0xa8,0xc2,0x65,0x00,0x00,0x00
+
+#===------------------------------------------------------------------------===#
+# s_buffer_atomic
+#===------------------------------------------------------------------------===#
+
+# GFX9: s_buffer_atomic_add s5, s[4:7], 0x0    ; encoding: [0x42,0x01,0x0a,0xc1,0x00,0x00,0x00,0x00]
+0x42,0x01,0x0a,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_add_x2 s[10:11], s[4:7], s0 glc    ; encoding: [0x82,0x02,0x89,0xc1,0x00,0x00,0x00,0x00]
+0x82,0x02,0x89,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_and s5, s[4:7], s0 glc    ; encoding: [0x42,0x01,0x21,0xc1,0x00,0x00,0x00,0x00]
+0x42,0x01,0x21,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_and_x2 s[10:11], s[4:7], s0 glc    ; encoding: [0x82,0x02,0xa1,0xc1,0x00,0x00,0x00,0x00]
+0x82,0x02,0xa1,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_cmpswap s[10:11], s[4:7], s101    ; encoding: [0x82,0x02,0x04,0xc1,0x65,0x00,0x00,0x00]
+0x82,0x02,0x04,0xc1,0x65,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_cmpswap s[10:11], s[4:7], 0x0    ; encoding: [0x82,0x02,0x06,0xc1,0x00,0x00,0x00,0x00]
+0x82,0x02,0x06,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_cmpswap s[10:11], s[4:7], s0 glc    ; encoding: [0x82,0x02,0x05,0xc1,0x00,0x00,0x00,0x00]
+0x82,0x02,0x05,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_cmpswap_x2 s[20:23], s[4:7], s101    ; encoding: [0x02,0x05,0x84,0xc1,0x65,0x00,0x00,0x00]
+0x02,0x05,0x84,0xc1,0x65,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_cmpswap_x2 s[20:23], s[4:7], 0x0    ; encoding: [0x02,0x05,0x86,0xc1,0x00,0x00,0x00,0x00]
+0x02,0x05,0x86,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_cmpswap_x2 s[20:23], s[4:7], s0 glc    ; encoding: [0x02,0x05,0x85,0xc1,0x00,0x00,0x00,0x00]
+0x02,0x05,0x85,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_dec s5, s[4:7], s0    ; encoding: [0x42,0x01,0x30,0xc1,0x00,0x00,0x00,0x00]
+0x42,0x01,0x30,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_dec_x2 s[10:11], s[4:7], s0    ; encoding: [0x82,0x02,0xb0,0xc1,0x00,0x00,0x00,0x00]
+0x82,0x02,0xb0,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_inc s101, s[4:7], s0    ; encoding: [0x42,0x19,0x2c,0xc1,0x00,0x00,0x00,0x00]
+0x42,0x19,0x2c,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_inc_x2 s[12:13], s[4:7], s0    ; encoding: [0x02,0x03,0xac,0xc1,0x00,0x00,0x00,0x00]
+0x02,0x03,0xac,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_or s5, s[8:11], s0    ; encoding: [0x44,0x01,0x24,0xc1,0x00,0x00,0x00,0x00]
+0x44,0x01,0x24,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_or_x2 s[100:101], s[4:7], s0    ; encoding: [0x02,0x19,0xa4,0xc1,0x00,0x00,0x00,0x00]
+0x02,0x19,0xa4,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_smax s5, s[96:99], s0    ; encoding: [0x70,0x01,0x18,0xc1,0x00,0x00,0x00,0x00]
+0x70,0x01,0x18,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_smax_x2 s[10:11], s[96:99], s0    ; encoding: [0xb0,0x02,0x98,0xc1,0x00,0x00,0x00,0x00]
+0xb0,0x02,0x98,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_smin s5, s[4:7], s101    ; encoding: [0x42,0x01,0x10,0xc1,0x65,0x00,0x00,0x00]
+0x42,0x01,0x10,0xc1,0x65,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_smin_x2 s[10:11], s[4:7], s101    ; encoding: [0x82,0x02,0x90,0xc1,0x65,0x00,0x00,0x00]
+0x82,0x02,0x90,0xc1,0x65,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_sub s5, s[4:7], 0x0    ; encoding: [0x42,0x01,0x0e,0xc1,0x00,0x00,0x00,0x00]
+0x42,0x01,0x0e,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_sub_x2 s[10:11], s[4:7], 0x0    ; encoding: [0x82,0x02,0x8e,0xc1,0x00,0x00,0x00,0x00]
+0x82,0x02,0x8e,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_swap s5, s[4:7], s0 glc    ; encoding: [0x42,0x01,0x01,0xc1,0x00,0x00,0x00,0x00]
+0x42,0x01,0x01,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_swap_x2 s[10:11], s[4:7], s0 glc    ; encoding: [0x82,0x02,0x81,0xc1,0x00,0x00,0x00,0x00]
+0x82,0x02,0x81,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_umax s5, s[4:7], s0    ; encoding: [0x42,0x01,0x1c,0xc1,0x00,0x00,0x00,0x00]
+0x42,0x01,0x1c,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_umax_x2 s[10:11], s[4:7], s0    ; encoding: [0x82,0x02,0x9c,0xc1,0x00,0x00,0x00,0x00]
+0x82,0x02,0x9c,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_umin s101, s[4:7], s0    ; encoding: [0x42,0x19,0x14,0xc1,0x00,0x00,0x00,0x00]
+0x42,0x19,0x14,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_umin_x2 s[12:13], s[4:7], s0    ; encoding: [0x02,0x03,0x94,0xc1,0x00,0x00,0x00,0x00]
+0x02,0x03,0x94,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_xor s5, s[8:11], s0    ; encoding: [0x44,0x01,0x28,0xc1,0x00,0x00,0x00,0x00]
+0x44,0x01,0x28,0xc1,0x00,0x00,0x00,0x00
+
+# GFX9: s_buffer_atomic_xor_x2 s[100:101], s[4:7], s0    ; encoding: [0x02,0x19,0xa8,0xc1,0x00,0x00,0x00,0x00]
+0x02,0x19,0xa8,0xc1,0x00,0x00,0x00,0x00