[AMDGPU] gfx1010 memory legalizer

[kzhuravl]

LGTM

llvm/trunk/lib/Target/AMDGPU/SIMemoryLegalizer.cpp

public:
 
   bool insertCacheInvalidate(MachineBasicBlock::iterator &MI,
                              SIAtomicScope Scope,
                              SIAtomicAddrSpace AddrSpace,
                              Position Pos) const override;
 
 };
 
+class SIGfx10CacheControl : public SIGfx7CacheControl {
+protected:
+  bool CuMode = false;
+
+  /// Sets DLC bit to "true" if present in \p MI. Returns true if \p MI
+  /// is modified, false otherwise.
+  bool enableDLCBit(const MachineBasicBlock::iterator &MI) const {
+    return enableNamedBit<AMDGPU::OpName::dlc>(MI);
+  }
+
+public:
+
+  SIGfx10CacheControl(const GCNSubtarget &ST, bool CuMode) :
+    SIGfx7CacheControl(ST), CuMode(CuMode) {};
+
+  bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI,
+                             SIAtomicScope Scope,
+                             SIAtomicAddrSpace AddrSpace) const override;
+
+  bool enableNonTemporal(const MachineBasicBlock::iterator &MI) const override;
+
+  bool insertCacheInvalidate(MachineBasicBlock::iterator &MI,
+                             SIAtomicScope Scope,
+                             SIAtomicAddrSpace AddrSpace,
+                             Position Pos) const override;
+
+  bool insertWait(MachineBasicBlock::iterator &MI,
+                  SIAtomicScope Scope,
+                  SIAtomicAddrSpace AddrSpace,
+                  SIMemOp Op,
+                  bool IsCrossAddrSpaceOrdering,
+                  Position Pos) const override;
+};
+
 class SIMemoryLegalizer final : public MachineFunctionPass {
 private:
 
   /// Cache Control.
   std::unique_ptr<SICacheControl> CC = nullptr;
 
   /// List of atomic pseudo instructions.
   std::list<MachineBasicBlock::iterator> AtomicPseudoMIs;
SICacheControl::SICacheControl(const GCNSubtarget &ST) {
   IV = getIsaVersion(ST.getCPU());
 }
 
 /* static */
 std::unique_ptr<SICacheControl> SICacheControl::create(const GCNSubtarget &ST) {
   GCNSubtarget::Generation Generation = ST.getGeneration();
   if (Generation <= AMDGPUSubtarget::SOUTHERN_ISLANDS)
     return make_unique<SIGfx6CacheControl>(ST);
+  if (Generation < AMDGPUSubtarget::GFX10)
     return make_unique<SIGfx7CacheControl>(ST);
+  return make_unique<SIGfx10CacheControl>(ST, ST.isCuModeEnabled());
 }
 
 bool SIGfx6CacheControl::enableLoadCacheBypass(
     const MachineBasicBlock::iterator &MI,
     SIAtomicScope Scope,
     SIAtomicAddrSpace AddrSpace) const {
   assert(MI->mayLoad() && !MI->mayStore());
   bool Changed = false;
bool SIGfx7CacheControl::insertCacheInvalidate(MachineBasicBlock::iterator &MI,
   /// Other address spaces do not hava a cache.
 
   if (Pos == Position::AFTER)
     --MI;
 
   return Changed;
 }
 
+bool SIGfx10CacheControl::enableLoadCacheBypass(
+    const MachineBasicBlock::iterator &MI,
+    SIAtomicScope Scope,
+    SIAtomicAddrSpace AddrSpace) const {
+  assert(MI->mayLoad() && !MI->mayStore());
+  bool Changed = false;
+
+  if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
+    /// TODO Do not set glc for rmw atomic operations as they
+    /// implicitly bypass the L0/L1 caches.
+
+    switch (Scope) {
+    case SIAtomicScope::SYSTEM:
+    case SIAtomicScope::AGENT:
+      Changed |= enableGLCBit(MI);
+      Changed |= enableDLCBit(MI);
+      break;
+    case SIAtomicScope::WORKGROUP:
+      // In WGP mode the waves of a work-group can be executing on either CU of
+      // the WGP. Therefore need to bypass the L0 which is per CU. Otherwise in
+      // CU mode and all waves of a work-group are on the same CU, and so the
+      // L0 does not need to be bypassed.
+      if (!CuMode) Changed |= enableGLCBit(MI);
+      break;
+    case SIAtomicScope::WAVEFRONT:
+    case SIAtomicScope::SINGLETHREAD:
+      // No cache to bypass.
+      break;
+    default:
+      llvm_unreachable("Unsupported synchronization scope");
+    }
+  }
+
+  /// The scratch address space does not need the global memory caches
+  /// to be bypassed as all memory operations by the same thread are
+  /// sequentially consistent, and no other thread can access scratch
+  /// memory.
+
+  /// Other address spaces do not hava a cache.
+
+  return Changed;
+}
+
+bool SIGfx10CacheControl::enableNonTemporal(
+    const MachineBasicBlock::iterator &MI) const {
+  assert(MI->mayLoad() ^ MI->mayStore());
+  bool Changed = false;
+
+  Changed |= enableSLCBit(MI);
+  /// TODO for store (non-rmw atomic) instructions also enableGLCBit(MI)
+
+  return Changed;
+}
+
+bool SIGfx10CacheControl::insertCacheInvalidate(MachineBasicBlock::iterator &MI,
+                                                SIAtomicScope Scope,
+                                                SIAtomicAddrSpace AddrSpace,
+                                                Position Pos) const {
+  bool Changed = false;
+
+  MachineBasicBlock &MBB = *MI->getParent();
+  DebugLoc DL = MI->getDebugLoc();
+
+  if (Pos == Position::AFTER)
+    ++MI;
+
+  if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
+    switch (Scope) {
+    case SIAtomicScope::SYSTEM:
+    case SIAtomicScope::AGENT:
+      BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_GL0_INV));
+      BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_GL1_INV));
+      Changed = true;
+      break;
+    case SIAtomicScope::WORKGROUP:
+      // In WGP mode the waves of a work-group can be executing on either CU of
+      // the WGP. Therefore need to invalidate the L0 which is per CU. Otherwise
+      // in CU mode and all waves of a work-group are on the same CU, and so the
+      // L0 does not need to be invalidated.
+      if (!CuMode) {
+        BuildMI(MBB, MI, DL, TII->get(AMDGPU::BUFFER_GL0_INV));
+        Changed = true;
+      }
+      break;
+    case SIAtomicScope::WAVEFRONT:
+    case SIAtomicScope::SINGLETHREAD:
+      // No cache to invalidate.
+      break;
+    default:
+      llvm_unreachable("Unsupported synchronization scope");
+    }
+  }
+
+  /// The scratch address space does not need the global memory cache
+  /// to be flushed as all memory operations by the same thread are
+  /// sequentially consistent, and no other thread can access scratch
+  /// memory.
+
+  /// Other address spaces do not hava a cache.
+
+  if (Pos == Position::AFTER)
+    --MI;
+
+  return Changed;
+}
+
+bool SIGfx10CacheControl::insertWait(MachineBasicBlock::iterator &MI,
+                                     SIAtomicScope Scope,
+                                     SIAtomicAddrSpace AddrSpace,
+                                     SIMemOp Op,
+                                     bool IsCrossAddrSpaceOrdering,
+                                     Position Pos) const {
+  bool Changed = false;
+
+  MachineBasicBlock &MBB = *MI->getParent();
+  DebugLoc DL = MI->getDebugLoc();
+
+  if (Pos == Position::AFTER)
+    ++MI;
+
+  bool VMCnt = false;
+  bool VSCnt = false;
+  bool LGKMCnt = false;
+
+  if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
+    switch (Scope) {
+    case SIAtomicScope::SYSTEM:
+    case SIAtomicScope::AGENT:
+      if ((Op & SIMemOp::LOAD) != SIMemOp::NONE)
+        VMCnt |= true;
+      if ((Op & SIMemOp::STORE) != SIMemOp::NONE)
+        VSCnt |= true;
+      break;
+    case SIAtomicScope::WORKGROUP:
+      // In WGP mode the waves of a work-group can be executing on either CU of
+      // the WGP. Therefore need to wait for operations to complete to ensure
+      // they are visible to waves in the other CU as the L0 is per CU.
+      // Otherwise in CU mode and all waves of a work-group are on the same CU
+      // which shares the same L0.
+      if (!CuMode) {
+        if ((Op & SIMemOp::LOAD) != SIMemOp::NONE)
+          VMCnt |= true;
+        if ((Op & SIMemOp::STORE) != SIMemOp::NONE)
+          VSCnt |= true;
+      }
+      break;
+    case SIAtomicScope::WAVEFRONT:
+    case SIAtomicScope::SINGLETHREAD:
+      // The L0 cache keeps all memory operations in order for
+      // work-items in the same wavefront.
+      break;
+    default:
+      llvm_unreachable("Unsupported synchronization scope");
+    }
+  }
+
+  if ((AddrSpace & SIAtomicAddrSpace::LDS) != SIAtomicAddrSpace::NONE) {
+    switch (Scope) {
+    case SIAtomicScope::SYSTEM:
+    case SIAtomicScope::AGENT:
+    case SIAtomicScope::WORKGROUP:
+      // If no cross address space ordering then an LDS waitcnt is not
+      // needed as LDS operations for all waves are executed in a
+      // total global ordering as observed by all waves. Required if
+      // also synchronizing with global/GDS memory as LDS operations
+      // could be reordered with respect to later global/GDS memory
+      // operations of the same wave.
+      LGKMCnt |= IsCrossAddrSpaceOrdering;
+      break;
+    case SIAtomicScope::WAVEFRONT:
+    case SIAtomicScope::SINGLETHREAD:
+      // The LDS keeps all memory operations in order for
+      // the same wavesfront.
+      break;
+    default:
+      llvm_unreachable("Unsupported synchronization scope");
+    }
+  }
+
+  if ((AddrSpace & SIAtomicAddrSpace::GDS) != SIAtomicAddrSpace::NONE) {
+    switch (Scope) {
+    case SIAtomicScope::SYSTEM:
+    case SIAtomicScope::AGENT:
+      // If no cross address space ordering then an GDS waitcnt is not
+      // needed as GDS operations for all waves are executed in a
+      // total global ordering as observed by all waves. Required if
+      // also synchronizing with global/LDS memory as GDS operations
+      // could be reordered with respect to later global/LDS memory
+      // operations of the same wave.
+      LGKMCnt |= IsCrossAddrSpaceOrdering;
+      break;
+    case SIAtomicScope::WORKGROUP:
+    case SIAtomicScope::WAVEFRONT:
+    case SIAtomicScope::SINGLETHREAD:
+      // The GDS keeps all memory operations in order for
+      // the same work-group.
+      break;
+    default:
+      llvm_unreachable("Unsupported synchronization scope");
+    }
+  }
+
+  if (VMCnt || LGKMCnt) {
+    unsigned WaitCntImmediate =
+      AMDGPU::encodeWaitcnt(IV,
+                            VMCnt ? 0 : getVmcntBitMask(IV),
+                            getExpcntBitMask(IV),
+                            LGKMCnt ? 0 : getLgkmcntBitMask(IV));
+    BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(WaitCntImmediate);
+    Changed = true;
+  }
+
+  if (VSCnt) {
+    BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT_VSCNT))
+      .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
+      .addImm(0);
+    Changed = true;
+  }
+
+  if (Pos == Position::AFTER)
+    --MI;
+
+  return Changed;
+}
+
 bool SIMemoryLegalizer::removeAtomicPseudoMIs() {
   if (AtomicPseudoMIs.empty())
     return false;
 
   for (auto &MI : AtomicPseudoMIs)
     MI->eraseFromParent();
 
   AtomicPseudoMIs.clear();

llvm/trunk/test/CodeGen/AMDGPU/memory-legalizer-atomic-cmpxchg.ll

 ; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+code-object-v3 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+code-object-v3,+cumode -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10CU %s
 
-; GCN-LABEL: {{^}}system_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_monotonic_monotonic:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @system_monotonic_monotonic(
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
+; GFX10:         .amdhsa_kernel system_one_as_monotonic_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_monotonic_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}system_acquire_monotonic:
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_acquire_monotonic:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_acquire_monotonic(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_acquire_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_acquire_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}system_release_monotonic:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_release_monotonic:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:    buffer_wbinvl1_vol
-define amdgpu_kernel void @system_release_monotonic(
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:   buffer_wbinvl1_vol
+; GFX10-NOT:  buffer_gl._inv
+; GFX10:         .amdhsa_kernel system_one_as_release_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_release_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}system_acq_rel_monotonic:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_acq_rel_monotonic:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_acq_rel_monotonic(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_acq_rel_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_acq_rel_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}system_seq_cst_monotonic:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_seq_cst_monotonic:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_seq_cst_monotonic(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_seq_cst_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_seq_cst_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}system_acquire_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_acquire_acquire:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_acquire_acquire(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_acquire_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_acquire_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}system_release_acquire:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_release_acquire:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_release_acquire(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_release_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_release_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}system_acq_rel_acquire:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_acq_rel_acquire:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_acq_rel_acquire(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_acq_rel_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_acq_rel_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}system_seq_cst_acquire:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_seq_cst_acquire:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_seq_cst_acquire(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_seq_cst_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_seq_cst_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}system_seq_cst_seq_cst:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_seq_cst_seq_cst:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_seq_cst_seq_cst(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_seq_cst_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_seq_cst_seq_cst(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
   ret void
 }
 
-; GCN-LABEL: {{^}}system_one_as_monotonic_monotonic:
+; GCN-LABEL: {{^}}singlethread_one_as_monotonic_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @system_one_as_monotonic_monotonic(
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
+; GFX10:         .amdhsa_kernel singlethread_one_as_monotonic_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_one_as_monotonic_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}system_one_as_acquire_monotonic:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GCN-LABEL: {{^}}singlethread_one_as_acquire_monotonic:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
+; GFX10:         .amdhsa_kernel singlethread_one_as_acquire_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_one_as_acquire_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_one_as_release_monotonic:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; Gfx8-NOT:  buffer_wbinvl1_vol
+; GCN-NOT:   buffer_gl{{[01]}}_inv
+; GFX10:         .amdhsa_kernel singlethread_one_as_release_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_one_as_release_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_one_as_acq_rel_monotonic:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel singlethread_one_as_acq_rel_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_one_as_acq_rel_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_one_as_seq_cst_monotonic:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel singlethread_one_as_seq_cst_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_one_as_seq_cst_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_one_as_acquire_acquire:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel singlethread_one_as_acquire_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_one_as_acquire_acquire(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_one_as_release_acquire:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel singlethread_one_as_release_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_one_as_release_acquire(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_one_as_acq_rel_acquire:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel singlethread_one_as_acq_rel_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_one_as_acq_rel_acquire(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_one_as_seq_cst_acquire:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel singlethread_one_as_seq_cst_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_one_as_seq_cst_acquire(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_one_as_seq_cst_seq_cst:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel singlethread_one_as_seq_cst_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_one_as_seq_cst_seq_cst(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_one_as_monotonic_monotonic:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel agent_one_as_monotonic_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_monotonic_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_one_as_acquire_monotonic:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_one_as_acquire_monotonic(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_acquire_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_acquire_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}system_one_as_release_monotonic:
+; GCN-LABEL: {{^}}agent_one_as_release_monotonic:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    buffer_wbinvl1_vol
-define amdgpu_kernel void @system_one_as_release_monotonic(
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:    buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel agent_one_as_release_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_release_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}system_one_as_acq_rel_monotonic:
+; GCN-LABEL: {{^}}agent_one_as_acq_rel_monotonic:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_one_as_acq_rel_monotonic(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_acq_rel_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_acq_rel_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}system_one_as_seq_cst_monotonic:
+; GCN-LABEL: {{^}}agent_one_as_seq_cst_monotonic:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_one_as_seq_cst_monotonic(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_seq_cst_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_seq_cst_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}system_one_as_acquire_acquire:
+; GCN-LABEL: {{^}}agent_one_as_acquire_acquire:
 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_one_as_acquire_acquire(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_acquire_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_acquire_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}system_one_as_release_acquire:
+; GCN-LABEL: {{^}}agent_one_as_release_acquire:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_one_as_release_acquire(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_release_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_release_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}system_one_as_acq_rel_acquire:
+; GCN-LABEL: {{^}}agent_one_as_acq_rel_acquire:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_one_as_acq_rel_acquire(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_acq_rel_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_acq_rel_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}system_one_as_seq_cst_acquire:
+; GCN-LABEL: {{^}}agent_one_as_seq_cst_acquire:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_one_as_seq_cst_acquire(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_seq_cst_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_seq_cst_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}system_one_as_seq_cst_seq_cst:
+; GCN-LABEL: {{^}}agent_one_as_seq_cst_seq_cst:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @system_one_as_seq_cst_seq_cst(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_seq_cst_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_seq_cst_seq_cst(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}workgroup_one_as_monotonic_monotonic:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_monotonic_monotonic(
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_monotonic_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_monotonic_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_acquire_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_acquire_monotonic(
+; GCN-LABEL:     {{^}}workgroup_one_as_acquire_monotonic:
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:     s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10:         .amdhsa_kernel workgroup_one_as_acquire_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_acquire_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_release_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_release_monotonic(
+; GCN-LABEL:     {{^}}workgroup_one_as_release_monotonic:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:       buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel workgroup_one_as_release_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_release_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_acq_rel_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_acq_rel_monotonic(
+; GCN-LABEL:     {{^}}workgroup_one_as_acq_rel_monotonic:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_acq_rel_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_acq_rel_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_seq_cst_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_seq_cst_monotonic(
+; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst_monotonic:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_seq_cst_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_acquire_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_acquire_acquire(
+; GCN-LABEL:     {{^}}workgroup_one_as_acquire_acquire:
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10:         .amdhsa_kernel workgroup_one_as_acquire_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_acquire_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_release_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_release_acquire(
+; GCN-LABEL:     {{^}}workgroup_one_as_release_acquire:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_release_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_release_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_acq_rel_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_acq_rel_acquire(
+; GCN-LABEL:     {{^}}workgroup_one_as_acq_rel_acquire:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP:      buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_acq_rel_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_acq_rel_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_seq_cst_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_seq_cst_acquire(
+; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst_acquire:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_seq_cst_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_seq_cst_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_seq_cst_seq_cst(
+; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst_seq_cst:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP:      buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_seq_cst_seq_cst(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_one_as_monotonic_monotonic:
+; GCN-LABEL: {{^}}wavefront_one_as_monotonic_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_one_as_monotonic_monotonic(
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_one_as_monotonic_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @wavefront_one_as_monotonic_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") monotonic monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_one_as_acquire_monotonic:
+; GCN-LABEL: {{^}}wavefront_one_as_acquire_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_one_as_acquire_monotonic(
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_one_as_acquire_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @wavefront_one_as_acquire_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_one_as_release_monotonic:
+; GCN-LABEL: {{^}}wavefront_one_as_release_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_one_as_release_monotonic(
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_one_as_release_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @wavefront_one_as_release_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_one_as_acq_rel_monotonic:
+; GCN-LABEL: {{^}}wavefront_one_as_acq_rel_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_one_as_acq_rel_monotonic(
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_one_as_acq_rel_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @wavefront_one_as_acq_rel_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_one_as_seq_cst_monotonic:
+; GCN-LABEL: {{^}}wavefront_one_as_seq_cst_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_one_as_seq_cst_monotonic(
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_one_as_seq_cst_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @wavefront_one_as_seq_cst_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_one_as_acquire_acquire:
+; GCN-LABEL: {{^}}wavefront_one_as_acquire_acquire:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_one_as_acquire_acquire(
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_one_as_acquire_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @wavefront_one_as_acquire_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_one_as_release_acquire:
+; GCN-LABEL: {{^}}wavefront_one_as_release_acquire:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_one_as_release_acquire(
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_one_as_release_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @wavefront_one_as_release_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_one_as_acq_rel_acquire:
+; GCN-LABEL: {{^}}wavefront_one_as_acq_rel_acquire:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_one_as_acq_rel_acquire(
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_one_as_acq_rel_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @wavefront_one_as_acq_rel_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_one_as_seq_cst_acquire:
+; GCN-LABEL: {{^}}wavefront_one_as_seq_cst_acquire:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_one_as_seq_cst_acquire(
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_one_as_seq_cst_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @wavefront_one_as_seq_cst_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}singlethread_one_as_seq_cst_seq_cst:
+; GCN-LABEL: {{^}}wavefront_one_as_seq_cst_seq_cst:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @singlethread_one_as_seq_cst_seq_cst(
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_one_as_seq_cst_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @wavefront_one_as_seq_cst_seq_cst(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst seq_cst
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_monotonic_monotonic(
+; GCN-LABEL: {{^}}system_one_as_acquire_monotonic_ret:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_acquire_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_acquire_monotonic_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_acquire_monotonic:
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_acq_rel_monotonic_ret:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_acquire_monotonic(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_acq_rel_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_acq_rel_monotonic_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_release_monotonic:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:    buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_release_monotonic(
+; GCN-LABEL: {{^}}system_one_as_seq_cst_monotonic_ret:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_seq_cst_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_seq_cst_monotonic_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_acq_rel_monotonic:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_acquire_acquire_ret:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_acq_rel_monotonic(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_acquire_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_acquire_acquire_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_seq_cst_monotonic:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_release_acquire_ret:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_seq_cst_monotonic(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_release_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_release_acquire_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_acquire_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_acq_rel_acquire_ret:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_acquire_acquire(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_acq_rel_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_acq_rel_acquire_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_release_acquire:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_seq_cst_acquire_ret:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_release_acquire(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_seq_cst_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_seq_cst_acquire_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_acq_rel_acquire:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}system_one_as_seq_cst_seq_cst_ret:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_acq_rel_acquire(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_seq_cst_seq_cst_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_seq_cst_seq_cst_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_seq_cst_acquire:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}agent_one_as_acquire_monotonic_ret:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_seq_cst_acquire(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_acquire_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_acquire_monotonic_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_seq_cst_seq_cst:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-LABEL: {{^}}agent_one_as_acq_rel_monotonic_ret:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_seq_cst_seq_cst(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_acq_rel_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_acq_rel_monotonic_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_one_as_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_one_as_monotonic_monotonic(
+; GCN-LABEL: {{^}}agent_one_as_seq_cst_monotonic_ret:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_seq_cst_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_seq_cst_monotonic_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_one_as_acquire_monotonic:
+; GCN-LABEL: {{^}}agent_one_as_acquire_acquire_ret:
 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
 ; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_one_as_acquire_monotonic(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_acquire_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_acquire_acquire_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_one_as_release_monotonic:
+; GCN-LABEL: {{^}}agent_one_as_release_acquire_ret:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_one_as_release_monotonic(
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_release_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_release_acquire_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_one_as_acq_rel_monotonic:
+; GCN-LABEL: {{^}}agent_one_as_acq_rel_acquire_ret:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
 ; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_one_as_acq_rel_monotonic(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_acq_rel_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_acq_rel_acquire_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_one_as_seq_cst_monotonic:
+; GCN-LABEL: {{^}}agent_one_as_seq_cst_acquire_ret:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
 ; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_one_as_seq_cst_monotonic(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_seq_cst_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_seq_cst_acquire_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_one_as_acquire_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-LABEL: {{^}}agent_one_as_seq_cst_seq_cst_ret:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
 ; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_one_as_acquire_acquire(
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_seq_cst_seq_cst_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_seq_cst_seq_cst_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_one_as_release_acquire:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_one_as_release_acquire(
+; GCN-LABEL:     {{^}}workgroup_one_as_acquire_monotonic_ret:
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10:         .amdhsa_kernel workgroup_one_as_acquire_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_acquire_monotonic_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_one_as_acq_rel_acquire:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_one_as_acq_rel_acquire(
+; GCN-LABEL:     {{^}}workgroup_one_as_acq_rel_monotonic_ret:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_acq_rel_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_acq_rel_monotonic_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_one_as_seq_cst_acquire:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_one_as_seq_cst_acquire(
+; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst_monotonic_ret:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_seq_cst_monotonic_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}agent_one_as_seq_cst_seq_cst:
-; GCN:        s_waitcnt vmcnt(0){{$}}
+; GCN-LABEL:     {{^}}workgroup_one_as_acquire_acquire_ret:
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10:         .amdhsa_kernel workgroup_one_as_acquire_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_acquire_acquire_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_one_as_release_acquire_ret:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_release_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_release_acquire_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_one_as_acq_rel_acquire_ret:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP:      buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_acq_rel_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_acq_rel_acquire_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst_acquire_ret:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_seq_cst_acquire_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst_seq_cst_ret:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP:      buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst_seq_cst_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_seq_cst_seq_cst_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_monotonic_monotonic:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
+; GFX10:         .amdhsa_kernel system_monotonic_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_monotonic_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic monotonic
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_acquire_monotonic:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_acquire_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_acquire_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_release_monotonic:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-define amdgpu_kernel void @agent_one_as_seq_cst_seq_cst(
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:   buffer_wbinvl1_vol
+; GFX10-NOT:  buffer_gl._inv
+; GFX10:         .amdhsa_kernel system_release_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_release_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_monotonic_monotonic(
+; GCN-LABEL: {{^}}system_acq_rel_monotonic:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_acq_rel_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_acq_rel_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_seq_cst_monotonic:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_seq_cst_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_seq_cst_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_acquire_acquire:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_acquire_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_acquire_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_acquire_monotonic:
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_acquire_monotonic(
+; GCN-LABEL: {{^}}system_release_acquire:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_release_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_release_acquire(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_acq_rel_acquire:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_acq_rel_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_acq_rel_acquire(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_seq_cst_acquire:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_seq_cst_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_seq_cst_acquire(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_seq_cst_seq_cst:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_seq_cst_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_seq_cst_seq_cst(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_monotonic_monotonic:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
+; GFX10:         .amdhsa_kernel singlethread_monotonic_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_monotonic_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_release_monotonic:
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:    buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_release_monotonic(
+; GCN-LABEL: {{^}}singlethread_acquire_monotonic:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
+; GFX10:         .amdhsa_kernel singlethread_acquire_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_acquire_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_acq_rel_monotonic:
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_acq_rel_monotonic(
+; GCN-LABEL: {{^}}singlethread_release_monotonic:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; Gfx8-NOT:  buffer_wbinvl1_vol
+; GCN-NOT:   buffer_gl{{[01]}}_inv
+; GFX10:         .amdhsa_kernel singlethread_release_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_release_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_acq_rel_monotonic:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel singlethread_acq_rel_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_acq_rel_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_seq_cst_monotonic:
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_seq_cst_monotonic(
+; GCN-LABEL: {{^}}singlethread_seq_cst_monotonic:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel singlethread_seq_cst_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_seq_cst_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_acquire_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_acquire_acquire(
+; GCN-LABEL: {{^}}singlethread_acquire_acquire:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel singlethread_acquire_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_acquire_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_release_acquire:
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_release_acquire(
+; GCN-LABEL: {{^}}singlethread_release_acquire:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel singlethread_release_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_release_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_acq_rel_acquire:
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_acq_rel_acquire(
+; GCN-LABEL: {{^}}singlethread_acq_rel_acquire:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel singlethread_acq_rel_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_acq_rel_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_seq_cst_acquire:
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_seq_cst_acquire(
+; GCN-LABEL: {{^}}singlethread_seq_cst_acquire:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel singlethread_seq_cst_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_seq_cst_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_seq_cst_seq_cst:
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_seq_cst_seq_cst(
+; GCN-LABEL: {{^}}singlethread_seq_cst_seq_cst:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel singlethread_seq_cst_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @singlethread_seq_cst_seq_cst(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_monotonic_monotonic:
+; GCN-LABEL: {{^}}agent_monotonic_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_one_as_monotonic_monotonic(
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel agent_monotonic_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_monotonic_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_acquire_monotonic:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_one_as_acquire_monotonic(
+; GCN-LABEL: {{^}}agent_acquire_monotonic:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_acquire_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_acquire_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_release_monotonic:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:    buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel agent_release_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_release_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_acq_rel_monotonic:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_acq_rel_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_acq_rel_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_seq_cst_monotonic:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_seq_cst_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_seq_cst_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_acquire_acquire:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_acquire_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_acquire_acquire(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_release_acquire:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_release_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_release_acquire(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_acq_rel_acquire:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_acq_rel_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_acq_rel_acquire(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_seq_cst_acquire:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_seq_cst_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_seq_cst_acquire(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_seq_cst_seq_cst:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_seq_cst_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_seq_cst_seq_cst(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
+  ret void
+}
+
+; GCN-LABEL: {{^}}workgroup_monotonic_monotonic:
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel workgroup_monotonic_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_monotonic_monotonic(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_acquire_monotonic:
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:     s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10:         .amdhsa_kernel workgroup_acquire_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_acquire_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_release_monotonic:
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_one_as_release_monotonic(
+; GCN-LABEL:     {{^}}workgroup_release_monotonic:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:       buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel workgroup_release_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_release_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_acq_rel_monotonic:
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_one_as_acq_rel_monotonic(
+; GCN-LABEL:     {{^}}workgroup_acq_rel_monotonic:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_acq_rel_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_acq_rel_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_seq_cst_monotonic:
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_one_as_seq_cst_monotonic(
+; GCN-LABEL:     {{^}}workgroup_seq_cst_monotonic:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_seq_cst_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_seq_cst_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_acquire_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_one_as_acquire_acquire(
+; GCN-LABEL:     {{^}}workgroup_acquire_acquire:
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10:         .amdhsa_kernel workgroup_acquire_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_acquire_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_release_acquire:
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_one_as_release_acquire(
+; GCN-LABEL:     {{^}}workgroup_release_acquire:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_release_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_release_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_acq_rel_acquire:
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_one_as_acq_rel_acquire(
+; GCN-LABEL:     {{^}}workgroup_acq_rel_acquire:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP:      buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_acq_rel_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_acq_rel_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_seq_cst_acquire:
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_one_as_seq_cst_acquire(
+; GCN-LABEL:     {{^}}workgroup_seq_cst_acquire:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_seq_cst_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_seq_cst_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_seq_cst_seq_cst:
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @workgroup_one_as_seq_cst_seq_cst(
+; GCN-LABEL:     {{^}}workgroup_seq_cst_seq_cst:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP:      buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_seq_cst_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_seq_cst_seq_cst(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_monotonic_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_monotonic_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
   %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") monotonic monotonic
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_acquire_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_acquire_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_acquire_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
   %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acquire monotonic
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_release_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_release_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_release_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
   %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") release monotonic
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_acq_rel_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_acq_rel_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_acq_rel_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
   %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel monotonic
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_seq_cst_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_seq_cst_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_seq_cst_monotonic(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
   %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst monotonic
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_acquire_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_acquire_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_acquire_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
   %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acquire acquire
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_release_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_release_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_release_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
   %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") release acquire
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_acq_rel_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_acq_rel_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_acq_rel_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
   %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel acquire
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_seq_cst_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_seq_cst_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_seq_cst_acquire(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
   %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst acquire
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_seq_cst_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX8-NOT:  buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}._inv
+; GFX10:         .amdhsa_kernel wavefront_seq_cst_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_seq_cst_seq_cst(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
   %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst seq_cst
   ret void
 }
 
-; GCN-LABEL: {{^}}wavefront_one_as_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @wavefront_one_as_monotonic_monotonic(
+; GCN-LABEL: {{^}}system_acquire_monotonic_ret:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_acquire_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_acquire_monotonic_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") monotonic monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}wavefront_one_as_acquire_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @wavefront_one_as_acquire_monotonic(
+; GCN-LABEL: {{^}}system_acq_rel_monotonic_ret:
+; GCN:        s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_acq_rel_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_acq_rel_monotonic_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}wavefront_one_as_release_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @wavefront_one_as_release_monotonic(
+; GCN-LABEL: {{^}}system_seq_cst_monotonic_ret:
+; GCN:        s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_seq_cst_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_seq_cst_monotonic_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}wavefront_one_as_acq_rel_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @wavefront_one_as_acq_rel_monotonic(
+; GCN-LABEL: {{^}}system_acquire_acquire_ret:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_acquire_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_acquire_acquire_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}wavefront_one_as_seq_cst_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @wavefront_one_as_seq_cst_monotonic(
+; GCN-LABEL: {{^}}system_release_acquire_ret:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_release_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_release_acquire_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst monotonic
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}wavefront_one_as_acquire_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @wavefront_one_as_acquire_acquire(
+; GCN-LABEL: {{^}}system_acq_rel_acquire_ret:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_acq_rel_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_acq_rel_acquire_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}wavefront_one_as_release_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @wavefront_one_as_release_acquire(
+; GCN-LABEL: {{^}}system_seq_cst_acquire_ret:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_seq_cst_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_seq_cst_acquire_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}wavefront_one_as_acq_rel_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @wavefront_one_as_acq_rel_acquire(
+; GCN-LABEL: {{^}}system_seq_cst_seq_cst_ret:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_seq_cst_seq_cst_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_seq_cst_seq_cst_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}wavefront_one_as_seq_cst_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @wavefront_one_as_seq_cst_acquire(
+; GCN-LABEL: {{^}}agent_acquire_monotonic_ret:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_acquire_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_acquire_monotonic_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_acq_rel_monotonic_ret:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_acq_rel_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_acq_rel_monotonic_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_seq_cst_monotonic_ret:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_seq_cst_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_seq_cst_monotonic_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_acquire_acquire_ret:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_acquire_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_acquire_acquire_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst acquire
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}wavefront_one_as_seq_cst_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
-define amdgpu_kernel void @wavefront_one_as_seq_cst_seq_cst(
+; GCN-LABEL: {{^}}agent_release_acquire_ret:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_release_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_release_acquire_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_acq_rel_acquire_ret:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_acq_rel_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_acq_rel_acquire_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_seq_cst_acquire_ret:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_seq_cst_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_seq_cst_acquire_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_seq_cst_seq_cst_ret:
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_seq_cst_seq_cst_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_seq_cst_seq_cst_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_acquire_monotonic_ret:
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8:          s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU:       s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10:         .amdhsa_kernel workgroup_acquire_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_acquire_monotonic_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_acq_rel_monotonic_ret:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8:          s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10CU:       s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_acq_rel_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_acq_rel_monotonic_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_seq_cst_monotonic_ret:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8:          s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10CU:       s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_seq_cst_monotonic_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_seq_cst_monotonic_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_acquire_acquire_ret:
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8:          s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU:       s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10:         .amdhsa_kernel workgroup_acquire_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_acquire_acquire_ret(
     i32* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst seq_cst
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_release_acquire_ret:
+; GFX8:          s_waitcnt lgkmcnt(0){{$}}
+; GFX8:          flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10:         flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX8:          s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10CU:       s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_release_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_release_acquire_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_acq_rel_acquire_ret:
+; GFX8:          s_waitcnt lgkmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10CU:       s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP:      buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_acq_rel_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_acq_rel_acquire_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_seq_cst_acquire_ret:
+; GFX8:          s_waitcnt lgkmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10CU:       s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_seq_cst_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_seq_cst_acquire_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_seq_cst_seq_cst_ret:
+; GFX8:          s_waitcnt lgkmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10CU:       s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP:      buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_seq_cst_seq_cst_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_seq_cst_seq_cst_ret(
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
   ret void
 }

llvm/trunk/test/CodeGen/AMDGPU/memory-legalizer-atomic-fence.ll

 ; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GFX6,GFX68 %s
 ; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GFX8,GFX68 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GFX8,GFX68 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+code-object-v3 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GFX10,GFX10WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+code-object-v3,+cumode -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GFX10,GFX10CU %s
 
 ; FUNC-LABEL: {{^}}system_one_as_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GFX6:       s_waitcnt vmcnt(0){{$}}
 ; GFX6-NEXT:  buffer_wbinvl1{{$}}
 ; GFX8:       s_waitcnt vmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol{{$}}
-; GCN:        s_endpgm
+; GFX10:      s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10-NEXT: buffer_gl0_inv{{$}}
+; GFX10-NEXT: buffer_gl1_inv{{$}}
+; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel system_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_acquire() {
 entry:
   fence syncscope("one-as") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_one_as_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel system_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_release() {
 entry:
   fence syncscope("one-as") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_one_as_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX6:       buffer_wbinvl1{{$}}
 ; GFX8:       buffer_wbinvl1_vol{{$}}
+; GFX10-NEXT: buffer_gl0_inv{{$}}
+; GFX10-NEXT: buffer_gl1_inv{{$}}
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel system_one_as_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_acq_rel() {
 entry:
   fence syncscope("one-as") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_one_as_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX6:       buffer_wbinvl1{{$}}
 ; GFX8:       buffer_wbinvl1_vol{{$}}
+; GFX10-NEXT: buffer_gl0_inv{{$}}
+; GFX10-NEXT: buffer_gl1_inv{{$}}
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel system_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_seq_cst() {
 entry:
   fence syncscope("one-as") seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}singlethread_one_as_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel singlethread_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_acquire() {
 entry:
   fence syncscope("singlethread-one-as") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}singlethread_one_as_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel singlethread_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_release() {
 entry:
   fence syncscope("singlethread-one-as") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}singlethread_one_as_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel singlethread_one_as_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_acq_rel() {
 entry:
   fence syncscope("singlethread-one-as") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}singlethread_one_as_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel singlethread_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_seq_cst() {
 entry:
   fence syncscope("singlethread-one-as") seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_one_as_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GFX6:       s_waitcnt vmcnt(0){{$}}
 ; GFX6-NEXT:  buffer_wbinvl1{{$}}
 ; GFX8:       s_waitcnt vmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol{{$}}
-; GCN:        s_endpgm
+; GFX10:      s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10-NEXT: buffer_gl0_inv{{$}}
+; GFX10-NEXT: buffer_gl1_inv{{$}}
+; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel agent_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_acquire() {
 entry:
   fence syncscope("agent-one-as") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_one_as_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel agent_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_release() {
 entry:
   fence syncscope("agent-one-as") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_one_as_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX6:       buffer_wbinvl1{{$}}
 ; GFX8:       buffer_wbinvl1_vol{{$}}
+; GFX10-NEXT: buffer_gl0_inv{{$}}
+; GFX10-NEXT: buffer_gl1_inv{{$}}
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel agent_one_as_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_acq_rel() {
 entry:
   fence syncscope("agent-one-as") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_one_as_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX6:       buffer_wbinvl1{{$}}
 ; GFX8:       buffer_wbinvl1_vol{{$}}
+; GFX10-NEXT: buffer_gl0_inv{{$}}
+; GFX10-NEXT: buffer_gl1_inv{{$}}
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel agent_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_seq_cst() {
 entry:
   fence syncscope("agent-one-as") seq_cst
   ret void
 }
 
-; FUNC-LABEL: {{^}}workgroup_one_as_acquire:
-; GCN:        %bb.0
-; GFX68-NOT:  s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
+; FUNC-LABEL:    {{^}}workgroup_one_as_acquire:
+; GCN:           %bb.0
+; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10WGP-NEXT: buffer_gl0_inv{{$}}
+; GFX10CU-NOT:   buffer_gl0_inv{{$}}
+; GCN-NOT:       ATOMIC_FENCE
+; GCN:           s_endpgm
+; GFX10:         .amdhsa_kernel workgroup_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_acquire() {
 entry:
   fence syncscope("workgroup-one-as") acquire
   ret void
 }
 
-; FUNC-LABEL: {{^}}workgroup_one_as_release:
-; GCN:        %bb.0
-; GFX68-NOT:  s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
+; FUNC-LABEL:    {{^}}workgroup_one_as_release:
+; GCN:           %bb.0
+; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10-NOT:     buffer_gl0_inv
+; GCN-NOT:       ATOMIC_FENCE
+; GCN:           s_endpgm
+; GFX10:         .amdhsa_kernel workgroup_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_release() {
 entry:
   fence syncscope("workgroup-one-as") release
   ret void
 }
 
-; FUNC-LABEL: {{^}}workgroup_one_as_acq_rel:
-; GCN:        %bb.0
-; GFX68-NOT:  s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
+; FUNC-LABEL:    {{^}}workgroup_one_as_acq_rel:
+; GCN:           %bb.0
+; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10WGP-NEXT: buffer_gl0_inv{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   buffer_gl0_inv{{$}}
+; GCN-NOT:       ATOMIC_FENCE
+; GCN:           s_endpgm
+; GFX10:         .amdhsa_kernel workgroup_one_as_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_acq_rel() {
 entry:
   fence syncscope("workgroup-one-as") acq_rel
   ret void
 }
 
-; FUNC-LABEL: {{^}}workgroup_one_as_seq_cst:
-; GCN:        %bb.0
-; GFX68-NOT:  s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
+; FUNC-LABEL:    {{^}}workgroup_one_as_seq_cst:
+; GCN:           %bb.0
+; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10WGP-NEXT: buffer_gl0_inv{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   buffer_gl0_inv{{$}}
+; GCN-NOT:       ATOMIC_FENCE
+; GCN:           s_endpgm
+; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_seq_cst() {
 entry:
   fence syncscope("workgroup-one-as") seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}wavefront_one_as_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel wavefront_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_acquire() {
 entry:
   fence syncscope("wavefront-one-as") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}wavefront_one_as_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel wavefront_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_release() {
 entry:
   fence syncscope("wavefront-one-as") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}wavefront_one_as_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel wavefront_one_as_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_acq_rel() {
 entry:
   fence syncscope("wavefront-one-as") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}wavefront_one_as_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel wavefront_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_seq_cst() {
 entry:
   fence syncscope("wavefront-one-as") seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GFX6-NEXT:  buffer_wbinvl1{{$}}
 ; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol{{$}}
-; GCN:        s_endpgm
+; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10-NEXT: buffer_gl0_inv{{$}}
+; GFX10-NEXT: buffer_gl1_inv{{$}}
+; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel system_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_acquire() {
 entry:
   fence acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel system_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_release() {
 entry:
   fence release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX6:       buffer_wbinvl1{{$}}
 ; GFX8:       buffer_wbinvl1_vol{{$}}
+; GFX10-NEXT: buffer_gl0_inv{{$}}
+; GFX10-NEXT: buffer_gl1_inv{{$}}
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel system_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_acq_rel() {
 entry:
   fence acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}system_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX6:       buffer_wbinvl1{{$}}
 ; GFX8:       buffer_wbinvl1_vol{{$}}
+; GFX10-NEXT: buffer_gl0_inv{{$}}
+; GFX10-NEXT: buffer_gl1_inv{{$}}
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel system_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_seq_cst() {
 entry:
   fence seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}singlethread_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel singlethread_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_acquire() {
 entry:
   fence syncscope("singlethread") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}singlethread_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel singlethread_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_release() {
 entry:
   fence syncscope("singlethread") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}singlethread_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel singlethread_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_acq_rel() {
 entry:
   fence syncscope("singlethread") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}singlethread_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel singlethread_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_seq_cst() {
 entry:
   fence syncscope("singlethread") seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GFX6-NEXT:  buffer_wbinvl1{{$}}
 ; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol{{$}}
-; GCN:        s_endpgm
+; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10-NEXT: buffer_gl0_inv{{$}}
+; GFX10-NEXT: buffer_gl1_inv{{$}}
+; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel agent_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_acquire() {
 entry:
   fence syncscope("agent") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel agent_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_release() {
 entry:
   fence syncscope("agent") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX6:       buffer_wbinvl1{{$}}
 ; GFX8:       buffer_wbinvl1_vol{{$}}
+; GFX10-NEXT: buffer_gl0_inv{{$}}
+; GFX10-NEXT: buffer_gl1_inv{{$}}
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel agent_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_acq_rel() {
 entry:
   fence syncscope("agent") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}agent_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX6:       buffer_wbinvl1{{$}}
 ; GFX8:       buffer_wbinvl1_vol{{$}}
+; GFX10-NEXT: buffer_gl0_inv{{$}}
+; GFX10-NEXT: buffer_gl1_inv{{$}}
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel agent_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_seq_cst() {
 entry:
   fence syncscope("agent") seq_cst
   ret void
 }
 
-; FUNC-LABEL: {{^}}workgroup_acquire:
-; GCN:        %bb.0
-; GFX68-NOT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
+; FUNC-LABEL:    {{^}}workgroup_acquire:
+; GCN:           %bb.0
+; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10WGP-NEXT: buffer_gl0_inv{{$}}
+; GFX10CU-NOT:   buffer_gl0_inv{{$}}
+; GCN-NOT:       ATOMIC_FENCE
+; GCN:           s_endpgm
+; GFX10:         .amdhsa_kernel workgroup_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_acquire() {
 entry:
   fence syncscope("workgroup") acquire
   ret void
 }
 
-; FUNC-LABEL: {{^}}workgroup_release:
-; GCN:        %bb.0
-; GFX68-NOT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
+; FUNC-LABEL:    {{^}}workgroup_release:
+; GCN:           %bb.0
+; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10-NOT:     buffer_gl0_inv
+; GCN-NOT:       ATOMIC_FENCE
+; GCN:           s_endpgm
+; GFX10:         .amdhsa_kernel workgroup_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_release() {
 entry:
   fence syncscope("workgroup") release
   ret void
 }
 
-; FUNC-LABEL: {{^}}workgroup_acq_rel:
-; GCN:        %bb.0
-; GFX68-NOT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
+; FUNC-LABEL:    {{^}}workgroup_acq_rel:
+; GCN:           %bb.0
+; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10WGP-NEXT: buffer_gl0_inv{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   buffer_gl0_inv{{$}}
+; GCN-NOT:       ATOMIC_FENCE
+; GCN:           s_endpgm
+; GFX10:         .amdhsa_kernel workgroup_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_acq_rel() {
 entry:
   fence syncscope("workgroup") acq_rel
   ret void
 }
 
-; FUNC-LABEL: {{^}}workgroup_seq_cst:
-; GCN:        %bb.0
-; GFX68-NOT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
+; FUNC-LABEL:    {{^}}workgroup_seq_cst:
+; GCN:           %bb.0
+; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10WGP-NEXT: buffer_gl0_inv{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   buffer_gl0_inv{{$}}
+; GCN-NOT:       ATOMIC_FENCE
+; GCN:           s_endpgm
+; GFX10:         .amdhsa_kernel workgroup_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_seq_cst() {
 entry:
   fence syncscope("workgroup") seq_cst
   ret void
 }
 
 ; FUNC-LABEL: {{^}}wavefront_acquire:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel wavefront_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_acquire() {
 entry:
   fence syncscope("wavefront") acquire
   ret void
 }
 
 ; FUNC-LABEL: {{^}}wavefront_release:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel wavefront_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_release() {
 entry:
   fence syncscope("wavefront") release
   ret void
 }
 
 ; FUNC-LABEL: {{^}}wavefront_acq_rel:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel wavefront_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_acq_rel() {
 entry:
   fence syncscope("wavefront") acq_rel
   ret void
 }
 
 ; FUNC-LABEL: {{^}}wavefront_seq_cst:
 ; GCN:        %bb.0
 ; GCN-NOT:    ATOMIC_FENCE
 ; GCN:        s_endpgm
+; GFX10:         .amdhsa_kernel wavefront_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_seq_cst() {
 entry:
   fence syncscope("wavefront") seq_cst
   ret void
 }

llvm/trunk/test/CodeGen/AMDGPU/memory-legalizer-atomic-rmw.ll

 ; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+code-object-v3 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+code-object-v3,+cumode -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10CU %s
 
 ; GCN-LABEL: {{^}}system_one_as_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel system_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_monotonic(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") monotonic
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_one_as_acquire:
 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_acquire(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acquire
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_one_as_release:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    buffer_wbinvl1_vol
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:    buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel system_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_release(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") release
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_one_as_acq_rel:
 ; GCN:         s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT:  s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:    flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NEXT:    s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT:  s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:   buffer_wbinvl1_vol
+; GFX10-NEXT:  buffer_gl0_inv
+; GFX10-NEXT:  buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_acq_rel(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acq_rel
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_one_as_seq_cst:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_seq_cst(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") seq_cst
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_one_as_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel singlethread_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_monotonic(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") monotonic
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_one_as_acquire:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel singlethread_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_acquire(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acquire
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_one_as_release:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel singlethread_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_release(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") release
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_one_as_acq_rel:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel singlethread_one_as_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_acq_rel(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acq_rel
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_one_as_seq_cst:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel singlethread_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_seq_cst(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") seq_cst
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_one_as_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel agent_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_monotonic(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") monotonic
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_one_as_acquire:
 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_acquire(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_one_as_release:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    buffer_wbinvl1_vol
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:    buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel agent_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_release(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") release
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_one_as_acq_rel:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_acq_rel(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_one_as_seq_cst:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_seq_cst(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
   ret void
 }
 
 ; GCN-LABEL: {{^}}workgroup_one_as_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel workgroup_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_monotonic(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
+; GCN-LABEL:     {{^}}workgroup_one_as_acquire:
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_acquire(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_release:
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    buffer_wbinvl1_vol
+; GCN-LABEL:     {{^}}workgroup_one_as_release:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:       buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel workgroup_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_release(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") release
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_acq_rel:
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
+; GCN-LABEL:     {{^}}workgroup_one_as_acq_rel:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_acq_rel(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_seq_cst:
-; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
+; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_seq_cst(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_one_as_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel wavefront_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_monotonic(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") monotonic
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_one_as_acquire:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel wavefront_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_acquire(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") acquire
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_one_as_release:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel wavefront_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_release(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") release
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_one_as_acq_rel:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel wavefront_one_as_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_acq_rel(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") acq_rel
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_one_as_seq_cst:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel wavefront_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_seq_cst(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") seq_cst
   ret void
 }
 
+; GCN-LABEL: {{^}}system_one_as_acquire_ret:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_acquire_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_one_as_acq_rel_ret:
+; GCN:         s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT:  s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:    flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN-NEXT:    s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:   buffer_wbinvl1_vol
+; GFX10-NEXT:  buffer_gl0_inv
+; GFX10-NEXT:  buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_acq_rel_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_acq_rel_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_one_as_seq_cst_ret:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_one_as_seq_cst_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_one_as_seq_cst_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_one_as_acquire_ret:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_acquire_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_one_as_acq_rel_ret:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_acq_rel_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_acq_rel_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_one_as_seq_cst_ret:
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_one_as_seq_cst_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_one_as_seq_cst_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_one_as_acquire_ret:
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:           flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_acquire_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_one_as_acq_rel_ret:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_acq_rel_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_acq_rel_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst_ret:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_one_as_seq_cst_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
 ; GCN-LABEL: {{^}}system_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel system_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_monotonic(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in monotonic
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_acquire(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_release:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:    buffer_wbinvl1_vol
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:    buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel system_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_release(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in release
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_acq_rel:
-; GCN:         s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:    flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:   buffer_wbinvl1_vol
+; GFX10-NEXT:  buffer_gl0_inv
+; GFX10-NEXT:  buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_acq_rel(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_seq_cst:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_seq_cst(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel singlethread_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_monotonic(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") monotonic
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel singlethread_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_acquire(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acquire
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_release:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel singlethread_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_release(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") release
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_acq_rel:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel singlethread_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_acq_rel(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acq_rel
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel singlethread_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_seq_cst(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") seq_cst
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel agent_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_monotonic(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") monotonic
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_acquire(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_release:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:    buffer_wbinvl1_vol
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:    buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel agent_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_release(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") release
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_acq_rel:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_acq_rel(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_seq_cst:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_seq_cst(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
   ret void
 }
 
 ; GCN-LABEL: {{^}}workgroup_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel workgroup_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_monotonic(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") monotonic
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
+; GCN-LABEL:     {{^}}workgroup_acquire:
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_acquire(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_release:
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:    buffer_wbinvl1_vol
+; GCN-LABEL:     {{^}}workgroup_release:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:       buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel workgroup_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_release(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") release
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_acq_rel:
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
+; GCN-LABEL:     {{^}}workgroup_acq_rel:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_acq_rel(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_seq_cst:
-; GFX8-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
+; GCN-LABEL:     {{^}}workgroup_seq_cst:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_seq_cst(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel wavefront_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_monotonic(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") monotonic
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel wavefront_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_acquire(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") acquire
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_release:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel wavefront_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_release(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") release
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_acq_rel:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel wavefront_acq_rel
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_acq_rel(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") acq_rel
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:   buffer_wbinvl1_vol
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
+; GFX10:         .amdhsa_kernel wavefront_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_seq_cst(
     i32* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") seq_cst
   ret void
 }
+
+; GCN-LABEL: {{^}}system_acquire_ret:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_acquire_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_acq_rel_ret:
+; GFX8:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:    flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GFX8-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:   buffer_wbinvl1_vol
+; GFX10-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT:  buffer_gl0_inv
+; GFX10-NEXT:  buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_acq_rel_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_acq_rel_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_seq_cst_ret:
+; GFX8:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GFX8-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:   buffer_wbinvl1_vol
+; GFX10-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT:  buffer_gl0_inv
+; GFX10-NEXT:  buffer_gl1_inv
+; GFX10:         .amdhsa_kernel system_seq_cst_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @system_seq_cst_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_acquire_ret:
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:        flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GFX8-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:   buffer_wbinvl1_vol
+; GFX10-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT:  buffer_gl0_inv
+; GFX10-NEXT:  buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_acquire_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_acq_rel_ret:
+; GFX8:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GFX8-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:   buffer_wbinvl1_vol
+; GFX10-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT:  buffer_gl0_inv
+; GFX10-NEXT:  buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_acq_rel_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_acq_rel_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_seq_cst_ret:
+; GFX8:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt lgkmcnt(0){{$}}
+; GFX10:       s_waitcnt_vscnt null, 0x0{{$}}
+; GCN-NEXT:   flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GFX8-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX8-NEXT:   buffer_wbinvl1_vol
+; GFX10-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT:  buffer_gl0_inv
+; GFX10-NEXT:  buffer_gl1_inv
+; GFX10:         .amdhsa_kernel agent_seq_cst_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @agent_seq_cst_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_acquire_ret:
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GCN:           flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_acquire_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_acquire_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_acq_rel_ret:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_acq_rel_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_acq_rel_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+; GCN-LABEL:     {{^}}workgroup_seq_cst_ret:
+; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX8-NOT:      buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GFX10:         .amdhsa_kernel workgroup_seq_cst_ret
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
+define amdgpu_kernel void @workgroup_seq_cst_ret(
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}

llvm/trunk/test/CodeGen/AMDGPU/memory-legalizer-invalid-syncscope.ll

 ; RUN: not llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s 2>&1 | FileCheck %s
 ; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s 2>&1 | FileCheck %s
 
 ; CHECK: error: <unknown>:0:0: in function invalid_fence void (): Unsupported atomic synchronization scope
 define amdgpu_kernel void @invalid_fence() {
 entry:
   fence syncscope("invalid") seq_cst
   ret void
 }
 

llvm/trunk/test/CodeGen/AMDGPU/memory-legalizer-load.ll

 ; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX8,GFX89 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX8,GFX89 %s
 ; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx900 -verify-machineinstrs -amdgpu-enable-global-sgpr-addr < %s | FileCheck --check-prefixes=GCN,GFX9,GFX89 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs -amdgpu-enable-global-sgpr-addr < %s | FileCheck --check-prefixes=GCN,GFX9,GFX89 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+code-object-v3 -verify-machineinstrs -amdgpu-enable-global-sgpr-addr < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+code-object-v3,+cumode -verify-machineinstrs -amdgpu-enable-global-sgpr-addr < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10CU %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 
 ; GCN-LABEL: {{^}}system_one_as_unordered:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel system_one_as_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_unordered(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("one-as") unordered, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_one_as_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel system_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_monotonic(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("one-as") monotonic, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_one_as_acquire:
 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX89:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
 ; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX89-NEXT: buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
 ; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel system_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_acquire(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("one-as") acquire, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_one_as_seq_cst:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN-NEXT:   flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX89-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
 ; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX89-NEXT: buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
 ; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel system_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_seq_cst(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("one-as") seq_cst, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_one_as_unordered:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel singlethread_one_as_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_unordered(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("singlethread-one-as") unordered, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_one_as_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel singlethread_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_monotonic(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("singlethread-one-as") monotonic, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_one_as_acquire:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel singlethread_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_acquire(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("singlethread-one-as") acquire, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_one_as_seq_cst:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel singlethread_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_seq_cst(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("singlethread-one-as") seq_cst, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_one_as_unordered:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel agent_one_as_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_unordered(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("agent-one-as") unordered, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_one_as_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel agent_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_monotonic(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("agent-one-as") monotonic, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_one_as_acquire:
 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX89:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
 ; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX89-NEXT: buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
 ; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel agent_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_acquire(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("agent-one-as") acquire, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_one_as_seq_cst:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN-NEXT:   flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX89-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
 ; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 ; GFX89-NEXT: buffer_wbinvl1_vol
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
 ; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel agent_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_seq_cst(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("agent-one-as") seq_cst, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}workgroup_one_as_unordered:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel workgroup_one_as_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_unordered(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("workgroup-one-as") unordered, align 4
   store i32 %val, i32* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX89:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GCN-LABEL:    {{^}}workgroup_one_as_monotonic:
+; GCN-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX89:        flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GFX10WGP:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10CU-NOT:  flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GCN-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX89-NOT:    buffer_wbinvl1_vol
+; GFX10-NOT:    buffer_gl{{[01]}}_inv
+; GCN:          flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel workgroup_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_monotonic(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("workgroup-one-as") monotonic, align 4
   store i32 %val, i32* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GFX89:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GFX89-NOT:  s_waitcnt vmcnt(0){{$}}
-; GFX89-NOT:  buffer_wbinvl1_vol
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GCN-LABEL:     {{^}}workgroup_one_as_acquire:
+; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:     s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX89:         flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GFX10WGP:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10CU-NOT:   flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX89-NOT:     buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GCN:           flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel workgroup_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_acquire(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("workgroup-one-as") acquire, align 4
   store i32 %val, i32* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_seq_cst:
-; GFX89-NOT:  s_waitcnt vmcnt(0){{$}}
-; GFX89:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GFX89-NOT:  s_waitcnt vmcnt(0){{$}}
-; GFX89-NOT:  buffer_wbinvl1_vol
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst:
+; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0
+; GFX89:         flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GFX10WGP:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10CU:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX89-NOT:     buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GCN:           flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_seq_cst(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("workgroup-one-as") seq_cst, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_one_as_unordered:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel wavefront_one_as_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_unordered(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("wavefront-one-as") unordered, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_one_as_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel wavefront_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_monotonic(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("wavefront-one-as") monotonic, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_one_as_acquire:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel wavefront_one_as_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_acquire(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("wavefront-one-as") acquire, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_one_as_seq_cst:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel wavefront_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_seq_cst(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("wavefront-one-as") seq_cst, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_private_0:
 ; GFX89: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen glc slc{{$}}
+; GFX10: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen slc{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_private_0
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_private_0(
     i32 addrspace(5)* %in, i32* %out) {
 entry:
   %val = load i32, i32 addrspace(5)* %in, align 4, !nontemporal !0
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_private_1:
 ; GFX89: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen glc slc{{$}}
+; GFX10: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen slc{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_private_1
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_private_1(
     i32 addrspace(5)* %in, i32* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val.gep = getelementptr inbounds i32, i32 addrspace(5)* %in, i32 %tid
   %val = load i32, i32 addrspace(5)* %val.gep, align 4, !nontemporal !0
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_global_0:
 ; GCN: s_load_dword s{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0x0{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_global_0
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_global_0(
     i32 addrspace(1)* %in, i32* %out) {
 entry:
   %val = load i32, i32 addrspace(1)* %in, align 4, !nontemporal !0
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_global_1:
 ; GFX8:  flat_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}] glc slc{{$}}
 ; GFX9:  global_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}] glc slc{{$}}
+; GFX10: global_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}] slc{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_global_1
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_global_1(
     i32 addrspace(1)* %in, i32* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val.gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i32 %tid
   %val = load i32, i32 addrspace(1)* %val.gep, align 4, !nontemporal !0
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_local_0:
 ; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_local_0
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_local_0(
     i32 addrspace(3)* %in, i32* %out) {
 entry:
   %val = load i32, i32 addrspace(3)* %in, align 4, !nontemporal !0
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_local_1:
 ; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_local_1
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_local_1(
     i32 addrspace(3)* %in, i32* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val.gep = getelementptr inbounds i32, i32 addrspace(3)* %in, i32 %tid
   %val = load i32, i32 addrspace(3)* %val.gep, align 4, !nontemporal !0
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_flat_0:
 ; GFX89: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}] glc slc{{$}}
+; GFX10: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}] slc{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_flat_0
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_flat_0(
     i32* %in, i32* %out) {
 entry:
   %val = load i32, i32* %in, align 4, !nontemporal !0
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_flat_1:
 ; GFX89: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}] glc slc{{$}}
+; GFX10: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}] slc{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_flat_1
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_flat_1(
     i32* %in, i32* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val.gep = getelementptr inbounds i32, i32* %in, i32 %tid
   %val = load i32, i32* %val.gep, align 4, !nontemporal !0
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel system_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_unordered(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in unordered, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GCN:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel system_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_monotonic(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in monotonic, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX89:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
+; GFX89-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GFX89-NEXT: buffer_wbinvl1_vol
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel system_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_acquire(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in acquire, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_seq_cst:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NEXT:   flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX89:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GFX89-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
+; GFX89-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GFX89-NEXT: buffer_wbinvl1_vol
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel system_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_seq_cst(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in seq_cst, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel singlethread_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_unordered(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("singlethread") unordered, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel singlethread_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_monotonic(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("singlethread") monotonic, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel singlethread_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_acquire(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("singlethread") acquire, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel singlethread_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_seq_cst(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("singlethread") seq_cst, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel agent_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_unordered(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("agent") unordered, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GCN:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel agent_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_monotonic(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("agent") monotonic, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX89:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
+; GFX89-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GFX89-NEXT: buffer_wbinvl1_vol
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel agent_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_acquire(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("agent") acquire, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_seq_cst:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NEXT:   flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX89:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
+; GFX89-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
+; GFX89-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
 ; GFX89-NEXT: buffer_wbinvl1_vol
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10-NEXT: buffer_gl0_inv
+; GFX10-NEXT: buffer_gl1_inv
+; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel agent_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_seq_cst(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("agent") seq_cst, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}workgroup_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel workgroup_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_unordered(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("workgroup") unordered, align 4
   store i32 %val, i32* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX89:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GCN-LABEL:    {{^}}workgroup_monotonic:
+; GCN-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX89:        flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GFX10WGP:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10CU-NOT:  flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GCN-NOT:      s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX89-NOT:    buffer_wbinvl1_vol
+; GFX10-NOT:    buffer_gl{{[01]}}_inv
+; GCN:          flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel workgroup_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_monotonic(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("workgroup") monotonic, align 4
   store i32 %val, i32* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX89:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GFX89:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX89-NOT:  buffer_wbinvl1_vol
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GCN-LABEL:     {{^}}workgroup_acquire:
+; GFX10-NOT:     s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
+; GFX89:         flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GFX10WGP:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10CU-NOT:   flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX89-NOT:     buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GCN:           flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel workgroup_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_acquire(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("workgroup") acquire, align 4
   store i32 %val, i32* %out
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_seq_cst:
-; GFX89-NOT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX89:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GFX89:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX89-NOT:  buffer_wbinvl1_vol
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GCN-LABEL:     {{^}}workgroup_seq_cst:
+; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0
+; GFX89:         flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GFX10WGP:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GFX10CU:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX89-NOT:     buffer_wbinvl1_vol
+; GFX10WGP-NEXT: buffer_gl0_inv
+; GFX10CU-NOT:   buffer_gl0_inv
+; GCN:           flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel workgroup_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_seq_cst(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("workgroup") seq_cst, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel wavefront_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_unordered(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("wavefront") unordered, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel wavefront_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_monotonic(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("wavefront") monotonic, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel wavefront_acquire
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_acquire(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("wavefront") acquire, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GFX89-NOT: buffer_wbinvl1_vol
+; GFX10-NOT: buffer_gl{{[01]}}_inv
 ; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+; GFX10:         .amdhsa_kernel wavefront_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_seq_cst(
     i32* %in, i32* %out) {
 entry:
   %val = load atomic i32, i32* %in syncscope("wavefront") seq_cst, align 4
   store i32 %val, i32* %out
   ret void
 }
 
 !0 = !{i32 1}

llvm/trunk/test/CodeGen/AMDGPU/memory-legalizer-store.ll

 ; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX8,GFX89 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX8,GFX89 %s
 ; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx900 -verify-machineinstrs -amdgpu-enable-global-sgpr-addr < %s | FileCheck --check-prefixes=GCN,GFX9,GFX89 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs -amdgpu-enable-global-sgpr-addr < %s | FileCheck --check-prefixes=GCN,GFX9,GFX89 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+code-object-v3 -verify-machineinstrs -amdgpu-enable-global-sgpr-addr < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+code-object-v3,+cumode -verify-machineinstrs -amdgpu-enable-global-sgpr-addr < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10CU %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 
 ; GCN-LABEL: {{^}}system_one_as_unordered:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel system_one_as_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_unordered(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("one-as") unordered, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_one_as_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel system_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_monotonic(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("one-as") monotonic, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_one_as_release:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel system_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_release(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("one-as") release, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_one_as_seq_cst:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel system_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_one_as_seq_cst(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("one-as") seq_cst, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_one_as_unordered:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel singlethread_one_as_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_unordered(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("singlethread-one-as") unordered, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_one_as_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel singlethread_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_monotonic(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("singlethread-one-as") monotonic, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_one_as_release:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel singlethread_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_release(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("singlethread-one-as") release, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_one_as_seq_cst:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel singlethread_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_one_as_seq_cst(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("singlethread-one-as") seq_cst, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_one_as_unordered:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel agent_one_as_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_unordered(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("agent-one-as") unordered, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_one_as_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel agent_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_monotonic(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("agent-one-as") monotonic, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_one_as_release:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel agent_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_release(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("agent-one-as") release, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_one_as_seq_cst:
 ; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel agent_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_one_as_seq_cst(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("agent-one-as") seq_cst, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}workgroup_one_as_unordered:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel workgroup_one_as_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_unordered(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("workgroup-one-as") unordered, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}workgroup_one_as_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel workgroup_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_monotonic(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("workgroup-one-as") monotonic, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_release:
-; GFX89-NOT:  s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GCN-LABEL:     {{^}}workgroup_one_as_release:
+; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel workgroup_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_release(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("workgroup-one-as") release, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_one_as_seq_cst:
-; GFX89-NOT:  s_waitcnt vmcnt(0){{$}}
-; GCN:        flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst:
+; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_one_as_seq_cst(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("workgroup-one-as") seq_cst, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_one_as_unordered:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel wavefront_one_as_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_unordered(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("wavefront-one-as") unordered, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_one_as_monotonic:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel wavefront_one_as_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_monotonic(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("wavefront-one-as") monotonic, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_one_as_release:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel wavefront_one_as_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_release(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("wavefront-one-as") release, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_one_as_seq_cst:
 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel wavefront_one_as_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_one_as_seq_cst(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("wavefront-one-as") seq_cst, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_private_0:
 ; GFX89: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen glc slc{{$}}
+; GFX10: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen slc{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_private_0
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_private_0(
     i32* %in, i32 addrspace(5)* %out) {
 entry:
   %val = load i32, i32* %in, align 4
   store i32 %val, i32 addrspace(5)* %out, !nontemporal !0
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_private_1:
 ; GFX89: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen glc slc{{$}}
+; GFX10: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen slc{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_private_1
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_private_1(
     i32* %in, i32 addrspace(5)* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val = load i32, i32* %in, align 4
   %out.gep = getelementptr inbounds i32, i32 addrspace(5)* %out, i32 %tid
   store i32 %val, i32 addrspace(5)* %out.gep, !nontemporal !0
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_global_0:
 ; GFX8:  flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc slc{{$}}
 ; GFX9:  global_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off glc slc{{$}}
+; GFX10: global_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off slc{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_global_0
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_global_0(
     i32* %in, i32 addrspace(1)* %out) {
 entry:
   %val = load i32, i32* %in, align 4
   store i32 %val, i32 addrspace(1)* %out, !nontemporal !0
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_global_1:
 ; GFX8:  flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc slc{{$}}
 ; GFX9:  global_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}] glc slc{{$}}
+; GFX10: global_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}] slc{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_global_1
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_global_1(
     i32* %in, i32 addrspace(1)* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val = load i32, i32* %in, align 4
   %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 %tid
   store i32 %val, i32 addrspace(1)* %out.gep, !nontemporal !0
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_local_0:
 ; GCN: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_local_0
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_local_0(
     i32* %in, i32 addrspace(3)* %out) {
 entry:
   %val = load i32, i32* %in, align 4
   store i32 %val, i32 addrspace(3)* %out, !nontemporal !0
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_local_1:
 ; GCN: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_local_1
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_local_1(
     i32* %in, i32 addrspace(3)* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val = load i32, i32* %in, align 4
   %out.gep = getelementptr inbounds i32, i32 addrspace(3)* %out, i32 %tid
   store i32 %val, i32 addrspace(3)* %out.gep, !nontemporal !0
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_flat_0:
 ; GFX89: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc slc{{$}}
+; GFX10: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} slc{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_flat_0
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_flat_0(
     i32* %in, i32* %out) {
 entry:
   %val = load i32, i32* %in, align 4
   store i32 %val, i32* %out, !nontemporal !0
   ret void
 }
 
 ; GCN-LABEL: {{^}}nontemporal_flat_1:
 ; GFX89: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc slc{{$}}
+; GFX10: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} slc{{$}}
+; GFX10:         .amdhsa_kernel nontemporal_flat_1
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @nontemporal_flat_1(
     i32* %in, i32* %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val = load i32, i32* %in, align 4
   %out.gep = getelementptr inbounds i32, i32* %out, i32 %tid
   store i32 %val, i32* %out.gep, !nontemporal !0
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel system_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_unordered(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out unordered, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel system_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_monotonic(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out monotonic, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_release:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX89:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel system_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_release(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out release, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}system_seq_cst:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX89:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel system_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @system_seq_cst(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out seq_cst, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel singlethread_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_unordered(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("singlethread") unordered, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel singlethread_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_monotonic(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("singlethread") monotonic, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_release:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel singlethread_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_release(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("singlethread") release, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}singlethread_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel singlethread_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @singlethread_seq_cst(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("singlethread") seq_cst, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel agent_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_unordered(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("agent") unordered, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel agent_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_monotonic(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("agent") monotonic, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_release:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX89:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel agent_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_release(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("agent") release, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}agent_seq_cst:
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX89:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt lgkmcnt(0){{$}}
+; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
 ; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel agent_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @agent_seq_cst(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("agent") seq_cst, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}workgroup_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel workgroup_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_unordered(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("workgroup") unordered, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}workgroup_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel workgroup_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_monotonic(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("workgroup") monotonic, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_release:
-; GFX89-NOT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GCN-LABEL:     {{^}}workgroup_release:
+; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel workgroup_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_release(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("workgroup") release, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}workgroup_seq_cst:
-; GFX89-NOT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GCN-LABEL:     {{^}}workgroup_seq_cst:
+; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
+; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
+; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
+; GCN:           flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel workgroup_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @workgroup_seq_cst(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("workgroup") seq_cst, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel wavefront_unordered
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_unordered(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("wavefront") unordered, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel wavefront_monotonic
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_monotonic(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("wavefront") monotonic, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_release:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel wavefront_release
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_release(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("wavefront") release, align 4
   ret void
 }
 
 ; GCN-LABEL: {{^}}wavefront_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
 ; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX10:         .amdhsa_kernel wavefront_seq_cst
+; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
+; GFX10CU:       .amdhsa_workgroup_processor_mode 0
+; GFX10-NOT:     .amdhsa_memory_ordered 0
 define amdgpu_kernel void @wavefront_seq_cst(
     i32 %in, i32* %out) {
 entry:
   store atomic i32 %in, i32* %out syncscope("wavefront") seq_cst, align 4
   ret void
 }
 
 !0 = !{i32 1}