[AMDGPU] Add patterns for i8/i16 local atomic load/store

[piotr]

Summary

Add patterns for i8/i16 local atomic load/store.

Added tests for new patterns.

Copied atomic_[store/load]_local.ll to GlobalISel directory.

[foad]

llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_local.ll

141

Depending on what kind of crash it is, you might be able to add -global-isel-abort=0 to the RUN lines to work around this? There are already some examples in this directory.

[piotr]

Added -global-isel-abort=0 and restored the tests.

llvm/lib/Target/AMDGPU/AMDGPUInstructions.td

def zextloadi8_#as  : PatFrag<(ops node:$ptr), (zextload node:$ptr)> {
   let MemoryVT = i8;
 }
 
 def zextloadi16_#as : PatFrag<(ops node:$ptr), (zextload node:$ptr)> {
   let IsLoad = 1;
   let MemoryVT = i16;
 }
 
+def atomic_load_8_#as : PatFrag<(ops node:$ptr), (atomic_load_8 node:$ptr)> {
+  let IsAtomic = 1;
+  let MemoryVT = i8;
+}
+
+def atomic_load_16_#as : PatFrag<(ops node:$ptr), (atomic_load_16 node:$ptr)> {
+  let IsAtomic = 1;
+  let MemoryVT = i16;
+}
+
 def atomic_load_32_#as : PatFrag<(ops node:$ptr), (atomic_load_32 node:$ptr)> {
   let IsAtomic = 1;
   let MemoryVT = i32;
 }
 
 def atomic_load_64_#as : PatFrag<(ops node:$ptr), (atomic_load_64 node:$ptr)> {
   let IsAtomic = 1;
   let MemoryVT = i64;

llvm/lib/Target/AMDGPU/DSInstructions.td

 defm : DSReadPat_mc <DS_READ_U16, i32, "extloadi16_local">;
 defm : DSReadPat_mc <DS_READ_U16, i32, "zextloadi16_local">;
 defm : DSReadPat_mc <DS_READ_U16, i16, "load_local">;
 
 foreach vt = Reg32Types.types in {
 defm : DSReadPat_mc <DS_READ_B32, vt, "load_local">;
 }
 
+defm : DSReadPat_mc <DS_READ_U8, i16, "atomic_load_8_local">;
+defm : DSReadPat_mc <DS_READ_U8, i32, "atomic_load_8_local">;
+defm : DSReadPat_mc <DS_READ_U16, i16, "atomic_load_16_local">;
+defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_16_local">;
 defm : DSReadPat_mc <DS_READ_B32, i32, "atomic_load_32_local">;
 defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_64_local">;
 
 let OtherPredicates = [D16PreservesUnusedBits] in {
 def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2i16>;
 def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2f16>;
 def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2i16>;
 def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2f16>;

 defm : DSWritePat_mc <DS_WRITE_B16, i32, "truncstorei16_local">;
 defm : DSWritePat_mc <DS_WRITE_B8, i16, "truncstorei8_local">;
 defm : DSWritePat_mc <DS_WRITE_B16, i16, "store_local">;
 
 foreach vt = Reg32Types.types in {
 defm : DSWritePat_mc <DS_WRITE_B32, vt, "store_local">;
 }
 
+defm : DSAtomicWritePat_mc <DS_WRITE_B8, i16, "atomic_store_local_8">;
+defm : DSAtomicWritePat_mc <DS_WRITE_B8, i32, "atomic_store_local_8">;
+defm : DSAtomicWritePat_mc <DS_WRITE_B16, i16, "atomic_store_local_16">;
+defm : DSAtomicWritePat_mc <DS_WRITE_B16, i32, "atomic_store_local_16">;
 defm : DSAtomicWritePat_mc <DS_WRITE_B32, i32, "atomic_store_local_32">;
 defm : DSAtomicWritePat_mc <DS_WRITE_B64, i64, "atomic_store_local_64">;
 
 let OtherPredicates = [D16PreservesUnusedBits] in {
 def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>;
 def : DSWritePat <DS_WRITE_B8_D16_HI, i32, truncstorei8_hi16_local>;
 }
 

llvm/lib/Target/AMDGPU/SIInstrInfo.td

def unindexedload_glue : PatFrag <(ops node:$ptr), (AMDGPUld_glue node:$ptr)> {
   let IsUnindexed = 1;
 }
 
 def load_glue : PatFrag <(ops node:$ptr), (unindexedload_glue node:$ptr)> {
   let IsLoad = 1;
   let IsNonExtLoad = 1;
 }
 
+def atomic_load_8_glue : PatFrag<(ops node:$ptr),
+  (AMDGPUatomic_ld_glue node:$ptr)> {
+  let IsAtomic = 1;
+  let MemoryVT = i8;
+}
+
+def atomic_load_16_glue : PatFrag<(ops node:$ptr),
+  (AMDGPUatomic_ld_glue node:$ptr)> {
+  let IsAtomic = 1;
+  let MemoryVT = i16;
+}
+
 def atomic_load_32_glue : PatFrag<(ops node:$ptr),
   (AMDGPUatomic_ld_glue node:$ptr)> {
   let IsAtomic = 1;
   let MemoryVT = i32;
 }
 
 def atomic_load_64_glue : PatFrag<(ops node:$ptr),
   (AMDGPUatomic_ld_glue node:$ptr)> {
def load_align16_local_m0 : PatFrag<(ops node:$ptr),
                                    (load_local_m0 node:$ptr)>, Aligned<16> {
   let IsLoad = 1;
   let IsNonExtLoad = 1;
 }
 
 } // End IsLoad = 1
 
 let IsAtomic = 1, AddressSpaces = LoadAddress_local.AddrSpaces in {
+def atomic_load_8_local_m0 : PatFrag<(ops node:$ptr),
+                                      (atomic_load_8_glue node:$ptr)> {
+  let MemoryVT = i8;
+}
+def atomic_load_16_local_m0 : PatFrag<(ops node:$ptr),
+                                      (atomic_load_16_glue node:$ptr)> {
+  let MemoryVT = i16;
+}
 def atomic_load_32_local_m0 : PatFrag<(ops node:$ptr),
                                       (atomic_load_32_glue node:$ptr)> {
   let MemoryVT = i32;
 }
 def atomic_load_64_local_m0 : PatFrag<(ops node:$ptr),
                                        (atomic_load_64_glue node:$ptr)> {
   let MemoryVT = i64;
 }
def store_align16_local_m0 : PatFrag <(ops node:$value, node:$ptr),
                                      (store_local_m0 node:$value, node:$ptr)>,
                             Aligned<16> {
   let IsStore = 1;
   let IsTruncStore = 0;
 }
 
 let AddressSpaces = StoreAddress_local.AddrSpaces in {
 
+def atomic_store_local_8_m0 : PatFrag <
+  (ops node:$value, node:$ptr),
+  (AMDGPUatomic_st_glue node:$value, node:$ptr)> {
+  let IsAtomic = 1;
+  let MemoryVT = i8;
+}
+def atomic_store_local_16_m0 : PatFrag <
+  (ops node:$value, node:$ptr),
+  (AMDGPUatomic_st_glue node:$value, node:$ptr)> {
+  let IsAtomic = 1;
+  let MemoryVT = i16;
+}
 def atomic_store_local_32_m0 : PatFrag <
   (ops node:$value, node:$ptr),
   (AMDGPUatomic_st_glue node:$value, node:$ptr)> {
   let IsAtomic = 1;
   let MemoryVT = i32;
 }
 def atomic_store_local_64_m0 : PatFrag <
   (ops node:$value, node:$ptr),

llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_local.ll

-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CI %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -global-isel-abort=0 -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CI %s
+; RUN: llc -global-isel -global-isel-abort=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+
+; GCN-LABEL: {{^}}atomic_load_monotonic_i8:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_read_u8 v0, v0{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define i8 @atomic_load_monotonic_i8(i8 addrspace(3)* %ptr) {
+  %load = load atomic i8, i8 addrspace(3)* %ptr monotonic, align 1
+  ret i8 %load
+}
+
+; GCN-LABEL: {{^}}atomic_load_monotonic_i8_offset:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_read_u8 v0, v0 offset:16{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define i8 @atomic_load_monotonic_i8_offset(i8 addrspace(3)* %ptr) {
+  %gep = getelementptr inbounds i8, i8 addrspace(3)* %ptr, i8 16
+  %load = load atomic i8, i8 addrspace(3)* %gep monotonic, align 1
+  ret i8 %load
+}
+
+; GCN-LABEL: {{^}}atomic_load_monotonic_i16:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_read_u16 v0, v0{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define i16 @atomic_load_monotonic_i16(i16 addrspace(3)* %ptr) {
+  %load = load atomic i16, i16 addrspace(3)* %ptr monotonic, align 2
+  ret i16 %load
+}
+
+; GCN-LABEL: {{^}}atomic_load_monotonic_i16_offset:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_read_u16 v0, v0 offset:32{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define i16 @atomic_load_monotonic_i16_offset(i16 addrspace(3)* %ptr) {
+  %gep = getelementptr inbounds i16, i16 addrspace(3)* %ptr, i16 16
+  %load = load atomic i16, i16 addrspace(3)* %gep monotonic, align 2
+  ret i16 %load
+}
 
 ; GCN-LABEL: {{^}}atomic_load_monotonic_i32:
 ; GCN: s_waitcnt
 ; GFX9-NOT: s_mov_b32 m0
 ; CI-NEXT: s_mov_b32 m0
 ; GCN-NEXT: ds_read_b32 v0, v0{{$}}
 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
 ; GCN-NEXT: s_setpc_b64

 ; CI-NEXT: s_mov_b32 m0
 ; GCN-NEXT: ds_read_b64 v[0:1], v0 offset:128{{$}}
 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
 ; GCN-NEXT: s_setpc_b64
 define i8* @atomic_load_monotonic_p0i8_offset(i8* addrspace(3)* %ptr) {
   %gep = getelementptr inbounds i8*, i8* addrspace(3)* %ptr, i32 16
   %load = load atomic i8*, i8* addrspace(3)* %gep monotonic, align 8
   ret i8* %load
 }
 
 ; GCN-LABEL: {{^}}atomic_load_monotonic_p3i8_offset:
 ; GCN: s_waitcnt
 ; GFX9-NOT: s_mov_b32 m0
 ; CI-NEXT: s_mov_b32 m0
 ; GCN-NEXT: ds_read_b32 v0, v0 offset:64{{$}}
 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
 ; GCN-NEXT: s_setpc_b64
 define i8 addrspace(3)* @atomic_load_monotonic_p3i8_offset(i8 addrspace(3)* addrspace(3)* %ptr) {
   %gep = getelementptr inbounds i8 addrspace(3)*, i8 addrspace(3)* addrspace(3)* %ptr, i32 16
   %load = load atomic i8 addrspace(3)*, i8 addrspace(3)* addrspace(3)* %gep monotonic, align 4
   ret i8 addrspace(3)* %load
 }

llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_store_local.ll

+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CI %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+
+; GCN-LABEL: {{^}}atomic_store_monotonic_i8:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_write_b8 v0, v1{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define void @atomic_store_monotonic_i8(i8 addrspace(3)* %ptr, i8 %val) {
+  store atomic i8 %val, i8 addrspace(3)* %ptr monotonic, align 1
+  ret void
+}
+
+; GCN-LABEL: {{^}}atomic_store_monotonic_offset_i8:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_write_b8 v0, v1 offset:16{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define void @atomic_store_monotonic_offset_i8(i8 addrspace(3)* %ptr, i8 %val) {
+  %gep = getelementptr inbounds i8, i8 addrspace(3)* %ptr, i8 16
+  store atomic i8 %val, i8 addrspace(3)* %gep monotonic, align 1
+  ret void
+}
+
+; GCN-LABEL: {{^}}atomic_store_monotonic_i16:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_write_b16 v0, v1{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define void @atomic_store_monotonic_i16(i16 addrspace(3)* %ptr, i16 %val) {
+  store atomic i16 %val, i16 addrspace(3)* %ptr monotonic, align 2
+  ret void
+}
+
+; GCN-LABEL: {{^}}atomic_store_monotonic_offset_i16:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_write_b16 v0, v1 offset:32{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define void @atomic_store_monotonic_offset_i16(i16 addrspace(3)* %ptr, i16 %val) {
+  %gep = getelementptr inbounds i16, i16 addrspace(3)* %ptr, i16 16
+  store atomic i16 %val, i16 addrspace(3)* %gep monotonic, align 2
+  ret void
+}
+
+; GCN-LABEL: {{^}}atomic_store_monotonic_i32:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_write_b32 v0, v1{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define void @atomic_store_monotonic_i32(i32 addrspace(3)* %ptr, i32 %val) {
+  store atomic i32 %val, i32 addrspace(3)* %ptr monotonic, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}atomic_store_monotonic_offset_i32:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_write_b32 v0, v1 offset:64{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define void @atomic_store_monotonic_offset_i32(i32 addrspace(3)* %ptr, i32 %val) {
+  %gep = getelementptr inbounds i32, i32 addrspace(3)* %ptr, i32 16
+  store atomic i32 %val, i32 addrspace(3)* %gep monotonic, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}atomic_store_monotonic_i64:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_write_b64 v0, v[1:2]{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define void @atomic_store_monotonic_i64(i64 addrspace(3)* %ptr, i64 %val) {
+  store atomic i64 %val, i64 addrspace(3)* %ptr monotonic, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}atomic_store_monotonic_offset_i64:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_write_b64 v0, v[1:2] offset:128{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define void @atomic_store_monotonic_offset_i64(i64 addrspace(3)* %ptr, i64 %val) {
+  %gep = getelementptr inbounds i64, i64 addrspace(3)* %ptr, i64 16
+  store atomic i64 %val, i64 addrspace(3)* %gep monotonic, align 8
+  ret void
+}
+

llvm/test/CodeGen/AMDGPU/atomic_load_local.ll

 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CI %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
 
+; GCN-LABEL: {{^}}atomic_load_monotonic_i8:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_read_u8 v0, v0{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define i8 @atomic_load_monotonic_i8(i8 addrspace(3)* %ptr) {
+  %load = load atomic i8, i8 addrspace(3)* %ptr monotonic, align 1
+  ret i8 %load
+}
+
+; GCN-LABEL: {{^}}atomic_load_monotonic_i8_offset:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_read_u8 v0, v0 offset:16{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define i8 @atomic_load_monotonic_i8_offset(i8 addrspace(3)* %ptr) {
+  %gep = getelementptr inbounds i8, i8 addrspace(3)* %ptr, i8 16
+  %load = load atomic i8, i8 addrspace(3)* %gep monotonic, align 1
+  ret i8 %load
+}
+
+; GCN-LABEL: {{^}}atomic_load_monotonic_i16:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_read_u16 v0, v0{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define i16 @atomic_load_monotonic_i16(i16 addrspace(3)* %ptr) {
+  %load = load atomic i16, i16 addrspace(3)* %ptr monotonic, align 2
+  ret i16 %load
+}
+
+; GCN-LABEL: {{^}}atomic_load_monotonic_i16_offset:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_read_u16 v0, v0 offset:32{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define i16 @atomic_load_monotonic_i16_offset(i16 addrspace(3)* %ptr) {
+  %gep = getelementptr inbounds i16, i16 addrspace(3)* %ptr, i16 16
+  %load = load atomic i16, i16 addrspace(3)* %gep monotonic, align 2
+  ret i16 %load
+}
+
 ; GCN-LABEL: {{^}}atomic_load_monotonic_i32:
 ; GCN: s_waitcnt
 ; GFX9-NOT: s_mov_b32 m0
 ; CI-NEXT: s_mov_b32 m0
 ; GCN-NEXT: ds_read_b32 v0, v0{{$}}
 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
 ; GCN-NEXT: s_setpc_b64
 define i32 @atomic_load_monotonic_i32(i32 addrspace(3)* %ptr) {

llvm/test/CodeGen/AMDGPU/atomic_store_local.ll

 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CI %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
 
+; GCN-LABEL: {{^}}atomic_store_monotonic_i8:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_write_b8 v0, v1{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define void @atomic_store_monotonic_i8(i8 addrspace(3)* %ptr, i8 %val) {
+  store atomic i8 %val, i8 addrspace(3)* %ptr monotonic, align 1
+  ret void
+}
+
+; GCN-LABEL: {{^}}atomic_store_monotonic_offset_i8:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_write_b8 v0, v1 offset:16{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define void @atomic_store_monotonic_offset_i8(i8 addrspace(3)* %ptr, i8 %val) {
+  %gep = getelementptr inbounds i8, i8 addrspace(3)* %ptr, i8 16
+  store atomic i8 %val, i8 addrspace(3)* %gep monotonic, align 1
+  ret void
+}
+
+; GCN-LABEL: {{^}}atomic_store_monotonic_i16:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_write_b16 v0, v1{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define void @atomic_store_monotonic_i16(i16 addrspace(3)* %ptr, i16 %val) {
+  store atomic i16 %val, i16 addrspace(3)* %ptr monotonic, align 2
+  ret void
+}
+
+; GCN-LABEL: {{^}}atomic_store_monotonic_offset_i16:
+; GCN: s_waitcnt
+; GFX9-NOT: s_mov_b32 m0
+; CI-NEXT: s_mov_b32 m0
+; GCN-NEXT: ds_write_b16 v0, v1 offset:32{{$}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define void @atomic_store_monotonic_offset_i16(i16 addrspace(3)* %ptr, i16 %val) {
+  %gep = getelementptr inbounds i16, i16 addrspace(3)* %ptr, i16 16
+  store atomic i16 %val, i16 addrspace(3)* %gep monotonic, align 2
+  ret void
+}
+
 ; GCN-LABEL: {{^}}atomic_store_monotonic_i32:
 ; GCN: s_waitcnt
 ; GFX9-NOT: s_mov_b32 m0
 ; CI-NEXT: s_mov_b32 m0
 ; GCN-NEXT: ds_write_b32 v0, v1{{$}}
 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
 ; GCN-NEXT: s_setpc_b64
 define void @atomic_store_monotonic_i32(i32 addrspace(3)* %ptr, i32 %val) {