[AMDGPU] Only use ds_read/write_b128 for alignment >= 16

[hsmhsm]

Summary

PS: Submitting on behalf of Jay.

llvm/lib/Target/AMDGPU/DSInstructions.td

 
 foreach vt = Reg32Types.types in {
 defm : DSReadPat_mc <DS_READ_B32, vt, "load_local">;
 }
 
 defm : DSReadPat_mc <DS_READ_B32, i32, "atomic_load_32_local">;
 defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_64_local">;
 
+// Prefer ds_read over ds_read2, all other things being equal, because it has
+// a larger immediate offset range.
 let AddedComplexity = 100 in {
 
 foreach vt = VReg_64.RegTypes in {
 defm : DSReadPat_mc <DS_READ_B64, vt, "load_align8_local">;
 }
 
 let SubtargetPredicate = isGFX7Plus in {
 
 foreach vt = VReg_96.RegTypes in {
 defm : DSReadPat_mc <DS_READ_B96, vt, "load_align16_local">;
 }
 
+// For performance reasons restrict this to alignment >= 16 even with
+// unaligned-access-mode. At lower alignments ds_read2_b64 is always a better
+// choice.
 foreach vt = VReg_128.RegTypes in {
 defm : DSReadPat_mc <DS_READ_B128, vt, "load_align16_local">;
 }
 
 let SubtargetPredicate = HasUnalignedAccessMode in {
 
 foreach vt = VReg_96.RegTypes in {
 defm : DSReadPat_mc <DS_READ_B96, vt, "load_local">;
 }
 
-foreach vt = VReg_128.RegTypes in {
-defm : DSReadPat_mc <DS_READ_B128, vt, "load_local">;
-}
-
 } // End SubtargetPredicate = HasUnalignedAccessMode
 
 } // End SubtargetPredicate = isGFX7Plus
 
 } // End AddedComplexity = 100
 
 let OtherPredicates = [D16PreservesUnusedBits] in {
 def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2i16>;

 foreach vt = VReg_64.RegTypes in {
 defm : DS64Bit4ByteAlignedPat_mc<vt>;
 }
 
 foreach vt = VReg_128.RegTypes in {
 defm : DS128Bit8ByteAlignedPat_mc<vt>;
 }
 
+// Prefer ds_write over ds_write2, all other things being equal, because it has
+// a larger immediate offset range.
 let AddedComplexity = 100 in {
 
 foreach vt = VReg_64.RegTypes in {
 defm : DSWritePat_mc <DS_WRITE_B64, vt, "store_align8_local">;
 }
 
 let SubtargetPredicate = isGFX7Plus in {
 
 foreach vt = VReg_96.RegTypes in {
 defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_align16_local">;
 }
 
+// For performance reasons restrict this to alignment >= 16 even with
+// unaligned-access-mode. At lower alignments ds_write2_b64 is always a better
+// choice.
 foreach vt = VReg_128.RegTypes in {
 defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align16_local">;
 }
 
 let SubtargetPredicate = HasUnalignedAccessMode in {
 
 foreach vt = VReg_96.RegTypes in {
 defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_local">;
 }
 
-foreach vt = VReg_128.RegTypes in {
-defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_local">;
-}
-
 } // End SubtargetPredicate = HasUnalignedAccessMode
 
 } // End SubtargetPredicate = isGFX7Plus
 
 } // End AddedComplexity = 100
 
 class DSAtomicRetPat<DS_Pseudo inst, ValueType vt, PatFrag frag, bit gds=0> : GCNPat <
   (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value),

llvm/test/CodeGen/AMDGPU/GlobalISel/lds-misaligned-bug.ll

bb:
   ret void
 }
 
 ; GCN-LABEL: test_local_misaligned_v4:
 ; ALIGNED-DAG: ds_read2_b32
 ; ALIGNED-DAG: ds_read2_b32
 ; ALIGNED-DAG: ds_write2_b32
 ; ALIGNED-DAG: ds_write2_b32
-; UNALIGNED-DAG: ds_read_b128
-; UNALIGNED-DAG: ds_write_b128
+; UNALIGNED-DAG: ds_read2_b64
+; UNALIGNED-DAG: ds_write2_b64
 define amdgpu_kernel void @test_local_misaligned_v4(i32 addrspace(3)* %arg) {
 bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
   %ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)*
   %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 4
   %v1 = extractelement <4 x i32> %load, i32 0
   %v2 = extractelement <4 x i32> %load, i32 1
bb:
   %v7 = insertelement <3 x i32> %v6, i32 %v2, i32 2
   store <3 x i32> %v7, <3 x i32> addrspace(3)* %ptr, align 16
   ret void
 }
 
 ; GCN-LABEL: test_local_v4_aligned8:
 ; ALIGNED-DAG: ds_read2_b64
 ; ALIGNED-DAG: ds_write2_b64
-; UNALIGNED-DAG: ds_read_b128
-; UNALIGNED-DAG: ds_write_b128
+; UNALIGNED-DAG: ds_read2_b64
+; UNALIGNED-DAG: ds_write2_b64
 define amdgpu_kernel void @test_local_v4_aligned8(i32 addrspace(3)* %arg) {
 bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
   %ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)*
   %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 8
   %v1 = extractelement <4 x i32> %load, i32 0
   %v2 = extractelement <4 x i32> %load, i32 1

llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -mattr=+unaligned-access-mode -verify-machineinstrs < %s | FileCheck --check-prefix=GFX9 %s
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=hawaii -mattr=+unaligned-access-mode -verify-machineinstrs < %s | FileCheck --check-prefix=GFX7 %s
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -mattr=+unaligned-access-mode -verify-machineinstrs < %s | FileCheck --check-prefix=GFX10 %s
 
 ; Unaligned DS access in available from GFX9 onwards.
 ; LDS alignment enforcement is controlled by a configuration register:
 ; SH_MEM_CONFIG.alignment_mode
 
 define <4 x i32> @load_lds_v4i32_align1(<4 x i32> addrspace(3)* %ptr) {
 ; GFX9-LABEL: load_lds_v4i32_align1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    ds_read_b128 v[0:3], v0
+; GFX9-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: load_lds_v4i32_align1:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    s_movk_i32 s4, 0xff
; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %load = load <3 x i32>, <3 x i32> addrspace(3)* %ptr, align 1
   ret <3 x i32> %load
 }
 
 define void @store_lds_v4i32_align1(<4 x i32> addrspace(3)* %out, <4 x i32> %x) {
 ; GFX9-LABEL: store_lds_v4i32_align1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    ds_write_b128 v0, v[1:4]
+; GFX9-NEXT:    ds_write2_b64 v0, v[1:2], v[3:4] offset1:1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: store_lds_v4i32_align1:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 8, v1

llvm/test/CodeGen/AMDGPU/ds-alignment.ll

 ; ALIGNED-GISEL-NEXT:    ds_write_b8 v16, v0 offset:15
 ; ALIGNED-GISEL-NEXT:    s_endpgm
 ;
 ; UNALIGNED-LABEL: ds16align1:
 ; UNALIGNED:       ; %bb.0:
 ; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
-; UNALIGNED-NEXT:    ds_read_b128 v[0:3], v0
+; UNALIGNED-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v4, s1
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
-; UNALIGNED-NEXT:    ds_write_b128 v4, v[0:3]
+; UNALIGNED-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
 ; UNALIGNED-NEXT:    s_endpgm
   %val = load <4 x i32>, <4 x i32> addrspace(3)* %in, align 1
   store <4 x i32> %val, <4 x i32> addrspace(3)* %out, align 1
   ret void
 }
 
 define amdgpu_kernel void @ds16align2(<4 x i32> addrspace(3)* %in, <4 x i32> addrspace(3)* %out) {
 ; ALIGNED-SDAG-LABEL: ds16align2:

 ; ALIGNED-GISEL-NEXT:    ds_write_b16 v1, v0 offset:14
 ; ALIGNED-GISEL-NEXT:    s_endpgm
 ;
 ; UNALIGNED-LABEL: ds16align2:
 ; UNALIGNED:       ; %bb.0:
 ; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
-; UNALIGNED-NEXT:    ds_read_b128 v[0:3], v0
+; UNALIGNED-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v4, s1
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
-; UNALIGNED-NEXT:    ds_write_b128 v4, v[0:3]
+; UNALIGNED-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
 ; UNALIGNED-NEXT:    s_endpgm
   %val = load <4 x i32>, <4 x i32> addrspace(3)* %in, align 2
   store <4 x i32> %val, <4 x i32> addrspace(3)* %out, align 2
   ret void
 }
 
 define amdgpu_kernel void @ds16align4(<4 x i32> addrspace(3)* %in, <4 x i32> addrspace(3)* %out) {
 ; ALIGNED-SDAG-LABEL: ds16align4:

 ; ALIGNED-GISEL-NEXT:    ds_write2_b32 v4, v2, v3 offset0:2 offset1:3
 ; ALIGNED-GISEL-NEXT:    s_endpgm
 ;
 ; UNALIGNED-LABEL: ds16align4:
 ; UNALIGNED:       ; %bb.0:
 ; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
-; UNALIGNED-NEXT:    ds_read_b128 v[0:3], v0
+; UNALIGNED-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
 ; UNALIGNED-NEXT:    v_mov_b32_e32 v4, s1
 ; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
-; UNALIGNED-NEXT:    ds_write_b128 v4, v[0:3]
+; UNALIGNED-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
 ; UNALIGNED-NEXT:    s_endpgm
   %val = load <4 x i32>, <4 x i32> addrspace(3)* %in, align 4
   store <4 x i32> %val, <4 x i32> addrspace(3)* %out, align 4
   ret void
 }
 
 define amdgpu_kernel void @ds16align8(<4 x i32> addrspace(3)* %in, <4 x i32> addrspace(3)* %out) {
-; ALIGNED-LABEL: ds16align8:
-; ALIGNED:       ; %bb.0:
-; ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
-; ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
-; ALIGNED-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
-; ALIGNED-NEXT:    v_mov_b32_e32 v4, s1
-; ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
-; ALIGNED-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
-; ALIGNED-NEXT:    s_endpgm
-;
-; UNALIGNED-LABEL: ds16align8:
-; UNALIGNED:       ; %bb.0:
-; UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
-; UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
-; UNALIGNED-NEXT:    ds_read_b128 v[0:3], v0
-; UNALIGNED-NEXT:    v_mov_b32_e32 v4, s1
-; UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
-; UNALIGNED-NEXT:    ds_write_b128 v4, v[0:3]
-; UNALIGNED-NEXT:    s_endpgm
+; GCN-LABEL: ds16align8:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
+; GCN-NEXT:    v_mov_b32_e32 v4, s1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
+; GCN-NEXT:    s_endpgm
   %val = load <4 x i32>, <4 x i32> addrspace(3)* %in, align 8
   store <4 x i32> %val, <4 x i32> addrspace(3)* %out, align 8
   ret void
 }
 
 define amdgpu_kernel void @ds16align16(<4 x i32> addrspace(3)* %in, <4 x i32> addrspace(3)* %out) {
 ; GCN-LABEL: ds16align16:
 ; GCN:       ; %bb.0:

llvm/test/CodeGen/AMDGPU/ds_read2.ll

 ; GFX9-ALIGNED-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v2
 ; GFX9-ALIGNED-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
 ; GFX9-ALIGNED-NEXT:    global_store_dwordx2 v4, v[0:1], s[0:1]
 ; GFX9-ALIGNED-NEXT:    s_endpgm
 ;
 ; GFX9-UNALIGNED-LABEL: load_misaligned64_constant_offsets:
 ; GFX9-UNALIGNED:       ; %bb.0:
 ; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-UNALIGNED-NEXT:    ds_read_b128 v[0:3], v4
+; GFX9-UNALIGNED-NEXT:    ds_read2_b64 v[0:3], v4 offset1:1
 ; GFX9-UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-UNALIGNED-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v2
 ; GFX9-UNALIGNED-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
 ; GFX9-UNALIGNED-NEXT:    global_store_dwordx2 v4, v[0:1], s[0:1]
 ; GFX9-UNALIGNED-NEXT:    s_endpgm
   %val0 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 0), align 4
   %val1 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 1), align 4

llvm/test/CodeGen/AMDGPU/ds_write2.ll

 ; GFX9-ALIGNED-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-ALIGNED-NEXT:    v_mov_b32_e32 v1, 0x7b
 ; GFX9-ALIGNED-NEXT:    ds_write2_b32 v0, v1, v0 offset1:1
 ; GFX9-ALIGNED-NEXT:    ds_write2_b32 v0, v1, v0 offset0:2 offset1:3
 ; GFX9-ALIGNED-NEXT:    s_endpgm
 ;
 ; GFX9-UNALIGNED-LABEL: store_misaligned64_constant_offsets:
 ; GFX9-UNALIGNED:       ; %bb.0:
-; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v0, 0x7b
-; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v2, v0
-; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v3, v1
-; GFX9-UNALIGNED-NEXT:    ds_write_b128 v1, v[0:3]
+; GFX9-UNALIGNED-NEXT:    s_movk_i32 s0, 0x7b
+; GFX9-UNALIGNED-NEXT:    s_mov_b32 s1, 0
+; GFX9-UNALIGNED-NEXT:    s_mov_b32 s2, s0
+; GFX9-UNALIGNED-NEXT:    s_mov_b32 s3, s1
+; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-UNALIGNED-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
 ; GFX9-UNALIGNED-NEXT:    s_endpgm
   store i64 123, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 0), align 4
   store i64 123, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 1), align 4
   ret void
 }
 
 @bar.large = addrspace(3) global [4096 x i64] undef, align 4
 

 ; GFX9-UNALIGNED:       ; %bb.0:
 ; GFX9-UNALIGNED-NEXT:    s_load_dword s4, s[0:1], 0x24
 ; GFX9-UNALIGNED-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
 ; GFX9-UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-UNALIGNED-NEXT:    v_lshl_add_u32 v4, v0, 4, s4
 ; GFX9-UNALIGNED-NEXT:    s_load_dwordx4 s[0:3], s[2:3], 0x0
 ; GFX9-UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-UNALIGNED-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-UNALIGNED-NEXT:    ds_write_b128 v4, v[0:3]
+; GFX9-UNALIGNED-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
 ; GFX9-UNALIGNED-NEXT:    s_endpgm
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %in
   %val0 = load <4 x float>, <4 x float> addrspace(1)* %in.gep, align 4
   %out.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(3)* %out, i32 %x.i
   store <4 x float> %val0, <4 x float> addrspace(3)* %out.gep, align 4
   ret void
 }

llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll

bb:
   ret void
 }
 
 ; GCN-LABEL: test_local_misaligned_v4:
 ; ALIGNED-DAG: ds_read2_b32
 ; ALIGNED-DAG: ds_read2_b32
 ; ALIGNED-DAG: ds_write2_b32
 ; ALIGNED-DAG: ds_write2_b32
-; UNALIGNED-DAG: ds_read_b128
-; UNALIGNED-DAG: ds_write_b128
+; UNALIGNED-DAG: ds_read2_b64
+; UNALIGNED-DAG: ds_write2_b64
 define amdgpu_kernel void @test_local_misaligned_v4(i32 addrspace(3)* %arg) {
 bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
   %ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)*
   %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 4
   %v1 = extractelement <4 x i32> %load, i32 0
   %v2 = extractelement <4 x i32> %load, i32 1
bb:
   %v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3
   store <4 x i32> %v8, <4 x i32>* %ptr, align 16
   ret void
 }
 
 ; GCN-LABEL: test_local_v4_aligned8:
 ; ALIGNED-DAG: ds_read2_b64
 ; ALIGNED-DAG: ds_write2_b64
-; UNALIGNED-DAG: ds_read_b128
-; UNALIGNED-DAG: ds_write_b128
+; UNALIGNED-DAG: ds_read2_b64
+; UNALIGNED-DAG: ds_write2_b64
 define amdgpu_kernel void @test_local_v4_aligned8(i32 addrspace(3)* %arg) {
 bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
   %ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)*
   %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 8
   %v1 = extractelement <4 x i32> %load, i32 0
   %v2 = extractelement <4 x i32> %load, i32 1