[AMDGPU] Resolve issues when picking between ds_read/write and ds_read2/write2

[mbrkusanin]

Summary

Both ds_read_b128 and ds_read2_b64 are valid for 128bit 16-byte aligned
loads but the one that will be selected is determined either by the order in
tablegen or by the AddedComplexity attribute. Currently ds_read_b128 has
priority.

While ds_read2_b64 has lower alignment requirements, we cannot always
restrict ds_read_b128 to 16-byte alignment because of unaligned-access-mode
option. This was causing ds_read_b128 to be selected for 8-byte aligned
loads regardles of chosen access mode.

To resolve this we use two patterns for selecting ds_read_b128. One
requires alignment of 16-byte and the other requires
unaligned-access-mode option.

Same goes for ds_write2_b64 and ds_write_b128.

[foad]

llvm/lib/Target/AMDGPU/DSInstructions.td

683

Can't we relax this predicate to "isGFX7Plus" instead of duplicating the patterns?

[mbrkusanin]

llvm/lib/Target/AMDGPU/DSInstructions.td

683

Right. That way we do not increase number of patterns.

[foad]

Looks good, thanks!

llvm/lib/Target/AMDGPU/AMDGPU.td

def isGFX7Only :
   Predicate<"Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS">,
   AssemblerPredicate<(all_of (not FeatureGCN3Encoding), FeatureCIInsts, (not FeatureGFX10Insts))>;
 
 def isGFX7GFX10 :
   Predicate<"Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS ||"
             "Subtarget->getGeneration() == AMDGPUSubtarget::GFX10">,
   AssemblerPredicate<(all_of (not FeatureGCN3Encoding), FeatureCIInsts)>;
 
-def isGFX7GFX8 :
-  Predicate<"Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS ||"
-            "Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS">,
-  AssemblerPredicate<(all_of FeatureSouthernIslands, FeatureCIInsts)>;
-
 def isGFX7GFX8GFX9 :
   Predicate<"Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS ||"
             "Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS ||"
             "Subtarget->getGeneration() == AMDGPUSubtarget::GFX9">,
   AssemblerPredicate<(all_of FeatureGFX7GFX8GFX9Insts)>;
 
 def isGFX6GFX7GFX8GFX9 :
   Predicate<"Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS ||"

 
 def EnableLateCFGStructurize : Predicate<
   "EnableLateStructurizeCFG">;
 
 def EnableFlatScratch : Predicate<"Subtarget->enableFlatScratch()">;
 
 def DisableFlatScratch : Predicate<"!Subtarget->enableFlatScratch()">;
 
+def HasUnalignedAccessMode : Predicate<"Subtarget->hasUnalignedAccessMode()">,
+  AssemblerPredicate<(all_of FeatureUnalignedAccessMode)>;
+
 // Include AMDGPU TD files
 include "SISchedule.td"
 include "GCNProcessors.td"
 include "AMDGPUInstrInfo.td"
 include "SIRegisterInfo.td"
 include "AMDGPURegisterBanks.td"
 include "AMDGPUInstructions.td"
 include "SIInstrInfo.td"
 include "AMDGPUCallingConv.td"
 include "AMDGPUSearchableTables.td"

llvm/lib/Target/AMDGPU/DSInstructions.td

 defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_64_local">;
 
 let AddedComplexity = 100 in {
 
 foreach vt = VReg_64.RegTypes in {
 defm : DSReadPat_mc <DS_READ_B64, vt, "load_align8_local">;
 }
 
-let SubtargetPredicate = isGFX7GFX8 in {
+let SubtargetPredicate = isGFX7Plus in {
 
 foreach vt = VReg_96.RegTypes in {
 defm : DSReadPat_mc <DS_READ_B96, vt, "load_align16_local">;
 }
 
 foreach vt = VReg_128.RegTypes in {
 defm : DSReadPat_mc <DS_READ_B128, vt, "load_align16_local">;
 }
 
-}
-
-let SubtargetPredicate = isGFX9Plus in {
+let SubtargetPredicate = HasUnalignedAccessMode in {
 
 foreach vt = VReg_96.RegTypes in {
 defm : DSReadPat_mc <DS_READ_B96, vt, "load_local">;
 }
 
 foreach vt = VReg_128.RegTypes in {
 defm : DSReadPat_mc <DS_READ_B128, vt, "load_local">;
 }
 
-}
+} // End SubtargetPredicate = HasUnalignedAccessMode
+
+} // End SubtargetPredicate = isGFX7Plus
 
 } // End AddedComplexity = 100
 
 let OtherPredicates = [D16PreservesUnusedBits] in {
 def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2i16>;
 def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2f16>;
 def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2i16>;
 def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2f16>;

 }
 
 let AddedComplexity = 100 in {
 
 foreach vt = VReg_64.RegTypes in {
 defm : DSWritePat_mc <DS_WRITE_B64, vt, "store_align8_local">;
 }
 
-let SubtargetPredicate = isGFX7GFX8 in {
+let SubtargetPredicate = isGFX7Plus in {
 
 foreach vt = VReg_96.RegTypes in {
 defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_align16_local">;
 }
 
 foreach vt = VReg_128.RegTypes in {
 defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align16_local">;
 }
 
-}
-
-let SubtargetPredicate = isGFX9Plus in {
+let SubtargetPredicate = HasUnalignedAccessMode in {
 
 foreach vt = VReg_96.RegTypes in {
 defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_local">;
 }
 
 foreach vt = VReg_128.RegTypes in {
 defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_local">;
 }
 
-}
+} // End SubtargetPredicate = HasUnalignedAccessMode
+
+} // End SubtargetPredicate = isGFX7Plus
 
 } // End AddedComplexity = 100
+
 class DSAtomicRetPat<DS_Pseudo inst, ValueType vt, PatFrag frag, bit gds=0> : GCNPat <
   (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value),
   (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 gds))
 >;
 
 multiclass DSAtomicRetPat_mc<DS_Pseudo inst, ValueType vt, string frag> {
   let OtherPredicates = [LDSRequiresM0Init] in {
     def : DSAtomicRetPat<inst, vt, !cast<PatFrag>(frag#"_local_m0_"#vt.Size)>;

llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local-128.mir

bb.0:
     ; GFX7: liveins: $vgpr0
     ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX7: $m0 = S_MOV_B32 -1
     ; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 [[COPY]], 0, 1, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3)
     ; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]]
     ; GFX9-LABEL: name: load_local_v4s32_align_8
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 16, align 8, addrspace 3)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]]
+    ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(<4 x  s32>) = G_LOAD %0 :: (load 16, align 8, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
 
 ...
 
 ---
 
bb.0:
     ; GFX7: liveins: $vgpr0
     ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX7: $m0 = S_MOV_B32 -1
     ; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 [[COPY]], 50, 51, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3)
     ; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]]
     ; GFX9-LABEL: name: load_local_v4s32_align_8_offset_160
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 400, 0, implicit $exec :: (load 16, align 8, addrspace 3)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]]
+    ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 50, 51, 0, implicit $exec :: (load 16, align 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 400
     %2:vgpr(p3) = G_PTR_ADD %0, %1
     %3:vgpr(<4 x  s32>) = G_LOAD %2 :: (load 16, align 8, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
 
 ...
 
bb.0:
     ; GFX7: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4000, implicit $exec
     ; GFX7: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_CO_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
     ; GFX7: $m0 = S_MOV_B32 -1
     ; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 %2, 0, 1, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3)
     ; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]]
     ; GFX9-LABEL: name: load_local_v4s32_align_8_offset_320
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 4000, 0, implicit $exec :: (load 16, align 8, addrspace 3)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]]
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4000, implicit $exec
+    ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[V_ADD_U32_e64_]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 4000
     %2:vgpr(p3) = G_PTR_ADD %0, %1
     %3:vgpr(<4 x  s32>) = G_LOAD %2 :: (load 16, align 8, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
 
 ...
 
bb.0:
     ; GFX7: liveins: $vgpr0
     ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX7: $m0 = S_MOV_B32 -1
     ; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 [[COPY]], 0, 1, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3)
     ; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]]
     ; GFX9-LABEL: name: load_local_v2s64
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 16, align 8, addrspace 3)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]]
+    ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(<2 x s64>) = G_LOAD %0 :: (load 16, align 8, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
 
 ...
 
 ---
 
bb.0:
     ; GFX7: liveins: $vgpr0
     ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX7: $m0 = S_MOV_B32 -1
     ; GFX7: [[DS_READ2_B64_:%[0-9]+]]:vreg_128 = DS_READ2_B64 [[COPY]], 0, 1, 0, implicit $m0, implicit $exec :: (load 16, align 8, addrspace 3)
     ; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_]]
     ; GFX9-LABEL: name: load_local_s128
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 16, align 8, addrspace 3)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]]
+    ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(s128) = G_LOAD %0 :: (load 16, align 8, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
 
 ...
 
 ---
 

llvm/test/CodeGen/AMDGPU/GlobalISel/lds-misaligned-bug.ll

-; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s
-; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s
-; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s
-; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,VECT %s
-; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=UNALIGNED %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,ALIGNED,VECT %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=GCN,UNALIGNED,VECT %s
 
 ; GCN-LABEL: test_local_misaligned_v2:
 ; GCN-DAG: ds_read2_b32
 ; GCN-DAG: ds_write2_b32
 define amdgpu_kernel void @test_local_misaligned_v2(i32 addrspace(3)* %arg) {
 bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
   %ptr = bitcast i32 addrspace(3)* %gep to <2 x i32> addrspace(3)*
   %load = load <2 x i32>, <2 x i32> addrspace(3)* %ptr, align 4
   %v1 = extractelement <2 x i32> %load, i32 0
   %v2 = extractelement <2 x i32> %load, i32 1
   %v3 = insertelement <2 x i32> undef, i32 %v2, i32 0
   %v4 = insertelement <2 x i32> %v3, i32 %v1, i32 1
   store <2 x i32> %v4, <2 x i32> addrspace(3)* %ptr, align 4
   ret void
 }
 
 ; GCN-LABEL: test_local_misaligned_v4:
-; GCN-DAG: ds_read2_b32
-; GCN-DAG: ds_read2_b32
-; GCN-DAG: ds_write2_b32
-; GCN-DAG: ds_write2_b32
+; ALIGNED-DAG: ds_read2_b32
+; ALIGNED-DAG: ds_read2_b32
+; ALIGNED-DAG: ds_write2_b32
+; ALIGNED-DAG: ds_write2_b32
 ; UNALIGNED-DAG: ds_read_b128
 ; UNALIGNED-DAG: ds_write_b128
 define amdgpu_kernel void @test_local_misaligned_v4(i32 addrspace(3)* %arg) {
 bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
   %ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)*
   %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 4
   %v1 = extractelement <4 x i32> %load, i32 0
   %v2 = extractelement <4 x i32> %load, i32 1
   %v3 = extractelement <4 x i32> %load, i32 2
   %v4 = extractelement <4 x i32> %load, i32 3
   %v5 = insertelement <4 x i32> undef, i32 %v4, i32 0
   %v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1
   %v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2
   %v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3
   store <4 x i32> %v8, <4 x i32> addrspace(3)* %ptr, align 4
   ret void
 }
 
 ; GCN-LABEL: test_local_misaligned_v3:
-; GCN-DAG: ds_read2_b32
-; GCN-DAG: ds_read_b32
-; GCN-DAG: ds_write2_b32
-; GCN-DAG: ds_write_b32
+; ALIGNED-DAG: ds_read2_b32
+; ALIGNED-DAG: ds_read_b32
+; ALIGNED-DAG: ds_write2_b32
+; ALIGNED-DAG: ds_write_b32
 ; UNALIGNED-DAG: ds_read_b96
 ; UNALIGNED-DAG: ds_write_b96
 define amdgpu_kernel void @test_local_misaligned_v3(i32 addrspace(3)* %arg) {
 bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
   %ptr = bitcast i32 addrspace(3)* %gep to <3 x i32> addrspace(3)*
   %load = load <3 x i32>, <3 x i32> addrspace(3)* %ptr, align 4
bb:
   %v5 = insertelement <3 x i32> undef, i32 %v3, i32 0
   %v6 = insertelement <3 x i32> %v5, i32 %v1, i32 1
   %v7 = insertelement <3 x i32> %v6, i32 %v2, i32 2
   store <3 x i32> %v7, <3 x i32> addrspace(3)* %ptr, align 16
   ret void
 }
 
 ; GCN-LABEL: test_local_v4_aligned8:
-; GCN-DAG: ds_read_b128
-; GCN-DAG: ds_write_b128
+; ALIGNED-DAG: ds_read2_b64
+; ALIGNED-DAG: ds_write2_b64
+; UNALIGNED-DAG: ds_read_b128
+; UNALIGNED-DAG: ds_write_b128
 define amdgpu_kernel void @test_local_v4_aligned8(i32 addrspace(3)* %arg) {
 bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
   %ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)*
   %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 8
   %v1 = extractelement <4 x i32> %load, i32 0
   %v2 = extractelement <4 x i32> %load, i32 1

llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll

; GFX7-NEXT:    s_setpc_b64 s[30:31]
   %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 4
   ret <4 x i32> %load
 }
 
 define <4 x i32> @load_lds_v4i32_align8(<4 x i32> addrspace(3)* %ptr) {
 ; GFX9-LABEL: load_lds_v4i32_align8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    ds_read_b128 v[0:3], v0
+; GFX9-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: load_lds_v4i32_align8:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1

llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll

 ; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NEXT:    ds_write_b128 v4, v[0:3]
+; GFX9-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: store_lds_v4i32_align8:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x9
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xd
 ; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)

llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll

-; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s
-; RUN: llc -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s
-; RUN: llc -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,VECT %s
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=UNALIGNED,VECT %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
+; RUN: llc -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
+; RUN: llc -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,ALIGNED,VECT %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=GCN,UNALIGNED,VECT %s
 
 ; GCN-LABEL: test_local_misaligned_v2:
 ; GCN-DAG: ds_read2_b32
 ; GCN-DAG: ds_write2_b32
 define amdgpu_kernel void @test_local_misaligned_v2(i32 addrspace(3)* %arg) {
 bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
   %ptr = bitcast i32 addrspace(3)* %gep to <2 x i32> addrspace(3)*
   %load = load <2 x i32>, <2 x i32> addrspace(3)* %ptr, align 4
   %v1 = extractelement <2 x i32> %load, i32 0
   %v2 = extractelement <2 x i32> %load, i32 1
   %v3 = insertelement <2 x i32> undef, i32 %v2, i32 0
   %v4 = insertelement <2 x i32> %v3, i32 %v1, i32 1
   store <2 x i32> %v4, <2 x i32> addrspace(3)* %ptr, align 4
   ret void
 }
 
 ; GCN-LABEL: test_local_misaligned_v4:
-; GCN-DAG: ds_read2_b32
-; GCN-DAG: ds_read2_b32
-; GCN-DAG: ds_write2_b32
-; GCN-DAG: ds_write2_b32
+; ALIGNED-DAG: ds_read2_b32
+; ALIGNED-DAG: ds_read2_b32
+; ALIGNED-DAG: ds_write2_b32
+; ALIGNED-DAG: ds_write2_b32
 ; UNALIGNED-DAG: ds_read_b128
 ; UNALIGNED-DAG: ds_write_b128
 define amdgpu_kernel void @test_local_misaligned_v4(i32 addrspace(3)* %arg) {
 bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
   %ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)*
   %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 4
   %v1 = extractelement <4 x i32> %load, i32 0
   %v2 = extractelement <4 x i32> %load, i32 1
   %v3 = extractelement <4 x i32> %load, i32 2
   %v4 = extractelement <4 x i32> %load, i32 3
   %v5 = insertelement <4 x i32> undef, i32 %v4, i32 0
   %v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1
   %v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2
   %v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3
   store <4 x i32> %v8, <4 x i32> addrspace(3)* %ptr, align 4
   ret void
 }
 
 ; GCN-LABEL: test_local_misaligned_v3:
-; GCN-DAG: ds_read2_b32
-; GCN-DAG: ds_read_b32
-; GCN-DAG: ds_write2_b32
-; GCN-DAG: ds_write_b32
+; ALIGNED-DAG: ds_read2_b32
+; ALIGNED-DAG: ds_read_b32
+; ALIGNED-DAG: ds_write2_b32
+; ALIGNED-DAG: ds_write_b32
 ; UNALIGNED-DAG: ds_read_b96
 ; UNALIGNED-DAG: ds_write_b96
 define amdgpu_kernel void @test_local_misaligned_v3(i32 addrspace(3)* %arg) {
 bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
   %ptr = bitcast i32 addrspace(3)* %gep to <3 x i32> addrspace(3)*
   %load = load <3 x i32>, <3 x i32> addrspace(3)* %ptr, align 4
bb:
   %v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1
   %v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2
   %v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3
   store <4 x i32> %v8, <4 x i32>* %ptr, align 16
   ret void
 }
 
 ; GCN-LABEL: test_local_v4_aligned8:
-; GCN-DAG: ds_read_b128
-; GCN-DAG: ds_write_b128
+; ALIGNED-DAG: ds_read2_b64
+; ALIGNED-DAG: ds_write2_b64
+; UNALIGNED-DAG: ds_read_b128
+; UNALIGNED-DAG: ds_write_b128
 define amdgpu_kernel void @test_local_v4_aligned8(i32 addrspace(3)* %arg) {
 bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %arg, i32 %lid
   %ptr = bitcast i32 addrspace(3)* %gep to <4 x i32> addrspace(3)*
   %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 8
   %v1 = extractelement <4 x i32> %load, i32 0
   %v2 = extractelement <4 x i32> %load, i32 1

llvm/test/CodeGen/AMDGPU/load-local.128.ll

; GFX6-NEXT:    s_setpc_b64 s[30:31]
   %load = load <4 x i32>, <4 x i32> addrspace(3)* %ptr, align 4
   ret <4 x i32> %load
 }
 
 define <4 x i32> @load_lds_v4i32_align8(<4 x i32> addrspace(3)* %ptr) {
 ; GFX9-LABEL: load_lds_v4i32_align8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    ds_read_b128 v[0:3], v0
+; GFX9-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: load_lds_v4i32_align8:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1

llvm/test/CodeGen/AMDGPU/store-local.128.ll

 define amdgpu_kernel void @store_lds_v4i32_align8(<4 x i32> addrspace(3)* %out, <4 x i32> %x) {
 ; GFX9-LABEL: store_lds_v4i32_align8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NEXT:    ds_write_b128 v4, v[0:3]
+; GFX9-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: store_lds_v4i32_align8:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_load_dword s4, s[0:1], 0x9
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xd
 ; GFX7-NEXT:    s_mov_b32 m0, -1
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)