[AMDGPU] Supported ds_read_b128 generation; Widened vector length for local address-space

[FarhanaAleen]

Summary

Starting from GCN 2nd generation, ISA supports ds_read_b128 on top of ds_read_b64. This patch supports ds_read_b128 instruction pattern and generation of this instruction.

In the vectorizer, this patch also widen the vector length so that vectorizer generates 128 bit loads for local address-space which gets translated to ds_read_b128.

[nhaehnle]

Have you tested this on real hardware? I remember reading that there is a hardware bug on gfx7 with this instruction. The bug may apply only to early gfx7 chips.

test/CodeGen/AMDGPU/reorder-stores.ll

2

What does SEA mean? We usually use CI for Sea Islands.

[arsenm]

I've implemented this before: https://github.com/arsenm/llvm/tree/ds-128

This looks mostly the same. It's not clear to me it's always better to use this. I don't think this executes any faster, and at least for ds_write_b128, this has an additional constraint that the inputs must now be in a contiguous 128-bit register instead of 2 independent 64-bit pairs, which increases register pressure and may require copies. It might be better to defer forming this until later, like in the LoadStoreOptimizer pass. Jeff had a benchmark he wanted to try with this.

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

246–248

It might be OK to say 128 anyway. You could still do adjacent ds_read2_b64 even when not using ds_read_b128. I don't think we try to do the same trick we do with 4-byte aligned 8 byte reads for the 64-bit equivalent, but you might want to look into that.

Anything you change here would also equally apply to REGION_ADDRESS

lib/Target/AMDGPU/SIISelLowering.cpp

5427

This should be hidden inside a Subtarget->hasDS128() check

5428

You don't need the isAligned16 helper. You just need to check that the alignment is >= 16, not % 16

test/CodeGen/AMDGPU/ds_read2_superreg.ll

100–135

These tests have the unfortunate side effect of breaking what this test intended, which was the pass forming the read2. Maybe change all of these to reduce the alignment so you still get read2?

[rampitec]

Given performance benefit is somewhat unclear can you put it under an option?

lib/Target/AMDGPU/SIISelLowering.cpp

5428

Second to that.

[FarhanaAleen]

Enabled ds_read_b128 under a switch and incorporated additional comments.

[rampitec]

lib/Target/AMDGPU/SIISelLowering.cpp

5433

You only have pattern for v4i32, but enable operation for all 128 bit. Will it work with v8i16 for example?

[FarhanaAleen]

lib/Target/AMDGPU/SIISelLowering.cpp

5433

Yes, it works for i16/i8.

During dag combine, AMDGPU loadCombiner combines vector types of 8/16/64 to vector types of 32 bit type.

[rampitec]

LGTM. Thanks.

lib/Target/AMDGPU/AMDGPUInstructions.td

 //===----------------------------------------------------------------------===//
 // Load/Store Pattern Fragments
 //===----------------------------------------------------------------------===//
 
 class Aligned8Bytes <dag ops, dag frag> : PatFrag <ops, frag, [{
   return cast<MemSDNode>(N)->getAlignment() % 8 == 0;
 }]>;
 
+class Aligned16Bytes <dag ops, dag frag> : PatFrag <ops, frag, [{
+  return cast<MemSDNode>(N)->getAlignment() % 16 == 0;
+}]>;
+
 class LoadFrag <SDPatternOperator op> : PatFrag<(ops node:$ptr), (op node:$ptr)>;
 
 class StoreFrag<SDPatternOperator op> : PatFrag <
   (ops node:$value, node:$ptr), (op node:$value, node:$ptr)
 >;
 
 class StoreHi16<SDPatternOperator op> : PatFrag <
   (ops node:$value, node:$ptr), (op (srl node:$value, (i32 16)), node:$ptr)

 def truncstorei16_local : LocalStore <truncstorei16>;
 def store_local_hi16 : StoreHi16 <truncstorei16>, LocalAddress;
 def truncstorei8_local_hi16 : StoreHi16<truncstorei8>, LocalAddress;
 
 def load_align8_local : Aligned8Bytes <
   (ops node:$ptr), (load_local node:$ptr)
 >;
 
+def load_align16_local : Aligned16Bytes <
+  (ops node:$ptr), (load_local node:$ptr)
+>;
+
 def store_align8_local : Aligned8Bytes <
   (ops node:$val, node:$ptr), (store_local node:$val, node:$ptr)
 >;
 
 
 def load_flat          : FlatLoad <load>;
 def az_extloadi8_flat  : FlatLoad <az_extloadi8>;
 def sextloadi8_flat    : FlatLoad <sextloadi8>;

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

 
 unsigned AMDGPUTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const {
   AMDGPUAS AS = ST->getAMDGPUAS();
   if (AddrSpace == AS.GLOBAL_ADDRESS ||
       AddrSpace == AS.CONSTANT_ADDRESS ||
       AddrSpace == AS.CONSTANT_ADDRESS_32BIT ||
       AddrSpace == AS.FLAT_ADDRESS)
     return 128;
-  if (AddrSpace == AS.LOCAL_ADDRESS ||
-      AddrSpace == AS.REGION_ADDRESS)
+
+  if (AddrSpace == AS.LOCAL_ADDRESS) {
+    if (ST->getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS)
+      return 128;
+    return 64;
+  }
+  if (AddrSpace == AS.REGION_ADDRESS)
     return 64;
   if (AddrSpace == AS.PRIVATE_ADDRESS)
     return 8 * ST->getMaxPrivateElementSize();
 
   if (ST->getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS &&
       (AddrSpace == AS.PARAM_D_ADDRESS ||
       AddrSpace == AS.PARAM_I_ADDRESS ||
       (AddrSpace >= AS.CONSTANT_BUFFER_0 &&

lib/Target/AMDGPU/DSInstructions.td

 defm : DSReadPat_mc <DS_READ_U16, i32, "az_extloadi16_local">;
 defm : DSReadPat_mc <DS_READ_U16, i16, "load_local">;
 defm : DSReadPat_mc <DS_READ_B32, i32, "load_local">;
 
 let AddedComplexity = 100 in {
 
 defm : DSReadPat_mc <DS_READ_B64, v2i32, "load_align8_local">;
 
+defm : DSReadPat_mc <DS_READ_B128, v4i32, "load_align16_local">;
+
 } // End AddedComplexity = 100
 
 let OtherPredicates = [HasD16LoadStore] in {
 let AddedComplexity = 100 in {
 defm : DSReadPat_Hi16<DS_READ_U16_D16_HI, load_local>;
 defm : DSReadPat_Hi16<DS_READ_U8_D16_HI, az_extloadi8_local>;
 defm : DSReadPat_Hi16<DS_READ_I8_D16_HI, sextloadi8_local>;
 

lib/Target/AMDGPU/SIISelLowering.cpp

static bool mayTailCallThisCC(CallingConv::ID CC) {
   switch (CC) {
   case CallingConv::C:
     return true;
   default:
     return canGuaranteeTCO(CC);
   }
 }
 
+static bool isAligned16(unsigned Alignment) {
+  return Alignment % 16 == 0;
+}
+
 bool SITargetLowering::isEligibleForTailCallOptimization(
     SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
     const SmallVectorImpl<ISD::OutputArg> &Outs,
     const SmallVectorImpl<SDValue> &OutVals,
     const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
   if (!mayTailCallThisCC(CalleeCC))
     return false;
 
case 16:
       // Same as global/flat
       if (NumElements > 4)
         return SplitVectorLoad(Op, DAG);
       return SDValue();
     default:
       llvm_unreachable("unsupported private_element_size");
     }
   } else if (AS == AMDGPUASI.LOCAL_ADDRESS) {
-    if (NumElements > 2)
-      return SplitVectorLoad(Op, DAG);
-
-    if (NumElements == 2)
+    unsigned Alignment = Load->getAlignment();
+    if (Subtarget->getGeneration() >= SISubtarget::SEA_ISLANDS &&
+        isAligned16(Alignment) && MemVT.getStoreSize() == 16)
         return SDValue();
 
-    // If properly aligned, if we split we might be able to use ds_read_b64.
+    if (NumElements > 2)
       return SplitVectorLoad(Op, DAG);
   }
   return SDValue();
 }
 
 SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
   if (Op.getValueType() != MVT::i64)
     return SDValue();
 
   SDLoc DL(Op);

lib/Target/AMDGPU/SIInstrInfo.td

 
 def sextloadi16_glue : PatFrag<(ops node:$ptr), (sextload_glue node:$ptr), [{
   return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i16;
 }]>;
 
 def load_glue_align8 : Aligned8Bytes <
   (ops node:$ptr), (load_glue node:$ptr)
 >;
+def load_glue_align16 : Aligned16Bytes <
+  (ops node:$ptr), (load_glue node:$ptr)
+>;
 
 
 def load_local_m0 : LoadFrag<load_glue>, LocalAddress;
 def sextloadi8_local_m0 : LoadFrag<sextloadi8_glue>, LocalAddress;
 def sextloadi16_local_m0 : LoadFrag<sextloadi16_glue>, LocalAddress;
 def az_extloadi8_local_m0 : LoadFrag<az_extloadi8_glue>, LocalAddress;
 def az_extloadi16_local_m0 : LoadFrag<az_extloadi16_glue>, LocalAddress;
 def load_align8_local_m0 : LoadFrag <load_glue_align8>, LocalAddress;
+def load_align16_local_m0 : LoadFrag <load_glue_align16>, LocalAddress;
 
 
 def AMDGPUst_glue : SDNode <"ISD::STORE", SDTStore,
   [SDNPHasChain, SDNPMayStore, SDNPMemOperand, SDNPInGlue]
 >;
 
 def unindexedstore_glue : PatFrag<(ops node:$val, node:$ptr),
                                    (AMDGPUst_glue node:$val, node:$ptr), [{

test/CodeGen/AMDGPU/ds_read2_superreg.ll

define amdgpu_kernel void @simple_read2_v4f32_superreg_align8(<4 x float> addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x <4 x float>], [512 x <4 x float>] addrspace(3)* @lds.v4, i32 0, i32 %x.i
   %val0 = load <4 x float>, <4 x float> addrspace(3)* %arrayidx0, align 8
   %out.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %out, i32 %x.i
   store <4 x float> %val0, <4 x float> addrspace(1)* %out.gep
   ret void
 }
 
 ; CI-LABEL: {{^}}simple_read2_v4f32_superreg:
-; CI-DAG: ds_read2_b64 [[REG_ZW:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}} offset1:1{{$}}
+; CI-DAG: ds_read_b128 [[REG_ZW:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}
 ; CI: buffer_store_dwordx4 [[REG_ZW]]
 ; CI: s_endpgm
 define amdgpu_kernel void @simple_read2_v4f32_superreg(<4 x float> addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x <4 x float>], [512 x <4 x float>] addrspace(3)* @lds.v4, i32 0, i32 %x.i
   %val0 = load <4 x float>, <4 x float> addrspace(3)* %arrayidx0
   %out.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %out, i32 %x.i
   store <4 x float> %val0, <4 x float> addrspace(1)* %out.gep
   ret void
 }
 
 ; FIXME: Extra moves shuffling superregister
 ; CI-LABEL: {{^}}simple_read2_v8f32_superreg:
-; CI-DAG: ds_read2_b64 [[VEC_HI:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}} offset0:2 offset1:3{{$}}
-; CI-DAG: ds_read2_b64 [[VEC_LO:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}} offset1:1{{$}}
+; CI-DAG: ds_read_b128 [[VEC_HI:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}} offset:16
+; CI-DAG: ds_read_b128 [[VEC_LO:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}
 ; CI-DAG: buffer_store_dwordx4 [[VEC_HI]], v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0 addr64 offset:16
 ; CI-DAG: buffer_store_dwordx4 [[VEC_LO]], v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0 addr64{{$}}
 ; CI: s_endpgm
 define amdgpu_kernel void @simple_read2_v8f32_superreg(<8 x float> addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x <8 x float>], [512 x <8 x float>] addrspace(3)* @lds.v8, i32 0, i32 %x.i
   %val0 = load <8 x float>, <8 x float> addrspace(3)* %arrayidx0
   %out.gep = getelementptr inbounds <8 x float>, <8 x float> addrspace(1)* %out, i32 %x.i
   store <8 x float> %val0, <8 x float> addrspace(1)* %out.gep
   ret void
 }
 
 ; FIXME: Extra moves shuffling superregister
 ; CI-LABEL: {{^}}simple_read2_v16f32_superreg:
-; CI-DAG: ds_read2_b64 [[VEC0_3:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}} offset1:1{{$}}
-; CI-DAG: ds_read2_b64 [[VEC4_7:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}} offset0:2 offset1:3{{$}}
-; CI-DAG: ds_read2_b64 [[VEC8_11:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}} offset0:4 offset1:5{{$}}
-; CI-DAG: ds_read2_b64 [[VEC12_15:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}} offset0:6 offset1:7{{$}}
+; CI-DAG: ds_read_b128 [[VEC0_3:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}
+; CI-DAG: ds_read_b128 [[VEC4_7:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}} offset:16
+; CI-DAG: ds_read_b128 [[VEC8_11:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}} offset:32
+; CI-DAG: ds_read_b128 [[VEC12_15:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}} offset:48
 ; CI: s_waitcnt lgkmcnt(0)
 ; CI-DAG: buffer_store_dwordx4 [[VEC0_3]], v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0 addr64{{$}}
 ; CI-DAG: buffer_store_dwordx4 [[VEC4_7]], v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0 addr64 offset:16
 ; CI-DAG: buffer_store_dwordx4 [[VEC8_11]], v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0 addr64 offset:32
 ; CI-DAG: buffer_store_dwordx4 [[VEC12_15]], v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0 addr64 offset:48
 ; CI: s_endpgm
 define amdgpu_kernel void @simple_read2_v16f32_superreg(<16 x float> addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x <16 x float>], [512 x <16 x float>] addrspace(3)* @lds.v16, i32 0, i32 %x.i

test/CodeGen/AMDGPU/indirect-private-64.ll

 ; SI-ALLOCA4: buffer_load_dword v
 ; SI-ALLOCA4: buffer_load_dword v
 
 ; SI-PROMOTE: ds_write_b64
 ; SI-PROMOTE: ds_write_b64
 ; SI-PROMOTE: ds_read_b64
 ; SI-PROMOTE: ds_read_b64
 ; CI-PROMOTE: ds_write2_b64
-; CI-PROMOTE: ds_read2_b64
+; CI-PROMOTE: ds_read_b128
 define amdgpu_kernel void @private_access_v2f64_alloca(<2 x double> addrspace(1)* noalias %out, <2 x double> addrspace(1)* noalias %in, i32 %b) #1 {
   %val = load <2 x double>, <2 x double> addrspace(1)* %in, align 16
   %array = alloca [4 x <2 x double>], align 16, addrspace(5)
   %ptr = getelementptr inbounds [4 x <2 x double>], [4 x <2 x double>] addrspace(5)* %array, i32 0, i32 %b
   store <2 x double> %val, <2 x double> addrspace(5)* %ptr, align 16
   call void @llvm.amdgcn.s.barrier()
   %result = load <2 x double>, <2 x double> addrspace(5)* %ptr, align 16
   store <2 x double> %result, <2 x double> addrspace(1)* %out, align 16

 ; SI-ALLOCA4: buffer_load_dword v
 ; SI-ALLOCA4: buffer_load_dword v
 
 ; SI-PROMOTE: ds_write_b64
 ; SI-PROMOTE: ds_write_b64
 ; SI-PROMOTE: ds_read_b64
 ; SI-PROMOTE: ds_read_b64
 ; CI-PROMOTE: ds_write2_b64
-; CI-PROMOTE: ds_read2_b64
+; CI-PROMOTE: ds_read_b128
 define amdgpu_kernel void @private_access_v2i64_alloca(<2 x i64> addrspace(1)* noalias %out, <2 x i64> addrspace(1)* noalias %in, i32 %b) #1 {
   %val = load <2 x i64>, <2 x i64> addrspace(1)* %in, align 16
   %array = alloca [4 x <2 x i64>], align 16, addrspace(5)
   %ptr = getelementptr inbounds [4 x <2 x i64>], [4 x <2 x i64>] addrspace(5)* %array, i32 0, i32 %b
   store <2 x i64> %val, <2 x i64> addrspace(5)* %ptr, align 16
   call void @llvm.amdgcn.s.barrier()
   %result = load <2 x i64>, <2 x i64> addrspace(5)* %ptr, align 16
   store <2 x i64> %result, <2 x i64> addrspace(1)* %out, align 16
   ret void
 }
 
 attributes #0 = { convergent nounwind }
 attributes #1 = { nounwind "amdgpu-waves-per-eu"="1,2" "amdgpu-flat-work-group-size"="64,128" }

test/CodeGen/AMDGPU/load-local-f32.ll

-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC,GCN-DEF %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC,GFX8 %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}load_f32_local:
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0
 ; GCN: ds_read_b32
 
 ; EG: LDS_READ_RET

 ; GCN-DAG: ds_write_b64
 ; GCN-DAG: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:8{{$}}
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_v3f32(<3 x float> addrspace(3)* %out, <3 x float> addrspace(3)* %in) #0 {
 entry:
-  %tmp0 = load <3 x float>, <3 x float> addrspace(3)* %in
+  %tmp0 = load <3 x float>, <3 x float> addrspace(3)* %in, align 8
   store <3 x float> %tmp0, <3 x float> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v4f32:
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0
 
-; GCN: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GFX8: ds_read_b128
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_v4f32(<4 x float> addrspace(3)* %out, <4 x float> addrspace(3)* %in) #0 {
 entry:
   %tmp0 = load <4 x float>, <4 x float> addrspace(3)* %in
   store <4 x float> %tmp0, <4 x float> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v8f32:
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0
 
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GFX8: ds_read_b128
+; GFX8: ds_read_b128
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_v8f32(<8 x float> addrspace(3)* %out, <8 x float> addrspace(3)* %in) #0 {
 entry:
   %tmp0 = load <8 x float>, <8 x float> addrspace(3)* %in
   store <8 x float> %tmp0, <8 x float> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v16f32:
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0
 
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+
+; GFX8: ds_read_b128
+; GFX8: ds_read_b128
+; GFX8: ds_read_b128
+; GFX8: ds_read_b128
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET

test/CodeGen/AMDGPU/load-local-f64.ll

-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC,GCN-DEF %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC,GFX789 %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC,GFX789 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC,GFX789 %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}local_load_f64:
 ; SICIV: s_mov_b32 m0
 ; GFX9-NOT: m0
 
 ; GCN: ds_read_b64 [[VAL:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}{{$}}
 ; GCN: ds_write_b64 v{{[0-9]+}}, [[VAL]]
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_f64(double addrspace(3)* %out, double addrspace(3)* %in) #0 {
   %ld = load double, double addrspace(3)* %in
   store double %ld, double addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v2f64:
 ; SICIV: s_mov_b32 m0
 ; GFX9-NOT: m0
 
-; GCN: ds_read2_b64
+; GCN-DEF: ds_read2_b64
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_v2f64(<2 x double> addrspace(3)* %out, <2 x double> addrspace(3)* %in) #0 {
 entry:
   %ld = load <2 x double>, <2 x double> addrspace(3)* %in

 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_v3f64(<3 x double> addrspace(3)* %out, <3 x double> addrspace(3)* %in) #0 {
 entry:
-  %ld = load <3 x double>, <3 x double> addrspace(3)* %in
+  %ld = load <3 x double>, <3 x double> addrspace(3)* %in, align 8
   store <3 x double> %ld, <3 x double> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v4f64:
 ; SICIV: s_mov_b32 m0
 ; GFX9-NOT: m0
 
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GFX789: ds_read_b128
+; GFX789: ds_read_b128
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_v4f64(<4 x double> addrspace(3)* %out, <4 x double> addrspace(3)* %in) #0 {
 entry:
   %ld = load <4 x double>, <4 x double> addrspace(3)* %in
   store <4 x double> %ld, <4 x double> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v8f64:
 ; SICIV: s_mov_b32 m0
 ; GFX9-NOT: m0
 
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+
+; GFX789: ds_read_b128
+; GFX789: ds_read_b128
+; GFX789: ds_read_b128
+; GFX789: ds_read_b128
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
entry:
   store <8 x double> %ld, <8 x double> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v16f64:
 ; SICIV: s_mov_b32 m0
 ; GFX9-NOT: m0
 
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+
+; GFX789: ds_read_b128
+; GFX789: ds_read_b128
+; GFX789: ds_read_b128
+; GFX789: ds_read_b128
+; GFX789: ds_read_b128
+; GFX789: ds_read_b128
+; GFX789: ds_read_b128
+; GFX789: ds_read_b128
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET

test/CodeGen/AMDGPU/load-local-i16.ll

-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,SICIVI,FUNC,GCN-LOAD %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,GFX89,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,GFX89,FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}local_load_i16:
 ; GFX9-NOT: m0
 ; SICIVI: s_mov_b32 m0
 
entry:
   store <4 x i16> %ld, <4 x i16> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v8i16:
 ; GFX9-NOT: m0
 ; SICIVI: s_mov_b32 m0
 
-; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_v8i16(<8 x i16> addrspace(3)* %out, <8 x i16> addrspace(3)* %in) {
 entry:
   %ld = load <8 x i16>, <8 x i16> addrspace(3)* %in
   store <8 x i16> %ld, <8 x i16> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v16i16:
 ; GFX9-NOT: m0
 ; SICIVI: s_mov_b32 m0
 
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:3{{$}}
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:1 offset1:2{{$}}
-
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:3{{$}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:1 offset1:2{{$}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:16
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
define amdgpu_kernel void @local_sextload_v4i16_to_v4i32(<4 x i32> addrspace(3)* %out, <4 x i16> addrspace(3)* %in) #0 {
   store <4 x i32> %ext, <4 x i32> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v8i16_to_v8i32:
 ; GFX9-NOT: m0
 ; SICIVI: s_mov_b32 m0
 
-; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_zextload_v8i16_to_v8i32(<8 x i32> addrspace(3)* %out, <8 x i16> addrspace(3)* %in) #0 {
   %load = load <8 x i16>, <8 x i16> addrspace(3)* %in
   %ext = zext <8 x i16> %load to <8 x i32>
   store <8 x i32> %ext, <8 x i32> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v8i16_to_v8i32:
 ; GFX9-NOT: m0
 ; SICIVI: s_mov_b32 m0
 
-; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG-DAG: BFE_INT
 ; EG-DAG: BFE_INT
 ; EG-DAG: BFE_INT
 ; EG-DAG: BFE_INT
 ; EG-DAG: BFE_INT
 ; EG-DAG: BFE_INT
 ; EG-DAG: BFE_INT
 ; EG-DAG: BFE_INT
 define amdgpu_kernel void @local_sextload_v8i16_to_v8i32(<8 x i32> addrspace(3)* %out, <8 x i16> addrspace(3)* %in) #0 {
   %load = load <8 x i16>, <8 x i16> addrspace(3)* %in
   %ext = sext <8 x i16> %load to <8 x i32>
   store <8 x i32> %ext, <8 x i32> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v16i16_to_v16i32:
 ; GFX9-NOT: m0
 ; SICIVI: s_mov_b32 m0
 
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:16
 
 ; GCN: ds_write2_b64
 ; GCN: ds_write2_b64
 ; GCN: ds_write2_b64
 ; GCN: ds_write2_b64
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
define amdgpu_kernel void @local_zextload_v16i16_to_v16i32(<16 x i32> addrspace(3)* %out, <16 x i16> addrspace(3)* %in) #0 {
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v16i16_to_v16i32:
 ; GFX9-NOT: m0
 ; SICIVI: s_mov_b32 m0
 
 
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:16
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
define amdgpu_kernel void @local_sextload_v16i16_to_v16i32(<16 x i32> addrspace(3)* %out, <16 x i16> addrspace(3)* %in) #0 {
   store <16 x i32> %ext, <16 x i32> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v32i16_to_v32i32:
 ; GFX9-NOT: m0
 ; SICIVI: s_mov_b32 m0
 
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:4 offset1:5
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:6 offset1:7
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:4 offset1:5
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:6 offset1:7
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:16
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:32
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:48
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
define amdgpu_kernel void @local_zextload_v32i16_to_v32i32(<32 x i32> addrspace(3)* %out, <32 x i16> addrspace(3)* %in) #0 {
   store <32 x i32> %ext, <32 x i32> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v32i16_to_v32i32:
 ; GFX9-NOT: m0
 ; SICIVI: s_mov_b32 m0
 
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:4 offset1:5
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:6 offset1:7
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:4 offset1:5{{$}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:6 offset1:7{{$}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:32
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:48
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:16
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:14 offset1:15
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:12 offset1:13
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:10 offset1:11
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:8 offset1:9
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:6 offset1:7
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:4 offset1:5
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:2 offset1:3
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset1:1
define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(<32 x i32> addrspace(3)* %out, <32 x i16> addrspace(3)* %in) #0 {
   store <32 x i32> %ext, <32 x i32> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v64i16_to_v64i32:
 ; GFX9-NOT: m0
 ; SICIVI: s_mov_b32 m0
 
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:14 offset1:15
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:4 offset1:5
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:6 offset1:7
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:8 offset1:9
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:12 offset1:13
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:10 offset1:11
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GCN-LOAD: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:30 offset1:31
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:28 offset1:29
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:26 offset1:27
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:24 offset1:25
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:22 offset1:23
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:20 offset1:21
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:18 offset1:19
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:16 offset1:17

test/CodeGen/AMDGPU/load-local-i32.ll

-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC,GCN-DEF %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC,GFX89 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC,GFX89 %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}local_load_i32:
 ; GCN-NOT: s_wqm_b64
 ; SICIVI: s_mov_b32 m0, -1
 ; GFX9-NOT: m0
 ; GCN: ds_read_b32
 

 ; FUNC-LABEL: {{^}}local_load_v3i32:
 ; SICIVI: s_mov_b32 m0, -1
 ; GFX9-NOT: m0
 
 ; GCN-DAG: ds_read_b64
 ; GCN-DAG: ds_read_b32
 define amdgpu_kernel void @local_load_v3i32(<3 x i32> addrspace(3)* %out, <3 x i32> addrspace(3)* %in) #0 {
 entry:
-  %ld = load <3 x i32>, <3 x i32> addrspace(3)* %in
-  store <3 x i32> %ld, <3 x i32> addrspace(3)* %out
+  %ld = load <3 x i32>, <3 x i32> addrspace(3)* %in, align 8
+  store <3 x i32> %ld, <3 x i32> addrspace(3)* %out, align 8
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v4i32:
 ; SICIVI: s_mov_b32 m0, -1
 ; GFX9-NOT: m0
 
-; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GCN-DEF: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
 
 define amdgpu_kernel void @local_load_v4i32(<4 x i32> addrspace(3)* %out, <4 x i32> addrspace(3)* %in) #0 {
 entry:
   %ld = load <4 x i32>, <4 x i32> addrspace(3)* %in
   store <4 x i32> %ld, <4 x i32> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v8i32:
 ; SICIVI: s_mov_b32 m0, -1
 ; GFX9-NOT: m0
 
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GCN-DEF: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
+; GCN-DEF: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:16
 define amdgpu_kernel void @local_load_v8i32(<8 x i32> addrspace(3)* %out, <8 x i32> addrspace(3)* %in) #0 {
 entry:
   %ld = load <8 x i32>, <8 x i32> addrspace(3)* %in
   store <8 x i32> %ld, <8 x i32> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v16i32:
 ; SICIVI: s_mov_b32 m0, -1
 ; GFX9-NOT: m0
 
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:6 offset1:7{{$}}
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:4 offset1:5{{$}}
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
-; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GCN-DEF: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:6 offset1:7{{$}}
+; GCN-DEF: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:4 offset1:5{{$}}
+; GCN-DEF: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
+; GCN-DEF: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:16
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:32
+; GFX89: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:48
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:6 offset1:7
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:4 offset1:5
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:2 offset1:3
 ; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset1:1
 define amdgpu_kernel void @local_load_v16i32(<16 x i32> addrspace(3)* %out, <16 x i32> addrspace(3)* %in) #0 {
 entry:
   %ld = load <16 x i32>, <16 x i32> addrspace(3)* %in
   store <16 x i32> %ld, <16 x i32> addrspace(3)* %out

test/CodeGen/AMDGPU/load-local-i64.ll

-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC,GCN-DEF %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC,GFX789 %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC,GFX789 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC,GFX789 %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}local_load_i64:
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0
 
 ; GCN: ds_read_b64 [[VAL:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}{{$}}
 ; GCN: ds_write_b64 v{{[0-9]+}}, [[VAL]]
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_i64(i64 addrspace(3)* %out, i64 addrspace(3)* %in) #0 {
   %ld = load i64, i64 addrspace(3)* %in
   store i64 %ld, i64 addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v2i64:
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0
 
-; GCN: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GFX789: ds_read_b128
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_v2i64(<2 x i64> addrspace(3)* %out, <2 x i64> addrspace(3)* %in) #0 {
 entry:
   %ld = load <2 x i64>, <2 x i64> addrspace(3)* %in

 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_v3i64(<3 x i64> addrspace(3)* %out, <3 x i64> addrspace(3)* %in) #0 {
 entry:
-  %ld = load <3 x i64>, <3 x i64> addrspace(3)* %in
+  %ld = load <3 x i64>, <3 x i64> addrspace(3)* %in, align 8
   store <3 x i64> %ld, <3 x i64> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v4i64:
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0
 
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GFX89: ds_read_b128
+; GFX89: ds_read_b128
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_v4i64(<4 x i64> addrspace(3)* %out, <4 x i64> addrspace(3)* %in) #0 {
 entry:
   %ld = load <4 x i64>, <4 x i64> addrspace(3)* %in
   store <4 x i64> %ld, <4 x i64> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v8i64:
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0
 
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GFX89: ds_read_b128
+; GFX89: ds_read_b128
+; GFX89: ds_read_b128
+; GFX89: ds_read_b128
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
entry:
   store <8 x i64> %ld, <8 x i64> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v16i64:
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0
 
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
-; GCN: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+; GCN-DEF: ds_read2_b64
+
+; GFX89: ds_read_b128
+; GFX89: ds_read_b128
+; GFX89: ds_read_b128
+; GFX89: ds_read_b128
+; GFX89: ds_read_b128
+; GFX89: ds_read_b128
+; GFX89: ds_read_b128
+; GFX89: ds_read_b128
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET

test/CodeGen/AMDGPU/load-local-i8.ll

-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,SICIVI,FUNC,GCN-LOAD %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,SICIVI,FUNC,GFX8 %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
 ; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 
 ; FUNC-LABEL: {{^}}local_load_i8:
 ; GCN-NOT: s_wqm_b64
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0

 entry:
   %ld = load <8 x i8>, <8 x i8> addrspace(3)* %in
   store <8 x i8> %ld, <8 x i8> addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v16i8:
 ; GFX9-NOT: m0
-; GCN: ds_read2_b64  v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GCN-LOAD: ds_read2_b64  v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1{{$}}
+; GFX8: ds_read_b128  v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, v{{[0-9]+}}
+; GFX9: ds_read_b128  v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, v{{[0-9]+}}
 ; GCN: ds_write2_b64 v{{[0-9]+}}, v{{\[}}[[LO]]:{{[0-9]+}}], v[{{[0-9]+}}:[[HI]]{{\]}} offset1:1{{$}}
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_v16i8(<16 x i8> addrspace(3)* %out, <16 x i8> addrspace(3)* %in) #0 {
 entry:

test/CodeGen/AMDGPU/reorder-stores.ll

-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn < %s | FileCheck -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=SI %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn < %s | FileCheck -check-prefixes=SI,GCN %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefixes=SI,GCN-SEA %s
 
 ; SI-LABEL: {{^}}no_reorder_v2f64_global_load_store:
 ; SI: buffer_load_dwordx4
 ; SI: buffer_load_dwordx4
 ; SI: buffer_store_dwordx4
 ; SI: buffer_store_dwordx4
 ; SI: s_endpgm
 define amdgpu_kernel void @no_reorder_v2f64_global_load_store(<2 x double> addrspace(1)* nocapture %x, <2 x double> addrspace(1)* nocapture %y) nounwind {
   %tmp1 = load <2 x double>, <2 x double> addrspace(1)* %x, align 16
   %tmp4 = load <2 x double>, <2 x double> addrspace(1)* %y, align 16
   store <2 x double> %tmp4, <2 x double> addrspace(1)* %x, align 16
   store <2 x double> %tmp1, <2 x double> addrspace(1)* %y, align 16
   ret void
 }
 
 ; SI-LABEL: {{^}}no_reorder_scalarized_v2f64_local_load_store:
-; SI: ds_read2_b64
+; GCN: ds_read2_b64
+; GCN-SEA: ds_read_b128
 ; SI: ds_write2_b64
 ; SI: s_endpgm
 define amdgpu_kernel void @no_reorder_scalarized_v2f64_local_load_store(<2 x double> addrspace(3)* nocapture %x, <2 x double> addrspace(3)* nocapture %y) nounwind {
   %tmp1 = load <2 x double>, <2 x double> addrspace(3)* %x, align 16
   %tmp4 = load <2 x double>, <2 x double> addrspace(3)* %y, align 16
   store <2 x double> %tmp4, <2 x double> addrspace(3)* %x, align 16
   store <2 x double> %tmp1, <2 x double> addrspace(3)* %y, align 16
   ret void