[AMDGPU]Supported ds_write_b128 generation.

[FarhanaAleen]

Summary

This is a follow-on patch of https://reviews.llvm.org/D44210

[msearles]

LGTM

lib/Target/AMDGPU/AMDGPUInstructions.td

 def load_align16_local : Aligned16Bytes <
   (ops node:$ptr), (load_local node:$ptr)
 >;
 
 def store_align8_local : Aligned8Bytes <
   (ops node:$val, node:$ptr), (store_local node:$val, node:$ptr)
 >;
 
+def store_align16_local : Aligned16Bytes <
+  (ops node:$val, node:$ptr), (store_local node:$val, node:$ptr)
+>;
 
 def load_flat          : FlatLoad <load>;
 def az_extloadi8_flat  : FlatLoad <az_extloadi8>;
 def sextloadi8_flat    : FlatLoad <sextloadi8>;
 def az_extloadi16_flat : FlatLoad <az_extloadi16>;
 def sextloadi16_flat   : FlatLoad <sextloadi16>;
 def atomic_load_flat   : FlatLoad<atomic_load>;
 

lib/Target/AMDGPU/DSInstructions.td

 def : DS64Bit4ByteAlignedReadPat<DS_READ2_B32_gfx9, load_local>;
 def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32_gfx9, store_local>;
 }
 
 
 let AddedComplexity = 100 in {
 
 defm : DSWritePat_mc <DS_WRITE_B64, v2i32, "store_align8_local">;
+defm : DSWritePat_mc <DS_WRITE_B128, v4i32, "store_align16_local">;
+
 } // End AddedComplexity = 100
 class DSAtomicRetPat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
   (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$value),
   (inst $ptr, $value, (as_i16imm $offset), (i1 0))
 >;
 
 multiclass DSAtomicRetPat_mc<DS_Pseudo inst, ValueType vt, string frag> {
   let OtherPredicates = [LDSRequiresM0Init] in {

lib/Target/AMDGPU/SIISelLowering.cpp

if (AS == AMDGPUASI.GLOBAL_ADDRESS ||
     case 16:
       if (NumElements > 4)
         return SplitVectorStore(Op, DAG);
       return SDValue();
     default:
       llvm_unreachable("unsupported private_element_size");
     }
   } else if (AS == AMDGPUASI.LOCAL_ADDRESS) {
-    if (NumElements > 2)
-      return SplitVectorStore(Op, DAG);
-
-    if (NumElements == 2)
-      return Op;
+    // Use ds_write_b128 if possible.
+    if (Subtarget->useDS128(EnableDS128) && Store->getAlignment() >= 16 &&
+        VT.getStoreSize() == 16)
+      return SDValue();
 
-    // If properly aligned, if we split we might be able to use ds_write_b64.
+    if (NumElements > 2)
       return SplitVectorStore(Op, DAG);
+    return SDValue();
   } else {
     llvm_unreachable("unhandled address space");
   }
 }
 
 SDValue SITargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
   SDLoc DL(Op);
   EVT VT = Op.getValueType();

lib/Target/AMDGPU/SIInstrInfo.td

def truncstorei16_glue : PatFrag<(ops node:$val, node:$ptr),
                            (truncstore_glue node:$val, node:$ptr), [{
   return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i16;
 }]>;
 
 def store_glue_align8 : Aligned8Bytes <
   (ops node:$value, node:$ptr), (store_glue node:$value, node:$ptr)
 >;
 
+def store_glue_align16 : Aligned16Bytes <
+  (ops node:$value, node:$ptr), (store_glue node:$value, node:$ptr)
+>;
+
 def store_local_m0 : StoreFrag<store_glue>, LocalAddress;
 def truncstorei8_local_m0 : StoreFrag<truncstorei8_glue>, LocalAddress;
 def truncstorei16_local_m0 : StoreFrag<truncstorei16_glue>, LocalAddress;
 
 def store_align8_local_m0 : StoreFrag<store_glue_align8>, LocalAddress;
+def store_align16_local_m0 : StoreFrag<store_glue_align16>, LocalAddress;
 
 def si_setcc_uniform : PatFrag <
   (ops node:$lhs, node:$rhs, node:$cond),
   (setcc node:$lhs, node:$rhs, node:$cond), [{
   for (SDNode *Use : N->uses()) {
     if (Use->isMachineOpcode() || Use->getOpcode() != ISD::CopyToReg)
       return false;
 

test/CodeGen/AMDGPU/load-local-f32.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
 
-; Testing for ds_read_128
+; Testing for ds_read/write_128
 ; RUN: llc -march=amdgcn -mcpu=tahiti -amdgpu-ds128 < %s | FileCheck -check-prefixes=SI,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s
 
 ; FUNC-LABEL: {{^}}load_f32_local:
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0
 ; GCN: ds_read_b32

 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_v16f32(<16 x float> addrspace(3)* %out, <16 x float> addrspace(3)* %in) #0 {
 entry:
   %tmp0 = load <16 x float>, <16 x float> addrspace(3)* %in
   store <16 x float> %tmp0, <16 x float> addrspace(3)* %out
   ret void
 }
 
-; Tests if ds_read_b128 gets generated for the 16 byte aligned load.
+; Tests if ds_read/write_b128 gets generated for the 16 byte aligned load.
 ; FUNC-LABEL: {{^}}local_v4f32_to_128:
+
 ; SI-NOT: ds_read_b128
+; SI-NOT: ds_write_b128
+
 ; CIVI: ds_read_b128
+; CIVI: ds_write_b128
+
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_v4f32_to_128(<4 x float> addrspace(3)* %out, <4 x float> addrspace(3)* %in) {
   %ld = load <4 x float>, <4 x float> addrspace(3)* %in, align 16
-  store <4 x float> %ld, <4 x float> addrspace(3)* %out
+  store <4 x float> %ld, <4 x float> addrspace(3)* %out, align 16
   ret void
 }
 
 attributes #0 = { nounwind }

test/CodeGen/AMDGPU/load-local-f64.ll

 entry:
   %ld = load <16 x double>, <16 x double> addrspace(3)* %in
   store <16 x double> %ld, <16 x double> addrspace(3)* %out
   ret void
 }
 
 ; Tests if ds_read_b128 gets generated for the 16 byte aligned load.
 ; FUNC-LABEL: {{^}}local_load_v2f64_to_128:
+
 ; CIVI: ds_read_b128
+; CIVI: ds_write_b128
+
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_v2f64_to_128(<2 x double> addrspace(3)* %out, <2 x double> addrspace(3)* %in) {
 entry:
   %ld = load <2 x double>, <2 x double> addrspace(3)* %in, align 16
-  store <2 x double> %ld, <2 x double> addrspace(3)* %out
+  store <2 x double> %ld, <2 x double> addrspace(3)* %out, align 16
   ret void
 }
 
 attributes #0 = { nounwind }

test/CodeGen/AMDGPU/load-local-i16.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,SICIVI,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,GFX89,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,GFX89,FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
-; Testing for ds_read_b128
+; Testing for ds_read/write_b128
 ; RUN: llc -march=amdgcn -mcpu=tonga -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s
 
 ; FUNC-LABEL: {{^}}local_load_i16:
 ; GFX9-NOT: m0
 ; SICIVI: s_mov_b32 m0
 
 ; GCN: ds_read_u16 v{{[0-9]+}}

 ; ; XFUNC-LABEL: {{^}}local_sextload_v64i16_to_v64i64:
 ; define amdgpu_kernel void @local_sextload_v64i16_to_v64i64(<64 x i64> addrspace(3)* %out, <64 x i16> addrspace(3)* %in) #0 {
 ;   %load = load <64 x i16>, <64 x i16> addrspace(3)* %in
 ;   %ext = sext <64 x i16> %load to <64 x i64>
 ;   store <64 x i64> %ext, <64 x i64> addrspace(3)* %out
 ;   ret void
 ; }
 
-; Tests if ds_read_b128 gets generated for the 16 byte aligned load.
+; Tests if ds_read/write_b128 gets generated for the 16 byte aligned load.
 ; FUNC-LABEL: {{^}}local_v8i16_to_128:
+
 ; SI-NOT: ds_read_b128
+; SI-NOT: ds_write_b128
+
 ; CIVI: ds_read_b128
+; CIVI: ds_write_b128
+
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_v8i16_to_128(<8 x i16> addrspace(3)* %out, <8 x i16> addrspace(3)* %in) {
   %ld = load <8 x i16>, <8 x i16> addrspace(3)* %in, align 16
-  store <8 x i16> %ld, <8 x i16> addrspace(3)* %out
+  store <8 x i16> %ld, <8 x i16> addrspace(3)* %out, align 16
   ret void
 }
 
 attributes #0 = { nounwind }

test/CodeGen/AMDGPU/load-local-i32.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
-; Testing for ds_read_128
+; Testing for ds_read/write_128
 ; RUN: llc -march=amdgcn -mcpu=tahiti -amdgpu-ds128 < %s | FileCheck -check-prefixes=SI,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s
 
 ; FUNC-LABEL: {{^}}local_load_i32:
 ; GCN-NOT: s_wqm_b64
 ; SICIVI: s_mov_b32 m0, -1
 ; GFX9-NOT: m0

 
 define amdgpu_kernel void @local_sextload_v4i32_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i32> addrspace(3)* %in) #0 {
   %ld = load <4 x i32>, <4 x i32> addrspace(3)* %in
   %ext = sext <4 x i32> %ld to <4 x i64>
   store <4 x i64> %ext, <4 x i64> addrspace(3)* %out
   ret void
 }
 
-; Tests if ds_read_b128 gets generated for the 16 byte aligned load.
+; Tests if ds_read/write_b128 gets generated for the 16 byte aligned load.
 ; FUNC-LABEL: {{^}}local_v4i32_to_128:
+
 ; SI-NOT: ds_read_b128
+; SI-NOT: ds_write_b128
+
 ; CIVI: ds_read_b128
+; CIVI: ds_write_b128
+
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_v4i32_to_128(<4 x i32> addrspace(3)* %out, <4 x i32> addrspace(3)* %in) {
   %ld = load <4 x i32>, <4 x i32> addrspace(3)* %in, align 16
-  store <4 x i32> %ld, <4 x i32> addrspace(3)* %out
+  store <4 x i32> %ld, <4 x i32> addrspace(3)* %out, align 16
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v8i32_to_v8i64:
 ; SICIVI: s_mov_b32 m0, -1
 ; GFX9-NOT: m0
 
 define amdgpu_kernel void @local_zextload_v8i32_to_v8i64(<8 x i64> addrspace(3)* %out, <8 x i32> addrspace(3)* %in) #0 {

test/CodeGen/AMDGPU/load-local-i64.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
 
-; Testing for ds_read_b128
+; Testing for ds_read/write_b128
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s
 
 ; FUNC-LABEL: {{^}}local_load_i64:
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0
 
 ; GCN: ds_read_b64 [[VAL:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}{{$}}

 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_load_v2i64(<2 x i64> addrspace(3)* %out, <2 x i64> addrspace(3)* %in) #0 {
 entry:
   %ld = load <2 x i64>, <2 x i64> addrspace(3)* %in
   store <2 x i64> %ld, <2 x i64> addrspace(3)* %out
   ret void
 }
 
-; Tests if ds_read_b128 gets generated for the 16 byte aligned load.
+; Tests if ds_read/write_b128 gets generated for the 16 byte aligned load.
 ; FUNC-LABEL: {{^}}local_load_v2i64_to_128:
+
 ; CIVI: ds_read_b128
+; CIVI: ds_write_b128
+
 define amdgpu_kernel void @local_load_v2i64_to_128(<2 x i64> addrspace(3)* %out, <2 x i64> addrspace(3)* %in) {
 entry:
-  %ld = load <2 x i64>, <2 x i64> addrspace(3)* %in
-  store <2 x i64> %ld, <2 x i64> addrspace(3)* %out
+  %ld = load <2 x i64>, <2 x i64> addrspace(3)* %in, align 16
+  store <2 x i64> %ld, <2 x i64> addrspace(3)* %out, align 16
   ret void
 }
 
 ; FUNC-LABEL: {{^}}local_load_v3i64:
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0
 
 ; GCN-DAG: ds_read2_b64

test/CodeGen/AMDGPU/load-local-i8.ll

 ; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,SICIVI,FUNC %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,SICIVI,FUNC %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
 ; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
-; Testing for ds_read_b128
+; Testing for ds_read/write_b128
 ; RUN: llc -march=amdgcn -mcpu=tonga -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-ds128 < %s | FileCheck -check-prefixes=CIVI,FUNC %s
 
 ; FUNC-LABEL: {{^}}local_load_i8:
 ; GCN-NOT: s_wqm_b64
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0
 ; GCN: ds_read_u8

 ; XFUNC-LABEL: {{^}}local_sextload_v64i8_to_v64i16:
 ; define amdgpu_kernel void @local_sextload_v64i8_to_v64i16(<64 x i16> addrspace(3)* %out, <64 x i8> addrspace(3)* %in) #0 {
 ;   %load = load <64 x i8>, <64 x i8> addrspace(3)* %in
 ;   %ext = sext <64 x i8> %load to <64 x i16>
 ;   store <64 x i16> %ext, <64 x i16> addrspace(3)* %out
 ;   ret void
 ; }
 
-; Tests if ds_read_b128 gets generated for the 16 byte aligned load.
+; Tests if ds_read/write_b128 gets generated for the 16 byte aligned load.
 ; FUNC-LABEL: {{^}}local_v16i8_to_128:
+
 ; SI-NOT: ds_read_b128
+; SI-NOT: ds_write_b128
+
 ; CIVI: ds_read_b128
+; CIVI: ds_write_b128
+
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_v16i8_to_128(<16 x i8> addrspace(3)* %out, <16 x i8> addrspace(3)* %in) {
   %ld = load <16 x i8>, <16 x i8> addrspace(3)* %in, align 16
-  store <16 x i8> %ld, <16 x i8> addrspace(3)* %out
+  store <16 x i8> %ld, <16 x i8> addrspace(3)* %out, align 16
   ret void
 }
 
 attributes #0 = { nounwind }