AMDGPU/R600: Expand unaligned writes to local and global AS

[jvesely]

Summary

DAGCombiner can pass us unaligned stores.

[jvesely]

fix store_v4i16 test

[arsenm]

lib/Target/AMDGPU/R600ISelLowering.cpp

1131

C++ style comments

1142–1143

Ditto

1151–1152

These can be in one && assert

[jvesely]

fix c style comments

[jvesely]

lib/Target/AMDGPU/R600ISelLowering.cpp

1151–1153

I'd prefer to keep them separate. the assertion message clearly indicates what part failed

[arsenm]

LGTM

lib/Target/AMDGPU/R600ISelLowering.cpp

return DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other,
                      DAG.getTargetConstant(0, DL, MVT::i32));
 }
 
 SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
   StoreSDNode *StoreNode = cast<StoreSDNode>(Op);
   unsigned AS = StoreNode->getAddressSpace();
   SDValue Value = StoreNode->getValue();
   EVT ValueVT = Value.getValueType();
+  EVT MemVT = StoreNode->getMemoryVT();
+  unsigned Align = StoreNode->getAlignment();
 
   if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS) &&
       ValueVT.isVector()) {
     return SplitVectorStore(Op, DAG);
   }
 
+  /* private AS needs special fixes */
+  if (Align < MemVT.getStoreSize() && (AS != AMDGPUAS::PRIVATE_ADDRESS) &&
+      !allowsMisalignedMemoryAccesses(MemVT, AS, Align, NULL)) {
+    return expandUnalignedStore(StoreNode, DAG);
+  }
+
   SDLoc DL(Op);
   SDValue Chain = StoreNode->getChain();
   SDValue Ptr = StoreNode->getBasePtr();
 
   if (AS == AMDGPUAS::GLOBAL_ADDRESS) {
+    /* It is beneficial to create MSKOR here instead of combiner to avoid
+     * artificial dependencies introduced by RMW */
     if (StoreNode->isTruncatingStore()) {
       EVT VT = Value.getValueType();
       assert(VT.bitsLE(MVT::i32));
-      EVT MemVT = StoreNode->getMemoryVT();
       SDValue MaskConstant;
       if (MemVT == MVT::i8) {
         MaskConstant = DAG.getConstant(0xFF, DL, MVT::i32);
       } else {
         assert(MemVT == MVT::i16);
+        assert(StoreNode->getAlignment() >= 2);
         MaskConstant = DAG.getConstant(0xFFFF, DL, MVT::i32);
       }
       SDValue DWordAddr = DAG.getNode(ISD::SRL, DL, VT, Ptr,
                                       DAG.getConstant(2, DL, MVT::i32));
       SDValue ByteIndex = DAG.getNode(ISD::AND, DL, Ptr.getValueType(), Ptr,
                                       DAG.getConstant(0x00000003, DL, VT));
       SDValue TruncValue = DAG.getNode(ISD::AND, DL, VT, Value, MaskConstant);
       SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, ByteIndex,
                                    DAG.getConstant(3, DL, VT));
if (StoreNode->isTruncatingStore()) {
       }
       return Chain;
     }
   }
 
   if (AS != AMDGPUAS::PRIVATE_ADDRESS)
     return SDValue();
 
-  EVT MemVT = StoreNode->getMemoryVT();
   if (MemVT.bitsLT(MVT::i32))
     return lowerPrivateTruncStore(StoreNode, DAG);
 
   // Lowering for indirect addressing
   const MachineFunction &MF = DAG.getMachineFunction();
   const R600FrameLowering *TFL = getSubtarget()->getFrameLowering();
   unsigned StackWidth = TFL->getStackWidth(MF);
 

test/CodeGen/AMDGPU/store-global.ll

 ; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}store_i1:
 ; EG: MEM_RAT MSKOR
+; EG-NOT: MEM_RAT MSKOR
+
+; CM: MEM_RAT MSKOR
+; CM-NOT: MEM_RAT MSKOR
+
 ; SI: buffer_store_byte
 define void @store_i1(i1 addrspace(1)* %out) {
 entry:
   store i1 true, i1 addrspace(1)* %out
   ret void
 }
 
 ; i8 store
 ; FUNC-LABEL: {{^}}store_i8:
 ; EG: MEM_RAT MSKOR T[[RW_GPR:[0-9]]].XW, T{{[0-9]}}.X
+; EG-NOT: MEM_RAT MSKOR
 
 ; IG 0: Get the byte index and truncate the value
 ; EG: AND_INT * T{{[0-9]}}.[[BI_CHAN:[XYZW]]], KC0[2].Y, literal.x
 ; EG: LSHL T{{[0-9]}}.[[SHIFT_CHAN:[XYZW]]], PV.[[BI_CHAN]], literal.x
 ; EG: AND_INT * T{{[0-9]}}.[[TRUNC_CHAN:[XYZW]]], KC0[2].Z, literal.y
 ; EG-NEXT: 3(4.203895e-45), 255(3.573311e-43)
 
 

 entry:
   store i8 %in, i8 addrspace(1)* %out
   ret void
 }
 
 ; i16 store
 ; FUNC-LABEL: {{^}}store_i16:
 ; EG: MEM_RAT MSKOR T[[RW_GPR:[0-9]]].XW, T{{[0-9]}}.X
+; EG-NOT: MEM_RAT MSKOR
 
 ; IG 0: Get the byte index and truncate the value
 
 
 ; EG: AND_INT * T{{[0-9]}}.[[BI_CHAN:[XYZW]]], KC0[2].Y, literal.x
 ; EG-NEXT: 3(4.203895e-45),
 
 ; EG: LSHL T{{[0-9]}}.[[SHIFT_CHAN:[XYZW]]], PV.[[BI_CHAN]], literal.x
entry:
   store i16 %in, i16 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}store_i24:
 ; SI: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 16
 ; SI-DAG: buffer_store_byte
 ; SI-DAG: buffer_store_short
+
+; EG: MEM_RAT MSKOR
+; EG: MEM_RAT MSKOR
 define void @store_i24(i24 addrspace(1)* %out, i24 %in) {
 entry:
   store i24 %in, i24 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}store_i25:
 ; SI: s_and_b32 [[AND:s[0-9]+]], s{{[0-9]+}}, 0x1ffffff{{$}}
 ; SI: v_mov_b32_e32 [[VAND:v[0-9]+]], [[AND]]
 ; SI: buffer_store_dword [[VAND]]
+
+; EG: MEM_RAT_CACHELESS STORE_RAW
+; EG-NOT: MEM_RAT
+
+; CM: MEM_RAT_CACHELESS STORE_DWORD
+; CM-NOT: MEM_RAT
 define void @store_i25(i25 addrspace(1)* %out, i25 %in) {
 entry:
   store i25 %in, i25 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}store_v2i8:
+; v2i8 is naturally 2B aligned
 ; EG: MEM_RAT MSKOR
 ; EG-NOT: MEM_RAT MSKOR
 
+; CM: MEM_RAT MSKOR
+; CM-NOT: MEM_RAT MSKOR
+
 ; SI: buffer_store_short
 define void @store_v2i8(<2 x i8> addrspace(1)* %out, <2 x i32> %in) {
 entry:
   %0 = trunc <2 x i32> %in to <2 x i8>
   store <2 x i8> %0, <2 x i8> addrspace(1)* %out
   ret void
 }
 
+; FUNC-LABEL: {{^}}store_v2i8_unaligned:
+; EG: MEM_RAT MSKOR
+; EG: MEM_RAT MSKOR
+; EG-NOT: MEM_RAT MSKOR
+
+; CM: MEM_RAT MSKOR
+; CM: MEM_RAT MSKOR
+; CM-NOT: MEM_RAT MSKOR
+
+; SI: buffer_store_byte
+define void @store_v2i8_unaligned(<2 x i8> addrspace(1)* %out, <2 x i32> %in) {
+entry:
+  %0 = trunc <2 x i32> %in to <2 x i8>
+  store <2 x i8> %0, <2 x i8> addrspace(1)* %out, align 1
+  ret void
+}
+
 
 ; FUNC-LABEL: {{^}}store_v2i16:
 ; EG: MEM_RAT_CACHELESS STORE_RAW
 
 ; CM: MEM_RAT_CACHELESS STORE_DWORD
 
 ; SI: buffer_store_dword
 define void @store_v2i16(<2 x i16> addrspace(1)* %out, <2 x i32> %in) {
 entry:
   %0 = trunc <2 x i32> %in to <2 x i16>
   store <2 x i16> %0, <2 x i16> addrspace(1)* %out
   ret void
 }
 
+; FUNC-LABEL: {{^}}store_v2i16_unaligned:
+; EG: MEM_RAT MSKOR
+; EG: MEM_RAT MSKOR
+; EG-NOT: MEM_RAT MSKOR
+; EG-NOT: MEM_RAT_CACHELESS STORE_RAW
+
+; CM: MEM_RAT MSKOR
+; CM: MEM_RAT MSKOR
+; CM-NOT: MEM_RAT MSKOR
+; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD
+
+; SI: buffer_store_short
+; SI: buffer_store_short
+define void @store_v2i16_unaligned(<2 x i16> addrspace(1)* %out, <2 x i32> %in) {
+entry:
+  %0 = trunc <2 x i32> %in to <2 x i16>
+  store <2 x i16> %0, <2 x i16> addrspace(1)* %out, align 2
+  ret void
+}
+
 ; FUNC-LABEL: {{^}}store_v4i8:
 ; EG: MEM_RAT_CACHELESS STORE_RAW
 
 ; CM: MEM_RAT_CACHELESS STORE_DWORD
 
 ; SI: buffer_store_dword
 define void @store_v4i8(<4 x i8> addrspace(1)* %out, <4 x i32> %in) {
 entry:
   %0 = trunc <4 x i32> %in to <4 x i8>
   store <4 x i8> %0, <4 x i8> addrspace(1)* %out
   ret void
 }
 
+; FUNC-LABEL: {{^}}store_v4i8_unaligned:
+; EG: MEM_RAT MSKOR
+; EG: MEM_RAT MSKOR
+; EG: MEM_RAT MSKOR
+; EG: MEM_RAT MSKOR
+; EG-NOT: MEM_RAT MSKOR
+; EG-NOT: MEM_RAT_CACHELESS STORE_RAW
+
+; CM: MEM_RAT MSKOR
+; CM: MEM_RAT MSKOR
+; CM: MEM_RAT MSKOR
+; CM: MEM_RAT MSKOR
+; CM-NOT: MEM_RAT MSKOR
+; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD
+
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI: buffer_store_byte
+; SI-NOT: buffer_store_dword
+define void @store_v4i8_unaligned(<4 x i8> addrspace(1)* %out, <4 x i32> %in) {
+entry:
+  %0 = trunc <4 x i32> %in to <4 x i8>
+  store <4 x i8> %0, <4 x i8> addrspace(1)* %out, align 1
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_v4i8_halfaligned:
+; EG: MEM_RAT MSKOR
+; EG: MEM_RAT MSKOR
+; EG-NOT: MEM_RAT MSKOR
+; EG-NOT: MEM_RAT_CACHELESS STORE_RAW
+
+; CM: MEM_RAT MSKOR
+; CM: MEM_RAT MSKOR
+; CM-NOT: MEM_RAT MSKOR
+; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD
+
+; SI: buffer_store_short
+; SI: buffer_store_short
+; SI-NOT: buffer_store_dword
+define void @store_v4i8_halfaligned(<4 x i8> addrspace(1)* %out, <4 x i32> %in) {
+entry:
+  %0 = trunc <4 x i32> %in to <4 x i8>
+  store <4 x i8> %0, <4 x i8> addrspace(1)* %out, align 2
+  ret void
+}
+
 ; floating-point store
 ; FUNC-LABEL: {{^}}store_f32:
 ; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+\.X, T[0-9]+\.X}}, 1
 
 ; CM: MEM_RAT_CACHELESS STORE_DWORD T{{[0-9]+\.X, T[0-9]+\.X}}
 
 ; SI: buffer_store_dword
 
 define void @store_f32(float addrspace(1)* %out, float %in) {
   store float %in, float addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}store_v4i16:
-; MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+}}.XYZW
+; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+}}.XY
+
+; CM: MEM_RAT_CACHELESS STORE_DWORD T{{[0-9]+}}
 
 ; SI: buffer_store_dwordx2
 define void @store_v4i16(<4 x i16> addrspace(1)* %out, <4 x i32> %in) {
 entry:
   %0 = trunc <4 x i32> %in to <4 x i16>
   store <4 x i16> %0, <4 x i16> addrspace(1)* %out
   ret void
 }

 
 ; SI: buffer_store_dwordx4
 define void @store_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %in) {
 entry:
   store <4 x i32> %in, <4 x i32> addrspace(1)* %out
   ret void
 }
 
+; FUNC-LABEL: {{^}}store_v4i32_unaligned:
+; EG: MEM_RAT_CACHELESS STORE_RAW {{T[0-9]+\.XYZW}}
+; EG-NOT: MEM_RAT_CACHELESS STORE_RAW
+
+; CM: MEM_RAT_CACHELESS STORE_DWORD
+; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD
+
+; SI: buffer_store_dwordx4
+define void @store_v4i32_unaligned(<4 x i32> addrspace(1)* %out, <4 x i32> %in) {
+entry:
+  store <4 x i32> %in, <4 x i32> addrspace(1)* %out, align 4
+  ret void
+}
+
 ; v4f32 store
 ; FUNC-LABEL: {{^}}store_v4f32:
 ; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+\.XYZW, T[0-9]+\.X}}, 1
 ; EG-NOT: MEM_RAT_CACHELESS STORE_RAW
 
 ; CM: MEM_RAT_CACHELESS STORE_DWORD
 ; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD
 
 ; SI: buffer_store_dwordx4
 define void @store_v4f32(<4 x float> addrspace(1)* %out, <4 x float> addrspace(1)* %in) {
   %1 = load <4 x float>, <4 x float> addrspace(1) * %in
   store <4 x float> %1, <4 x float> addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}store_i64_i8:
 ; EG: MEM_RAT MSKOR
+
+; CM: MEM_RAT MSKOR
+
 ; SI: buffer_store_byte
 define void @store_i64_i8(i8 addrspace(1)* %out, i64 %in) {
 entry:
   %0 = trunc i64 %in to i8
   store i8 %0, i8 addrspace(1)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}store_i64_i16:
 ; EG: MEM_RAT MSKOR
 ; SI: buffer_store_short
 define void @store_i64_i16(i16 addrspace(1)* %out, i64 %in) {
 entry:
   %0 = trunc i64 %in to i16
   store i16 %0, i16 addrspace(1)* %out
   ret void
 }
 
 ; The stores in this function are combined by the optimizer to create a
-; 64-bit store with 32-bit alignment.  This is legal for SI and the legalizer
+; 64-bit store with 32-bit alignment.  This is legal and the legalizer
 ; should not try to split the 64-bit store back into 2 32-bit stores.
-;
-; Evergreen / Northern Islands don't support 64-bit stores yet, so there should
-; be two 32-bit stores.
 
 ; FUNC-LABEL: {{^}}vecload2:
-; EG: MEM_RAT_CACHELESS STORE_RAW
+; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+\.XY, T[0-9]+\.X}}, 1
+; EG-NOT: MEM_RAT_CACHELESS STORE_RAW
 
 ; CM: MEM_RAT_CACHELESS STORE_DWORD
+; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD
 
 ; SI: buffer_store_dwordx2
 define void @vecload2(i32 addrspace(1)* nocapture %out, i32 addrspace(2)* nocapture %mem) #0 {
 entry:
   %0 = load i32, i32 addrspace(2)* %mem, align 4
   %arrayidx1.i = getelementptr inbounds i32, i32 addrspace(2)* %mem, i64 1
   %1 = load i32, i32 addrspace(2)* %arrayidx1.i, align 4
   store i32 %0, i32 addrspace(1)* %out, align 4

test/CodeGen/AMDGPU/store-local.ll

 ; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}store_local_i1:
 ; EG: LDS_BYTE_WRITE
+
+; CM: LDS_BYTE_WRITE
+
 ; SI: ds_write_b8
 define void @store_local_i1(i1 addrspace(3)* %out) {
 entry:
   store i1 true, i1 addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}store_local_i8:
 ; EG: LDS_BYTE_WRITE
 
+; CM: LDS_BYTE_WRITE
+
 ; SI: ds_write_b8
 define void @store_local_i8(i8 addrspace(3)* %out, i8 %in) {
   store i8 %in, i8 addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}store_local_i16:
 ; EG: LDS_SHORT_WRITE
 
+; CM: LDS_SHORT_WRITE
+
 ; SI: ds_write_b16
 define void @store_local_i16(i16 addrspace(3)* %out, i16 %in) {
   store i16 %in, i16 addrspace(3)* %out
   ret void
 }
 
 ; FUNC-LABEL: {{^}}store_local_v2i16:
 ; EG: LDS_WRITE

 
 ; SI: ds_write_b32
 define void @store_local_v4i8(<4 x i8> addrspace(3)* %out, <4 x i8> %in) {
 entry:
   store <4 x i8> %in, <4 x i8> addrspace(3)* %out
   ret void
 }
 
+; FUNC-LABEL: {{^}}store_local_v4i8_unaligned:
+; EG: LDS_BYTE_WRITE
+; EG: LDS_BYTE_WRITE
+; EG: LDS_BYTE_WRITE
+; EG: LDS_BYTE_WRITE
+; EG-NOT: LDS_WRITE
+
+; CM: LDS_BYTE_WRITE
+; CM: LDS_BYTE_WRITE
+; CM: LDS_BYTE_WRITE
+; CM: LDS_BYTE_WRITE
+; CM-NOT: LDS_WRITE
+
+; SI: ds_write_b8
+; SI: ds_write_b8
+; SI: ds_write_b8
+; SI: ds_write_b8
+define void @store_local_v4i8_unaligned(<4 x i8> addrspace(3)* %out, <4 x i8> %in) {
+entry:
+  store <4 x i8> %in, <4 x i8> addrspace(3)* %out, align 1
+  ret void
+}
+
+; FUNC-LABEL: {{^}}store_local_v4i8_halfaligned:
+; EG: LDS_SHORT_WRITE
+; EG: LDS_SHORT_WRITE
+; EG-NOT: LDS_WRITE
+
+; CM: LDS_SHORT_WRITE
+; CM: LDS_SHORT_WRITE
+; CM-NOT: LDS_WRITE
+
+; SI: ds_write_b16
+; SI: ds_write_b16
+define void @store_local_v4i8_halfaligned(<4 x i8> addrspace(3)* %out, <4 x i8> %in) {
+entry:
+  store <4 x i8> %in, <4 x i8> addrspace(3)* %out, align 2
+  ret void
+}
+
 ; FUNC-LABEL: {{^}}store_local_v2i32:
 ; EG: LDS_WRITE
 ; EG: LDS_WRITE
+; EG-NOT: LDS_WRITE
 
 ; CM: LDS_WRITE
 ; CM: LDS_WRITE
+; CM-NOT: LDS_WRITE
 
 ; SI: ds_write_b64
 define void @store_local_v2i32(<2 x i32> addrspace(3)* %out, <2 x i32> %in) {
 entry:
   store <2 x i32> %in, <2 x i32> addrspace(3)* %out
   ret void
 }