AMDGPU: Fix dropping memref for ds append/consume

[arsenm]

Summary

The way SelectionDAG treats memory operands is very frustrating, and
by default drops them unless a property is set on the pattern. There
is no pattern for manually selected instructions, so this requires
manually setting them.

[rampitec]

LGTM

[arsenm]

r363455

include/llvm/IR/IntrinsicsAMDGPU.td

class AMDGPUDSOrderedIntrinsic : Intrinsic<
    ImmArg<5>, ImmArg<6>, ImmArg<7>
   ]
 >;
 
 class AMDGPUDSAppendConsumedIntrinsic : Intrinsic<
   [llvm_i32_ty],
   [llvm_anyptr_ty, // LDS or GDS ptr
    llvm_i1_ty], // isVolatile
-   [IntrConvergent, IntrArgMemOnly, NoCapture<0>, ImmArg<1>]
+   [IntrConvergent, IntrArgMemOnly, NoCapture<0>, ImmArg<1>],
+   "",
+   [SDNPMemOperand]
 >;
 
 def int_amdgcn_ds_ordered_add : AMDGPUDSOrderedIntrinsic;
 def int_amdgcn_ds_ordered_swap : AMDGPUDSOrderedIntrinsic;
 
 // The pointer argument is assumed to be dynamically uniform if a VGPR.
 def int_amdgcn_ds_append : AMDGPUDSAppendConsumedIntrinsic;
 def int_amdgcn_ds_consume : AMDGPUDSAppendConsumedIntrinsic;

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
   // The address is assumed to be uniform, so if it ends up in a VGPR, it will
   // be copied to an SGPR with readfirstlane.
   unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ?
     AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
 
   SDValue Chain = N->getOperand(0);
   SDValue Ptr = N->getOperand(2);
   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
+  MachineMemOperand *MMO = M->getMemOperand();
   bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
 
   SDValue Offset;
   if (CurDAG->isBaseWithConstantOffset(Ptr)) {
     SDValue PtrBase = Ptr.getOperand(0);
     SDValue PtrOffset = Ptr.getOperand(1);
 
     const APInt &OffsetVal = cast<ConstantSDNode>(PtrOffset)->getAPIntValue();
void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
 
   SDValue Ops[] = {
     Offset,
     CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32),
     Chain,
     N->getOperand(N->getNumOperands() - 1) // New glue
   };
 
-  CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
+  SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
+  CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
 }
 
 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) {
   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
   switch (IntrID) {
   case Intrinsic::amdgcn_ds_append:
   case Intrinsic::amdgcn_ds_consume: {
     if (N->getValueType(0) != MVT::i32)

test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll

 ; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,NOTGFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CIPLUS,NOTGFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CIPLUS,NOTGFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CIPLUS,GFX9 %s
 
 ; GCN-LABEL: {{^}}ds_append_lds:
 ; GCN: s_load_dword [[PTR:s[0-9]+]]
 ; GCN: s_mov_b32 m0, [[PTR]]
 ; GCN: ds_append [[RESULT:v[0-9]+]]{{$}}
+; GCN-NOT: buffer_wbinvl1
 ; GCN: {{.*}}store{{.*}} [[RESULT]]
 define amdgpu_kernel void @ds_append_lds(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
   %val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %lds, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_append_lds_max_offset:
 ; GCN: s_load_dword [[PTR:s[0-9]+]]
 ; GCN: s_mov_b32 m0, [[PTR]]
 ; GCN: ds_append [[RESULT:v[0-9]+]] offset:65532{{$}}
+; GCN-NOT: buffer_wbinvl1
 ; GCN: {{.*}}store{{.*}} [[RESULT]]
 define amdgpu_kernel void @ds_append_lds_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16383
   %val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %gep, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_append_no_fold_offset_si:
 ; GCN: s_load_dword [[PTR:s[0-9]+]]
 
 ; SI: s_add_i32 [[PTR]], [[PTR]], 16
 ; SI: s_mov_b32 m0, [[PTR]]
 ; SI: ds_append [[RESULT:v[0-9]+]]{{$}}
 
 ; CIPLUS: s_mov_b32 m0, [[PTR]]
 ; CIPLUS: ds_append [[RESULT:v[0-9]+]] offset:16{{$}}
 
+; GCN-NOT: buffer_wbinvl1
 ; GCN: {{.*}}store{{.*}} [[RESULT]]
 define amdgpu_kernel void @ds_append_no_fold_offset_si(i32 addrspace(3)* addrspace(4)* %lds.ptr, i32 addrspace(1)* %out) #0 {
   %lds = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(4)* %lds.ptr, align 4
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 4
   %val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %gep, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_append_lds_over_max_offset:
 ; GCN: s_load_dword [[PTR:s[0-9]+]]
 
 ; SI: s_bitset1_b32 [[PTR]], 16
 ; CIPLUS: s_add_i32 [[PTR]], [[PTR]], 0x10000
 
 ; GCN: s_mov_b32 m0, [[PTR]]
 ; GCN: ds_append [[RESULT:v[0-9]+]]{{$}}
+; GCN-NOT: buffer_wbinvl1
 ; GCN: {{.*}}store{{.*}} [[RESULT]]
 define amdgpu_kernel void @ds_append_lds_over_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16384
   %val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %gep, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_append_lds_vgpr_addr:
 ; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
 ; GCN: s_mov_b32 m0, [[READLANE]]
 ; GCN: ds_append [[RESULT:v[0-9]+]]{{$}}
+; GCN-NOT: buffer_wbinvl1
 ; GCN: {{.*}}store{{.*}} [[RESULT]]
 define void @ds_append_lds_vgpr_addr(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
   %val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %lds, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_append_gds:
 ; GCN: s_load_dword [[PTR:s[0-9]+]]
 ; GCN: s_mov_b32 m0, [[PTR]]
 ; GCN: ds_append [[RESULT:v[0-9]+]] gds{{$}}
+; GCN-NOT: buffer_wbinvl1
 ; GCN: {{.*}}store{{.*}} [[RESULT]]
 define amdgpu_kernel void @ds_append_gds(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
   %val = call i32 @llvm.amdgcn.ds.append.p2i32(i32 addrspace(2)* %gds, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_append_gds_max_offset:
 ; GCN: s_load_dword [[PTR:s[0-9]+]]
 ; GCN: s_mov_b32 m0, [[PTR]]
 ; GCN: ds_append [[RESULT:v[0-9]+]] offset:65532 gds{{$}}
+; GCN-NOT: buffer_wbinvl1
 ; GCN: {{.*}}store{{.*}} [[RESULT]]
 define amdgpu_kernel void @ds_append_gds_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
   %gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16383
   %val = call i32 @llvm.amdgcn.ds.append.p2i32(i32 addrspace(2)* %gep, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_append_gds_over_max_offset:
+; GCN-NOT: buffer_wbinvl1
 define amdgpu_kernel void @ds_append_gds_over_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
   %gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16384
   %val = call i32 @llvm.amdgcn.ds.append.p2i32(i32 addrspace(2)* %gep, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_append_lds_m0_restore:
 ; GCN: s_load_dword [[PTR:s[0-9]+]]
 ; GCN: s_mov_b32 m0, [[PTR]]
 ; GCN: ds_append [[RESULT:v[0-9]+]]{{$}}
+; GCN-NOT: buffer_wbinvl1
 ; NOTGFX9: s_mov_b32 m0, -1
 ; GFX9-NOT: m0
 ; GCN: _store_dword
 ; GCN: ds_read_b32
 define amdgpu_kernel void @ds_append_lds_m0_restore(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
   %val0 = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %lds, i1 false)
   store i32 %val0, i32 addrspace(1)* %out
   %val1 = load volatile i32, i32 addrspace(3)* %lds
   ret void
 }
 
 declare i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* nocapture, i1) #1
 declare i32 @llvm.amdgcn.ds.append.p2i32(i32 addrspace(2)* nocapture, i1) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { argmemonly convergent nounwind }

test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll

 ; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,NOTGFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CIPLUS,NOTGFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CIPLUS,NOTGFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CIPLUS,GFX9 %s
 
 ; GCN-LABEL: {{^}}ds_consume_lds:
 ; GCN: s_load_dword [[PTR:s[0-9]+]]
 ; GCN: s_mov_b32 m0, [[PTR]]
 ; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}}
+; GCN-NOT: buffer_wbinvl1
 ; GCN: {{.*}}store{{.*}} [[RESULT]]
 define amdgpu_kernel void @ds_consume_lds(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
   %val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %lds, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_consume_lds_max_offset:
 ; GCN: s_load_dword [[PTR:s[0-9]+]]
 ; GCN: s_mov_b32 m0, [[PTR]]
 ; GCN: ds_consume [[RESULT:v[0-9]+]] offset:65532{{$}}
+; GCN-NOT: buffer_wbinvl1
 ; GCN: {{.*}}store{{.*}} [[RESULT]]
 define amdgpu_kernel void @ds_consume_lds_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16383
   %val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %gep, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_consume_no_fold_offset_si:
 ; GCN: s_load_dword [[PTR:s[0-9]+]]
 
 ; SI: s_add_i32 [[PTR]], [[PTR]], 16
 ; SI: s_mov_b32 m0, [[PTR]]
 ; SI: ds_consume [[RESULT:v[0-9]+]]{{$}}
 
 ; CIPLUS: s_mov_b32 m0, [[PTR]]
 ; CIPLUS: ds_consume [[RESULT:v[0-9]+]] offset:16{{$}}
 
+; GCN-NOT: buffer_wbinvl1
 ; GCN: {{.*}}store{{.*}} [[RESULT]]
 define amdgpu_kernel void @ds_consume_no_fold_offset_si(i32 addrspace(3)* addrspace(4)* %lds.ptr, i32 addrspace(1)* %out) #0 {
   %lds = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(4)* %lds.ptr, align 4
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 4
   %val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %gep, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_consume_lds_over_max_offset:
 ; GCN: s_load_dword [[PTR:s[0-9]+]]
 
 ; SI: s_bitset1_b32 [[PTR]], 16
 ; CIPLUS: s_add_i32 [[PTR]], [[PTR]], 0x10000
 
 ; GCN: s_mov_b32 m0, [[PTR]]
 ; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}}
+; GCN-NOT: buffer_wbinvl1
 ; GCN: {{.*}}store{{.*}} [[RESULT]]
 define amdgpu_kernel void @ds_consume_lds_over_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
   %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16384
   %val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %gep, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_consume_lds_vgpr_addr:
 ; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
 ; GCN: s_mov_b32 m0, [[READLANE]]
 ; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}}
+; GCN-NOT: buffer_wbinvl1
 ; GCN: {{.*}}store{{.*}} [[RESULT]]
 define void @ds_consume_lds_vgpr_addr(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
   %val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %lds, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_consume_gds:
 ; GCN: s_load_dword [[PTR:s[0-9]+]]
 ; GCN: s_mov_b32 m0, [[PTR]]
 ; GCN: ds_consume [[RESULT:v[0-9]+]] gds{{$}}
+; GCN-NOT: buffer_wbinvl1
 ; GCN: {{.*}}store{{.*}} [[RESULT]]
 define amdgpu_kernel void @ds_consume_gds(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
   %val = call i32 @llvm.amdgcn.ds.consume.p2i32(i32 addrspace(2)* %gds, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_consume_gds_max_offset:
 ; GCN: s_load_dword [[PTR:s[0-9]+]]
 ; GCN: s_mov_b32 m0, [[PTR]]
 ; GCN: ds_consume [[RESULT:v[0-9]+]] offset:65532 gds{{$}}
+; GCN-NOT: buffer_wbinvl1
 ; GCN: {{.*}}store{{.*}} [[RESULT]]
 define amdgpu_kernel void @ds_consume_gds_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
   %gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16383
   %val = call i32 @llvm.amdgcn.ds.consume.p2i32(i32 addrspace(2)* %gep, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_consume_gds_over_max_offset:
+; GCN-NOT: buffer_wbinvl1
 define amdgpu_kernel void @ds_consume_gds_over_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
   %gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16384
   %val = call i32 @llvm.amdgcn.ds.consume.p2i32(i32 addrspace(2)* %gep, i1 false)
   store i32 %val, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}ds_consume_lds_m0_restore:
 ; GCN: s_load_dword [[PTR:s[0-9]+]]
 ; GCN: s_mov_b32 m0, [[PTR]]
 ; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}}
+; GCN-NOT: buffer_wbinvl1
 ; NOTGFX9: s_mov_b32 m0, -1
 ; GFX9-NOT: m0
 ; GCN: _store_dword
 ; GCN: ds_read_b32
 define amdgpu_kernel void @ds_consume_lds_m0_restore(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
   %val0 = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %lds, i1 false)
   store i32 %val0, i32 addrspace(1)* %out
   %val1 = load volatile i32, i32 addrspace(3)* %lds
   ret void
 }
 
 declare i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* nocapture, i1) #1
 declare i32 @llvm.amdgcn.ds.consume.p2i32(i32 addrspace(2)* nocapture, i1) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { argmemonly convergent nounwind }