AMDGPU/SI: Add llvm.amdgcn.s.buffer.load intrinsic

[airlied]

Summary

This patch also adds a new address space to represent loads from the constant
address space that use a buffer resource.

Note that this new address space does not support GEP instructions due to
limitations in the instruction selector.

Written by Tom Stellard

[airlied]

This adds support for the x2/x4 paths. Probably could be done better, and I've tested this with a few vulkan games.

[arsenm]

include/llvm/IR/IntrinsicsAMDGPU.td

471

I think these need slc as well. I believe slc started meaning something for scalar ops in VI

lib/Target/AMDGPU/AMDGPUISelLowering.h

392

Probably should be S_BUFFER_LOAD to match the instruction name pattern

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

498

cast<>, not static_cast

lib/Target/AMDGPU/SIISelLowering.cpp

2175–2193

All of this seems to just be a workaround for getTgtMemIntrinsic not setting MODerefenrencable. It would be better to just fix visitTargetIntrinsic to add this to the MMO when it first creates it.

lib/Target/AMDGPU/SMInstructions.td

280

There's no reason to give any of these patterns names

[arsenm]

test/CodeGen/AMDGPU/mubuf.ll

94

name test values

96

Why is this using the old intrinsic? It would be better to only use the new intrinsic in a single test and move any for the old intrinsic into the legacy intrinsic test file

[airlied]

include/llvm/IR/IntrinsicsAMDGPU.td

471

I don't see slc as a bit in the GCN3 docs for s_buffer_load.

lib/Target/AMDGPU/SIISelLowering.cpp

2175–2193

do you mean just the MMO code here?, the Ops changes is needed to cast the 128-bit constant address space pointer into the v4i32 the hw gets.

include/llvm/IR/IntrinsicsAMDGPU.td

 def int_amdgcn_buffer_load_format : AMDGPUBufferLoad;
 def int_amdgcn_buffer_load : AMDGPUBufferLoad;
 
+
+def int_amdgcn_s_buffer_load : Intrinsic <
+  [llvm_anyint_ty],
+  [LLVMQualPointerType<LLVMMatchType<0>, 42>,
+   llvm_i32_ty, // byte offset
+   llvm_i1_ty], // glc
+  [IntrReadMem, IntrArgMemOnly, NoCapture<0>]>;
+
 class AMDGPUBufferStore : Intrinsic <
   [],
   [llvm_anyfloat_ty,  // vdata(VGPR) -- can currently only select f32, v2f32, v4f32

lib/Target/AMDGPU/AMDGPU.h

   const static unsigned CONSTANT_BUFFER_14 = 22;
   const static unsigned CONSTANT_BUFFER_15 = 23;
 
+  const static unsigned CONSTANT_ADDRESS_W_RSRC = 42;
   // Some places use this if the address space can't be determined.
   const static unsigned UNKNOWN_ADDRESS_SPACE = ~0u;
 };

lib/Target/AMDGPU/AMDGPUISelLowering.h

   ATOMIC_DEC,
   BUFFER_LOAD,
   BUFFER_LOAD_FORMAT,
+  SBUFFER_LOAD,
   LAST_AMDGPU_ISD_NUMBER
 };
 

lib/Target/AMDGPU/AMDGPUISelLowering.cpp

   NODE_NAME_CASE(ATOMIC_DEC)
   NODE_NAME_CASE(BUFFER_LOAD)
   NODE_NAME_CASE(BUFFER_LOAD_FORMAT)
+  NODE_NAME_CASE(SBUFFER_LOAD)
   case AMDGPUISD::LAST_AMDGPU_ISD_NUMBER: break;
   }
   return nullptr;

lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

     return "e-p:64:64-p1:64:64-p2:64:64-p3:32:32-p4:32:32-p5:32:32"
          "-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128"
          "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-A5";
-  return "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32"
+  return "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p42:128:128"
       "-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128"
       "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64";
 }

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h

   }
 
   unsigned getVectorSplitCost() { return 0; }
+
+  bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info);
+
+  Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
+                                           Type *ExpectedType);
+
 };
 
 } // end namespace llvm

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

   AMDGPUAS AS = ST->getAMDGPUAS();
   if (AddrSpace == AS.GLOBAL_ADDRESS ||
       AddrSpace == AS.CONSTANT_ADDRESS ||
+      AddrSpace == AS.CONSTANT_ADDRESS_W_RSRC ||
       AddrSpace == AS.FLAT_ADDRESS)
     return 128;
   if (AddrSpace == AS.LOCAL_ADDRESS ||

 
   return false;
 }
+
+bool AMDGPUTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
+                                      MemIntrinsicInfo &Info) {
+  IRBuilder<> Builder(Inst);
+  switch (Inst->getIntrinsicID()) {
+  default:
+    return false;
+  case Intrinsic::amdgcn_s_buffer_load:
+    Info.ReadMem = true;
+    Info.WriteMem = false;
+
+    // We can only set this if the intrinsic is functionally equivalent to a
+    // load/store.
+    if (auto Offset = dyn_cast<ConstantInt>(Inst->getArgOperand(1))) {
+      if (Offset->isZero() &&
+          static_cast<ConstantInt*>(Inst->getArgOperand(2))->isZero()) {
+        Info.PtrVal = Inst->getArgOperand(0);
+      }
+    }
+    break;
+  }
+  return true;
+}
+
+Value *AMDGPUTTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
+                                                        Type *ExpectedType) {
+  if (Inst->getType() == ExpectedType)
+    return Inst;
+
+  return nullptr;
+}

lib/Target/AMDGPU/BUFInstructions.td

 
 // Offset in an 32-bit VGPR
 def : Pat <
-  (SIload_constant v4i32:$sbase, i32:$voff),
-  (BUFFER_LOAD_DWORD_OFFEN $voff, $sbase, (i32 0), 0, 0, 0, 0)
+  (i32 (SIsbuffer_load v4i32:$sbase, i32:$offset, i1:$glc)),
+  (BUFFER_LOAD_DWORD_OFFEN $offset, $sbase, (i32 0), 0, (as_i1imm $glc), 0, 0)
+>;
+
+// Offset in an 32-bit VGPR
+def : Pat <
+  (v2i32 (SIsbuffer_load v4i32:$sbase, i32:$offset, i1:$glc)),
+  (BUFFER_LOAD_DWORDX2_OFFEN $offset, $sbase, (i32 0), 0, (as_i1imm $glc), 0, 0)
+>;
+
+// Offset in an 32-bit VGPR
+def : Pat <
+  (v4i32 (SIsbuffer_load v4i32:$sbase, i32:$offset, i1:$glc)),
+  (BUFFER_LOAD_DWORDX4_OFFEN $offset, $sbase, (i32 0), 0, (as_i1imm $glc), 0, 0)
 >;
 
 

lib/Target/AMDGPU/SIISelLowering.h

                          EVT VT) const override;
   MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override;
   bool isFMAFasterThanFMulAndFAdd(EVT VT) const override;
+  void LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results,
+                             SelectionDAG &DAG) const override;
   SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
   void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
                           SelectionDAG &DAG) const override;

lib/Target/AMDGPU/SIISelLowering.cpp

 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/Loads.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/DAGCombine.h"

     Info.writeMem = true;
     return true;
   }
+  case Intrinsic::amdgcn_s_buffer_load:
+    Info.opc = AMDGPUISD::SBUFFER_LOAD;
+    Info.memVT = MVT::getVT(CI.getType());
+    Info.ptrVal = CI.getOperand(0);
+    Info.align = 0;
+    Info.vol = false;
+    Info.readMem = true;
+    Info.writeMem = false;
+    return true;
   default:
     return false;
   }

     }
 
     return isLegalMUBUFAddressingMode(AM);
-  } else if (AS == AMDGPUASI.CONSTANT_ADDRESS) {
+  } else if (AS == AMDGPUASI.CONSTANT_ADDRESS ||
+       AS == AMDGPUASI.CONSTANT_ADDRESS_W_RSRC) {
     // If the offset isn't a multiple of 4, it probably isn't going to be
     // correctly aligned.
     // FIXME: Can we get the real alignment here?

 // Custom DAG Lowering Operations
 //===----------------------------------------------------------------------===//
 
+void SITargetLowering::LowerOperationWrapper(SDNode *N,
+                                             SmallVectorImpl<SDValue> &Results,
+                                             SelectionDAG &DAG) const {
+
+  if (N->getOpcode() != AMDGPUISD::SBUFFER_LOAD) {
+    TargetLowering::LowerOperationWrapper(N, Results, DAG);
+    return;
+  }
+
+  SDLoc DL(N);
+  MemSDNode *M = cast<MemSDNode>(N);
+  SDValue Ops[] = {
+    M->getOperand(0), // Chain
+    DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, M->getOperand(1)), // Ptr
+    M->getOperand(2), // Offset
+    DAG.getTargetConstant(cast<ConstantSDNode>(
+      M->getOperand(3))->getZExtValue(), DL, MVT::i1) // glc
+  };
+
+  auto MMO = M->getMemOperand();
+  if (isDereferenceablePointer(MMO->getValue(), DAG.getDataLayout()))
+    MMO->setFlags(MachineMemOperand::MODereferenceable);
+
+  SDValue LD = DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_LOAD, DL,
+                                       M->getVTList(), Ops, M->getMemoryVT(),
+                                       M->getMemOperand());
+  Results.push_back(LD);
+  Results.push_back(LD.getValue(1));
+}
+
 SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
   default: return AMDGPUTargetLowering::LowerOperation(Op, DAG);

       TRI->getPreloadedValue(MF, SIRegisterInfo::WORKITEM_ID_Z), VT);
   case AMDGPUIntrinsic::SI_load_const: {
     SDValue Ops[] = {
-      Op.getOperand(1),
-      Op.getOperand(2)
+      DAG.getEntryNode(), // Chain
+      Op.getOperand(1), // Ptr
+      Op.getOperand(2), // Offset
+      DAG.getTargetConstant(0, DL, MVT::i1) // glc
     };
 
     MachineMemOperand *MMO = MF.getMachineMemOperand(

         MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
             MachineMemOperand::MOInvariant,
         VT.getStoreSize(), 4);
-    return DAG.getMemIntrinsicNode(AMDGPUISD::LOAD_CONSTANT, DL,
-                                   Op->getVTList(), Ops, VT, MMO);
+    SDVTList VTList = DAG.getVTList(MVT::i32, MVT::Other);
+    SDValue Load = DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_LOAD, DL,
+                                           VTList, Ops, MVT::i32, MMO);
+
+    SDValue MergeOps[] = {
+      DAG.getNode(ISD::BITCAST, DL, MVT::f32, Load),
+      Load.getValue(1)
+    };
+
+    return DAG.getMergeValues(MergeOps, DL);
   }
   case Intrinsic::amdgcn_fdiv_fast:
     return lowerFDIV_FAST(Op, DAG);

lib/Target/AMDGPU/SIInstrInfo.td

 // SI DAG Nodes
 //===----------------------------------------------------------------------===//
 
-def SIload_constant : SDNode<"AMDGPUISD::LOAD_CONSTANT",
-  SDTypeProfile<1, 2, [SDTCisVT<0, f32>, SDTCisVT<1, v4i32>, SDTCisVT<2, i32>]>,
-                      [SDNPMayLoad, SDNPMemOperand]
->;
-
 def SIatomic_inc : SDNode<"AMDGPUISD::ATOMIC_INC", SDTAtomic2,
   [SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain]
 >;

 defm atomic_inc_global : global_binary_atomic_op<SIatomic_inc>;
 defm atomic_dec_global : global_binary_atomic_op<SIatomic_dec>;
 
+def SIsbuffer_load : SDNode<"AMDGPUISD::SBUFFER_LOAD",
+  SDTypeProfile<1, 3, [SDTCisVT<1, v4i32>, SDTCisVT<2, i32>, SDTCisVT<3, i1>]>,
+  [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]
+>;
+
 //===----------------------------------------------------------------------===//
 // SDNodes and PatFrag for local loads and stores to enable s_mov_b32 m0, -1
 // to be glued to the memory instructions.

lib/Target/AMDGPU/SMInstructions.td

 defm : SMRD_Pattern <"S_LOAD_DWORDX16", v16i32>;
 
 // 1. Offset as an immediate
-def SM_LOAD_PATTERN : Pat <  // name this pattern to reuse AddedComplexity on CI
-  (SIload_constant v4i32:$sbase, (SMRDBufferImm i32:$offset)),
-  (S_BUFFER_LOAD_DWORD_IMM $sbase, $offset, 0)
+// name this pattern to reuse AddedComplexity on CI
+def SM_LOAD_PATTERN : Pat <
+  (i32 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), i1:$glc)),
+  (S_BUFFER_LOAD_DWORD_IMM $sbase, $offset, (as_i1imm $glc))
 >;
 
 // 2. Offset loaded in an 32bit SGPR
 def : Pat <
-  (SIload_constant v4i32:$sbase, (SMRDBufferSgpr i32:$offset)),
-  (S_BUFFER_LOAD_DWORD_SGPR $sbase, $offset, 0)
+  (i32 (SIsbuffer_load v4i32:$sbase, (SMRDBufferSgpr i32:$offset), i1:$glc)),
+  (S_BUFFER_LOAD_DWORD_SGPR $sbase, $offset, (as_i1imm $glc))
 >;
 
+def SM_LOAD_PATTERNX2  : Pat <
+  (v2i32 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), i1:$glc)),
+  (S_BUFFER_LOAD_DWORDX2_IMM $sbase, $offset, (as_i1imm $glc))
+>;
+
+// 2. Offset loaded in an 32bit SGPR
+def : Pat <
+  (v2i32 (SIsbuffer_load v4i32:$sbase, (SMRDBufferSgpr i32:$offset), i1:$glc)),
+  (S_BUFFER_LOAD_DWORDX2_SGPR $sbase, $offset, (as_i1imm $glc))
+>;
+
+def SM_LOAD_PATTERNX4  : Pat <
+  (v4i32 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), i1:$glc)),
+  (S_BUFFER_LOAD_DWORDX4_IMM $sbase, $offset, (as_i1imm $glc))
+>;
+
+// 2. Offset loaded in an 32bit SGPR
+def : Pat <
+  (v4i32 (SIsbuffer_load v4i32:$sbase, (SMRDBufferSgpr i32:$offset), i1:$glc)),
+  (S_BUFFER_LOAD_DWORDX4_SGPR $sbase, $offset, (as_i1imm $glc))
+>;
+
+
+
 } // End let AddedComplexity = 100
 
 } // let Predicates = [isGCN]

 def : SMRD_Pattern_ci <"S_LOAD_DWORDX16", v16i32>;
 
 def : Pat <
-  (SIload_constant v4i32:$sbase, (SMRDBufferImm32 i32:$offset)),
-  (S_BUFFER_LOAD_DWORD_IMM_ci $sbase, $offset, 0)> {
+  (i32 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), i1:$glc)),
+  (S_BUFFER_LOAD_DWORD_IMM_ci $sbase, $offset, $glc)> {
   let Predicates = [isCI]; // should this be isCIOnly?
 }
-
 } // End let AddedComplexity = SM_LOAD_PATTERN.AddedComplexity
 
+let AddedComplexity = SM_LOAD_PATTERNX2.AddedComplexity in {
+def : Pat <
+  (v2i32 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), i1:$glc)),
+  (S_BUFFER_LOAD_DWORDX2_IMM_ci $sbase, $offset, $glc)> {
+  let Predicates = [isCI]; // should this be isCIOnly?
+}
+
+} // End let AddedComplexity = SM_LOAD_PATTERNX2.AddedComplexity
+
+let AddedComplexity = SM_LOAD_PATTERNX4.AddedComplexity in {
+def : Pat <
+  (v4i32 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), i1:$glc)),
+  (S_BUFFER_LOAD_DWORDX4_IMM_ci $sbase, $offset, $glc)> {
+  let Predicates = [isCI]; // should this be isCIOnly?
+}
+
+} // End let AddedComplexity = SM_LOAD_PATTERNX4.AddedComplexity
+

test/CodeGen/AMDGPU/mubuf.ll

   ret void
 }
 
+; Using the load.const intrinsic with an vgpr offset
+; CHECK-LABEL: {{^}}s_buffer_load:
+; CHECK: buffer_load_dword v{{[0-9+]}}, v{{[0-9+]}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offen
+; CHECK: buffer_load_dword v{{[0-9+]}}, v{{[0-9+]}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offen
+define amdgpu_ps void @s_buffer_load(<16 x i8> addrspace(2)* inreg, <16 x i8> addrspace(2)* inreg, <32 x i8> addrspace(2)* inreg, i32 inreg, <2 x i32>, <2 x i32>, <2 x i32>, <3 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, float, float, float, float, float, float, float, float, float, i32 addrspace(42)* addrspace(2)* inreg %in) {
+main_body:
+  %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %20 = getelementptr <16 x i8>, <16 x i8> addrspace(2)* %0, i32 0
+  %21 = load <16 x i8>, <16 x i8> addrspace(2)* %20
+  %22 = call float @llvm.SI.load.const(<16 x i8> %21, i32 %tid)
+  %23 = load i32 addrspace(42)*, i32 addrspace(42)* addrspace(2)* %in
+  %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(i32 addrspace(42)* %23, i32 %tid, i1 false)
+  %s.buffer.float = bitcast i32 %s.buffer to float
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %22, float %22, float %22, float %s.buffer.float, i1 1, i1 0)
+  ret void
+}
+
 ;;;==========================================================================;;;
 ;;; MUBUF STORE TESTS
 ;;;==========================================================================;;;

   ret void
 }
 
+declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #1
 declare i32 @llvm.SI.buffer.load.dword.i32.i32(<16 x i8>, i32, i32, i32, i32, i32, i32, i32, i32) #0
 declare void @llvm.SI.tbuffer.store.i32(<16 x i8>, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32)
+declare float @llvm.SI.load.const(<16 x i8>, i32) #1
+declare i32 @llvm.amdgcn.s.buffer.load.i32(i32 addrspace(42)* nocapture, i32, i1)
+declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1)
 
 attributes #0 = { nounwind readonly }
+attributes #1 = { nounwind readnone }