AMDGPU/R600: Implement loads from constant AS

[jvesely]

Summary

Everything is in place

[• tstellarAMD]

lib/Target/AMDGPU/R600ISelLowering.cpp

1070

Why can't you lower this directly to an immediate instead of using the CONST_DATA_PTR node?

[jvesely]

rename IMM_CONST_ADDRESS -> IMM_GLOBAL_ADDRESS
add tests for D19790

[jvesely]

lib/Target/AMDGPU/R600ISelLowering.cpp

1070

I read somewhere that conversion from GA to TGA needed some kind of wrapper pseudo instruction and CONST_DATA_PTR filled the spot.

I tried using:
Pat<
(globaladdress:$a)
(MOV_IMM_CONST_ADDR tglobaladdress:$a)>;
to eliminate Lowering conversion entirely but that produced:
%vreg1<def> = MOV_IMM_CONST_ADDR %vreg1; R600_Reg32:%vreg1
I guess conversions in patterns don't work.

I also tried using:
Pat<
(tglobaladdress:$a)
(MOV_IMM_CONST_ADDR tglobaladdress:$a)>;
and lower to TGA, but that hit assertions in a lot of places that expect register operands and the easiest way to convert TGA to a register was a wrapper node.
%vreg2<def> = ADD_INT 0, 0, 1, 0, 0, 0, <ga:@float_gv>, 0, 0, 0, -1, %vreg0, 0, 0, 0, -1, 1, pred:%PRED_SEL_OFF, 0, 0; R600_TReg32_X:%vreg2 R600_Reg32:%vreg0

[• tstellarAMD]

lib/Target/AMDGPU/R600ISelLowering.cpp

1070

You are correct about the wrapper, but what I'm saying is why even emit a GlobalAddress at all. Why not do return DAG.getConstant(0, MVT::i32), like we do when lowering LDS global addresses.

[jvesely]

lib/Target/AMDGPU/R600ISelLowering.cpp

1070

the value is not zero in this case.
RO data is appended at the end of code section so we need that offset at least.
even if we used separate rodata section for r600 the address would be 0 only for the first global variable, the other variables would still need section relative addresses (that's what fixup in D19791 does).

LDS path computes section offset in lowering, which is OK for uninitialized variables (or if we emit initialization code somewhere). I don't think it'd work with const data, unless we force rodata layout to use the lowered addresses.
I think we can move the LDS approach to use targetglobaladdress and fixups, it would fix the TODO about minimizing wasted space in LDS.

[• tstellarAMD]

LGTM.

[jvesely]

rebase on top of changed D19789

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

 //===----------------------------------------------------------------------===//
 
 #include "AMDGPUInstrInfo.h"
 #include "AMDGPUIntrinsicInfo.h"
 #include "AMDGPUISelLowering.h" // For AMDGPUISD
 #include "AMDGPUSubtarget.h"
 #include "SIISelLowering.h"
 #include "SIMachineFunctionInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/CodeGen/FunctionLoweringInfo.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/PseudoSourceValue.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/IR/DiagnosticInfo.h"
 
 using namespace llvm;
if (CbId == -1)
     return checkType(MemVal, AMDGPUAS::CONSTANT_ADDRESS);
 
   return checkType(MemVal, AMDGPUAS::CONSTANT_BUFFER_0 + CbId);
 }
 
 bool AMDGPUDAGToDAGISel::isGlobalLoad(const MemSDNode *N) const {
   if (!N->readMem())
     return false;
-  if (N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS)
-    if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS ||
-        N->getMemoryVT().bitsLT(MVT::i32))
+  if (N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS) {
+    if (Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS)
+      return !isa<GlobalValue>(
+        GetUnderlyingObject(N->getMemOperand()->getValue(),
+  CurDAG->getDataLayout()));
+
+    //TODO: Why do we need this?
+    if (N->getMemoryVT().bitsLT(MVT::i32))
       return true;
+  }
 
   return checkType(N->getMemOperand()->getValue(), AMDGPUAS::GLOBAL_ADDRESS);
 }
 
 bool AMDGPUDAGToDAGISel::isParamLoad(const LoadSDNode *N) const {
   return checkType(N->getMemOperand()->getValue(), AMDGPUAS::PARAM_I_ADDRESS);
 }
 

lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp

for (MachineBasicBlock::instr_iterator E = MBB.instr_end(); I != E;) {
         I++;
       }
       for (unsigned i = 0, e = Literals.size(); i < e; i += 2) {
         MachineInstrBuilder MILit = BuildMI(MBB, I, I->getDebugLoc(),
             TII->get(AMDGPU::LITERALS));
         if (Literals[i]->isImm()) {
             MILit.addImm(Literals[i]->getImm());
         } else {
-            MILit.addImm(0);
+            MILit.addGlobalAddress(Literals[i]->getGlobal(),
+                                   Literals[i]->getOffset());
         }
         if (i + 1 < e) {
           if (Literals[i + 1]->isImm()) {
             MILit.addImm(Literals[i + 1]->getImm());
           } else {
-            MILit.addImm(0);
+            MILit.addGlobalAddress(Literals[i + 1]->getGlobal(),
+                                   Literals[i + 1]->getOffset());
           }
         } else
           MILit.addImm(0);
         ClauseContent.push_back(MILit);
       }
     }
     assert(ClauseContent.size() < 128 && "ALU clause is too big");
     ClauseHead->getOperand(7).setImm(ClauseContent.size() - 1);

lib/Target/AMDGPU/R600ISelLowering.cpp

case AMDGPU::MOV_IMM_F32:
     TII->buildMovImm(*BB, I, MI->getOperand(0).getReg(),
                      MI->getOperand(1).getFPImm()->getValueAPF()
                          .bitcastToAPInt().getZExtValue());
     break;
   case AMDGPU::MOV_IMM_I32:
     TII->buildMovImm(*BB, I, MI->getOperand(0).getReg(),
                      MI->getOperand(1).getImm());
     break;
+  case AMDGPU::MOV_IMM_GLOBAL_ADDR: {
+    //TODO: Perhaps combine this instruction with the next if possible
+    auto MIB = TII->buildDefaultInstruction(*BB, MI, AMDGPU::MOV,
+                                 MI->getOperand(0).getReg(),
+                                 AMDGPU::ALU_LITERAL_X);
+    int Idx = TII->getOperandIdx(*MIB, AMDGPU::OpName::literal);
+    //TODO: Ugh this is rather ugly
+    MIB->getOperand(Idx) = MI->getOperand(1);
+    break;
+  }
   case AMDGPU::CONST_COPY: {
     MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, MI, AMDGPU::MOV,
         MI->getOperand(0).getReg(), AMDGPU::ALU_CONST);
     TII->setImmOperand(NewMI, AMDGPU::OpName::src0_sel,
         MI->getOperand(1).getImm());
     break;
   }
 
SDValue R600TargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI,
                                                SelectionDAG &DAG) const {
 
   GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Op);
   if (GSD->getAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS)
     return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG);
 
   const DataLayout &DL = DAG.getDataLayout();
   const GlobalValue *GV = GSD->getGlobal();
-  MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
-  Type *EltType = GV->getValueType();
-  unsigned Size = DL.getTypeAllocSize(EltType);
-  unsigned Alignment = DL.getPrefTypeAlignment(EltType);
-
-  MVT PrivPtrVT = getPointerTy(DL, AMDGPUAS::PRIVATE_ADDRESS);
   MVT ConstPtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS);
 
-  int FI = FrameInfo->CreateStackObject(Size, Alignment, false);
-  SDValue InitPtr = DAG.getFrameIndex(FI, PrivPtrVT);
-
-  const GlobalVariable *Var = cast<GlobalVariable>(GV);
-  if (!Var->hasInitializer()) {
-    // This has no use, but bugpoint will hit it.
-    return DAG.getZExtOrTrunc(InitPtr, SDLoc(Op), ConstPtrVT);
-  }
-
-  const Constant *Init = Var->getInitializer();
-  SmallVector<SDNode*, 8> WorkList;
-
-  for (SDNode::use_iterator I = DAG.getEntryNode()->use_begin(),
-                            E = DAG.getEntryNode()->use_end(); I != E; ++I) {
-    if (I->getOpcode() != AMDGPUISD::REGISTER_LOAD && I->getOpcode() != ISD::LOAD)
-      continue;
-    WorkList.push_back(*I);
-  }
-  SDValue Chain = LowerConstantInitializer(Init, GV, InitPtr, DAG.getEntryNode(), DAG);
-  for (SmallVector<SDNode*, 8>::iterator I = WorkList.begin(),
-                                         E = WorkList.end(); I != E; ++I) {
-    SmallVector<SDValue, 8> Ops;
-    Ops.push_back(Chain);
-    for (unsigned i = 1; i < (*I)->getNumOperands(); ++i) {
-      Ops.push_back((*I)->getOperand(i));
-    }
-    DAG.UpdateNodeOperands(*I, Ops);
-  }
-  return DAG.getZExtOrTrunc(InitPtr, SDLoc(Op), ConstPtrVT);
+  SDValue GA = DAG.getTargetGlobalAddress(GV, SDLoc(GSD), ConstPtrVT);
+  return DAG.getNode(AMDGPUISD::CONST_DATA_PTR, SDLoc(GSD), ConstPtrVT, GA);
 }
 
 SDValue R600TargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
   // On hw >= R700, COS/SIN input must be between -1. and 1.
   // Thus we lower them to TRIG ( FRACT ( x / 2Pi + 0.5) - 0.5)
   EVT VT = Op.getValueType();
   SDValue Arg = Op.getOperand(0);
   SDLoc DL(Op);
return DAG.getNode(
       ISD::SETCC,
       DL,
       MVT::i1,
       Op, DAG.getConstantFP(0.0f, DL, MVT::f32),
       DAG.getCondCode(ISD::SETNE)
       );
 }
 
 SDValue R600TargetLowering::LowerImplicitParameter(SelectionDAG &DAG, EVT VT,
                                                    SDLoc DL,
                                                    unsigned DwordOffset) const {
   unsigned ByteOffset = DwordOffset * 4;
   PointerType * PtrType = PointerType::get(VT.getTypeForEVT(*DAG.getContext()),
                                       AMDGPUAS::CONSTANT_BUFFER_0);
 
   // We shouldn't be using an offset wider than 16-bits for implicit parameters.
   assert(isInt<16>(ByteOffset));
if (AS == AMDGPUAS::PRIVATE_ADDRESS &&
     return lowerPrivateExtLoad(Op, DAG);
   }
 
   SDLoc DL(Op);
   EVT VT = Op.getValueType();
   SDValue Chain = LoadNode->getChain();
   SDValue Ptr = LoadNode->getBasePtr();
 
-  // Lower loads constant address space global variable loads
-  if (LoadNode->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS &&
-      isa<GlobalVariable>(GetUnderlyingObject(
-          LoadNode->getMemOperand()->getValue(), DAG.getDataLayout()))) {
-
-    SDValue Ptr = DAG.getZExtOrTrunc(
-        LoadNode->getBasePtr(), DL,
-        getPointerTy(DAG.getDataLayout(), AMDGPUAS::PRIVATE_ADDRESS));
-    Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr,
-        DAG.getConstant(2, DL, MVT::i32));
-    return DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, Op->getVTList(),
-                       LoadNode->getChain(), Ptr,
-                       DAG.getTargetConstant(0, DL, MVT::i32),
-                       Op.getOperand(2));
-  }
-
   if (LoadNode->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && VT.isVector()) {
     SDValue MergedValues[2] = {
       scalarizeVectorLoad(LoadNode, DAG),
       Chain
     };
     return DAG.getMergeValues(MergedValues, DL);
   }
 

lib/Target/AMDGPU/R600Instructions.td

 } // end let isPseudo = 1, isCodeGenOnly = 1, usesCustomInserter = 1
 
 def MOV_IMM_I32 : MOV_IMM<i32, i32imm>;
 def : Pat <
   (imm:$val),
   (MOV_IMM_I32 imm:$val)
 >;
 
+def MOV_IMM_GLOBAL_ADDR : MOV_IMM<iPTR, i32imm>;
+def : Pat <
+  (AMDGPUconstdata_ptr tglobaladdr:$addr),
+  (MOV_IMM_GLOBAL_ADDR tglobaladdr:$addr)
+>;
+
+
 def MOV_IMM_F32 : MOV_IMM<f32, f32imm>;
 def : Pat <
   (fpimm:$val),
   (MOV_IMM_F32  fpimm:$val)
 >;
 
 def PRED_SETE : R600_2OP <0x20, "PRED_SETE", []>;
 def PRED_SETGT : R600_2OP <0x21, "PRED_SETGT", []>;

test/CodeGen/AMDGPU/gv-const-addrspace-fail.ll

-; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; XUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
-
-
-@a = internal addrspace(2) constant [1 x i8] [ i8 7 ], align 1
-
-; FUNC-LABEL: {{^}}test_i8:
-; EG: CF_END
-; SI: buffer_store_byte
-; SI: s_endpgm
-define void @test_i8( i32 %s, i8 addrspace(1)* %out) #3 {
-  %arrayidx = getelementptr inbounds [1 x i8], [1 x i8] addrspace(2)* @a, i32 0, i32 %s
-  %1 = load i8, i8 addrspace(2)* %arrayidx, align 1
-  store i8 %1, i8 addrspace(1)* %out
-  ret void
-}
-
-@b = internal addrspace(2) constant [1 x i16] [ i16 7 ], align 2
-
-; FUNC-LABEL: {{^}}test_i16:
-; EG: CF_END
-; SI: buffer_store_short
-; SI: s_endpgm
-define void @test_i16( i32 %s, i16 addrspace(1)* %out) #3 {
-  %arrayidx = getelementptr inbounds [1 x i16], [1 x i16] addrspace(2)* @b, i32 0, i32 %s
-  %1 = load i16, i16 addrspace(2)* %arrayidx, align 2
-  store i16 %1, i16 addrspace(1)* %out
-  ret void
-}
-
-%struct.bar = type { float, [5 x i8] }
-
-; The illegal i8s aren't handled
-@struct_bar_gv = internal addrspace(2) constant [1 x %struct.bar] [ %struct.bar { float 16.0, [5 x i8] [i8 0, i8 1, i8 2, i8 3, i8 4] } ]
-
-; FUNC-LABEL: {{^}}struct_bar_gv_load:
-define void @struct_bar_gv_load(i8 addrspace(1)* %out, i32 %index) {
-  %gep = getelementptr inbounds [1 x %struct.bar], [1 x %struct.bar] addrspace(2)* @struct_bar_gv, i32 0, i32 0, i32 1, i32 %index
-  %load = load i8, i8 addrspace(2)* %gep, align 1
-  store i8 %load, i8 addrspace(1)* %out, align 1
-  ret void
-}
-
-
-; The private load isn't scalarzied.
-@array_vector_gv = internal addrspace(2) constant [4 x <4 x i32>] [ <4 x i32> <i32 1, i32 2, i32 3, i32 4>,
-                                                                    <4 x i32> <i32 5, i32 6, i32 7, i32 8>,
-                                                                    <4 x i32> <i32 9, i32 10, i32 11, i32 12>,
-                                                                    <4 x i32> <i32 13, i32 14, i32 15, i32 16> ]
-
-; FUNC-LABEL: {{^}}array_vector_gv_load:
-define void @array_vector_gv_load(<4 x i32> addrspace(1)* %out, i32 %index) {
-  %gep = getelementptr inbounds [4 x <4 x i32>], [4 x <4 x i32>] addrspace(2)* @array_vector_gv, i32 0, i32 %index
-  %load = load <4 x i32>, <4 x i32> addrspace(2)* %gep, align 16
-  store <4 x i32> %load, <4 x i32> addrspace(1)* %out, align 16
-  ret void
-}

test/CodeGen/AMDGPU/gv-const-addrspace.ll

 ; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN -check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 
 @b = internal addrspace(2) constant [1 x i16] [ i16 7 ], align 2
 
 @float_gv = internal unnamed_addr addrspace(2) constant [5 x float] [float 0.0, float 1.0, float 2.0, float 3.0, float 4.0], align 4
 
 ; FUNC-LABEL: {{^}}float:
 ; GCN: s_load_dword
 
-; EG-DAG: MOV {{\** *}}T2.X
-; EG-DAG: MOV {{\** *}}T3.X
-; EG-DAG: MOV {{\** *}}T4.X
-; EG-DAG: MOV {{\** *}}T5.X
-; EG-DAG: MOV {{\** *}}T6.X
-; EG: MOVA_INT
-
+; EG: VTX_READ_32
+; EG: @float_gv
+; EG-NOT: MOVA_INT
 define void @float(float addrspace(1)* %out, i32 %index) {
 entry:
   %0 = getelementptr inbounds [5 x float], [5 x float] addrspace(2)* @float_gv, i32 0, i32 %index
   %1 = load float, float addrspace(2)* %0
   store float %1, float addrspace(1)* %out
   ret void
 }
 
 @i32_gv = internal unnamed_addr addrspace(2) constant [5 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4], align 4
 
 ; FUNC-LABEL: {{^}}i32:
 
 ; GCN: s_load_dword
 
-; EG-DAG: MOV {{\** *}}T2.X
-; EG-DAG: MOV {{\** *}}T3.X
-; EG-DAG: MOV {{\** *}}T4.X
-; EG-DAG: MOV {{\** *}}T5.X
-; EG-DAG: MOV {{\** *}}T6.X
-; EG: MOVA_INT
-
+; EG: VTX_READ_32
+; EG: @i32_gv
+; EG-NOT: MOVA_INT
 define void @i32(i32 addrspace(1)* %out, i32 %index) {
 entry:
   %0 = getelementptr inbounds [5 x i32], [5 x i32] addrspace(2)* @i32_gv, i32 0, i32 %index
   %1 = load i32, i32 addrspace(2)* %0
   store i32 %1, i32 addrspace(1)* %out
   ret void
 }
 
 
 %struct.foo = type { float, [5 x i32] }
 
 @struct_foo_gv = internal unnamed_addr addrspace(2) constant [1 x %struct.foo] [ %struct.foo { float 16.0, [5 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4] } ]
 
 ; FUNC-LABEL: {{^}}struct_foo_gv_load:
 ; GCN: s_load_dword
 
+; EG: VTX_READ_32
+; EG: @struct_foo_gv
+; EG-NOT: MOVA_INT
 define void @struct_foo_gv_load(i32 addrspace(1)* %out, i32 %index) {
   %gep = getelementptr inbounds [1 x %struct.foo], [1 x %struct.foo] addrspace(2)* @struct_foo_gv, i32 0, i32 0, i32 1, i32 %index
   %load = load i32, i32 addrspace(2)* %gep, align 4
   store i32 %load, i32 addrspace(1)* %out, align 4
   ret void
 }
 
 @array_v1_gv = internal addrspace(2) constant [4 x <1 x i32>] [ <1 x i32> <i32 1>,
                                                                 <1 x i32> <i32 2>,
                                                                 <1 x i32> <i32 3>,
                                                                 <1 x i32> <i32 4> ]
 
 ; FUNC-LABEL: {{^}}array_v1_gv_load:
 ; GCN: s_load_dword
+
+; EG: VTX_READ_32
+; EG: @array_v1_gv
+; EG-NOT: MOVA_INT
 define void @array_v1_gv_load(<1 x i32> addrspace(1)* %out, i32 %index) {
   %gep = getelementptr inbounds [4 x <1 x i32>], [4 x <1 x i32>] addrspace(2)* @array_v1_gv, i32 0, i32 %index
   %load = load <1 x i32>, <1 x i32> addrspace(2)* %gep, align 4
   store <1 x i32> %load, <1 x i32> addrspace(1)* %out, align 4
   ret void
 }
 
+; FUNC-LABEL: {{^}}gv_addressing_in_branch:
+
+; EG: VTX_READ_32
+; EG: @float_gv
+; EG-NOT: MOVA_INT
 define void @gv_addressing_in_branch(float addrspace(1)* %out, i32 %index, i32 %a) {
 entry:
   %0 = icmp eq i32 0, %a
   br i1 %0, label %if, label %else
 
 if:
   %1 = getelementptr inbounds [5 x float], [5 x float] addrspace(2)* @float_gv, i32 0, i32 %index
   %2 = load float, float addrspace(2)* %1