AMDGPU: Select s_buffer_load_dword with a non-constant SGPR offset

[mareko]

Summary

Apps that benefit:

• alien isolation
• bioshock infinite
• civilization: beyond earth
• company of heroes 2
• dirt showdown
• dota 2
• F1 2015
• grid autosport
• hitman
• legend of grimrock
• serious sam 3: bfe
• shadow warrior
• talos principle
• total war: warhammer
• UE4 demos: effects cave, elemental, sun temple

[arsenm]

lib/Target/AMDGPU/SIInstrInfo.cpp

3720–3723 ↗

(On Diff #119000)

Don't these need to be preserved from the input? Especially the offsets

[mareko]

lib/Target/AMDGPU/SIInstrInfo.cpp

3720–3723 ↗

(On Diff #119000)

S_BUFFER_LOAD_DWORD_SGPR doesn't have soffset, offset, slc. It only has soff (which is now VGPR) and glc, but glc can't be 1 currently.

[mareko]

Preserve GLC.

[nhaehnle]

That should indeed help a lot :)
LGTM.

llvm/trunk/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset,
                         bool &Imm) const;
   bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
                   bool &Imm) const;
   bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
   bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
   bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
   bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
   bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
-  bool SelectSMRDBufferSgpr(SDValue Addr, SDValue &Offset) const;
   bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
 
   bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const;
   bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods) const;
   bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
   bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
   bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
                        SDValue &Clamp, SDValue &Omod) const;
bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr,
 
   bool Imm;
   if (!SelectSMRDOffset(Addr, Offset, Imm))
     return false;
 
   return !Imm && isa<ConstantSDNode>(Offset);
 }
 
-bool AMDGPUDAGToDAGISel::SelectSMRDBufferSgpr(SDValue Addr,
-                                              SDValue &Offset) const {
-  bool Imm;
-  return SelectSMRDOffset(Addr, Offset, Imm) && !Imm &&
-         !isa<ConstantSDNode>(Offset);
-}
-
 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
                                             SDValue &Base,
                                             SDValue &Offset) const {
   SDLoc DL(Index);
 
   if (CurDAG->isBaseWithConstantOffset(Index)) {
     SDValue N0 = Index.getOperand(0);
     SDValue N1 = Index.getOperand(1);

llvm/trunk/lib/Target/AMDGPU/BUFInstructions.td

def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",
                                            int_amdgcn_buffer_wbinvl1_vol>;
 
 } // End let SubtargetPredicate = isCIVI
 
 //===----------------------------------------------------------------------===//
 // MUBUF Patterns
 //===----------------------------------------------------------------------===//
 
-// Offset in an 32-bit VGPR
-def : GCNPat <
-  (SIload_constant v4i32:$sbase, i32:$voff),
-  (BUFFER_LOAD_DWORD_OFFEN $voff, $sbase, (i32 0), 0, 0, 0, 0)
->;
-
-
 //===----------------------------------------------------------------------===//
 // buffer_load/store_format patterns
 //===----------------------------------------------------------------------===//
 
 multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
                                   string opcode> {
   def : GCNPat<
     (vt (name v4i32:$rsrc, 0,

llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.cpp

case AMDGPU::S_XNOR_B32:
       lowerScalarXnor(Worklist, Inst);
       Inst.eraseFromParent();
       continue;
 
     case AMDGPU::S_XNOR_B64:
       splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32);
       Inst.eraseFromParent();
       continue;
+
+    case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR: {
+      unsigned VDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+
+      BuildMI(*MBB, Inst, Inst.getDebugLoc(),
+              get(AMDGPU::BUFFER_LOAD_DWORD_OFFEN), VDst)
+        .add(*getNamedOperand(Inst, AMDGPU::OpName::soff)) // vaddr
+        .add(*getNamedOperand(Inst, AMDGPU::OpName::sbase)) // srsrc
+        .addImm(0) // soffset
+        .addImm(0) // offset
+        .addImm(getNamedOperand(Inst, AMDGPU::OpName::glc)->getImm())
+        .addImm(0) // slc
+        .addImm(0) // tfe
+        .setMemRefs(Inst.memoperands_begin(), Inst.memoperands_end());
+
+      MRI.replaceRegWith(getNamedOperand(Inst, AMDGPU::OpName::sdst)->getReg(),
+                         VDst);
+      addUsersToMoveToVALUWorklist(VDst, MRI, Worklist);
+      Inst.eraseFromParent();
+      continue;
+    }
     }
 
     if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
       // We cannot move this instruction to the VALU, so we should try to
       // legalize its operands instead.
       legalizeOperands(Inst);
       continue;
     }

llvm/trunk/lib/Target/AMDGPU/SMInstructions.td

return Ld->getAlignment() >= 4  &&
     static_cast<const SITargetLowering *>(getTargetLowering())->isMemOpHasNoClobberedMemOperand(N)));
 }]>;
 
 def SMRDImm         : ComplexPattern<i64, 2, "SelectSMRDImm">;
 def SMRDImm32       : ComplexPattern<i64, 2, "SelectSMRDImm32">;
 def SMRDSgpr        : ComplexPattern<i64, 2, "SelectSMRDSgpr">;
 def SMRDBufferImm   : ComplexPattern<i32, 1, "SelectSMRDBufferImm">;
 def SMRDBufferImm32 : ComplexPattern<i32, 1, "SelectSMRDBufferImm32">;
-def SMRDBufferSgpr  : ComplexPattern<i32, 1, "SelectSMRDBufferSgpr">;
 
 multiclass SMRD_Pattern <string Instr, ValueType vt> {
 
   // 1. IMM offset
   def : GCNPat <
     (smrd_load (SMRDImm i64:$sbase, i32:$offset)),
     (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0))
   >;

 // 1. Offset as an immediate
 def SM_LOAD_PATTERN : GCNPat <  // name this pattern to reuse AddedComplexity on CI
   (SIload_constant v4i32:$sbase, (SMRDBufferImm i32:$offset)),
   (S_BUFFER_LOAD_DWORD_IMM $sbase, $offset, 0)
 >;
 
 // 2. Offset loaded in an 32bit SGPR
 def : GCNPat <
-  (SIload_constant v4i32:$sbase, (SMRDBufferSgpr i32:$offset)),
+  (SIload_constant v4i32:$sbase, i32:$offset),
   (S_BUFFER_LOAD_DWORD_SGPR $sbase, $offset, 0)
 >;
 
 } // End let AddedComplexity = 100
 
 let OtherPredicates = [isVI] in {
 
 def : GCNPat <

llvm/trunk/test/CodeGen/AMDGPU/smrd.ll

 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(2)* %arg, i32 0
   %tmp20 = load <4 x i32>, <4 x i32> addrspace(2)* %tmp
   %tmp21 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 1048576)
   call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %tmp21, i1 true, i1 true) #0
   ret void
 }
 
+; GCN-LABEL: {{^}}smrd_sgpr_offset:
+; GCN: s_buffer_load_dword s{{[0-9]}}, s[0:3], s4
+define amdgpu_ps float @smrd_sgpr_offset(<4 x i32> inreg %desc, i32 inreg %offset) #0 {
+main_body:
+  %r = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %offset)
+  ret float %r
+}
+
+; GCN-LABEL: {{^}}smrd_vgpr_offset:
+; GCN: buffer_load_dword v{{[0-9]}}, v0, s[0:3], 0 offen ;
+define amdgpu_ps float @smrd_vgpr_offset(<4 x i32> inreg %desc, i32 %offset) #0 {
+main_body:
+  %r = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %offset)
+  ret float %r
+}
+
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
 declare float @llvm.SI.load.const.v4i32(<4 x i32>, i32) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }