AMDGPU: Fold CI-specific complex SMRD patterns into existing complex patterns

[arsenm]

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1382

Can you add a Subtarget->has32BitLiteralSMRDOffset()?

1403

Ditto

[mareko]

This is just a cleanup. Also, it adds checking that ByteCount is aligned to 4.

[mareko]

add one more use of has32BitLiteralSMRDOffset.

[arsenm]

LGTM

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

private:
 
   bool SelectFlat(SDValue Addr, SDValue &VAddr, SDValue &SLC) const;
 
   bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset,
                         bool &Imm) const;
   bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
                   bool &Imm) const;
   bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
-  bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
   bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
   bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
-  bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
   bool SelectSMRDBufferSgpr(SDValue Addr, SDValue &Offset) const;
   bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
 
   bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const;
   bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
   bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
   bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
                        SDValue &Clamp, SDValue &Omod) const;
if (AMDGPU::isLegalSMRDImmOffset(*Subtarget, ByteOffset)) {
     Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32);
     Imm = true;
     return true;
   }
 
   if (!isUInt<32>(EncodedOffset) || !isUInt<32>(ByteOffset))
     return false;
 
-  if (Gen == AMDGPUSubtarget::SEA_ISLANDS && isUInt<32>(EncodedOffset)) {
-    // 32-bit Immediates are supported on Sea Islands.
+  if (Subtarget->has32BitLiteralSMRDOffset() &&
+      ByteOffset % 4 == 0 && isUInt<32>(EncodedOffset)) {
     Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32);
   } else {
     SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
     Offset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32,
                                             C32Bit), 0);
   }
   Imm = false;
   return true;
bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
   Offset = CurDAG->getTargetConstant(0, SL, MVT::i32);
   Imm = true;
   return true;
 }
 
 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
                                        SDValue &Offset) const {
   bool Imm;
-  return SelectSMRD(Addr, SBase, Offset, Imm) && Imm;
-}
-
-bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
-                                         SDValue &Offset) const {
-
-  if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS)
-    return false;
 
-  bool Imm;
   if (!SelectSMRD(Addr, SBase, Offset, Imm))
     return false;
 
-  return !Imm && isa<ConstantSDNode>(Offset);
+  if (Subtarget->has32BitLiteralSMRDOffset() &&
+      isa<ConstantSDNode>(Offset))
+    return true;
+
+  return Imm;
 }
 
 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
                                         SDValue &Offset) const {
   bool Imm;
   return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm &&
          !isa<ConstantSDNode>(Offset);
 }
 
 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr,
                                              SDValue &Offset) const {
   bool Imm;
-  return SelectSMRDOffset(Addr, Offset, Imm) && Imm;
-}
-
-bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr,
-                                               SDValue &Offset) const {
-  if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS)
-    return false;
 
-  bool Imm;
   if (!SelectSMRDOffset(Addr, Offset, Imm))
     return false;
 
-  return !Imm && isa<ConstantSDNode>(Offset);
+  if (Subtarget->has32BitLiteralSMRDOffset() &&
+      isa<ConstantSDNode>(Offset))
+    return true;
+
+  return Imm;
 }
 
 bool AMDGPUDAGToDAGISel::SelectSMRDBufferSgpr(SDValue Addr,
                                               SDValue &Offset) const {
   bool Imm;
   return SelectSMRDOffset(Addr, Offset, Imm) && !Imm &&
          !isa<ConstantSDNode>(Offset);
 }

lib/Target/AMDGPU/AMDGPUSubtarget.h

public:
   bool hasFlatGlobalInsts() const {
     return FlatGlobalInsts;
   }
 
   bool hasFlatScratchInsts() const {
     return FlatScratchInsts;
   }
 
+  bool has32BitLiteralSMRDOffset() const {
+    return getGeneration() == SEA_ISLANDS;
+  }
+
   bool isMesaKernel(const MachineFunction &MF) const {
     return isMesa3DOS() && !AMDGPU::isShader(MF.getFunction()->getCallingConv());
   }
 
   // Covers VS/PS/CS graphics shaders
   bool isMesaGfxShader(const MachineFunction &MF) const {
     return isMesa3DOS() && AMDGPU::isShader(MF.getFunction()->getCallingConv());
   }

lib/Target/AMDGPU/SMInstructions.td

return Ld->getAlignment() >= 4  &&
     ((Ld->getAddressSpace() == AMDGPUASI.CONSTANT_ADDRESS &&
     static_cast<const SITargetLowering *>(getTargetLowering())->isMemOpUniform(N)) ||
     (Subtarget->getScalarizeGlobalBehavior() && Ld->getAddressSpace() == AMDGPUASI.GLOBAL_ADDRESS &&
     static_cast<const SITargetLowering *>(getTargetLowering())->isMemOpUniform(N) &&
     static_cast<const SITargetLowering *>(getTargetLowering())->isMemOpHasNoClobberedMemOperand(N)));
 }]>;
 
 def SMRDImm         : ComplexPattern<i64, 2, "SelectSMRDImm">;
-def SMRDImm32       : ComplexPattern<i64, 2, "SelectSMRDImm32">;
 def SMRDSgpr        : ComplexPattern<i64, 2, "SelectSMRDSgpr">;
 def SMRDBufferImm   : ComplexPattern<i32, 1, "SelectSMRDBufferImm">;
-def SMRDBufferImm32 : ComplexPattern<i32, 1, "SelectSMRDBufferImm32">;
 def SMRDBufferSgpr  : ComplexPattern<i32, 1, "SelectSMRDBufferSgpr">;
 
 let Predicates = [isGCN] in {
 
 multiclass SMRD_Pattern <string Instr, ValueType vt> {
 
   // 1. IMM offset
   def : Pat <

 
 defm : SMRD_Pattern <"S_LOAD_DWORD",    i32>;
 defm : SMRD_Pattern <"S_LOAD_DWORDX2",  v2i32>;
 defm : SMRD_Pattern <"S_LOAD_DWORDX4",  v4i32>;
 defm : SMRD_Pattern <"S_LOAD_DWORDX8",  v8i32>;
 defm : SMRD_Pattern <"S_LOAD_DWORDX16", v16i32>;
 
 // 1. Offset as an immediate
-def SM_LOAD_PATTERN : Pat <  // name this pattern to reuse AddedComplexity on CI
+def : Pat <
   (SIload_constant v4i32:$sbase, (SMRDBufferImm i32:$offset)),
   (S_BUFFER_LOAD_DWORD_IMM $sbase, $offset, 0)
 >;
 
 // 2. Offset loaded in an 32bit SGPR
 def : Pat <
   (SIload_constant v4i32:$sbase, (SMRDBufferSgpr i32:$offset)),
   (S_BUFFER_LOAD_DWORD_SGPR $sbase, $offset, 0)
class SMRD_Real_ci <bits<5> op, SM_Pseudo ps>
   let Inst{8}     = imm;
   let Inst{14-9}  = !if(ps.has_sbase, sbase{6-1}, ?);
   let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?);
   let Inst{26-22} = op;
   let Inst{31-27} = 0x18; //encoding
 }
 
 def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>;
-
-let AddedComplexity = SM_LOAD_PATTERN.AddedComplexity in {
-
-class SMRD_Pattern_ci <string Instr, ValueType vt> : Pat <
-  (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)),
-  (vt (!cast<SM_Pseudo>(Instr#"_IMM_ci") $sbase, $offset, 0))> {
-  let Predicates = [isCIOnly];
-}
-
-def : SMRD_Pattern_ci <"S_LOAD_DWORD",    i32>;
-def : SMRD_Pattern_ci <"S_LOAD_DWORDX2",  v2i32>;
-def : SMRD_Pattern_ci <"S_LOAD_DWORDX4",  v4i32>;
-def : SMRD_Pattern_ci <"S_LOAD_DWORDX8",  v8i32>;
-def : SMRD_Pattern_ci <"S_LOAD_DWORDX16", v16i32>;
-
-def : Pat <
-  (SIload_constant v4i32:$sbase, (SMRDBufferImm32 i32:$offset)),
-  (S_BUFFER_LOAD_DWORD_IMM_ci $sbase, $offset, 0)> {
-  let Predicates = [isCI]; // should this be isCIOnly?
-}
-
-} // End let AddedComplexity = SM_LOAD_PATTERN.AddedComplexity
-

lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

int64_t getSMRDEncodedOffset(const MCSubtargetInfo &ST, int64_t ByteOffset) {
   if (isSI(ST) || isCI(ST))
     return ByteOffset >> 2;
 
   return ByteOffset;
 }
 
 bool isLegalSMRDImmOffset(const MCSubtargetInfo &ST, int64_t ByteOffset) {
   int64_t EncodedOffset = getSMRDEncodedOffset(ST, ByteOffset);
-  return isSI(ST) || isCI(ST) ? isUInt<8>(EncodedOffset) :
+  return isSI(ST) || isCI(ST) ? ByteOffset % 4 == 0 && isUInt<8>(EncodedOffset) :
                                 isUInt<20>(EncodedOffset);
 }
 } // end namespace AMDGPU
 
 } // end namespace llvm
 
 const unsigned AMDGPUAS::MAX_COMMON_ADDRESS;
 const unsigned AMDGPUAS::GLOBAL_ADDRESS;