[AMDGPU] Add and use SIInstrFlags::GWS. NFC.

[foad]

Summary

This reduces the number of places where we have to check for a list of
DS_GWS_* opcodes.

llvm/lib/Target/AMDGPU/DSInstructions.td

 //===-- DSInstructions.td - DS Instruction Definitions --------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 class DS_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> :
   InstSI <outs, ins, "", pattern>,
   SIMCInstr <opName, SIEncodingFamily.NONE> {
 
   let LGKM_CNT = 1;
   let DS = 1;
+  let GWS = 0;
   let Size = 8;
   let UseNamedOperandTable = 1;
 
   // Most instruction load and store data, so set this as the default.
   let mayLoad = 1;
   let mayStore = 1;
   let maybeAtomic = 1;
 
class DS_Real <DS_Pseudo ps, string opName = ps.Mnemonic> :
 
   let isPseudo = 0;
   let isCodeGenOnly = 0;
   let LGKM_CNT = 1;
   let DS = 1;
   let UseNamedOperandTable = 1;
 
   // copy relevant pseudo op flags
+  let GWS                = ps.GWS;
   let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates    = ps.OtherPredicates;
   let SchedRW            = ps.SchedRW;
   let mayLoad            = ps.mayLoad;
   let mayStore           = ps.mayStore;
   let IsAtomicRet        = ps.IsAtomicRet;
   let IsAtomicNoRet      = ps.IsAtomicNoRet;
 
multiclass DS_1A_mc <string opName> {
   let has_m0_read = 0 in {
     def _gfx9 : DS_1A<opName>;
   }
 }
 
 
 class DS_GWS <string opName, dag ins, string asmOps>
 : DS_Pseudo<opName, (outs), ins, asmOps> {
+  let GWS = 1;
 
   let has_vdst  = 0;
   let has_addr  = 0;
   let has_data0 = 0;
   let has_data1 = 0;
 
   let has_gds   = 0;
   let gdsValue  = 1;

llvm/lib/Target/AMDGPU/MCA/AMDGPUCustomBehaviour.h

class AMDGPUCustomBehaviour : public CustomBehaviour {
   /// Unfortunately, some of the logic from that function is not available to us
   /// in this scope so we conservatively end up assuming that some
   /// instructions interact with more CNTs than they do in reality.
   void generateWaitCntInfo();
   /// Helper function used in generateWaitCntInfo()
   bool hasModifiersSet(const std::unique_ptr<Instruction> &Inst,
                        unsigned OpName) const;
   /// Helper function used in generateWaitCntInfo()
+  bool isGWS(uint16_t Opcode) const;
+  /// Helper function used in generateWaitCntInfo()
   bool isAlwaysGDS(uint16_t Opcode) const;
   /// Helper function used in generateWaitCntInfo()
   bool isVMEM(const MCInstrDesc &MCID);
   /// This method gets called from checkCustomHazard when mca is attempting to
   /// dispatch an s_waitcnt instruction (or one of its variants). The method
   /// looks at each of the instructions that are still executing in the pipeline
   /// to determine if the waitcnt should force a wait.
   unsigned handleWaitCnt(ArrayRef<InstRef> IssuedInst, const InstRef &IR);

llvm/lib/Target/AMDGPU/MCA/AMDGPUCustomBehaviour.cpp

bool AMDGPUCustomBehaviour::hasModifiersSet(
 
   const MCAOperand *Op = Inst->getOperand(Idx);
   if (Op == nullptr || !Op->isImm() || !Op->getImm())
     return false;
 
   return true;
 }
 
+// taken from SIInstrInfo::isGWS()
+bool AMDGPUCustomBehaviour::isGWS(uint16_t Opcode) const {
+  const MCInstrDesc &MCID = MCII.get(Opcode);
+  return MCID.TSFlags & SIInstrFlags::GWS;
+}
+
 // taken from SIInstrInfo::isAlwaysGDS()
 bool AMDGPUCustomBehaviour::isAlwaysGDS(uint16_t Opcode) const {
-  return Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::DS_GWS_INIT ||
-         Opcode == AMDGPU::DS_GWS_SEMA_V || Opcode == AMDGPU::DS_GWS_SEMA_BR ||
-         Opcode == AMDGPU::DS_GWS_SEMA_P ||
-         Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL ||
-         Opcode == AMDGPU::DS_GWS_BARRIER;
+  return Opcode == AMDGPU::DS_ORDERED_COUNT || isGWS(Opcode);
 }
 
 } // namespace mca
 } // namespace llvm
 
 using namespace llvm;
 using namespace mca;
 

llvm/lib/Target/AMDGPU/SIDefines.h

enum : uint64_t {
   // Is a WMMA instruction.
   IsWMMA = UINT64_C(1) << 59,
 
   // Whether tied sources will be read.
   TiedSourceNotRead = UINT64_C(1) << 60,
 
   // Is never uniform.
   IsNeverUniform = UINT64_C(1) << 61,
+
+  // ds_gws_* instructions.
+  GWS = UINT64_C(1) << 62,
 };
 
 // v_cmp_class_* etc. use a 10-bit mask for what operation is checked.
 // The result is true if any of these tests are true.
 enum ClassFlags : unsigned {
   S_NAN = 1 << 0,        // Signaling NaN
   Q_NAN = 1 << 1,        // Quiet NaN
   N_INFINITY = 1 << 2,   // Negative infinity

llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp

if (TII->isDS(Inst) && (Inst.mayStore() || Inst.mayLoad())) {
               CurrScore);
         }
         if (AMDGPU::hasNamedOperand(Inst.getOpcode(), AMDGPU::OpName::data1)) {
           setExpScore(&Inst, TII, TRI, MRI,
                       AMDGPU::getNamedOperandIdx(Inst.getOpcode(),
                                                  AMDGPU::OpName::data1),
                       CurrScore);
         }
-      } else if (SIInstrInfo::isAtomicRet(Inst) &&
-                 Inst.getOpcode() != AMDGPU::DS_GWS_INIT &&
-                 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_V &&
-                 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_BR &&
-                 Inst.getOpcode() != AMDGPU::DS_GWS_SEMA_P &&
-                 Inst.getOpcode() != AMDGPU::DS_GWS_BARRIER &&
+      } else if (SIInstrInfo::isAtomicRet(Inst) && !SIInstrInfo::isGWS(Inst) &&
                  Inst.getOpcode() != AMDGPU::DS_APPEND &&
                  Inst.getOpcode() != AMDGPU::DS_CONSUME &&
                  Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
         for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
           const MachineOperand &Op = Inst.getOperand(I);
           if (Op.isReg() && !Op.isDef() &&
               TRI->isVectorRegister(*MRI, Op.getReg())) {
             setExpScore(&Inst, TII, TRI, MRI, I, CurrScore);

llvm/lib/Target/AMDGPU/SIInstrFormats.td

class InstSI <dag outs, dag ins, string asm = "",
   field bit IsWMMA = 0;
 
   // This bit indicates that tied source will not be read.
   field bit TiedSourceNotRead = 0;
 
   // This bit indicates that the instruction is never-uniform/divergent
   field bit IsNeverUniform = 0;
 
+  // ds_gws_* instructions.
+  field bit GWS = 0;
+
   // These need to be kept in sync with the enum in SIInstrFlags.
   let TSFlags{0} = SALU;
   let TSFlags{1} = VALU;
 
   let TSFlags{2} = SOP1;
   let TSFlags{3} = SOP2;
   let TSFlags{4} = SOPC;
   let TSFlags{5} = SOPK;
class InstSI <dag outs, dag ins, string asm = "",
   let TSFlags{58} = IsAtomicRet;
 
   let TSFlags{59} = IsWMMA;
 
   let TSFlags{60} = TiedSourceNotRead;
 
   let TSFlags{61} = IsNeverUniform;
   
+  let TSFlags{62} = GWS;
+
   let SchedRW = [Write32Bit];
 
   let AsmVariantName = AMDGPUAsmVariants.Default;
 
   // Avoid changing source registers in a way that violates constant bus read limitations.
   let hasExtraSrcRegAllocReq = !or(VOP1, VOP2, VOP3, VOPC, SDWA, VALU);
 }
 

llvm/lib/Target/AMDGPU/SIInstrInfo.h

public:
   static bool isDS(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::DS;
   }
 
   bool isDS(uint16_t Opcode) const {
     return get(Opcode).TSFlags & SIInstrFlags::DS;
   }
 
+  static bool isGWS(const MachineInstr &MI) {
+    return MI.getDesc().TSFlags & SIInstrFlags::GWS;
+  }
+
+  bool isGWS(uint16_t Opcode) const {
+    return get(Opcode).TSFlags & SIInstrFlags::GWS;
+  }
+
   bool isAlwaysGDS(uint16_t Opcode) const;
 
   static bool isMIMG(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::MIMG;
   }
 
   bool isMIMG(uint16_t Opcode) const {
     return get(Opcode).TSFlags & SIInstrFlags::MIMG;

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
   return MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
          MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
          MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
          MI.getOpcode() == AMDGPU::S_SETPRIO ||
          changesVGPRIndexingMode(MI);
 }
 
 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {
-  return Opcode == AMDGPU::DS_ORDERED_COUNT ||
-         Opcode == AMDGPU::DS_GWS_INIT ||
-         Opcode == AMDGPU::DS_GWS_SEMA_V ||
-         Opcode == AMDGPU::DS_GWS_SEMA_BR ||
-         Opcode == AMDGPU::DS_GWS_SEMA_P ||
-         Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL ||
-         Opcode == AMDGPU::DS_GWS_BARRIER;
+  return Opcode == AMDGPU::DS_ORDERED_COUNT || isGWS(Opcode);
 }
 
 bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) {
   // Skip the full operand and register alias search modifiesRegister
   // does. There's only a handful of instructions that touch this, it's only an
   // implicit def, and doesn't alias any other registers.
   return is_contained(MI.getDesc().implicit_defs(), AMDGPU::MODE);
 }

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h

 }
 
 bool hasXNACK(const MCSubtargetInfo &STI);
 bool hasSRAMECC(const MCSubtargetInfo &STI);
 bool hasMIMG_R128(const MCSubtargetInfo &STI);
 bool hasA16(const MCSubtargetInfo &STI);
 bool hasG16(const MCSubtargetInfo &STI);
 bool hasPackedD16(const MCSubtargetInfo &STI);
+bool hasGDS(const MCSubtargetInfo &STI);
 unsigned getNSAMaxSize(const MCSubtargetInfo &STI);
 
 bool isSI(const MCSubtargetInfo &STI);
 bool isCI(const MCSubtargetInfo &STI);
 bool isVI(const MCSubtargetInfo &STI);
 bool isGFX9(const MCSubtargetInfo &STI);
 bool isGFX9_GFX10(const MCSubtargetInfo &STI);
 bool isGFX8_GFX9_GFX10(const MCSubtargetInfo &STI);

llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

bool hasG16(const MCSubtargetInfo &STI) {
   return STI.hasFeature(AMDGPU::FeatureG16);
 }
 
 bool hasPackedD16(const MCSubtargetInfo &STI) {
   return !STI.hasFeature(AMDGPU::FeatureUnpackedD16VMem) && !isCI(STI) &&
          !isSI(STI);
 }
 
+bool hasGDS(const MCSubtargetInfo &STI) {
+  return STI.hasFeature(AMDGPU::FeatureGDS);
+}
+
 unsigned getNSAMaxSize(const MCSubtargetInfo &STI) {
   auto Version = getIsaVersion(STI.getCPU());
   if (Version.Major == 10)
     return Version.Minor >= 3 ? 13 : 5;
   if (Version.Major == 11)
     return 5;
   return 0;
 }