[AMDGPU] Introduce divergence and uniform bit fields in tablegen

[yassingh]

Summary

IsSourceOfDivergence and IsAlwaysUniform can be set to 1 to mark instructions
which are inherently divergent or uniform. Adding them to Readlane and
Writelane instruction for now. To be extended to all required instructions.

[foad]

llvm/lib/Target/AMDGPU/SIDefines.h

137–138

Now would be a good time to adopt the name "IsNeverUniform" instead of "IsSourceOfDivergence", to match the InstructionUniformity enum (https://llvm.org/doxygen/namespacellvm.html#ae58c751054b01f206f9b9e34e461d25f)

[arsenm]

I’m not sure we really need a bit for just these 3 cases

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

8422

Space after if

[yassingh]

In D143154#4099243, @arsenm wrote:

I’m not sure we really need a bit for just these 3 cases

At least for divergent there are a lot more instructions, atomic buffers, flat atomics, some lds instructions, etc. Will submit subsequent patches covering them.

[ruiling]

llvm/lib/Target/AMDGPU/SIInstrFormats.td

242

As we have only very few alwaysUniform instruction, maybe we don't need this to save one bit.

[yassingh]

Addressed review comments

• Using isNeverUniform instead of isSourceOfDivergence.
• Removed the bit for always uniform.

[sameerds]

LGTM. But please wait for at least one other reviewer.

Also update the git commit message before submitting

llvm/lib/Target/AMDGPU/SIDefines.h

enum : uint64_t {
   // Atomic with return.
   IsAtomicRet = UINT64_C(1) << 58,
 
   // Is a WMMA instruction.
   IsWMMA = UINT64_C(1) << 59,
 
   // Whether tied sources will be read.
   TiedSourceNotRead = UINT64_C(1) << 60,
+
+  // Is never uniform.
+  IsNeverUniform = UINT64_C(1) << 61,
 };
 
 // v_cmp_class_* etc. use a 10-bit mask for what operation is checked.
 // The result is true if any of these tests are true.
 enum ClassFlags : unsigned {
   S_NAN = 1 << 0,        // Signaling NaN
   Q_NAN = 1 << 1,        // Quiet NaN
   N_INFINITY = 1 << 2,   // Negative infinity

llvm/lib/Target/AMDGPU/SIInstrFormats.td

class InstSI <dag outs, dag ins, string asm = "",
   field bit IsAtomicRet = 0;
 
   // This bit indicates that this is one of WMMA instructions.
   field bit IsWMMA = 0;
 
   // This bit indicates that tied source will not be read.
   field bit TiedSourceNotRead = 0;
 
+  // This bit indicates that the instruction is never-uniform/divergent
+  field bit IsNeverUniform = 0;
+
   // These need to be kept in sync with the enum in SIInstrFlags.
   let TSFlags{0} = SALU;
   let TSFlags{1} = VALU;
 
   let TSFlags{2} = SOP1;
   let TSFlags{3} = SOP2;
   let TSFlags{4} = SOPC;
   let TSFlags{5} = SOPK;
class InstSI <dag outs, dag ins, string asm = "",
   let TSFlags{57} = IsAtomicNoRet;
 
   let TSFlags{58} = IsAtomicRet;
 
   let TSFlags{59} = IsWMMA;
 
   let TSFlags{60} = TiedSourceNotRead;
 
+  let TSFlags{61} = IsNeverUniform;
+  
   let SchedRW = [Write32Bit];
 
   let AsmVariantName = AMDGPUAsmVariants.Default;
 
   // Avoid changing source registers in a way that violates constant bus read limitations.
   let hasExtraSrcRegAllocReq = !or(VOP1, VOP2, VOP3, VOPC, SDWA, VALU);
 }
 
 class PseudoInstSI<dag outs, dag ins, list<dag> pattern = [], string asm = "">

llvm/lib/Target/AMDGPU/SIInstrInfo.h

public:
   static bool isFPAtomic(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::FPAtomic;
   }
 
   bool isFPAtomic(uint16_t Opcode) const {
     return get(Opcode).TSFlags & SIInstrFlags::FPAtomic;
   }
 
+  static bool isNeverUniform(const MachineInstr &MI){
+    return MI.getDesc().TSFlags & SIInstrFlags::IsNeverUniform;
+  }
+
   static bool doesNotReadTiedSource(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::TiedSourceNotRead;
   }
 
   bool doesNotReadTiedSource(uint16_t Opcode) const {
     return get(Opcode).TSFlags & SIInstrFlags::TiedSourceNotRead;
   }
 

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

if (SIInstrInfo::isGenericAtomicRMWOpcode(opcode) ||
       opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS) {
     return InstructionUniformity::NeverUniform;
   }
   return InstructionUniformity::Default;
 }
 
 InstructionUniformity
 SIInstrInfo::getInstructionUniformity(const MachineInstr &MI) const {
+
+  if (isNeverUniform(MI))
+    return InstructionUniformity::NeverUniform;
+
   unsigned opcode = MI.getOpcode();
+  if (opcode == AMDGPU::V_READLANE_B32 || opcode == AMDGPU::V_READFIRSTLANE_B32)
+    return InstructionUniformity::AlwaysUniform;
+
   if (MI.isCopy()) {
     const MachineOperand &srcOp = MI.getOperand(1);
     if (srcOp.isReg() && srcOp.getReg().isPhysical()) {
       const TargetRegisterClass *regClass =
           RI.getPhysRegBaseClass(srcOp.getReg());
       return RI.isSGPRClass(regClass) ? InstructionUniformity::AlwaysUniform
                                       : InstructionUniformity::NeverUniform;
     }
if (llvm::any_of(MI.memoperands(), [](const MachineMemOperand *mmo) {
         })) {
       // At least one MMO in a non-global address space.
       return InstructionUniformity::NeverUniform;
     }
 
     return InstructionUniformity::Default;
   }
 
-  if (opcode == AMDGPU::V_READLANE_B32 || opcode == AMDGPU::V_READFIRSTLANE_B32)
-    return InstructionUniformity::AlwaysUniform;
-
-  if (opcode == AMDGPU::V_WRITELANE_B32)
-    return InstructionUniformity::NeverUniform;
-
   const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
   const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo();
 
   // FIXME: It's conceptually broken to report this for an instruction, and not
   // a specific def operand. For inline asm in particular, there could be mixed
   // uniform and divergent results.
   for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
     const MachineOperand &SrcOp = MI.getOperand(I);

llvm/lib/Target/AMDGPU/VOP2Instructions.td

 }
 
 } // End isCommutable = 1
 
 // These are special and do not read the exec mask.
 let isConvergent = 1, Uses = []<Register> in {
 def V_READLANE_B32 : VOP2_Pseudo<"v_readlane_b32", VOP_READLANE,
   [(set i32:$vdst, (int_amdgcn_readlane i32:$src0, i32:$src1))]>;
-
-let Constraints = "$vdst = $vdst_in", DisableEncoding="$vdst_in" in {
+let IsNeverUniform = 1, Constraints = "$vdst = $vdst_in", DisableEncoding="$vdst_in" in {
 def V_WRITELANE_B32 : VOP2_Pseudo<"v_writelane_b32", VOP_WRITELANE,
   [(set i32:$vdst, (int_amdgcn_writelane i32:$src0, i32:$src1, i32:$vdst_in))]>;
-} // End $vdst = $vdst_in, DisableEncoding $vdst_in
+} // End IsNeverUniform, $vdst = $vdst_in, DisableEncoding $vdst_in
 } // End isConvergent = 1
 
 let isReMaterializable = 1 in {
 defm V_BFM_B32 : VOP2Inst <"v_bfm_b32", VOP_I32_I32_I32>;
 defm V_BCNT_U32_B32 : VOP2Inst <"v_bcnt_u32_b32", VOP_I32_I32_I32, add_ctpop>;
 defm V_MBCNT_LO_U32_B32 : VOP2Inst <"v_mbcnt_lo_u32_b32", VOP_I32_I32_I32, int_amdgcn_mbcnt_lo>;
 defm V_MBCNT_HI_U32_B32 : VOP2Inst <"v_mbcnt_hi_u32_b32", VOP_I32_I32_I32, int_amdgcn_mbcnt_hi>;
 defm V_LDEXP_F32 : VOP2Inst <"v_ldexp_f32", VOP_F32_F32_I32, AMDGPUldexp>;

llvm/test/Analysis/DivergenceAnalysis/AMDGPU/MIR/always-uniform.mir

-# RUN: llc -mtriple=amdgcn-- -run-pass=print-machine-uniformity -o - %s 2>&1 | FileCheck %s
+# RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -run-pass=print-machine-uniformity -o - %s 2>&1 | FileCheck %s
+
 # readlane, readfirstlane is always uniform
 
 ---
 name:            readlane
 machineFunctionInfo:
   isEntryFunction: true
 body:             |
   bb.0:

llvm/test/Analysis/DivergenceAnalysis/AMDGPU/MIR/never-uniform.mir

-# RUN: llc -mtriple=amdgcn-- -run-pass=print-machine-uniformity -o - %s 2>&1 | FileCheck %s
+# RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -run-pass=print-machine-uniformity -o - %s 2>&1 | FileCheck %s
 # loads from flat non uniform
 ---
 name:            flatloads
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
 
 body:             |