[AMDGPU] Add options for waitcnt pass debugging; add instr count in debug output.

[msearles]

Summary

-amdgpu-waitcnt-forcezero={1|0}  Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-amdgpu-waitcnt-forceexp=<n>  Force emit a s_waitcnt expcnt(0) before the first <n> instrs
-amdgpu-waitcnt-forcelgkm=<n> Force emit a s_waitcnt lgkmcnt(0) before the first <n> instrs
-amdgpu-waitcnt-forcevm=<n>   Force emit a s_waitcnt vmcnt(0) before the first <n> instrs

[kzhuravl]

This will be pretty useful. Thanks!

lib/Target/AMDGPU/SIInsertWaitcnts.cpp

425–427

tabs?

1167–1173

tabs?

1185–1188

tabs?

[kzhuravl]

lib/Target/AMDGPU/SIInsertWaitcnts.cpp

423

const?

[arsenm]

lib/Target/AMDGPU/SIInsertWaitcnts.cpp

1644

You can directly << the Inst, it doesn't need a separate print line

test/CodeGen/AMDGPU/waitcnt-debug.mir

10

You don't add CHECK as a check-prefix, so this doesn't do anything

[msearles]

Adjust per reviewer comments.

[msearles]

Ping

[kzhuravl]

LGTM.

[msearles]

Patch was submitted/reverted (caused a buildbot failure: private field 'InstCnt' is not used ); changed InstCnt to be a function-scope static instead.

[msearles]

Reopening; patch was submitted/reverted (caused a buildbot failure: private field 'InstCnt' is not used ); changed InstCnt to be a function-scope static instead.

lib/Target/AMDGPU/SIInsertWaitcnts.cpp

 #include <memory>
 #include <utility>
 #include <vector>
 
 #define DEBUG_TYPE "si-insert-waitcnts"
 
 using namespace llvm;
 
+static cl::opt<unsigned> ForceZeroFlag(
+  "amdgpu-waitcnt-forcezero",
+  cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
+  cl::init(0), cl::Hidden);
+
+static cl::opt<unsigned> ForceExpFlag(
+  "amdgpu-waitcnt-forceexp",
+  cl::desc("Force emit a s_waitcnt expcnt(0) before the first <n> instrs"),
+  cl::init(0), cl::Hidden);
+
+static cl::opt<unsigned> ForceLgkmFlag(
+  "amdgpu-waitcnt-forcelgkm",
+  cl::desc("Force emit a s_waitcnt lgkmcnt(0) before the first <n> instrs"),
+  cl::init(0), cl::Hidden);
+
+static cl::opt<unsigned> ForceVmFlag(
+  "amdgpu-waitcnt-forcevm",
+  cl::desc("Force emit a s_waitcnt vmcnt(0) before the first <n> instrs"),
+  cl::init(0), cl::Hidden);
+
 namespace {
 
 // Class of object that encapsulates latest instruction counter score
 // associated with the operand.  Used for determining whether
 // s_waitcnt instruction needs to be emited.
 
 #define CNT_MASK(t) (1u << (t))
 
DenseMap<MachineBasicBlock *, std::unique_ptr<BlockWaitcntBrackets>>
       BlockWaitcntBracketsMap;
 
   DenseSet<MachineBasicBlock *> BlockWaitcntProcessedSet;
 
   DenseMap<MachineLoop *, std::unique_ptr<LoopWaitcntData>> LoopWaitcntDataMap;
 
   std::vector<std::unique_ptr<BlockWaitcntBrackets>> KillWaitBrackets;
 
+  bool ForceZero = false;
+  int32_t ForceSwaitcnt[NUM_INST_CNTS];
+
 public:
   static char ID;
 
   SIInsertWaitcnts() : MachineFunctionPass(ID) {}
 
   bool runOnMachineFunction(MachineFunction &MF) override;
 
   StringRef getPassName() const override {
     return "SI insert wait instructions";
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesCFG();
     AU.addRequired<MachineLoopInfo>();
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 
   void addKillWaitBracket(BlockWaitcntBrackets *Bracket) {
     // The waitcnt information is copied because it changes as the block is
     // traversed.
     KillWaitBrackets.push_back(
         llvm::make_unique<BlockWaitcntBrackets>(*Bracket));
   }
 
+  bool ForceEmit() const {
+    for (enum InstCounterType T = VM_CNT; T < NUM_INST_CNTS;
+         T = (enum InstCounterType)(T + 1))
+      if (ForceSwaitcnt[T] > 0)
+        return true;
+    return false;
+  }
+
   bool mayAccessLDSThroughFlat(const MachineInstr &MI) const;
   MachineInstr *generateSWaitCntInstBefore(MachineInstr &MI,
                                            BlockWaitcntBrackets *ScoreBrackets);
   void updateEventWaitCntAfter(MachineInstr &Inst,
                                BlockWaitcntBrackets *ScoreBrackets);
   void mergeInputScoreBrackets(MachineBasicBlock &Block);
   MachineBasicBlock *loopBottom(const MachineLoop *Loop);
   void insertWaitcntInBlock(MachineFunction &MF, MachineBasicBlock &Block);
for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
               EXP_CNT, ScoreBrackets->getRegScore(RegNo, EXP_CNT));
         }
         EmitSwaitcnt |= ScoreBrackets->updateByWait(
             LGKM_CNT, ScoreBrackets->getRegScore(RegNo, LGKM_CNT));
       }
     } // End of for loop that looks at all dest operands.
   }
 
-  // TODO: Tie force zero to a compiler triage option.
-  bool ForceZero = false;
-
   // Check to see if this is an S_BARRIER, and if an implicit S_WAITCNT 0
   // occurs before the instruction. Doing it here prevents any additional
   // S_WAITCNTs from being emitted if the instruction was marked as
   // requiring a WAITCNT beforehand.
   if (MI.getOpcode() == AMDGPU::S_BARRIER &&
       !ST->hasAutoWaitcntBeforeBarrier()) {
     EmitSwaitcnt |=
         ScoreBrackets->updateByWait(VM_CNT, ScoreBrackets->getScoreUB(VM_CNT));
if (ScoreBrackets->getScoreLB(LGKM_CNT) <
       // another s_waitcnt inserted right after this if there are non-LGKM
       // instructions still outstanding.
       ForceZero = true;
       EmitSwaitcnt = true;
     }
   }
 
   // Does this operand processing indicate s_wait counter update?
-  if (EmitSwaitcnt) {
+  if (EmitSwaitcnt || ForceEmit()) {
     int CntVal[NUM_INST_CNTS];
 
     bool UseDefaultWaitcntStrategy = true;
     if (ForceZero) {
       // Force all waitcnts to 0.
       for (enum InstCounterType T = VM_CNT; T < NUM_INST_CNTS;
            T = (enum InstCounterType)(T + 1)) {
         ScoreBrackets->setScoreLB(T, ScoreBrackets->getScoreUB(T));
if (UseDefaultWaitcntStrategy) {
           // If we are not waiting for a particular counter then encode
           // it as -1 which means "don't care."
           CntVal[T] = -1;
         }
       }
     }
 
     // If we are not waiting on any counter we can skip the wait altogether.
-    if (EmitSwaitcnt != 0) {
+    if (EmitSwaitcnt != 0 || ForceEmit()) {
       MachineInstr *OldWaitcnt = ScoreBrackets->getWaitcnt();
       int Imm = (!OldWaitcnt) ? 0 : OldWaitcnt->getOperand(0).getImm();
       if (!OldWaitcnt || (AMDGPU::decodeVmcnt(IV, Imm) !=
                           (CntVal[VM_CNT] & AMDGPU::getVmcntBitMask(IV))) ||
           (AMDGPU::decodeExpcnt(IV, Imm) !=
            (CntVal[EXP_CNT] & AMDGPU::getExpcntBitMask(IV))) ||
           (AMDGPU::decodeLgkmcnt(IV, Imm) !=
            (CntVal[LGKM_CNT] & AMDGPU::getLgkmcntBitMask(IV)))) {
if (EmitSwaitcnt != 0 || ForceEmit()) {
       if (OldWaitcnt && OldWaitcnt->getOpcode() != AMDGPU::S_WAITCNT) {
         SWaitInst = OldWaitcnt;
       } else {
         SWaitInst = MF.CreateMachineInstr(TII->get(AMDGPU::S_WAITCNT),
                                           MI.getDebugLoc());
         CompilerGeneratedWaitcntSet.insert(SWaitInst);
       }
 
+      if (!EmitSwaitcnt) {
+        for (enum InstCounterType T = VM_CNT; T < NUM_INST_CNTS;
+             T = (enum InstCounterType)(T + 1)) {
+          if (ForceSwaitcnt[T] > 0 ) {
+            DEBUG(dbgs() << "ForceSwaitcnt[" << T << "]: "
+                  << ForceSwaitcnt[T] << '\n';);
+          }
+        }
+      }
+
       const MachineOperand &Op =
           MachineOperand::CreateImm(AMDGPU::encodeWaitcnt(
-              IV, CntVal[VM_CNT], CntVal[EXP_CNT], CntVal[LGKM_CNT]));
+              IV,
+              (ForceSwaitcnt[VM_CNT]   > 0) ? 0 : CntVal[VM_CNT],
+              (ForceSwaitcnt[EXP_CNT]  > 0) ? 0 : CntVal[EXP_CNT],
+              (ForceSwaitcnt[LGKM_CNT] > 0) ? 0 : CntVal[LGKM_CNT]));
       SWaitInst->addOperand(MF, Op);
 
+      if (!EmitSwaitcnt) {
+        for (enum InstCounterType T = VM_CNT; T < NUM_INST_CNTS;
+             T = (enum InstCounterType)(T + 1)) {
+          --ForceSwaitcnt[T];
+        }
+      }
+
       if (CntVal[EXP_CNT] == 0) {
         ScoreBrackets->setMixedExpTypes(false);
       }
     }
   }
 
   return SWaitInst;
 }
for (MachineBasicBlock *MBB : Loop->blocks())
     if (MBB->getNumber() > Bottom->getNumber())
       Bottom = MBB;
   return Bottom;
 }
 
 // Generate s_waitcnt instructions where needed.
 void SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
                                             MachineBasicBlock &Block) {
+  static int32_t InstCnt = 0;
+
   // Initialize the state information.
   mergeInputScoreBrackets(Block);
 
   BlockWaitcntBrackets *ScoreBrackets = BlockWaitcntBracketsMap[&Block].get();
 
   DEBUG({
-    dbgs() << "Block" << Block.getNumber();
+    dbgs() << "*** Block" << Block.getNumber() << " ***";
     ScoreBrackets->dump();
   });
 
   // Walk over the instructions.
   for (MachineBasicBlock::iterator Iter = Block.begin(), E = Block.end();
        Iter != E;) {
     MachineInstr &Inst = *Iter;
     // Remove any previously existing waitcnts.

 #endif
 
     ScoreBrackets->clearWaitcnt();
 
     if (SWaitInst) {
       DEBUG({ SWaitInst->print(dbgs() << '\n'); });
     }
     DEBUG({
-      Inst.print(dbgs());
+      dbgs() << "Instr" << ++InstCnt << ": " << Inst;
       ScoreBrackets->dump();
     });
 
     // Check to see if this is a GWS instruction. If so, and if this is CI or
     // VI, then the generated code sequence will include an S_WAITCNT 0.
     // TODO: Are these the only GWS instructions?
     if (Inst.getOpcode() == AMDGPU::DS_GWS_INIT ||
         Inst.getOpcode() == AMDGPU::DS_GWS_SEMA_V ||
bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
   TII = ST->getInstrInfo();
   TRI = &TII->getRegisterInfo();
   MRI = &MF.getRegInfo();
   MLI = &getAnalysis<MachineLoopInfo>();
   IV = AMDGPU::IsaInfo::getIsaVersion(ST->getFeatureBits());
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
   AMDGPUASI = ST->getAMDGPUAS();
 
+  ForceZero = ForceZeroFlag;
+  ForceSwaitcnt[VM_CNT] = ForceVmFlag;
+  ForceSwaitcnt[EXP_CNT] = ForceExpFlag;
+  ForceSwaitcnt[LGKM_CNT] = ForceLgkmFlag;
+
   HardwareLimits.VmcntMax = AMDGPU::getVmcntBitMask(IV);
   HardwareLimits.ExpcntMax = AMDGPU::getExpcntBitMask(IV);
   HardwareLimits.LgkmcntMax = AMDGPU::getLgkmcntBitMask(IV);
 
   HardwareLimits.NumVGPRsMax = ST->getAddressableNumVGPRs();
   HardwareLimits.NumSGPRsMax = ST->getAddressableNumSGPRs();
   assert(HardwareLimits.NumVGPRsMax <= SQ_MAX_PGM_VGPRS);
   assert(HardwareLimits.NumSGPRsMax <= SQ_MAX_PGM_SGPRS);

test/CodeGen/AMDGPU/waitcnt-debug.mir

+# RUN: llc -mtriple=amdgcn -verify-machineinstrs -run-pass si-insert-waitcnts -amdgpu-waitcnt-forcelgkm=1 -o - %s | FileCheck -check-prefixes=GCN,LGKM %s
+# RUN: llc -mtriple=amdgcn -verify-machineinstrs -run-pass si-insert-waitcnts -amdgpu-waitcnt-forceexp=2 -o - %s | FileCheck -check-prefixes=GCN,EXP %s
+# RUN: llc -mtriple=amdgcn -verify-machineinstrs -run-pass si-insert-waitcnts -amdgpu-waitcnt-forcevm=3 -o - %s | FileCheck -check-prefixes=GCN,VM %s
+# RUN: llc -mtriple=amdgcn -verify-machineinstrs -run-pass si-insert-waitcnts -amdgpu-waitcnt-forcezero=1 -amdgpu-waitcnt-forcevm=2 -o - %s | FileCheck -check-prefixes=GCN,ZERO %s
+
+# check that the waitcnt pass options that force insertion of waitcnt instructions are working as expected
+
+...
+# GCN-LABEL: name: waitcnt-debug
+# LGKM: S_WAITCNT 127
+# LGKM-NEXT: S_NOP 0
+# LGKM-NEXT: S_NOP 0
+
+# EXP: S_WAITCNT 3855
+# EXP-NEXT: S_NOP 0
+# EXP-NEXT: S_WAITCNT 3855
+# EXP-NEXT: S_NOP 0
+
+# VM: S_WAITCNT 3952
+# VM-NEXT: S_NOP 0
+# VM-NEXT: S_WAITCNT 3952
+# VM-NEXT: S_NOP 0
+# VM-NEXT: S_WAITCNT 3952
+# VM-NEXT: S_NOP 0
+
+# ZERO: S_WAITCNT 0
+# ZERO-NEXT: S_WAITCNT 0
+# ZERO-NEXT: S_NOP 0
+# ZERO-NEXT: S_WAITCNT 0
+# ZERO-NEXT: S_NOP 0
+
+name:            waitcnt-debug
+liveins:
+body:             |
+  bb.0:
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+...
+