AMDGPU/SI: Use hazard recognizer to detect DPP hazards

[• tstellarAMD]

Ping.

[• tstellarAMD]

Ping.

[nhaehnle]

LGTM

[arsenm]

LGTM

lib/Target/AMDGPU/GCNHazardRecognizer.h

class GCNHazardRecognizer : public ScheduleHazardRecognizer {
   const MachineFunction &MF;
 
   int getWaitStatesSinceDef(unsigned Reg,
                             std::function<bool(MachineInstr*)> IsHazardDef =
                             [](MachineInstr*) {return true;});
 
   int checkSMRDHazards(MachineInstr *SMRD);
   int checkVMEMHazards(MachineInstr* VMEM);
+  int checkDPPHazards(MachineInstr *DPP);
 public:
   GCNHazardRecognizer(const MachineFunction &MF);
   // We can only issue on instruction be cycle.
   bool atIssueLimit() const override { return true; }
   void EmitInstruction(SUnit *SU) override;
   void EmitInstruction(MachineInstr *MI) override;
   HazardType getHazardType(SUnit *SU, int Stalls) override;
   void EmitNoop() override;
   unsigned PreEmitNoops(SUnit *SU) override;
   unsigned PreEmitNoops(MachineInstr *) override;
   void RecedeCycle() override;
 };
 
 } // end namespace llvm
 
 #endif //LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H

lib/Target/AMDGPU/GCNHazardRecognizer.cpp

GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
   MachineInstr *MI = SU->getInstr();
 
   if (TII->isSMRD(*MI) && checkSMRDHazards(MI) > 0)
     return NoopHazard;
 
   if (TII->isVMEM(*MI) && checkVMEMHazards(MI) > 0)
     return NoopHazard;
 
+  if (TII->isDPP(*MI) && checkDPPHazards(MI) > 0)
+    return NoopHazard;
+
   return NoHazard;
 }
 
 unsigned GCNHazardRecognizer::PreEmitNoops(SUnit *SU) {
   return PreEmitNoops(SU->getInstr());
 }
 
 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
   const SIInstrInfo *TII =
       static_cast<const SIInstrInfo*>(MF.getSubtarget().getInstrInfo());
 
   if (TII->isSMRD(*MI))
     return std::max(0, checkSMRDHazards(MI));
 
   if (TII->isVMEM(*MI))
     return std::max(0, checkVMEMHazards(MI));
 
+  if (TII->isDPP(*MI))
+    return std::max(0, checkDPPHazards(MI));
+
   return 0;
 }
 
 void GCNHazardRecognizer::EmitNoop() {
   EmittedInstrs.push_front(nullptr);
 }
 
 void GCNHazardRecognizer::RecedeCycle() {
if (!Use.isReg() || TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
       continue;
 
     int WaitStatesNeededForUse =
         VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn);
     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
   }
   return WaitStatesNeeded;
 }
+
+int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
+  const AMDGPUSubtarget &ST = MF.getSubtarget<AMDGPUSubtarget>();
+  const SIRegisterInfo *TRI =
+      static_cast<const SIRegisterInfo*>(ST.getRegisterInfo());
+
+  // Check for DPP VGPR read after VALU VGPR write.
+  int DppVgprWaitStates = 2;
+  int WaitStatesNeeded = 0;
+
+  for (const MachineOperand &Use : DPP->uses()) {
+    if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
+      continue;
+    int WaitStatesNeededForUse =
+        DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg());
+    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
+  }
+
+  return WaitStatesNeeded;
+}

lib/Target/AMDGPU/SIInsertWaits.cpp

private:
   bool unorderedDefines(MachineInstr &MI);
 
   /// \brief Resolve all operand dependencies to counter requirements
   Counters handleOperands(MachineInstr &MI);
 
   /// \brief Insert S_NOP between an instruction writing M0 and S_SENDMSG.
   void handleSendMsg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I);
 
-  /// \param DPP The DPP instruction
-  /// \param SearchI The iterator to start look for hazards.
-  /// \param SearchMBB The basic block we are operating on.
-  /// \param WaitStates Then number of wait states that need to be inserted
-  ///                    When a hazard is detected.
-  void insertDPPWaitStates(MachineBasicBlock::iterator DPP,
-                           MachineBasicBlock::reverse_iterator SearchI,
-                           MachineBasicBlock *SearchMBB,
-                           unsigned WaitStates);
-
-  void insertDPPWaitStates(MachineBasicBlock::iterator DPP);
-
   /// Return true if there are LGKM instrucitons that haven't been waited on
   /// yet.
   bool hasOutstandingLGKM() const;
 
 public:
   static char ID;
 
   SIInsertWaits() :
void SIInsertWaits::handleSendMsg(MachineBasicBlock &MBB,
   for (unsigned i = 0; i < NumOperands; i++) {
     const MachineOperand &Op = I->getOperand(i);
 
     if (Op.isReg() && Op.isDef() && Op.getReg() == AMDGPU::M0)
       LastInstWritesM0 = true;
   }
 }
 
-void SIInsertWaits::insertDPPWaitStates(MachineBasicBlock::iterator DPP,
-                                        MachineBasicBlock::reverse_iterator SearchI,
-                                        MachineBasicBlock *SearchMBB,
-                                        unsigned WaitStates) {
-
-  MachineBasicBlock::reverse_iterator E = SearchMBB->rend();
-
-  for (; WaitStates > 0; --WaitStates, ++SearchI) {
-
-    // If we have reached the start of the block, we need to check predecessors.
-    if (SearchI == E) {
-      for (MachineBasicBlock *Pred : SearchMBB->predecessors()) {
-        // We only need to check fall-through blocks.  Branch instructions
-        // give us enough wait states.
-        if (Pred->getFirstTerminator() == Pred->end()) {
-          insertDPPWaitStates(DPP, Pred->rbegin(), Pred, WaitStates);
-          break;
-        }
-      }
-      return;
-    }
-
-    for (MachineOperand &Op : SearchI->operands()) {
-      if (!Op.isReg() || !Op.isDef())
-        continue;
-
-      if (DPP->readsRegister(Op.getReg(), TRI)) {
-        TII->insertWaitStates(*DPP->getParent(), DPP, WaitStates);
-        return;
-      }
-    }
-  }
-}
-
-void SIInsertWaits::insertDPPWaitStates(MachineBasicBlock::iterator DPP) {
-  MachineBasicBlock::reverse_iterator I(DPP);
-  insertDPPWaitStates(DPP, I, DPP->getParent(), 2);
-}
-
 // FIXME: Insert waits listed in Table 4.2 "Required User-Inserted Wait States"
 // around other non-memory instructions.
 bool SIInsertWaits::runOnMachineFunction(MachineFunction &MF) {
   bool Changes = false;
 
   TII = static_cast<const SIInstrInfo *>(MF.getSubtarget().getInstrInfo());
   TRI =
       static_cast<const SIRegisterInfo *>(MF.getSubtarget().getRegisterInfo());
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
           // bit is updated, so we can restore the bit by reading the value of
           // vcc and then writing it back to the register.
           BuildMI(MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_MOV_B64),
                   AMDGPU::VCC)
                   .addReg(AMDGPU::VCC);
         }
       }
 
-      if (TII->isDPP(*I)) {
-        insertDPPWaitStates(I);
-      }
-
       // Wait for everything before a barrier.
       if (I->getOpcode() == AMDGPU::S_BARRIER)
         Changes |= insertWait(MBB, I, LastIssued);
       else
         Changes |= insertWait(MBB, I, handleOperands(*I));
 
       pushInstruction(MBB, I);
       handleSendMsg(MBB, I);
     }
 
     // Wait for everything at the end of the MBB
     Changes |= insertWait(MBB, MBB.getFirstTerminator(), LastIssued);
   }
 
   return Changes;
 }

test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll

-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=VI -check-prefix=VI-OPT %s
+; RUN: llc -O0 -march=amdgcn -mcpu=tonga -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=VI -check-prefix=VI-NOOPT %s
 
 ; FIXME: The register allocator / scheduler should be able to avoid these hazards.
 
 ; VI-LABEL: {{^}}dpp_test:
 ; VI: v_mov_b32_e32 v0, s{{[0-9]+}}
 ; VI: s_nop 1
 ; VI: v_mov_b32_dpp v0, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 ; encoding: [0xfa,0x02,0x00,0x7e,0x00,0x01,0x08,0x11]
 define void @dpp_test(i32 addrspace(1)* %out, i32 %in) {

 define void @dpp_wait_states(i32 addrspace(1)* %out, i32 %in) {
   %tmp0 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %in, i32 1, i32 1, i32 1, i1 1) #0
   %tmp1 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %tmp0, i32 1, i32 1, i32 1, i1 1) #0
   store i32 %tmp1, i32 addrspace(1)* %out
   ret void
 }
 
 ; VI-LABEL: {{^}}dpp_first_in_bb:
-; VI: s_nop 1
+; VI: ; %endif
+; VI-OPT: s_mov_b32
+; VI-OPT: s_mov_b32
+; VI-NOOPT: s_nop 1
 ; VI: v_mov_b32_dpp [[VGPR0:v[0-9]+]], v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
 ; VI: s_nop 1
 ; VI: v_mov_b32_dpp [[VGPR1:v[0-9]+]], [[VGPR0]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
 ; VI: s_nop 1
 ; VI: v_mov_b32_dpp v{{[0-9]+}}, [[VGPR1]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
 define void @dpp_first_in_bb(float addrspace(1)* %out, float addrspace(1)* %in, float %cond, float %a, float %b) {
   %cmp = fcmp oeq float %cond, 0.0
   br i1 %cmp, label %if, label %else