AMDGPU/SI: Handle s_getreg hazard in GCNHazardRecognizer

[• tstellarAMD]

Fix whitespace.

[arsenm]

LGTM

lib/Target/AMDGPU/GCNHazardRecognizer.cpp

320 ↗

(On Diff #74406)

HWReg != getHWReg looks confusing to me. getHWReg(TII, *MI) != GetRegHWReg?

[• tstellarAMD]

Refactor to make it easier to share code when adding support for other hazards.

llvm/trunk/lib/Target/AMDGPU/GCNHazardRecognizer.h

class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
   MachineInstr *CurrCycleInstr;
   std::list<MachineInstr*> EmittedInstrs;
   const MachineFunction &MF;
   const SISubtarget &ST;
 
   int getWaitStatesSinceDef(unsigned Reg,
                             function_ref<bool(MachineInstr *)> IsHazardDef =
                                 [](MachineInstr *) { return true; });
+  int getWaitStatesSinceSetReg(function_ref<bool(MachineInstr *)> IsHazard);
 
   int checkSMEMSoftClauseHazards(MachineInstr *SMEM);
   int checkSMRDHazards(MachineInstr *SMRD);
   int checkVMEMHazards(MachineInstr* VMEM);
   int checkDPPHazards(MachineInstr *DPP);
   int checkDivFMasHazards(MachineInstr *DivFMas);
+  int checkGetRegHazards(MachineInstr *GetRegInstr);
 public:
   GCNHazardRecognizer(const MachineFunction &MF);
   // We can only issue one instruction per cycle.
   bool atIssueLimit() const override { return true; }
   void EmitInstruction(SUnit *SU) override;
   void EmitInstruction(MachineInstr *MI) override;
   HazardType getHazardType(SUnit *SU, int Stalls) override;
   void EmitNoop() override;

llvm/trunk/lib/Target/AMDGPU/GCNHazardRecognizer.cpp

 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
   CurrCycleInstr = MI;
 }
 
 static bool isDivFMas(unsigned Opcode) {
   return Opcode == AMDGPU::V_DIV_FMAS_F32 || Opcode == AMDGPU::V_DIV_FMAS_F64;
 }
 
+static bool isSGetReg(unsigned Opcode) {
+  return Opcode == AMDGPU::S_GETREG_B32;
+}
+
+static bool isSSetReg(unsigned Opcode) {
+  return Opcode == AMDGPU::S_SETREG_B32 || Opcode == AMDGPU::S_SETREG_IMM32_B32;
+}
+
+static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
+
+  const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
+                                                     AMDGPU::OpName::simm16);
+  return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
+}
+
 ScheduleHazardRecognizer::HazardType
 GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
   MachineInstr *MI = SU->getInstr();
 
   if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
     return NoopHazard;
 
   if (SIInstrInfo::isVMEM(*MI) && checkVMEMHazards(MI) > 0)
     return NoopHazard;
 
   if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
     return NoopHazard;
 
   if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
     return NoopHazard;
 
+  if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
+    return NoopHazard;
+
   return NoHazard;
 }
 
 unsigned GCNHazardRecognizer::PreEmitNoops(SUnit *SU) {
   return PreEmitNoops(SU->getInstr());
 }
 
 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
   if (SIInstrInfo::isSMRD(*MI))
     return std::max(0, checkSMRDHazards(MI));
 
   if (SIInstrInfo::isVMEM(*MI))
     return std::max(0, checkVMEMHazards(MI));
 
   if (SIInstrInfo::isDPP(*MI))
     return std::max(0, checkDPPHazards(MI));
 
   if (isDivFMas(MI->getOpcode()))
     return std::max(0, checkDivFMasHazards(MI));
 
+  if (isSGetReg(MI->getOpcode()))
+    return std::max(0, checkGetRegHazards(MI));
+
   return 0;
 }
 
 void GCNHazardRecognizer::EmitNoop() {
   EmittedInstrs.push_front(nullptr);
 }
 
 void GCNHazardRecognizer::AdvanceCycle() {
for (MachineInstr *MI : EmittedInstrs) {
     if (!MI || !IsHazardDef(MI))
       continue;
     if (MI->modifiesRegister(Reg, TRI))
       return WaitStates;
   }
   return std::numeric_limits<int>::max();
 }
 
+int GCNHazardRecognizer::getWaitStatesSinceSetReg(
+    function_ref<bool(MachineInstr *)> IsHazard) {
+
+  int WaitStates = -1;
+  for (MachineInstr *MI : EmittedInstrs) {
+    ++WaitStates;
+    if (!MI || !isSSetReg(MI->getOpcode()) || !IsHazard(MI))
+      continue;
+    return WaitStates;
+  }
+  return std::numeric_limits<int>::max();
+}
+
 //===----------------------------------------------------------------------===//
 // No-op Hazard Detection
 //===----------------------------------------------------------------------===//
 
 static void addRegsToSet(iterator_range<MachineInstr::const_mop_iterator> Ops,
                          std::set<unsigned> &Set) {
   for (const MachineOperand &Op : Ops) {
     if (Op.isReg())
int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
   // v_div_fmas requires 4 wait states after a write to vcc from a VALU
   // instruction.
   const int DivFMasWaitStates = 4;
   auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
   int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn);
 
   return DivFMasWaitStates - WaitStatesNeeded;
 }
+
+int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
+  const SIInstrInfo *TII = ST.getInstrInfo();
+  unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
+
+  const int GetRegWaitStates = 2;
+  auto IsHazardFn = [TII, GetRegHWReg] (MachineInstr *MI) {
+    return GetRegHWReg == getHWReg(TII, *MI);
+  };
+  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
+
+  return GetRegWaitStates - WaitStatesNeeded;
+}

llvm/trunk/test/CodeGen/MIR/AMDGPU/inserted-wait-states.mir

 # RUN: llc -march=amdgcn -run-pass post-RA-hazard-rec  %s -o - | FileCheck %s
 
+--- |
+  define void @div_fmas() { ret void }
+  define void @s_getreg() { ret void }
+...
+---
+# CHECK-LABEL: name: div_fmas
+
 # CHECK-LABEL: bb.0:
 # CHECK: S_MOV_B64
 # CHECK-NOT: S_NOP
 # CHECK: V_DIV_FMAS
 
 # CHECK-LABEL: bb.1:
 # CHECK: V_CMP_EQ_I32
 # CHECK: S_NOP

 
 # CHECK-LABEL: bb.3:
 # CHECK: V_DIV_SCALE_F32
 # CHECK: S_NOP
 # CHECK: S_NOP
 # CHECK: S_NOP
 # CHECK: S_NOP
 # CHECK: V_DIV_FMAS_F32
---- |
-  define void @test0() { ret void }
-...
----
-name: test0
+name: div_fmas
 
 body: |
   bb.0:
     successors: %bb.1
     %vcc = S_MOV_B64 0
     %vgpr0 = V_DIV_FMAS_F32 0, %vgpr1, 0, %vgpr2, 0, %vgpr3, 0, 0, implicit %vcc, implicit %exec
     S_BRANCH %bb.1
 
   bb.1:
     successors: %bb.2
     implicit %vcc = V_CMP_EQ_I32_e32 %vgpr1, %vgpr2, implicit %exec
     %vgpr0 = V_DIV_FMAS_F32 0, %vgpr1, 0, %vgpr2, 0, %vgpr3, 0, 0, implicit %vcc, implicit %exec
     S_BRANCH %bb.2
 
   bb.2:
     successors: %bb.3
     %vcc = V_CMP_EQ_I32_e64 %vgpr1, %vgpr2, implicit %exec
     %vgpr0 = V_DIV_FMAS_F32 0, %vgpr1, 0, %vgpr2, 0, %vgpr3, 0, 0, implicit %vcc, implicit %exec
     S_BRANCH %bb.3
 
   bb.3:
     %vgpr4, %vcc = V_DIV_SCALE_F32 0, %vgpr1, 0, %vgpr1, 0, %vgpr3, 0, 0, implicit %exec
     %vgpr0 = V_DIV_FMAS_F32 0, %vgpr1, 0, %vgpr2, 0, %vgpr3, 0, 0, implicit %vcc, implicit %exec
     S_ENDPGM
+
+...
+
+...
+---
+# CHECK-LABEL: name: s_getreg
+
+# CHECK-LABEL: bb.0:
+# CHECK: S_SETREG
+# CHECK: S_NOP 0
+# CHECK: S_NOP 0
+# CHECK: S_GETREG
+
+# CHECK-LABEL: bb.1:
+# CHECK: S_SETREG_IMM32
+# CHECK: S_NOP 0
+# CHECK: S_NOP 0
+# CHECK: S_GETREG
+
+# CHECK-LABEL: bb.2:
+# CHECK: S_SETREG
+# CHECK: S_NOP 0
+# CHECK: S_GETREG
+
+# CHECK-LABEL: bb.3:
+# CHECK: S_SETREG
+# CHECK-NEXT: S_GETREG
+
+name: s_getreg
+
+body: |
+  bb.0:
+    successors: %bb.1
+    S_SETREG_B32 %sgpr0, 1
+    %sgpr1 = S_GETREG_B32 1
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.2
+    S_SETREG_IMM32_B32 0, 1
+    %sgpr1 = S_GETREG_B32 1
+    S_BRANCH %bb.2
+
+  bb.2:
+    successors: %bb.3
+    S_SETREG_B32 %sgpr0, 1
+    %sgpr1 = S_MOV_B32 0
+    %sgpr2 = S_GETREG_B32 1
+    S_BRANCH %bb.3
+
+  bb.3:
+    S_SETREG_B32 %sgpr0, 0
+    %sgpr1 = S_GETREG_B32 1
+    S_ENDPGM
 ...