AMDGPU/SI: Handle s_setreg hazard in GCNHazardRecognizer

[• tstellarAMD]

Ping.

[nhaehnle]

One small comment, apart from that LGTM.

test/CodeGen/MIR/AMDGPU/inserted-wait-states.mir

140 ↗

(On Diff #74416)

GCN-NEXT?

llvm/trunk/lib/Target/AMDGPU/GCNHazardRecognizer.h

class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
   int getWaitStatesSinceSetReg(function_ref<bool(MachineInstr *)> IsHazard);
 
   int checkSMEMSoftClauseHazards(MachineInstr *SMEM);
   int checkSMRDHazards(MachineInstr *SMRD);
   int checkVMEMHazards(MachineInstr* VMEM);
   int checkDPPHazards(MachineInstr *DPP);
   int checkDivFMasHazards(MachineInstr *DivFMas);
   int checkGetRegHazards(MachineInstr *GetRegInstr);
+  int checkSetRegHazards(MachineInstr *SetRegInstr);
 public:
   GCNHazardRecognizer(const MachineFunction &MF);
   // We can only issue one instruction per cycle.
   bool atIssueLimit() const override { return true; }
   void EmitInstruction(SUnit *SU) override;
   void EmitInstruction(MachineInstr *MI) override;
   HazardType getHazardType(SUnit *SU, int Stalls) override;
   void EmitNoop() override;

llvm/trunk/lib/Target/AMDGPU/GCNHazardRecognizer.cpp

if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
     return NoopHazard;
 
   if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
     return NoopHazard;
 
   if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
     return NoopHazard;
 
+  if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
+    return NoopHazard;
+
   return NoHazard;
 }
 
 unsigned GCNHazardRecognizer::PreEmitNoops(SUnit *SU) {
   return PreEmitNoops(SU->getInstr());
 }
 
 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
   if (SIInstrInfo::isSMRD(*MI))
     return std::max(0, checkSMRDHazards(MI));
 
   if (SIInstrInfo::isVMEM(*MI))
     return std::max(0, checkVMEMHazards(MI));
 
   if (SIInstrInfo::isDPP(*MI))
     return std::max(0, checkDPPHazards(MI));
 
   if (isDivFMas(MI->getOpcode()))
     return std::max(0, checkDivFMasHazards(MI));
 
   if (isSGetReg(MI->getOpcode()))
     return std::max(0, checkGetRegHazards(MI));
 
+  if (isSSetReg(MI->getOpcode()))
+    return std::max(0, checkSetRegHazards(MI));
+
   return 0;
 }
 
 void GCNHazardRecognizer::EmitNoop() {
   EmittedInstrs.push_front(nullptr);
 }
 
 void GCNHazardRecognizer::AdvanceCycle() {
int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
   const int GetRegWaitStates = 2;
   auto IsHazardFn = [TII, GetRegHWReg] (MachineInstr *MI) {
     return GetRegHWReg == getHWReg(TII, *MI);
   };
   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
 
   return GetRegWaitStates - WaitStatesNeeded;
 }
+
+int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
+  const SIInstrInfo *TII = ST.getInstrInfo();
+  unsigned HWReg = getHWReg(TII, *SetRegInstr);
+
+  const int SetRegWaitStates =
+      ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ? 1 : 2;
+  auto IsHazardFn = [TII, HWReg] (MachineInstr *MI) {
+    return HWReg == getHWReg(TII, *MI);
+  };
+  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
+  return SetRegWaitStates - WaitStatesNeeded;
+}

llvm/trunk/test/CodeGen/MIR/AMDGPU/inserted-wait-states.mir

-# RUN: llc -march=amdgcn -run-pass post-RA-hazard-rec  %s -o - | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass post-RA-hazard-rec  %s -o - | FileCheck %s -check-prefixes=GCN
+# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass post-RA-hazard-rec  %s -o - | FileCheck %s -check-prefixes=GCN
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass post-RA-hazard-rec  %s -o - | FileCheck %s -check-prefixes=GCN,VI
 
 --- |
   define void @div_fmas() { ret void }
   define void @s_getreg() { ret void }
+  define void @s_setreg() { ret void }
 ...
 ---
-# CHECK-LABEL: name: div_fmas
+# GCN-LABEL: name: div_fmas
 
-# CHECK-LABEL: bb.0:
-# CHECK: S_MOV_B64
-# CHECK-NOT: S_NOP
-# CHECK: V_DIV_FMAS
-
-# CHECK-LABEL: bb.1:
-# CHECK: V_CMP_EQ_I32
-# CHECK: S_NOP
-# CHECK: S_NOP
-# CHECK: S_NOP
-# CHECK: S_NOP
-# CHECK: V_DIV_FMAS_F32
-
-# CHECK-LABEL: bb.2:
-# CHECK: V_CMP_EQ_I32
-# CHECK: S_NOP
-# CHECK: S_NOP
-# CHECK: S_NOP
-# CHECK: S_NOP
-# CHECK: V_DIV_FMAS_F32
-
-# CHECK-LABEL: bb.3:
-# CHECK: V_DIV_SCALE_F32
-# CHECK: S_NOP
-# CHECK: S_NOP
-# CHECK: S_NOP
-# CHECK: S_NOP
-# CHECK: V_DIV_FMAS_F32
+# GCN-LABEL: bb.0:
+# GCN: S_MOV_B64
+# GCN-NOT: S_NOP
+# GCN: V_DIV_FMAS
+
+# GCN-LABEL: bb.1:
+# GCN: V_CMP_EQ_I32
+# GCN: S_NOP
+# GCN: S_NOP
+# GCN: S_NOP
+# GCN: S_NOP
+# GCN: V_DIV_FMAS_F32
+
+# GCN-LABEL: bb.2:
+# GCN: V_CMP_EQ_I32
+# GCN: S_NOP
+# GCN: S_NOP
+# GCN: S_NOP
+# GCN: S_NOP
+# GCN: V_DIV_FMAS_F32
+
+# GCN-LABEL: bb.3:
+# GCN: V_DIV_SCALE_F32
+# GCN: S_NOP
+# GCN: S_NOP
+# GCN: S_NOP
+# GCN: S_NOP
+# GCN: V_DIV_FMAS_F32
 name: div_fmas
 
 body: |
   bb.0:
     successors: %bb.1
     %vcc = S_MOV_B64 0
     %vgpr0 = V_DIV_FMAS_F32 0, %vgpr1, 0, %vgpr2, 0, %vgpr3, 0, 0, implicit %vcc, implicit %exec
     S_BRANCH %bb.1
bb.3:
     %vgpr4, %vcc = V_DIV_SCALE_F32 0, %vgpr1, 0, %vgpr1, 0, %vgpr3, 0, 0, implicit %exec
     %vgpr0 = V_DIV_FMAS_F32 0, %vgpr1, 0, %vgpr2, 0, %vgpr3, 0, 0, implicit %vcc, implicit %exec
     S_ENDPGM
 
 ...
 
 ...
 ---
-# CHECK-LABEL: name: s_getreg
+# GCN-LABEL: name: s_getreg
 
-# CHECK-LABEL: bb.0:
-# CHECK: S_SETREG
-# CHECK: S_NOP 0
-# CHECK: S_NOP 0
-# CHECK: S_GETREG
-
-# CHECK-LABEL: bb.1:
-# CHECK: S_SETREG_IMM32
-# CHECK: S_NOP 0
-# CHECK: S_NOP 0
-# CHECK: S_GETREG
-
-# CHECK-LABEL: bb.2:
-# CHECK: S_SETREG
-# CHECK: S_NOP 0
-# CHECK: S_GETREG
-
-# CHECK-LABEL: bb.3:
-# CHECK: S_SETREG
-# CHECK-NEXT: S_GETREG
+# GCN-LABEL: bb.0:
+# GCN: S_SETREG
+# GCN: S_NOP 0
+# GCN: S_NOP 0
+# GCN: S_GETREG
+
+# GCN-LABEL: bb.1:
+# GCN: S_SETREG_IMM32
+# GCN: S_NOP 0
+# GCN: S_NOP 0
+# GCN: S_GETREG
+
+# GCN-LABEL: bb.2:
+# GCN: S_SETREG
+# GCN: S_NOP 0
+# GCN: S_GETREG
+
+# GCN-LABEL: bb.3:
+# GCN: S_SETREG
+# GCN-NEXT: S_GETREG
 
 name: s_getreg
 
 body: |
   bb.0:
     successors: %bb.1
     S_SETREG_B32 %sgpr0, 1
     %sgpr1 = S_GETREG_B32 1
bb.2:
     %sgpr2 = S_GETREG_B32 1
     S_BRANCH %bb.3
 
   bb.3:
     S_SETREG_B32 %sgpr0, 0
     %sgpr1 = S_GETREG_B32 1
     S_ENDPGM
 ...
+
+...
+---
+# GCN-LABEL: name: s_setreg
+
+# GCN-LABEL: bb.0:
+# GCN: S_SETREG
+# GCN: S_NOP 0
+# VI: S_NOP 0
+# GCN-NEXT: S_SETREG
+
+# GCN-LABEL: bb.1:
+# GCN: S_SETREG
+# GCN: S_NOP 0
+# VI: S_NOP 0
+# GCN-NEXT: S_SETREG
+
+# GCN-LABEL: bb.2:
+# GCN: S_SETREG
+# GCN-NEXT: S_SETREG
+
+name: s_setreg
+
+body: |
+  bb.0:
+    successors: %bb.1
+    S_SETREG_B32 %sgpr0, 1
+    S_SETREG_B32 %sgpr1, 1
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.2
+    S_SETREG_B32 %sgpr0, 64
+    S_SETREG_B32 %sgpr1, 128
+    S_BRANCH %bb.2
+
+  bb.2:
+    S_SETREG_B32 %sgpr0, 1
+    S_SETREG_B32 %sgpr1, 0
+    S_ENDPGM
+...