AMDGPU/SI: Handle hazard with s_rfe_b64

[• tstellarAMD]

Remove debug statements.

[nhaehnle]

LGTM

lib/Target/AMDGPU/GCNHazardRecognizer.h

class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
   int checkVMEMHazards(MachineInstr* VMEM);
   int checkDPPHazards(MachineInstr *DPP);
   int checkDivFMasHazards(MachineInstr *DivFMas);
   int checkGetRegHazards(MachineInstr *GetRegInstr);
   int checkSetRegHazards(MachineInstr *SetRegInstr);
   int createsVALUHazard(const MachineInstr &MI);
   int checkVALUHazards(MachineInstr *VALU);
   int checkRWLaneHazards(MachineInstr *RWLane);
+  int checkRFEHazards(MachineInstr *RFE);
 public:
   GCNHazardRecognizer(const MachineFunction &MF);
   // We can only issue one instruction per cycle.
   bool atIssueLimit() const override { return true; }
   void EmitInstruction(SUnit *SU) override;
   void EmitInstruction(MachineInstr *MI) override;
   HazardType getHazardType(SUnit *SU, int Stalls) override;
   void EmitNoop() override;

lib/Target/AMDGPU/GCNHazardRecognizer.cpp

 static bool isSSetReg(unsigned Opcode) {
   return Opcode == AMDGPU::S_SETREG_B32 || Opcode == AMDGPU::S_SETREG_IMM32_B32;
 }
 
 static bool isRWLane(unsigned Opcode) {
   return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
 }
 
-static bool getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
+static bool isRFE(unsigned Opcode) {
+  return Opcode == AMDGPU::S_RFE_B64;
+}
+
+static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
 
   const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
                                                      AMDGPU::OpName::simm16);
   return RegOp->getImm() & 0x3f;
 }
 
 ScheduleHazardRecognizer::HazardType
 GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
     return NoopHazard;
 
   if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
     return NoopHazard;
 
   if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
     return NoopHazard;
 
+  if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
+    return NoopHazard;
+
   return NoHazard;
 }
 
 unsigned GCNHazardRecognizer::PreEmitNoops(SUnit *SU) {
   return PreEmitNoops(SU->getInstr());
 }
 
 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
   }
 
   if (isSGetReg(MI->getOpcode()))
     return std::max(0, checkGetRegHazards(MI));
 
   if (isSSetReg(MI->getOpcode()))
     return std::max(0, checkSetRegHazards(MI));
 
+  if (isRFE(MI->getOpcode()))
+    return std::max(0, checkRFEHazards(MI));
+
   return 0;
 }
 
 void GCNHazardRecognizer::EmitNoop() {
   EmittedInstrs.push_front(nullptr);
 }
 
 void GCNHazardRecognizer::AdvanceCycle() {
int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
   auto IsHazardFn = [TII] (MachineInstr *MI) {
     return TII->isVALU(*MI);
   };
 
   const int RWLaneWaitStates = 4;
   int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn);
   return RWLaneWaitStates - WaitStatesSince;
 }
+
+int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
+
+  if (ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
+    return 0;
+
+  const SIInstrInfo *TII = ST.getInstrInfo();
+
+  const int RFEWaitStates = 1;
+
+  auto IsHazardFn = [TII] (MachineInstr *MI) {
+    return getHWReg(TII, *MI) == AMDGPU::Hwreg::ID_TRAPSTS;
+  };
+  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
+  return RFEWaitStates - WaitStatesNeeded;
+}

lib/Target/AMDGPU/SIDefines.h

 
 } // namespace SendMsg
 
 namespace Hwreg { // Encoding of SIMM16 used in s_setreg/getreg* insns.
 
 enum Id { // HwRegCode, (6) [5:0]
   ID_UNKNOWN_ = -1,
   ID_SYMBOLIC_FIRST_ = 1, // There are corresponding symbolic names defined.
+  ID_MODE = 1,
+  ID_STATUS = 2,
+  ID_TRAPSTS = 3,
+  ID_HW_ID = 4,
+  ID_GPR_ALLOC = 5,
+  ID_LDS_ALLOC = 6,
+  ID_IB_STS = 7,
   ID_SYMBOLIC_LAST_ = 8,
   ID_SHIFT_ = 0,
   ID_WIDTH_ = 6,
   ID_MASK_ = (((1 << ID_WIDTH_) - 1) << ID_SHIFT_)
 };
 
 enum Offset { // Offset, (5) [10:6]
   OFFSET_DEFAULT_ = 0,

test/CodeGen/MIR/AMDGPU/inserted-wait-states.mir

 # RUN: llc -march=amdgcn -mcpu=tahiti -run-pass post-RA-hazard-rec  %s -o - | FileCheck %s -check-prefixes=GCN
 # RUN: llc -march=amdgcn -mcpu=hawaii -run-pass post-RA-hazard-rec  %s -o - | FileCheck %s -check-prefixes=GCN,CIVI
 # RUN: llc -march=amdgcn -mcpu=fiji -run-pass post-RA-hazard-rec  %s -o - | FileCheck %s -check-prefixes=GCN,CIVI,VI
 
 --- |
   define void @div_fmas() { ret void }
   define void @s_getreg() { ret void }
   define void @s_setreg() { ret void }
   define void @vmem_gt_8dw_store() { ret void }
   define void @readwrite_lane() { ret void }
+  define void @rfe() { ret void }
 ...
 ---
 # GCN-LABEL: name: div_fmas
 
 # GCN-LABEL: bb.0:
 # GCN: S_MOV_B64
 # GCN-NOT: S_NOP
 # GCN: V_DIV_FMAS
bb.2:
     S_BRANCH %bb.3
 
   bb.3:
     %vgpr0,implicit %vcc = V_ADD_I32_e32 %vgpr1, %vgpr2, implicit %vcc, implicit %exec
     %vgpr4 = V_WRITELANE_B32 %sgpr4, %vcc_lo
     S_ENDPGM
 
 ...
+
+...
+---
+
+# GCN-LABEL: name: rfe
+
+# GCN-LABEL: bb.0:
+# GCN: S_SETREG
+# VI: S_NOP
+# GCN-NEXT: S_RFE_B64
+
+# GCN-LABEL: bb.1:
+# GCN: S_SETREG
+# GCN-NEXT: S_RFE_B64
+
+name: rfe
+
+body: |
+  bb.0:
+    successors: %bb.1
+    S_SETREG_B32 %sgpr0, 3
+    S_RFE_B64 %sgpr2_sgpr3
+    S_BRANCH %bb.1
+
+  bb.1:
+    S_SETREG_B32 %sgpr0, 0
+    S_RFE_B64 %sgpr2_sgpr3
+    S_ENDPGM
+
+...