diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h @@ -48,6 +48,7 @@ const SIInstrInfo &TII; const SIRegisterInfo &TRI; TargetSchedModel TSchedModel; + bool RunLdsBranchVmemWARHazardFixup; /// RegUnits of uses in the current soft memory clause. BitVector ClauseUses; diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp @@ -23,6 +23,9 @@ // Hazard Recoginizer Implementation //===----------------------------------------------------------------------===// +static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, + const GCNSubtarget &ST); + GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) : IsHazardRecognizerMode(false), CurrCycleInstr(nullptr), @@ -34,6 +37,7 @@ ClauseDefs(TRI.getNumRegUnits()) { MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5; TSchedModel.init(&ST); + RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST); } void GCNHazardRecognizer::Reset() { @@ -1074,10 +1078,33 @@ return true; } -bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) { +static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, + const GCNSubtarget &ST) { if (!ST.hasLdsBranchVmemWARHazard()) return false; + // Check if the necessary condition for the hazard is met: both LDS and VMEM + // instructions need to appear in the same function. + bool HasLds = false; + bool HasVmem = false; + for (auto &MBB : MF) { + for (auto &MI : MBB) { + HasLds |= SIInstrInfo::isDS(MI); + HasVmem |= + SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI); + if (HasLds && HasVmem) + return true; + } + } + return false; +} + +bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) { + if (!RunLdsBranchVmemWARHazardFixup) + return false; + + assert(ST.hasLdsBranchVmemWARHazard()); + auto IsHazardInst = [](const MachineInstr &MI) { if (SIInstrInfo::isDS(MI)) return 1;