diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp @@ -1391,20 +1391,14 @@ if (MFMAPaddingRatio == 0) return 0; - auto IsMFMAFn = [](const MachineInstr &MI) { - return SIInstrInfo::isMAI(MI) && - MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && - MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; - }; - const SIMachineFunctionInfo *MFI = MF.getInfo(); - if (!IsMFMAFn(*MI) || MFI->getOccupancy() < 2) + if (!SIInstrInfo::isMFMA(*MI) || MFI->getOccupancy() < 2) return 0; int NeighborMFMALatency = 0; - auto IsNeighboringMFMA = [&IsMFMAFn, &NeighborMFMALatency, + auto IsNeighboringMFMA = [&NeighborMFMALatency, this](const MachineInstr &MI) { - if (!IsMFMAFn(MI)) + if (!SIInstrInfo::isMFMA(MI)) return false; NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI); @@ -1456,12 +1450,6 @@ } } - auto IsMFMAFn = [](const MachineInstr &MI) { - return SIInstrInfo::isMAI(MI) && - MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && - MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; - }; - for (const MachineOperand &Op : MI->explicit_operands()) { if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg())) continue; @@ -1481,9 +1469,9 @@ Register Reg = Op.getReg(); unsigned HazardDefLatency = 0; - auto IsOverlappedMFMAFn = [Reg, &IsMFMAFn, &HazardDefLatency, + auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency, this](const MachineInstr &MI) { - if (!IsMFMAFn(MI)) + if (!SIInstrInfo::isMFMA(MI)) return false; Register DstReg = MI.getOperand(0).getReg(); if (DstReg == Reg) @@ -1560,9 +1548,9 @@ Register DstReg = MI->getOperand(0).getReg(); unsigned HazardDefLatency = 0; - auto IsSrcCMFMAFn = [DstReg, &IsMFMAFn, &HazardDefLatency, + auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency, this](const MachineInstr &MI) { - if (!IsMFMAFn(MI)) + if (!SIInstrInfo::isMFMA(MI)) return false; Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg(); HazardDefLatency = @@ -1596,21 +1584,16 @@ int WaitStatesNeeded = 0; unsigned Opc = MI->getOpcode(); - auto IsMFMAFn = [](const MachineInstr &MI) { - return SIInstrInfo::isMAI(MI) && - MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && - MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; - }; - - auto IsLegacyVALUFn = [&IsMFMAFn](const MachineInstr &MI) { - return SIInstrInfo::isVALU(MI) && !IsMFMAFn(MI); + auto IsLegacyVALUFn = [](const MachineInstr &MI) { + return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI); }; - auto IsLegacyVALUNotDotFn = [&IsMFMAFn](const MachineInstr &MI) { - return SIInstrInfo::isVALU(MI) && !IsMFMAFn(MI) && !SIInstrInfo::isDOT(MI); + auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) { + return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI) && + !SIInstrInfo::isDOT(MI); }; - if (!IsMFMAFn(*MI)) + if (!SIInstrInfo::isMFMA(*MI)) return WaitStatesNeeded; const int VALUWritesExecWaitStates = 4; @@ -1662,9 +1645,9 @@ bool FullReg; const MachineInstr *MI1; - auto IsOverlappedMFMAFn = [Reg, &IsMFMAFn, &FullReg, &MI1, + auto IsOverlappedMFMAFn = [Reg, &FullReg, &MI1, this](const MachineInstr &MI) { - if (!IsMFMAFn(MI)) + if (!SIInstrInfo::isMFMA(MI)) return false; Register DstReg = MI.getOperand(0).getReg(); FullReg = (DstReg == Reg); @@ -1858,18 +1841,12 @@ if (!ST.hasGFX90AInsts()) return 0; - auto IsMFMAFn = [](const MachineInstr &MI) -> bool { - return SIInstrInfo::isMAI(MI) && - MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && - MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; - }; - auto IsDGEMMFn = [](const MachineInstr &MI) -> bool { return isDGEMM(MI.getOpcode()); }; // This is checked in checkMAIHazards90A() - if (IsMFMAFn(*MI)) + if (SIInstrInfo::isMFMA(*MI)) return 0; int WaitStatesNeeded = 0; @@ -1882,8 +1859,9 @@ const MachineInstr *MFMA = nullptr; unsigned Reg; - auto IsMFMAWriteFn = [&Reg, &IsMFMAFn, &MFMA, this](const MachineInstr &MI) { - if (!IsMFMAFn(MI) || !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) + auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) { + if (!SIInstrInfo::isMFMA(MI) || + !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) return false; MFMA = &MI; return true; @@ -2091,9 +2069,8 @@ break; } - auto IsSMFMAReadAsCFn = [&Reg, &IsMFMAFn, &MFMA, - this](const MachineInstr &MI) { - if (!IsMFMAFn(MI) || isDGEMM(MI.getOpcode()) || + auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) { + if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) || !MI.readsRegister(Reg, &TRI)) return false; @@ -2143,11 +2120,10 @@ return false; const MachineInstr *MAI = nullptr; + auto IsMFMAFn = [&MAI](const MachineInstr &MI) { MAI = nullptr; - if (SIInstrInfo::isMAI(MI) && - MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && - MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64) + if (SIInstrInfo::isMFMA(MI)) MAI = &MI; return MAI != nullptr; }; diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -15,6 +15,7 @@ #define LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H #include "AMDGPUMIRFormatter.h" +#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIRegisterInfo.h" #include "Utils/AMDGPUBaseInfo.h" #include "llvm/ADT/SetVector.h" @@ -655,6 +656,11 @@ return get(Opcode).TSFlags & SIInstrFlags::IsMAI; } + static bool isMFMA(const MachineInstr &MI) { + return isMAI(MI) && MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && + MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; + } + static bool isDOT(const MachineInstr &MI) { return MI.getDesc().TSFlags & SIInstrFlags::IsDOT; } diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -16,7 +16,6 @@ #include "AMDGPUInstrInfo.h" #include "GCNHazardRecognizer.h" #include "GCNSubtarget.h" -#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIMachineFunctionInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/LiveIntervals.h"