Index: llvm/trunk/lib/Target/AMDGPU/SIMemoryLegalizer.cpp =================================================================== --- llvm/trunk/lib/Target/AMDGPU/SIMemoryLegalizer.cpp +++ llvm/trunk/lib/Target/AMDGPU/SIMemoryLegalizer.cpp @@ -47,7 +47,8 @@ namespace { -struct SIMemOpInfo final { +class SIMemOpInfo final { +private: SyncScope::ID SSID = SyncScope::System; AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent; AtomicOrdering FailureOrdering = AtomicOrdering::SequentiallyConsistent; @@ -65,6 +66,42 @@ : SSID(MMO->getSyncScopeID()), Ordering(MMO->getOrdering()), FailureOrdering(MMO->getFailureOrdering()) {} + +public: + /// \returns Synchronization scope ID of the machine instruction used to + /// create this SIMemOpInfo. + SyncScope::ID getSSID() const { + return SSID; + } + /// \returns Ordering constraint of the machine instruction used to + /// create this SIMemOpInfo. + AtomicOrdering getOrdering() const { + return Ordering; + } + /// \returns Failure ordering constraint of the machine instruction used to + /// create this SIMemOpInfo. + AtomicOrdering getFailureOrdering() const { + return FailureOrdering; + } + + /// \returns Load info if \p MI is a load operation, "None" otherwise. + static Optional getLoadInfo( + const MachineBasicBlock::iterator &MI); + /// \returns Store info if \p MI is a store operation, "None" otherwise. + static Optional getStoreInfo( + const MachineBasicBlock::iterator &MI); + /// \returns Atomic fence info if \p MI is an atomic fence operation, + /// "None" otherwise. + static Optional getAtomicFenceInfo( + const MachineBasicBlock::iterator &MI); + /// \returns Atomic cmpxchg info if \p MI is an atomic cmpxchg operation, + /// "None" otherwise. + static Optional getAtomicCmpxchgInfo( + const MachineBasicBlock::iterator &MI); + /// \returns Atomic rmw info if \p MI is an atomic rmw operation, + /// "None" otherwise. + static Optional getAtomicRmwInfo( + const MachineBasicBlock::iterator &MI); }; class SIMemoryLegalizer final : public MachineFunctionPass { @@ -108,25 +145,6 @@ /// context. void reportUnknownSynchScope(const MachineBasicBlock::iterator &MI); - /// \returns Load info if \p MI is a load operation, "None" otherwise. - Optional getLoadInfo( - const MachineBasicBlock::iterator &MI) const; - /// \returns Store info if \p MI is a store operation, "None" otherwise. - Optional getStoreInfo( - const MachineBasicBlock::iterator &MI) const; - /// \returns Atomic fence info if \p MI is an atomic fence operation, - /// "None" otherwise. - Optional getAtomicFenceInfo( - const MachineBasicBlock::iterator &MI) const; - /// \returns Atomic cmpxchg info if \p MI is an atomic cmpxchg operation, - /// "None" otherwise. - Optional getAtomicCmpxchgInfo( - const MachineBasicBlock::iterator &MI) const; - /// \returns Atomic rmw info if \p MI is an atomic rmw operation, - /// "None" otherwise. - Optional getAtomicRmwInfo( - const MachineBasicBlock::iterator &MI) const; - /// \brief Expands load operation \p MI. Returns true if instructions are /// added/deleted or \p MI is modified, false otherwise. bool expandLoad(const SIMemOpInfo &MOI, @@ -167,71 +185,9 @@ } // end namespace anonymous -bool SIMemoryLegalizer::insertBufferWbinvl1Vol(MachineBasicBlock::iterator &MI, - bool Before) const { - MachineBasicBlock &MBB = *MI->getParent(); - DebugLoc DL = MI->getDebugLoc(); - - if (!Before) - ++MI; - - BuildMI(MBB, MI, DL, TII->get(Wbinvl1Opcode)); - - if (!Before) - --MI; - - return true; -} - -bool SIMemoryLegalizer::insertWaitcntVmcnt0(MachineBasicBlock::iterator &MI, - bool Before) const { - MachineBasicBlock &MBB = *MI->getParent(); - DebugLoc DL = MI->getDebugLoc(); - - if (!Before) - ++MI; - - BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(Vmcnt0Immediate); - - if (!Before) - --MI; - - return true; -} - -bool SIMemoryLegalizer::setGLC(const MachineBasicBlock::iterator &MI) const { - int GLCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::glc); - if (GLCIdx == -1) - return false; - - MachineOperand &GLC = MI->getOperand(GLCIdx); - if (GLC.getImm() == 1) - return false; - - GLC.setImm(1); - return true; -} - -bool SIMemoryLegalizer::removeAtomicPseudoMIs() { - if (AtomicPseudoMIs.empty()) - return false; - - for (auto &MI : AtomicPseudoMIs) - MI->eraseFromParent(); - - AtomicPseudoMIs.clear(); - return true; -} - -void SIMemoryLegalizer::reportUnknownSynchScope( +/* static */ +Optional SIMemOpInfo::getLoadInfo( const MachineBasicBlock::iterator &MI) { - DiagnosticInfoUnsupported Diag(*MI->getParent()->getParent()->getFunction(), - "Unsupported synchronization scope"); - CTX->diagnose(Diag); -} - -Optional SIMemoryLegalizer::getLoadInfo( - const MachineBasicBlock::iterator &MI) const { assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); if (!(MI->mayLoad() && !MI->mayStore())) @@ -246,8 +202,9 @@ return SIMemOpInfo(MMO); } -Optional SIMemoryLegalizer::getStoreInfo( - const MachineBasicBlock::iterator &MI) const { +/* static */ +Optional SIMemOpInfo::getStoreInfo( + const MachineBasicBlock::iterator &MI) { assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); if (!(!MI->mayLoad() && MI->mayStore())) @@ -262,8 +219,9 @@ return SIMemOpInfo(MMO); } -Optional SIMemoryLegalizer::getAtomicFenceInfo( - const MachineBasicBlock::iterator &MI) const { +/* static */ +Optional SIMemOpInfo::getAtomicFenceInfo( + const MachineBasicBlock::iterator &MI) { assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); if (MI->getOpcode() != AMDGPU::ATOMIC_FENCE) @@ -276,8 +234,9 @@ return SIMemOpInfo(SSID, Ordering, AtomicOrdering::NotAtomic); } -Optional SIMemoryLegalizer::getAtomicCmpxchgInfo( - const MachineBasicBlock::iterator &MI) const { +/* static */ +Optional SIMemOpInfo::getAtomicCmpxchgInfo( + const MachineBasicBlock::iterator &MI) { assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); if (!(MI->mayLoad() && MI->mayStore())) @@ -294,8 +253,9 @@ return SIMemOpInfo(MMO); } -Optional SIMemoryLegalizer::getAtomicRmwInfo( - const MachineBasicBlock::iterator &MI) const { +/* static */ +Optional SIMemOpInfo::getAtomicRmwInfo( + const MachineBasicBlock::iterator &MI) { assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic); if (!(MI->mayLoad() && MI->mayStore())) @@ -312,30 +272,93 @@ return SIMemOpInfo(MMO); } +bool SIMemoryLegalizer::insertBufferWbinvl1Vol(MachineBasicBlock::iterator &MI, + bool Before) const { + MachineBasicBlock &MBB = *MI->getParent(); + DebugLoc DL = MI->getDebugLoc(); + + if (!Before) + ++MI; + + BuildMI(MBB, MI, DL, TII->get(Wbinvl1Opcode)); + + if (!Before) + --MI; + + return true; +} + +bool SIMemoryLegalizer::insertWaitcntVmcnt0(MachineBasicBlock::iterator &MI, + bool Before) const { + MachineBasicBlock &MBB = *MI->getParent(); + DebugLoc DL = MI->getDebugLoc(); + + if (!Before) + ++MI; + + BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(Vmcnt0Immediate); + + if (!Before) + --MI; + + return true; +} + +bool SIMemoryLegalizer::setGLC(const MachineBasicBlock::iterator &MI) const { + int GLCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::glc); + if (GLCIdx == -1) + return false; + + MachineOperand &GLC = MI->getOperand(GLCIdx); + if (GLC.getImm() == 1) + return false; + + GLC.setImm(1); + return true; +} + +bool SIMemoryLegalizer::removeAtomicPseudoMIs() { + if (AtomicPseudoMIs.empty()) + return false; + + for (auto &MI : AtomicPseudoMIs) + MI->eraseFromParent(); + + AtomicPseudoMIs.clear(); + return true; +} + +void SIMemoryLegalizer::reportUnknownSynchScope( + const MachineBasicBlock::iterator &MI) { + DiagnosticInfoUnsupported Diag(*MI->getParent()->getParent()->getFunction(), + "Unsupported synchronization scope"); + CTX->diagnose(Diag); +} + bool SIMemoryLegalizer::expandLoad(const SIMemOpInfo &MOI, MachineBasicBlock::iterator &MI) { assert(MI->mayLoad() && !MI->mayStore()); bool Changed = false; - if (MOI.SSID == SyncScope::System || - MOI.SSID == MMI->getAgentSSID()) { - if (MOI.Ordering == AtomicOrdering::Acquire || - MOI.Ordering == AtomicOrdering::SequentiallyConsistent) + if (MOI.getSSID() == SyncScope::System || + MOI.getSSID() == MMI->getAgentSSID()) { + if (MOI.getOrdering() == AtomicOrdering::Acquire || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) Changed |= setGLC(MI); - if (MOI.Ordering == AtomicOrdering::SequentiallyConsistent) + if (MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) Changed |= insertWaitcntVmcnt0(MI); - if (MOI.Ordering == AtomicOrdering::Acquire || - MOI.Ordering == AtomicOrdering::SequentiallyConsistent) { + if (MOI.getOrdering() == AtomicOrdering::Acquire || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) { Changed |= insertWaitcntVmcnt0(MI, false); Changed |= insertBufferWbinvl1Vol(MI, false); } return Changed; - } else if (MOI.SSID == SyncScope::SingleThread || - MOI.SSID == MMI->getWorkgroupSSID() || - MOI.SSID == MMI->getWavefrontSSID()) { + } else if (MOI.getSSID() == SyncScope::SingleThread || + MOI.getSSID() == MMI->getWorkgroupSSID() || + MOI.getSSID() == MMI->getWavefrontSSID()) { return Changed; } else { reportUnknownSynchScope(MI); @@ -348,16 +371,16 @@ assert(!MI->mayLoad() && MI->mayStore()); bool Changed = false; - if (MOI.SSID == SyncScope::System || - MOI.SSID == MMI->getAgentSSID()) { - if (MOI.Ordering == AtomicOrdering::Release || - MOI.Ordering == AtomicOrdering::SequentiallyConsistent) + if (MOI.getSSID() == SyncScope::System || + MOI.getSSID() == MMI->getAgentSSID()) { + if (MOI.getOrdering() == AtomicOrdering::Release || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) Changed |= insertWaitcntVmcnt0(MI); return Changed; - } else if (MOI.SSID == SyncScope::SingleThread || - MOI.SSID == MMI->getWorkgroupSSID() || - MOI.SSID == MMI->getWavefrontSSID()) { + } else if (MOI.getSSID() == SyncScope::SingleThread || + MOI.getSSID() == MMI->getWorkgroupSSID() || + MOI.getSSID() == MMI->getWavefrontSSID()) { return Changed; } else { reportUnknownSynchScope(MI); @@ -370,24 +393,24 @@ assert(MI->getOpcode() == AMDGPU::ATOMIC_FENCE); bool Changed = false; - if (MOI.SSID == SyncScope::System || - MOI.SSID == MMI->getAgentSSID()) { - if (MOI.Ordering == AtomicOrdering::Acquire || - MOI.Ordering == AtomicOrdering::Release || - MOI.Ordering == AtomicOrdering::AcquireRelease || - MOI.Ordering == AtomicOrdering::SequentiallyConsistent) + if (MOI.getSSID() == SyncScope::System || + MOI.getSSID() == MMI->getAgentSSID()) { + if (MOI.getOrdering() == AtomicOrdering::Acquire || + MOI.getOrdering() == AtomicOrdering::Release || + MOI.getOrdering() == AtomicOrdering::AcquireRelease || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) Changed |= insertWaitcntVmcnt0(MI); - if (MOI.Ordering == AtomicOrdering::Acquire || - MOI.Ordering == AtomicOrdering::AcquireRelease || - MOI.Ordering == AtomicOrdering::SequentiallyConsistent) + if (MOI.getOrdering() == AtomicOrdering::Acquire || + MOI.getOrdering() == AtomicOrdering::AcquireRelease || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) Changed |= insertBufferWbinvl1Vol(MI); AtomicPseudoMIs.push_back(MI); return Changed; - } else if (MOI.SSID == SyncScope::SingleThread || - MOI.SSID == MMI->getWorkgroupSSID() || - MOI.SSID == MMI->getWavefrontSSID()) { + } else if (MOI.getSSID() == SyncScope::SingleThread || + MOI.getSSID() == MMI->getWorkgroupSSID() || + MOI.getSSID() == MMI->getWavefrontSSID()) { AtomicPseudoMIs.push_back(MI); return Changed; } else { @@ -401,27 +424,27 @@ assert(MI->mayLoad() && MI->mayStore()); bool Changed = false; - if (MOI.SSID == SyncScope::System || - MOI.SSID == MMI->getAgentSSID()) { - if (MOI.Ordering == AtomicOrdering::Release || - MOI.Ordering == AtomicOrdering::AcquireRelease || - MOI.Ordering == AtomicOrdering::SequentiallyConsistent || - MOI.FailureOrdering == AtomicOrdering::SequentiallyConsistent) + if (MOI.getSSID() == SyncScope::System || + MOI.getSSID() == MMI->getAgentSSID()) { + if (MOI.getOrdering() == AtomicOrdering::Release || + MOI.getOrdering() == AtomicOrdering::AcquireRelease || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent || + MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) Changed |= insertWaitcntVmcnt0(MI); - if (MOI.Ordering == AtomicOrdering::Acquire || - MOI.Ordering == AtomicOrdering::AcquireRelease || - MOI.Ordering == AtomicOrdering::SequentiallyConsistent || - MOI.FailureOrdering == AtomicOrdering::Acquire || - MOI.FailureOrdering == AtomicOrdering::SequentiallyConsistent) { + if (MOI.getOrdering() == AtomicOrdering::Acquire || + MOI.getOrdering() == AtomicOrdering::AcquireRelease || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent || + MOI.getFailureOrdering() == AtomicOrdering::Acquire || + MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) { Changed |= insertWaitcntVmcnt0(MI, false); Changed |= insertBufferWbinvl1Vol(MI, false); } return Changed; - } else if (MOI.SSID == SyncScope::SingleThread || - MOI.SSID == MMI->getWorkgroupSSID() || - MOI.SSID == MMI->getWavefrontSSID()) { + } else if (MOI.getSSID() == SyncScope::SingleThread || + MOI.getSSID() == MMI->getWorkgroupSSID() || + MOI.getSSID() == MMI->getWavefrontSSID()) { Changed |= setGLC(MI); return Changed; } else { @@ -435,24 +458,24 @@ assert(MI->mayLoad() && MI->mayStore()); bool Changed = false; - if (MOI.SSID == SyncScope::System || - MOI.SSID == MMI->getAgentSSID()) { - if (MOI.Ordering == AtomicOrdering::Release || - MOI.Ordering == AtomicOrdering::AcquireRelease || - MOI.Ordering == AtomicOrdering::SequentiallyConsistent) + if (MOI.getSSID() == SyncScope::System || + MOI.getSSID() == MMI->getAgentSSID()) { + if (MOI.getOrdering() == AtomicOrdering::Release || + MOI.getOrdering() == AtomicOrdering::AcquireRelease || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) Changed |= insertWaitcntVmcnt0(MI); - if (MOI.Ordering == AtomicOrdering::Acquire || - MOI.Ordering == AtomicOrdering::AcquireRelease || - MOI.Ordering == AtomicOrdering::SequentiallyConsistent) { + if (MOI.getOrdering() == AtomicOrdering::Acquire || + MOI.getOrdering() == AtomicOrdering::AcquireRelease || + MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) { Changed |= insertWaitcntVmcnt0(MI, false); Changed |= insertBufferWbinvl1Vol(MI, false); } return Changed; - } else if (MOI.SSID == SyncScope::SingleThread || - MOI.SSID == MMI->getWorkgroupSSID() || - MOI.SSID == MMI->getWavefrontSSID()) { + } else if (MOI.getSSID() == SyncScope::SingleThread || + MOI.getSSID() == MMI->getWorkgroupSSID() || + MOI.getSSID() == MMI->getWavefrontSSID()) { Changed |= setGLC(MI); return Changed; } else { @@ -480,15 +503,15 @@ if (!(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic)) continue; - if (const auto &MOI = getLoadInfo(MI)) + if (const auto &MOI = SIMemOpInfo::getLoadInfo(MI)) Changed |= expandLoad(MOI.getValue(), MI); - else if (const auto &MOI = getStoreInfo(MI)) + else if (const auto &MOI = SIMemOpInfo::getStoreInfo(MI)) Changed |= expandStore(MOI.getValue(), MI); - else if (const auto &MOI = getAtomicFenceInfo(MI)) + else if (const auto &MOI = SIMemOpInfo::getAtomicFenceInfo(MI)) Changed |= expandAtomicFence(MOI.getValue(), MI); - else if (const auto &MOI = getAtomicCmpxchgInfo(MI)) + else if (const auto &MOI = SIMemOpInfo::getAtomicCmpxchgInfo(MI)) Changed |= expandAtomicCmpxchg(MOI.getValue(), MI); - else if (const auto &MOI = getAtomicRmwInfo(MI)) + else if (const auto &MOI = SIMemOpInfo::getAtomicRmwInfo(MI)) Changed |= expandAtomicRmw(MOI.getValue(), MI); } }