diff --git a/llvm/include/llvm/Analysis/LoopNestAnalysis.h b/llvm/include/llvm/Analysis/LoopNestAnalysis.h --- a/llvm/include/llvm/Analysis/LoopNestAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopNestAnalysis.h @@ -128,6 +128,8 @@ [](const Loop *L) { return L->isLoopSimplifyForm(); }); } + StringRef getName() const { return Loops.front()->getName(); } + protected: const unsigned MaxPerfectDepth; // maximum perfect nesting depth level. LoopVectorTy Loops; // the loops in the nest (in breadth first order). diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h --- a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h @@ -44,6 +44,7 @@ #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/LoopNestAnalysis.h" #include "llvm/Analysis/MemorySSA.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" @@ -61,17 +62,136 @@ // Forward declarations of an update tracking API used in the pass manager. class LPMUpdater; +namespace detail { + +// SFINAE helper class that distinguish loop passes and loop-nest passes. +template +struct IsLoopPass : std::false_type {}; + +template +struct IsLoopPass< + PassT, + std::enable_if_t().run( + std::declval(), std::declval(), + std::declval(), + std::declval()))>::value>> : std::true_type {}; + +} // namespace detail + // Explicit specialization and instantiation declarations for the pass manager. // See the comments on the definition of the specialization for details on how // it differs from the primary template. template <> -PreservedAnalyses -PassManager::run(Loop &InitialL, LoopAnalysisManager &AM, - LoopStandardAnalysisResults &AnalysisResults, - LPMUpdater &U); -extern template class PassManager; +class PassManager + : public PassInfoMixin< + PassManager> { +public: + /// Construct a pass manager. + /// + /// If \p DebugLogging is true, we'll log our progress to llvm::dbgs(). + explicit PassManager(bool DebugLogging = false) + : DebugLogging(DebugLogging) {} + + // FIXME: These are equivalent to the default move constructor/move + // assignment. However, using = default triggers linker errors due to the + // explicit instantiations below. Find away to use the default and remove the + // duplicated code here. + PassManager(PassManager &&Arg) + : PassCategories(std::move(Arg.PassCategories)), + LoopPasses(std::move(Arg.LoopPasses)), + LoopNestPasses(std::move(Arg.LoopNestPasses)), + DebugLogging(std::move(Arg.DebugLogging)) {} + + PassManager &operator=(PassManager &&RHS) { + PassCategories = std::move(RHS.PassCategories); + LoopPasses = std::move(RHS.LoopPasses); + LoopNestPasses = std::move(RHS.LoopNestPasses); + DebugLogging = std::move(RHS.DebugLogging); + return *this; + } + + PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, LPMUpdater &U); + + template + std::enable_if_t::value> addPass(PassT Pass) { + using LoopPassModelT = + detail::PassModel; + PassCategories.push_back(false); + LoopPasses.emplace_back(new LoopPassModelT(std::move(Pass))); + } + + template + std::enable_if_t::value> addPass(PassT Pass) { + using LoopNestPassModelT = + detail::PassModel; + PassCategories.push_back(true); + LoopNestPasses.emplace_back(new LoopNestPassModelT(std::move(Pass))); + } + + // Specializations of `addPass` for `RepeatedPass`. These are necessary since + // `RepeatedPass` has a templated `run` method that will result in incorrect + // behavior of `IsLoopNestPass`. + template + std::enable_if_t::value> + addPass(RepeatedPass Pass) { + using RepeatedLoopPassModelT = + detail::PassModel, PreservedAnalyses, + LoopAnalysisManager, LoopStandardAnalysisResults &, + LPMUpdater &>; + PassCategories.push_back(false); + LoopPasses.emplace_back(new RepeatedLoopPassModelT(std::move(Pass))); + } + + template + std::enable_if_t::value> + addPass(RepeatedPass Pass) { + using RepeatedLoopNestPassModelT = + detail::PassModel, PreservedAnalyses, + LoopAnalysisManager, LoopStandardAnalysisResults &, + LPMUpdater &>; + PassCategories.push_back(true); + LoopNestPasses.emplace_back( + new RepeatedLoopNestPassModelT(std::move(Pass))); + } + + static bool isRequired() { return true; } + +protected: + using LoopPassConceptT = + detail::PassConcept; + using LoopNestPassConceptT = + detail::PassConcept; + + BitVector PassCategories; + std::vector> LoopPasses; + std::vector> LoopNestPasses; + + /// Flag indicating whether we should do debug logging. + bool DebugLogging; + + template + Optional + runSinglePass(IRUnitT &IR, PassT &Pass, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, LPMUpdater &U, + PassInstrumentation &PI); + + PreservedAnalyses runWithLoopNestPasses(Loop &L, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, + LPMUpdater &U); + PreservedAnalyses runWithoutLoopNestPasses(Loop &L, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, + LPMUpdater &U); +}; /// The Loop pass manager. /// @@ -221,6 +341,29 @@ : Worklist(Worklist), LAM(LAM) {} }; +template +Optional LoopPassManager::runSinglePass( + IRUnitT &IR, PassT &Pass, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, LPMUpdater &U, PassInstrumentation &PI) { + // Check the PassInstrumentation's BeforePass callbacks before running the + // pass, skip its execution completely if asked to (callback returns false). + if (!PI.runBeforePass(*Pass, IR)) + return None; + + PreservedAnalyses PA; + { + TimeTraceScope TimeScope(Pass->name(), IR.getName()); + PA = Pass->run(IR, AM, AR, U); + } + + // do not pass deleted Loop into the instrumentation + if (U.skipCurrentLoop()) + PI.runAfterPassInvalidated(*Pass, PA); + else + PI.runAfterPass(*Pass, IR, PA); + return PA; +} + /// Adaptor that maps from a function to its loops. /// /// Designed to allow composition of a LoopPass(Manager) and a diff --git a/llvm/lib/Analysis/LoopNestAnalysis.cpp b/llvm/lib/Analysis/LoopNestAnalysis.cpp --- a/llvm/lib/Analysis/LoopNestAnalysis.cpp +++ b/llvm/lib/Analysis/LoopNestAnalysis.cpp @@ -306,6 +306,8 @@ return true; } +AnalysisKey LoopNestAnalysis::Key; + raw_ostream &llvm::operator<<(raw_ostream &OS, const LoopNest &LN) { OS << "IsPerfect="; if (LN.getMaxPerfectDepth() == LN.getNestDepth()) diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp --- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp +++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp @@ -12,58 +12,88 @@ using namespace llvm; -// Explicit template instantiations and specialization defininitions for core -// template typedefs. namespace llvm { -template class PassManager; /// Explicitly specialize the pass manager's run method to handle loop nest /// structure updates. -template <> PreservedAnalyses PassManager::run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U) { - PreservedAnalyses PA = PreservedAnalyses::all(); + PreservedAnalyses PA; if (DebugLogging) dbgs() << "Starting Loop pass manager run.\n"; // Request PassInstrumentation from analysis manager, will use it to run // instrumenting callbacks for the passes later. - PassInstrumentation PI = AM.getResult(L, AR); - for (auto &Pass : Passes) { - // Check the PassInstrumentation's BeforePass callbacks before running the - // pass, skip its execution completely if asked to (callback returns false). - if (!PI.runBeforePass(*Pass, L)) - continue; + if (!L.getParentLoop() && !LoopNestPasses.empty()) + PA = runWithLoopNestPasses(L, AM, AR, U); + else + PA = runWithoutLoopNestPasses(L, AM, AR, U); + + // Invalidation for the current loop should be handled above, and other loop + // analysis results shouldn't be impacted by runs over this loop. Therefore, + // the remaining analysis results in the AnalysisManager are preserved. We + // mark this with a set so that we don't need to inspect each one + // individually. + // FIXME: This isn't correct! This loop and all nested loops' analyses should + // be preserved, but unrolling should invalidate the parent loop's analyses. + PA.preserveSet>(); + + if (DebugLogging) + dbgs() << "Finished Loop pass manager run.\n"; + + return PA; +} - PreservedAnalyses PassPA; - { - TimeTraceScope TimeScope(Pass->name(), L.getName()); - PassPA = Pass->run(L, AM, AR, U); +PreservedAnalyses +LoopPassManager::runWithLoopNestPasses(Loop &L, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, + LPMUpdater &U) { + PreservedAnalyses PA = PreservedAnalyses::all(); + PassInstrumentation PI = AM.getResult(L, AR); + + unsigned LoopPassIndex = 0, LoopNestPassIndex = 0; + std::unique_ptr LoopNestPtr; + bool IsLoopNestPtrValid = false; + + for (size_t I = 0, E = PassCategories.size(); I != E; ++I) { + Optional PassPA; + if (!PassCategories.test(I)) { + auto &Pass = LoopPasses[LoopPassIndex++]; + PassPA = runSinglePass(L, Pass, AM, AR, U, PI); + } else { + auto &Pass = LoopNestPasses[LoopNestPassIndex++]; + + // If the loop-nest object calculated before is no longer valid, + // re-calculate it here before running the loop-nest pass. + if (!IsLoopNestPtrValid) { + LoopNestPtr = LoopNest::getLoopNest(L, AR.SE); + IsLoopNestPtrValid = true; + } + PassPA = runSinglePass(*LoopNestPtr, Pass, AM, AR, U, PI); } - // do not pass deleted Loop into the instrumentation - if (U.skipCurrentLoop()) - PI.runAfterPassInvalidated(*Pass, PassPA); - else - PI.runAfterPass(*Pass, L, PassPA); + if (!PassPA) + continue; // If the loop was deleted, abort the run and return to the outer walk. if (U.skipCurrentLoop()) { - PA.intersect(std::move(PassPA)); + PA.intersect(std::move(*PassPA)); break; } // Update the analysis manager as each pass runs and potentially // invalidates analyses. - AM.invalidate(L, PassPA); + AM.invalidate(L, *PassPA); // Finally, we intersect the final preserved analyses to compute the // aggregate preserved set for this pass manager. - PA.intersect(std::move(PassPA)); + PA.intersect(std::move(*PassPA)); + + // Check if the current pass preserved the loop-nest object or not. + IsLoopNestPtrValid = PassPA->getChecker().preserved(); // FIXME: Historically, the pass managers all called the LLVM context's // yield function here. We don't have a generic way to acquire the @@ -71,22 +101,43 @@ // in the new pass manager so it is currently omitted. // ...getContext().yield(); } + return PA; +} - // Invalidation for the current loop should be handled above, and other loop - // analysis results shouldn't be impacted by runs over this loop. Therefore, - // the remaining analysis results in the AnalysisManager are preserved. We - // mark this with a set so that we don't need to inspect each one - // individually. - // FIXME: This isn't correct! This loop and all nested loops' analyses should - // be preserved, but unrolling should invalidate the parent loop's analyses. - PA.preserveSet>(); +PreservedAnalyses +LoopPassManager::runWithoutLoopNestPasses(Loop &L, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, + LPMUpdater &U) { + PreservedAnalyses PA = PreservedAnalyses::all(); + PassInstrumentation PI = AM.getResult(L, AR); + for (auto &Pass : LoopPasses) { + auto PassPA = runSinglePass(L, Pass, AM, AR, U, PI); + if (!PassPA) + continue; - if (DebugLogging) - dbgs() << "Finished Loop pass manager run.\n"; + // If the loop was deleted, abort the run and return to the outer walk. + if (U.skipCurrentLoop()) { + PA.intersect(std::move(*PassPA)); + break; + } + + // Update the analysis manager as each pass runs and potentially + // invalidates analyses. + AM.invalidate(L, *PassPA); + + // Finally, we intersect the final preserved analyses to compute the + // aggregate preserved set for this pass manager. + PA.intersect(std::move(*PassPA)); + // FIXME: Historically, the pass managers all called the LLVM context's + // yield function here. We don't have a generic way to acquire the + // context and it isn't yet clear what the right pattern is for yielding + // in the new pass manager so it is currently omitted. + // ...getContext().yield(); + } return PA; } -} +} // namespace llvm PrintLoopPass::PrintLoopPass() : OS(dbgs()) {} PrintLoopPass::PrintLoopPass(raw_ostream &OS, const std::string &Banner)