diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h --- a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h @@ -53,6 +53,16 @@ // Forward declarations of an update tracking API used in the pass manager. class LPMUpdater; +namespace { + +template +using HasRunOnLoopT = decltype(std::declval().run( + std::declval(), std::declval(), + std::declval(), + std::declval())); + +} // namespace + // Explicit specialization and instantiation declarations for the pass manager. // See the comments on the definition of the specialization for details on how // it differs from the primary template. @@ -62,13 +72,6 @@ : public PassInfoMixin< PassManager> { -private: - template - using HasRunOnLoopT = decltype(std::declval().run( - std::declval(), std::declval(), - std::declval(), - std::declval())); - public: /// Construct a pass manager. /// @@ -154,6 +157,9 @@ static bool isRequired() { return true; } + size_t getNumLoopPasses() const { return LoopPasses.size(); } + size_t getNumLoopNestPasses() const { return LoopNestPasses.size(); } + protected: using LoopPassConceptT = detail::PassConceptcontains(&L)) && "Cannot delete a loop outside of the " @@ -263,6 +278,8 @@ /// loops within them will be visited in postorder as usual for the loop pass /// manager. void addChildLoops(ArrayRef NewChildLoops) { + assert(!LoopNestMode && + "Child loops should not be pushed in loop-nest mode."); // Insert ourselves back into the worklist first, as this loop should be // revisited after all the children have been processed. Worklist.insert(CurrentL); @@ -294,7 +311,10 @@ "All of the new loops must be siblings of the current loop!"); #endif - appendLoopsToWorklist(NewSibLoops, Worklist); + if (LoopNestMode) + Worklist.insert(NewSibLoops); + else + appendLoopsToWorklist(NewSibLoops, Worklist); // No need to skip the current loop or revisit it, as sibling loops // shouldn't impact anything. @@ -324,6 +344,7 @@ Loop *CurrentL; bool SkipCurrentLoop; + const bool LoopNestMode; #ifndef NDEBUG // In debug builds we also track the parent loop to implement asserts even in @@ -332,8 +353,8 @@ #endif LPMUpdater(SmallPriorityWorklist &Worklist, - LoopAnalysisManager &LAM) - : Worklist(Worklist), LAM(LAM) {} + LoopAnalysisManager &LAM, bool LoopNestMode = false) + : Worklist(Worklist), LAM(LAM), LoopNestMode(LoopNestMode) {} }; template @@ -366,6 +387,15 @@ /// FunctionAnalysisManager it will run the \c LoopAnalysisManagerFunctionProxy /// analysis prior to running the loop passes over the function to enable a \c /// LoopAnalysisManager to be used within this run safely. +/// +/// The adaptor comes with two modes: the loop mode and the loop-nest mode, and +/// the worklist updater lived inside will be in the same mode as the adaptor +/// (refer to the documentation of \c LPMUpdater for more detailed explanation). +/// Specifically, in loop mode, all loops in the funciton will be pushed into +/// the worklist and processed by \p Pass, while only top-level loops are +/// processed in loop-nest mode. Please refer to the various specializations of +/// \fn createLoopFunctionToLoopPassAdaptor to see when loop mode and loop-nest +/// mode are used. class FunctionToLoopPassAdaptor : public PassInfoMixin { public: @@ -376,10 +406,12 @@ explicit FunctionToLoopPassAdaptor(std::unique_ptr Pass, bool UseMemorySSA = false, bool UseBlockFrequencyInfo = false, - bool DebugLogging = false) + bool DebugLogging = false, + bool LoopNestMode = false) : Pass(std::move(Pass)), LoopCanonicalizationFPM(DebugLogging), UseMemorySSA(UseMemorySSA), - UseBlockFrequencyInfo(UseBlockFrequencyInfo) { + UseBlockFrequencyInfo(UseBlockFrequencyInfo), + LoopNestMode(LoopNestMode) { LoopCanonicalizationFPM.addPass(LoopSimplifyPass()); LoopCanonicalizationFPM.addPass(LCSSAPass()); } @@ -389,6 +421,8 @@ static bool isRequired() { return true; } + bool isLoopNestMode() const { return LoopNestMode; } + private: std::unique_ptr Pass; @@ -396,12 +430,16 @@ bool UseMemorySSA = false; bool UseBlockFrequencyInfo = false; + const bool LoopNestMode; }; /// A function to deduce a loop pass type and wrap it in the templated /// adaptor. +/// +/// If \p Pass is a loop pass, the returned adaptor will be in loop mode. template -FunctionToLoopPassAdaptor +inline std::enable_if_t::value, + FunctionToLoopPassAdaptor> createFunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA = false, bool UseBlockFrequencyInfo = false, bool DebugLogging = false) { @@ -410,7 +448,46 @@ LoopStandardAnalysisResults &, LPMUpdater &>; return FunctionToLoopPassAdaptor( std::make_unique(std::move(Pass)), UseMemorySSA, - UseBlockFrequencyInfo, DebugLogging); + UseBlockFrequencyInfo, DebugLogging, false); +} + +/// If \p Pass is a loop-nest pass, \p Pass will first be wrapped into a +/// \c LoopPassManager and the returned adaptor will be in loop-nest mode. +template +inline std::enable_if_t::value, + FunctionToLoopPassAdaptor> +createFunctionToLoopPassAdaptor(LoopNestPassT Pass, bool UseMemorySSA = false, + bool UseBlockFrequencyInfo = false, + bool DebugLogging = false) { + LoopPassManager LPM(DebugLogging); + LPM.addPass(std::move(Pass)); + using PassModelT = + detail::PassModel; + return FunctionToLoopPassAdaptor(std::make_unique(std::move(LPM)), + UseMemorySSA, UseBlockFrequencyInfo, + DebugLogging, true); +} + +/// If \p Pass is an instance of \c LoopPassManager, the returned adaptor will +/// be in loop-nest mode if the pass manager contains only loop-nest passes. +template <> +inline FunctionToLoopPassAdaptor +createFunctionToLoopPassAdaptor(LoopPassManager LPM, + bool UseMemorySSA, + bool UseBlockFrequencyInfo, + bool DebugLogging) { + // Check if LPM contains any loop pass and if it does not, returns an adaptor + // in loop-nest mode. + using PassModelT = + detail::PassModel; + bool LoopNestMode = (LPM.getNumLoopPasses() == 0); + return FunctionToLoopPassAdaptor(std::make_unique(std::move(LPM)), + UseMemorySSA, UseBlockFrequencyInfo, + DebugLogging, LoopNestMode); } /// Pass for printing a loop's contents as textual IR. diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp --- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp +++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp @@ -222,11 +222,16 @@ // Register the worklist and loop analysis manager so that loop passes can // update them when they mutate the loop nest structure. - LPMUpdater Updater(Worklist, LAM); + LPMUpdater Updater(Worklist, LAM, LoopNestMode); // Add the loop nests in the reverse order of LoopInfo. See method // declaration. - appendLoopsToWorklist(LI, Worklist); + if (!LoopNestMode) { + appendLoopsToWorklist(LI, Worklist); + } else { + for (Loop *L : LI) + Worklist.insert(L); + } #ifndef NDEBUG PI.pushBeforeNonSkippedPassCallback([&LAR, &LI](StringRef PassID, Any IR) { @@ -247,6 +252,8 @@ do { Loop *L = Worklist.pop_back_val(); + assert(!(LoopNestMode && L->getParentLoop()) && + "L should be a top-level loop in loop-nest mode."); // Reset the update structure for this loop. Updater.CurrentL = L; diff --git a/llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp b/llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp --- a/llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp +++ b/llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp @@ -1603,28 +1603,69 @@ } TEST_F(LoopPassManagerTest, HandleLoopNestPass) { - ::testing::InSequence MakeExpectationsSequenced; + ::testing::Sequence FSequence, GSequence; - EXPECT_CALL(MLPHandle, run(HasName("loop.0.0"), _, _, _)).Times(2); - EXPECT_CALL(MLPHandle, run(HasName("loop.0.1"), _, _, _)).Times(2); - EXPECT_CALL(MLPHandle, run(HasName("loop.0"), _, _, _)); - EXPECT_CALL(MLNPHandle, run(HasName("loop.0"), _, _, _)); - EXPECT_CALL(MLPHandle, run(HasName("loop.0"), _, _, _)); - EXPECT_CALL(MLNPHandle, run(HasName("loop.0"), _, _, _)); - EXPECT_CALL(MLPHandle, run(HasName("loop.g.0"), _, _, _)); - EXPECT_CALL(MLNPHandle, run(HasName("loop.g.0"), _, _, _)); - EXPECT_CALL(MLPHandle, run(HasName("loop.g.0"), _, _, _)); - EXPECT_CALL(MLNPHandle, run(HasName("loop.g.0"), _, _, _)); + EXPECT_CALL(MLPHandle, run(HasName("loop.0.0"), _, _, _)) + .Times(2) + .InSequence(FSequence); + EXPECT_CALL(MLPHandle, run(HasName("loop.0.1"), _, _, _)) + .Times(2) + .InSequence(FSequence); + EXPECT_CALL(MLPHandle, run(HasName("loop.0"), _, _, _)).InSequence(FSequence); + EXPECT_CALL(MLNPHandle, run(HasName("loop.0"), _, _, _)) + .InSequence(FSequence); + EXPECT_CALL(MLPHandle, run(HasName("loop.0"), _, _, _)).InSequence(FSequence); + EXPECT_CALL(MLNPHandle, run(HasName("loop.0"), _, _, _)) + .InSequence(FSequence); + EXPECT_CALL(MLPHandle, run(HasName("loop.g.0"), _, _, _)) + .InSequence(GSequence); + EXPECT_CALL(MLNPHandle, run(HasName("loop.g.0"), _, _, _)) + .InSequence(GSequence); + EXPECT_CALL(MLPHandle, run(HasName("loop.g.0"), _, _, _)) + .InSequence(GSequence); + EXPECT_CALL(MLNPHandle, run(HasName("loop.g.0"), _, _, _)) + .InSequence(GSequence); - LoopPassManager LPM(true); - LPM.addPass(MLPHandle.getPass()); - LPM.addPass(MLNPHandle.getPass()); - LPM.addPass(MLPHandle.getPass()); - LPM.addPass(MLNPHandle.getPass()); + EXPECT_CALL(MLNPHandle, run(HasName("loop.0"), _, _, _)) + .InSequence(FSequence); + EXPECT_CALL(MLNPHandle, run(HasName("loop.g.0"), _, _, _)) + .InSequence(GSequence); + + EXPECT_CALL(MLNPHandle, run(HasName("loop.0"), _, _, _)) + .InSequence(FSequence); + EXPECT_CALL(MLNPHandle, run(HasName("loop.g.0"), _, _, _)) + .InSequence(GSequence); ModulePassManager MPM(true); - MPM.addPass(createModuleToFunctionPassAdaptor( - createFunctionToLoopPassAdaptor(std::move(LPM)))); + FunctionPassManager FPM(true); + + { + LoopPassManager LPM(true); + LPM.addPass(MLPHandle.getPass()); + LPM.addPass(MLNPHandle.getPass()); + LPM.addPass(MLPHandle.getPass()); + LPM.addPass(MLNPHandle.getPass()); + + auto Adaptor = createFunctionToLoopPassAdaptor(std::move(LPM)); + ASSERT_FALSE(Adaptor.isLoopNestMode()); + FPM.addPass(std::move(Adaptor)); + } + + { + auto Adaptor = createFunctionToLoopPassAdaptor(MLNPHandle.getPass()); + ASSERT_TRUE(Adaptor.isLoopNestMode()); + FPM.addPass(std::move(Adaptor)); + } + + { + LoopPassManager LPM(true); + LPM.addPass(MLNPHandle.getPass()); + auto Adaptor = createFunctionToLoopPassAdaptor(MLNPHandle.getPass()); + ASSERT_TRUE(Adaptor.isLoopNestMode()); + FPM.addPass(std::move(Adaptor)); + } + + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); MPM.run(*M, MAM); }