diff --git a/llvm/include/llvm/Analysis/CGSCCPassManager.h b/llvm/include/llvm/Analysis/CGSCCPassManager.h --- a/llvm/include/llvm/Analysis/CGSCCPassManager.h +++ b/llvm/include/llvm/Analysis/CGSCCPassManager.h @@ -336,18 +336,15 @@ /// \c CGSCCAnalysisManagerModuleProxy analysis prior to running the CGSCC /// pass over the module to enable a \c FunctionAnalysisManager to be used /// within this run safely. -template class ModuleToPostOrderCGSCCPassAdaptor - : public PassInfoMixin> { + : public PassInfoMixin { public: - explicit ModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT Pass) - : Pass(std::move(Pass)) {} + using PassConceptT = + detail::PassConcept; - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - ModuleToPostOrderCGSCCPassAdaptor( - const ModuleToPostOrderCGSCCPassAdaptor &Arg) - : Pass(Arg.Pass) {} + explicit ModuleToPostOrderCGSCCPassAdaptor(std::unique_ptr Pass) + : Pass(std::move(Pass)) {} ModuleToPostOrderCGSCCPassAdaptor(ModuleToPostOrderCGSCCPassAdaptor &&Arg) : Pass(std::move(Arg.Pass)) {} @@ -369,15 +366,19 @@ static bool isRequired() { return true; } private: - CGSCCPassT Pass; + std::unique_ptr Pass; }; /// A function to deduce a function pass type and wrap it in the /// templated adaptor. template -ModuleToPostOrderCGSCCPassAdaptor +ModuleToPostOrderCGSCCPassAdaptor createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT Pass) { - return ModuleToPostOrderCGSCCPassAdaptor(std::move(Pass)); + using PassModelT = detail::PassModel; + return ModuleToPostOrderCGSCCPassAdaptor( + std::make_unique(std::move(Pass))); } /// A proxy from a \c FunctionAnalysisManager to an \c SCC. @@ -455,17 +456,13 @@ /// \c FunctionAnalysisManagerCGSCCProxy analysis prior to running the function /// pass over the SCC to enable a \c FunctionAnalysisManager to be used /// within this run safely. -template class CGSCCToFunctionPassAdaptor - : public PassInfoMixin> { + : public PassInfoMixin { public: - explicit CGSCCToFunctionPassAdaptor(FunctionPassT Pass) - : Pass(std::move(Pass)) {} + using PassConceptT = detail::PassConcept; - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - CGSCCToFunctionPassAdaptor(const CGSCCToFunctionPassAdaptor &Arg) - : Pass(Arg.Pass) {} + explicit CGSCCToFunctionPassAdaptor(std::unique_ptr Pass) + : Pass(std::move(Pass)) {} CGSCCToFunctionPassAdaptor(CGSCCToFunctionPassAdaptor &&Arg) : Pass(std::move(Arg.Pass)) {} @@ -482,92 +479,24 @@ /// Runs the function pass across every function in the module. PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, - LazyCallGraph &CG, CGSCCUpdateResult &UR) { - // Setup the function analysis manager from its proxy. - FunctionAnalysisManager &FAM = - AM.getResult(C, CG).getManager(); - - SmallVector Nodes; - for (LazyCallGraph::Node &N : C) - Nodes.push_back(&N); - - // The SCC may get split while we are optimizing functions due to deleting - // edges. If this happens, the current SCC can shift, so keep track of - // a pointer we can overwrite. - LazyCallGraph::SCC *CurrentC = &C; - - LLVM_DEBUG(dbgs() << "Running function passes across an SCC: " << C - << "\n"); - - PreservedAnalyses PA = PreservedAnalyses::all(); - for (LazyCallGraph::Node *N : Nodes) { - // Skip nodes from other SCCs. These may have been split out during - // processing. We'll eventually visit those SCCs and pick up the nodes - // there. - if (CG.lookupSCC(*N) != CurrentC) - continue; - - Function &F = N->getFunction(); - - PassInstrumentation PI = FAM.getResult(F); - if (!PI.runBeforePass(Pass, F)) - continue; - - PreservedAnalyses PassPA; - { - TimeTraceScope TimeScope(Pass.name()); - PassPA = Pass.run(F, FAM); - } - - PI.runAfterPass(Pass, F, PassPA); - - // We know that the function pass couldn't have invalidated any other - // function's analyses (that's the contract of a function pass), so - // directly handle the function analysis manager's invalidation here. - FAM.invalidate(F, PassPA); - - // Then intersect the preserved set so that invalidation of module - // analyses will eventually occur when the module pass completes. - PA.intersect(std::move(PassPA)); - - // If the call graph hasn't been preserved, update it based on this - // function pass. This may also update the current SCC to point to - // a smaller, more refined SCC. - auto PAC = PA.getChecker(); - if (!PAC.preserved() && !PAC.preservedSet>()) { - CurrentC = &updateCGAndAnalysisManagerForFunctionPass(CG, *CurrentC, *N, - AM, UR, FAM); - assert( - CG.lookupSCC(*N) == CurrentC && - "Current SCC not updated to the SCC containing the current node!"); - } - } - - // By definition we preserve the proxy. And we preserve all analyses on - // Functions. This precludes *any* invalidation of function analyses by the - // proxy, but that's OK because we've taken care to invalidate analyses in - // the function analysis manager incrementally above. - PA.preserveSet>(); - PA.preserve(); - - // We've also ensured that we updated the call graph along the way. - PA.preserve(); - - return PA; - } + LazyCallGraph &CG, CGSCCUpdateResult &UR); static bool isRequired() { return true; } private: - FunctionPassT Pass; + std::unique_ptr Pass; }; /// A function to deduce a function pass type and wrap it in the /// templated adaptor. template -CGSCCToFunctionPassAdaptor +CGSCCToFunctionPassAdaptor createCGSCCToFunctionPassAdaptor(FunctionPassT Pass) { - return CGSCCToFunctionPassAdaptor(std::move(Pass)); + using PassModelT = + detail::PassModel; + return CGSCCToFunctionPassAdaptor( + std::make_unique(std::move(Pass))); } /// Checks -abort-on-max-devirt-iterations-reached to see if we should report an @@ -588,401 +517,36 @@ /// This repetition has the potential to be very large however, as each one /// might refine a single call site. As a consequence, in practice we use an /// upper bound on the number of repetitions to limit things. -template -class DevirtSCCRepeatedPass - : public PassInfoMixin> { +class DevirtSCCRepeatedPass : public PassInfoMixin { public: - explicit DevirtSCCRepeatedPass(PassT Pass, int MaxIterations) + using PassConceptT = + detail::PassConcept; + + explicit DevirtSCCRepeatedPass(std::unique_ptr Pass, + int MaxIterations) : Pass(std::move(Pass)), MaxIterations(MaxIterations) {} /// Runs the wrapped pass up to \c MaxIterations on the SCC, iterating /// whenever an indirect call is refined. PreservedAnalyses run(LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, - LazyCallGraph &CG, CGSCCUpdateResult &UR) { - PreservedAnalyses PA = PreservedAnalyses::all(); - PassInstrumentation PI = - AM.getResult(InitialC, CG); - - // The SCC may be refined while we are running passes over it, so set up - // a pointer that we can update. - LazyCallGraph::SCC *C = &InitialC; - - // Struct to track the counts of direct and indirect calls in each function - // of the SCC. - struct CallCount { - int Direct; - int Indirect; - }; - - // Put value handles on all of the indirect calls and return the number of - // direct calls for each function in the SCC. - auto ScanSCC = - [](LazyCallGraph::SCC &C, - SmallMapVector &CallHandles) { - assert(CallHandles.empty() && - "Must start with a clear set of handles."); - - SmallDenseMap CallCounts; - CallCount CountLocal = {0, 0}; - for (LazyCallGraph::Node &N : C) { - CallCount &Count = - CallCounts.insert(std::make_pair(&N.getFunction(), CountLocal)) - .first->second; - for (Instruction &I : instructions(N.getFunction())) - if (auto *CB = dyn_cast(&I)) { - if (CB->getCalledFunction()) { - ++Count.Direct; - } else { - ++Count.Indirect; - CallHandles.insert({CB, WeakTrackingVH(CB)}); - } - } - } - - return CallCounts; - }; - - UR.IndirectVHs.clear(); - // Populate the initial call handles and get the initial call counts. - auto CallCounts = ScanSCC(*C, UR.IndirectVHs); - - for (int Iteration = 0;; ++Iteration) { - if (!PI.runBeforePass(Pass, *C)) - continue; - - PreservedAnalyses PassPA = Pass.run(*C, AM, CG, UR); - - if (UR.InvalidatedSCCs.count(C)) - PI.runAfterPassInvalidated(Pass, PassPA); - else - PI.runAfterPass(Pass, *C, PassPA); - - // If the SCC structure has changed, bail immediately and let the outer - // CGSCC layer handle any iteration to reflect the refined structure. - if (UR.UpdatedC && UR.UpdatedC != C) { - PA.intersect(std::move(PassPA)); - break; - } - - // Check that we didn't miss any update scenario. - assert(!UR.InvalidatedSCCs.count(C) && "Processing an invalid SCC!"); - assert(C->begin() != C->end() && "Cannot have an empty SCC!"); - - // Check whether any of the handles were devirtualized. - bool Devirt = llvm::any_of(UR.IndirectVHs, [](auto &P) -> bool { - if (P.second) { - if (CallBase *CB = dyn_cast(P.second)) { - if (CB->getCalledFunction()) { - LLVM_DEBUG(dbgs() << "Found devirtualized call: " << *CB << "\n"); - return true; - } - } - } - return false; - }); - - // Rescan to build up a new set of handles and count how many direct - // calls remain. If we decide to iterate, this also sets up the input to - // the next iteration. - UR.IndirectVHs.clear(); - auto NewCallCounts = ScanSCC(*C, UR.IndirectVHs); - - // If we haven't found an explicit devirtualization already see if we - // have decreased the number of indirect calls and increased the number - // of direct calls for any function in the SCC. This can be fooled by all - // manner of transformations such as DCE and other things, but seems to - // work well in practice. - if (!Devirt) - // Iterate over the keys in NewCallCounts, if Function also exists in - // CallCounts, make the check below. - for (auto &Pair : NewCallCounts) { - auto &CallCountNew = Pair.second; - auto CountIt = CallCounts.find(Pair.first); - if (CountIt != CallCounts.end()) { - const auto &CallCountOld = CountIt->second; - if (CallCountOld.Indirect > CallCountNew.Indirect && - CallCountOld.Direct < CallCountNew.Direct) { - Devirt = true; - break; - } - } - } - - if (!Devirt) { - PA.intersect(std::move(PassPA)); - break; - } - - // Otherwise, if we've already hit our max, we're done. - if (Iteration >= MaxIterations) { - maxDevirtIterationsReached(); - LLVM_DEBUG( - dbgs() << "Found another devirtualization after hitting the max " - "number of repetitions (" - << MaxIterations << ") on SCC: " << *C << "\n"); - PA.intersect(std::move(PassPA)); - break; - } - - LLVM_DEBUG( - dbgs() - << "Repeating an SCC pass after finding a devirtualization in: " << *C - << "\n"); - - // Move over the new call counts in preparation for iterating. - CallCounts = std::move(NewCallCounts); - - // Update the analysis manager with each run and intersect the total set - // of preserved analyses so we're ready to iterate. - AM.invalidate(*C, PassPA); - - PA.intersect(std::move(PassPA)); - } - - // Note that we don't add any preserved entries here unlike a more normal - // "pass manager" because we only handle invalidation *between* iterations, - // not after the last iteration. - return PA; - } + LazyCallGraph &CG, CGSCCUpdateResult &UR); private: - PassT Pass; + std::unique_ptr Pass; int MaxIterations; }; /// A function to deduce a function pass type and wrap it in the /// templated adaptor. -template -DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(PassT Pass, - int MaxIterations) { - return DevirtSCCRepeatedPass(std::move(Pass), MaxIterations); -} - -// Out-of-line implementation details for templates below this point. - template -PreservedAnalyses -ModuleToPostOrderCGSCCPassAdaptor::run(Module &M, - ModuleAnalysisManager &AM) { - // Setup the CGSCC analysis manager from its proxy. - CGSCCAnalysisManager &CGAM = - AM.getResult(M).getManager(); - - // Get the call graph for this module. - LazyCallGraph &CG = AM.getResult(M); - - // Get Function analysis manager from its proxy. - FunctionAnalysisManager &FAM = - AM.getCachedResult(M)->getManager(); - - // We keep worklists to allow us to push more work onto the pass manager as - // the passes are run. - SmallPriorityWorklist RCWorklist; - SmallPriorityWorklist CWorklist; - - // Keep sets for invalidated SCCs and RefSCCs that should be skipped when - // iterating off the worklists. - SmallPtrSet InvalidRefSCCSet; - SmallPtrSet InvalidSCCSet; - - SmallDenseSet, 4> - InlinedInternalEdges; - - CGSCCUpdateResult UR = { - RCWorklist, CWorklist, InvalidRefSCCSet, InvalidSCCSet, - nullptr, nullptr, PreservedAnalyses::all(), InlinedInternalEdges, - {}}; - - // Request PassInstrumentation from analysis manager, will use it to run - // instrumenting callbacks for the passes later. - PassInstrumentation PI = AM.getResult(M); - - PreservedAnalyses PA = PreservedAnalyses::all(); - CG.buildRefSCCs(); - for (auto RCI = CG.postorder_ref_scc_begin(), - RCE = CG.postorder_ref_scc_end(); - RCI != RCE;) { - assert(RCWorklist.empty() && - "Should always start with an empty RefSCC worklist"); - // The postorder_ref_sccs range we are walking is lazily constructed, so - // we only push the first one onto the worklist. The worklist allows us - // to capture *new* RefSCCs created during transformations. - // - // We really want to form RefSCCs lazily because that makes them cheaper - // to update as the program is simplified and allows us to have greater - // cache locality as forming a RefSCC touches all the parts of all the - // functions within that RefSCC. - // - // We also eagerly increment the iterator to the next position because - // the CGSCC passes below may delete the current RefSCC. - RCWorklist.insert(&*RCI++); - - do { - LazyCallGraph::RefSCC *RC = RCWorklist.pop_back_val(); - if (InvalidRefSCCSet.count(RC)) { - LLVM_DEBUG(dbgs() << "Skipping an invalid RefSCC...\n"); - continue; - } - - assert(CWorklist.empty() && - "Should always start with an empty SCC worklist"); - - LLVM_DEBUG(dbgs() << "Running an SCC pass across the RefSCC: " << *RC - << "\n"); - - // The top of the worklist may *also* be the same SCC we just ran over - // (and invalidated for). Keep track of that last SCC we processed due - // to SCC update to avoid redundant processing when an SCC is both just - // updated itself and at the top of the worklist. - LazyCallGraph::SCC *LastUpdatedC = nullptr; - - // Push the initial SCCs in reverse post-order as we'll pop off the - // back and so see this in post-order. - for (LazyCallGraph::SCC &C : llvm::reverse(*RC)) - CWorklist.insert(&C); - - do { - LazyCallGraph::SCC *C = CWorklist.pop_back_val(); - // Due to call graph mutations, we may have invalid SCCs or SCCs from - // other RefSCCs in the worklist. The invalid ones are dead and the - // other RefSCCs should be queued above, so we just need to skip both - // scenarios here. - if (InvalidSCCSet.count(C)) { - LLVM_DEBUG(dbgs() << "Skipping an invalid SCC...\n"); - continue; - } - if (LastUpdatedC == C) { - LLVM_DEBUG(dbgs() << "Skipping redundant run on SCC: " << *C << "\n"); - continue; - } - if (&C->getOuterRefSCC() != RC) { - LLVM_DEBUG(dbgs() << "Skipping an SCC that is now part of some other " - "RefSCC...\n"); - continue; - } - - // Ensure we can proxy analysis updates from the CGSCC analysis manager - // into the the Function analysis manager by getting a proxy here. - // This also needs to update the FunctionAnalysisManager, as this may be - // the first time we see this SCC. - CGAM.getResult(*C, CG).updateFAM( - FAM); - - // Each time we visit a new SCC pulled off the worklist, - // a transformation of a child SCC may have also modified this parent - // and invalidated analyses. So we invalidate using the update record's - // cross-SCC preserved set. This preserved set is intersected by any - // CGSCC pass that handles invalidation (primarily pass managers) prior - // to marking its SCC as preserved. That lets us track everything that - // might need invalidation across SCCs without excessive invalidations - // on a single SCC. - // - // This essentially allows SCC passes to freely invalidate analyses - // of any ancestor SCC. If this becomes detrimental to successfully - // caching analyses, we could force each SCC pass to manually - // invalidate the analyses for any SCCs other than themselves which - // are mutated. However, that seems to lose the robustness of the - // pass-manager driven invalidation scheme. - CGAM.invalidate(*C, UR.CrossSCCPA); - - do { - // Check that we didn't miss any update scenario. - assert(!InvalidSCCSet.count(C) && "Processing an invalid SCC!"); - assert(C->begin() != C->end() && "Cannot have an empty SCC!"); - assert(&C->getOuterRefSCC() == RC && - "Processing an SCC in a different RefSCC!"); - - LastUpdatedC = UR.UpdatedC; - UR.UpdatedRC = nullptr; - UR.UpdatedC = nullptr; - - // Check the PassInstrumentation's BeforePass callbacks before - // running the pass, skip its execution completely if asked to - // (callback returns false). - if (!PI.runBeforePass(Pass, *C)) - continue; - - PreservedAnalyses PassPA; - { - TimeTraceScope TimeScope(Pass.name()); - PassPA = Pass.run(*C, CGAM, CG, UR); - } - - if (UR.InvalidatedSCCs.count(C)) - PI.runAfterPassInvalidated(Pass, PassPA); - else - PI.runAfterPass(Pass, *C, PassPA); - - // Update the SCC and RefSCC if necessary. - C = UR.UpdatedC ? UR.UpdatedC : C; - RC = UR.UpdatedRC ? UR.UpdatedRC : RC; - - if (UR.UpdatedC) { - // If we're updating the SCC, also update the FAM inside the proxy's - // result. - CGAM.getResult(*C, CG).updateFAM( - FAM); - } - - // If the CGSCC pass wasn't able to provide a valid updated SCC, - // the current SCC may simply need to be skipped if invalid. - if (UR.InvalidatedSCCs.count(C)) { - LLVM_DEBUG(dbgs() << "Skipping invalidated root or island SCC!\n"); - break; - } - // Check that we didn't miss any update scenario. - assert(C->begin() != C->end() && "Cannot have an empty SCC!"); - - // We handle invalidating the CGSCC analysis manager's information - // for the (potentially updated) SCC here. Note that any other SCCs - // whose structure has changed should have been invalidated by - // whatever was updating the call graph. This SCC gets invalidated - // late as it contains the nodes that were actively being - // processed. - CGAM.invalidate(*C, PassPA); - - // Then intersect the preserved set so that invalidation of module - // analyses will eventually occur when the module pass completes. - // Also intersect with the cross-SCC preserved set to capture any - // cross-SCC invalidation. - UR.CrossSCCPA.intersect(PassPA); - PA.intersect(std::move(PassPA)); - - // The pass may have restructured the call graph and refined the - // current SCC and/or RefSCC. We need to update our current SCC and - // RefSCC pointers to follow these. Also, when the current SCC is - // refined, re-run the SCC pass over the newly refined SCC in order - // to observe the most precise SCC model available. This inherently - // cannot cycle excessively as it only happens when we split SCCs - // apart, at most converging on a DAG of single nodes. - // FIXME: If we ever start having RefSCC passes, we'll want to - // iterate there too. - if (UR.UpdatedC) - LLVM_DEBUG(dbgs() - << "Re-running SCC passes after a refinement of the " - "current SCC: " - << *UR.UpdatedC << "\n"); - - // Note that both `C` and `RC` may at this point refer to deleted, - // invalid SCC and RefSCCs respectively. But we will short circuit - // the processing when we check them in the loop above. - } while (UR.UpdatedC); - } while (!CWorklist.empty()); - - // We only need to keep internal inlined edge information within - // a RefSCC, clear it to save on space and let the next time we visit - // any of these functions have a fresh start. - InlinedInternalEdges.clear(); - } while (!RCWorklist.empty()); - } - - // By definition we preserve the call garph, all SCC analyses, and the - // analysis proxies by handling them above and in any nested pass managers. - PA.preserveSet>(); - PA.preserve(); - PA.preserve(); - PA.preserve(); - return PA; +DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(CGSCCPassT Pass, + int MaxIterations) { + using PassModelT = detail::PassModel; + return DevirtSCCRepeatedPass(std::make_unique(std::move(Pass)), + MaxIterations); } // Clear out the debug logging macro. diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h --- a/llvm/include/llvm/IR/PassManager.h +++ b/llvm/include/llvm/IR/PassManager.h @@ -1228,73 +1228,34 @@ /// Note that although function passes can access module analyses, module /// analyses are not invalidated while the function passes are running, so they /// may be stale. Function analyses will not be stale. -template class ModuleToFunctionPassAdaptor - : public PassInfoMixin> { + : public PassInfoMixin { public: - explicit ModuleToFunctionPassAdaptor(FunctionPassT Pass) + using PassConceptT = detail::PassConcept; + + explicit ModuleToFunctionPassAdaptor(std::unique_ptr Pass) : Pass(std::move(Pass)) {} /// Runs the function pass across every function in the module. - PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM) { - FunctionAnalysisManager &FAM = - AM.getResult(M).getManager(); - - // Request PassInstrumentation from analysis manager, will use it to run - // instrumenting callbacks for the passes later. - PassInstrumentation PI = AM.getResult(M); - - PreservedAnalyses PA = PreservedAnalyses::all(); - for (Function &F : M) { - if (F.isDeclaration()) - continue; - - // Check the PassInstrumentation's BeforePass callbacks before running the - // pass, skip its execution completely if asked to (callback returns - // false). - if (!PI.runBeforePass(Pass, F)) - continue; - - PreservedAnalyses PassPA; - { - TimeTraceScope TimeScope(Pass.name(), F.getName()); - PassPA = Pass.run(F, FAM); - } - - PI.runAfterPass(Pass, F, PassPA); - - // We know that the function pass couldn't have invalidated any other - // function's analyses (that's the contract of a function pass), so - // directly handle the function analysis manager's invalidation here. - FAM.invalidate(F, PassPA); - - // Then intersect the preserved set so that invalidation of module - // analyses will eventually occur when the module pass completes. - PA.intersect(std::move(PassPA)); - } - - // The FunctionAnalysisManagerModuleProxy is preserved because (we assume) - // the function passes we ran didn't add or remove any functions. - // - // We also preserve all analyses on Functions, because we did all the - // invalidation we needed to do above. - PA.preserveSet>(); - PA.preserve(); - return PA; - } + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); static bool isRequired() { return true; } private: - FunctionPassT Pass; + std::unique_ptr Pass; }; /// A function to deduce a function pass type and wrap it in the /// templated adaptor. template -ModuleToFunctionPassAdaptor +ModuleToFunctionPassAdaptor createModuleToFunctionPassAdaptor(FunctionPassT Pass) { - return ModuleToFunctionPassAdaptor(std::move(Pass)); + using PassModelT = + detail::PassModel; + + return ModuleToFunctionPassAdaptor( + std::make_unique(std::move(Pass))); } /// A utility pass template to force an analysis result to be available. diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h --- a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h @@ -56,6 +56,7 @@ #include "llvm/Transforms/Utils/LCSSA.h" #include "llvm/Transforms/Utils/LoopSimplify.h" #include "llvm/Transforms/Utils/LoopUtils.h" +#include namespace llvm { @@ -105,7 +106,7 @@ RequireAnalysisPass; -template class FunctionToLoopPassAdaptor; +class FunctionToLoopPassAdaptor; /// This class provides an interface for updating the loop pass manager based /// on mutations to the loop nest. @@ -200,7 +201,7 @@ } private: - template friend class llvm::FunctionToLoopPassAdaptor; + friend class llvm::FunctionToLoopPassAdaptor; /// The \c FunctionToLoopPassAdaptor's worklist of loops to process. SmallPriorityWorklist &Worklist; @@ -229,11 +230,15 @@ /// FunctionAnalysisManager it will run the \c LoopAnalysisManagerFunctionProxy /// analysis prior to running the loop passes over the function to enable a \c /// LoopAnalysisManager to be used within this run safely. -template class FunctionToLoopPassAdaptor - : public PassInfoMixin> { + : public PassInfoMixin { public: - explicit FunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA = false, + using PassConceptT = + detail::PassConcept; + + explicit FunctionToLoopPassAdaptor(std::unique_ptr Pass, + bool UseMemorySSA = false, bool UseBlockFrequencyInfo = false, bool DebugLogging = false) : Pass(std::move(Pass)), LoopCanonicalizationFPM(DebugLogging), @@ -244,156 +249,12 @@ } /// Runs the loop passes across every loop in the function. - PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) { - // Before we even compute any loop analyses, first run a miniature function - // pass pipeline to put loops into their canonical form. Note that we can - // directly build up function analyses after this as the function pass - // manager handles all the invalidation at that layer. - PassInstrumentation PI = AM.getResult(F); - - PreservedAnalyses PA = PreservedAnalyses::all(); - // Check the PassInstrumentation's BeforePass callbacks before running the - // canonicalization pipeline. - if (PI.runBeforePass(LoopCanonicalizationFPM, F)) { - PA = LoopCanonicalizationFPM.run(F, AM); - PI.runAfterPass(LoopCanonicalizationFPM, F, PA); - } - - // Get the loop structure for this function - LoopInfo &LI = AM.getResult(F); - - // If there are no loops, there is nothing to do here. - if (LI.empty()) - return PA; - - // Get the analysis results needed by loop passes. - MemorySSA *MSSA = UseMemorySSA - ? (&AM.getResult(F).getMSSA()) - : nullptr; - BlockFrequencyInfo *BFI = UseBlockFrequencyInfo && F.hasProfileData() - ? (&AM.getResult(F)) - : nullptr; - LoopStandardAnalysisResults LAR = {AM.getResult(F), - AM.getResult(F), - AM.getResult(F), - AM.getResult(F), - AM.getResult(F), - AM.getResult(F), - AM.getResult(F), - BFI, - MSSA}; - - // Setup the loop analysis manager from its proxy. It is important that - // this is only done when there are loops to process and we have built the - // LoopStandardAnalysisResults object. The loop analyses cached in this - // manager have access to those analysis results and so it must invalidate - // itself when they go away. - auto &LAMFP = AM.getResult(F); - if (UseMemorySSA) - LAMFP.markMSSAUsed(); - LoopAnalysisManager &LAM = LAMFP.getManager(); - - // A postorder worklist of loops to process. - SmallPriorityWorklist Worklist; - - // Register the worklist and loop analysis manager so that loop passes can - // update them when they mutate the loop nest structure. - LPMUpdater Updater(Worklist, LAM); - - // Add the loop nests in the reverse order of LoopInfo. See method - // declaration. - appendLoopsToWorklist(LI, Worklist); - -#ifndef NDEBUG - PI.pushBeforeNonSkippedPassCallback([&LAR, &LI](StringRef PassID, Any IR) { - if (isSpecialPass(PassID, {"PassManager"})) - return; - assert(any_isa(IR)); - const Loop *L = any_cast(IR); - assert(L && "Loop should be valid for printing"); - - // Verify the loop structure and LCSSA form before visiting the loop. - L->verifyLoop(); - assert(L->isRecursivelyLCSSAForm(LAR.DT, LI) && - "Loops must remain in LCSSA form!"); - }); -#endif - - do { - Loop *L = Worklist.pop_back_val(); - - // Reset the update structure for this loop. - Updater.CurrentL = L; - Updater.SkipCurrentLoop = false; - -#ifndef NDEBUG - // Save a parent loop pointer for asserts. - Updater.ParentL = L->getParentLoop(); -#endif - // Check the PassInstrumentation's BeforePass callbacks before running the - // pass, skip its execution completely if asked to (callback returns - // false). - if (!PI.runBeforePass(Pass, *L)) - continue; - - PreservedAnalyses PassPA; - { - TimeTraceScope TimeScope(Pass.name()); - PassPA = Pass.run(*L, LAM, LAR, Updater); - } - - // Do not pass deleted Loop into the instrumentation. - if (Updater.skipCurrentLoop()) - PI.runAfterPassInvalidated(Pass, PassPA); - else - PI.runAfterPass(Pass, *L, PassPA); - - // FIXME: We should verify the set of analyses relevant to Loop passes - // are preserved. - - // If the loop hasn't been deleted, we need to handle invalidation here. - if (!Updater.skipCurrentLoop()) - // We know that the loop pass couldn't have invalidated any other - // loop's analyses (that's the contract of a loop pass), so directly - // handle the loop analysis manager's invalidation here. - LAM.invalidate(*L, PassPA); - - // Then intersect the preserved set so that invalidation of module - // analyses will eventually occur when the module pass completes. - PA.intersect(std::move(PassPA)); - } while (!Worklist.empty()); - -#ifndef NDEBUG - PI.popBeforeNonSkippedPassCallback(); -#endif - - // By definition we preserve the proxy. We also preserve all analyses on - // Loops. This precludes *any* invalidation of loop analyses by the proxy, - // but that's OK because we've taken care to invalidate analyses in the - // loop analysis manager incrementally above. - PA.preserveSet>(); - PA.preserve(); - // We also preserve the set of standard analyses. - PA.preserve(); - PA.preserve(); - PA.preserve(); - if (UseBlockFrequencyInfo && F.hasProfileData()) - PA.preserve(); - if (UseMemorySSA) - PA.preserve(); - // FIXME: What we really want to do here is preserve an AA category, but - // that concept doesn't exist yet. - PA.preserve(); - PA.preserve(); - PA.preserve(); - PA.preserve(); - return PA; - } + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); static bool isRequired() { return true; } private: - LoopPassT Pass; + std::unique_ptr Pass; FunctionPassManager LoopCanonicalizationFPM; @@ -404,12 +265,16 @@ /// A function to deduce a loop pass type and wrap it in the templated /// adaptor. template -FunctionToLoopPassAdaptor +FunctionToLoopPassAdaptor createFunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA = false, bool UseBlockFrequencyInfo = false, bool DebugLogging = false) { - return FunctionToLoopPassAdaptor( - std::move(Pass), UseMemorySSA, UseBlockFrequencyInfo, DebugLogging); + using PassModelT = + detail::PassModel; + return FunctionToLoopPassAdaptor( + std::make_unique(std::move(Pass)), UseMemorySSA, + UseBlockFrequencyInfo, DebugLogging); } /// Pass for printing a loop's contents as textual IR. diff --git a/llvm/lib/Analysis/CGSCCPassManager.cpp b/llvm/lib/Analysis/CGSCCPassManager.cpp --- a/llvm/lib/Analysis/CGSCCPassManager.cpp +++ b/llvm/lib/Analysis/CGSCCPassManager.cpp @@ -148,6 +148,451 @@ return PA; } +PreservedAnalyses +ModuleToPostOrderCGSCCPassAdaptor::run(Module &M, ModuleAnalysisManager &AM) { + // Setup the CGSCC analysis manager from its proxy. + CGSCCAnalysisManager &CGAM = + AM.getResult(M).getManager(); + + // Get the call graph for this module. + LazyCallGraph &CG = AM.getResult(M); + + // Get Function analysis manager from its proxy. + FunctionAnalysisManager &FAM = + AM.getCachedResult(M)->getManager(); + + // We keep worklists to allow us to push more work onto the pass manager as + // the passes are run. + SmallPriorityWorklist RCWorklist; + SmallPriorityWorklist CWorklist; + + // Keep sets for invalidated SCCs and RefSCCs that should be skipped when + // iterating off the worklists. + SmallPtrSet InvalidRefSCCSet; + SmallPtrSet InvalidSCCSet; + + SmallDenseSet, 4> + InlinedInternalEdges; + + CGSCCUpdateResult UR = { + RCWorklist, CWorklist, InvalidRefSCCSet, InvalidSCCSet, + nullptr, nullptr, PreservedAnalyses::all(), InlinedInternalEdges, + {}}; + + // Request PassInstrumentation from analysis manager, will use it to run + // instrumenting callbacks for the passes later. + PassInstrumentation PI = AM.getResult(M); + + PreservedAnalyses PA = PreservedAnalyses::all(); + CG.buildRefSCCs(); + for (auto RCI = CG.postorder_ref_scc_begin(), + RCE = CG.postorder_ref_scc_end(); + RCI != RCE;) { + assert(RCWorklist.empty() && + "Should always start with an empty RefSCC worklist"); + // The postorder_ref_sccs range we are walking is lazily constructed, so + // we only push the first one onto the worklist. The worklist allows us + // to capture *new* RefSCCs created during transformations. + // + // We really want to form RefSCCs lazily because that makes them cheaper + // to update as the program is simplified and allows us to have greater + // cache locality as forming a RefSCC touches all the parts of all the + // functions within that RefSCC. + // + // We also eagerly increment the iterator to the next position because + // the CGSCC passes below may delete the current RefSCC. + RCWorklist.insert(&*RCI++); + + do { + LazyCallGraph::RefSCC *RC = RCWorklist.pop_back_val(); + if (InvalidRefSCCSet.count(RC)) { + LLVM_DEBUG(dbgs() << "Skipping an invalid RefSCC...\n"); + continue; + } + + assert(CWorklist.empty() && + "Should always start with an empty SCC worklist"); + + LLVM_DEBUG(dbgs() << "Running an SCC pass across the RefSCC: " << *RC + << "\n"); + + // The top of the worklist may *also* be the same SCC we just ran over + // (and invalidated for). Keep track of that last SCC we processed due + // to SCC update to avoid redundant processing when an SCC is both just + // updated itself and at the top of the worklist. + LazyCallGraph::SCC *LastUpdatedC = nullptr; + + // Push the initial SCCs in reverse post-order as we'll pop off the + // back and so see this in post-order. + for (LazyCallGraph::SCC &C : llvm::reverse(*RC)) + CWorklist.insert(&C); + + do { + LazyCallGraph::SCC *C = CWorklist.pop_back_val(); + // Due to call graph mutations, we may have invalid SCCs or SCCs from + // other RefSCCs in the worklist. The invalid ones are dead and the + // other RefSCCs should be queued above, so we just need to skip both + // scenarios here. + if (InvalidSCCSet.count(C)) { + LLVM_DEBUG(dbgs() << "Skipping an invalid SCC...\n"); + continue; + } + if (LastUpdatedC == C) { + LLVM_DEBUG(dbgs() << "Skipping redundant run on SCC: " << *C << "\n"); + continue; + } + if (&C->getOuterRefSCC() != RC) { + LLVM_DEBUG(dbgs() << "Skipping an SCC that is now part of some other " + "RefSCC...\n"); + continue; + } + + // Ensure we can proxy analysis updates from the CGSCC analysis manager + // into the the Function analysis manager by getting a proxy here. + // This also needs to update the FunctionAnalysisManager, as this may be + // the first time we see this SCC. + CGAM.getResult(*C, CG).updateFAM( + FAM); + + // Each time we visit a new SCC pulled off the worklist, + // a transformation of a child SCC may have also modified this parent + // and invalidated analyses. So we invalidate using the update record's + // cross-SCC preserved set. This preserved set is intersected by any + // CGSCC pass that handles invalidation (primarily pass managers) prior + // to marking its SCC as preserved. That lets us track everything that + // might need invalidation across SCCs without excessive invalidations + // on a single SCC. + // + // This essentially allows SCC passes to freely invalidate analyses + // of any ancestor SCC. If this becomes detrimental to successfully + // caching analyses, we could force each SCC pass to manually + // invalidate the analyses for any SCCs other than themselves which + // are mutated. However, that seems to lose the robustness of the + // pass-manager driven invalidation scheme. + CGAM.invalidate(*C, UR.CrossSCCPA); + + do { + // Check that we didn't miss any update scenario. + assert(!InvalidSCCSet.count(C) && "Processing an invalid SCC!"); + assert(C->begin() != C->end() && "Cannot have an empty SCC!"); + assert(&C->getOuterRefSCC() == RC && + "Processing an SCC in a different RefSCC!"); + + LastUpdatedC = UR.UpdatedC; + UR.UpdatedRC = nullptr; + UR.UpdatedC = nullptr; + + // Check the PassInstrumentation's BeforePass callbacks before + // running the pass, skip its execution completely if asked to + // (callback returns false). + if (!PI.runBeforePass(*Pass, *C)) + continue; + + PreservedAnalyses PassPA; + { + TimeTraceScope TimeScope(Pass->name()); + PassPA = Pass->run(*C, CGAM, CG, UR); + } + + if (UR.InvalidatedSCCs.count(C)) + PI.runAfterPassInvalidated(*Pass, PassPA); + else + PI.runAfterPass(*Pass, *C, PassPA); + + // Update the SCC and RefSCC if necessary. + C = UR.UpdatedC ? UR.UpdatedC : C; + RC = UR.UpdatedRC ? UR.UpdatedRC : RC; + + if (UR.UpdatedC) { + // If we're updating the SCC, also update the FAM inside the proxy's + // result. + CGAM.getResult(*C, CG).updateFAM( + FAM); + } + + // If the CGSCC pass wasn't able to provide a valid updated SCC, + // the current SCC may simply need to be skipped if invalid. + if (UR.InvalidatedSCCs.count(C)) { + LLVM_DEBUG(dbgs() << "Skipping invalidated root or island SCC!\n"); + break; + } + // Check that we didn't miss any update scenario. + assert(C->begin() != C->end() && "Cannot have an empty SCC!"); + + // We handle invalidating the CGSCC analysis manager's information + // for the (potentially updated) SCC here. Note that any other SCCs + // whose structure has changed should have been invalidated by + // whatever was updating the call graph. This SCC gets invalidated + // late as it contains the nodes that were actively being + // processed. + CGAM.invalidate(*C, PassPA); + + // Then intersect the preserved set so that invalidation of module + // analyses will eventually occur when the module pass completes. + // Also intersect with the cross-SCC preserved set to capture any + // cross-SCC invalidation. + UR.CrossSCCPA.intersect(PassPA); + PA.intersect(std::move(PassPA)); + + // The pass may have restructured the call graph and refined the + // current SCC and/or RefSCC. We need to update our current SCC and + // RefSCC pointers to follow these. Also, when the current SCC is + // refined, re-run the SCC pass over the newly refined SCC in order + // to observe the most precise SCC model available. This inherently + // cannot cycle excessively as it only happens when we split SCCs + // apart, at most converging on a DAG of single nodes. + // FIXME: If we ever start having RefSCC passes, we'll want to + // iterate there too. + if (UR.UpdatedC) + LLVM_DEBUG(dbgs() + << "Re-running SCC passes after a refinement of the " + "current SCC: " + << *UR.UpdatedC << "\n"); + + // Note that both `C` and `RC` may at this point refer to deleted, + // invalid SCC and RefSCCs respectively. But we will short circuit + // the processing when we check them in the loop above. + } while (UR.UpdatedC); + } while (!CWorklist.empty()); + + // We only need to keep internal inlined edge information within + // a RefSCC, clear it to save on space and let the next time we visit + // any of these functions have a fresh start. + InlinedInternalEdges.clear(); + } while (!RCWorklist.empty()); + } + + // By definition we preserve the call garph, all SCC analyses, and the + // analysis proxies by handling them above and in any nested pass managers. + PA.preserveSet>(); + PA.preserve(); + PA.preserve(); + PA.preserve(); + return PA; +} + +PreservedAnalyses DevirtSCCRepeatedPass::run(LazyCallGraph::SCC &InitialC, + CGSCCAnalysisManager &AM, + LazyCallGraph &CG, + CGSCCUpdateResult &UR) { + PreservedAnalyses PA = PreservedAnalyses::all(); + PassInstrumentation PI = + AM.getResult(InitialC, CG); + + // The SCC may be refined while we are running passes over it, so set up + // a pointer that we can update. + LazyCallGraph::SCC *C = &InitialC; + + // Struct to track the counts of direct and indirect calls in each function + // of the SCC. + struct CallCount { + int Direct; + int Indirect; + }; + + // Put value handles on all of the indirect calls and return the number of + // direct calls for each function in the SCC. + auto ScanSCC = [](LazyCallGraph::SCC &C, + SmallMapVector &CallHandles) { + assert(CallHandles.empty() && "Must start with a clear set of handles."); + + SmallDenseMap CallCounts; + CallCount CountLocal = {0, 0}; + for (LazyCallGraph::Node &N : C) { + CallCount &Count = + CallCounts.insert(std::make_pair(&N.getFunction(), CountLocal)) + .first->second; + for (Instruction &I : instructions(N.getFunction())) + if (auto *CB = dyn_cast(&I)) { + if (CB->getCalledFunction()) { + ++Count.Direct; + } else { + ++Count.Indirect; + CallHandles.insert({CB, WeakTrackingVH(CB)}); + } + } + } + + return CallCounts; + }; + + UR.IndirectVHs.clear(); + // Populate the initial call handles and get the initial call counts. + auto CallCounts = ScanSCC(*C, UR.IndirectVHs); + + for (int Iteration = 0;; ++Iteration) { + if (!PI.runBeforePass(*Pass, *C)) + continue; + + PreservedAnalyses PassPA = Pass->run(*C, AM, CG, UR); + + if (UR.InvalidatedSCCs.count(C)) + PI.runAfterPassInvalidated(*Pass, PassPA); + else + PI.runAfterPass(*Pass, *C, PassPA); + + // If the SCC structure has changed, bail immediately and let the outer + // CGSCC layer handle any iteration to reflect the refined structure. + if (UR.UpdatedC && UR.UpdatedC != C) { + PA.intersect(std::move(PassPA)); + break; + } + + // Check that we didn't miss any update scenario. + assert(!UR.InvalidatedSCCs.count(C) && "Processing an invalid SCC!"); + assert(C->begin() != C->end() && "Cannot have an empty SCC!"); + + // Check whether any of the handles were devirtualized. + bool Devirt = llvm::any_of(UR.IndirectVHs, [](auto &P) -> bool { + if (P.second) { + if (CallBase *CB = dyn_cast(P.second)) { + if (CB->getCalledFunction()) { + LLVM_DEBUG(dbgs() << "Found devirtualized call: " << *CB << "\n"); + return true; + } + } + } + return false; + }); + + // Rescan to build up a new set of handles and count how many direct + // calls remain. If we decide to iterate, this also sets up the input to + // the next iteration. + UR.IndirectVHs.clear(); + auto NewCallCounts = ScanSCC(*C, UR.IndirectVHs); + + // If we haven't found an explicit devirtualization already see if we + // have decreased the number of indirect calls and increased the number + // of direct calls for any function in the SCC. This can be fooled by all + // manner of transformations such as DCE and other things, but seems to + // work well in practice. + if (!Devirt) + // Iterate over the keys in NewCallCounts, if Function also exists in + // CallCounts, make the check below. + for (auto &Pair : NewCallCounts) { + auto &CallCountNew = Pair.second; + auto CountIt = CallCounts.find(Pair.first); + if (CountIt != CallCounts.end()) { + const auto &CallCountOld = CountIt->second; + if (CallCountOld.Indirect > CallCountNew.Indirect && + CallCountOld.Direct < CallCountNew.Direct) { + Devirt = true; + break; + } + } + } + + if (!Devirt) { + PA.intersect(std::move(PassPA)); + break; + } + + // Otherwise, if we've already hit our max, we're done. + if (Iteration >= MaxIterations) { + maxDevirtIterationsReached(); + LLVM_DEBUG( + dbgs() << "Found another devirtualization after hitting the max " + "number of repetitions (" + << MaxIterations << ") on SCC: " << *C << "\n"); + PA.intersect(std::move(PassPA)); + break; + } + + LLVM_DEBUG( + dbgs() << "Repeating an SCC pass after finding a devirtualization in: " + << *C << "\n"); + + // Move over the new call counts in preparation for iterating. + CallCounts = std::move(NewCallCounts); + + // Update the analysis manager with each run and intersect the total set + // of preserved analyses so we're ready to iterate. + AM.invalidate(*C, PassPA); + + PA.intersect(std::move(PassPA)); + } + + // Note that we don't add any preserved entries here unlike a more normal + // "pass manager" because we only handle invalidation *between* iterations, + // not after the last iteration. + return PA; +} + +PreservedAnalyses CGSCCToFunctionPassAdaptor::run(LazyCallGraph::SCC &C, + CGSCCAnalysisManager &AM, + LazyCallGraph &CG, + CGSCCUpdateResult &UR) { + // Setup the function analysis manager from its proxy. + FunctionAnalysisManager &FAM = + AM.getResult(C, CG).getManager(); + + SmallVector Nodes; + for (LazyCallGraph::Node &N : C) + Nodes.push_back(&N); + + // The SCC may get split while we are optimizing functions due to deleting + // edges. If this happens, the current SCC can shift, so keep track of + // a pointer we can overwrite. + LazyCallGraph::SCC *CurrentC = &C; + + LLVM_DEBUG(dbgs() << "Running function passes across an SCC: " << C << "\n"); + + PreservedAnalyses PA = PreservedAnalyses::all(); + for (LazyCallGraph::Node *N : Nodes) { + // Skip nodes from other SCCs. These may have been split out during + // processing. We'll eventually visit those SCCs and pick up the nodes + // there. + if (CG.lookupSCC(*N) != CurrentC) + continue; + + Function &F = N->getFunction(); + + PassInstrumentation PI = FAM.getResult(F); + if (!PI.runBeforePass(*Pass, F)) + continue; + + PreservedAnalyses PassPA; + { + TimeTraceScope TimeScope(Pass->name()); + PassPA = Pass->run(F, FAM); + } + + PI.runAfterPass(*Pass, F, PassPA); + + // We know that the function pass couldn't have invalidated any other + // function's analyses (that's the contract of a function pass), so + // directly handle the function analysis manager's invalidation here. + FAM.invalidate(F, PassPA); + + // Then intersect the preserved set so that invalidation of module + // analyses will eventually occur when the module pass completes. + PA.intersect(std::move(PassPA)); + + // If the call graph hasn't been preserved, update it based on this + // function pass. This may also update the current SCC to point to + // a smaller, more refined SCC. + auto PAC = PA.getChecker(); + if (!PAC.preserved() && !PAC.preservedSet>()) { + CurrentC = &updateCGAndAnalysisManagerForFunctionPass(CG, *CurrentC, *N, + AM, UR, FAM); + assert(CG.lookupSCC(*N) == CurrentC && + "Current SCC not updated to the SCC containing the current node!"); + } + } + + // By definition we preserve the proxy. And we preserve all analyses on + // Functions. This precludes *any* invalidation of function analyses by the + // proxy, but that's OK because we've taken care to invalidate analyses in + // the function analysis manager incrementally above. + PA.preserveSet>(); + PA.preserve(); + + // We've also ensured that we updated the call graph along the way. + PA.preserve(); + + return PA; +} + bool CGSCCAnalysisManagerModuleProxy::Result::invalidate( Module &M, const PreservedAnalyses &PA, ModuleAnalysisManager::Invalidator &Inv) { diff --git a/llvm/lib/IR/PassManager.cpp b/llvm/lib/IR/PassManager.cpp --- a/llvm/lib/IR/PassManager.cpp +++ b/llvm/lib/IR/PassManager.cpp @@ -91,6 +91,54 @@ } } // namespace llvm +PreservedAnalyses ModuleToFunctionPassAdaptor::run(Module &M, + ModuleAnalysisManager &AM) { + FunctionAnalysisManager &FAM = + AM.getResult(M).getManager(); + + // Request PassInstrumentation from analysis manager, will use it to run + // instrumenting callbacks for the passes later. + PassInstrumentation PI = AM.getResult(M); + + PreservedAnalyses PA = PreservedAnalyses::all(); + for (Function &F : M) { + if (F.isDeclaration()) + continue; + + // Check the PassInstrumentation's BeforePass callbacks before running the + // pass, skip its execution completely if asked to (callback returns + // false). + if (!PI.runBeforePass(*Pass, F)) + continue; + + PreservedAnalyses PassPA; + { + TimeTraceScope TimeScope(Pass->name(), F.getName()); + PassPA = Pass->run(F, FAM); + } + + PI.runAfterPass(*Pass, F, PassPA); + + // We know that the function pass couldn't have invalidated any other + // function's analyses (that's the contract of a function pass), so + // directly handle the function analysis manager's invalidation here. + FAM.invalidate(F, PassPA); + + // Then intersect the preserved set so that invalidation of module + // analyses will eventually occur when the module pass completes. + PA.intersect(std::move(PassPA)); + } + + // The FunctionAnalysisManagerModuleProxy is preserved because (we assume) + // the function passes we ran didn't add or remove any functions. + // + // We also preserve all analyses on Functions, because we did all the + // invalidation we needed to do above. + PA.preserveSet>(); + PA.preserve(); + return PA; +} + AnalysisSetKey CFGAnalyses::SetKey; AnalysisSetKey PreservedAnalyses::AllAnalysesKey; diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp --- a/llvm/lib/Passes/StandardInstrumentations.cpp +++ b/llvm/lib/Passes/StandardInstrumentations.cpp @@ -473,7 +473,7 @@ } void PrintIRInstrumentation::printBeforePass(StringRef PassID, Any IR) { - if (PassID.startswith("PassManager<") || PassID.contains("PassAdaptor<")) + if (isIgnored(PassID)) return; // Saving Module for AfterPassInvalidated operations. @@ -492,7 +492,7 @@ } void PrintIRInstrumentation::printAfterPass(StringRef PassID, Any IR) { - if (PassID.startswith("PassManager<") || PassID.contains("PassAdaptor<")) + if (isIgnored(PassID)) return; if (!llvm::shouldPrintAfterPass(PassID)) @@ -509,7 +509,7 @@ if (!StoreModuleDesc || !llvm::shouldPrintAfterPass(PassID)) return; - if (PassID.startswith("PassManager<") || PassID.contains("PassAdaptor<")) + if (isIgnored(PassID)) return; const Module *M; @@ -599,13 +599,9 @@ if (!isEnabled()) return; - std::vector SpecialPasses = {"PassManager", "PassAdaptor"}; - - PIC.registerShouldRunOptionalPassCallback( - [this, SpecialPasses](StringRef PassID, Any IR) { - return isSpecialPass(PassID, SpecialPasses) || - checkPass(PassID, getBisectDescription(IR)); - }); + PIC.registerShouldRunOptionalPassCallback([this](StringRef PassID, Any IR) { + return isIgnored(PassID) || checkPass(PassID, getBisectDescription(IR)); + }); } void PrintPassInstrumentation::registerCallbacks( diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp --- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp +++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp @@ -86,6 +86,152 @@ return PA; } +} // namespace llvm + +PreservedAnalyses FunctionToLoopPassAdaptor::run(Function &F, + FunctionAnalysisManager &AM) { + // Before we even compute any loop analyses, first run a miniature function + // pass pipeline to put loops into their canonical form. Note that we can + // directly build up function analyses after this as the function pass + // manager handles all the invalidation at that layer. + PassInstrumentation PI = AM.getResult(F); + + PreservedAnalyses PA = PreservedAnalyses::all(); + // Check the PassInstrumentation's BeforePass callbacks before running the + // canonicalization pipeline. + if (PI.runBeforePass(LoopCanonicalizationFPM, F)) { + PA = LoopCanonicalizationFPM.run(F, AM); + PI.runAfterPass(LoopCanonicalizationFPM, F, PA); + } + + // Get the loop structure for this function + LoopInfo &LI = AM.getResult(F); + + // If there are no loops, there is nothing to do here. + if (LI.empty()) + return PA; + + // Get the analysis results needed by loop passes. + MemorySSA *MSSA = + UseMemorySSA ? (&AM.getResult(F).getMSSA()) : nullptr; + BlockFrequencyInfo *BFI = UseBlockFrequencyInfo && F.hasProfileData() + ? (&AM.getResult(F)) + : nullptr; + LoopStandardAnalysisResults LAR = {AM.getResult(F), + AM.getResult(F), + AM.getResult(F), + AM.getResult(F), + AM.getResult(F), + AM.getResult(F), + AM.getResult(F), + BFI, + MSSA}; + + // Setup the loop analysis manager from its proxy. It is important that + // this is only done when there are loops to process and we have built the + // LoopStandardAnalysisResults object. The loop analyses cached in this + // manager have access to those analysis results and so it must invalidate + // itself when they go away. + auto &LAMFP = AM.getResult(F); + if (UseMemorySSA) + LAMFP.markMSSAUsed(); + LoopAnalysisManager &LAM = LAMFP.getManager(); + + // A postorder worklist of loops to process. + SmallPriorityWorklist Worklist; + + // Register the worklist and loop analysis manager so that loop passes can + // update them when they mutate the loop nest structure. + LPMUpdater Updater(Worklist, LAM); + + // Add the loop nests in the reverse order of LoopInfo. See method + // declaration. + appendLoopsToWorklist(LI, Worklist); + +#ifndef NDEBUG + PI.pushBeforeNonSkippedPassCallback([&LAR, &LI](StringRef PassID, Any IR) { + if (isSpecialPass(PassID, {"PassManager"})) + return; + assert(any_isa(IR)); + const Loop *L = any_cast(IR); + assert(L && "Loop should be valid for printing"); + + // Verify the loop structure and LCSSA form before visiting the loop. + L->verifyLoop(); + assert(L->isRecursivelyLCSSAForm(LAR.DT, LI) && + "Loops must remain in LCSSA form!"); + }); +#endif + + do { + Loop *L = Worklist.pop_back_val(); + + // Reset the update structure for this loop. + Updater.CurrentL = L; + Updater.SkipCurrentLoop = false; + +#ifndef NDEBUG + // Save a parent loop pointer for asserts. + Updater.ParentL = L->getParentLoop(); +#endif + // Check the PassInstrumentation's BeforePass callbacks before running the + // pass, skip its execution completely if asked to (callback returns + // false). + if (!PI.runBeforePass(*Pass, *L)) + continue; + + PreservedAnalyses PassPA; + { + TimeTraceScope TimeScope(Pass->name()); + PassPA = Pass->run(*L, LAM, LAR, Updater); + } + + // Do not pass deleted Loop into the instrumentation. + if (Updater.skipCurrentLoop()) + PI.runAfterPassInvalidated(*Pass, PassPA); + else + PI.runAfterPass(*Pass, *L, PassPA); + + // FIXME: We should verify the set of analyses relevant to Loop passes + // are preserved. + + // If the loop hasn't been deleted, we need to handle invalidation here. + if (!Updater.skipCurrentLoop()) + // We know that the loop pass couldn't have invalidated any other + // loop's analyses (that's the contract of a loop pass), so directly + // handle the loop analysis manager's invalidation here. + LAM.invalidate(*L, PassPA); + + // Then intersect the preserved set so that invalidation of module + // analyses will eventually occur when the module pass completes. + PA.intersect(std::move(PassPA)); + } while (!Worklist.empty()); + +#ifndef NDEBUG + PI.popBeforeNonSkippedPassCallback(); +#endif + + // By definition we preserve the proxy. We also preserve all analyses on + // Loops. This precludes *any* invalidation of loop analyses by the proxy, + // but that's OK because we've taken care to invalidate analyses in the + // loop analysis manager incrementally above. + PA.preserveSet>(); + PA.preserve(); + // We also preserve the set of standard analyses. + PA.preserve(); + PA.preserve(); + PA.preserve(); + if (UseBlockFrequencyInfo && F.hasProfileData()) + PA.preserve(); + if (UseMemorySSA) + PA.preserve(); + // FIXME: What we really want to do here is preserve an AA category, but + // that concept doesn't exist yet. + PA.preserve(); + PA.preserve(); + PA.preserve(); + PA.preserve(); + return PA; } PrintLoopPass::PrintLoopPass() : OS(dbgs()) {} diff --git a/llvm/test/Other/change-printer.ll b/llvm/test/Other/change-printer.ll --- a/llvm/test/Other/change-printer.ll +++ b/llvm/test/Other/change-printer.ll @@ -66,7 +66,7 @@ ; CHECK-SIMPLE: *** IR Dump After InstSimplifyPass *** (function: f) ; CHECK-SIMPLE-NEXT: define i32 @f() ; CHECK-SIMPLE: *** IR Pass PassManager{{.*}} (function: f) ignored *** -; CHECK-SIMPLE: *** IR Pass ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}> (module) ignored *** +; CHECK-SIMPLE: *** IR Pass ModuleToFunctionPassAdaptor (module) ignored *** ; CHECK-SIMPLE: *** IR Dump After VerifierPass (module) omitted because no change *** ; CHECK-SIMPLE: *** IR Dump After PrintModulePass (module) omitted because no change *** ; CHECK-SIMPLE-NOT: *** IR @@ -150,7 +150,7 @@ ; CHECK-SIMPLE-BEFORE: *** IR Dump After InstSimplifyPass *** (function: f) ; CHECK-SIMPLE-BEFORE-NEXT: define i32 @f() ; CHECK-SIMPLE-BEFORE: *** IR Pass PassManager{{.*}} (function: f) ignored *** -; CHECK-SIMPLE-BEFORE: *** IR Pass ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}> (module) ignored *** +; CHECK-SIMPLE-BEFORE: *** IR Pass ModuleToFunctionPassAdaptor (module) ignored *** ; CHECK-SIMPLE-BEFORE: *** IR Dump After VerifierPass (module) omitted because no change *** ; CHECK-SIMPLE-BEFORE: *** IR Dump After PrintModulePass (module) omitted because no change *** ; CHECK-SIMPLE-BEFORE-NOT: *** IR diff --git a/llvm/test/Other/pass-pipeline-parsing.ll b/llvm/test/Other/pass-pipeline-parsing.ll --- a/llvm/test/Other/pass-pipeline-parsing.ll +++ b/llvm/test/Other/pass-pipeline-parsing.ll @@ -161,24 +161,24 @@ ; RUN: -passes='module(no-op-function,no-op-loop,no-op-cgscc,cgscc(no-op-function,no-op-loop),function(no-op-loop))' %s 2>&1 \ ; RUN: | FileCheck %s --check-prefix=CHECK-ADAPTORS ; CHECK-ADAPTORS: Starting llvm::Module pass manager run -; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor<{{.*}}NoOpFunctionPass> +; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor ; CHECK-ADAPTORS: Running pass: NoOpFunctionPass -; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor<{{.*}}FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass>{{.*}}> -; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass> +; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor +; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor ; CHECK-ADAPTORS: Running pass: NoOpLoopPass on Loop at depth 1 containing: %loop -; CHECK-ADAPTORS: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}NoOpCGSCCPass> +; CHECK-ADAPTORS: Running pass: ModuleToPostOrderCGSCCPassAdaptor ; CHECK-ADAPTORS: Running pass: NoOpCGSCCPass -; CHECK-ADAPTORS: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}LazyCallGraph{{.*}}> +; CHECK-ADAPTORS: Running pass: ModuleToPostOrderCGSCCPassAdaptor ; CHECK-ADAPTORS: Starting CGSCC pass manager run -; CHECK-ADAPTORS: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}NoOpFunctionPass> +; CHECK-ADAPTORS: Running pass: CGSCCToFunctionPassAdaptor ; CHECK-ADAPTORS: Running pass: NoOpFunctionPass -; CHECK-ADAPTORS: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass>{{.*}}> -; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass> +; CHECK-ADAPTORS: Running pass: CGSCCToFunctionPassAdaptor +; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor ; CHECK-ADAPTORS: Running pass: NoOpLoopPass on Loop at depth 1 containing: %loop ; CHECK-ADAPTORS: Finished CGSCC pass manager run -; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}> +; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor ; CHECK-ADAPTORS: Starting llvm::Function pass manager run -; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass> +; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor ; CHECK-ADAPTORS: Running pass: NoOpLoopPass on Loop at depth 1 containing: %loop ; CHECK-ADAPTORS: Finished llvm::Function pass manager run ; CHECK-ADAPTORS: Finished llvm::Module pass manager run