diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h --- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -802,10 +802,11 @@ void getAnalysisUsage(AnalysisUsage &AU) const override; - /// Query the result of the loop access information for the loop \p L. + /// Return the proxy object for retrieving LoopAccessInfo for individual + /// loops. /// /// If there is no cached result available run the analysis. - const LoopAccessInfo &getInfo(Loop *L) { return LAIs->getInfo(*L); } + LoopAccessInfoManager &getLAIs() { return *LAIs; } void releaseMemory() override { // Invalidate the cache when the pass is freed. diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h --- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h +++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h @@ -240,16 +240,18 @@ /// induction variable and the different reduction variables. class LoopVectorizationLegality { public: - LoopVectorizationLegality( - Loop *L, PredicatedScalarEvolution &PSE, DominatorTree *DT, - TargetTransformInfo *TTI, TargetLibraryInfo *TLI, AAResults *AA, - Function *F, std::function *GetLAA, - LoopInfo *LI, OptimizationRemarkEmitter *ORE, - LoopVectorizationRequirements *R, LoopVectorizeHints *H, DemandedBits *DB, - AssumptionCache *AC, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI) - : TheLoop(L), LI(LI), PSE(PSE), TTI(TTI), TLI(TLI), DT(DT), - GetLAA(GetLAA), ORE(ORE), Requirements(R), Hints(H), DB(DB), AC(AC), - BFI(BFI), PSI(PSI) {} + LoopVectorizationLegality(Loop *L, PredicatedScalarEvolution &PSE, + DominatorTree *DT, TargetTransformInfo *TTI, + TargetLibraryInfo *TLI, AAResults *AA, Function *F, + LoopAccessInfoManager &LAIs, LoopInfo *LI, + OptimizationRemarkEmitter *ORE, + LoopVectorizationRequirements *R, + LoopVectorizeHints *H, DemandedBits *DB, + AssumptionCache *AC, BlockFrequencyInfo *BFI, + ProfileSummaryInfo *PSI) + : TheLoop(L), LI(LI), PSE(PSE), TTI(TTI), TLI(TLI), DT(DT), LAIs(LAIs), + ORE(ORE), Requirements(R), Hints(H), DB(DB), AC(AC), BFI(BFI), + PSI(PSI) {} /// ReductionList contains the reduction descriptors for all /// of the reductions that were found in the loop. @@ -486,10 +488,8 @@ DominatorTree *DT; // LoopAccess analysis. - std::function *GetLAA; + LoopAccessInfoManager &LAIs; - // And the loop-accesses info corresponding to this loop. This pointer is - // null until canVectorizeMemory sets it up. const LoopAccessInfo *LAI = nullptr; /// Interface to emit optimization remarks. diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h --- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h +++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h @@ -69,7 +69,7 @@ class DominatorTree; class Function; class Loop; -class LoopAccessInfo; +class LoopAccessInfoManager; class LoopInfo; class OptimizationRemarkEmitter; class ProfileSummaryInfo; @@ -180,7 +180,7 @@ DemandedBits *DB; AAResults *AA; AssumptionCache *AC; - std::function *GetLAA; + LoopAccessInfoManager *LAIs; OptimizationRemarkEmitter *ORE; ProfileSummaryInfo *PSI; @@ -193,8 +193,7 @@ runImpl(Function &F, ScalarEvolution &SE_, LoopInfo &LI_, TargetTransformInfo &TTI_, DominatorTree &DT_, BlockFrequencyInfo &BFI_, TargetLibraryInfo *TLI_, DemandedBits &DB_, - AAResults &AA_, AssumptionCache &AC_, - std::function &GetLAA_, + AAResults &AA_, AssumptionCache &AC_, LoopAccessInfoManager &LAIs_, OptimizationRemarkEmitter &ORE_, ProfileSummaryInfo *PSI_); bool processLoop(Loop *L); diff --git a/llvm/lib/Transforms/Scalar/LoopDistribute.cpp b/llvm/lib/Transforms/Scalar/LoopDistribute.cpp --- a/llvm/lib/Transforms/Scalar/LoopDistribute.cpp +++ b/llvm/lib/Transforms/Scalar/LoopDistribute.cpp @@ -653,13 +653,14 @@ class LoopDistributeForLoop { public: LoopDistributeForLoop(Loop *L, Function *F, LoopInfo *LI, DominatorTree *DT, - ScalarEvolution *SE, OptimizationRemarkEmitter *ORE) - : L(L), F(F), LI(LI), DT(DT), SE(SE), ORE(ORE) { + ScalarEvolution *SE, LoopAccessInfoManager &LAIs, + OptimizationRemarkEmitter *ORE) + : L(L), F(F), LI(LI), DT(DT), SE(SE), LAIs(LAIs), ORE(ORE) { setForced(); } /// Try to distribute an inner-most loop. - bool processLoop(std::function &GetLAA) { + bool processLoop() { assert(L->isInnermost() && "Only process inner loops."); LLVM_DEBUG(dbgs() << "\nLDist: In \"" @@ -677,7 +678,7 @@ BasicBlock *PH = L->getLoopPreheader(); - LAI = &GetLAA(*L); + LAI = &LAIs.getInfo(*L); // Currently, we only distribute to isolate the part of the loop with // dependence cycles to enable partial vectorization. @@ -953,6 +954,7 @@ const LoopAccessInfo *LAI = nullptr; DominatorTree *DT; ScalarEvolution *SE; + LoopAccessInfoManager &LAIs; OptimizationRemarkEmitter *ORE; /// Indicates whether distribution is forced to be enabled/disabled for @@ -969,7 +971,7 @@ /// Shared implementation between new and old PMs. static bool runImpl(Function &F, LoopInfo *LI, DominatorTree *DT, ScalarEvolution *SE, OptimizationRemarkEmitter *ORE, - std::function &GetLAA) { + LoopAccessInfoManager &LAIs) { // Build up a worklist of inner-loops to vectorize. This is necessary as the // act of distributing a loop creates new loops and can invalidate iterators // across the loops. @@ -984,12 +986,12 @@ // Now walk the identified inner loops. bool Changed = false; for (Loop *L : Worklist) { - LoopDistributeForLoop LDL(L, &F, LI, DT, SE, ORE); + LoopDistributeForLoop LDL(L, &F, LI, DT, SE, LAIs, ORE); // If distribution was forced for the specific loop to be // enabled/disabled, follow that. Otherwise use the global flag. if (LDL.isForced().value_or(EnableLoopDistribute)) - Changed |= LDL.processLoop(GetLAA); + Changed |= LDL.processLoop(); } // Process each loop nest in the function. @@ -1013,14 +1015,12 @@ return false; auto *LI = &getAnalysis().getLoopInfo(); - auto *LAA = &getAnalysis(); auto *DT = &getAnalysis().getDomTree(); auto *SE = &getAnalysis().getSE(); auto *ORE = &getAnalysis().getORE(); - std::function GetLAA = - [&](Loop &L) -> const LoopAccessInfo & { return LAA->getInfo(&L); }; + auto &LAIs = getAnalysis().getLAIs(); - return runImpl(F, LI, DT, SE, ORE, GetLAA); + return runImpl(F, LI, DT, SE, ORE, LAIs); } void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -1045,10 +1045,7 @@ auto &ORE = AM.getResult(F); LoopAccessInfoManager &LAIs = AM.getResult(F); - std::function GetLAA = - [&](Loop &L) -> const LoopAccessInfo & { return LAIs.getInfo(L); }; - - bool Changed = runImpl(F, &LI, &DT, &SE, &ORE, GetLAA); + bool Changed = runImpl(F, &LI, &DT, &SE, &ORE, LAIs); if (!Changed) return PreservedAnalyses::all(); PreservedAnalyses PA; diff --git a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp --- a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp +++ b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp @@ -621,11 +621,12 @@ } // end anonymous namespace -static bool -eliminateLoadsAcrossLoops(Function &F, LoopInfo &LI, DominatorTree &DT, - BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, - ScalarEvolution *SE, AssumptionCache *AC, - function_ref GetLAI) { +static bool eliminateLoadsAcrossLoops(Function &F, LoopInfo &LI, + DominatorTree &DT, + BlockFrequencyInfo *BFI, + ProfileSummaryInfo *PSI, + ScalarEvolution *SE, AssumptionCache *AC, + LoopAccessInfoManager &LAIs) { // Build up a worklist of inner-loops to transform to avoid iterator // invalidation. // FIXME: This logic comes from other passes that actually change the loop @@ -649,7 +650,7 @@ if (!L->isRotatedForm() || !L->getExitingBlock()) continue; // The actual work is performed by LoadEliminationForLoop. - LoadEliminationForLoop LEL(L, &LI, GetLAI(*L), &DT, BFI, PSI); + LoadEliminationForLoop LEL(L, &LI, LAIs.getInfo(*L), &DT, BFI, PSI); Changed |= LEL.processLoop(); } return Changed; @@ -672,7 +673,7 @@ return false; auto &LI = getAnalysis().getLoopInfo(); - auto &LAA = getAnalysis(); + auto &LAIs = getAnalysis().getLAIs(); auto &DT = getAnalysis().getDomTree(); auto *PSI = &getAnalysis().getPSI(); auto *BFI = (PSI && PSI->hasProfileSummary()) ? @@ -681,9 +682,8 @@ auto *SE = &getAnalysis().getSE(); // Process each loop nest in the function. - return eliminateLoadsAcrossLoops( - F, LI, DT, BFI, PSI, SE, /*AC*/ nullptr, - [&LAA](Loop &L) -> const LoopAccessInfo & { return LAA.getInfo(&L); }); + return eliminateLoadsAcrossLoops(F, LI, DT, BFI, PSI, SE, /*AC*/ nullptr, + LAIs); } void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -736,9 +736,7 @@ &AM.getResult(F) : nullptr; LoopAccessInfoManager &LAIs = AM.getResult(F); - bool Changed = eliminateLoadsAcrossLoops( - F, LI, DT, BFI, PSI, &SE, &AC, - [&](Loop &L) -> const LoopAccessInfo & { return LAIs.getInfo(L); }); + bool Changed = eliminateLoadsAcrossLoops(F, LI, DT, BFI, PSI, &SE, &AC, LAIs); if (!Changed) return PreservedAnalyses::all(); diff --git a/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp b/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp --- a/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp +++ b/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp @@ -147,9 +147,8 @@ // LoopAccessInfo will take place only when it's necessary. LoopVersioningLICM(AliasAnalysis *AA, ScalarEvolution *SE, OptimizationRemarkEmitter *ORE, - function_ref GetLAI) - : AA(AA), SE(SE), GetLAI(GetLAI), - LoopDepthThreshold(LVLoopDepthThreshold), + LoopAccessInfoManager &LAIs) + : AA(AA), SE(SE), LAIs(LAIs), LoopDepthThreshold(LVLoopDepthThreshold), InvariantThreshold(LVInvarThreshold), ORE(ORE) {} bool runOnLoop(Loop *L, LoopInfo *LI, DominatorTree *DT); @@ -185,7 +184,7 @@ const LoopAccessInfo *LAI = nullptr; // Proxy for retrieving LoopAccessInfo. - function_ref GetLAI; + LoopAccessInfoManager &LAIs; // The current loop we are working on. Loop *CurLoop = nullptr; @@ -413,7 +412,7 @@ } } // Get LoopAccessInfo from current loop via the proxy. - LAI = &GetLAI(CurLoop); + LAI = &LAIs.getInfo(*CurLoop); // Check LoopAccessInfo for need of runtime check. if (LAI->getRuntimePointerChecking()->getChecks().empty()) { LLVM_DEBUG(dbgs() << " LAA: Runtime check not found !!\n"); @@ -584,12 +583,9 @@ &getAnalysis().getORE(); LoopInfo *LI = &getAnalysis().getLoopInfo(); DominatorTree *DT = &getAnalysis().getDomTree(); + auto &LAIs = getAnalysis().getLAIs(); - auto GetLAI = [&](Loop *L) -> const LoopAccessInfo & { - return getAnalysis().getInfo(L); - }; - - return LoopVersioningLICM(AA, SE, ORE, GetLAI).runOnLoop(L, LI, DT); + return LoopVersioningLICM(AA, SE, ORE, LAIs).runOnLoop(L, LI, DT); } bool LoopVersioningLICM::runOnLoop(Loop *L, LoopInfo *LI, DominatorTree *DT) { @@ -672,9 +668,7 @@ OptimizationRemarkEmitter ORE(F); LoopAccessInfoManager LAIs(*SE, *AA, *DT, *LI, nullptr); - std::function GetLAI = - [&](Loop *L) -> const LoopAccessInfo & { return LAIs.getInfo(*L); }; - if (!LoopVersioningLICM(AA, SE, &ORE, GetLAI).runOnLoop(&L, LI, DT)) + if (!LoopVersioningLICM(AA, SE, &ORE, LAIs).runOnLoop(&L, LI, DT)) return PreservedAnalyses::all(); return getLoopPassPreservedAnalyses(); } diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp --- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp +++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp @@ -256,8 +256,8 @@ } namespace { -bool runImpl(LoopInfo *LI, function_ref GetLAA, - DominatorTree *DT, ScalarEvolution *SE) { +bool runImpl(LoopInfo *LI, LoopAccessInfoManager &LAIs, DominatorTree *DT, + ScalarEvolution *SE) { // Build up a worklist of inner-loops to version. This is necessary as the // act of versioning a loop creates new loops and can invalidate iterators // across the loops. @@ -275,7 +275,7 @@ if (!L->isLoopSimplifyForm() || !L->isRotatedForm() || !L->getExitingBlock()) continue; - const LoopAccessInfo &LAI = GetLAA(*L); + const LoopAccessInfo &LAI = LAIs.getInfo(*L); if (!LAI.hasConvergentOp() && (LAI.getNumRuntimePointerChecks() || !LAI.getPSE().getPredicate().isAlwaysTrue())) { @@ -301,14 +301,11 @@ bool runOnFunction(Function &F) override { auto *LI = &getAnalysis().getLoopInfo(); - auto GetLAA = [&](Loop &L) -> const LoopAccessInfo & { - return getAnalysis().getInfo(&L); - }; - + auto &LAIs = getAnalysis().getLAIs(); auto *DT = &getAnalysis().getDomTree(); auto *SE = &getAnalysis().getSE(); - return runImpl(LI, GetLAA, DT, SE); + return runImpl(LI, LAIs, DT, SE); } void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -348,14 +345,10 @@ FunctionAnalysisManager &AM) { auto &SE = AM.getResult(F); auto &LI = AM.getResult(F); - auto &DT = AM.getResult(F); - LoopAccessInfoManager &LAIs = AM.getResult(F); - auto GetLAA = [&](Loop &L) -> const LoopAccessInfo & { - return LAIs.getInfo(L); - }; + auto &DT = AM.getResult(F); - if (runImpl(&LI, GetLAA, &DT, &SE)) + if (runImpl(&LI, LAIs, &DT, &SE)) return PreservedAnalyses::none(); return PreservedAnalyses::all(); } diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -916,7 +916,7 @@ } bool LoopVectorizationLegality::canVectorizeMemory() { - LAI = &(*GetLAA)(*TheLoop); + LAI = &LAIs.getInfo(*TheLoop); const OptimizationRemarkAnalysis *LAR = LAI->getReport(); if (LAR) { ORE->emit([&]() { diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2212,16 +2212,15 @@ auto *TLI = TLIP ? &TLIP->getTLI(F) : nullptr; auto *AA = &getAnalysis().getAAResults(); auto *AC = &getAnalysis().getAssumptionCache(F); - auto *LAA = &getAnalysis(); + auto &LAIs = getAnalysis().getLAIs(); auto *DB = &getAnalysis().getDemandedBits(); auto *ORE = &getAnalysis().getORE(); auto *PSI = &getAnalysis().getPSI(); - std::function GetLAA = - [&](Loop &L) -> const LoopAccessInfo & { return LAA->getInfo(&L); }; - - return Impl.runImpl(F, *SE, *LI, *TTI, *DT, *BFI, TLI, *DB, *AA, *AC, - GetLAA, *ORE, PSI).MadeAnyChange; + return Impl + .runImpl(F, *SE, *LI, *TTI, *DT, *BFI, TLI, *DB, *AA, *AC, LAIs, *ORE, + PSI) + .MadeAnyChange; } void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -10202,7 +10201,7 @@ // Check if it is legal to vectorize the loop. LoopVectorizationRequirements Requirements; - LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, AA, F, GetLAA, LI, ORE, + LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, AA, F, *LAIs, LI, ORE, &Requirements, &Hints, DB, AC, BFI, PSI); if (!LVL.canVectorize(EnableVPlanNativePath)) { LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n"); @@ -10563,8 +10562,8 @@ Function &F, ScalarEvolution &SE_, LoopInfo &LI_, TargetTransformInfo &TTI_, DominatorTree &DT_, BlockFrequencyInfo &BFI_, TargetLibraryInfo *TLI_, DemandedBits &DB_, AAResults &AA_, AssumptionCache &AC_, - std::function &GetLAA_, - OptimizationRemarkEmitter &ORE_, ProfileSummaryInfo *PSI_) { + LoopAccessInfoManager &LAIs_, OptimizationRemarkEmitter &ORE_, + ProfileSummaryInfo *PSI_) { SE = &SE_; LI = &LI_; TTI = &TTI_; @@ -10573,7 +10572,7 @@ TLI = TLI_; AA = &AA_; AC = &AC_; - GetLAA = &GetLAA_; + LAIs = &LAIs_; DB = &DB_; ORE = &ORE_; PSI = PSI_; @@ -10643,13 +10642,11 @@ auto &ORE = AM.getResult(F); LoopAccessInfoManager &LAIs = AM.getResult(F); - std::function GetLAA = - [&](Loop &L) -> const LoopAccessInfo & { return LAIs.getInfo(L); }; auto &MAMProxy = AM.getResult(F); ProfileSummaryInfo *PSI = MAMProxy.getCachedResult(*F.getParent()); LoopVectorizeResult Result = - runImpl(F, SE, LI, TTI, DT, BFI, &TLI, DB, AA, AC, GetLAA, ORE, PSI); + runImpl(F, SE, LI, TTI, DT, BFI, &TLI, DB, AA, AC, LAIs, ORE, PSI); if (!Result.MadeAnyChange) return PreservedAnalyses::all(); PreservedAnalyses PA;