Index: llvm/include/llvm/Transforms/Scalar/LoopPassManager.h =================================================================== --- llvm/include/llvm/Transforms/Scalar/LoopPassManager.h +++ llvm/include/llvm/Transforms/Scalar/LoopPassManager.h @@ -258,7 +258,7 @@ /// state, this routine will mark that the current loop should be skipped by /// the rest of the pass management infrastructure. void markLoopAsDeleted(Loop &L, llvm::StringRef Name) { - assert((!LoopNestMode || L.isOutermost()) && + assert((!LoopNestMode || CurrentL == &L) && "L should be a top-level loop in loop-nest mode."); LAM.clear(L, Name); assert((&L == CurrentL || CurrentL->contains(&L)) && Index: llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h =================================================================== --- llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h +++ llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h @@ -10,6 +10,7 @@ #define LLVM_TRANSFORMS_SCALAR_LOOPUNROLLANDJAMPASS_H #include "llvm/IR/PassManager.h" +#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { class Function; @@ -20,7 +21,8 @@ public: explicit LoopUnrollAndJamPass(int OptLevel = 2) : OptLevel(OptLevel) {} - PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + PreservedAnalyses run(LoopNest &L, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, LPMUpdater &U); }; } // end namespace llvm Index: llvm/lib/Passes/PassBuilder.cpp =================================================================== --- llvm/lib/Passes/PassBuilder.cpp +++ llvm/lib/Passes/PassBuilder.cpp @@ -1207,7 +1207,8 @@ // across the loop nests. // We do UnrollAndJam in a separate LPM to ensure it happens before unroll if (EnableUnrollAndJam && PTO.LoopUnrolling) - FPM.addPass(LoopUnrollAndJamPass(Level.getSpeedupLevel())); + FPM.addPass(createFunctionToLoopPassAdaptor( + LoopUnrollAndJamPass(Level.getSpeedupLevel()))); FPM.addPass(LoopUnrollPass(LoopUnrollOptions( Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling, PTO.ForgetAllSCEVInLoopUnroll))); @@ -1287,7 +1288,8 @@ // across the loop nests. // We do UnrollAndJam in a separate LPM to ensure it happens before unroll if (EnableUnrollAndJam && PTO.LoopUnrolling) { - FPM.addPass(LoopUnrollAndJamPass(Level.getSpeedupLevel())); + FPM.addPass(createFunctionToLoopPassAdaptor( + LoopUnrollAndJamPass(Level.getSpeedupLevel()))); } FPM.addPass(LoopUnrollPass(LoopUnrollOptions( Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling, Index: llvm/lib/Passes/PassRegistry.def =================================================================== --- llvm/lib/Passes/PassRegistry.def +++ llvm/lib/Passes/PassRegistry.def @@ -247,7 +247,6 @@ FUNCTION_PASS("load-store-vectorizer", LoadStoreVectorizerPass()) FUNCTION_PASS("loop-simplify", LoopSimplifyPass()) FUNCTION_PASS("loop-sink", LoopSinkPass()) -FUNCTION_PASS("loop-unroll-and-jam", LoopUnrollAndJamPass()) FUNCTION_PASS("lowerinvoke", LowerInvokePass()) FUNCTION_PASS("lowerswitch", LowerSwitchPass()) FUNCTION_PASS("mem2reg", PromotePass()) @@ -399,6 +398,7 @@ LOOP_PASS("loop-simplifycfg", LoopSimplifyCFGPass()) LOOP_PASS("loop-reduce", LoopStrengthReducePass()) LOOP_PASS("indvars", IndVarSimplifyPass()) +LOOP_PASS("loop-unroll-and-jam", LoopUnrollAndJamPass()) LOOP_PASS("loop-unroll-full", LoopFullUnrollPass()) LOOP_PASS("print-access-info", LoopAccessInfoPrinterPass(dbgs())) LOOP_PASS("print", DDGAnalysisPrinterPass(dbgs())) Index: llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp =================================================================== --- llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp +++ llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp @@ -22,6 +22,7 @@ #include "llvm/Analysis/DependenceAnalysis.h" #include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/LoopPass.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/TargetTransformInfo.h" @@ -41,6 +42,8 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Scalar.h" +#include "llvm/Transforms/Utils.h" +#include "llvm/Transforms/Utils/LCSSA.h" #include "llvm/Transforms/Utils/LoopPeel.h" #include "llvm/Transforms/Utils/LoopSimplify.h" #include "llvm/Transforms/Utils/LoopUtils.h" @@ -424,35 +427,29 @@ return UnrollResult; } -static bool tryToUnrollAndJamLoop(Function &F, DominatorTree &DT, LoopInfo &LI, +static bool tryToUnrollAndJamLoop(LoopNest &LN, DominatorTree &DT, LoopInfo &LI, ScalarEvolution &SE, const TargetTransformInfo &TTI, AssumptionCache &AC, DependenceInfo &DI, - OptimizationRemarkEmitter &ORE, - int OptLevel) { + OptimizationRemarkEmitter &ORE, int OptLevel, + LPMUpdater &U) { bool DidSomething = false; + ArrayRef Loops = LN.getLoops(); + Loop *OutmostLoop = &LN.getOutermostLoop(); - // The loop unroll and jam pass requires loops to be in simplified form, and - // also needs LCSSA. Since simplification may add new inner loops, it has to - // run before the legality and profitability checks. This means running the - // loop unroll and jam pass will simplify all loops, regardless of whether - // anything end up being unroll and jammed. - for (auto &L : LI) { - DidSomething |= - simplifyLoop(L, &DT, &LI, &SE, &AC, nullptr, false /* PreserveLCSSA */); - DidSomething |= formLCSSARecursively(*L, DT, &LI, &SE); - } - - // Add the loop nests in the reverse order of LoopInfo. See method + // Add the loop nests in the reverse order of LN. See method // declaration. SmallPriorityWorklist Worklist; - appendLoopsToWorklist(LI, Worklist); + appendLoopsToWorklist(Loops, Worklist); while (!Worklist.empty()) { Loop *L = Worklist.pop_back_val(); + std::string LoopName = std::string(L->getName()); LoopUnrollResult Result = tryToUnrollAndJamLoop(L, DT, &LI, SE, TTI, AC, DI, ORE, OptLevel); if (Result != LoopUnrollResult::Unmodified) DidSomething = true; + if (L == OutmostLoop && Result == LoopUnrollResult::FullyUnrolled) + U.markLoopAsDeleted(*L, LoopName); } return DidSomething; @@ -460,29 +457,35 @@ namespace { -class LoopUnrollAndJam : public FunctionPass { +class LoopUnrollAndJam : public LoopPass { public: static char ID; // Pass ID, replacement for typeid unsigned OptLevel; - LoopUnrollAndJam(int OptLevel = 2) : FunctionPass(ID), OptLevel(OptLevel) { + LoopUnrollAndJam(int OptLevel = 2) : LoopPass(ID), OptLevel(OptLevel) { initializeLoopUnrollAndJamPass(*PassRegistry::getPassRegistry()); } - bool runOnFunction(Function &F) override { - if (skipFunction(F)) + bool runOnLoop(Loop *L, LPPassManager &LPM) override { + if (skipLoop(L)) return false; - auto &DT = getAnalysis().getDomTree(); - LoopInfo &LI = getAnalysis().getLoopInfo(); - ScalarEvolution &SE = getAnalysis().getSE(); - const TargetTransformInfo &TTI = - getAnalysis().getTTI(F); - auto &AC = getAnalysis().getAssumptionCache(F); + auto *F = L->getHeader()->getParent(); + auto &SE = getAnalysis().getSE(); + auto *LI = &getAnalysis().getLoopInfo(); auto &DI = getAnalysis().getDI(); + auto &DT = getAnalysis().getDomTree(); + auto &TTI = getAnalysis().getTTI(*F); auto &ORE = getAnalysis().getORE(); + auto &AC = getAnalysis().getAssumptionCache(*F); - return tryToUnrollAndJamLoop(F, DT, LI, SE, TTI, AC, DI, ORE, OptLevel); + LoopUnrollResult Result = + tryToUnrollAndJamLoop(L, DT, LI, SE, TTI, AC, DI, ORE, OptLevel); + + if (Result == LoopUnrollResult::FullyUnrolled) + LPM.markLoopAsDeleted(*L); + + return Result != LoopUnrollResult::Unmodified; } /// This transformation requires natural loop information & requires that @@ -495,6 +498,7 @@ AU.addRequired(); AU.addRequired(); AU.addRequired(); + getLoopAnalysisUsage(AU); } }; @@ -505,7 +509,10 @@ INITIALIZE_PASS_BEGIN(LoopUnrollAndJam, "loop-unroll-and-jam", "Unroll and Jam loops", false, false) INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) +INITIALIZE_PASS_DEPENDENCY(LoopPass) INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(LoopSimplify) +INITIALIZE_PASS_DEPENDENCY(LCSSAWrapperPass) INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) @@ -518,19 +525,20 @@ return new LoopUnrollAndJam(OptLevel); } -PreservedAnalyses LoopUnrollAndJamPass::run(Function &F, - FunctionAnalysisManager &AM) { - ScalarEvolution &SE = AM.getResult(F); - LoopInfo &LI = AM.getResult(F); - TargetTransformInfo &TTI = AM.getResult(F); - AssumptionCache &AC = AM.getResult(F); - DominatorTree &DT = AM.getResult(F); - DependenceInfo &DI = AM.getResult(F); - OptimizationRemarkEmitter &ORE = - AM.getResult(F); - - if (!tryToUnrollAndJamLoop(F, DT, LI, SE, TTI, AC, DI, ORE, OptLevel)) +PreservedAnalyses LoopUnrollAndJamPass::run(LoopNest &LN, + LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, + LPMUpdater &U) { + Function &F = *LN.getParent(); + + DependenceInfo DI(&F, &AR.AA, &AR.SE, &AR.LI); + OptimizationRemarkEmitter ORE(&F); + + if (!tryToUnrollAndJamLoop(LN, AR.DT, AR.LI, AR.SE, AR.TTI, AR.AC, DI, ORE, + OptLevel, U)) return PreservedAnalyses::all(); - return getLoopPassPreservedAnalyses(); + auto PA = getLoopPassPreservedAnalyses(); + PA.preserve(); + return PA; } Index: llvm/test/Transforms/LoopUnrollAndJam/innerloop.ll =================================================================== --- llvm/test/Transforms/LoopUnrollAndJam/innerloop.ll +++ llvm/test/Transforms/LoopUnrollAndJam/innerloop.ll @@ -1,5 +1,5 @@ ; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -verify-loop-info < %s -S | FileCheck %s -; RUN: opt -passes='loop-unroll-and-jam,verify' -allow-unroll-and-jam < %s -S | FileCheck %s +; RUN: opt -passes='loop(loop-unroll-and-jam),verify' -allow-unroll-and-jam < %s -S | FileCheck %s ; Check that the newly created loops to not fail to be added to LI ; This test deliberately disables UnJ on the middle loop, performing it instead on the