Index: include/llvm/Transforms/Scalar.h =================================================================== --- include/llvm/Transforms/Scalar.h +++ include/llvm/Transforms/Scalar.h @@ -184,7 +184,7 @@ // Pass *createLoopUnrollPass(int OptLevel = 2, int Threshold = -1, int Count = -1, int AllowPartial = -1, int Runtime = -1, - int UpperBound = -1); + int UpperBound = -1, int AllowPeeling = -1); // Create an unrolling pass for full unrolling that uses exact trip count only. Pass *createSimpleLoopUnrollPass(int OptLevel = 2); Index: lib/Transforms/Scalar/LoopUnrollPass.cpp =================================================================== --- lib/Transforms/Scalar/LoopUnrollPass.cpp +++ lib/Transforms/Scalar/LoopUnrollPass.cpp @@ -134,7 +134,7 @@ Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI, int OptLevel, Optional UserThreshold, Optional UserCount, Optional UserAllowPartial, Optional UserRuntime, - Optional UserUpperBound) { + Optional UserUpperBound, Optional UserAllowPeeling) { TargetTransformInfo::UnrollingPreferences UP; // Set up the defaults @@ -201,6 +201,8 @@ UP.Runtime = *UserRuntime; if (UserUpperBound.hasValue()) UP.UpperBound = *UserUpperBound; + if (UserAllowPeeling.hasValue()) + UP.AllowPeeling = *UserAllowPeeling; return UP; } @@ -927,15 +929,13 @@ return ExplicitUnroll; } -static bool tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, - ScalarEvolution &SE, const TargetTransformInfo &TTI, - AssumptionCache &AC, OptimizationRemarkEmitter &ORE, - bool PreserveLCSSA, int OptLevel, - Optional ProvidedCount, - Optional ProvidedThreshold, - Optional ProvidedAllowPartial, - Optional ProvidedRuntime, - Optional ProvidedUpperBound) { +static bool tryToUnrollLoop( + Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE, + const TargetTransformInfo &TTI, AssumptionCache &AC, + OptimizationRemarkEmitter &ORE, bool PreserveLCSSA, int OptLevel, + Optional ProvidedCount, Optional ProvidedThreshold, + Optional ProvidedAllowPartial, Optional ProvidedRuntime, + Optional ProvidedUpperBound, Optional ProvidedAllowPeeling) { DEBUG(dbgs() << "Loop Unroll: F[" << L->getHeader()->getParent()->getName() << "] Loop %" << L->getHeader()->getName() << "\n"); if (HasUnrollDisablePragma(L)) @@ -951,7 +951,8 @@ bool Convergent; TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences( L, SE, TTI, OptLevel, ProvidedThreshold, ProvidedCount, - ProvidedAllowPartial, ProvidedRuntime, ProvidedUpperBound); + ProvidedAllowPartial, ProvidedRuntime, ProvidedUpperBound, + ProvidedAllowPeeling); // Exit early if unrolling is disabled. if (UP.Threshold == 0 && (!UP.Partial || UP.PartialThreshold == 0)) return false; @@ -1053,10 +1054,12 @@ LoopUnroll(int OptLevel = 2, Optional Threshold = None, Optional Count = None, Optional AllowPartial = None, Optional Runtime = None, - Optional UpperBound = None) + Optional UpperBound = None, + Optional AllowPeeling = None) : LoopPass(ID), OptLevel(OptLevel), ProvidedCount(std::move(Count)), ProvidedThreshold(Threshold), ProvidedAllowPartial(AllowPartial), - ProvidedRuntime(Runtime), ProvidedUpperBound(UpperBound) { + ProvidedRuntime(Runtime), ProvidedUpperBound(UpperBound), + ProvidedAllowPeeling(AllowPeeling) { initializeLoopUnrollPass(*PassRegistry::getPassRegistry()); } @@ -1066,6 +1069,7 @@ Optional ProvidedAllowPartial; Optional ProvidedRuntime; Optional ProvidedUpperBound; + Optional ProvidedAllowPeeling; bool runOnLoop(Loop *L, LPPassManager &) override { if (skipLoop(L)) @@ -1088,7 +1092,7 @@ return tryToUnrollLoop(L, DT, LI, SE, TTI, AC, ORE, PreserveLCSSA, OptLevel, ProvidedCount, ProvidedThreshold, ProvidedAllowPartial, ProvidedRuntime, - ProvidedUpperBound); + ProvidedUpperBound, ProvidedAllowPeeling); } /// This transformation requires natural loop information & requires that @@ -1112,8 +1116,8 @@ INITIALIZE_PASS_END(LoopUnroll, "loop-unroll", "Unroll loops", false, false) Pass *llvm::createLoopUnrollPass(int OptLevel, int Threshold, int Count, - int AllowPartial, int Runtime, - int UpperBound) { + int AllowPartial, int Runtime, int UpperBound, + int AllowPeeling) { // TODO: It would make more sense for this function to take the optionals // directly, but that's dangerous since it would silently break out of tree // callers. @@ -1122,11 +1126,12 @@ Count == -1 ? None : Optional(Count), AllowPartial == -1 ? None : Optional(AllowPartial), Runtime == -1 ? None : Optional(Runtime), - UpperBound == -1 ? None : Optional(UpperBound)); + UpperBound == -1 ? None : Optional(UpperBound), + AllowPeeling == -1 ? None : Optional(AllowPeeling)); } Pass *llvm::createSimpleLoopUnrollPass(int OptLevel) { - return llvm::createLoopUnrollPass(OptLevel, -1, -1, 0, 0, 0); + return llvm::createLoopUnrollPass(OptLevel, -1, -1, 0, 0, 0, 0); } PreservedAnalyses LoopFullUnrollPass::run(Loop &L, LoopAnalysisManager &AM, @@ -1156,7 +1161,8 @@ tryToUnrollLoop(&L, AR.DT, &AR.LI, AR.SE, AR.TTI, AR.AC, *ORE, /*PreserveLCSSA*/ true, OptLevel, /*Count*/ None, /*Threshold*/ None, /*AllowPartial*/ false, - /*Runtime*/ false, /*UpperBound*/ false); + /*Runtime*/ false, /*UpperBound*/ false, + /*AllowPeeling*/ false); if (!Changed) return PreservedAnalyses::all(); @@ -1274,11 +1280,13 @@ // states we support: partial and full (or "simple") unrolling. However, to // enable these things we actually pass "None" in for the optional to avoid // providing an explicit choice. - Optional AllowPartialParam, RuntimeParam, UpperBoundParam; + Optional AllowPartialParam, RuntimeParam, UpperBoundParam, + AllowPeeling; bool CurChanged = tryToUnrollLoop( &L, DT, &LI, SE, TTI, AC, ORE, /*PreserveLCSSA*/ true, OptLevel, /*Count*/ None, - /*Threshold*/ None, AllowPartialParam, RuntimeParam, UpperBoundParam); + /*Threshold*/ None, AllowPartialParam, RuntimeParam, UpperBoundParam, + /*AllowPeeling*/ None); Changed |= CurChanged; // The parent must not be damaged by unrolling! Index: test/Transforms/LoopUnroll/peel-loop-pgo.ll =================================================================== --- test/Transforms/LoopUnroll/peel-loop-pgo.ll +++ test/Transforms/LoopUnroll/peel-loop-pgo.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -S -debug-only=loop-unroll -loop-unroll 2>&1 | FileCheck %s +; RUN: opt < %s -S -debug-only=loop-unroll -passes='require,unroll' 2>&1 | FileCheck %s ; REQUIRES: asserts ; Make sure we use the profile information correctly to peel-off 3 iterations @@ -9,6 +10,10 @@ ; CHECK: Loop Unroll: F[optsize] ; CHECK-NOT: PEELING +; Confirm that no peeling occurs when we are performing full unrolling. +; RUN: opt < %s -S -debug-only=loop-unroll -passes='require,loop(unroll-full)' 2>&1 | FileCheck %s --check-prefix=FULLUNROLL +; FULLUNROLL-NOT: PEELING + ; CHECK-LABEL: @basic ; CHECK: br i1 %{{.*}}, label %[[NEXT0:.*]], label %for.cond.for.end_crit_edge, !prof !1 ; CHECK: [[NEXT0]]: