Index: include/llvm/Transforms/Scalar/LoopUnrollPass.h
===================================================================
--- include/llvm/Transforms/Scalar/LoopUnrollPass.h
+++ include/llvm/Transforms/Scalar/LoopUnrollPass.h
@@ -10,6 +10,7 @@
#ifndef LLVM_TRANSFORMS_SCALAR_LOOPUNROLLPASS_H
#define LLVM_TRANSFORMS_SCALAR_LOOPUNROLLPASS_H
+#include "llvm/ADT/Optional.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/IR/PassManager.h"
@@ -30,16 +31,59 @@
LoopStandardAnalysisResults &AR, LPMUpdater &U);
};
+struct LoopUnrollOptions {
+ // Selectors for optional behaviors - set to true to enable, false to disable,
+ // None - to use global default.
+ Optional<bool> AllowPartial;
+ Optional<bool> AllowPeeling;
+ Optional<bool> AllowRuntime;
+ Optional<bool> AllowUpperBound;
+
+ LoopUnrollOptions(Optional<bool> Partial = None,
+ Optional<bool> Peeling = None,
+ Optional<bool> Runtime = None,
+ Optional<bool> UpperBound = None)
+ : AllowPartial(Partial), AllowPeeling(Peeling), AllowRuntime(Runtime),
+ AllowUpperBound(UpperBound) {}
+
+ // Use 'builder' pattern to set members by name at construction time
+
+ /// Selector for the optional behavior - Partial unrolling.
+ LoopUnrollOptions &setPartial(bool Partial) {
+ AllowPartial = Partial;
+ return *this;
+ }
+
+ /// Selector for the optional behavior - Runtime unrolling.
+ LoopUnrollOptions &setRuntime(bool Runtime) {
+ AllowRuntime = Runtime;
+ return *this;
+ }
+
+ /// Selector for the optional behavior - Peeling.
+ LoopUnrollOptions &setPeeling(bool Peeling) {
+ AllowPeeling = Peeling;
+ return *this;
+ }
+ /// Selector for the optional behavior - UpperBound.
+ LoopUnrollOptions &setUpperBound(bool UpperBound) {
+ AllowUpperBound = UpperBound;
+ return *this;
+ }
+};
+
/// Loop unroll pass that will support both full and partial unrolling.
/// It is a function pass to have access to function and module analyses.
/// It will also put loops into canonical form (simplified and LCSSA).
class LoopUnrollPass : public PassInfoMixin<LoopUnrollPass> {
const int OptLevel;
+ LoopUnrollOptions UnrollOpts;
public:
/// This uses the target information (or flags) to control the thresholds for
/// different unrolling stategies but supports all of them.
- explicit LoopUnrollPass(int OptLevel = 2) : OptLevel(OptLevel) {}
+ explicit LoopUnrollPass(int OptLevel = 2, LoopUnrollOptions UnrollOpts = {})
+ : OptLevel(OptLevel), UnrollOpts(UnrollOpts) {}
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
};
Index: lib/Passes/PassRegistry.def
===================================================================
--- lib/Passes/PassRegistry.def
+++ lib/Passes/PassRegistry.def
@@ -217,6 +217,7 @@
FUNCTION_PASS("tailcallelim", TailCallElimPass())
FUNCTION_PASS("unreachableblockelim", UnreachableBlockElimPass())
FUNCTION_PASS("unroll", LoopUnrollPass())
+FUNCTION_PASS("unroll-peeling",LoopUnrollPass(2, LoopUnrollOptions().setPartial(false).setPeeling(true).setRuntime(false).setUpperBound(false)))
FUNCTION_PASS("verify", VerifierPass())
FUNCTION_PASS("verify<domtree>", DominatorTreeVerifierPass())
FUNCTION_PASS("verify<loops>", LoopVerifierPass())
Index: lib/Transforms/Scalar/LoopUnrollPass.cpp
===================================================================
--- lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -1333,23 +1333,22 @@
Loop *ParentL = L.getParentLoop();
#endif
- // The API here is quite complex to call, but there are only two interesting
- // states we support: partial and full (or "simple") unrolling. However, to
- // enable these things we actually pass "None" in for the optional to avoid
- // providing an explicit choice.
- Optional<bool> AllowPartialParam, RuntimeParam, UpperBoundParam,
- AllowPeeling;
// Check if the profile summary indicates that the profiled application
// has a huge working set size, in which case we disable peeling to avoid
// bloating it further.
+ Optional<bool> LocalAllowPeeling = UnrollOpts.AllowPeeling;
if (PSI && PSI->hasHugeWorkingSetSize())
- AllowPeeling = false;
+ LocalAllowPeeling = false;
std::string LoopName = L.getName();
- LoopUnrollResult Result =
- tryToUnrollLoop(&L, DT, &LI, SE, TTI, AC, ORE,
- /*PreserveLCSSA*/ true, OptLevel, /*Count*/ None,
- /*Threshold*/ None, AllowPartialParam, RuntimeParam,
- UpperBoundParam, AllowPeeling);
+ // The API here is quite complex to call and we allow to select some
+ // flavors of unrolling during construction time (by setting Allow* values).
+ // As soon as the caller does not care we pass None for the corresponding
+ // optional to avoid providing an explicit choice.
+ LoopUnrollResult Result = tryToUnrollLoop(
+ &L, DT, &LI, SE, TTI, AC, ORE,
+ /*PreserveLCSSA*/ true, OptLevel, /*Count*/ None,
+ /*Threshold*/ None, UnrollOpts.AllowPartial, UnrollOpts.AllowRuntime,
+ UnrollOpts.AllowUpperBound, LocalAllowPeeling);
Changed |= Result != LoopUnrollResult::Unmodified;
// The parent must not be damaged by unrolling!
Index: test/Transforms/LoopUnroll/peel-loop.ll
===================================================================
--- test/Transforms/LoopUnroll/peel-loop.ll
+++ test/Transforms/LoopUnroll/peel-loop.ll
@@ -1,4 +1,6 @@
; RUN: opt < %s -S -loop-unroll -unroll-force-peel-count=3 -verify-dom-info -simplifycfg -instcombine | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll,simplify-cfg,instcombine' -unroll-force-peel-count=3 -verify-dom-info | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll-peeling,simplify-cfg,instcombine' -unroll-force-peel-count=3 -verify-dom-info | FileCheck %s
; Basic loop peeling - check that we can peel-off the first 3 loop iterations
; when explicitly requested.
Index: test/Transforms/LoopUnroll/runtime-loop.ll
===================================================================
--- test/Transforms/LoopUnroll/runtime-loop.ll
+++ test/Transforms/LoopUnroll/runtime-loop.ll
@@ -1,33 +1,53 @@
-; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=true | FileCheck %s -check-prefix=EPILOG
-; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
-
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime=true -unroll-runtime-epilog=true | FileCheck %s -check-prefix=EPILOG
-; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
+; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=true | FileCheck %s -check-prefixes=EPILOG,COMMON
+; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=PROLOG,COMMON
+
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime=true -unroll-runtime-epilog=true | FileCheck %s -check-prefixes=EPILOG,COMMON
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=PROLOG,COMMON
+;
+; Restricted versions of unroll (unroll-noruntime, unroll-full) should not be doing runtime unrolling
+; even if it is globally enabled through -unroll-runtime option
+;
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll-peeling' -unroll-runtime=true -unroll-runtime-epilog=true | FileCheck %s -check-prefixes=NOEPILOG,COMMON
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll-peeling' -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=NOPROLOG,COMMON
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-runtime=true -unroll-runtime-epilog=true | FileCheck %s -check-prefixes=NOEPILOG,COMMON
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=NOPROLOG,COMMON
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
; Tests for unrolling loops with run-time trip counts
+; COMMON-LABEL: @test(
+
; EPILOG: %xtraiter = and i32 %n
; EPILOG: %lcmp.mod = icmp ne i32 %xtraiter, 0
; EPILOG: br i1 %lcmp.mod, label %for.body.epil.preheader, label %for.end.loopexit
+; NOEPILOG-NOT: %xtraiter = and i32 %n
+
; PROLOG: %xtraiter = and i32 %n
; PROLOG: %lcmp.mod = icmp ne i32 %xtraiter, 0
; PROLOG: br i1 %lcmp.mod, label %for.body.prol.preheader, label %for.body.prol.loopexit
+; NOPROLOG-NOT: %xtraiter = and i32 %n
+
; EPILOG: for.body.epil:
; EPILOG: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %for.body.epil ], [ %indvars.iv.unr, %for.body.epil.preheader ]
; EPILOG: %epil.iter.sub = sub i32 %epil.iter, 1
; EPILOG: %epil.iter.cmp = icmp ne i32 %epil.iter.sub, 0
; EPILOG: br i1 %epil.iter.cmp, label %for.body.epil, label %for.end.loopexit.epilog-lcssa, !llvm.loop !0
+; NOEPILOG: for.body:
+; NOEPILOG-NOT: for.body.epil:
+
; PROLOG: for.body.prol:
; PROLOG: %indvars.iv.prol = phi i64 [ %indvars.iv.next.prol, %for.body.prol ], [ 0, %for.body.prol.preheader ]
; PROLOG: %prol.iter.sub = sub i32 %prol.iter, 1
; PROLOG: %prol.iter.cmp = icmp ne i32 %prol.iter.sub, 0
; PROLOG: br i1 %prol.iter.cmp, label %for.body.prol, label %for.body.prol.loopexit.unr-lcssa, !llvm.loop !0
+; NOPROLOG: for.body:
+; NOPROLOG-NOT: for.body.prol:
+
define i32 @test(i32* nocapture %a, i32 %n) nounwind uwtable readonly {
entry:
@@ -54,11 +74,10 @@
; Still try to completely unroll loops with compile-time trip counts
; even if the -unroll-runtime is specified
-; EPILOG: for.body:
-; EPILOG-NOT: for.body.epil:
-
-; PROLOG: for.body:
-; PROLOG-NOT: for.body.prol:
+; COMMON-LABEL: @test1(
+; COMMON: for.body:
+; COMMON-NOT: for.body.epil:
+; COMMON-NOT: for.body.prol:
define i32 @test1(i32* nocapture %a) nounwind uwtable readonly {
entry:
@@ -82,8 +101,11 @@
; This is test 2007-05-09-UnknownTripCount.ll which can be unrolled now
; if the -unroll-runtime option is turned on
+; COMMON-LABEL: @foo(
; EPILOG: bb72.2:
; PROLOG: bb72.2:
+; NOEPILOG-NOT: bb72.2:
+; NOPROLOG-NOT: bb72.2:
define void @foo(i32 %trips) {
entry:
@@ -105,12 +127,19 @@
; Test run-time unrolling for a loop that counts down by -2.
+; COMMON-LABEL: @down(
; EPILOG: for.body.epil:
; EPILOG: br i1 %epil.iter.cmp, label %for.body.epil, label %for.cond.for.end_crit_edge.epilog-lcssa
+; NOEPILOG: for.body:
+; NOEPILOG-NOT: for.body.epil:
+
; PROLOG: for.body.prol:
; PROLOG: br i1 %prol.iter.cmp, label %for.body.prol, label %for.body.prol.loopexit
+; NOPROLOG: for.body:
+; NOPROLOG-NOT: for.body.prol:
+
define zeroext i16 @down(i16* nocapture %p, i32 %len) nounwind uwtable readonly {
entry:
%cmp2 = icmp eq i32 %len, 0
@@ -138,12 +167,20 @@
}
; Test run-time unrolling disable metadata.
+; COMMON-LABEL: @test2(
+
; EPILOG: for.body:
; EPILOG-NOT: for.body.epil:
+; NOEPILOG: for.body:
+; NOEPILOG-NOT: for.body.epil:
+
; PROLOG: for.body:
; PROLOG-NOT: for.body.prol:
+; NOPROLOG: for.body:
+; NOPROLOG-NOT: for.body.prol:
+
define zeroext i16 @test2(i16* nocapture %p, i32 %len) nounwind uwtable readonly {
entry:
%cmp2 = icmp eq i32 %len, 0
@@ -174,11 +211,9 @@
; -runtime-unroll-multi-exit=true
; single exit, multiple exiting blocks.
define void @unique_exit(i32 %arg) {
-; PROLOG: unique_exit(
-; PROLOG-NOT: .unr
+; COMMON-LABEL: @unique_exit(
+; COMMON-NOT: .unr
-; EPILOG: unique_exit(
-; EPILOG-NOT: .unr
entry:
%tmp = icmp sgt i32 undef, %arg
br i1 %tmp, label %preheader, label %returnblock
@@ -206,11 +241,9 @@
; multiple exit blocks. don't unroll
define void @multi_exit(i64 %trip, i1 %cond) {
-; PROLOG: multi_exit(
-; PROLOG-NOT: .unr
+; COMMON-LABEL: @multi_exit(
+; COMMON-NOT: .unr
-; EPILOG: multi_exit(
-; EPILOG-NOT: .unr
entry:
br label %loop_header
@@ -238,11 +271,15 @@
exit2.loopexit:
ret void
}
+
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.unroll.runtime.disable"}
-; EPILOG: !0 = distinct !{!0, !1}
+; need to use LABEL here to separate function IR matching from metadata matching
+; COMMON-LABEL: {{^}}!0 =
+
+; EPILOG-SAME: distinct !{!0, !1}
; EPILOG: !1 = !{!"llvm.loop.unroll.disable"}
-; PROLOG: !0 = distinct !{!0, !1}
+; PROLOG-SAME: distinct !{!0, !1}
; PROLOG: !1 = !{!"llvm.loop.unroll.disable"}