Index: lib/Transforms/Scalar/LoopUnrollPass.cpp =================================================================== --- lib/Transforms/Scalar/LoopUnrollPass.cpp +++ lib/Transforms/Scalar/LoopUnrollPass.cpp @@ -20,6 +20,7 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Metadata.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -36,7 +37,8 @@ static cl::opt UnrollCount("unroll-count", cl::init(0), cl::Hidden, - cl::desc("Use this unroll count for all loops, for testing purposes")); + cl::desc("Use this unroll count for all loops including those with " + "unroll_count pragma values, for testing purposes")); static cl::opt UnrollAllowPartial("unroll-allow-partial", cl::init(false), cl::Hidden, @@ -47,6 +49,13 @@ UnrollRuntime("unroll-runtime", cl::ZeroOrMore, cl::init(false), cl::Hidden, cl::desc("Unroll loops with run-time trip counts")); +// Maximum allowed unroll count for a loop being fully unrolled +// because of a pragma unroll(enable) statement (ie, metadata +// "llvm.loopunroll.enable" is true). This prevents unexpected +// behavior like crashing when using this pragma on high trip count +// loops. +static const unsigned PragmaFullUnrollCountLimit = 1024; + namespace { class LoopUnroll : public LoopPass { public: @@ -151,6 +160,63 @@ return LoopSize; } +// Returns the value associated with the given metadata node name (for +// example, "llvm.loopunroll.count"). If no such named metadata node +// exists, then nullptr is returned. +static const ConstantInt *GetUnrollMetadataValue(const Loop *L, + StringRef Name) { + MDNode *LoopID = L->getLoopID(); + if (!LoopID) return nullptr; + + // First operand should refer to the loop id itself. + assert(LoopID->getNumOperands() > 0 && "requires at least one operand"); + assert(LoopID->getOperand(0) == LoopID && "invalid loop id"); + + for (unsigned i = 1, e = LoopID->getNumOperands(); i < e; ++i) { + const MDNode *MD = dyn_cast(LoopID->getOperand(i)); + if (!MD) continue; + + const MDString *S = dyn_cast(MD->getOperand(0)); + if (!S) continue; + + if (Name.equals(S->getString())) { + assert(MD->getNumOperands() == 2 && + "Unroll hint metadata should have two operands."); + return cast(MD->getOperand(1)); + } + } + return nullptr; +} + +// Returns true if the loop has an unroll(enable) pragma. +static bool HasUnrollEnablePragma(const Loop *L) { + const ConstantInt *EnableValue = + GetUnrollMetadataValue(L, "llvm.loopunroll.enable"); + return (EnableValue && EnableValue->getZExtValue()); + return false; +} + +// Returns true if the loop has an unroll(disable) pragma. +static bool HasUnrollDisablePragma(const Loop *L) { + const ConstantInt *EnableValue = + GetUnrollMetadataValue(L, "llvm.loopunroll.enable"); + return (EnableValue && !EnableValue->getZExtValue()); + return false; +} + +// Check for unroll_count(N) pragma. If found, return true and set +// Count to the integer parameter of the pragma. +static bool HasUnrollCountPragma(const Loop *L, int &Count) { + const ConstantInt *CountValue = + GetUnrollMetadataValue(L, "llvm.loopunroll.count"); + if (CountValue) { + Count = CountValue->getZExtValue(); + assert(Count >= 1 && "Unroll count must be positive."); + return true; + } + return false; +} + bool LoopUnroll::runOnLoop(Loop *L, LPPassManager &LPM) { if (skipOptnoneFunction(L)) return false; @@ -202,12 +268,49 @@ TripMultiple = SE->getSmallConstantTripMultiple(L, LatchBlock); } - bool Runtime = UserRuntime ? CurrentRuntime : UP.Runtime; + // User-specified count (either as a command-line option or + // constructor parameter) has highest precedence. + unsigned Count = UserCount ? CurrentCount : 0; - // Use a default unroll-count if the user doesn't specify a value - // and the trip count is a run-time value. The default is different - // for run-time or compile-time trip count loops. - unsigned Count = UserCount ? CurrentCount : UP.Count; + // If there is no user-specified count, unroll pragmas have the next + // highest precendence. + if (Count == 0) { + if (HasUnrollDisablePragma(L)) { + // Loop has unroll(disable) pragma. + return false; + } + + int PragmaCount; + if (HasUnrollCountPragma(L, PragmaCount)) { + if (PragmaCount == 1) { + // Nothing to do. + return false; + } + Count = PragmaCount; + Threshold = NoThreshold; + } else if (HasUnrollEnablePragma(L)) { + // Loop has unroll(enable) pragma without a unroll_count pragma, + // so unroll loop fully if possible. + if (TripCount == 0) { + DEBUG(dbgs() << " Loop has unroll(enable) pragma but loop cannot be " + "fully unrolled because trip count is unknown.\n"); + // Continue with standard heuristic unrolling. + } else if (TripCount > PragmaFullUnrollCountLimit) { + DEBUG(dbgs() << " Loop has unroll(enable) pragma but loop cannot be " + "fully unrolled because loop count is greater than " + << PragmaFullUnrollCountLimit); + // Continue with standard heuristic unrolling. + } else { + Count = TripCount; + Threshold = NoThreshold; + } + } + } + + if (Count == 0) + Count = UP.Count; + + bool Runtime = UserRuntime ? CurrentRuntime : UP.Runtime; if (Runtime && Count == 0 && TripCount == 0) Count = UnrollRuntimeCount; Index: test/Transforms/LoopUnroll/unroll-pragmas.ll =================================================================== --- test/Transforms/LoopUnroll/unroll-pragmas.ll +++ test/Transforms/LoopUnroll/unroll-pragmas.ll @@ -0,0 +1,285 @@ +; RUN: opt < %s -loop-unroll -S | FileCheck %s + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; loop4 contains a small loop which should be completely unrolled by +; the default unrolling heuristics. It serves as a control for the +; unroll(disable) pragma test loop4_with_disable. +; +; CHECK-LABEL: @loop4( +; CHECK-NOT: br i1 +define void @loop4(i32* nocapture %a) { +entry: + br label %for.body + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, i32* %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 4 + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body + ret void +} + +; #pragma clang loop unroll(disable) +; +; CHECK-LABEL: @loop4_with_disable( +; CHECK: store i32 +; CHECK-NOT: store i32 +; CHECK: br i1 +define void @loop4_with_disable(i32* nocapture %a) { +entry: + br label %for.body + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, i32* %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 4 + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1 + +for.end: ; preds = %for.body + ret void +} +!1 = metadata !{metadata !1, metadata !2} +!2 = metadata !{metadata !"llvm.loopunroll.enable", i1 false} + +; loop64 has a high enough count that it should *not* be unrolled by +; the default unrolling heuristic. It serves as the control for the +; unroll(enable) pragma test loop64_with_.* tests below. +; +; CHECK-LABEL: @loop64( +; CHECK: store i32 +; CHECK-NOT: store i32 +; CHECK: br i1 +define void @loop64(i32* nocapture %a) { +entry: + br label %for.body + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, i32* %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 64 + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body + ret void +} + +; #pragma clang loop unroll(enable) +; Loop should be fully unrolled. +; +; CHECK-LABEL: @loop64_with_enable( +; CHECK-NOT: br i1 +define void @loop64_with_enable(i32* nocapture %a) { +entry: + br label %for.body + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, i32* %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 64 + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !3 + +for.end: ; preds = %for.body + ret void +} +!3 = metadata !{metadata !3, metadata !4} +!4 = metadata !{metadata !"llvm.loopunroll.enable", i1 true} + +; #pragma clang loop unroll_count(4) +; Loop should be unrolled 4 times. +; +; CHECK-LABEL: @loop64_with_count4( +; CHECK: store i32 +; CHECK: store i32 +; CHECK: store i32 +; CHECK: store i32 +; CHECK-NOT: store i32 +; CHECK: br i1 +define void @loop64_with_count4(i32* nocapture %a) { +entry: + br label %for.body + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, i32* %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 64 + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !5 + +for.end: ; preds = %for.body + ret void +} +!5 = metadata !{metadata !5, metadata !6} +!6 = metadata !{metadata !"llvm.loopunroll.count", i32 4} + + +; #pragma clang loop unroll_count(enable) unroll_count(4) +; Loop should be unrolled 4 times. +; +; CHECK-LABEL: @loop64_with_enable_and_count4( +; CHECK: store i32 +; CHECK: store i32 +; CHECK: store i32 +; CHECK: store i32 +; CHECK-NOT: store i32 +; CHECK: br i1 +define void @loop64_with_enable_and_count4(i32* nocapture %a) { +entry: + br label %for.body + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, i32* %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 64 + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !7 + +for.end: ; preds = %for.body + ret void +} +!7 = metadata !{metadata !7, metadata !6, metadata !4} + +; #pragma clang loop unroll_count(enable) +; Full unrolling is requested, but loop has a dynamic trip count so +; no unrolling should occur. +; +; CHECK-LABEL: @dynamic_loop_with_enable( +; CHECK: store i32 +; CHECK-NOT: store i32 +; CHECK: br i1 +define void @dynamic_loop_with_enable(i32* nocapture %a, i32 %b) { +entry: + %cmp3 = icmp sgt i32 %b, 0 + br i1 %cmp3, label %for.body, label %for.end, !llvm.loop !8 + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, i32* %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %b + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !8 + +for.end: ; preds = %for.body, %entry + ret void +} +!8 = metadata !{metadata !8, metadata !4} + +; #pragma clang loop unroll_count(4) +; Loop has a dynamic trip count. Unrolling should occur, but no +; conditional branches can be removed. +; +; CHECK-LABEL: @dynamic_loop_with_count4( +; CHECK-NOT: store +; CHECK: br i1 +; CHECK: store +; CHECK: br i1 +; CHECK: store +; CHECK: br i1 +; CHECK: store +; CHECK: br i1 +; CHECK: store +; CHECK: br i1 +; CHECK-NOT: br i1 +define void @dynamic_loop_with_count4(i32* nocapture %a, i32 %b) { +entry: + %cmp3 = icmp sgt i32 %b, 0 + br i1 %cmp3, label %for.body, label %for.end, !llvm.loop !9 + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, i32* %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %b + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !9 + +for.end: ; preds = %for.body, %entry + ret void +} +!9 = metadata !{metadata !9, metadata !6} + +; #pragma clang loop unroll_count(1) +; Loop should not be unrolled +; +; CHECK-LABEL: @unroll_1( +; CHECK: store i32 +; CHECK-NOT: store i32 +; CHECK: br i1 +define void @unroll_1(i32* nocapture %a, i32 %b) { +entry: + br label %for.body + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, i32* %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 4 + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !10 + +for.end: ; preds = %for.body + ret void +} +!10 = metadata !{metadata !10, metadata !11} +!11 = metadata !{metadata !"llvm.loopunroll.count", i32 1} + +; #pragma clang loop unroll(enable) +; Loop has very high loop count (1 million) and full unrolling was requested. +; Loop should not be unrolled. +; +; CHECK-LABEL: @unroll_1M( +; CHECK: store i32 +; CHECK-NOT: store i32 +; CHECK: br i1 +define void @unroll_1M(i32* nocapture %a, i32 %b) { +entry: + br label %for.body + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, i32* %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 1000000 + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !12 + +for.end: ; preds = %for.body + ret void +} +!12 = metadata !{metadata !12, metadata !4}