Index: lib/Transforms/Scalar/LoopUnrollPass.cpp =================================================================== --- lib/Transforms/Scalar/LoopUnrollPass.cpp +++ lib/Transforms/Scalar/LoopUnrollPass.cpp @@ -274,6 +274,42 @@ return 0; } +// Remove existing unroll metadata and add unroll disable metadata to +// indicate the loop has already been unrolled. This prevents a loop +// from being unrolled more than is directed by a pragma if the loop +// unrolling pass is run more than once (which it generally is). +static void SetLoopAlreadyUnrolled(Loop *L) { + MDNode *LoopID = L->getLoopID(); + if (!LoopID) return; + + // First remove any existing loop unrolling metadata. + SmallVector Vals; + // Reserve first location for self reference to the LoopID metadata node. + Vals.push_back(nullptr); + for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) { + bool IsUnrollMetadata = false; + MDNode *MD = dyn_cast(LoopID->getOperand(i)); + if (MD) { + const MDString *S = dyn_cast(MD->getOperand(0)); + IsUnrollMetadata = S && S->getString().startswith("llvm.loop.unroll."); + } + if (!IsUnrollMetadata) Vals.push_back(LoopID->getOperand(i)); + } + + // Add unroll(disable) metadata to disable future unrolling. + LLVMContext &Context = L->getHeader()->getContext(); + MDNode *DisableNode = + MDNode::get(Context, {MDString::get(Context, "llvm.loop.unroll.enable"), + ConstantInt::get(Type::getInt1Ty(Context), 0)}); + Vals.push_back(DisableNode); + + MDNode *NewLoopID = MDNode::get(Context, Vals); + // Set operand 0 to refer to the loop id itself. + NewLoopID->replaceOperandWith(0, NewLoopID); + L->setLoopID(NewLoopID); + LoopID->replaceAllUsesWith(NewLoopID); +} + unsigned LoopUnroll::selectUnrollCount( const Loop *L, unsigned TripCount, bool HasEnablePragma, unsigned PragmaCount, const TargetTransformInfo::UnrollingPreferences &UP, @@ -428,6 +464,10 @@ } if (HasPragma) { + // Mark loop as unrolled to prevent unrolling beyond that + // requested by the pragma. + SetLoopAlreadyUnrolled(L); + // Emit optimization remarks if we are unable to unroll the loop // as directed by a pragma. DebugLoc LoopLoc = L->getStartLoc(); Index: test/Transforms/LoopUnroll/unroll-pragmas-disabled.ll =================================================================== --- test/Transforms/LoopUnroll/unroll-pragmas-disabled.ll +++ test/Transforms/LoopUnroll/unroll-pragmas-disabled.ll @@ -0,0 +1,69 @@ +; RUN: opt < %s -loop-unroll -S | FileCheck %s +; +; Verify that the unrolling pass removes existing loop unrolling metadata +; and adds a disable unrolling node after unrolling is complete. + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; #pragma clang loop vectorize(enable) unroll(enable) unroll_count(4) vectorize_width(8) +; +; Unroll metadata should be replaces with unroll(disable). Vectorize +; metadata should be untouched. +; +; CHECK-LABEL: @loop1( +; CHECK: br i1 {{.*}}, label {{.*}}, label {{.*}}, !llvm.loop ![[LOOP_1:.*]] +define void @loop1(i32* nocapture %a) { +entry: + br label %for.body + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, i32* %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 64 + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1 + +for.end: ; preds = %for.body + ret void +} +!1 = metadata !{metadata !1, metadata !2, metadata !3, metadata !4, metadata !5} +!2 = metadata !{metadata !"llvm.loop.vectorize.enable", i1 true} +!3 = metadata !{metadata !"llvm.loop.unroll.enable", i1 true} +!4 = metadata !{metadata !"llvm.loop.unroll.count", i32 4} +!5 = metadata !{metadata !"llvm.loop.vectorize.width", i32 8} + +; #pragma clang loop unroll(disable) +; +; Unroll metadata should not change. +; +; CHECK-LABEL: @loop2( +; CHECK: br i1 {{.*}}, label {{.*}}, label {{.*}}, !llvm.loop ![[LOOP_2:.*]] +define void @loop2(i32* nocapture %a) { +entry: + br label %for.body + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %inc = add nsw i32 %0, 1 + store i32 %inc, i32* %arrayidx, align 4 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 64 + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !6 + +for.end: ; preds = %for.body + ret void +} +!6 = metadata !{metadata !6, metadata !7} +!7 = metadata !{metadata !"llvm.loop.unroll.enable", i1 false} + +; CHECK: ![[LOOP_1]] = metadata !{metadata ![[LOOP_1]], metadata ![[VEC_ENABLE:.*]], metadata ![[WIDTH_8:.*]], metadata ![[UNROLL_DISABLE:.*]]} +; CHECK: ![[VEC_ENABLE]] = metadata !{metadata !"llvm.loop.vectorize.enable", i1 true} +; CHECK: ![[WIDTH_8]] = metadata !{metadata !"llvm.loop.vectorize.width", i32 8} +; CHECK: ![[UNROLL_DISABLE]] = metadata !{metadata !"llvm.loop.unroll.enable", i1 false} +; CHECK: ![[LOOP_2]] = metadata !{metadata ![[LOOP_2]], metadata ![[UNROLL_DISABLE:.*]]} Index: test/Transforms/LoopUnroll/unroll-pragmas.ll =================================================================== --- test/Transforms/LoopUnroll/unroll-pragmas.ll +++ test/Transforms/LoopUnroll/unroll-pragmas.ll @@ -1,4 +1,8 @@ ; RUN: opt < %s -loop-unroll -S | FileCheck %s +; RUN: opt < %s -loop-unroll -loop-unroll -S | FileCheck %s +; +; Run loop unrolling twice to verify that loop unrolling metadata is properly +; removed and further unrolling is disabled after the pass is run once. target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu"