Index: lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- lib/Transforms/Vectorize/LoopVectorize.cpp +++ lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1219,7 +1219,7 @@ /// for example 'force', means a decision has been made. So, we need to be /// careful NOT to add them if the user hasn't specifically asked so. class LoopVectorizeHints { - enum HintKind { HK_WIDTH, HK_UNROLL, HK_FORCE }; + enum HintKind { HK_WIDTH, HK_UNROLL, HK_FORCE, HK_ISVECTORIZED }; /// Hint - associates name and validation with the hint value. struct Hint { @@ -1238,6 +1238,8 @@ return isPowerOf2_32(Val) && Val <= MaxInterleaveFactor; case HK_FORCE: return (Val <= 1); + case HK_ISVECTORIZED: + return (Val==0 || Val==1); } return false; } @@ -1250,6 +1252,8 @@ /// Vectorization forced Hint Force; + /// Already Vectorized + Hint IsVectorized; /// Return the loop metadata prefix. static StringRef Prefix() { return "llvm.loop."; } @@ -1269,6 +1273,7 @@ HK_WIDTH), Interleave("interleave.count", DisableInterleaving, HK_UNROLL), Force("vectorize.enable", FK_Undefined, HK_FORCE), + IsVectorized("isvectorized", 0, HK_ISVECTORIZED), PotentiallyUnsafe(false), TheLoop(L), ORE(ORE) { // Populate values with existing loop metadata. getHintsFromMetadata(); @@ -1277,14 +1282,19 @@ if (VectorizerParams::isInterleaveForced()) Interleave.Value = VectorizerParams::VectorizationInterleave; + if (IsVectorized.Value != 1) + // If the vectorization width and interleaving count are both 1 then consider + // the loop to have been already vectorized because there's nothing more that we + // can do. + IsVectorized.Value = Width.Value == 1 && Interleave.Value == 1; DEBUG(if (DisableInterleaving && Interleave.Value == 1) dbgs() << "LV: Interleaving disabled by the pass manager\n"); } /// Mark the loop L as already vectorized by setting the width to 1. void setAlreadyVectorized() { - Width.Value = Interleave.Value = 1; - Hint Hints[] = {Width, Interleave}; + IsVectorized.Value = 1; + Hint Hints[] = {IsVectorized}; writeHintsToMetadata(Hints); } @@ -1301,9 +1311,7 @@ return false; } - if (getWidth() == 1 && getInterleave() == 1) { - // FIXME: Add a separate metadata to indicate when the loop has already - // been vectorized instead of setting width and count to 1. + if (getIsVectorized() == 1) { DEBUG(dbgs() << "LV: Not vectorizing: Disabled/already vectorized.\n"); // FIXME: Add interleave.disable metadata. This will allow // vectorize.disable to be used without disabling the pass and errors @@ -1312,8 +1320,8 @@ "AllDisabled", L->getStartLoc(), L->getHeader()) << "loop not vectorized: vectorization and interleaving are " - "explicitly disabled, or vectorize width and interleave " - "count are both set to 1"); + "explicitly disabled, or the loop has already been " + "vectorized"); return false; } @@ -1346,6 +1354,7 @@ unsigned getWidth() const { return Width.Value; } unsigned getInterleave() const { return Interleave.Value; } + unsigned getIsVectorized() const { return IsVectorized.Value; } enum ForceKind getForce() const { return (ForceKind)Force.Value; } /// \brief If hints are provided that force vectorization, use the AlwaysPrint @@ -1429,7 +1438,7 @@ return; unsigned Val = C->getZExtValue(); - Hint *Hints[] = {&Width, &Interleave, &Force}; + Hint *Hints[] = {&Width, &Interleave, &Force, &IsVectorized}; for (auto H : Hints) { if (Name == H->Name) { if (H->validate(Val)) Index: test/Transforms/LoopVectorize/X86/already-vectorized.ll =================================================================== --- test/Transforms/LoopVectorize/X86/already-vectorized.ll +++ test/Transforms/LoopVectorize/X86/already-vectorized.ll @@ -39,9 +39,8 @@ } ; Now, we check for the Hint metadata -; CHECK: [[vect]] = distinct !{[[vect]], [[width:![0-9]+]], [[unroll:![0-9]+]]} -; CHECK: [[width]] = !{!"llvm.loop.vectorize.width", i32 1} -; CHECK: [[unroll]] = !{!"llvm.loop.interleave.count", i32 1} -; CHECK: [[scalar]] = distinct !{[[scalar]], [[runtime_unroll:![0-9]+]], [[width]], [[unroll]]} +; CHECK: [[vect]] = distinct !{[[vect]], [[width:![0-9]+]]} +; CHECK: [[width]] = !{!"llvm.loop.isvectorized", i32 1} +; CHECK: [[scalar]] = distinct !{[[scalar]], [[runtime_unroll:![0-9]+]], [[width]]} ; CHECK: [[runtime_unroll]] = !{!"llvm.loop.unroll.runtime.disable"} Index: test/Transforms/LoopVectorize/X86/vectorization-remarks-loopid-dbg.ll =================================================================== --- test/Transforms/LoopVectorize/X86/vectorization-remarks-loopid-dbg.ll +++ test/Transforms/LoopVectorize/X86/vectorization-remarks-loopid-dbg.ll @@ -8,7 +8,7 @@ ; VECTORIZED: remark: vectorization-remarks.c:17:8: vectorized loop (vectorization width: 4, interleaved count: 1) ; UNROLLED: remark: vectorization-remarks.c:17:8: interleaved loop (interleaved count: 4) -; NONE: remark: vectorization-remarks.c:17:8: loop not vectorized: vectorization and interleaving are explicitly disabled, or vectorize width and interleave count are both set to 1 +; NONE: remark: vectorization-remarks.c:17:8: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" Index: test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll =================================================================== --- test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll +++ test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll @@ -20,7 +20,7 @@ ; for (int i = 0; i < Length; i++) ; A[i] = i; ; } -; CHECK: remark: source.cpp:13:5: loop not vectorized: vectorization and interleaving are explicitly disabled, or vectorize width and interleave count are both set to 1 +; CHECK: remark: source.cpp:13:5: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized ; void test_array_bounds(int *A, int *B, int Length) { ; #pragma clang loop vectorize(enable) @@ -67,7 +67,7 @@ ; YAML-NEXT: DebugLoc: { File: source.cpp, Line: 13, Column: 5 } ; YAML-NEXT: Function: _Z13test_disabledPii ; YAML-NEXT: Args: -; YAML-NEXT: - String: 'loop not vectorized: vectorization and interleaving are explicitly disabled, or vectorize width and interleave count are both set to 1' +; YAML-NEXT: - String: 'loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized ; YAML-NEXT: ... ; YAML-NEXT: --- !Analysis ; YAML-NEXT: Pass: '' Index: test/Transforms/LoopVectorize/X86/vectorization-remarks.ll =================================================================== --- test/Transforms/LoopVectorize/X86/vectorization-remarks.ll +++ test/Transforms/LoopVectorize/X86/vectorization-remarks.ll @@ -8,7 +8,7 @@ ; VECTORIZED: remark: vectorization-remarks.c:17:8: vectorized loop (vectorization width: 4, interleaved count: 1) ; UNROLLED: remark: vectorization-remarks.c:17:8: interleaved loop (interleaved count: 4) -; NONE: remark: vectorization-remarks.c:17:8: loop not vectorized: vectorization and interleaving are explicitly disabled, or vectorize width and interleave count are both set to 1 +; NONE: remark: vectorization-remarks.c:17:8: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" Index: test/Transforms/LoopVectorize/duplicated-metadata.ll =================================================================== --- test/Transforms/LoopVectorize/duplicated-metadata.ll +++ test/Transforms/LoopVectorize/duplicated-metadata.ll @@ -26,4 +26,4 @@ !0 = !{!0, !1} !1 = !{!"llvm.loop.vectorize.width", i32 4} ; CHECK-NOT: !{metadata !"llvm.loop.vectorize.width", i32 4} -; CHECK: !{!"llvm.loop.interleave.count", i32 1} +; CHECK: !{!"llvm.loop.isvectorized", i32 1} Index: test/Transforms/LoopVectorize/multiple-strides-vectorization.ll =================================================================== --- test/Transforms/LoopVectorize/multiple-strides-vectorization.ll +++ test/Transforms/LoopVectorize/multiple-strides-vectorization.ll @@ -25,7 +25,7 @@ ; CHECK-LABEL: Test ; CHECK: <4 x i64> ; CHECK: <4 x i32>, <4 x i32> -; CHECK: llvm.loop.vectorize.width +; CHECK: !{!"llvm.loop.isvectorized", i32 1} target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" Index: test/Transforms/LoopVectorize/vectorize-once.ll =================================================================== --- test/Transforms/LoopVectorize/vectorize-once.ll +++ test/Transforms/LoopVectorize/vectorize-once.ll @@ -67,11 +67,10 @@ attributes #0 = { nounwind readonly ssp uwtable "fp-contract-model"="standard" "no-frame-pointer-elim" "no-frame-pointer-elim-non-leaf" "realign-stack" "relocation-model"="pic" "ssp-buffers-size"="8" } -; CHECK: !0 = distinct !{!0, !1, !2} -; CHECK: !1 = !{!"llvm.loop.vectorize.width", i32 1} -; CHECK: !2 = !{!"llvm.loop.interleave.count", i32 1} -; CHECK: !3 = distinct !{!3, !4, !1, !2} -; CHECK: !4 = !{!"llvm.loop.unroll.runtime.disable"} +; CHECK: !0 = distinct !{!0, !1} +; CHECK: !1 = !{!"llvm.loop.isvectorized", i32 1} +; CHECK: !2 = distinct !{!2, !3, !1} +; CHECK: !3 = !{!"llvm.loop.unroll.runtime.disable"} !0 = !{!0, !1} !1 = !{!"llvm.loop.vectorize.width", i32 1}