Index: include/llvm/Transforms/Vectorize.h =================================================================== --- include/llvm/Transforms/Vectorize.h +++ include/llvm/Transforms/Vectorize.h @@ -110,8 +110,8 @@ // // LoopVectorize - Create a loop vectorization pass. // -Pass *createLoopVectorizePass(bool NoUnrolling = false, - bool AlwaysVectorize = true); +Pass *createLoopVectorizePass(bool InterleaveOnlyWhenForced = false, + bool VectorizeOnlyWhenForced = false); //===----------------------------------------------------------------------===// // Index: include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h =================================================================== --- include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h +++ include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h @@ -95,7 +95,7 @@ FK_Enabled = 1, ///< Forcing enabled. }; - LoopVectorizeHints(const Loop *L, bool DisableInterleaving, + LoopVectorizeHints(const Loop *L, bool InterleaveOnlyWhenForced, OptimizationRemarkEmitter &ORE); /// Mark the loop L as already vectorized by setting the width to 1. @@ -105,7 +105,8 @@ writeHintsToMetadata(Hints); } - bool allowVectorization(Function *F, Loop *L, bool AlwaysVectorize) const; + bool allowVectorization(Function *F, Loop *L, + bool VectorizeOnlyWhenForced) const; /// Dumps all the hint information. void emitRemarkWithHints() const; Index: include/llvm/Transforms/Vectorize/LoopVectorize.h =================================================================== --- include/llvm/Transforms/Vectorize/LoopVectorize.h +++ include/llvm/Transforms/Vectorize/LoopVectorize.h @@ -78,12 +78,13 @@ /// The LoopVectorize Pass. struct LoopVectorizePass : public PassInfoMixin { - bool DisableUnrolling = false; + /// If false, consider all loops for interleaving. + /// If true, only loops that explicitly request interleaving are considered. + bool InterleaveOnlyWhenForced = false; - /// If true, consider all loops for vectorization. - /// If false, only loops that explicitly request vectorization are - /// considered. - bool AlwaysVectorize = true; + /// If false, consider all loops for vectorization. + /// If true, only loops that explicitly request vectorization are considered. + bool VectorizeOnlyWhenForced = false; ScalarEvolution *SE; LoopInfo *LI; Index: lib/Transforms/IPO/PassManagerBuilder.cpp =================================================================== --- lib/Transforms/IPO/PassManagerBuilder.cpp +++ lib/Transforms/IPO/PassManagerBuilder.cpp @@ -637,7 +637,7 @@ // llvm.loop.distribute=true or when -enable-loop-distribute is specified. MPM.add(createLoopDistributePass()); - MPM.add(createLoopVectorizePass(DisableUnrollLoops, LoopVectorize)); + MPM.add(createLoopVectorizePass(DisableUnrollLoops, !LoopVectorize)); // Eliminate loads by forwarding stores from the previous iteration to loads // of the current iteration. @@ -874,7 +874,7 @@ if (!DisableUnrollLoops) PM.add(createSimpleLoopUnrollPass(OptLevel)); // Unroll small loops - PM.add(createLoopVectorizePass(true, LoopVectorize)); + PM.add(createLoopVectorizePass(true, !LoopVectorize)); // The vectorizer may have significantly shortened a loop body; unroll again. if (!DisableUnrollLoops) PM.add(createLoopUnrollPass(OptLevel)); Index: lib/Transforms/Vectorize/LoopVectorizationLegality.cpp =================================================================== --- lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -80,10 +80,11 @@ return false; } -LoopVectorizeHints::LoopVectorizeHints(const Loop *L, bool DisableInterleaving, +LoopVectorizeHints::LoopVectorizeHints(const Loop *L, + bool InterleaveOnlyWhenForced, OptimizationRemarkEmitter &ORE) : Width("vectorize.width", VectorizerParams::VectorizationFactor, HK_WIDTH), - Interleave("interleave.count", DisableInterleaving, HK_UNROLL), + Interleave("interleave.count", InterleaveOnlyWhenForced, HK_UNROLL), Force("vectorize.enable", FK_Undefined, HK_FORCE), IsVectorized("isvectorized", 0, HK_ISVECTORIZED), TheLoop(L), ORE(ORE) { // Populate values with existing loop metadata. @@ -98,19 +99,19 @@ // consider the loop to have been already vectorized because there's // nothing more that we can do. IsVectorized.Value = Width.Value == 1 && Interleave.Value == 1; - LLVM_DEBUG(if (DisableInterleaving && Interleave.Value == 1) dbgs() + LLVM_DEBUG(if (InterleaveOnlyWhenForced && Interleave.Value == 1) dbgs() << "LV: Interleaving disabled by the pass manager\n"); } -bool LoopVectorizeHints::allowVectorization(Function *F, Loop *L, - bool AlwaysVectorize) const { +bool LoopVectorizeHints::allowVectorization( + Function *F, Loop *L, bool VectorizeOnlyWhenForced) const { if (getForce() == LoopVectorizeHints::FK_Disabled) { LLVM_DEBUG(dbgs() << "LV: Not vectorizing: #pragma vectorize disable.\n"); emitRemarkWithHints(); return false; } - if (!AlwaysVectorize && getForce() != LoopVectorizeHints::FK_Enabled) { + if (VectorizeOnlyWhenForced && getForce() != LoopVectorizeHints::FK_Enabled) { LLVM_DEBUG(dbgs() << "LV: Not vectorizing: No #pragma vectorize enable.\n"); emitRemarkWithHints(); return false; Index: lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- lib/Transforms/Vectorize/LoopVectorize.cpp +++ lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1359,7 +1359,8 @@ return false; Function *Fn = OuterLp->getHeader()->getParent(); - if (!Hints.allowVectorization(Fn, OuterLp, false /*AlwaysVectorize*/)) { + if (!Hints.allowVectorization(Fn, OuterLp, + true /*VectorizeOnlyWhenForced*/)) { LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n"); return false; } @@ -1415,10 +1416,11 @@ LoopVectorizePass Impl; - explicit LoopVectorize(bool NoUnrolling = false, bool AlwaysVectorize = true) + explicit LoopVectorize(bool InterleaveOnlyWhenForced = false, + bool VectorizeOnlyWhenForced = false) : FunctionPass(ID) { - Impl.DisableUnrolling = NoUnrolling; - Impl.AlwaysVectorize = AlwaysVectorize; + Impl.InterleaveOnlyWhenForced = InterleaveOnlyWhenForced; + Impl.VectorizeOnlyWhenForced = VectorizeOnlyWhenForced; initializeLoopVectorizePass(*PassRegistry::getPassRegistry()); } @@ -6022,8 +6024,9 @@ namespace llvm { -Pass *createLoopVectorizePass(bool NoUnrolling, bool AlwaysVectorize) { - return new LoopVectorize(NoUnrolling, AlwaysVectorize); +Pass *createLoopVectorizePass(bool InterleaveOnlyWhenForced, + bool VectorizeOnlyWhenForced) { + return new LoopVectorize(InterleaveOnlyWhenForced, VectorizeOnlyWhenForced); } } // end namespace llvm @@ -7141,7 +7144,7 @@ << L->getHeader()->getParent()->getName() << "\" from " << DebugLocStr << "\n"); - LoopVectorizeHints Hints(L, DisableUnrolling, *ORE); + LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE); LLVM_DEBUG( dbgs() << "LV: Loop hints:" @@ -7165,7 +7168,7 @@ // less verbose reporting vectorized loops and unvectorized loops that may // benefit from vectorization, respectively. - if (!Hints.allowVectorization(F, L, AlwaysVectorize)) { + if (!Hints.allowVectorization(F, L, VectorizeOnlyWhenForced)) { LLVM_DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n"); return false; }