diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -171,6 +171,9 @@ unsigned Width; // Cost of the loop with that width unsigned Cost; + + // Width 1 means no vectorization, cost 0 means uncomputed cost. + static VectorizationFactor Disabled() { return {1, 0}; } }; /// Planner drives the vectorization process after having passed @@ -221,8 +224,9 @@ LoopVectorizationCostModel &CM) : OrigLoop(L), LI(LI), TLI(TLI), TTI(TTI), Legal(Legal), CM(CM) {} - /// Plan how to best vectorize, return the best VF and its cost. - VectorizationFactor plan(bool OptForSize, unsigned UserVF); + /// Plan how to best vectorize, return the best VF and its cost, or None if + /// vectorization and interleaving should be avoided up front. + Optional plan(bool OptForSize, unsigned UserVF); /// Use the VPlan-native path to plan how to best vectorize, return the best /// VF and its cost. diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -843,7 +843,7 @@ AC(AC), ORE(ORE), TheFunction(F), Hints(Hints), InterleaveInfo(IAI) {} /// \return An upper bound for the vectorization factor, or None if - /// vectorization should be avoided up front. + /// vectorization and interleaving should be avoided up front. Optional computeMaxVF(bool OptForSize); /// \return The most profitable vectorization factor and the cost of that VF. @@ -6079,9 +6079,6 @@ VectorizationFactor LoopVectorizationPlanner::planInVPlanNativePath(bool OptForSize, unsigned UserVF) { - // Width 1 means no vectorization, cost 0 means uncomputed cost. - const VectorizationFactor NoVectorization = {1U, 0U}; - // Outer loop handling: They may require CFG and instruction level // transformations before even evaluating whether vectorization is profitable. // Since we cannot modify the incoming IR, we need to build VPlan upfront in @@ -6101,7 +6098,7 @@ // For VPlan build stress testing, we bail out after VPlan construction. if (VPlanBuildStressTest) - return NoVectorization; + return VectorizationFactor::Disabled(); return {UserVF, 0}; } @@ -6109,17 +6106,15 @@ LLVM_DEBUG( dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the " "VPlan-native path.\n"); - return NoVectorization; + return VectorizationFactor::Disabled(); } -VectorizationFactor -LoopVectorizationPlanner::plan(bool OptForSize, unsigned UserVF) { +Optional LoopVectorizationPlanner::plan(bool OptForSize, + unsigned UserVF) { assert(OrigLoop->empty() && "Inner loop expected."); - // Width 1 means no vectorization, cost 0 means uncomputed cost. - const VectorizationFactor NoVectorization = {1U, 0U}; Optional MaybeMaxVF = CM.computeMaxVF(OptForSize); if (!MaybeMaxVF.hasValue()) // Cases considered too costly to vectorize. - return NoVectorization; + return None; // Invalidate interleave groups if all blocks of loop will be predicated. if (CM.blockNeedsPredication(OrigLoop->getHeader()) && @@ -6139,7 +6134,7 @@ CM.selectUserVectorizationFactor(UserVF); buildVPlansWithVPRecipes(UserVF, UserVF); LLVM_DEBUG(printPlans(dbgs())); - return {UserVF, 0}; + return {{UserVF, 0}}; } unsigned MaxVF = MaybeMaxVF.getValue(); @@ -6158,7 +6153,7 @@ buildVPlansWithVPRecipes(1, MaxVF); LLVM_DEBUG(printPlans(dbgs())); if (MaxVF == 1) - return NoVectorization; + return VectorizationFactor::Disabled(); // Select the optimal vectorization factor. return CM.selectVectorizationFactor(MaxVF); @@ -7322,58 +7317,65 @@ // Get user vectorization factor. unsigned UserVF = Hints.getWidth(); - // Plan how to best vectorize, return the best VF and its cost. - VectorizationFactor VF = LVP.plan(OptForSize, UserVF); - - // Select the interleave count. - unsigned IC = CM.selectInterleaveCount(OptForSize, VF.Width, VF.Cost); - - // Get user interleave count. - unsigned UserIC = Hints.getInterleave(); + VectorizationFactor VF = VectorizationFactor::Disabled(); + unsigned IC = 1; // Identify the diagnostic messages that should be produced. std::pair VecDiagMsg, IntDiagMsg; - bool VectorizeLoop = true, InterleaveLoop = true; - if (Requirements.doesNotMeet(F, L, Hints)) { - LLVM_DEBUG(dbgs() << "LV: Not vectorizing: loop did not meet vectorization " - "requirements.\n"); - Hints.emitRemarkWithHints(); - return false; - } + bool VectorizeLoop = false, InterleaveLoop = false; + + // Plan how to best vectorize, return the best VF and its cost. + Optional MaybeVF = LVP.plan(OptForSize, UserVF); + if (MaybeVF) { + VF = *MaybeVF; + IC = CM.selectInterleaveCount(OptForSize, VF.Width, VF.Cost); - if (VF.Width == 1) { - LLVM_DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n"); - VecDiagMsg = std::make_pair( - "VectorizationNotBeneficial", - "the cost-model indicates that vectorization is not beneficial"); - VectorizeLoop = false; - } - - if (IC == 1 && UserIC <= 1) { - // Tell the user interleaving is not beneficial. - LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n"); - IntDiagMsg = std::make_pair( - "InterleavingNotBeneficial", - "the cost-model indicates that interleaving is not beneficial"); - InterleaveLoop = false; - if (UserIC == 1) { - IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled"; - IntDiagMsg.second += - " and is explicitly disabled or interleave count is set to 1"; + if (Requirements.doesNotMeet(F, L, Hints)) { + LLVM_DEBUG( + dbgs() << "LV: Not vectorizing: loop did not meet vectorization " + "requirements.\n"); + Hints.emitRemarkWithHints(); + return false; } - } else if (IC > 1 && UserIC == 1) { - // Tell the user interleaving is beneficial, but it explicitly disabled. - LLVM_DEBUG( - dbgs() << "LV: Interleaving is beneficial but is explicitly disabled."); - IntDiagMsg = std::make_pair( - "InterleavingBeneficialButDisabled", - "the cost-model indicates that interleaving is beneficial " - "but is explicitly disabled or interleave count is set to 1"); - InterleaveLoop = false; - } - // Override IC if user provided an interleave count. - IC = UserIC > 0 ? UserIC : IC; + // Check if loop should be vectorized and generate diagnostics. + if (VF.Width == 1) { + LLVM_DEBUG( + dbgs() << "LV: Vectorization is possible but not beneficial.\n"); + VecDiagMsg = std::make_pair( + "VectorizationNotBeneficial", + "the cost-model indicates that vectorization is not beneficial"); + } else + VectorizeLoop = true; + + // Check if loop should be interleaved and generate diagnostics. + unsigned UserIC = Hints.getInterleave(); // Get user interleave count. + if (IC == 1 && UserIC <= 1) { + // Tell the user interleaving is not beneficial. + LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n"); + IntDiagMsg = std::make_pair( + "InterleavingNotBeneficial", + "the cost-model indicates that interleaving is not beneficial"); + if (UserIC == 1) { + IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled"; + IntDiagMsg.second += + " and is explicitly disabled or interleave count is set to 1"; + } + } else if (IC > 1 && UserIC == 1) { + // Tell the user interleaving is beneficial, but it explicitly disabled. + LLVM_DEBUG( + dbgs() + << "LV: Interleaving is beneficial but is explicitly disabled."); + IntDiagMsg = std::make_pair( + "InterleavingBeneficialButDisabled", + "the cost-model indicates that interleaving is beneficial " + "but is explicitly disabled or interleave count is set to 1"); + } else + InterleaveLoop = true; + + // Override IC if user provided an interleave count. + IC = UserIC > 0 ? UserIC : IC; + } // Emit diagnostic messages, if any. const char *VAPassName = Hints.vectorizeAnalysisPassName(); diff --git a/llvm/test/Transforms/LoopVectorize/no-interleave-up-front.ll b/llvm/test/Transforms/LoopVectorize/no-interleave-up-front.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/no-interleave-up-front.ll @@ -0,0 +1,35 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -loop-vectorize -S | FileCheck %s + +; Test case based on reproducer for +; http://bugs.chromium.org/p/oss-fuzz/issues/detail?id=6477 + +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" + +define void @test1(i32 %n) #0 { +; CHECK-LABEL: @test1( +; CHECK-NEXT: br i1 false, label [[DOTLR_PH_PREHEADER:%.*]], label [[DOT_CRIT_EDGE:%.*]] +; CHECK: .lr.ph.preheader: +; CHECK-NEXT: br label [[DOTLR_PH:%.*]] +; CHECK: .lr.ph: +; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ 0, [[DOTLR_PH_PREHEADER]] ] +; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1 +; CHECK-NEXT: br i1 true, label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[DOTLR_PH]], !llvm.loop !0 +; CHECK: ._crit_edge.loopexit: +; CHECK-NEXT: br label [[DOT_CRIT_EDGE]] +; CHECK: ._crit_edge: +; CHECK-NEXT: ret void +; + br i1 false, label %.lr.ph, label %._crit_edge + +.lr.ph: ; preds = %.lr.ph, %0 + %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ] + %indvars.iv.next = add i64 %indvars.iv, 1 + br i1 true, label %._crit_edge, label %.lr.ph, !llvm.loop !0 + +._crit_edge: ; preds = %.lr.ph, %0 + ret void +} + +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.interleave.count", i32 2}