diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -6936,20 +6936,23 @@ return new VPWidenCallRecipe(*CI, VPValues); } +bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const { + assert(!isa(I) && !isa(I) && !isa(I) && + "Called with unsupported instruction"); + // Instruction should be widened, unless it is scalar after vectorization, + // scalarization is profitable or it is predicated. + auto willWiden = [this, I](unsigned VF) -> bool { + return !CM.isScalarAfterVectorization(I, VF) && + !CM.isProfitableToScalarize(I, VF) && + !CM.isScalarWithPredication(I, VF); + }; + return LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range); +} + VPWidenSelectRecipe *VPRecipeBuilder::tryToWidenSelect(Instruction *I, VFRange &Range) { auto *SI = dyn_cast(I); - if (!SI) - return nullptr; - - // SI should be widened, unless it is scalar after vectorization, - // scalarization is profitable or it is predicated. - auto willWiden = [this, SI](unsigned VF) -> bool { - return !CM.isScalarAfterVectorization(SI, VF) && - !CM.isProfitableToScalarize(SI, VF) && - !CM.isScalarWithPredication(SI, VF); - }; - if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range)) + if (!SI || !shouldWiden(I, Range)) return nullptr; auto *SE = PSE.getSE(); @@ -6961,12 +6964,6 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range, VPlan &Plan) { - bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange( - [&](unsigned VF) { return CM.isScalarWithPredication(I, VF); }, Range); - - if (IsPredicated) - return nullptr; - auto IsVectorizableOpcode = [](unsigned Opcode) { switch (Opcode) { case Instruction::Add: @@ -7010,17 +7007,7 @@ return false; }; - if (!IsVectorizableOpcode(I->getOpcode())) - return nullptr; - - auto willWiden = [&](unsigned VF) -> bool { - if (!isa(I) && (CM.isScalarAfterVectorization(I, VF) || - CM.isProfitableToScalarize(I, VF))) - return false; - return true; - }; - - if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range)) + if (!IsVectorizableOpcode(I->getOpcode()) || !shouldWiden(I, Range)) return nullptr; // Success: widen this instruction. diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -64,6 +64,11 @@ Ingredient2Recipe[I] = R; } + /// Check if \p I can be widened at the start of \p Range and possibly + /// decrease the range such that the returned value holds for the entire \p + /// Range. The function should not be called for memory instructions or calls. + bool shouldWiden(Instruction *I, VFRange &Range) const; + public: /// A helper function that computes the predicate of the block BB, assuming /// that the header block of the loop is set to True. It returns the *entry*