diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -6931,22 +6931,24 @@ return new VPWidenCallRecipe(*CI, VPValues); } -VPWidenSelectRecipe *VPRecipeBuilder::tryToWidenSelect(Instruction *I, - VFRange &Range) { - auto *SI = dyn_cast(I); - if (!SI) - return nullptr; - - // SI should be widened, unless it is scalar after vectorization, +bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const { + assert(!isa(I) && !isa(I) && !isa(I) && + "Instruction should have been handled earlier"); + // Instruction should be widened, unless it is scalar after vectorization, // scalarization is profitable or it is predicated. - auto willWiden = [this, SI](unsigned VF) -> bool { - return !CM.isScalarAfterVectorization(SI, VF) && - !CM.isProfitableToScalarize(SI, VF) && - !CM.isScalarWithPredication(SI, VF); + auto WillScalarize = [this, I](unsigned VF) -> bool { + return CM.isScalarAfterVectorization(I, VF) || + CM.isProfitableToScalarize(I, VF) || + CM.isScalarWithPredication(I, VF); }; - if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range)) - return nullptr; + return !LoopVectorizationPlanner::getDecisionAndClampRange(WillScalarize, + Range); +} +VPWidenSelectRecipe *VPRecipeBuilder::tryToWidenSelect(Instruction *I) { + auto *SI = dyn_cast(I); + if (!SI) + return nullptr; auto *SE = PSE.getSE(); bool InvariantCond = SE->isLoopInvariant(PSE.getSCEV(SI->getOperand(0)), OrigLoop); @@ -6954,14 +6956,7 @@ return new VPWidenSelectRecipe(*SI, InvariantCond); } -VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range) { - bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange( - [&](unsigned VF) { return CM.isScalarWithPredication(I, VF); }, Range); - - if (IsPredicated) - return nullptr; - - assert(!isa(I) && "PHIs should have been handled earlier"); +VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VPlan &Plan) { auto IsVectorizableOpcode = [](unsigned Opcode) { switch (Opcode) { case Instruction::Add: @@ -7007,14 +7002,6 @@ if (!IsVectorizableOpcode(I->getOpcode())) return nullptr; - auto willWiden = [&](unsigned VF) -> bool { - return !CM.isScalarAfterVectorization(I, VF) && - !CM.isProfitableToScalarize(I, VF); - }; - - if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range)) - return nullptr; - // Success: widen this instruction. return new VPWidenRecipe(*I); } @@ -7095,7 +7082,6 @@ // operations, inductions and Phi nodes. if ((Recipe = tryToWidenCall(Instr, Range, *Plan)) || (Recipe = tryToWidenMemory(Instr, Range, Plan)) || - (Recipe = tryToWidenSelect(Instr, Range)) || (Recipe = tryToOptimizeInduction(Instr, Range)) || (Recipe = tryToBlend(Instr, Plan)) || (isa(Instr) && @@ -7105,24 +7091,19 @@ return true; } - // Handle GEP widening. - if (GetElementPtrInst *GEP = dyn_cast(Instr)) { - auto Scalarize = [&](unsigned VF) { - return CM.isScalarWithPredication(Instr, VF) || - CM.isScalarAfterVectorization(Instr, VF) || - CM.isProfitableToScalarize(Instr, VF); - }; - if (LoopVectorizationPlanner::getDecisionAndClampRange(Scalarize, Range)) - return false; - VPWidenGEPRecipe *Recipe = new VPWidenGEPRecipe(GEP, OrigLoop); - setRecipe(Instr, Recipe); - VPBB->appendRecipe(Recipe); - return true; - } + // Calls and memory instructions are widened by the specialized recipes above, + // or scalarized. + if (isa(Instr) || isa(Instr) || isa(Instr)) + return false; + + if (!shouldWiden(Instr, Range)) + return false; - // Check if Instr is to be widened by a general VPWidenRecipe, after - // having first checked for specific widening recipes. - if ((Recipe = tryToWiden(Instr, Range))) { + if ((Recipe = tryToWidenSelect(Instr)) || + (isa(Instr) && + (Recipe = + new VPWidenGEPRecipe(cast(Instr), OrigLoop))) || + (Recipe = tryToWiden(Instr, *Plan))) { setRecipe(Instr, Recipe); VPBB->appendRecipe(Recipe); return true; diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -64,6 +64,11 @@ Ingredient2Recipe[I] = R; } + /// Check if \p I can be widened at the start of \p Range and possibly + /// decrease the range such that the returned value holds for the entire \p + /// Range. The function should not be called for memory instructions or calls. + bool shouldWiden(Instruction *I, VFRange &Range) const; + public: /// A helper function that computes the predicate of the block BB, assuming /// that the header block of the loop is set to True. It returns the *entry* @@ -114,14 +119,15 @@ /// ensure same decision from \p Range.Start to \p Range.End. VPWidenCallRecipe *tryToWidenCall(Instruction *I, VFRange &Range, VPlan &Plan); - - VPWidenSelectRecipe *tryToWidenSelect(Instruction *I, VFRange &Range); - - /// Check if \p I can be widened within the given VF \p Range. If \p I can be - /// widened for \p Range.Start, build a new VPWidenRecipe and return it. - /// Range.End may be decreased to ensure same decision from \p Range.Start to - /// \p Range.End. - VPWidenRecipe *tryToWiden(Instruction *I, VFRange &Range); + /// Check if \p I is a SelectInst and return a VPWidenSelectRecipe if it is. + /// The function should only be called if the cost-model indicates that + /// widening should be performed. + VPWidenSelectRecipe *tryToWidenSelect(Instruction *I); + + /// Check if \p I has an opcode that can be widened and return a VPWidenRecipe + /// if it can. The function should only be called if the cost-model indicates + /// that widening should be performed. + VPWidenRecipe *tryToWiden(Instruction *I, VPlan &Plan); /// Create a replicating region for instruction \p I that requires /// predication. \p PredRecipe is a VPReplicateRecipe holding \p I.