diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8577,6 +8577,53 @@ return Region; } +VPRecipeOrVPValueTy +VPRecipeBuilder::createPhiRecipe(PHINode *Phi, ArrayRef Operands, + VFRange &Range, VPlanPtr &Plan) { + if (Phi->getParent() != OrigLoop->getHeader()) + return tryToBlend(Phi, Operands, Plan); + + // Always record recipes for header phis. Later first-order recurrence phis + // can have earlier phis as incoming values. + recordRecipeOf(Phi); + + VPRecipeBase *Recipe; + if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, *Plan, Range))) + return toVPRecipeResult(Recipe); + + VPHeaderPHIRecipe *PhiRecipe = nullptr; + VPValue *StartV = Operands[0]; + if (Legal->isReductionVariable(Phi)) { + const RecurrenceDescriptor &RdxDesc = + Legal->getReductionVars().find(Phi)->second; + assert(RdxDesc.getRecurrenceStartValue() == + Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())); + PhiRecipe = new VPReductionPHIRecipe(Phi, RdxDesc, *StartV, + CM.isInLoopReduction(Phi), + CM.useOrderedReductions(RdxDesc)); + } else if (Legal->isFixedOrderRecurrence(Phi)) { + // TODO: Currently fixed-order recurrences are modeled as chains of + // first-order recurrences. If there are no users of the intermediate + // recurrences in the chain, the fixed order recurrence should be modeled + // directly, enabling more efficient codegen. + PhiRecipe = new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV); + } else { + llvm_unreachable( + "can only widen reductions and fixed-order recurrences here"); + } + + // Record the incoming value from the backedge, so we can add the incoming + // value from the backedge after all recipes have been created. + auto *Inc = cast( + Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch())); + auto RecipeIter = Ingredient2Recipe.find(Inc); + if (RecipeIter == Ingredient2Recipe.end()) + recordRecipeOf(Inc); + + PhisToFix.push_back(PhiRecipe); + return toVPRecipeResult(PhiRecipe); +} + VPRecipeOrVPValueTy VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr, ArrayRef Operands, @@ -8584,51 +8631,11 @@ VPlanPtr &Plan) { // First, check for specific widening recipes that deal with inductions, Phi // nodes, calls and memory operations. - VPRecipeBase *Recipe; - if (auto Phi = dyn_cast(Instr)) { - if (Phi->getParent() != OrigLoop->getHeader()) - return tryToBlend(Phi, Operands, Plan); - - // Always record recipes for header phis. Later first-order recurrence phis - // can have earlier phis as incoming values. - recordRecipeOf(Phi); - - if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, *Plan, Range))) - return toVPRecipeResult(Recipe); - - VPHeaderPHIRecipe *PhiRecipe = nullptr; - assert((Legal->isReductionVariable(Phi) || - Legal->isFixedOrderRecurrence(Phi)) && - "can only widen reductions and fixed-order recurrences here"); - VPValue *StartV = Operands[0]; - if (Legal->isReductionVariable(Phi)) { - const RecurrenceDescriptor &RdxDesc = - Legal->getReductionVars().find(Phi)->second; - assert(RdxDesc.getRecurrenceStartValue() == - Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())); - PhiRecipe = new VPReductionPHIRecipe(Phi, RdxDesc, *StartV, - CM.isInLoopReduction(Phi), - CM.useOrderedReductions(RdxDesc)); - } else { - // TODO: Currently fixed-order recurrences are modeled as chains of - // first-order recurrences. If there are no users of the intermediate - // recurrences in the chain, the fixed order recurrence should be modeled - // directly, enabling more efficient codegen. - PhiRecipe = new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV); - } - - // Record the incoming value from the backedge, so we can add the incoming - // value from the backedge after all recipes have been created. - auto *Inc = cast( - Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch())); - auto RecipeIter = Ingredient2Recipe.find(Inc); - if (RecipeIter == Ingredient2Recipe.end()) - recordRecipeOf(Inc); - - PhisToFix.push_back(PhiRecipe); - return toVPRecipeResult(PhiRecipe); + if (auto *Phi = dyn_cast(Instr)) { + return createPhiRecipe(Phi, Operands, Range, Plan); } + VPRecipeBase *Recipe; if (isa(Instr) && (Recipe = tryToOptimizeInductionTruncate(cast(Instr), Operands, Range, *Plan))) diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -106,6 +106,12 @@ /// Return a VPRecipeOrValueTy with VPRecipeBase * being set. This can be used to force the use as VPRecipeBase* for recipe sub-types that also inherit from VPValue. VPRecipeOrVPValueTy toVPRecipeResult(VPRecipeBase *R) const { return R; } + /// Handle Phi instructions. Return a VPRecipeOrValueTy with VPRecipeBase * + /// being set that describes how to handle the Phi. + VPRecipeOrVPValueTy createPhiRecipe(PHINode *Phi, + ArrayRef Operands, + VFRange &Range, VPlanPtr &Plan); + public: VPRecipeBuilder(Loop *OrigLoop, const TargetLibraryInfo *TLI, LoopVectorizationLegality *Legal,