diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -526,7 +526,8 @@ /// Widen an integer or floating-point induction variable \p IV. If \p Trunc /// is provided, the integer induction variable will first be truncated to /// the corresponding type. - void widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc = nullptr); + void widenIntOrFpInduction(PHINode *IV, Value *Start, + TruncInst *Trunc = nullptr); /// getOrCreateVectorValue and getOrCreateScalarValue coordinate to generate a /// vector or scalar value on-demand if one is not yet available. When @@ -666,7 +667,8 @@ /// truncate instruction, instead of widening the original IV, we widen a /// version of the IV truncated to \p EntryVal's type. void createVectorIntOrFpInductionPHI(const InductionDescriptor &II, - Value *Step, Instruction *EntryVal); + Value *Step, Value *Start, + Instruction *EntryVal); /// Returns true if an instruction \p I should be scalarized instead of /// vectorized for the chosen vectorization factor. @@ -2005,10 +2007,10 @@ } void InnerLoopVectorizer::createVectorIntOrFpInductionPHI( - const InductionDescriptor &II, Value *Step, Instruction *EntryVal) { + const InductionDescriptor &II, Value *Step, Value *Start, + Instruction *EntryVal) { assert((isa(EntryVal) || isa(EntryVal)) && "Expected either an induction phi-node or a truncate of it!"); - Value *Start = II.getStartValue(); // Construct the initial value of the vector IV in the vector loop preheader auto CurrIP = Builder.saveIP(); @@ -2126,7 +2128,8 @@ VectorLoopValueMap.setVectorValue(CastInst, Part, VectorLoopVal); } -void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc) { +void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start, + TruncInst *Trunc) { assert((IV->getType()->isIntegerTy() || IV != OldInduction) && "Primary induction variable must have an integer type"); @@ -2208,7 +2211,7 @@ // least one user in the loop that is not widened. auto NeedsScalarIV = needsScalarInduction(EntryVal); if (!NeedsScalarIV) { - createVectorIntOrFpInductionPHI(ID, Step, EntryVal); + createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal); return; } @@ -2216,7 +2219,7 @@ // create the phi node, we will splat the scalar induction variable in each // loop iteration. if (!shouldScalarizeInstruction(EntryVal)) { - createVectorIntOrFpInductionPHI(ID, Step, EntryVal); + createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal); Value *ScalarIV = CreateScalarIV(Step); // Create scalar steps that can be used by instructions we will later // scalarize. Note that the addition of the scalar steps will not increase @@ -8061,20 +8064,22 @@ } VPWidenIntOrFpInductionRecipe * -VPRecipeBuilder::tryToOptimizeInductionPHI(PHINode *Phi) const { +VPRecipeBuilder::tryToOptimizeInductionPHI(PHINode *Phi, VPlan &Plan) const { // Check if this is an integer or fp induction. If so, build the recipe that // produces its scalar and vector values. InductionDescriptor II = Legal->getInductionVars().lookup(Phi); if (II.getKind() == InductionDescriptor::IK_IntInduction || - II.getKind() == InductionDescriptor::IK_FpInduction) - return new VPWidenIntOrFpInductionRecipe(Phi); + II.getKind() == InductionDescriptor::IK_FpInduction) { + VPValue *Start = Plan.getOrAddVPValue(II.getStartValue()); + return new VPWidenIntOrFpInductionRecipe(Phi, Start); + } return nullptr; } VPWidenIntOrFpInductionRecipe * -VPRecipeBuilder::tryToOptimizeInductionTruncate(TruncInst *I, - VFRange &Range) const { +VPRecipeBuilder::tryToOptimizeInductionTruncate(TruncInst *I, VFRange &Range, + VPlan &Plan) const { // Optimize the special case where the source is a constant integer // induction variable. Notice that we can only optimize the 'trunc' case // because (a) FP conversions lose precision, (b) sext/zext may wrap, and @@ -8090,9 +8095,14 @@ }; if (LoopVectorizationPlanner::getDecisionAndClampRange( - isOptimizableIVTruncate(I), Range)) + isOptimizableIVTruncate(I), Range)) { + + InductionDescriptor II = + Legal->getInductionVars().lookup(cast(I->getOperand(0))); + VPValue *Start = Plan.getOrAddVPValue(II.getStartValue()); return new VPWidenIntOrFpInductionRecipe(cast(I->getOperand(0)), - I); + Start, I); + } return nullptr; } @@ -8304,13 +8314,13 @@ if (auto Phi = dyn_cast(Instr)) { if (Phi->getParent() != OrigLoop->getHeader()) return tryToBlend(Phi, Plan); - if ((Recipe = tryToOptimizeInductionPHI(Phi))) + if ((Recipe = tryToOptimizeInductionPHI(Phi, *Plan))) return Recipe; return new VPWidenPHIRecipe(Phi); } - if (isa(Instr) && - (Recipe = tryToOptimizeInductionTruncate(cast(Instr), Range))) + if (isa(Instr) && (Recipe = tryToOptimizeInductionTruncate( + cast(Instr), Range, *Plan))) return Recipe; if (!shouldWiden(Instr, Range)) @@ -8705,7 +8715,8 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) { assert(!State.Instance && "Int or FP induction being replicated."); - State.ILV->widenIntOrFpInduction(IV, Trunc); + State.ILV->widenIntOrFpInduction(IV, getStartValue()->getLiveInIRValue(), + Trunc); } void VPWidenPHIRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -66,12 +66,14 @@ /// Check if an induction recipe should be constructed for \I. If so build and /// return it. If not, return null. - VPWidenIntOrFpInductionRecipe *tryToOptimizeInductionPHI(PHINode *Phi) const; + VPWidenIntOrFpInductionRecipe *tryToOptimizeInductionPHI(PHINode *Phi, + VPlan &Plan) const; /// Optimize the special case where the operand of \p I is a constant integer /// induction variable. VPWidenIntOrFpInductionRecipe * - tryToOptimizeInductionTruncate(TruncInst *I, VFRange &Range) const; + tryToOptimizeInductionTruncate(TruncInst *I, VFRange &Range, + VPlan &Plan) const; /// Handle non-loop phi nodes. Currently all such phi nodes are turned into /// a sequence of select instructions as the vectorizer currently performs diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -914,13 +914,15 @@ /// A recipe for handling phi nodes of integer and floating-point inductions, /// producing their vector and scalar values. -class VPWidenIntOrFpInductionRecipe : public VPRecipeBase { +class VPWidenIntOrFpInductionRecipe : public VPRecipeBase, public VPUser { PHINode *IV; TruncInst *Trunc; public: - VPWidenIntOrFpInductionRecipe(PHINode *IV, TruncInst *Trunc = nullptr) - : VPRecipeBase(VPWidenIntOrFpInductionSC), IV(IV), Trunc(Trunc) { + VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, + TruncInst *Trunc = nullptr) + : VPRecipeBase(VPWidenIntOrFpInductionSC), VPUser({Start}), IV(IV), + Trunc(Trunc) { if (Trunc) new VPValue(Trunc, this); else @@ -940,6 +942,8 @@ /// Print the recipe. void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override; + + VPValue *getStartValue() { return getOperand(0); } }; /// A recipe for handling all phi nodes except for integer and FP inductions. diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -68,7 +68,9 @@ InductionDescriptor II = Inductions.lookup(Phi); if (II.getKind() == InductionDescriptor::IK_IntInduction || II.getKind() == InductionDescriptor::IK_FpInduction) { - NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi); + + VPValue *Start = Plan->getOrAddVPValue(II.getStartValue()); + NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start); } else NewRecipe = new VPWidenPHIRecipe(Phi); } else if (GetElementPtrInst *GEP = dyn_cast(Inst)) {