diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8313,7 +8313,9 @@ if (Legal->isReductionVariable(Phi)) { RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi]; - return new VPWidenPHIRecipe(Phi, RdxDesc); + VPValue *StartV = + Plan->getOrAddVPValue(RdxDesc.getRecurrenceStartValue()); + return new VPWidenPHIRecipe(Phi, RdxDesc, *StartV); } return new VPWidenPHIRecipe(Phi); @@ -8720,9 +8722,8 @@ } void VPWidenPHIRecipe::execute(VPTransformState &State) { - Value *StartV = nullptr; - if (RdxDesc) - StartV = RdxDesc->getRecurrenceStartValue(); + Value *StartV = + getStartValue() ? getStartValue()->getLiveInIRValue() : nullptr; State.ILV->widenPHIInstruction(Phi, RdxDesc, StartV, State.UF, State.VF); } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -948,8 +948,9 @@ }; /// A recipe for handling all phi nodes except for integer and FP inductions. -/// For reduction PHIs, RdxDesc needs to be non-null -class VPWidenPHIRecipe : public VPRecipeBase { +/// For reduction PHIs, RdxDesc needs to be non-null and the start value is the +/// first operand of the recipe. +class VPWidenPHIRecipe : public VPRecipeBase, public VPUser { PHINode *Phi; /// Descriptor for a reduction PHI. @@ -958,9 +959,10 @@ public: /// Create a new VPWidenPHIRecipe for the reduction \p Phi described by \p /// RdxDesc. - VPWidenPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc) + VPWidenPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc, VPValue &Start) : VPWidenPHIRecipe(Phi) { this->RdxDesc = &RdxDesc; + addOperand(&Start); } /// Create a VPWidenPHIRecipe for \p Phi @@ -980,6 +982,11 @@ /// Print the recipe. void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override; + + /// Returns the start value of the phi, if it is a reduction. + VPValue *getStartValue() { + return getNumOperands() == 0 ? nullptr : getOperand(0); + } }; /// A recipe for vectorizing a phi-node as a sequence of mask-based select