[VPlan] Keep start value of VPWidenPHIRecipe as VPValue.

[fhahn]

Summary

Similar to D92129, update VPWidenPHIRecipe to manage the start value as
VPValue. This allows adjusting the start value as a VPlan transform,
which will be used in a follow-up patch to support reductions during
epilogue vectorization.

[fhahn]

Update after renaming to getLiveInIRValue.

Also updated documentation for VPWidenPHIRecipe to describe when the start value is managed as first operand.

[gilr]

I wonder if the code from fixReductions() could be moved (almost as-is) to widenPHIInstruction() as a first step?

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

4589

RK seems to be the only use of RdxDesc. Can be extracted in tryToCreateWidenRecipe() and stored as an ingredient in the recipe.

4593

Intentionally assigning StartV to itself? (symmetry with the else clause?)

[fhahn]

In D93975#2480166, @gilr wrote:

I wonder if the code from fixReductions() could be moved (almost as-is) to widenPHIInstruction() as a first step?

Thanks for the suggestion, I put up D94175 to move the code and keep RdxDesc in the recipe as a first step. The current patch is much simpler now.

[fhahn]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

4593

It's not needed. (can. be Iden = StartV;). Applied to D94175.

[gilr]

LGTM!

[fhahn]

rebase after parent patch landed.

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

if (Legal->isReductionVariable(P) || Legal->isFirstOrderRecurrence(P)) {
     Value *Iden = nullptr;
     bool ScalarPHI =
         (VF.isScalar()) || Cost->isInLoopReduction(cast<PHINode>(PN));
     Type *VecTy =
         ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), VF);
 
     if (RdxDesc) {
       RecurKind RK = RdxDesc->getRecurrenceKind();
       if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
         // MinMax reduction have the start value as their identify.
         if (ScalarPHI) {
           StartV = Iden = StartV;
         } else {
           IRBuilderBase::InsertPointGuard IPBuilder(Builder);
           Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
           StartV = Iden = Builder.CreateVectorSplat(VF, StartV, "minmax.ident");
         }
       } else {
         Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity(
             RK, VecTy->getScalarType());
         Iden = IdenC;
VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
   if (auto Phi = dyn_cast<PHINode>(Instr)) {
     if (Phi->getParent() != OrigLoop->getHeader())
       return tryToBlend(Phi, Plan);
     if ((Recipe = tryToOptimizeInductionPHI(Phi, *Plan)))
       return Recipe;
 
     if (Legal->isReductionVariable(Phi)) {
       RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi];
-      return new VPWidenPHIRecipe(Phi, RdxDesc);
+      VPValue *StartV =
+          Plan->getOrAddVPValue(RdxDesc.getRecurrenceStartValue());
+      return new VPWidenPHIRecipe(Phi, RdxDesc, *StartV);
     }
 
     return new VPWidenPHIRecipe(Phi);
   }
 
   if (isa<TruncInst>(Instr) && (Recipe = tryToOptimizeInductionTruncate(
                                     cast<TruncInst>(Instr), Range, *Plan)))
     return Recipe;

 
 void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
   assert(!State.Instance && "Int or FP induction being replicated.");
   State.ILV->widenIntOrFpInduction(IV, getStartValue()->getLiveInIRValue(),
                                    Trunc);
 }
 
 void VPWidenPHIRecipe::execute(VPTransformState &State) {
-  Value *StartV = nullptr;
-  if (RdxDesc)
-    StartV = RdxDesc->getRecurrenceStartValue();
+  Value *StartV =
+      getStartValue() ? getStartValue()->getLiveInIRValue() : nullptr;
   State.ILV->widenPHIInstruction(Phi, RdxDesc, StartV, State.UF, State.VF);
 }
 
 void VPBlendRecipe::execute(VPTransformState &State) {
   State.ILV->setDebugLocFromInst(State.Builder, Phi);
   // We know that all PHIs in non-header blocks are converted into
   // selects, so we don't have to worry about the insertion order and we
   // can just use the builder.

llvm/lib/Transforms/Vectorize/VPlan.h

public:
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
 
   /// Returns the start value of the induction.
   VPValue *getStartValue() { return getOperand(0); }
 };
 
 /// A recipe for handling all phi nodes except for integer and FP inductions.
-/// For reduction PHIs, RdxDesc needs to be non-null
-class VPWidenPHIRecipe : public VPRecipeBase {
+/// For reduction PHIs, RdxDesc needs to be non-null and the start value is the
+/// first operand of the recipe.
+class VPWidenPHIRecipe : public VPRecipeBase, public VPUser {
   PHINode *Phi;
 
   /// Descriptor for a reduction PHI.
   RecurrenceDescriptor *RdxDesc = nullptr;
 
 public:
   /// Create a new VPWidenPHIRecipe for the reduction \p Phi described by \p
   /// RdxDesc.
-  VPWidenPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc)
+  VPWidenPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc, VPValue &Start)
       : VPWidenPHIRecipe(Phi) {
     this->RdxDesc = &RdxDesc;
+    addOperand(&Start);
   }
 
   /// Create a VPWidenPHIRecipe for \p Phi
   VPWidenPHIRecipe(PHINode *Phi) : VPRecipeBase(VPWidenPHISC), Phi(Phi) {
     new VPValue(Phi, this);
   }
   ~VPWidenPHIRecipe() override = default;
 
   /// Method to support type inquiry through isa, cast, and dyn_cast.
   static inline bool classof(const VPDef *D) {
     return D->getVPDefID() == VPRecipeBase::VPWidenPHISC;
   }
 
   /// Generate the phi/select nodes.
   void execute(VPTransformState &State) override;
 
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
+
+  /// Returns the start value of the phi, if it is a reduction.
+  VPValue *getStartValue() {
+    return getNumOperands() == 0 ? nullptr : getOperand(0);
+  }
 };
 
 /// A recipe for vectorizing a phi-node as a sequence of mask-based select
 /// instructions.
 class VPBlendRecipe : public VPRecipeBase, public VPUser {
   PHINode *Phi;
 
 public: