diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -582,7 +582,8 @@ /// Create the exit value of first order recurrences in the middle block and /// update their users. - void fixFirstOrderRecurrence(VPWidenPHIRecipe *PhiR, VPTransformState &State); + void fixFirstOrderRecurrence(VPFirstOrderRecurrencePHIRecipe *PhiR, + VPTransformState &State); /// Create code for the loop exit value of the reduction. void fixReduction(VPReductionPHIRecipe *Phi, VPTransformState &State); @@ -4096,8 +4097,8 @@ } } -void InnerLoopVectorizer::fixFirstOrderRecurrence(VPWidenPHIRecipe *PhiR, - VPTransformState &State) { +void InnerLoopVectorizer::fixFirstOrderRecurrence( + VPFirstOrderRecurrencePHIRecipe *PhiR, VPTransformState &State) { // This is the second phase of vectorizing first-order recurrences. An // overview of the transformation is described below. Suppose we have the // following loop. @@ -8712,7 +8713,7 @@ void VPRecipeBuilder::fixHeaderPhis() { BasicBlock *OrigLatch = OrigLoop->getLoopLatch(); - for (VPWidenPHIRecipe *R : PhisToFix) { + for (VPHeaderPHIRecipe *R : PhisToFix) { auto *PN = cast(R->getUnderlyingValue()); VPRecipeBase *IncR = getRecipe(cast(PN->getIncomingValueForBlock(OrigLatch))); @@ -8854,7 +8855,7 @@ if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands))) return toVPRecipeResult(Recipe); - VPWidenPHIRecipe *PhiRecipe = nullptr; + VPHeaderPHIRecipe *PhiRecipe = nullptr; if (Legal->isReductionVariable(Phi) || Legal->isFirstOrderRecurrence(Phi)) { VPValue *StartV = Operands[0]; if (Legal->isReductionVariable(Phi)) { diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -59,7 +59,7 @@ /// Cross-iteration reduction & first-order recurrence phis for which we need /// to add the incoming value from the backedge after all recipes have been /// created. - SmallVector PhisToFix; + SmallVector PhisToFix; /// Check if \p I can be widened at the start of \p Range and possibly /// decrease the range such that the returned value holds for the entire \p diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1059,34 +1059,25 @@ const InductionDescriptor &getInductionDescriptor() const { return IndDesc; } }; -/// A recipe for handling first order recurrences and pointer inductions. For -/// first-order recurrences, the start value is the first operand of the recipe -/// and the incoming value from the backedge is the second operand. It also -/// serves as base class for VPReductionPHIRecipe. In the VPlan native path, all -/// incoming VPValues & VPBasicBlock pairs are managed in the recipe directly. -class VPWidenPHIRecipe : public VPRecipeBase, public VPValue { +/// A base class for all recipes modeling header phis, including phis for first +/// order recurrences, pointer inductions.and reductions. The start value is the +/// first operand of the recipe and the incoming value from the backedge is the +/// second operand. In the VPlan native path, all incoming VPValues & +/// VPBasicBlock pairs are managed in the recipe directly. +class VPHeaderPHIRecipe : public VPRecipeBase, public VPValue { /// List of incoming blocks. Only used in the VPlan native path. SmallVector IncomingBlocks; protected: - VPWidenPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi, - VPValue *Start = nullptr) + VPHeaderPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi, + VPValue *Start = nullptr) : VPRecipeBase(VPDefID, {}), VPValue(VPVID, Phi, this) { if (Start) addOperand(Start); } public: - /// Create a VPWidenPHIRecipe for \p Phi - VPWidenPHIRecipe(PHINode *Phi) - : VPWidenPHIRecipe(VPVWidenPHISC, VPWidenPHISC, Phi) {} - - /// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start. - VPWidenPHIRecipe(PHINode *Phi, VPValue &Start) : VPWidenPHIRecipe(Phi) { - addOperand(&Start); - } - - ~VPWidenPHIRecipe() override = default; + ~VPHeaderPHIRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *B) { @@ -1101,22 +1092,20 @@ } /// Generate the phi/select nodes. - void execute(VPTransformState &State) override; + void execute(VPTransformState &State) override = 0; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// Print the recipe. void print(raw_ostream &O, const Twine &Indent, - VPSlotTracker &SlotTracker) const override; + VPSlotTracker &SlotTracker) const override = 0; #endif - /// Returns the start value of the phi, if it is a reduction or first-order - /// recurrence. + /// Returns the start value of the phi, it one is set. VPValue *getStartValue() { return getNumOperands() == 0 ? nullptr : getOperand(0); } - /// Returns the incoming value from the loop backedge, if it is a reduction or - /// first-order recurrence. + /// Returns the incoming value from the loop backedge, if there is one. VPValue *getBackedgeValue() { return getOperand(1); } @@ -1140,13 +1129,44 @@ VPBasicBlock *getIncomingBlock(unsigned I) { return IncomingBlocks[I]; } }; +/// A recipe for handling header phis that are widened in the vector loop. +struct VPWidenPHIRecipe : public VPHeaderPHIRecipe { + /// Create a VPWidenPHIRecipe for \p Phi + VPWidenPHIRecipe(PHINode *Phi) + : VPHeaderPHIRecipe(VPVWidenPHISC, VPWidenPHISC, Phi) {} + + /// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start. + VPWidenPHIRecipe(PHINode *Phi, VPValue &Start) : VPWidenPHIRecipe(Phi) { + addOperand(&Start); + } + + ~VPWidenPHIRecipe() override = default; + + /// Method to support type inquiry through isa, cast, and dyn_cast. + static inline bool classof(const VPRecipeBase *B) { + return B->getVPDefID() == VPRecipeBase::VPWidenPHISC; + } + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPValue::VPVWidenPHISC; + } + + /// Generate the phi/select nodes. + void execute(VPTransformState &State) override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + /// Print the recipe. + void print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const override; +#endif +}; + /// A recipe for handling first-order recurrence phis. The start value is the /// first operand of the recipe and the incoming value from the backedge is the /// second operand. -struct VPFirstOrderRecurrencePHIRecipe : public VPWidenPHIRecipe { +struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe { VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start) - : VPWidenPHIRecipe(VPVFirstOrderRecurrencePHISC, - VPFirstOrderRecurrencePHISC, Phi, &Start) {} + : VPHeaderPHIRecipe(VPVFirstOrderRecurrencePHISC, + VPFirstOrderRecurrencePHISC, Phi, &Start) {} /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *R) { @@ -1171,7 +1191,7 @@ /// A recipe for handling reduction phis. The start value is the first operand /// of the recipe and the incoming value from the backedge is the second /// operand. -class VPReductionPHIRecipe : public VPWidenPHIRecipe { +class VPReductionPHIRecipe : public VPHeaderPHIRecipe { /// Descriptor for the reduction. const RecurrenceDescriptor &RdxDesc; @@ -1187,7 +1207,7 @@ VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc, VPValue &Start, bool IsInLoop = false, bool IsOrdered = false) - : VPWidenPHIRecipe(VPVReductionPHISC, VPReductionPHISC, Phi, &Start), + : VPHeaderPHIRecipe(VPVReductionPHISC, VPReductionPHISC, Phi, &Start), RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) { assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop"); } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -838,7 +838,7 @@ // vector loop. VPBasicBlock *Header = Entry->getEntryBasicBlock(); for (VPRecipeBase &R : Header->phis()) { - auto *PhiR = dyn_cast(&R); + auto *PhiR = dyn_cast(&R); if (!PhiR || !(isa(&R) || isa(&R))) continue; @@ -1461,7 +1461,7 @@ InterleavedAccessInfo &IAI) { if (VPBasicBlock *VPBB = dyn_cast(Block)) { for (VPRecipeBase &VPI : *VPBB) { - if (isa(&VPI)) + if (isa(&VPI)) continue; assert(isa(&VPI) && "Can only handle VPInstructions"); auto *VPInst = cast(&VPI);