diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1021,78 +1021,6 @@ #endif }; -/// A recipe for handling phi nodes of integer and floating-point inductions, -/// producing their vector values. -class VPWidenIntOrFpInductionRecipe : public VPRecipeBase, public VPValue { - PHINode *IV; - const InductionDescriptor &IndDesc; - bool NeedsVectorIV; - -public: - VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, - const InductionDescriptor &IndDesc, - bool NeedsVectorIV) - : VPRecipeBase(VPDef::VPWidenIntOrFpInductionSC, {Start, Step}), - VPValue(this, IV), IV(IV), IndDesc(IndDesc), - NeedsVectorIV(NeedsVectorIV) {} - - VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, - const InductionDescriptor &IndDesc, - TruncInst *Trunc, bool NeedsVectorIV) - : VPRecipeBase(VPDef::VPWidenIntOrFpInductionSC, {Start, Step}), - VPValue(this, Trunc), IV(IV), IndDesc(IndDesc), - NeedsVectorIV(NeedsVectorIV) {} - - ~VPWidenIntOrFpInductionRecipe() override = default; - - VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionSC) - - /// Generate the vectorized and scalarized versions of the phi node as - /// needed by their users. - void execute(VPTransformState &State) override; - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - /// Print the recipe. - void print(raw_ostream &O, const Twine &Indent, - VPSlotTracker &SlotTracker) const override; -#endif - - /// Returns the start value of the induction. - VPValue *getStartValue() { return getOperand(0); } - const VPValue *getStartValue() const { return getOperand(0); } - - /// Returns the step value of the induction. - VPValue *getStepValue() { return getOperand(1); } - const VPValue *getStepValue() const { return getOperand(1); } - - /// Returns the first defined value as TruncInst, if it is one or nullptr - /// otherwise. - TruncInst *getTruncInst() { - return dyn_cast_or_null(getVPValue(0)->getUnderlyingValue()); - } - const TruncInst *getTruncInst() const { - return dyn_cast_or_null(getVPValue(0)->getUnderlyingValue()); - } - - PHINode *getPHINode() { return IV; } - - /// Returns the induction descriptor for the recipe. - const InductionDescriptor &getInductionDescriptor() const { return IndDesc; } - - /// Returns true if the induction is canonical, i.e. starting at 0 and - /// incremented by UF * VF (= the original IV is incremented by 1). - bool isCanonical() const; - - /// Returns the scalar type of the induction. - const Type *getScalarType() const { - const TruncInst *TruncI = getTruncInst(); - return TruncI ? TruncI->getType() : IV->getType(); - } - - /// Returns true if a vector phi needs to be created for the induction. - bool needsVectorIV() const { return NeedsVectorIV; } -}; - /// A pure virtual base class for all recipes modeling header phis, including /// phis for first order recurrences, pointer inductions and reductions. The /// start value is the first operand of the recipe and the incoming value from @@ -1118,9 +1046,9 @@ /// per-lane based on the canonical induction. class VPHeaderPHIRecipe : public VPRecipeBase, public VPValue { protected: - VPHeaderPHIRecipe(unsigned char VPDefID, PHINode *Phi, + VPHeaderPHIRecipe(unsigned char VPDefID, Instruction *UnderlyingInstr, VPValue *Start = nullptr) - : VPRecipeBase(VPDefID, {}), VPValue(this, Phi) { + : VPRecipeBase(VPDefID, {}), VPValue(this, UnderlyingInstr) { if (Start) addOperand(Start); } @@ -1131,12 +1059,12 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *B) { return B->getVPDefID() >= VPDef::VPFirstHeaderPHISC && - B->getVPDefID() <= VPDef::VPLastPHISC; + B->getVPDefID() <= VPDef::VPLastHeaderPHISC; } static inline bool classof(const VPValue *V) { auto *B = V->getDefiningRecipe(); return B && B->getVPDefID() >= VPRecipeBase::VPFirstHeaderPHISC && - B->getVPDefID() <= VPRecipeBase::VPLastPHISC; + B->getVPDefID() <= VPRecipeBase::VPLastHeaderPHISC; } /// Generate the phi nodes. @@ -1160,17 +1088,93 @@ void setStartValue(VPValue *V) { setOperand(0, V); } /// Returns the incoming value from the loop backedge. - VPValue *getBackedgeValue() { + virtual VPValue *getBackedgeValue() { return getOperand(1); } /// Returns the backedge value as a recipe. The backedge value is guaranteed /// to be a recipe. - VPRecipeBase &getBackedgeRecipe() { + virtual VPRecipeBase &getBackedgeRecipe() { return *getBackedgeValue()->getDefiningRecipe(); } }; +/// A recipe for handling phi nodes of integer and floating-point inductions, +/// producing their vector values. +class VPWidenIntOrFpInductionRecipe : public VPHeaderPHIRecipe { + PHINode *IV; + TruncInst *Trunc; + const InductionDescriptor &IndDesc; + bool NeedsVectorIV; + +public: + VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, + const InductionDescriptor &IndDesc, + bool NeedsVectorIV) + : VPHeaderPHIRecipe(VPDef::VPWidenIntOrFpInductionSC, IV, Start), IV(IV), + Trunc(nullptr), IndDesc(IndDesc), NeedsVectorIV(NeedsVectorIV) { + addOperand(Step); + } + + VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, + const InductionDescriptor &IndDesc, + TruncInst *Trunc, bool NeedsVectorIV) + : VPHeaderPHIRecipe(VPDef::VPWidenIntOrFpInductionSC, Trunc, Start), + IV(IV), Trunc(Trunc), IndDesc(IndDesc), NeedsVectorIV(NeedsVectorIV) { + addOperand(Step); + } + + ~VPWidenIntOrFpInductionRecipe() override = default; + + VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionSC) + + /// Generate the vectorized and scalarized versions of the phi node as + /// needed by their users. + void execute(VPTransformState &State) override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + /// Print the recipe. + void print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const override; +#endif + + VPValue *getBackedgeValue() override { + llvm_unreachable( + "VPWidenIntOrFpInductionRecipe generates its own backedge value"); + } + + VPRecipeBase &getBackedgeRecipe() override { + llvm_unreachable( + "VPWidenIntOrFpInductionRecipe generates its own backedge value"); + } + + /// Returns the step value of the induction. + VPValue *getStepValue() { return getOperand(1); } + const VPValue *getStepValue() const { return getOperand(1); } + + /// Returns the first defined value as TruncInst, if it is one or nullptr + /// otherwise. + TruncInst *getTruncInst() { return Trunc; } + const TruncInst *getTruncInst() const { return Trunc; } + + PHINode *getPHINode() { return IV; } + + /// Returns the induction descriptor for the recipe. + const InductionDescriptor &getInductionDescriptor() const { return IndDesc; } + + /// Returns true if the induction is canonical, i.e. starting at 0 and + /// incremented by UF * VF (= the original IV is incremented by 1). + bool isCanonical() const; + + /// Returns the scalar type of the induction. + const Type *getScalarType() const { + return Trunc ? Trunc->getType() : IV->getType(); + } + + /// Returns true if a vector phi needs to be created for the induction. + bool needsVectorIV() const { return NeedsVectorIV; } +}; + class VPWidenPointerInductionRecipe : public VPHeaderPHIRecipe { const InductionDescriptor &IndDesc; diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -346,11 +346,11 @@ VPWidenMemoryInstructionSC, VPWidenSC, VPWidenSelectSC, - - // Phi-like recipes. Need to be kept together. + // START: Phi-like recipes. Need to be kept together. VPBlendSC, VPPredInstPHISC, - // Header-phi recipes. Need to be kept together. + // START: SubclassID for recipes that inherit VPHeaderPHIRecipe. + // VPHeaderPHIRecipe need to be kept together. VPCanonicalIVPHISC, VPActiveLaneMaskPHISC, VPFirstOrderRecurrencePHISC, @@ -358,8 +358,11 @@ VPWidenIntOrFpInductionSC, VPWidenPointerInductionSC, VPReductionPHISC, + // END: SubclassID for recipes that inherit VPHeaderPHIRecipe + // END: Phi-like recipes VPFirstPHISC = VPBlendSC, VPFirstHeaderPHISC = VPCanonicalIVPHISC, + VPLastHeaderPHISC = VPReductionPHISC, VPLastPHISC = VPReductionPHISC, };