diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -618,46 +618,19 @@ /// VPRecipeBase is a base class modeling a sequence of one or more output IR /// instructions. -class VPRecipeBase : public ilist_node_with_parent { +class VPRecipeBase : public ilist_node_with_parent, + public VPDef { friend VPBasicBlock; friend class VPBlockUtils; - const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast). /// Each VPRecipe belongs to a single VPBasicBlock. VPBasicBlock *Parent = nullptr; public: - /// An enumeration for keeping track of the concrete subclass of VPRecipeBase - /// that is actually instantiated. Values of this enumeration are kept in the - /// SubclassID field of the VPRecipeBase objects. They are used for concrete - /// type identification. - using VPRecipeTy = enum { - VPBlendSC, - VPBranchOnMaskSC, - VPInstructionSC, - VPInterleaveSC, - VPPredInstPHISC, - VPReductionSC, - VPReplicateSC, - VPWidenCallSC, - VPWidenCanonicalIVSC, - VPWidenGEPSC, - VPWidenIntOrFpInductionSC, - VPWidenMemoryInstructionSC, - VPWidenPHISC, - VPWidenSC, - VPWidenSelectSC - }; - - VPRecipeBase(const unsigned char SC) : SubclassID(SC) {} + VPRecipeBase(const unsigned char SC) : VPDef(SC) {} virtual ~VPRecipeBase() = default; - /// \return an ID for the concrete type of this object. - /// This is used to implement the classof checks. This should not be used - /// for any other purpose, as the values may change as LLVM evolves. - unsigned getVPRecipeID() const { return SubclassID; } - /// \return the VPBasicBlock which this VPRecipe belongs to. VPBasicBlock *getParent() { return Parent; } const VPBasicBlock *getParent() const { return Parent; } @@ -717,24 +690,24 @@ } }; -inline bool VPUser::classof(const VPRecipeBase *Recipe) { - return Recipe->getVPRecipeID() == VPRecipeBase::VPInstructionSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenCallSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenSelectSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenGEPSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPBlendSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPInterleaveSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPReplicateSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPBranchOnMaskSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenMemoryInstructionSC; +inline bool VPUser::classof(const VPDef *Recipe) { + return Recipe->getVPDefID() == VPRecipeBase::VPInstructionSC || + Recipe->getVPDefID() == VPRecipeBase::VPWidenSC || + Recipe->getVPDefID() == VPRecipeBase::VPWidenCallSC || + Recipe->getVPDefID() == VPRecipeBase::VPWidenSelectSC || + Recipe->getVPDefID() == VPRecipeBase::VPWidenGEPSC || + Recipe->getVPDefID() == VPRecipeBase::VPBlendSC || + Recipe->getVPDefID() == VPRecipeBase::VPInterleaveSC || + Recipe->getVPDefID() == VPRecipeBase::VPReplicateSC || + Recipe->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC || + Recipe->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC; } /// This is a concrete Recipe that models a single VPlan-level instruction. /// While as any Recipe it may generate a sequence of IR instructions when /// executed, these instructions would always form a single-def expression as /// the VPInstruction is also a single def-use vertex. -class VPInstruction : public VPUser, public VPValue, public VPRecipeBase { +class VPInstruction : public VPValue, public VPUser, public VPRecipeBase { friend class VPlanSlp; public: @@ -760,9 +733,16 @@ public: VPInstruction(unsigned Opcode, ArrayRef Operands) - : VPUser(Operands), VPValue(VPValue::VPInstructionSC), + : VPValue(VPValue::VPInstructionSC), VPUser(Operands), VPRecipeBase(VPRecipeBase::VPInstructionSC), Opcode(Opcode) {} + VPInstruction(unsigned Opcode, ArrayRef Operands) + : VPValue(VPValue::VPInstructionSC), VPUser({}), + VPRecipeBase(VPRecipeBase::VPInstructionSC), Opcode(Opcode) { + for (auto *I : Operands) + addOperand(I->getVPValue()); + } + VPInstruction(unsigned Opcode, std::initializer_list Operands) : VPInstruction(Opcode, ArrayRef(Operands)) {} @@ -777,8 +757,8 @@ } /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *R) { - return R->getVPRecipeID() == VPRecipeBase::VPInstructionSC; + static inline bool classof(const VPDef *R) { + return R->getVPDefID() == VPRecipeBase::VPInstructionSC; } unsigned getOpcode() const { return Opcode; } @@ -835,13 +815,15 @@ public: template VPWidenRecipe(Instruction &I, iterator_range Operands) - : VPRecipeBase(VPWidenSC), VPUser(Operands), Ingredient(I) {} + : VPRecipeBase(VPWidenSC), VPUser(Operands), Ingredient(I) { + new VPValue(&I, this); + } ~VPWidenRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenSC; + static inline bool classof(const VPDef *V) { + return V->getVPDefID() == VPRecipeBase::VPWidenSC; } /// Produce widened copies of all Ingredients. @@ -853,10 +835,7 @@ }; /// A recipe for widening Call instructions. -class VPWidenCallRecipe : public VPRecipeBase, - public VPDef, - public VPUser, - public VPValue { +class VPWidenCallRecipe : public VPRecipeBase, public VPUser, public VPValue { public: template @@ -867,8 +846,8 @@ ~VPWidenCallRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenCallSC; + static inline bool classof(const VPDef *V) { + return V->getVPDefID() == VPRecipeBase::VPWidenCallSC; } /// Produce a widened version of the call instruction. @@ -880,10 +859,7 @@ }; /// A recipe for widening select instructions. -class VPWidenSelectRecipe : public VPRecipeBase, - public VPDef, - public VPUser, - public VPValue { +class VPWidenSelectRecipe : public VPRecipeBase, public VPUser, public VPValue { /// Is the condition of the select loop invariant? bool InvariantCond; @@ -899,8 +875,8 @@ ~VPWidenSelectRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenSelectSC; + static inline bool classof(const VPDef *V) { + return V->getVPDefID() == VPRecipeBase::VPWidenSelectSC; } /// Produce a widened version of the select instruction. @@ -913,7 +889,6 @@ /// A recipe for handling GEP instructions. class VPWidenGEPRecipe : public VPRecipeBase, - public VPDef, public VPUser, public VPValue { bool IsPtrLoopInvariant; @@ -940,8 +915,8 @@ ~VPWidenGEPRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenGEPSC; + static inline bool classof(const VPDef *V) { + return V->getVPDefID() == VPRecipeBase::VPWidenGEPSC; } /// Generate the gep nodes. @@ -960,12 +935,17 @@ public: VPWidenIntOrFpInductionRecipe(PHINode *IV, TruncInst *Trunc = nullptr) - : VPRecipeBase(VPWidenIntOrFpInductionSC), IV(IV), Trunc(Trunc) {} + : VPRecipeBase(VPWidenIntOrFpInductionSC), IV(IV), Trunc(Trunc) { + if (Trunc) + new VPValue(Trunc, this); + else + new VPValue(IV, this); + } ~VPWidenIntOrFpInductionRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenIntOrFpInductionSC; + static inline bool classof(const VPDef *V) { + return V->getVPDefID() == VPRecipeBase::VPWidenIntOrFpInductionSC; } /// Generate the vectorized and scalarized versions of the phi node as @@ -982,12 +962,14 @@ PHINode *Phi; public: - VPWidenPHIRecipe(PHINode *Phi) : VPRecipeBase(VPWidenPHISC), Phi(Phi) {} + VPWidenPHIRecipe(PHINode *Phi) : VPRecipeBase(VPWidenPHISC), Phi(Phi) { + new VPValue(Phi, this); + } ~VPWidenPHIRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenPHISC; + static inline bool classof(const VPDef *V) { + return V->getVPDefID() == VPRecipeBase::VPWidenPHISC; } /// Generate the phi/select nodes. @@ -1009,6 +991,7 @@ /// might be incoming with a full mask for which there is no VPValue. VPBlendRecipe(PHINode *Phi, ArrayRef Operands) : VPRecipeBase(VPBlendSC), VPUser(Operands), Phi(Phi) { + new VPValue(Phi, this); assert(Operands.size() > 0 && ((Operands.size() == 1) || (Operands.size() % 2 == 0)) && "Expected either a single incoming value or a positive even number " @@ -1016,8 +999,8 @@ } /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPBlendSC; + static inline bool classof(const VPDef *V) { + return V->getVPDefID() == VPRecipeBase::VPBlendSC; } /// Return the number of incoming values, taking into account that a single @@ -1040,7 +1023,7 @@ /// VPInterleaveRecipe is a recipe for transforming an interleave group of load /// or stores into one wide load/store and shuffles. -class VPInterleaveRecipe : public VPRecipeBase, public VPDef, public VPUser { +class VPInterleaveRecipe : public VPRecipeBase, public VPUser { const InterleaveGroup *IG; public: @@ -1060,8 +1043,8 @@ ~VPInterleaveRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPInterleaveSC; + static inline bool classof(const VPDef *V) { + return V->getVPDefID() == VPRecipeBase::VPInterleaveSC; } /// Return the address accessed by this recipe. @@ -1109,13 +1092,15 @@ VPValue *VecOp, VPValue *CondOp, bool NoNaN, const TargetTransformInfo *TTI) : VPRecipeBase(VPReductionSC), RdxDesc(R), I(I), VecOp(VecOp), - ChainOp(ChainOp), CondOp(CondOp), NoNaN(NoNaN), TTI(TTI) {} + ChainOp(ChainOp), CondOp(CondOp), NoNaN(NoNaN), TTI(TTI) { + new VPValue(I, this); + } ~VPReductionRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPReductionSC; + static inline bool classof(const VPDef *V) { + return V->getVPDefID() == VPRecipeBase::VPReductionSC; } /// Generate the reduction in the loop @@ -1155,13 +1140,14 @@ // having predicated instructions also pack their values into a vector by // default unless they have a replicated user which uses their scalar value. AlsoPack = IsPredicated && !I->use_empty(); + new VPValue(I, this); } ~VPReplicateRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPReplicateSC; + static inline bool classof(const VPDef *V) { + return V->getVPDefID() == VPRecipeBase::VPReplicateSC; } /// Generate replicas of the desired Ingredient. Replicas will be generated @@ -1180,13 +1166,14 @@ class VPBranchOnMaskRecipe : public VPRecipeBase, public VPUser { public: VPBranchOnMaskRecipe(VPValue *BlockInMask) : VPRecipeBase(VPBranchOnMaskSC) { + new VPValue(nullptr, this); if (BlockInMask) // nullptr means all-one mask. addOperand(BlockInMask); } /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPBranchOnMaskSC; + static inline bool classof(const VPDef *V) { + return V->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC; } /// Generate the extraction of the appropriate bit from the block mask and the @@ -1225,12 +1212,14 @@ /// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi /// nodes after merging back from a Branch-on-Mask. VPPredInstPHIRecipe(Instruction *PredInst) - : VPRecipeBase(VPPredInstPHISC), PredInst(PredInst) {} + : VPRecipeBase(VPPredInstPHISC), PredInst(PredInst) { + new VPValue(PredInst, this); + } ~VPPredInstPHIRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPPredInstPHISC; + static inline bool classof(const VPDef *V) { + return V->getVPDefID() == VPRecipeBase::VPPredInstPHISC; } /// Generates phi nodes for live-outs as needed to retain SSA form. @@ -1248,7 +1237,6 @@ /// TODO: We currently execute only per-part unless a specific instance is /// provided. class VPWidenMemoryInstructionRecipe : public VPRecipeBase, - public VPDef, public VPUser, public VPValue { void setMask(VPValue *Mask) { @@ -1276,8 +1264,8 @@ } /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenMemoryInstructionSC; + static inline bool classof(const VPDef *V) { + return V->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC; } /// Return the address accessed by this recipe. @@ -1310,21 +1298,16 @@ /// A Recipe for widening the canonical induction variable of the vector loop. class VPWidenCanonicalIVRecipe : public VPRecipeBase { - /// A VPValue representing the canonical vector IV. - VPValue Val; - public: - VPWidenCanonicalIVRecipe() : VPRecipeBase(VPWidenCanonicalIVSC) {} - ~VPWidenCanonicalIVRecipe() override = default; + VPWidenCanonicalIVRecipe() : VPRecipeBase(VPWidenCanonicalIVSC) { + new VPValue(nullptr, this); + } - /// Return the VPValue representing the canonical vector induction variable of - /// the vector loop. - const VPValue *getVPValue() const { return &Val; } - VPValue *getVPValue() { return &Val; } + ~VPWidenCanonicalIVRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenCanonicalIVSC; + static inline bool classof(const VPDef *V) { + return V->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC; } /// Generate a canonical vector induction variable of the vector loop, with diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -32,9 +32,8 @@ class Value; class VPSlotTracker; class VPUser; -class VPValue; -class VPRecipeBase; class VPDef; +class VPRecipeBase; // This is the base class of the VPlan Def/Use graph, used for modeling the data // flow into, within and out of the VPlan. VPValues can stand for live-ins @@ -48,6 +47,7 @@ friend class VPSlotTracker; friend class VPRecipeBase; friend class VPDef; + friend class VPInstruction; const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast). @@ -225,14 +225,37 @@ } /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *Recipe); + static inline bool classof(const VPDef *Recipe); }; class VPDef { + const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast). TinyPtrVector DefinedValues; public: - VPDef() {} + /// An enumeration for keeping track of the concrete subclass of VPRecipeBase + /// that is actually instantiated. Values of this enumeration are kept in the + /// SubclassID field of the VPRecipeBase objects. They are used for concrete + /// type identification. + using VPRecipeTy = enum { + VPBlendSC, + VPBranchOnMaskSC, + VPInstructionSC, + VPInterleaveSC, + VPPredInstPHISC, + VPReductionSC, + VPReplicateSC, + VPWidenCallSC, + VPWidenCanonicalIVSC, + VPWidenGEPSC, + VPWidenIntOrFpInductionSC, + VPWidenMemoryInstructionSC, + VPWidenPHISC, + VPWidenSC, + VPWidenSelectSC + }; + + VPDef(const unsigned char SC) : SubclassID(SC) {} virtual ~VPDef() { for (VPValue *D : DefinedValues) @@ -245,6 +268,7 @@ assert(DefinedValues[i]); return DefinedValues[i]; } + const VPValue *getVPValue(unsigned i = 0) const { assert(DefinedValues[i]); return DefinedValues[i]; @@ -253,6 +277,11 @@ ArrayRef defined_values() { return DefinedValues; } unsigned getNumDefinedValues() const { return DefinedValues.size(); } + + /// \return an ID for the concrete type of this object. + /// This is used to implement the classof checks. This should not be used + /// for any other purpose, as the values may change as LLVM evolves. + unsigned getVPDefID() const { return SubclassID; } }; class VPlan; diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -365,7 +365,7 @@ EXPECT_TRUE(isa(&Recipe)); VPRecipeBase *BaseR = &Recipe; EXPECT_TRUE(isa(BaseR)); - EXPECT_EQ(&Recipe, BaseR->toVPUser()); + EXPECT_EQ(&Recipe, BaseR); } TEST(VPRecipeTest, CastVPWidenRecipeToVPUser) { @@ -383,7 +383,7 @@ EXPECT_TRUE(isa(&WidenR)); VPRecipeBase *WidenRBase = &WidenR; EXPECT_TRUE(isa(WidenRBase)); - EXPECT_EQ(&WidenR, WidenRBase->toVPUser()); + EXPECT_EQ(&WidenR, WidenRBase); delete AI; } @@ -402,7 +402,7 @@ EXPECT_TRUE(isa(&Recipe)); VPRecipeBase *BaseR = &Recipe; EXPECT_TRUE(isa(BaseR)); - EXPECT_EQ(&Recipe, BaseR->toVPUser()); + EXPECT_EQ(&Recipe, BaseR); delete Call; } @@ -425,7 +425,7 @@ EXPECT_TRUE(isa(&WidenSelectR)); VPRecipeBase *BaseR = &WidenSelectR; EXPECT_TRUE(isa(BaseR)); - EXPECT_EQ(&WidenSelectR, BaseR->toVPUser()); + EXPECT_EQ(&WidenSelectR, BaseR); delete SelectI; } @@ -445,7 +445,7 @@ EXPECT_TRUE(isa(&Recipe)); VPRecipeBase *BaseR = &Recipe; EXPECT_TRUE(isa(BaseR)); - EXPECT_EQ(&Recipe, BaseR->toVPUser()); + EXPECT_EQ(&Recipe, BaseR); delete GEP; } @@ -476,7 +476,7 @@ EXPECT_TRUE(isa(&Recipe)); VPRecipeBase *BaseR = &Recipe; EXPECT_TRUE(isa(BaseR)); - EXPECT_EQ(&Recipe, BaseR->toVPUser()); + EXPECT_EQ(&Recipe, BaseR); } TEST(VPRecipeTest, CastVPReplicateRecipeToVPUser) { @@ -503,7 +503,7 @@ EXPECT_TRUE(isa(&Recipe)); VPRecipeBase *BaseR = &Recipe; EXPECT_TRUE(isa(BaseR)); - EXPECT_EQ(&Recipe, BaseR->toVPUser()); + EXPECT_EQ(&Recipe, BaseR); } TEST(VPRecipeTest, CastVPWidenMemoryInstructionRecipeToVPUser) { @@ -519,14 +519,14 @@ EXPECT_TRUE(isa(&Recipe)); VPRecipeBase *BaseR = &Recipe; EXPECT_TRUE(isa(BaseR)); - EXPECT_EQ(&Recipe, BaseR->toVPUser()); + EXPECT_EQ(&Recipe, BaseR); delete Load; } -struct VPMultiDef : public VPDef { +struct VPMultiDef : public VPUser, public VPDef { SmallVector Defs; - VPMultiDef() : VPDef({}) { + VPMultiDef() : VPUser({}), VPDef(0) { new VPValue(nullptr, this); new VPValue(nullptr, this); }