diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -606,46 +606,18 @@ /// VPRecipeBase is a base class modeling a sequence of one or more output IR /// instructions. -class VPRecipeBase : public ilist_node_with_parent { +class VPRecipeBase : public ilist_node_with_parent, + public VPValue { friend VPBasicBlock; friend class VPBlockUtils; - const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast). - /// Each VPRecipe belongs to a single VPBasicBlock. VPBasicBlock *Parent = nullptr; public: - /// An enumeration for keeping track of the concrete subclass of VPRecipeBase - /// that is actually instantiated. Values of this enumeration are kept in the - /// SubclassID field of the VPRecipeBase objects. They are used for concrete - /// type identification. - using VPRecipeTy = enum { - VPBlendSC, - VPBranchOnMaskSC, - VPInstructionSC, - VPInterleaveSC, - VPPredInstPHISC, - VPReductionSC, - VPReplicateSC, - VPWidenCallSC, - VPWidenCanonicalIVSC, - VPWidenGEPSC, - VPWidenIntOrFpInductionSC, - VPWidenMemoryInstructionSC, - VPWidenPHISC, - VPWidenSC, - VPWidenSelectSC - }; - - VPRecipeBase(const unsigned char SC) : SubclassID(SC) {} + VPRecipeBase(const unsigned char SC, Value *V = nullptr) : VPValue(SC, V) {} virtual ~VPRecipeBase() = default; - /// \return an ID for the concrete type of this object. - /// This is used to implement the classof checks. This should not be used - /// for any other purpose, as the values may change as LLVM evolves. - unsigned getVPRecipeID() const { return SubclassID; } - /// \return the VPBasicBlock which this VPRecipe belongs to. VPBasicBlock *getParent() { return Parent; } const VPBasicBlock *getParent() const { return Parent; } @@ -692,23 +664,23 @@ }; inline bool VPUser::classof(const VPRecipeBase *Recipe) { - return Recipe->getVPRecipeID() == VPRecipeBase::VPInstructionSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenCallSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenSelectSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenGEPSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPBlendSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPInterleaveSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPReplicateSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPBranchOnMaskSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenMemoryInstructionSC; + return Recipe->getVPValueID() == VPRecipeBase::VPInstructionSC || + Recipe->getVPValueID() == VPRecipeBase::VPWidenSC || + Recipe->getVPValueID() == VPRecipeBase::VPWidenCallSC || + Recipe->getVPValueID() == VPRecipeBase::VPWidenSelectSC || + Recipe->getVPValueID() == VPRecipeBase::VPWidenGEPSC || + Recipe->getVPValueID() == VPRecipeBase::VPBlendSC || + Recipe->getVPValueID() == VPRecipeBase::VPInterleaveSC || + Recipe->getVPValueID() == VPRecipeBase::VPReplicateSC || + Recipe->getVPValueID() == VPRecipeBase::VPBranchOnMaskSC || + Recipe->getVPValueID() == VPRecipeBase::VPWidenMemoryInstructionSC; } /// This is a concrete Recipe that models a single VPlan-level instruction. /// While as any Recipe it may generate a sequence of IR instructions when /// executed, these instructions would always form a single-def expression as /// the VPInstruction is also a single def-use vertex. -class VPInstruction : public VPUser, public VPValue, public VPRecipeBase { +class VPInstruction : public VPUser, public VPRecipeBase { friend class VPlanSlp; public: @@ -738,8 +710,8 @@ public: VPInstruction(unsigned Opcode, ArrayRef Operands) - : VPUser(Operands), VPValue(VPValue::VPInstructionSC), - VPRecipeBase(VPRecipeBase::VPInstructionSC), Opcode(Opcode) {} + : VPUser(Operands), VPRecipeBase(VPValue::VPInstructionSC), + Opcode(Opcode) {} VPInstruction(unsigned Opcode, std::initializer_list Operands) : VPInstruction(Opcode, ArrayRef(Operands)) {} @@ -754,11 +726,6 @@ return new VPInstruction(Opcode, Operands); } - /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *R) { - return R->getVPRecipeID() == VPRecipeBase::VPInstructionSC; - } - unsigned getOpcode() const { return Opcode; } /// Generate the instruction. @@ -818,8 +785,8 @@ ~VPWidenRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenSC; } /// Produce widened copies of all Ingredients. @@ -831,19 +798,18 @@ }; /// A recipe for widening Call instructions. -class VPWidenCallRecipe : public VPRecipeBase, public VPValue, public VPUser { +class VPWidenCallRecipe : public VPRecipeBase, public VPUser { public: template VPWidenCallRecipe(CallInst &I, iterator_range CallArguments) - : VPRecipeBase(VPWidenCallSC), VPValue(VPValue::VPVWidenCallSC, &I), - VPUser(CallArguments) {} + : VPRecipeBase(VPWidenCallSC, &I), VPUser(CallArguments) {} ~VPWidenCallRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenCallSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenCallSC; } /// Produce a widened version of the call instruction. @@ -860,7 +826,7 @@ }; /// A recipe for widening select instructions. -class VPWidenSelectRecipe : public VPRecipeBase, public VPValue, public VPUser { +class VPWidenSelectRecipe : public VPRecipeBase, public VPUser { /// Is the condition of the select loop invariant? bool InvariantCond; @@ -869,14 +835,14 @@ template VPWidenSelectRecipe(SelectInst &I, iterator_range Operands, bool InvariantCond) - : VPRecipeBase(VPWidenSelectSC), VPValue(VPValue::VPVWidenSelectSC, &I), - VPUser(Operands), InvariantCond(InvariantCond) {} + : VPRecipeBase(VPValue::VPWidenSelectSC, &I), VPUser(Operands), + InvariantCond(InvariantCond) {} ~VPWidenSelectRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenSelectSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenSelectSC; } /// Produce a widened version of the select instruction. @@ -892,7 +858,7 @@ }; /// A recipe for handling GEP instructions. -class VPWidenGEPRecipe : public VPRecipeBase, public VPValue, public VPUser { +class VPWidenGEPRecipe : public VPRecipeBase, public VPUser { GetElementPtrInst *GEP; bool IsPtrLoopInvariant; @@ -907,8 +873,8 @@ template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands, Loop *OrigLoop) - : VPRecipeBase(VPWidenGEPSC), VPValue(VPValue::VPVWidenGEPSC, GEP), - VPUser(Operands), IsIndexLoopInvariant(GEP->getNumIndices(), false) { + : VPRecipeBase(VPValue::VPWidenGEPSC, GEP), VPUser(Operands), + IsIndexLoopInvariant(GEP->getNumIndices(), false) { IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand()); for (auto Index : enumerate(GEP->indices())) IsIndexLoopInvariant[Index.index()] = @@ -917,8 +883,8 @@ ~VPWidenGEPRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenGEPSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenGEPSC; } /// Generate the gep nodes. @@ -948,8 +914,8 @@ ~VPWidenIntOrFpInductionRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenIntOrFpInductionSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenIntOrFpInductionSC; } /// Generate the vectorized and scalarized versions of the phi node as @@ -970,8 +936,8 @@ ~VPWidenPHIRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenPHISC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenPHISC; } /// Generate the phi/select nodes. @@ -1000,8 +966,8 @@ } /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPBlendSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPBlendSC; } /// Return the number of incoming values, taking into account that a single @@ -1024,7 +990,7 @@ /// VPInterleaveRecipe is a recipe for transforming an interleave group of load /// or stores into one wide load/store and shuffles. -class VPInterleaveRecipe : public VPRecipeBase, public VPValue, public VPUser { +class VPInterleaveRecipe : public VPRecipeBase, public VPUser { const InterleaveGroup *IG; public: @@ -1032,8 +998,7 @@ VPInterleaveRecipe(const InterleaveGroup *IG, VPValue *Addr, VPValue *Mask) - : VPRecipeBase(VPInterleaveSC), VPValue(VPValue::VPVInterleaveSC), - VPUser({Addr}), IG(IG) { + : VPRecipeBase(VPValue::VPInterleaveSC), VPUser({Addr}), IG(IG) { if (Mask) addOperand(Mask); for (unsigned i = 0; i < IG->getNumMembers(); i++) @@ -1042,11 +1007,8 @@ ~VPInterleaveRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPInterleaveSC; - } static inline bool classof(const VPValue *V) { - return V->getVPValueID() == VPValue::VPVInterleaveSC; + return V->getVPValueID() == VPRecipeBase::VPInterleaveSC; } /// Return the address accessed by this recipe. @@ -1099,8 +1061,8 @@ ~VPReductionRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPReductionSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPReductionSC; } /// Generate the reduction in the loop @@ -1145,8 +1107,8 @@ ~VPReplicateRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPReplicateSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPReplicateSC; } /// Generate replicas of the desired Ingredient. Replicas will be generated @@ -1170,8 +1132,8 @@ } /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPBranchOnMaskSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPBranchOnMaskSC; } /// Generate the extraction of the appropriate bit from the block mask and the @@ -1214,8 +1176,8 @@ ~VPPredInstPHIRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPPredInstPHISC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPPredInstPHISC; } /// Generates phi nodes for live-outs as needed to retain SSA form. @@ -1232,9 +1194,7 @@ /// - For store: Address, stored value, optional mask /// TODO: We currently execute only per-part unless a specific instance is /// provided. -class VPWidenMemoryInstructionRecipe : public VPRecipeBase, - public VPValue, - public VPUser { +class VPWidenMemoryInstructionRecipe : public VPRecipeBase, public VPUser { void setMask(VPValue *Mask) { if (!Mask) @@ -1249,22 +1209,20 @@ public: VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask) - : VPRecipeBase(VPWidenMemoryInstructionSC), - VPValue(VPValue::VPMemoryInstructionSC, &Load), VPUser({Addr}) { + : VPRecipeBase(VPWidenMemoryInstructionSC, &Load), VPUser({Addr}) { setMask(Mask); } VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredValue, VPValue *Mask) - : VPRecipeBase(VPWidenMemoryInstructionSC), - VPValue(VPValue::VPMemoryInstructionSC, &Store), + : VPRecipeBase(VPWidenMemoryInstructionSC, &Store), VPUser({Addr, StoredValue}) { setMask(Mask); } /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenMemoryInstructionSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenMemoryInstructionSC; } /// Return the address accessed by this recipe. @@ -1316,8 +1274,8 @@ VPValue *getVPValue() { return &Val; } /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenCanonicalIVSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenCanonicalIVSC; } /// Generate a canonical vector induction variable of the vector loop, with diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -128,12 +128,22 @@ VPValueSC, VPVirtualValueSC, VPMultiValueSC, + + VPBlendSC, + VPBranchOnMaskSC, VPInstructionSC, - VPMemoryInstructionSC, - VPVWidenCallSC, - VPVWidenSelectSC, - VPVWidenGEPSC, - VPVInterleaveSC + VPInterleaveSC, + VPPredInstPHISC, + VPReductionSC, + VPReplicateSC, + VPWidenCallSC, + VPWidenCanonicalIVSC, + VPWidenGEPSC, + VPWidenIntOrFpInductionSC, + VPWidenMemoryInstructionSC, + VPWidenPHISC, + VPWidenSC, + VPWidenSelectSC, }; VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV) {} @@ -216,7 +226,10 @@ bool isSubValue() const { return UnderlyingOrProducer.is(); } - bool isVirtual() const { return getVPValueID() == VPVirtualValueSC; } + bool isVirtual() const { + return getVPValueID() == VPVirtualValueSC || + getVPValueID() == VPInterleaveSC; + } }; // Check size assertions made in the PointerLikeTypeTraits specialization for