diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7930,11 +7930,11 @@ unsigned FirstOpId; if (Kind == RecurrenceDescriptor::RK_IntegerMinMax || Kind == RecurrenceDescriptor::RK_FloatMinMax) { - assert(WidenRecipe->getVPRecipeID() == VPRecipeBase::VPWidenSelectSC && + assert(isa(WidenRecipe) && "Expected to replace a VPWidenSelectSC"); FirstOpId = 1; } else { - assert(WidenRecipe->getVPRecipeID() == VPRecipeBase::VPWidenSC && + assert(isa(WidenRecipe) && "Expected to replace a VPWidenSC"); FirstOpId = 0; } @@ -7951,7 +7951,7 @@ Kind == RecurrenceDescriptor::RK_FloatMinMax) { VPRecipeBase *CompareRecipe = RecipeBuilder.getRecipe(cast(R->getOperand(0))); - assert(CompareRecipe->getVPRecipeID() == VPRecipeBase::VPWidenSC && + assert(isa(CompareRecipe) && "Expected to replace a VPWidenSC"); CompareRecipe->eraseFromParent(); } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -606,46 +606,18 @@ /// VPRecipeBase is a base class modeling a sequence of one or more output IR /// instructions. -class VPRecipeBase : public ilist_node_with_parent { +class VPRecipeBase : public ilist_node_with_parent, + public VPValue { friend VPBasicBlock; friend class VPBlockUtils; - const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast). - /// Each VPRecipe belongs to a single VPBasicBlock. VPBasicBlock *Parent = nullptr; public: - /// An enumeration for keeping track of the concrete subclass of VPRecipeBase - /// that is actually instantiated. Values of this enumeration are kept in the - /// SubclassID field of the VPRecipeBase objects. They are used for concrete - /// type identification. - using VPRecipeTy = enum { - VPBlendSC, - VPBranchOnMaskSC, - VPInstructionSC, - VPInterleaveSC, - VPPredInstPHISC, - VPReductionSC, - VPReplicateSC, - VPWidenCallSC, - VPWidenCanonicalIVSC, - VPWidenGEPSC, - VPWidenIntOrFpInductionSC, - VPWidenMemoryInstructionSC, - VPWidenPHISC, - VPWidenSC, - VPWidenSelectSC - }; - - VPRecipeBase(const unsigned char SC) : SubclassID(SC) {} + VPRecipeBase(const unsigned char SC, Value *V = nullptr) : VPValue(SC, V) {} virtual ~VPRecipeBase() = default; - /// \return an ID for the concrete type of this object. - /// This is used to implement the classof checks. This should not be used - /// for any other purpose, as the values may change as LLVM evolves. - unsigned getVPRecipeID() const { return SubclassID; } - /// \return the VPBasicBlock which this VPRecipe belongs to. VPBasicBlock *getParent() { return Parent; } const VPBasicBlock *getParent() const { return Parent; } @@ -684,22 +656,22 @@ }; inline bool VPUser::classof(const VPRecipeBase *Recipe) { - return Recipe->getVPRecipeID() == VPRecipeBase::VPWidenSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenCallSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenSelectSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenGEPSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPBlendSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPInterleaveSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPReplicateSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPBranchOnMaskSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenMemoryInstructionSC; + return Recipe->getVPValueID() == VPRecipeBase::VPWidenSC || + Recipe->getVPValueID() == VPRecipeBase::VPWidenCallSC || + Recipe->getVPValueID() == VPRecipeBase::VPWidenSelectSC || + Recipe->getVPValueID() == VPRecipeBase::VPWidenGEPSC || + Recipe->getVPValueID() == VPRecipeBase::VPBlendSC || + Recipe->getVPValueID() == VPRecipeBase::VPInterleaveSC || + Recipe->getVPValueID() == VPRecipeBase::VPReplicateSC || + Recipe->getVPValueID() == VPRecipeBase::VPBranchOnMaskSC || + Recipe->getVPValueID() == VPRecipeBase::VPWidenMemoryInstructionSC; } /// This is a concrete Recipe that models a single VPlan-level instruction. /// While as any Recipe it may generate a sequence of IR instructions when /// executed, these instructions would always form a single-def expression as /// the VPInstruction is also a single def-use vertex. -class VPInstruction : public VPUser, public VPValue, public VPRecipeBase { +class VPInstruction : public VPUser, public VPRecipeBase { friend class VPlanSlp; public: @@ -729,8 +701,8 @@ public: VPInstruction(unsigned Opcode, ArrayRef Operands) - : VPUser(Operands), VPValue(VPValue::VPInstructionSC), - VPRecipeBase(VPRecipeBase::VPInstructionSC), Opcode(Opcode) {} + : VPUser(Operands), VPRecipeBase(VPValue::VPInstructionSC), + Opcode(Opcode) {} VPInstruction(unsigned Opcode, std::initializer_list Operands) : VPInstruction(Opcode, ArrayRef(Operands)) {} @@ -747,7 +719,7 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *R) { - return R->getVPRecipeID() == VPRecipeBase::VPInstructionSC; + return R->getVPValueID() == VPRecipeBase::VPInstructionSC; } unsigned getOpcode() const { return Opcode; } @@ -810,7 +782,7 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenSC; + return V->getVPValueID() == VPRecipeBase::VPWidenSC; } /// Produce widened copies of all Ingredients. @@ -822,19 +794,18 @@ }; /// A recipe for widening Call instructions. -class VPWidenCallRecipe : public VPRecipeBase, public VPValue, public VPUser { +class VPWidenCallRecipe : public VPRecipeBase, public VPUser { public: template VPWidenCallRecipe(CallInst &I, iterator_range CallArguments) - : VPRecipeBase(VPWidenCallSC), VPValue(VPValue::VPVWidenCallSC, &I), - VPUser(CallArguments) {} + : VPRecipeBase(VPWidenCallSC, &I), VPUser(CallArguments) {} ~VPWidenCallRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenCallSC; + return V->getVPValueID() == VPRecipeBase::VPWidenCallSC; } /// Produce a widened version of the call instruction. @@ -851,7 +822,7 @@ }; /// A recipe for widening select instructions. -class VPWidenSelectRecipe : public VPRecipeBase, public VPValue, public VPUser { +class VPWidenSelectRecipe : public VPRecipeBase, public VPUser { /// Is the condition of the select loop invariant? bool InvariantCond; @@ -860,14 +831,14 @@ template VPWidenSelectRecipe(SelectInst &I, iterator_range Operands, bool InvariantCond) - : VPRecipeBase(VPWidenSelectSC), VPValue(VPValue::VPVWidenSelectSC, &I), - VPUser(Operands), InvariantCond(InvariantCond) {} + : VPRecipeBase(VPValue::VPWidenSelectSC, &I), VPUser(Operands), + InvariantCond(InvariantCond) {} ~VPWidenSelectRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenSelectSC; + return V->getVPValueID() == VPRecipeBase::VPWidenSelectSC; } /// Produce a widened version of the select instruction. @@ -883,7 +854,7 @@ }; /// A recipe for handling GEP instructions. -class VPWidenGEPRecipe : public VPRecipeBase, public VPValue, public VPUser { +class VPWidenGEPRecipe : public VPRecipeBase, public VPUser { GetElementPtrInst *GEP; bool IsPtrLoopInvariant; @@ -898,8 +869,8 @@ template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands, Loop *OrigLoop) - : VPRecipeBase(VPWidenGEPSC), VPValue(VPValue::VPVWidenGEPSC, GEP), - VPUser(Operands), IsIndexLoopInvariant(GEP->getNumIndices(), false) { + : VPRecipeBase(VPValue::VPWidenGEPSC, GEP), VPUser(Operands), + IsIndexLoopInvariant(GEP->getNumIndices(), false) { IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand()); for (auto Index : enumerate(GEP->indices())) IsIndexLoopInvariant[Index.index()] = @@ -909,7 +880,7 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenGEPSC; + return V->getVPValueID() == VPRecipeBase::VPWidenGEPSC; } /// Generate the gep nodes. @@ -940,7 +911,7 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenIntOrFpInductionSC; + return V->getVPValueID() == VPRecipeBase::VPWidenIntOrFpInductionSC; } /// Generate the vectorized and scalarized versions of the phi node as @@ -962,7 +933,7 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenPHISC; + return V->getVPValueID() == VPRecipeBase::VPWidenPHISC; } /// Generate the phi/select nodes. @@ -992,7 +963,7 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPBlendSC; + return V->getVPValueID() == VPRecipeBase::VPBlendSC; } /// Return the number of incoming values, taking into account that a single @@ -1015,15 +986,14 @@ /// VPInterleaveRecipe is a recipe for transforming an interleave group of load /// or stores into one wide load/store and shuffles. -class VPInterleaveRecipe : public VPRecipeBase, public VPValue, public VPUser { +class VPInterleaveRecipe : public VPRecipeBase, public VPUser { const InterleaveGroup *IG; SmallVector Defs; public: VPInterleaveRecipe(const InterleaveGroup *IG, VPValue *Addr, VPValue *Mask) - : VPRecipeBase(VPInterleaveSC), VPValue(VPValue::VPVInterleaveSC), - VPUser({Addr}), IG(IG) { + : VPRecipeBase(VPValue::VPInterleaveSC), VPUser({Addr}), IG(IG) { if (Mask) addOperand(Mask); for (unsigned i = 0; i < IG->getNumMembers(); i++) @@ -1033,10 +1003,10 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPInterleaveSC; + return V->getVPValueID() == VPRecipeBase::VPInterleaveSC; } static inline bool classof(const VPValue *V) { - return V->getVPValueID() == VPValue::VPVInterleaveSC; + return V->getVPValueID() == VPValue::VPInterleaveSC; } /// Return the address accessed by this recipe. @@ -1090,7 +1060,7 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPReductionSC; + return V->getVPValueID() == VPRecipeBase::VPReductionSC; } /// Generate the reduction in the loop @@ -1136,7 +1106,7 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPReplicateSC; + return V->getVPValueID() == VPRecipeBase::VPReplicateSC; } /// Generate replicas of the desired Ingredient. Replicas will be generated @@ -1161,7 +1131,7 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPBranchOnMaskSC; + return V->getVPValueID() == VPRecipeBase::VPBranchOnMaskSC; } /// Generate the extraction of the appropriate bit from the block mask and the @@ -1205,7 +1175,7 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPPredInstPHISC; + return V->getVPValueID() == VPRecipeBase::VPPredInstPHISC; } /// Generates phi nodes for live-outs as needed to retain SSA form. @@ -1222,7 +1192,7 @@ /// - For store: Address, stored value, optional mask /// TODO: We currently execute only per-part unless a specific instance is /// provided. -class VPWidenMemoryInstructionRecipe : public VPRecipeBase, public VPValue, public VPUser { +class VPWidenMemoryInstructionRecipe : public VPRecipeBase, public VPUser { void setMask(VPValue *Mask) { if (!Mask) @@ -1239,22 +1209,20 @@ public: VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask) - : VPRecipeBase(VPWidenMemoryInstructionSC), - VPValue(VPValue::VPMemoryInstructionSC, &Load), VPUser({Addr}) { + : VPRecipeBase(VPWidenMemoryInstructionSC, &Load), VPUser({Addr}) { setMask(Mask); } VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredValue, VPValue *Mask) - : VPRecipeBase(VPWidenMemoryInstructionSC), - VPValue(VPValue::VPMemoryInstructionSC, &Store), + : VPRecipeBase(VPWidenMemoryInstructionSC, &Store), VPUser({Addr, StoredValue}) { setMask(Mask); } /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenMemoryInstructionSC; + return V->getVPValueID() == VPRecipeBase::VPWidenMemoryInstructionSC; } /// Return the address accessed by this recipe. @@ -1307,7 +1275,7 @@ /// Method to support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenCanonicalIVSC; + return V->getVPValueID() == VPRecipeBase::VPWidenCanonicalIVSC; } /// Generate a canonical vector induction variable of the vector loop, with diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -127,12 +127,22 @@ VPValueSC, VPVirtualValueSC, VPMultiValueSC, + + VPBlendSC, + VPBranchOnMaskSC, VPInstructionSC, - VPMemoryInstructionSC, - VPVWidenCallSC, - VPVWidenSelectSC, - VPVWidenGEPSC, - VPVInterleaveSC + VPInterleaveSC, + VPPredInstPHISC, + VPReductionSC, + VPReplicateSC, + VPWidenCallSC, + VPWidenCanonicalIVSC, + VPWidenGEPSC, + VPWidenIntOrFpInductionSC, + VPWidenMemoryInstructionSC, + VPWidenPHISC, + VPWidenSC, + VPWidenSelectSC, }; VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV) {} @@ -190,7 +200,10 @@ bool isSubValue() const { return UnderlyingOrProducer.is(); } - bool isVirtual() const { return getVPValueID() == VPVirtualValueSC; } + bool isVirtual() const { + return getVPValueID() == VPVirtualValueSC || + getVPValueID() == VPInterleaveSC; + } void addUser(VPUser &User) { Users.push_back(&User);