diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7791,14 +7791,11 @@ if (auto Recipe = RecipeBuilder.tryToCreateWidenRecipe(Instr, Range, Plan)) { - // Check if the recipe can be converted to a VPValue. We need the extra - // down-casting step until VPRecipeBase inherits from VPValue. - VPValue *MaybeVPValue = Recipe->toVPValue(); - if (!Instr->getType()->isVoidTy() && MaybeVPValue) { + if (!Instr->getType()->isVoidTy() && !Recipe->isVirtual()) { if (NeedDef.contains(Instr)) - Plan->addOrReplaceVPValue(Instr, MaybeVPValue); + Plan->addOrReplaceVPValue(Instr, Recipe); else - Plan->addVPValue(Instr, MaybeVPValue); + Plan->addVPValue(Instr, Recipe); } RecipeBuilder.setRecipe(Instr, Recipe); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -606,46 +606,18 @@ /// VPRecipeBase is a base class modeling a sequence of one or more output IR /// instructions. -class VPRecipeBase : public ilist_node_with_parent { +class VPRecipeBase : public ilist_node_with_parent, + public VPValue { friend VPBasicBlock; friend class VPBlockUtils; - const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast). - /// Each VPRecipe belongs to a single VPBasicBlock. VPBasicBlock *Parent = nullptr; public: - /// An enumeration for keeping track of the concrete subclass of VPRecipeBase - /// that is actually instantiated. Values of this enumeration are kept in the - /// SubclassID field of the VPRecipeBase objects. They are used for concrete - /// type identification. - using VPRecipeTy = enum { - VPBlendSC, - VPBranchOnMaskSC, - VPInstructionSC, - VPInterleaveSC, - VPPredInstPHISC, - VPReductionSC, - VPReplicateSC, - VPWidenCallSC, - VPWidenCanonicalIVSC, - VPWidenGEPSC, - VPWidenIntOrFpInductionSC, - VPWidenMemoryInstructionSC, - VPWidenPHISC, - VPWidenSC, - VPWidenSelectSC - }; - - VPRecipeBase(const unsigned char SC) : SubclassID(SC) {} + VPRecipeBase(const unsigned char SC, Value *V = nullptr) : VPValue(SC, V) {} virtual ~VPRecipeBase() = default; - /// \return an ID for the concrete type of this object. - /// This is used to implement the classof checks. This should not be used - /// for any other purpose, as the values may change as LLVM evolves. - unsigned getVPRecipeID() const { return SubclassID; } - /// \return the VPBasicBlock which this VPRecipe belongs to. VPBasicBlock *getParent() { return Parent; } const VPBasicBlock *getParent() const { return Parent; } @@ -686,43 +658,35 @@ /// nullptr otherwise. VPUser *toVPUser(); - /// Returns a pointer to a VPValue, if the recipe inherits from VPValue or - /// nullptr otherwise. - VPValue *toVPValue(); - const VPValue *toVPValue() const; /// Returns the underlying instruction, if the recipe is a VPValue or nullptr /// otherwise. Instruction *getUnderlyingInstr() { - if (auto *VPV = toVPValue()) - return cast_or_null(VPV->getUnderlyingValue()); - return nullptr; + return cast_or_null(getUnderlyingValue()); } const Instruction *getUnderlyingInstr() const { - if (auto *VPV = toVPValue()) - return cast_or_null(VPV->getUnderlyingValue()); - return nullptr; + return cast_or_null(getUnderlyingValue()); } }; inline bool VPUser::classof(const VPRecipeBase *Recipe) { - return Recipe->getVPRecipeID() == VPRecipeBase::VPInstructionSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenCallSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenSelectSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenGEPSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPBlendSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPInterleaveSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPReplicateSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPBranchOnMaskSC || - Recipe->getVPRecipeID() == VPRecipeBase::VPWidenMemoryInstructionSC; + return Recipe->getVPValueID() == VPRecipeBase::VPInstructionSC || + Recipe->getVPValueID() == VPRecipeBase::VPWidenSC || + Recipe->getVPValueID() == VPRecipeBase::VPWidenCallSC || + Recipe->getVPValueID() == VPRecipeBase::VPWidenSelectSC || + Recipe->getVPValueID() == VPRecipeBase::VPWidenGEPSC || + Recipe->getVPValueID() == VPRecipeBase::VPBlendSC || + Recipe->getVPValueID() == VPRecipeBase::VPVInterleaveSC || + Recipe->getVPValueID() == VPRecipeBase::VPReplicateSC || + Recipe->getVPValueID() == VPRecipeBase::VPBranchOnMaskSC || + Recipe->getVPValueID() == VPRecipeBase::VPWidenMemoryInstructionSC; } /// This is a concrete Recipe that models a single VPlan-level instruction. /// While as any Recipe it may generate a sequence of IR instructions when /// executed, these instructions would always form a single-def expression as /// the VPInstruction is also a single def-use vertex. -class VPInstruction : public VPUser, public VPValue, public VPRecipeBase { +class VPInstruction : public VPUser, public VPRecipeBase { friend class VPlanSlp; public: @@ -748,8 +712,8 @@ public: VPInstruction(unsigned Opcode, ArrayRef Operands) - : VPUser(Operands), VPValue(VPValue::VPInstructionSC), - VPRecipeBase(VPRecipeBase::VPInstructionSC), Opcode(Opcode) {} + : VPUser(Operands), VPRecipeBase(VPValue::VPInstructionSC), + Opcode(Opcode) {} VPInstruction(unsigned Opcode, std::initializer_list Operands) : VPInstruction(Opcode, ArrayRef(Operands)) {} @@ -764,11 +728,6 @@ return new VPInstruction(Opcode, Operands); } - /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *R) { - return R->getVPRecipeID() == VPRecipeBase::VPInstructionSC; - } - unsigned getOpcode() const { return Opcode; } /// Generate the instruction. @@ -828,8 +787,8 @@ ~VPWidenRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenSC; } /// Produce widened copies of all Ingredients. @@ -841,19 +800,18 @@ }; /// A recipe for widening Call instructions. -class VPWidenCallRecipe : public VPRecipeBase, public VPValue, public VPUser { +class VPWidenCallRecipe : public VPRecipeBase, public VPUser { public: template VPWidenCallRecipe(CallInst &I, iterator_range CallArguments) - : VPRecipeBase(VPRecipeBase::VPWidenCallSC), - VPValue(VPValue::VPVWidenCallSC, &I), VPUser(CallArguments) {} + : VPRecipeBase(VPWidenCallSC, &I), VPUser(CallArguments) {} ~VPWidenCallRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenCallSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenCallSC; } /// Produce a widened version of the call instruction. @@ -865,7 +823,7 @@ }; /// A recipe for widening select instructions. -class VPWidenSelectRecipe : public VPRecipeBase, public VPValue, public VPUser { +class VPWidenSelectRecipe : public VPRecipeBase, public VPUser { /// Is the condition of the select loop invariant? bool InvariantCond; @@ -874,15 +832,14 @@ template VPWidenSelectRecipe(SelectInst &I, iterator_range Operands, bool InvariantCond) - : VPRecipeBase(VPRecipeBase::VPWidenSelectSC), - VPValue(VPValue::VPVWidenSelectSC, &I), VPUser(Operands), + : VPRecipeBase(VPValue::VPWidenSelectSC, &I), VPUser(Operands), InvariantCond(InvariantCond) {} ~VPWidenSelectRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenSelectSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenSelectSC; } /// Produce a widened version of the select instruction. @@ -894,21 +851,21 @@ }; /// A recipe for handling GEP instructions. -class VPWidenGEPRecipe : public VPRecipeBase, public VPValue, public VPUser { +class VPWidenGEPRecipe : public VPRecipeBase, public VPUser { bool IsPtrLoopInvariant; SmallBitVector IsIndexLoopInvariant; public: template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands) - : VPRecipeBase(VPRecipeBase::VPWidenGEPSC), VPValue(VPWidenGEPSC, GEP), - VPUser(Operands), IsIndexLoopInvariant(GEP->getNumIndices(), false) {} + : VPRecipeBase(VPWidenGEPSC, GEP), VPUser(Operands), + IsIndexLoopInvariant(GEP->getNumIndices(), false) {} template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands, Loop *OrigLoop) - : VPRecipeBase(VPWidenGEPSC), VPValue(VPValue::VPVWidenGEPSC, GEP), - VPUser(Operands), IsIndexLoopInvariant(GEP->getNumIndices(), false) { + : VPRecipeBase(VPValue::VPWidenGEPSC, GEP), VPUser(Operands), + IsIndexLoopInvariant(GEP->getNumIndices(), false) { IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand()); for (auto Index : enumerate(GEP->indices())) IsIndexLoopInvariant[Index.index()] = @@ -917,8 +874,8 @@ ~VPWidenGEPRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenGEPSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenGEPSC; } /// Generate the gep nodes. @@ -941,8 +898,8 @@ ~VPWidenIntOrFpInductionRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenIntOrFpInductionSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenIntOrFpInductionSC; } /// Generate the vectorized and scalarized versions of the phi node as @@ -963,8 +920,8 @@ ~VPWidenPHIRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenPHISC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenPHISC; } /// Generate the phi/select nodes. @@ -993,8 +950,8 @@ } /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPBlendSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPBlendSC; } /// Return the number of incoming values, taking into account that a single @@ -1017,7 +974,7 @@ /// VPInterleaveRecipe is a recipe for transforming an interleave group of load /// or stores into one wide load/store and shuffles. -class VPInterleaveRecipe : public VPRecipeBase, public VPValue, public VPUser { +class VPInterleaveRecipe : public VPRecipeBase, public VPUser { const InterleaveGroup *IG; public: @@ -1025,8 +982,7 @@ VPInterleaveRecipe(const InterleaveGroup *IG, VPValue *Addr, VPValue *Mask) - : VPRecipeBase(VPRecipeBase::VPInterleaveSC), - VPValue(VPValue::VPVInterleaveSC), VPUser({Addr}), IG(IG) { + : VPRecipeBase(VPValue::VPVInterleaveSC), VPUser({Addr}), IG(IG) { if (Mask) addOperand(Mask); for (unsigned i = 0; i < IG->getNumMembers(); i++) @@ -1038,11 +994,8 @@ } /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPInterleaveSC; - } static inline bool classof(const VPValue *V) { - return V->getVPValueID() == VPValue::VPVInterleaveSC; + return V->getVPValueID() == VPRecipeBase::VPVInterleaveSC; } /// Return the address accessed by this recipe. @@ -1095,8 +1048,8 @@ ~VPReductionRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPReductionSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPReductionSC; } /// Generate the reduction in the loop @@ -1141,8 +1094,8 @@ ~VPReplicateRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPReplicateSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPReplicateSC; } /// Generate replicas of the desired Ingredient. Replicas will be generated @@ -1166,8 +1119,8 @@ } /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPBranchOnMaskSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPBranchOnMaskSC; } /// Generate the extraction of the appropriate bit from the block mask and the @@ -1210,8 +1163,8 @@ ~VPPredInstPHIRecipe() override = default; /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPPredInstPHISC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPPredInstPHISC; } /// Generates phi nodes for live-outs as needed to retain SSA form. @@ -1228,9 +1181,7 @@ /// - For store: Address, stored value, optional mask /// TODO: We currently execute only per-part unless a specific instance is /// provided. -class VPWidenMemoryInstructionRecipe : public VPRecipeBase, - public VPValue, - public VPUser { +class VPWidenMemoryInstructionRecipe : public VPRecipeBase, public VPUser { void setMask(VPValue *Mask) { if (!Mask) @@ -1245,22 +1196,20 @@ public: VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask) - : VPRecipeBase(VPWidenMemoryInstructionSC), - VPValue(VPValue::VPMemoryInstructionSC, &Load), VPUser({Addr}) { + : VPRecipeBase(VPWidenMemoryInstructionSC, &Load), VPUser({Addr}) { setMask(Mask); } VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredValue, VPValue *Mask) - : VPRecipeBase(VPWidenMemoryInstructionSC), - VPValue(VPValue::VPMemoryInstructionSC, &Store), + : VPRecipeBase(VPWidenMemoryInstructionSC, &Store), VPUser({Addr, StoredValue}) { setMask(Mask); } /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenMemoryInstructionSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenMemoryInstructionSC; } /// Return the address accessed by this recipe. @@ -1305,8 +1254,8 @@ VPValue *getVPValue() { return &Val; } /// Method to support type inquiry through isa, cast, and dyn_cast. - static inline bool classof(const VPRecipeBase *V) { - return V->getVPRecipeID() == VPRecipeBase::VPWidenCanonicalIVSC; + static inline bool classof(const VPValue *V) { + return V->getVPValueID() == VPRecipeBase::VPWidenCanonicalIVSC; } /// Generate a canonical vector induction variable of the vector loop, with diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -101,34 +101,6 @@ return nullptr; } -VPValue *VPRecipeBase::toVPValue() { - if (auto *Widen = dyn_cast(this)) - return Widen; - if (auto *Widen = dyn_cast(this)) - return Widen; - if (auto *V = dyn_cast(this)) - return V; - if (auto *V = dyn_cast(this)) - return V; - if (auto *V = dyn_cast(this)) - return V; - return nullptr; -} - -const VPValue *VPRecipeBase::toVPValue() const { - if (auto *Widen = dyn_cast(this)) - return Widen; - if (auto *Widen = dyn_cast(this)) - return Widen; - if (auto *V = dyn_cast(this)) - return V; - if (auto *V = dyn_cast(this)) - return V; - if (auto *V = dyn_cast(this)) - return V; - return nullptr; -} - // Get the top-most entry block of \p Start. This is the entry block of the // containing VPlan. This function is templated to support both const and non-const blocks template static T *getPlanEntry(T *Start) { @@ -346,10 +318,8 @@ for (auto *Def : InterleaveR->Defs) Def->replaceAllUsesWith(NewValue); } else { - if (auto *VPV = R.toVPValue()) { - assert(VPV->isConcrete() && "virtual/sub-values need special handling"); - VPV->replaceAllUsesWith(NewValue); - } + assert(R.isConcrete() && "virtual/sub-values need special handling"); + R.replaceAllUsesWith(NewValue); } if (auto *User = R.toVPUser()) diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -114,12 +114,21 @@ enum { VPValueSC, VPVSubValueSC, + VPBlendSC, + VPBranchOnMaskSC, VPInstructionSC, - VPMemoryInstructionSC, - VPVWidenCallSC, - VPVWidenSelectSC, - VPVWidenGEPSC, - VPVInterleaveSC + VPVInterleaveSC, + VPPredInstPHISC, + VPReductionSC, + VPReplicateSC, + VPWidenCallSC, + VPWidenCanonicalIVSC, + VPWidenGEPSC, + VPWidenIntOrFpInductionSC, + VPWidenMemoryInstructionSC, + VPWidenPHISC, + VPWidenSC, + VPWidenSelectSC, }; VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV) {}