diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -770,12 +770,13 @@ } // Helper macro to define common classof implementations for recipes. -#define VP_CLASSOF_IMPL(VPDefID, VPValueID) \ +#define VP_CLASSOF_IMPL(VPDefID) \ static inline bool classof(const VPDef *D) { \ return D->getVPDefID() == VPDefID; \ } \ static inline bool classof(const VPValue *V) { \ - return V->getVPValueID() == VPValueID; \ + auto *R = V->getDefiningRecipe(); \ + return R && R->getVPDefID() == VPDefID; \ } \ static inline bool classof(const VPUser *U) { \ auto *R = dyn_cast(U); \ @@ -832,15 +833,14 @@ public: VPInstruction(unsigned Opcode, ArrayRef Operands, DebugLoc DL, const Twine &Name = "") - : VPRecipeBase(VPRecipeBase::VPInstructionSC, Operands), - VPValue(VPValue::VPVInstructionSC, nullptr, this), Opcode(Opcode), - DL(DL), Name(Name.str()) {} + : VPRecipeBase(VPRecipeBase::VPInstructionSC, Operands), VPValue(this), + Opcode(Opcode), DL(DL), Name(Name.str()) {} VPInstruction(unsigned Opcode, std::initializer_list Operands, DebugLoc DL = {}, const Twine &Name = "") : VPInstruction(Opcode, ArrayRef(Operands), DL, Name) {} - VP_CLASSOF_IMPL(VPRecipeBase::VPInstructionSC, VPValue::VPVInstructionSC) + VP_CLASSOF_IMPL(VPRecipeBase::VPInstructionSC) VPInstruction *clone() const { SmallVector Operands(operands()); @@ -924,12 +924,11 @@ public: template VPWidenRecipe(Instruction &I, iterator_range Operands) - : VPRecipeBase(VPRecipeBase::VPWidenSC, Operands), - VPValue(VPValue::VPVWidenSC, &I, this) {} + : VPRecipeBase(VPWidenSC, Operands), VPValue(this, &I) {} ~VPWidenRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPWidenSC, VPValue::VPVWidenSC) + VP_CLASSOF_IMPL(VPRecipeBase::VPWidenSC) /// Produce widened copies of all Ingredients. void execute(VPTransformState &State) override; @@ -952,12 +951,11 @@ VPWidenCallRecipe(CallInst &I, iterator_range CallArguments, Intrinsic::ID VectorIntrinsicID) : VPRecipeBase(VPRecipeBase::VPWidenCallSC, CallArguments), - VPValue(VPValue::VPVWidenCallSC, &I, this), - VectorIntrinsicID(VectorIntrinsicID) {} + VPValue(this, &I), VectorIntrinsicID(VectorIntrinsicID) {} ~VPWidenCallRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPWidenCallSC, VPValue::VPVWidenCallSC) + VP_CLASSOF_IMPL(VPRecipeBase::VPWidenCallSC) /// Produce a widened version of the call instruction. void execute(VPTransformState &State) override; @@ -980,12 +978,11 @@ VPWidenSelectRecipe(SelectInst &I, iterator_range Operands, bool InvariantCond) : VPRecipeBase(VPRecipeBase::VPWidenSelectSC, Operands), - VPValue(VPValue::VPVWidenSelectSC, &I, this), - InvariantCond(InvariantCond) {} + VPValue(this, &I), InvariantCond(InvariantCond) {} ~VPWidenSelectRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPWidenSelectSC, VPValue::VPVWidenSelectSC) + VP_CLASSOF_IMPL(VPRecipeBase::VPWidenSelectSC) /// Produce a widened version of the select instruction. void execute(VPTransformState &State) override; @@ -1005,15 +1002,13 @@ public: template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands) - : VPRecipeBase(VPRecipeBase::VPWidenGEPSC, Operands), - VPValue(VPWidenGEPSC, GEP, this), + : VPRecipeBase(VPRecipeBase::VPWidenGEPSC, Operands), VPValue(this, GEP), IsIndexLoopInvariant(GEP->getNumIndices(), false) {} template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands, Loop *OrigLoop) - : VPRecipeBase(VPRecipeBase::VPWidenGEPSC, Operands), - VPValue(VPValue::VPVWidenGEPSC, GEP, this), + : VPRecipeBase(VPRecipeBase::VPWidenGEPSC, Operands), VPValue(this, GEP), IsIndexLoopInvariant(GEP->getNumIndices(), false) { IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand()); for (auto Index : enumerate(GEP->indices())) @@ -1022,7 +1017,7 @@ } ~VPWidenGEPRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPWidenGEPSC, VPValue::VPVWidenGEPSC) + VP_CLASSOF_IMPL(VPRecipeBase::VPWidenGEPSC) /// Generate the gep nodes. void execute(VPTransformState &State) override; @@ -1046,20 +1041,19 @@ const InductionDescriptor &IndDesc, bool NeedsVectorIV) : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start, Step}), - VPValue(IV, this), IV(IV), IndDesc(IndDesc), + VPValue(this, IV), IV(IV), IndDesc(IndDesc), NeedsVectorIV(NeedsVectorIV) {} VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, TruncInst *Trunc, bool NeedsVectorIV) : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start, Step}), - VPValue(Trunc, this), IV(IV), IndDesc(IndDesc), + VPValue(this, Trunc), IV(IV), IndDesc(IndDesc), NeedsVectorIV(NeedsVectorIV) {} ~VPWidenIntOrFpInductionRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPWidenIntOrFpInductionSC, - VPValue::VPVWidenIntOrFpInductionSC) + VP_CLASSOF_IMPL(VPRecipeBase::VPWidenIntOrFpInductionSC) /// Generate the vectorized and scalarized versions of the phi node as /// needed by their users. @@ -1132,9 +1126,9 @@ /// per-lane based on the canonical induction. class VPHeaderPHIRecipe : public VPRecipeBase, public VPValue { protected: - VPHeaderPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi, + VPHeaderPHIRecipe(unsigned char VPDefID, PHINode *Phi, VPValue *Start = nullptr) - : VPRecipeBase(VPDefID, {}), VPValue(VPVID, Phi, this) { + : VPRecipeBase(VPDefID, {}), VPValue(this, Phi) { if (Start) addOperand(Start); } @@ -1148,8 +1142,9 @@ B->getVPDefID() <= VPRecipeBase::VPLastPHISC; } static inline bool classof(const VPValue *V) { - return V->getVPValueID() >= VPValue::VPVFirstHeaderPHISC && - V->getVPValueID() <= VPValue::VPVLastPHISC; + auto *B = V->getDefiningRecipe(); + return B && B->getVPDefID() >= VPRecipeBase::VPFirstHeaderPHISC && + B->getVPDefID() <= VPRecipeBase::VPLastPHISC; } /// Generate the phi nodes. @@ -1195,9 +1190,7 @@ VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, bool IsScalarAfterVectorization) - : VPHeaderPHIRecipe(VPVWidenPointerInductionSC, VPWidenPointerInductionSC, - Phi), - IndDesc(IndDesc), + : VPHeaderPHIRecipe(VPWidenPointerInductionSC, Phi), IndDesc(IndDesc), IsScalarAfterVectorization(IsScalarAfterVectorization) { addOperand(Start); addOperand(Step); @@ -1205,8 +1198,7 @@ ~VPWidenPointerInductionRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPWidenPointerInductionSC, - VPValue::VPVWidenPointerInductionSC) + VP_CLASSOF_IMPL(VPRecipeBase::VPWidenPointerInductionSC) /// Generate vector values for the pointer induction. void execute(VPTransformState &State) override; @@ -1234,14 +1226,14 @@ public: /// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start. VPWidenPHIRecipe(PHINode *Phi, VPValue *Start = nullptr) - : VPHeaderPHIRecipe(VPVWidenPHISC, VPWidenPHISC, Phi) { + : VPHeaderPHIRecipe(VPWidenPHISC, Phi) { if (Start) addOperand(Start); } ~VPWidenPHIRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPWidenPHISC, VPValue::VPVWidenPHISC) + VP_CLASSOF_IMPL(VPRecipeBase::VPWidenPHISC) /// Generate the phi/select nodes. void execute(VPTransformState &State) override; @@ -1270,11 +1262,9 @@ /// second operand. struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe { VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start) - : VPHeaderPHIRecipe(VPVFirstOrderRecurrencePHISC, - VPFirstOrderRecurrencePHISC, Phi, &Start) {} + : VPHeaderPHIRecipe(VPFirstOrderRecurrencePHISC, Phi, &Start) {} - VP_CLASSOF_IMPL(VPRecipeBase::VPFirstOrderRecurrencePHISC, - VPValue::VPVFirstOrderRecurrencePHISC) + VP_CLASSOF_IMPL(VPRecipeBase::VPFirstOrderRecurrencePHISC) static inline bool classof(const VPHeaderPHIRecipe *R) { return R->getVPDefID() == VPRecipeBase::VPFirstOrderRecurrencePHISC; @@ -1308,14 +1298,14 @@ VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc, VPValue &Start, bool IsInLoop = false, bool IsOrdered = false) - : VPHeaderPHIRecipe(VPVReductionPHISC, VPReductionPHISC, Phi, &Start), - RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) { + : VPHeaderPHIRecipe(VPReductionPHISC, Phi, &Start), RdxDesc(RdxDesc), + IsInLoop(IsInLoop), IsOrdered(IsOrdered) { assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop"); } ~VPReductionPHIRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPReductionPHISC, VPValue::VPVReductionPHISC) + VP_CLASSOF_IMPL(VPRecipeBase::VPReductionPHISC) static inline bool classof(const VPHeaderPHIRecipe *R) { return R->getVPDefID() == VPRecipeBase::VPReductionPHISC; @@ -1351,15 +1341,14 @@ /// respective masks, ordered [I0, M0, I1, M1, ...]. Note that a single value /// might be incoming with a full mask for which there is no VPValue. VPBlendRecipe(PHINode *Phi, ArrayRef Operands) - : VPRecipeBase(VPBlendSC, Operands), - VPValue(VPValue::VPVBlendSC, Phi, this), Phi(Phi) { + : VPRecipeBase(VPBlendSC, Operands), VPValue(this, Phi), Phi(Phi) { assert(Operands.size() > 0 && ((Operands.size() == 1) || (Operands.size() % 2 == 0)) && "Expected either a single incoming value or a positive even number " "of operands"); } - VP_CLASSOF_IMPL(VPRecipeBase::VPBlendSC, VPValue::VPVBlendSC) + VP_CLASSOF_IMPL(VPRecipeBase::VPBlendSC) /// Return the number of incoming values, taking into account that a single /// incoming value has no mask. @@ -1484,14 +1473,14 @@ VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp, const TargetTransformInfo *TTI) : VPRecipeBase(VPRecipeBase::VPReductionSC, {ChainOp, VecOp}), - VPValue(VPValue::VPVReductionSC, I, this), RdxDesc(R), TTI(TTI) { + VPValue(this, I), RdxDesc(R), TTI(TTI) { if (CondOp) addOperand(CondOp); } ~VPReductionRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPReductionSC, VPValue::VPVReductionSC) + VP_CLASSOF_IMPL(VPRecipeBase::VPReductionSC) /// Generate the reduction in the loop void execute(VPTransformState &State) override; @@ -1530,7 +1519,7 @@ template VPReplicateRecipe(Instruction *I, iterator_range Operands, bool IsUniform, bool IsPredicated = false) - : VPRecipeBase(VPReplicateSC, Operands), VPValue(VPVReplicateSC, I, this), + : VPRecipeBase(VPReplicateSC, Operands), VPValue(this, I), IsUniform(IsUniform), IsPredicated(IsPredicated) { // Retain the previous behavior of predicateInstructions(), where an // insert-element of a predicated instruction got hoisted into the @@ -1542,7 +1531,7 @@ ~VPReplicateRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPReplicateSC, VPValue::VPVReplicateSC) + VP_CLASSOF_IMPL(VPRecipeBase::VPReplicateSC) /// Generate replicas of the desired Ingredient. Replicas will be generated /// for all parts and lanes unless a specific part and lane are specified in @@ -1634,11 +1623,10 @@ /// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi /// nodes after merging back from a Branch-on-Mask. VPPredInstPHIRecipe(VPValue *PredV) - : VPRecipeBase(VPPredInstPHISC, PredV), - VPValue(VPValue::VPVPredInstPHI, nullptr, this) {} + : VPRecipeBase(VPPredInstPHISC, PredV), VPValue(this) {} ~VPPredInstPHIRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPPredInstPHISC, VPValue::VPVPredInstPHI) + VP_CLASSOF_IMPL(VPRecipeBase::VPPredInstPHISC) /// Generates phi nodes for live-outs as needed to retain SSA form. void execute(VPTransformState &State) override; @@ -1688,7 +1676,7 @@ : VPRecipeBase(VPWidenMemoryInstructionSC, {Addr}), Ingredient(Load), Consecutive(Consecutive), Reverse(Reverse) { assert((Consecutive || !Reverse) && "Reverse implies consecutive"); - new VPValue(VPValue::VPVMemoryInstructionSC, &Load, this); + new VPValue(this, &Load); setMask(Mask); } @@ -1701,8 +1689,7 @@ setMask(Mask); } - VP_CLASSOF_IMPL(VPRecipeBase::VPWidenMemoryInstructionSC, - VPValue::VPVMemoryInstructionSC) + VP_CLASSOF_IMPL(VPRecipeBase::VPWidenMemoryInstructionSC) /// Return the address accessed by this recipe. VPValue *getAddr() const { @@ -1763,8 +1750,7 @@ public: VPExpandSCEVRecipe(const SCEV *Expr, ScalarEvolution &SE) - : VPRecipeBase(VPExpandSCEVSC, {}), VPValue(nullptr, this), Expr(Expr), - SE(SE) {} + : VPRecipeBase(VPExpandSCEVSC, {}), VPValue(this), Expr(Expr), SE(SE) {} ~VPExpandSCEVRecipe() override = default; @@ -1794,14 +1780,11 @@ public: VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL) - : VPHeaderPHIRecipe(VPValue::VPVCanonicalIVPHISC, VPCanonicalIVPHISC, - nullptr, StartV), - DL(DL) {} + : VPHeaderPHIRecipe(VPCanonicalIVPHISC, nullptr, StartV), DL(DL) {} ~VPCanonicalIVPHIRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPCanonicalIVPHISC, - VPValue::VPVCanonicalIVPHISC) + VP_CLASSOF_IMPL(VPRecipeBase::VPCanonicalIVPHISC) static inline bool classof(const VPHeaderPHIRecipe *D) { return D->getVPDefID() == VPCanonicalIVPHISC; @@ -1842,14 +1825,11 @@ public: VPActiveLaneMaskPHIRecipe(VPValue *StartMask, DebugLoc DL) - : VPHeaderPHIRecipe(VPValue::VPVActiveLaneMaskPHISC, - VPActiveLaneMaskPHISC, nullptr, StartMask), - DL(DL) {} + : VPHeaderPHIRecipe(VPActiveLaneMaskPHISC, nullptr, StartMask), DL(DL) {} ~VPActiveLaneMaskPHIRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPActiveLaneMaskPHISC, - VPValue::VPVActiveLaneMaskPHISC) + VP_CLASSOF_IMPL(VPRecipeBase::VPActiveLaneMaskPHISC) static inline bool classof(const VPHeaderPHIRecipe *D) { return D->getVPDefID() == VPActiveLaneMaskPHISC; @@ -1869,13 +1849,11 @@ class VPWidenCanonicalIVRecipe : public VPRecipeBase, public VPValue { public: VPWidenCanonicalIVRecipe(VPCanonicalIVPHIRecipe *CanonicalIV) - : VPRecipeBase(VPWidenCanonicalIVSC, {CanonicalIV}), - VPValue(VPValue::VPVWidenCanonicalIVSC, nullptr, this) {} + : VPRecipeBase(VPWidenCanonicalIVSC, {CanonicalIV}), VPValue(this) {} ~VPWidenCanonicalIVRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPWidenCanonicalIVSC, - VPValue::VPVWidenCanonicalIVSC) + VP_CLASSOF_IMPL(VPRecipeBase::VPWidenCanonicalIVSC) /// Generate a canonical vector induction variable of the vector loop, with /// start = { for 0 <= Part < UF}, and @@ -1910,13 +1888,12 @@ VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPValue *Start, VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step, Type *ResultTy) - : VPRecipeBase(VPDerivedIVSC, {Start, CanonicalIV, Step}), - VPValue(VPVDerivedIVSC, nullptr, this), ResultTy(ResultTy), - IndDesc(IndDesc) {} + : VPRecipeBase(VPDerivedIVSC, {Start, CanonicalIV, Step}), VPValue(this), + ResultTy(ResultTy), IndDesc(IndDesc) {} ~VPDerivedIVRecipe() override = default; - VP_CLASSOF_IMPL(VPRecipeBase::VPDerivedIVSC, VPValue::VPVInstructionSC) + VP_CLASSOF_IMPL(VPRecipeBase::VPDerivedIVSC) /// Generate the transformed value of the induction at offset StartValue (1. /// operand) + IV (2. operand) * StepValue (3, operand). @@ -1948,7 +1925,7 @@ public: VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV, VPValue *Step) - : VPRecipeBase(VPScalarIVStepsSC, {IV, Step}), VPValue(nullptr, this), + : VPRecipeBase(VPScalarIVStepsSC, {IV, Step}), VPValue(this), IndDesc(IndDesc) {} ~VPScalarIVStepsRecipe() override = default; diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -85,40 +85,17 @@ const Value *getUnderlyingValue() const { return UnderlyingVal; } /// An enumeration for keeping track of the concrete subclass of VPValue that - /// are actually instantiated. Values of this enumeration are kept in the - /// SubclassID field of the VPValue objects. They are used for concrete - /// type identification. + /// are actually instantiated. enum { - VPValueSC, - VPVDerivedIVSC, - VPVInstructionSC, - VPVMemoryInstructionSC, - VPVReductionSC, - VPVReplicateSC, - VPVWidenSC, - VPVWidenCallSC, - VPVWidenCanonicalIVSC, - VPVWidenGEPSC, - VPVWidenSelectSC, - - // Phi-like VPValues. Need to be kept together. - VPVBlendSC, - VPVPredInstPHI, - // Header-phi recipes. Need to be kept together. - VPVCanonicalIVPHISC, - VPVActiveLaneMaskPHISC, - VPVFirstOrderRecurrencePHISC, - VPVWidenPHISC, - VPVWidenIntOrFpInductionSC, - VPVWidenPointerInductionSC, - VPVReductionPHISC, - VPVFirstHeaderPHISC = VPVCanonicalIVPHISC, - VPVLastPHISC = VPVReductionPHISC, + VPValueSC, /// A generic VPValue, like live-in values. + VPVRecipeSC // A VPValue sub-class that is a VPRecipeBase and also inherits + // from VPValue }; VPValue(Value *UV = nullptr, VPDef *Def = nullptr) : VPValue(VPValueSC, UV, Def) {} VPValue(const VPValue &) = delete; + VPValue(VPDef *Def, Value *UV = nullptr) : VPValue(VPVRecipeSC, UV, Def) {} VPValue &operator=(const VPValue &) = delete; virtual ~VPValue();