diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -756,12 +756,13 @@ }; // Helper macro to define common classof implementations for recipes. -#define VP_CLASSOF_IMPL(VPDefID, VPValueID) \ +#define VP_CLASSOF_IMPL(VPDefID) \ static inline bool classof(const VPDef *D) { \ return D->getVPDefID() == VPDefID; \ } \ static inline bool classof(const VPValue *V) { \ - return V->getVPValueID() == VPValueID; \ + auto *R = V->getDefiningRecipe(); \ + return R && R->getVPDefID() == VPDefID; \ } \ static inline bool classof(const VPUser *U) { \ auto *R = dyn_cast(U); \ @@ -818,15 +819,14 @@ public: VPInstruction(unsigned Opcode, ArrayRef Operands, DebugLoc DL, const Twine &Name = "") - : VPRecipeBase(VPDef::VPInstructionSC, Operands), - VPValue(VPValue::VPVInstructionSC, nullptr, this), Opcode(Opcode), - DL(DL), Name(Name.str()) {} + : VPRecipeBase(VPDef::VPInstructionSC, Operands), VPValue(this), + Opcode(Opcode), DL(DL), Name(Name.str()) {} VPInstruction(unsigned Opcode, std::initializer_list Operands, DebugLoc DL = {}, const Twine &Name = "") : VPInstruction(Opcode, ArrayRef(Operands), DL, Name) {} - VP_CLASSOF_IMPL(VPDef::VPInstructionSC, VPValue::VPVInstructionSC) + VP_CLASSOF_IMPL(VPDef::VPInstructionSC) VPInstruction *clone() const { SmallVector Operands(operands()); @@ -910,12 +910,11 @@ public: template VPWidenRecipe(Instruction &I, iterator_range Operands) - : VPRecipeBase(VPDef::VPWidenSC, Operands), - VPValue(VPValue::VPVWidenSC, &I, this) {} + : VPRecipeBase(VPDef::VPWidenSC, Operands), VPValue(this, &I) {} ~VPWidenRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPWidenSC, VPValue::VPVWidenSC) + VP_CLASSOF_IMPL(VPDef::VPWidenSC) /// Produce widened copies of all Ingredients. void execute(VPTransformState &State) override; @@ -937,13 +936,12 @@ template VPWidenCallRecipe(CallInst &I, iterator_range CallArguments, Intrinsic::ID VectorIntrinsicID) - : VPRecipeBase(VPDef::VPWidenCallSC, CallArguments), - VPValue(VPValue::VPVWidenCallSC, &I, this), + : VPRecipeBase(VPDef::VPWidenCallSC, CallArguments), VPValue(this, &I), VectorIntrinsicID(VectorIntrinsicID) {} ~VPWidenCallRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPWidenCallSC, VPValue::VPVWidenCallSC) + VP_CLASSOF_IMPL(VPDef::VPWidenCallSC) /// Produce a widened version of the call instruction. void execute(VPTransformState &State) override; @@ -965,13 +963,12 @@ template VPWidenSelectRecipe(SelectInst &I, iterator_range Operands, bool InvariantCond) - : VPRecipeBase(VPDef::VPWidenSelectSC, Operands), - VPValue(VPValue::VPVWidenSelectSC, &I, this), + : VPRecipeBase(VPDef::VPWidenSelectSC, Operands), VPValue(this, &I), InvariantCond(InvariantCond) {} ~VPWidenSelectRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC, VPValue::VPVWidenSelectSC) + VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC) /// Produce a widened version of the select instruction. void execute(VPTransformState &State) override; @@ -991,15 +988,13 @@ public: template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands) - : VPRecipeBase(VPDef::VPWidenGEPSC, Operands), - VPValue(VPWidenGEPSC, GEP, this), + : VPRecipeBase(VPDef::VPWidenGEPSC, Operands), VPValue(this, GEP), IsIndexLoopInvariant(GEP->getNumIndices(), false) {} template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands, Loop *OrigLoop) - : VPRecipeBase(VPDef::VPWidenGEPSC, Operands), - VPValue(VPValue::VPVWidenGEPSC, GEP, this), + : VPRecipeBase(VPDef::VPWidenGEPSC, Operands), VPValue(this, GEP), IsIndexLoopInvariant(GEP->getNumIndices(), false) { IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand()); for (auto Index : enumerate(GEP->indices())) @@ -1008,7 +1003,7 @@ } ~VPWidenGEPRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPWidenGEPSC, VPValue::VPVWidenGEPSC) + VP_CLASSOF_IMPL(VPDef::VPWidenGEPSC) /// Generate the gep nodes. void execute(VPTransformState &State) override; @@ -1032,20 +1027,19 @@ const InductionDescriptor &IndDesc, bool NeedsVectorIV) : VPRecipeBase(VPDef::VPWidenIntOrFpInductionSC, {Start, Step}), - VPValue(IV, this), IV(IV), IndDesc(IndDesc), + VPValue(this, IV), IV(IV), IndDesc(IndDesc), NeedsVectorIV(NeedsVectorIV) {} VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, TruncInst *Trunc, bool NeedsVectorIV) : VPRecipeBase(VPDef::VPWidenIntOrFpInductionSC, {Start, Step}), - VPValue(Trunc, this), IV(IV), IndDesc(IndDesc), + VPValue(this, Trunc), IV(IV), IndDesc(IndDesc), NeedsVectorIV(NeedsVectorIV) {} ~VPWidenIntOrFpInductionRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionSC, - VPValue::VPVWidenIntOrFpInductionSC) + VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionSC) /// Generate the vectorized and scalarized versions of the phi node as /// needed by their users. @@ -1118,9 +1112,9 @@ /// per-lane based on the canonical induction. class VPHeaderPHIRecipe : public VPRecipeBase, public VPValue { protected: - VPHeaderPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi, + VPHeaderPHIRecipe(unsigned char VPDefID, PHINode *Phi, VPValue *Start = nullptr) - : VPRecipeBase(VPDefID, {}), VPValue(VPVID, Phi, this) { + : VPRecipeBase(VPDefID, {}), VPValue(this, Phi) { if (Start) addOperand(Start); } @@ -1134,8 +1128,9 @@ B->getVPDefID() <= VPDef::VPLastPHISC; } static inline bool classof(const VPValue *V) { - return V->getVPValueID() >= VPValue::VPVFirstHeaderPHISC && - V->getVPValueID() <= VPValue::VPVLastPHISC; + auto *B = V->getDefiningRecipe(); + return B && B->getVPDefID() >= VPRecipeBase::VPFirstHeaderPHISC && + B->getVPDefID() <= VPRecipeBase::VPLastPHISC; } /// Generate the phi nodes. @@ -1181,8 +1176,7 @@ VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, bool IsScalarAfterVectorization) - : VPHeaderPHIRecipe(VPVWidenPointerInductionSC, - VPDef::VPWidenPointerInductionSC, Phi), + : VPHeaderPHIRecipe(VPDef::VPWidenPointerInductionSC, Phi), IndDesc(IndDesc), IsScalarAfterVectorization(IsScalarAfterVectorization) { addOperand(Start); @@ -1191,8 +1185,7 @@ ~VPWidenPointerInductionRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPWidenPointerInductionSC, - VPValue::VPVWidenPointerInductionSC) + VP_CLASSOF_IMPL(VPDef::VPWidenPointerInductionSC) /// Generate vector values for the pointer induction. void execute(VPTransformState &State) override; @@ -1220,14 +1213,14 @@ public: /// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start. VPWidenPHIRecipe(PHINode *Phi, VPValue *Start = nullptr) - : VPHeaderPHIRecipe(VPVWidenPHISC, VPDef::VPWidenPHISC, Phi) { + : VPHeaderPHIRecipe(VPDef::VPWidenPHISC, Phi) { if (Start) addOperand(Start); } ~VPWidenPHIRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPWidenPHISC, VPValue::VPVWidenPHISC) + VP_CLASSOF_IMPL(VPDef::VPWidenPHISC) /// Generate the phi/select nodes. void execute(VPTransformState &State) override; @@ -1256,11 +1249,9 @@ /// second operand. struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe { VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start) - : VPHeaderPHIRecipe(VPVFirstOrderRecurrencePHISC, - VPDef::VPFirstOrderRecurrencePHISC, Phi, &Start) {} + : VPHeaderPHIRecipe(VPDef::VPFirstOrderRecurrencePHISC, Phi, &Start) {} - VP_CLASSOF_IMPL(VPDef::VPFirstOrderRecurrencePHISC, - VPValue::VPVFirstOrderRecurrencePHISC) + VP_CLASSOF_IMPL(VPDef::VPFirstOrderRecurrencePHISC) static inline bool classof(const VPHeaderPHIRecipe *R) { return R->getVPDefID() == VPDef::VPFirstOrderRecurrencePHISC; @@ -1294,15 +1285,14 @@ VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc, VPValue &Start, bool IsInLoop = false, bool IsOrdered = false) - : VPHeaderPHIRecipe(VPVReductionPHISC, VPDef::VPReductionPHISC, Phi, - &Start), + : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) { assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop"); } ~VPReductionPHIRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPReductionPHISC, VPValue::VPVReductionPHISC) + VP_CLASSOF_IMPL(VPDef::VPReductionPHISC) static inline bool classof(const VPHeaderPHIRecipe *R) { return R->getVPDefID() == VPDef::VPReductionPHISC; @@ -1338,15 +1328,14 @@ /// respective masks, ordered [I0, M0, I1, M1, ...]. Note that a single value /// might be incoming with a full mask for which there is no VPValue. VPBlendRecipe(PHINode *Phi, ArrayRef Operands) - : VPRecipeBase(VPDef::VPBlendSC, Operands), - VPValue(VPValue::VPVBlendSC, Phi, this), Phi(Phi) { + : VPRecipeBase(VPDef::VPBlendSC, Operands), VPValue(this, Phi), Phi(Phi) { assert(Operands.size() > 0 && ((Operands.size() == 1) || (Operands.size() % 2 == 0)) && "Expected either a single incoming value or a positive even number " "of operands"); } - VP_CLASSOF_IMPL(VPDef::VPBlendSC, VPValue::VPVBlendSC) + VP_CLASSOF_IMPL(VPDef::VPBlendSC) /// Return the number of incoming values, taking into account that a single /// incoming value has no mask. @@ -1470,15 +1459,15 @@ VPReductionRecipe(const RecurrenceDescriptor *R, Instruction *I, VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp, const TargetTransformInfo *TTI) - : VPRecipeBase(VPDef::VPReductionSC, {ChainOp, VecOp}), - VPValue(VPValue::VPVReductionSC, I, this), RdxDesc(R), TTI(TTI) { + : VPRecipeBase(VPDef::VPReductionSC, {ChainOp, VecOp}), VPValue(this, I), + RdxDesc(R), TTI(TTI) { if (CondOp) addOperand(CondOp); } ~VPReductionRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPReductionSC, VPValue::VPVReductionSC) + VP_CLASSOF_IMPL(VPDef::VPReductionSC) /// Generate the reduction in the loop void execute(VPTransformState &State) override; @@ -1517,9 +1506,8 @@ template VPReplicateRecipe(Instruction *I, iterator_range Operands, bool IsUniform, bool IsPredicated = false) - : VPRecipeBase(VPDef::VPReplicateSC, Operands), - VPValue(VPVReplicateSC, I, this), IsUniform(IsUniform), - IsPredicated(IsPredicated) { + : VPRecipeBase(VPDef::VPReplicateSC, Operands), VPValue(this, I), + IsUniform(IsUniform), IsPredicated(IsPredicated) { // Retain the previous behavior of predicateInstructions(), where an // insert-element of a predicated instruction got hoisted into the // predicated basic block iff it was its only user. This is achieved by @@ -1530,7 +1518,7 @@ ~VPReplicateRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPReplicateSC, VPValue::VPVReplicateSC) + VP_CLASSOF_IMPL(VPDef::VPReplicateSC) /// Generate replicas of the desired Ingredient. Replicas will be generated /// for all parts and lanes unless a specific part and lane are specified in @@ -1622,11 +1610,10 @@ /// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi /// nodes after merging back from a Branch-on-Mask. VPPredInstPHIRecipe(VPValue *PredV) - : VPRecipeBase(VPDef::VPPredInstPHISC, PredV), - VPValue(VPValue::VPVPredInstPHI, nullptr, this) {} + : VPRecipeBase(VPDef::VPPredInstPHISC, PredV), VPValue(this) {} ~VPPredInstPHIRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPPredInstPHISC, VPValue::VPVPredInstPHI) + VP_CLASSOF_IMPL(VPDef::VPPredInstPHISC) /// Generates phi nodes for live-outs as needed to retain SSA form. void execute(VPTransformState &State) override; @@ -1676,7 +1663,7 @@ : VPRecipeBase(VPDef::VPWidenMemoryInstructionSC, {Addr}), Ingredient(Load), Consecutive(Consecutive), Reverse(Reverse) { assert((Consecutive || !Reverse) && "Reverse implies consecutive"); - new VPValue(VPValue::VPVMemoryInstructionSC, &Load, this); + new VPValue(this, &Load); setMask(Mask); } @@ -1689,8 +1676,7 @@ setMask(Mask); } - VP_CLASSOF_IMPL(VPDef::VPWidenMemoryInstructionSC, - VPValue::VPVMemoryInstructionSC) + VP_CLASSOF_IMPL(VPDef::VPWidenMemoryInstructionSC) /// Return the address accessed by this recipe. VPValue *getAddr() const { @@ -1751,8 +1737,8 @@ public: VPExpandSCEVRecipe(const SCEV *Expr, ScalarEvolution &SE) - : VPRecipeBase(VPDef::VPExpandSCEVSC, {}), VPValue(nullptr, this), - Expr(Expr), SE(SE) {} + : VPRecipeBase(VPDef::VPExpandSCEVSC, {}), VPValue(this), Expr(Expr), + SE(SE) {} ~VPExpandSCEVRecipe() override = default; @@ -1782,13 +1768,11 @@ public: VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL) - : VPHeaderPHIRecipe(VPValue::VPVCanonicalIVPHISC, - VPDef::VPCanonicalIVPHISC, nullptr, StartV), - DL(DL) {} + : VPHeaderPHIRecipe(VPDef::VPCanonicalIVPHISC, nullptr, StartV), DL(DL) {} ~VPCanonicalIVPHIRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPCanonicalIVPHISC, VPValue::VPVCanonicalIVPHISC) + VP_CLASSOF_IMPL(VPDef::VPCanonicalIVPHISC) static inline bool classof(const VPHeaderPHIRecipe *D) { return D->getVPDefID() == VPDef::VPCanonicalIVPHISC; @@ -1829,13 +1813,12 @@ public: VPActiveLaneMaskPHIRecipe(VPValue *StartMask, DebugLoc DL) - : VPHeaderPHIRecipe(VPValue::VPVActiveLaneMaskPHISC, - VPDef::VPActiveLaneMaskPHISC, nullptr, StartMask), + : VPHeaderPHIRecipe(VPDef::VPActiveLaneMaskPHISC, nullptr, StartMask), DL(DL) {} ~VPActiveLaneMaskPHIRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPActiveLaneMaskPHISC, VPValue::VPVActiveLaneMaskPHISC) + VP_CLASSOF_IMPL(VPDef::VPActiveLaneMaskPHISC) static inline bool classof(const VPHeaderPHIRecipe *D) { return D->getVPDefID() == VPDef::VPActiveLaneMaskPHISC; @@ -1856,11 +1839,11 @@ public: VPWidenCanonicalIVRecipe(VPCanonicalIVPHIRecipe *CanonicalIV) : VPRecipeBase(VPDef::VPWidenCanonicalIVSC, {CanonicalIV}), - VPValue(VPValue::VPVWidenCanonicalIVSC, nullptr, this) {} + VPValue(this) {} ~VPWidenCanonicalIVRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPWidenCanonicalIVSC, VPValue::VPVWidenCanonicalIVSC) + VP_CLASSOF_IMPL(VPDef::VPWidenCanonicalIVSC) /// Generate a canonical vector induction variable of the vector loop, with /// start = { for 0 <= Part < UF}, and @@ -1896,12 +1879,11 @@ VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step, Type *ResultTy) : VPRecipeBase(VPDef::VPDerivedIVSC, {Start, CanonicalIV, Step}), - VPValue(VPVDerivedIVSC, nullptr, this), ResultTy(ResultTy), - IndDesc(IndDesc) {} + VPValue(this), ResultTy(ResultTy), IndDesc(IndDesc) {} ~VPDerivedIVRecipe() override = default; - VP_CLASSOF_IMPL(VPDef::VPDerivedIVSC, VPValue::VPVInstructionSC) + VP_CLASSOF_IMPL(VPDef::VPDerivedIVSC) /// Generate the transformed value of the induction at offset StartValue (1. /// operand) + IV (2. operand) * StepValue (3, operand). @@ -1933,8 +1915,8 @@ public: VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV, VPValue *Step) - : VPRecipeBase(VPDef::VPScalarIVStepsSC, {IV, Step}), - VPValue(nullptr, this), IndDesc(IndDesc) {} + : VPRecipeBase(VPDef::VPScalarIVStepsSC, {IV, Step}), VPValue(this), + IndDesc(IndDesc) {} ~VPScalarIVStepsRecipe() override = default; diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -85,39 +85,19 @@ const Value *getUnderlyingValue() const { return UnderlyingVal; } /// An enumeration for keeping track of the concrete subclass of VPValue that - /// are actually instantiated. Values of this enumeration are kept in the - /// SubclassID field of the VPValue objects. They are used for concrete - /// type identification. + /// are actually instantiated. enum { - VPValueSC, - VPVDerivedIVSC, - VPVInstructionSC, - VPVMemoryInstructionSC, - VPVReductionSC, - VPVReplicateSC, - VPVWidenSC, - VPVWidenCallSC, - VPVWidenCanonicalIVSC, - VPVWidenGEPSC, - VPVWidenSelectSC, - - // Phi-like VPValues. Need to be kept together. - VPVBlendSC, - VPVPredInstPHI, - // Header-phi recipes. Need to be kept together. - VPVCanonicalIVPHISC, - VPVActiveLaneMaskPHISC, - VPVFirstOrderRecurrencePHISC, - VPVWidenPHISC, - VPVWidenIntOrFpInductionSC, - VPVWidenPointerInductionSC, - VPVReductionPHISC, - VPVFirstHeaderPHISC = VPVCanonicalIVPHISC, - VPVLastPHISC = VPVReductionPHISC, + VPValueSC, /// A generic VPValue, like live-in values or defined by a recipe + /// that defines multiple values. + VPVRecipeSC /// A VPValue sub-class that is a VPRecipeBase. }; - VPValue(Value *UV = nullptr, VPDef *Def = nullptr) - : VPValue(VPValueSC, UV, Def) {} + /// Create a live-in VPValue. + VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV, nullptr) {} + /// Create a VPValue for a \p Def which is a subclass of VPValue. + VPValue(VPDef *Def, Value *UV = nullptr) : VPValue(VPVRecipeSC, UV, Def) {} + /// Create a VPValue for a \p Def which defines multiple values. + VPValue(Value *UV, VPDef *Def) : VPValue(VPValueSC, UV, Def) {} VPValue(const VPValue &) = delete; VPValue &operator=(const VPValue &) = delete;