diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -923,9 +923,9 @@ VPWidenIntOrFpInductionRecipe(PHINode *IV, TruncInst *Trunc = nullptr) : VPRecipeBase(VPWidenIntOrFpInductionSC), IV(IV), Trunc(Trunc) { if (Trunc) - new VPValue(Trunc, this); + new VPValue(VPValue::VPValueSubSC, Trunc, this); else - new VPValue(IV, this); + new VPValue(VPValue::VPValueSubSC, IV, this); } ~VPWidenIntOrFpInductionRecipe() override = default; @@ -949,7 +949,7 @@ public: VPWidenPHIRecipe(PHINode *Phi) : VPRecipeBase(VPWidenPHISC), Phi(Phi) { - new VPValue(Phi, this); + new VPValue(VPValue::VPValueSubSC, Phi, this); } ~VPWidenPHIRecipe() override = default; @@ -977,7 +977,7 @@ /// might be incoming with a full mask for which there is no VPValue. VPBlendRecipe(PHINode *Phi, ArrayRef Operands) : VPRecipeBase(VPBlendSC), VPUser(Operands), Phi(Phi) { - new VPValue(Phi, this); + new VPValue(VPValue::VPValueSubSC, Phi, this); assert(Operands.size() > 0 && ((Operands.size() == 1) || (Operands.size() % 2 == 0)) && "Expected either a single incoming value or a positive even number " @@ -1024,7 +1024,7 @@ if (Instruction *I = IG->getMember(i)) { if (I->getType()->isVoidTy()) continue; - new VPValue(I, this); + new VPValue(VPValue::VPValueSubSC, I, this); } for (auto *SV : StoredValues) @@ -1226,7 +1226,7 @@ /// nodes after merging back from a Branch-on-Mask. VPPredInstPHIRecipe(VPValue *PredV) : VPRecipeBase(VPPredInstPHISC), VPUser(PredV) { - new VPValue(VPValue::VPValueSC, PredV->getUnderlyingValue(), this); + new VPValue(VPValue::VPValueSubSC, PredV->getUnderlyingValue(), this); } ~VPPredInstPHIRecipe() override = default; @@ -1314,7 +1314,7 @@ class VPWidenCanonicalIVRecipe : public VPRecipeBase { public: VPWidenCanonicalIVRecipe() : VPRecipeBase(VPWidenCanonicalIVSC) { - new VPValue(nullptr, this); + new VPValue(VPValue::VPValueSubSC, nullptr, this); } ~VPWidenCanonicalIVRecipe() override = default; diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1049,6 +1049,12 @@ OS << "vp<%" << Tracker.getSlot(this) << ">"; } +Value *VPValue::getOutOfScopeIRValue() { + if (getVPValueID() != VPValue::VPValueSC) + return nullptr; + return getUnderlyingValue(); +} + void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const { bool First = true; for (VPValue *Op : operands()) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -34,8 +34,13 @@ class VPDef; class VPSlotTracker; class VPUser; -class VPRecipeBase; +class VPBlendRecipe; +class VPInterleaveRecipe; +class VPWidenCanonicalIVRecipe; +class VPWidenIntOrFpInductionRecipe; +class VPWidenPHIRecipe; class VPPredInstPHIRecipe; +class VPRecipeBase; // This is the base class of the VPlan Def/Use graph, used for modeling the data // flow into, within and out of the VPlan. VPValues can stand for live-ins @@ -49,8 +54,13 @@ friend class VPBasicBlock; friend class VPInterleavedAccessInfo; friend class VPSlotTracker; - friend class VPRecipeBase; + friend class VPBlendRecipe; + friend class VPInterleaveRecipe; + friend class VPWidenCanonicalIVRecipe; + friend class VPWidenIntOrFpInductionRecipe; + friend class VPWidenPHIRecipe; friend class VPPredInstPHIRecipe; + friend class VPRecipeBase; const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast). @@ -89,7 +99,8 @@ /// SubclassID field of the VPValue objects. They are used for concrete /// type identification. enum { - VPValueSC, + VPValueSC, /// VPValue defined outside of VPlan. + VPValueSubSC, // VPValue defined by a VPDef that defines multiple VPValues. VPVInstructionSC, VPVMemoryInstructionSC, VPVReductionSC, @@ -166,6 +177,10 @@ void replaceAllUsesWith(VPValue *New); VPDef *getDef() { return Def; } + + /// Returns the underlying IR value, if this VPValue is defined outside the + /// scope of VPlan. + Value *getOutOfScopeIRValue(); }; typedef DenseMap Value2VPValueTy;