diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -506,8 +506,8 @@ /// Widen an integer or floating-point induction variable \p IV. If \p Trunc /// is provided, the integer induction variable will first be truncated to /// the corresponding type. - void widenIntOrFpInduction(PHINode *IV, Value *Start, TruncInst *Trunc, - VPValue *Def, VPValue *CastDef, + void widenIntOrFpInduction(PHINode *IV, InductionDescriptor &ID, Value *Start, + TruncInst *Trunc, VPValue *Def, VPValue *CastDef, VPTransformState &State); /// Construct the vector value of a scalarized value \p V one lane at a time. @@ -2487,17 +2487,13 @@ State.set(CastDef, VectorLoopVal, Part); } -void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start, - TruncInst *Trunc, VPValue *Def, - VPValue *CastDef, +void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, + InductionDescriptor &ID, + Value *Start, TruncInst *Trunc, + VPValue *Def, VPValue *CastDef, VPTransformState &State) { assert((IV->getType()->isIntegerTy() || IV != OldInduction) && "Primary induction variable must have an integer type"); - - auto II = Legal->getInductionVars().find(IV); - assert(II != Legal->getInductionVars().end() && "IV is not an induction"); - - auto ID = II->second; assert(IV->getType() == ID.getStartValue()->getType() && "Types must match"); // The value from the original loop to which we are mapping the new induction @@ -8591,14 +8587,17 @@ ArrayRef Operands) const { // Check if this is an integer or fp induction. If so, build the recipe that // produces its scalar and vector values. - InductionDescriptor II = Legal->getInductionVars().lookup(Phi); + auto I = Legal->getInductionVars().find(Phi); + if (I == Legal->getInductionVars().end()) + return nullptr; + InductionDescriptor &II = I->second; if (II.getKind() == InductionDescriptor::IK_IntInduction || II.getKind() == InductionDescriptor::IK_FpInduction) { assert(II.getStartValue() == Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())); const SmallVectorImpl &Casts = II.getCastInsts(); return new VPWidenIntOrFpInductionRecipe( - Phi, Operands[0], Casts.empty() ? nullptr : Casts.front()); + Phi, Operands[0], II, Casts.empty() ? nullptr : Casts.front()); } return nullptr; @@ -8624,11 +8623,11 @@ if (LoopVectorizationPlanner::getDecisionAndClampRange( isOptimizableIVTruncate(I), Range)) { - InductionDescriptor II = - Legal->getInductionVars().lookup(cast(I->getOperand(0))); + InductionDescriptor &II = + Legal->getInductionVars().find(cast(I->getOperand(0)))->second; VPValue *Start = Plan.getOrAddVPValue(II.getStartValue()); return new VPWidenIntOrFpInductionRecipe(cast(I->getOperand(0)), - Start, I); + Start, II, I); } return nullptr; } @@ -9714,9 +9713,9 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) { assert(!State.Instance && "Int or FP induction being replicated."); - State.ILV->widenIntOrFpInduction(IV, getStartValue()->getLiveInIRValue(), - getTruncInst(), getVPValue(0), - getCastValue(), State); + State.ILV->widenIntOrFpInduction( + IV, getDescriptor(), getStartValue()->getLiveInIRValue(), getTruncInst(), + getVPValue(0), getCastValue(), State); } void VPWidenPHIRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -61,6 +61,7 @@ class VPlan; class VPReplicateRecipe; class VPlanSlp; +class InductionDescriptor; /// Returns a calculation for the total number of elements for a given \p VF. /// For fixed width vectors this value is a constant, whereas for scalable @@ -1005,19 +1006,24 @@ /// producing their vector and scalar values. class VPWidenIntOrFpInductionRecipe : public VPRecipeBase { PHINode *IV; + InductionDescriptor &IndDesc; public: VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, + InductionDescriptor &IndDesc, Instruction *Cast = nullptr) - : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start}), IV(IV) { + : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start}), IV(IV), + IndDesc(IndDesc) { new VPValue(IV, this); if (Cast) new VPValue(Cast, this); } - VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, TruncInst *Trunc) - : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start}), IV(IV) { + VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, + InductionDescriptor &IndDesc, TruncInst *Trunc) + : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start}), IV(IV), + IndDesc(IndDesc) { new VPValue(Trunc, this); } @@ -1056,6 +1062,9 @@ const TruncInst *getTruncInst() const { return dyn_cast_or_null(getVPValue(0)->getUnderlyingValue()); } + + /// Returns to induction descriptor for the recipe. + InductionDescriptor &getDescriptor() { return IndDesc; } }; /// A recipe for handling first order recurrences and pointer inductions. For diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -44,11 +44,12 @@ VPRecipeBase *NewRecipe = nullptr; if (auto *VPPhi = dyn_cast(&Ingredient)) { auto *Phi = cast(VPPhi->getUnderlyingValue()); - InductionDescriptor II = Inductions.lookup(Phi); - if (II.getKind() == InductionDescriptor::IK_IntInduction || - II.getKind() == InductionDescriptor::IK_FpInduction) { - VPValue *Start = Plan->getOrAddVPValue(II.getStartValue()); - NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start); + auto II = Inductions.find(Phi); + if (II != Inductions.end() && + (II->second.getKind() == InductionDescriptor::IK_IntInduction || + II->second.getKind() == InductionDescriptor::IK_FpInduction)) { + VPValue *Start = Plan->getOrAddVPValue(II->second.getStartValue()); + NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, II->second); } else { Plan->addVPValue(Phi, VPPhi); continue;