diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -522,8 +522,8 @@ /// is provided, the integer induction variable will first be truncated to /// the corresponding type. void widenIntOrFpInduction(PHINode *IV, Value *Start, TruncInst *Trunc, - VPValue *Def, VPValue *CastDef, - VPTransformState &State); + VPValue *Def, VPValue *CastDef, VPValue *StepDef, + VPValue *PhiDef, VPTransformState &State); /// Construct the vector value of a scalarized value \p V one lane at a time. void packScalarIntoVectorValue(VPValue *Def, const VPIteration &Instance, @@ -642,7 +642,8 @@ void createVectorIntOrFpInductionPHI(const InductionDescriptor &II, Value *Step, Value *Start, Instruction *EntryVal, VPValue *Def, - VPValue *CastDef, + VPValue *CastDef, VPValue *StepDef, + VPValue *PhiDef, VPTransformState &State); /// Returns true if an instruction \p I should be scalarized instead of @@ -1207,8 +1208,10 @@ // instruction. Widen memory instructions involved in address computation // will lead to gather/scatter instructions, which don't need to be // handled. + // There is also nothing to drop from VPWidenIntOrFpInductionRecipe. if (isa(CurRec) || - isa(CurRec)) + isa(CurRec) || + isa(CurRec)) continue; // This recipe contributes to the address computation of a widen @@ -2369,8 +2372,8 @@ void InnerLoopVectorizer::createVectorIntOrFpInductionPHI( const InductionDescriptor &II, Value *Step, Value *Start, - Instruction *EntryVal, VPValue *Def, VPValue *CastDef, - VPTransformState &State) { + Instruction *EntryVal, VPValue *Def, VPValue *CastDef, VPValue *StepDef, + VPValue *PhiDef, VPTransformState &State) { assert((isa(EntryVal) || isa(EntryVal)) && "Expected either an induction phi-node or a truncate of it!"); @@ -2440,17 +2443,9 @@ Builder.CreateBinOp(AddOp, LastInduction, SplatVF, "step.add")); LastInduction->setDebugLoc(EntryVal->getDebugLoc()); } - - // Move the last step to the end of the latch block. This ensures consistent - // placement of all induction updates. - auto *LoopVectorLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch(); - auto *Br = cast(LoopVectorLatch->getTerminator()); - auto *ICmp = cast(Br->getCondition()); - LastInduction->moveBefore(ICmp); - LastInduction->setName("vec.ind.next"); - + State.set(StepDef, LastInduction, 0); + State.set(PhiDef, VecInd, 0); VecInd->addIncoming(SteppedStart, LoopVectorPreHeader); - VecInd->addIncoming(LastInduction, LoopVectorLatch); } bool InnerLoopVectorizer::shouldScalarizeInstruction(Instruction *I) const { @@ -2500,10 +2495,9 @@ State.set(CastDef, VectorLoopVal, Part); } -void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start, - TruncInst *Trunc, VPValue *Def, - VPValue *CastDef, - VPTransformState &State) { +void InnerLoopVectorizer::widenIntOrFpInduction( + PHINode *IV, Value *Start, TruncInst *Trunc, VPValue *Def, VPValue *CastDef, + VPValue *StepDef, VPValue *PhiDef, VPTransformState &State) { assert((IV->getType()->isIntegerTy() || IV != OldInduction) && "Primary induction variable must have an integer type"); @@ -2597,7 +2591,7 @@ auto NeedsScalarIV = needsScalarInduction(EntryVal); if (!NeedsScalarIV) { createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal, Def, CastDef, - State); + StepDef, PhiDef, State); return; } @@ -2606,7 +2600,7 @@ // loop iteration. if (!shouldScalarizeInstruction(EntryVal)) { createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal, Def, CastDef, - State); + StepDef, PhiDef, State); Value *ScalarIV = CreateScalarIV(Step); // Create scalar steps that can be used by instructions we will later // scalarize. Note that the addition of the scalar steps will not increase @@ -9170,7 +9164,7 @@ auto *PN = cast(R->getUnderlyingValue()); VPRecipeBase *IncR = getRecipe(cast(PN->getIncomingValueForBlock(OrigLatch))); - R->addOperand(IncR->getVPSingleValue()); + R->addOperand(IncR->getVPValue(0)); } } @@ -9526,8 +9520,8 @@ // Otherwise, add the new recipe. VPRecipeBase *Recipe = RecipeOrValue.get(); for (auto *Def : Recipe->definedValues()) { - auto *UV = Def->getUnderlyingValue(); - Plan->addVPValue(UV, Def); + if (auto *UV = Def->getUnderlyingValue()) + Plan->addVPValue(UV, Def); } if (isa(Recipe) && @@ -9911,9 +9905,9 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) { assert(!State.Instance && "Int or FP induction being replicated."); - State.ILV->widenIntOrFpInduction(IV, getStartValue()->getLiveInIRValue(), - getTruncInst(), getVPValue(0), - getCastValue(), State); + State.ILV->widenIntOrFpInduction( + IV, getStartValue()->getLiveInIRValue(), getTruncInst(), getVPValue(0), + getCastValue(), getStepValue(), getPhiValue(), State); } void VPWidenPHIRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1022,6 +1022,13 @@ if (Cast) new VPValue(Cast, this); + + // The VPValues below are going to be mapped to the PHI and increment + // instructions created for the recipe. Note that those are only introduced + // to facilitate the transition to modeling the increment explicitly in + // VPlan. + new VPValue(nullptr, this); + new VPValue(nullptr, this); } ~VPWidenIntOrFpInductionRecipe() override = default; @@ -1045,11 +1052,18 @@ /// Returns the cast VPValue, if one is attached, or nullptr otherwise. VPValue *getCastValue() { - if (getNumDefinedValues() != 2) + if (getNumDefinedValues() != 4) return nullptr; return getVPValue(1); } + /// Return the VPValue representing the induction increment (= incoming value + /// from the latch). + VPValue *getStepValue() { return getVPValue(getNumDefinedValues() - 1); } + + /// Return the VPValue representing the PHI node for the recipe. + VPValue *getPhiValue() { return getVPValue(getNumDefinedValues() - 2); } + /// Returns the first defined value as TruncInst, if it is one or nullptr /// otherwise. TruncInst *getTruncInst() { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -840,7 +840,30 @@ // Fix the latch value of reduction and first-order recurrences phis in the // vector loop. VPBasicBlock *Header = Entry->getEntryBasicBlock(); + if (Header->empty()) { + assert(EnableVPlanNativePath); + Header = cast(Header->getSingleSuccessor()); + } for (VPRecipeBase &R : Header->phis()) { + if (auto *IV = dyn_cast(&R)) { + if (!State->hasAnyVectorValue(IV->getStepValue())) + continue; + PHINode *VecInd = cast(State->get(IV->getPhiValue(), 0)); + Instruction *LastInduction = + cast(State->get(IV->getStepValue(), 0)); + // Move the last step to the end of the latch block. This ensures + // consistent + // placement of all induction updates. + auto *LoopVectorLatch = + State->LI->getLoopFor(State->CFG.PrevBB)->getLoopLatch(); + auto *Br = cast(LoopVectorLatch->getTerminator()); + auto *ICmp = cast(Br->getCondition()); + LastInduction->moveBefore(ICmp); + LastInduction->setName("vec.ind.next"); + + VecInd->addIncoming(LastInduction, LoopVectorLatch); + continue; + } auto *PhiR = dyn_cast(&R); if (!PhiR || !(isa(&R) || isa(&R))) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -87,7 +87,13 @@ NewRecipe->insertBefore(&Ingredient); if (NewRecipe->getNumDefinedValues() == 1) VPV->replaceAllUsesWith(NewRecipe->getVPSingleValue()); - else + else if (isa(NewRecipe)) { + VPV->replaceAllUsesWith(NewRecipe->getVPValue(0)); + Ingredient.eraseFromParent(); + Plan->removeVPValueFor(Inst); + Plan->addVPValue(Inst, NewRecipe->getVPValue(0)); + continue; + } else assert(NewRecipe->getNumDefinedValues() == 0 && "Only recpies with zero or one defined values expected"); Ingredient.eraseFromParent();