diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -521,8 +521,8 @@ /// is provided, the integer induction variable will first be truncated to /// the corresponding type. void widenIntOrFpInduction(PHINode *IV, Value *Start, TruncInst *Trunc, - VPValue *Def, VPValue *CastDef, - VPTransformState &State); + VPValue *Def, VPValue *CastDef, VPValue *StepDef, + VPValue *PhiDef, VPTransformState &State); /// Construct the vector value of a scalarized value \p V one lane at a time. void packScalarIntoVectorValue(VPValue *Def, const VPIteration &Instance, @@ -641,7 +641,8 @@ void createVectorIntOrFpInductionPHI(const InductionDescriptor &II, Value *Step, Value *Start, Instruction *EntryVal, VPValue *Def, - VPValue *CastDef, + VPValue *CastDef, VPValue *StepDef, + VPValue *PhiDef, VPTransformState &State); /// Returns true if an instruction \p I should be scalarized instead of @@ -2278,8 +2279,8 @@ void InnerLoopVectorizer::createVectorIntOrFpInductionPHI( const InductionDescriptor &II, Value *Step, Value *Start, - Instruction *EntryVal, VPValue *Def, VPValue *CastDef, - VPTransformState &State) { + Instruction *EntryVal, VPValue *Def, VPValue *CastDef, VPValue *StepDef, + VPValue *PhiDef, VPTransformState &State) { assert((isa(EntryVal) || isa(EntryVal)) && "Expected either an induction phi-node or a truncate of it!"); @@ -2349,17 +2350,9 @@ Builder.CreateBinOp(AddOp, LastInduction, SplatVF, "step.add")); LastInduction->setDebugLoc(EntryVal->getDebugLoc()); } - - // Move the last step to the end of the latch block. This ensures consistent - // placement of all induction updates. - auto *LoopVectorLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch(); - auto *Br = cast(LoopVectorLatch->getTerminator()); - auto *ICmp = cast(Br->getCondition()); - LastInduction->moveBefore(ICmp); - LastInduction->setName("vec.ind.next"); - + State.set(StepDef, LastInduction, 0); + State.set(PhiDef, VecInd, 0); VecInd->addIncoming(SteppedStart, LoopVectorPreHeader); - VecInd->addIncoming(LastInduction, LoopVectorLatch); } bool InnerLoopVectorizer::shouldScalarizeInstruction(Instruction *I) const { @@ -2405,10 +2398,9 @@ State.set(CastDef, VectorLoopVal, Part); } -void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start, - TruncInst *Trunc, VPValue *Def, - VPValue *CastDef, - VPTransformState &State) { +void InnerLoopVectorizer::widenIntOrFpInduction( + PHINode *IV, Value *Start, TruncInst *Trunc, VPValue *Def, VPValue *CastDef, + VPValue *StepDef, VPValue *PhiDef, VPTransformState &State) { assert((IV->getType()->isIntegerTy() || IV != OldInduction) && "Primary induction variable must have an integer type"); @@ -2502,7 +2494,7 @@ auto NeedsScalarIV = needsScalarInduction(EntryVal); if (!NeedsScalarIV) { createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal, Def, CastDef, - State); + StepDef, PhiDef, State); return; } @@ -2511,7 +2503,7 @@ // loop iteration. if (!shouldScalarizeInstruction(EntryVal)) { createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal, Def, CastDef, - State); + StepDef, PhiDef, State); Value *ScalarIV = CreateScalarIV(Step); // Create scalar steps that can be used by instructions we will later // scalarize. Note that the addition of the scalar steps will not increase @@ -8838,7 +8830,7 @@ Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())); const SmallVectorImpl &Casts = II.getCastInsts(); return new VPWidenIntOrFpInductionRecipe( - Phi, Operands[0], Casts.empty() ? nullptr : Casts.front()); + Phi, II, Operands[0], Casts.empty() ? nullptr : Casts.front()); } return nullptr; @@ -8868,7 +8860,7 @@ Legal->getInductionVars().lookup(cast(I->getOperand(0))); VPValue *Start = Plan.getOrAddVPValue(II.getStartValue()); return new VPWidenIntOrFpInductionRecipe(cast(I->getOperand(0)), - Start, nullptr, I); + II, Start, nullptr, I); } return nullptr; } @@ -9367,7 +9359,8 @@ VPRecipeBase *Recipe = RecipeOrValue.get(); for (auto *Def : Recipe->definedValues()) { auto *UV = Def->getUnderlyingValue(); - Plan->addVPValue(UV, Def); + if (UV) + Plan->addVPValue(UV, Def); } RecipeBuilder.setRecipe(Instr, Recipe); @@ -9725,9 +9718,9 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) { assert(!State.Instance && "Int or FP induction being replicated."); - State.ILV->widenIntOrFpInduction(IV, getStartValue()->getLiveInIRValue(), - getTruncInst(), getVPValue(0), - getCastValue(), State); + State.ILV->widenIntOrFpInduction( + IV, getStartValue()->getLiveInIRValue(), getTruncInst(), getVPValue(0), + getCastValue(), getStepValue(), getPhiValue(), State); } void VPWidenPHIRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1008,11 +1008,13 @@ /// producing their vector and scalar values. class VPWidenIntOrFpInductionRecipe : public VPRecipeBase { PHINode *IV; + InductionDescriptor ⅈ public: - VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, Instruction *Cast, + VPWidenIntOrFpInductionRecipe(PHINode *IV, InductionDescriptor &II, + VPValue *Start, Instruction *Cast, TruncInst *Trunc = nullptr) - : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start}), IV(IV) { + : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start}), IV(IV), II(II) { if (Trunc) new VPValue(Trunc, this); else @@ -1020,6 +1022,9 @@ if (Cast) new VPValue(Cast, this); + + new VPValue(nullptr, this); + new VPValue(nullptr, this); } ~VPWidenIntOrFpInductionRecipe() override = default; @@ -1043,11 +1048,14 @@ /// Returns the cast VPValue, if one is attached, or nullptr otherwise. VPValue *getCastValue() { - if (getNumDefinedValues() != 2) + if (getNumDefinedValues() != 4) return nullptr; return getVPValue(1); } + VPValue *getStepValue() { return getVPValue(getNumDefinedValues() - 1); } + VPValue *getPhiValue() { return getVPValue(getNumDefinedValues() - 2); } + /// Returns the first defined value as TruncInst, if it is one or nullptr /// otherwise. TruncInst *getTruncInst() { @@ -1056,6 +1064,8 @@ const TruncInst *getTruncInst() const { return dyn_cast_or_null(getVPValue(0)->getUnderlyingValue()); } + + InductionDescriptor &getInductionDescriptor() { return II; } }; /// A recipe for handling first order recurrences and pointer inductions. For diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -826,7 +826,30 @@ // Fix the latch value of reduction and first-order recurrences phis in the // vector loop. VPBasicBlock *Header = Entry->getEntryBasicBlock(); + if (Header->empty()) { + assert(EnableVPlanNativePath); + Header = cast(Header->getSingleSuccessor()); + } for (VPRecipeBase &R : Header->phis()) { + if (auto *IV = dyn_cast(&R)) { + if (!State->hasAnyVectorValue(IV->getStepValue())) + continue; + PHINode *VecInd = cast(State->get(IV->getPhiValue(), 0)); + Instruction *LastInduction = + cast(State->get(IV->getStepValue(), 0)); + // Move the last step to the end of the latch block. This ensures + // consistent + // placement of all induction updates. + auto *LoopVectorLatch = + State->LI->getLoopFor(State->CFG.PrevBB)->getLoopLatch(); + auto *Br = cast(LoopVectorLatch->getTerminator()); + auto *ICmp = cast(Br->getCondition()); + LastInduction->moveBefore(ICmp); + LastInduction->setName("vec.ind.next"); + + VecInd->addIncoming(LastInduction, LoopVectorLatch); + continue; + } auto *PhiR = dyn_cast(&R); if (!PhiR || !(isa(&R) || isa(&R))) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -48,7 +48,8 @@ if (II.getKind() == InductionDescriptor::IK_IntInduction || II.getKind() == InductionDescriptor::IK_FpInduction) { VPValue *Start = Plan->getOrAddVPValue(II.getStartValue()); - NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, nullptr); + NewRecipe = + new VPWidenIntOrFpInductionRecipe(Phi, II, Start, nullptr); } else { Plan->addVPValue(Phi, VPPhi); continue; @@ -87,7 +88,13 @@ NewRecipe->insertBefore(&Ingredient); if (NewRecipe->getNumDefinedValues() == 1) VPV->replaceAllUsesWith(NewRecipe->getVPSingleValue()); - else + else if (isa(NewRecipe)) { + VPV->replaceAllUsesWith(NewRecipe->getVPValue(0)); + Ingredient.eraseFromParent(); + Plan->removeVPValueFor(Inst); + Plan->addVPValue(Inst, NewRecipe->getVPValue(0)); + continue; + } else assert(NewRecipe->getNumDefinedValues() == 0 && "Only recpies with zero or one defined values expected"); Ingredient.eraseFromParent();