diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -9535,16 +9535,12 @@ LastInduction->setDebugLoc(EntryVal->getDebugLoc()); } - // Move the last step to the end of the latch block. This ensures consistent - // placement of all induction updates. - auto *LoopVectorLatch = - State.LI->getLoopFor(State.CFG.PrevBB)->getLoopLatch(); - auto *Br = cast(LoopVectorLatch->getTerminator()); - LastInduction->moveBefore(Br); LastInduction->setName("vec.ind.next"); - VecInd->addIncoming(SteppedStart, State.CFG.VectorPreHeader); - VecInd->addIncoming(LastInduction, LoopVectorLatch); + // Add induction update using an incorrect block temporarily. The phi node + // will be fixed after VPlan execution. Note that at this point the latch + // block cannot be used, as it does not exist yet. + VecInd->addIncoming(LastInduction, State.CFG.VectorPreHeader); } void VPWidenPointerInductionRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -984,10 +984,22 @@ for (VPRecipeBase &R : Header->phis()) { // Skip phi-like recipes that generate their backedege values themselves. // TODO: Model their backedge values explicitly. - if (isa(&R) || isa(&R) || - isa(&R)) + if (isa(&R) || isa(&R)) continue; + // Set the correct incoming block for backedge values and move induction to + // latch. + if (auto *IndR = dyn_cast(&R)) { + auto *Phi = cast(State->get(IndR, 0)); + Phi->setIncomingBlock(1, VectorLatchBB); + + // Move the last step to the end of the latch block. This ensures + // consistent placement of all induction updates. + Instruction *Inc = cast(Phi->getIncomingValue(1)); + Inc->moveBefore(VectorLatchBB->getTerminator()->getPrevNode()); + continue; + } + auto *PhiR = cast(&R); // For canonical IV, first-order recurrences and in-order reduction phis, // only a single part is generated, which provides the last part from the