diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -9599,11 +9599,9 @@ NewPointerPhi->addIncoming(ScalarStartValue, State.CFG.VectorPreHeader); // A pointer induction, performed by using a gep - BasicBlock *LoopLatch = - State.LI->getLoopFor(State.CFG.PrevBB)->getLoopLatch(); + const DataLayout &DL = NewPointerPhi->getModule()->getDataLayout(); + Instruction *InductionLoc = &*State.Builder.GetInsertPoint(); - const DataLayout &DL = LoopLatch->getModule()->getDataLayout(); - Instruction *InductionLoc = LoopLatch->getTerminator(); const SCEV *ScalarStep = IndDesc.getStep(); SCEVExpander Exp(SE, DL, "induction"); Value *ScalarStepValue = Exp.expandCodeFor(ScalarStep, PhiType, InductionLoc); @@ -9614,7 +9612,12 @@ IndDesc.getElementType(), NewPointerPhi, State.Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind", InductionLoc); - NewPointerPhi->addIncoming(InductionGEP, LoopLatch); + // Add induction update using an incorrect block temporarily. The phi node + // will be fixed after VPlan execution. Note that at this point the latch + // block cannot be used, as it does not exist yet. + // TODO: Model increment value in VPlan, by turning the recipe into a + // multi-def and a subclass of VPHeaderPHIRecipe. + NewPointerPhi->addIncoming(InductionGEP, State.CFG.VectorPreHeader); // Create UF many actual address geps that use the pointer // phi as base and a vectorized version of the step value diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -981,14 +981,27 @@ } for (VPRecipeBase &R : Header->phis()) { // Skip phi-like recipes that generate their backedege values themselves. - // TODO: Model their backedge values explicitly. - if (isa(&R) || isa(&R)) + if (isa(&R)) continue; - // Set the correct incoming block for backedge values and move induction to - // latch. - if (auto *IndR = dyn_cast(&R)) { - auto *Phi = cast(State->get(IndR, 0)); + if (isa(&R) || + isa(&R)) { + PHINode *Phi = nullptr; + if (isa(&R)) { + Phi = cast(State->get(R.getVPSingleValue(), 0)); + } else { + auto *WidenPhi = cast(&R); + // TODO: Split off the case that all users of a pointer phi are scalar + // from the VPWidenPointerInductionRecipe. + if (all_of(WidenPhi->users(), [WidenPhi](const VPUser *U) { + return cast(U)->usesScalars(WidenPhi); + })) + continue; + + auto *GEP = cast(State->get(WidenPhi, 0)); + Phi = cast(GEP->getPointerOperand()); + } + Phi->setIncomingBlock(1, VectorLatchBB); // Move the last step to the end of the latch block. This ensures