diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -9489,16 +9489,20 @@ if (!RecurPhi) continue; + VPRecipeBase *PrevRecipe = RecurPhi->getBackedgeRecipe(); + VPBasicBlock *InsertBlock = PrevRecipe->getParent(); + auto *Region = GetReplicateRegion(PrevRecipe); + if (Region) + InsertBlock = cast(Region->getSingleSuccessor()); + if (Region || PrevRecipe->isPhi()) + Builder.setInsertPoint(InsertBlock, InsertBlock->getFirstNonPhi()); + else + Builder.setInsertPoint(InsertBlock, std::next(PrevRecipe->getIterator())); + auto *RecurSplice = cast( Builder.createNaryOp(VPInstruction::FirstOrderRecurrenceSplice, {RecurPhi, RecurPhi->getBackedgeValue()})); - VPRecipeBase *PrevRecipe = RecurPhi->getBackedgeRecipe(); - if (auto *Region = GetReplicateRegion(PrevRecipe)) { - VPBasicBlock *Succ = cast(Region->getSingleSuccessor()); - RecurSplice->moveBefore(*Succ, Succ->getFirstNonPhi()); - } else - RecurSplice->moveAfter(PrevRecipe); RecurPhi->replaceAllUsesWith(RecurSplice); // Set the first operand of RecurSplice to RecurPhi again, after replacing // all users. diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll --- a/llvm/test/Transforms/LoopVectorize/induction.ll +++ b/llvm/test/Transforms/LoopVectorize/induction.ll @@ -851,8 +851,10 @@ } ; Ensure that the shuffle vector for first order recurrence is inserted -; correctly after all the phis. These new phis correspond to new IVs -; that are generated by optimizing non-free truncs of IVs to IVs themselves +; correctly after all the phis. These new phis correspond to new IVs +; that are generated by optimizing non-free truncs of IVs to IVs themselves. +; This also ensures the first-order recurrence splice recipe is placed +; correctly if it is fed by an induction. define i64 @trunc_with_first_order_recurrence() { ; CHECK-LABEL: trunc_with_first_order_recurrence ; CHECK-LABEL: vector.body: