diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3900,24 +3900,22 @@ Phi->setIncomingValueForBlock(LoopScalarPreHeader, Start); Phi->setName("scalar.recur"); - // Finally, fix users of the recurrence outside the loop. The users will need - // either the last value of the scalar recurrence or the last value of the - // vector recurrence we extracted in the middle block. Since the loop is in - // LCSSA form, we just need to find all the phi nodes for the original scalar - // recurrence in the exit block, and then add an edge for the middle block. - // Note that LCSSA does not imply single entry when the original scalar loop - // had multiple exiting edges (as we always run the last iteration in the - // scalar epilogue); in that case, there is no edge from middle to exit and - // and thus no phis which needed updated. - if (!Cost->requiresScalarEpilogue(VF)) { - SmallPtrSet ToFix; - for (User *U : Phi->users()) - if (isa(U) && cast(U)->getParent() == LoopExitBlock) - ToFix.insert(cast(U)); - for (PHINode *LCSSAPhi : ToFix) { - LCSSAPhi->addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock); - State.Plan->removeLiveOut(LCSSAPhi); - } + auto RecurSplice = cast(*PhiR->user_begin()); + assert(PhiR->getNumUsers() == 1 && + RecurSplice->getOpcode() == + VPInstruction::FirstOrderRecurrenceSplice && + "recurrence phi must have a single user: FirstOrderRecurrenceSplice"); + SmallVector LiveOuts; + for (VPUser *U : RecurSplice->users()) + if (auto *LiveOut = dyn_cast(U)) + LiveOuts.push_back(LiveOut); + + for (VPLiveOut *LiveOut : LiveOuts) { + assert(!Cost->requiresScalarEpilogue(VF) && + "no live-outs expected when scalar epilogue is required"); + PHINode *LCSSAPhi = LiveOut->getPhi(); + LCSSAPhi->addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock); + State.Plan->removeLiveOut(LCSSAPhi); } } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -666,6 +666,10 @@ VPLiveOut(PHINode *Phi, VPValue *Op) : VPUser({Op}, VPUser::VPUserID::LiveOut), Phi(Phi) {} + static inline bool classof(const VPUser *U) { + return U->getVPUserID() == VPUser::VPUserID::LiveOut; + } + /// Fixup the wrapped LCSSA phi node in the unique exit block. This simply /// means we need to add the appropriate incoming value from the middle /// block as exiting edges from the scalar epilogue loop (if present) are