diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3890,12 +3890,16 @@ // had multiple exiting edges (as we always run the last iteration in the // scalar epilogue); in that case, there is no edge from middle to exit and // and thus no phis which needed updated. - if (!Cost->requiresScalarEpilogue(VF)) - for (PHINode &LCSSAPhi : LoopExitBlock->phis()) - if (llvm::is_contained(LCSSAPhi.incoming_values(), Phi)) { - LCSSAPhi.addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock); - State.Plan->removeLiveOut(&LCSSAPhi); - } + if (!Cost->requiresScalarEpilogue(VF)) { + SmallPtrSet ToFix; + for (User *U : Phi->users()) + if (isa(U) && cast(U)->getParent() == LoopExitBlock) + ToFix.insert(cast(U)); + for (PHINode *LCSSAPhi : ToFix) { + LCSSAPhi->addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock); + State.Plan->removeLiveOut(LCSSAPhi); + } + } } void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,