diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4100,12 +4100,18 @@ // We know that the loop is in LCSSA form. We need to update the PHI nodes // in the exit blocks. See comment on analogous loop in // fixFixedOrderRecurrence for a more complete explaination of the logic. - if (!Cost->requiresScalarEpilogue(VF)) - for (PHINode &LCSSAPhi : LoopExitBlock->phis()) - if (llvm::is_contained(LCSSAPhi.incoming_values(), LoopExitInst)) { - LCSSAPhi.addIncoming(ReducedPartRdx, LoopMiddleBlock); - State.Plan->removeLiveOut(&LCSSAPhi); - } + if (!Cost->requiresScalarEpilogue(VF)) { + SmallPtrSet ToFix; + auto *Phi = cast(PhiR->getUnderlyingValue()); + for (User *U : Phi->users()) + if (auto *UPhi = dyn_cast(U); + UPhi && UPhi->getParent() == LoopExitBlock) + ToFix.insert(UPhi); + for (PHINode *LCSSAPhi : ToFix) { + LCSSAPhi->addIncoming(ReducedPartRdx, LoopMiddleBlock); + State.Plan->removeLiveOut(LCSSAPhi); + } + } // Fix the scalar loop reduction variable with the incoming reduction sum // from the vector body and from the backedge value.