diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1915,7 +1915,7 @@ SmallPtrSet ElementTypesInLoop; /// Profitable vector factors. - SmallVector ProfitableVFs; + SmallVector CachedVFs; /// Cached cost of one scalar iteraton; VectorizationFactor ScalarVF = {ElementCount::getFixed(1), @@ -6108,9 +6108,9 @@ continue; } - // If profitable add it to ProfitableVF list. - if (isMoreProfitable(Candidate, getScalarVF())) - ProfitableVFs.push_back(Candidate); + // Cache calculated cost of the candidate since it will be needed again + // during epilogue vectorization cost modeling. + CachedVFs.push_back(Candidate); if (isMoreProfitable(Candidate, ChosenFactor)) ChosenFactor = Candidate; @@ -6250,7 +6250,7 @@ VectorizationFactor LoopVectorizationCostModel::selectEpilogueVectorizationFactor( const ElementCount MainLoopVF, const LoopVectorizationPlanner &LVP) { - VectorizationFactor Result = VectorizationFactor::Disabled(); + VectorizationFactor Result = getScalarVF(); if (!EnableEpilogueVectorization) { LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n";); return Result; @@ -6305,12 +6305,14 @@ if (!isEpilogueVectorizationProfitable(MainLoopVF)) return Result; - for (auto &NextVF : ProfitableVFs) + for (auto &NextVF : CachedVFs) { + if (NextVF.Width.isScalar()) + continue; if (ElementCount::isKnownLT(NextVF.Width, MainLoopVF) && - (Result.Width.getFixedValue() == 1 || - isMoreProfitable(NextVF, Result)) && + isMoreProfitable(NextVF, Result) && LVP.hasPlanWithVFs({MainLoopVF, NextVF.Width})) Result = NextVF; + } if (Result != VectorizationFactor::Disabled()) LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "