diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -14134,10 +14134,9 @@ template static bool tryToVectorizeSequence( - SmallVectorImpl &Incoming, function_ref Limit, - function_ref AreCompatible, + SmallVectorImpl &Incoming, function_ref AreCompatible, function_ref, bool)> TryToVectorizeHelper, - bool LimitForRegisterSize) { + bool LimitForRegisterSize, BoUpSLP &R) { bool Changed = false; // Try to vectorize elements base on their type. @@ -14166,10 +14165,18 @@ TryToVectorizeHelper(ArrayRef(IncIt, NumElts), LimitForRegisterSize)) { // Success start over because instructions might have been changed. Changed = true; - } else if (NumElts < Limit(*IncIt) && - (Candidates.empty() || - Candidates.front()->getType() == (*IncIt)->getType())) { - Candidates.append(IncIt, std::next(IncIt, NumElts)); + } else { + /// \Returns the minimum number of elements that we will attempt to + /// vectorize. + auto GetMinNumElements = [&R](Value *V) { + unsigned EltSize = R.getVectorElementSize(V); + return std::max(2U, R.getMaxVecRegSize() / EltSize); + }; + if (NumElts < GetMinNumElements(*IncIt) && + (Candidates.empty() || + Candidates.front()->getType() == (*IncIt)->getType())) { + Candidates.append(IncIt, std::next(IncIt, NumElts)); + } } // Final attempt to vectorize instructions with the same types. if (Candidates.size() > 1 && @@ -14310,16 +14317,12 @@ return compareCmp(V1, V2, *TLI, [&R](Instruction *I) { return R.isDeleted(I); }); }; - auto Limit = [&R](Value *V) { - unsigned EltSize = R.getVectorElementSize(V); - return std::max(2U, R.getMaxVecRegSize() / EltSize); - }; SmallVector Vals(PostponedCmps.begin(), PostponedCmps.end()); stable_sort(Vals, CompareSorter); OpsChanged |= tryToVectorizeSequence( - Vals, Limit, AreCompatibleCompares, + Vals, AreCompatibleCompares, [this, &R](ArrayRef Candidates, bool LimitForRegisterSize) { // Exclude possible reductions from other blocks. bool ArePossiblyReducedInOtherBlock = @@ -14334,7 +14337,7 @@ return false; return tryToVectorizeList(Candidates, R, LimitForRegisterSize); }, - /*LimitForRegisterSize=*/true); + /*LimitForRegisterSize=*/true, R); Instructions.clear(); } else { Instructions.clear(); @@ -14437,10 +14440,6 @@ } return true; }; - auto Limit = [&R](Value *V) { - unsigned EltSize = R.getVectorElementSize(V); - return std::max(2U, R.getMaxVecRegSize() / EltSize); - }; bool HaveVectorizedPhiNodes = false; do { @@ -14484,11 +14483,11 @@ stable_sort(Incoming, PHICompare); HaveVectorizedPhiNodes = tryToVectorizeSequence( - Incoming, Limit, AreCompatiblePHIs, + Incoming, AreCompatiblePHIs, [this, &R](ArrayRef Candidates, bool LimitForRegisterSize) { return tryToVectorizeList(Candidates, R, LimitForRegisterSize); }, - /*LimitForRegisterSize=*/true); + /*LimitForRegisterSize=*/true, R); Changed |= HaveVectorizedPhiNodes; VisitedInstrs.insert(Incoming.begin(), Incoming.end()); } while (HaveVectorizedPhiNodes); @@ -14764,10 +14763,6 @@ return V1->getValueOperand()->getValueID() == V2->getValueOperand()->getValueID(); }; - auto Limit = [&R, this](StoreInst *SI) { - unsigned EltSize = DL->getTypeSizeInBits(SI->getValueOperand()->getType()); - return R.getMinVF(EltSize); - }; // Attempt to sort and vectorize each of the store-groups. for (auto &Pair : Stores) { @@ -14783,11 +14778,11 @@ stable_sort(Pair.second, StoreSorter); Changed |= tryToVectorizeSequence( - Pair.second, Limit, AreCompatibleStores, + Pair.second, AreCompatibleStores, [this, &R](ArrayRef Candidates, bool) { return vectorizeStores(Candidates, R); }, - /*LimitForRegisterSize=*/false); + /*LimitForRegisterSize=*/false, R); } return Changed; }