diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1855,6 +1855,19 @@ /// \p VF is the vectorization factor chosen for the original loop. bool isEpilogueVectorizationProfitable(const ElementCount VF) const; + /// Returns vectorization factor for scalar loop. The cost is computed on + /// the first call and cached value is returned on all consequent calls. + VectorizationFactor getScalarVF() const { + if (ScalarVF.Cost.isValid()) + return ScalarVF; + const_cast(this)->ScalarVF.Cost = + expectedCost(ScalarVF.Width).first; + assert(ScalarVF.Cost.isValid() && + "Unexpected invalid cost for scalar loop"); + LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ScalarVF.Cost << ".\n"); + return ScalarVF; + } + public: /// The loop that we evaluate. Loop *TheLoop; @@ -1903,6 +1916,10 @@ /// Profitable vector factors. SmallVector ProfitableVFs; + + /// Cached cost of one scalar iteraton; + VectorizationFactor ScalarVF = {ElementCount::getFixed(1), + InstructionCost::getInvalid()}; }; } // end namespace llvm @@ -6057,14 +6074,10 @@ VectorizationFactor LoopVectorizationCostModel::selectVectorizationFactor( const ElementCountSet &VFCandidates) { - InstructionCost ExpectedCost = expectedCost(ElementCount::getFixed(1)).first; - LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ExpectedCost << ".\n"); - assert(ExpectedCost.isValid() && "Unexpected invalid cost for scalar loop"); assert(VFCandidates.count(ElementCount::getFixed(1)) && "Expected Scalar VF to be a candidate"); - const VectorizationFactor ScalarCost(ElementCount::getFixed(1), ExpectedCost); - VectorizationFactor ChosenFactor = ScalarCost; + VectorizationFactor ChosenFactor = getScalarVF(); bool ForceVectorization = Hints->getForce() == LoopVectorizeHints::FK_Enabled; if (ForceVectorization && VFCandidates.size() > 1) { @@ -6096,7 +6109,7 @@ } // If profitable add it to ProfitableVF list. - if (isMoreProfitable(Candidate, ScalarCost)) + if (isMoreProfitable(Candidate, getScalarVF())) ProfitableVFs.push_back(Candidate); if (isMoreProfitable(Candidate, ChosenFactor)) @@ -6167,11 +6180,11 @@ reportVectorizationFailure("There are conditional stores.", "store that is conditionally executed prevents vectorization", "ConditionalStore", ORE, TheLoop); - ChosenFactor = ScalarCost; + ChosenFactor = getScalarVF(); } LLVM_DEBUG(if (ForceVectorization && !ChosenFactor.Width.isScalar() && - ChosenFactor.Cost >= ScalarCost.Cost) dbgs() + ChosenFactor.Cost >= getScalarVF().Cost) dbgs() << "LV: Vectorization seems to be not beneficial, " << "but was forced by a user.\n"); LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << ChosenFactor.Width << ".\n");