diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h --- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h +++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h @@ -40,7 +40,6 @@ private: Module &M; std::unique_ptr Summary; - bool computeSummary(); void computeThresholds(); // Count thresholds to answer isHotCount and isColdCount queries. Optional HotCountThreshold, ColdCountThreshold; @@ -56,15 +55,17 @@ Optional computeThreshold(int PercentileCutoff); // The map that caches the threshold values. The keys are the percentile // cutoff values and the values are the corresponding threshold values. - DenseMap ThresholdCache; + mutable DenseMap ThresholdCache; public: - ProfileSummaryInfo(Module &M) : M(M) {} - ProfileSummaryInfo(ProfileSummaryInfo &&Arg) - : M(Arg.M), Summary(std::move(Arg.Summary)) {} + ProfileSummaryInfo(Module &M) : M(M) { refresh(); } + ProfileSummaryInfo(ProfileSummaryInfo &&Arg) = default; + + /// If no summary is present, attempt to refresh. + void refresh(); /// Returns true if profile summary is available. - bool hasProfileSummary() { return computeSummary(); } + bool hasProfileSummary() const { return Summary != nullptr; } /// Returns true if module \c M has sample profile. bool hasSampleProfile() { @@ -141,12 +142,12 @@ bool isColdBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI); /// Returns true if BasicBlock \p BB is considered hot with regard to a given /// hot percentile cutoff value. - bool isHotBlockNthPercentile(int PercentileCutoff, - const BasicBlock *BB, BlockFrequencyInfo *BFI); + bool isHotBlockNthPercentile(int PercentileCutoff, const BasicBlock *BB, + BlockFrequencyInfo *BFI); /// Returns true if BasicBlock \p BB is considered cold with regard to a given /// cold percentile cutoff value. - bool isColdBlockNthPercentile(int PercentileCutoff, - const BasicBlock *BB, BlockFrequencyInfo *BFI); + bool isColdBlockNthPercentile(int PercentileCutoff, const BasicBlock *BB, + BlockFrequencyInfo *BFI); /// Returns true if the call site \p CB is considered hot. bool isHotCallSite(const CallBase &CB, BlockFrequencyInfo *BFI); /// Returns true if call site \p CB is considered cold. @@ -167,15 +168,16 @@ } private: - template - bool isFunctionHotOrColdInCallGraphNthPercentile(int PercentileCutoff, - const Function *F, - BlockFrequencyInfo &BFI); - template - bool isHotOrColdCountNthPercentile(int PercentileCutoff, uint64_t C); - template - bool isHotOrColdBlockNthPercentile(int PercentileCutoff, const BasicBlock *BB, - BlockFrequencyInfo *BFI); + template + bool isFunctionHotOrColdInCallGraphNthPercentile(int PercentileCutoff, + const Function *F, + BlockFrequencyInfo &BFI); + template + bool isHotOrColdCountNthPercentile(int PercentileCutoff, uint64_t C); + template + bool isHotOrColdBlockNthPercentile(int PercentileCutoff, + const BasicBlock *BB, + BlockFrequencyInfo *BFI); }; /// An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo. diff --git a/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/llvm/lib/Analysis/ProfileSummaryInfo.cpp --- a/llvm/lib/Analysis/ProfileSummaryInfo.cpp +++ b/llvm/lib/Analysis/ProfileSummaryInfo.cpp @@ -86,23 +86,24 @@ // The profile summary metadata may be attached either by the frontend or by // any backend passes (IR level instrumentation, for example). This method // checks if the Summary is null and if so checks if the summary metadata is now -// available in the module and parses it to get the Summary object. Returns true -// if a valid Summary is available. -bool ProfileSummaryInfo::computeSummary() { - if (Summary) - return true; +// available in the module and parses it to get the Summary object. +void ProfileSummaryInfo::refresh() { + if (hasProfileSummary()) + return; // First try to get context sensitive ProfileSummary. auto *SummaryMD = M.getProfileSummary(/* IsCS */ true); - if (SummaryMD) { + if (SummaryMD) Summary.reset(ProfileSummary::getFromMD(SummaryMD)); - return true; + + if (!hasProfileSummary()) { + // This will actually return PSK_Instr or PSK_Sample summary. + SummaryMD = M.getProfileSummary(/* IsCS */ false); + if (SummaryMD) + Summary.reset(ProfileSummary::getFromMD(SummaryMD)); } - // This will actually return PSK_Instr or PSK_Sample summary. - SummaryMD = M.getProfileSummary(/* IsCS */ false); - if (!SummaryMD) - return false; - Summary.reset(ProfileSummary::getFromMD(SummaryMD)); - return true; + if (!hasProfileSummary()) + return; + computeThresholds(); } Optional ProfileSummaryInfo::getProfileCount(const CallBase &Call, @@ -129,7 +130,7 @@ /// either means it is not hot or it is unknown whether it is hot or not (for /// example, no profile data is available). bool ProfileSummaryInfo::isFunctionEntryHot(const Function *F) { - if (!F || !computeSummary()) + if (!F || !hasProfileSummary()) return false; auto FunctionCount = F->getEntryCount(); // FIXME: The heuristic used below for determining hotness is based on @@ -145,7 +146,7 @@ /// (for example, no profile data is available). bool ProfileSummaryInfo::isFunctionHotInCallGraph(const Function *F, BlockFrequencyInfo &BFI) { - if (!F || !computeSummary()) + if (!F || !hasProfileSummary()) return false; if (auto FunctionCount = F->getEntryCount()) if (isHotCount(FunctionCount.getCount())) @@ -174,7 +175,7 @@ /// (for example, no profile data is available). bool ProfileSummaryInfo::isFunctionColdInCallGraph(const Function *F, BlockFrequencyInfo &BFI) { - if (!F || !computeSummary()) + if (!F || !hasProfileSummary()) return false; if (auto FunctionCount = F->getEntryCount()) if (!isColdCount(FunctionCount.getCount())) @@ -201,10 +202,10 @@ return !F.getEntryCount().hasValue(); } -template +template bool ProfileSummaryInfo::isFunctionHotOrColdInCallGraphNthPercentile( int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) { - if (!F || !computeSummary()) + if (!F || !hasProfileSummary()) return false; if (auto FunctionCount = F->getEntryCount()) { if (isHot && @@ -256,7 +257,7 @@ return false; if (F->hasFnAttribute(Attribute::Cold)) return true; - if (!computeSummary()) + if (!hasProfileSummary()) return false; auto FunctionCount = F->getEntryCount(); // FIXME: The heuristic used below for determining coldness is based on @@ -267,8 +268,6 @@ /// Compute the hot and cold thresholds. void ProfileSummaryInfo::computeThresholds() { - if (!computeSummary()) - return; auto &DetailedSummary = Summary->getDetailedSummary(); auto &HotEntry = getEntryForPercentile(DetailedSummary, ProfileSummaryCutoffHot); @@ -289,7 +288,7 @@ } Optional ProfileSummaryInfo::computeThreshold(int PercentileCutoff) { - if (!computeSummary()) + if (!hasProfileSummary()) return None; auto iter = ThresholdCache.find(PercentileCutoff); if (iter != ThresholdCache.end()) { @@ -304,30 +303,22 @@ } bool ProfileSummaryInfo::hasHugeWorkingSetSize() { - if (!HasHugeWorkingSetSize) - computeThresholds(); return HasHugeWorkingSetSize && HasHugeWorkingSetSize.getValue(); } bool ProfileSummaryInfo::hasLargeWorkingSetSize() { - if (!HasLargeWorkingSetSize) - computeThresholds(); return HasLargeWorkingSetSize && HasLargeWorkingSetSize.getValue(); } bool ProfileSummaryInfo::isHotCount(uint64_t C) { - if (!HotCountThreshold) - computeThresholds(); return HotCountThreshold && C >= HotCountThreshold.getValue(); } bool ProfileSummaryInfo::isColdCount(uint64_t C) { - if (!ColdCountThreshold) - computeThresholds(); return ColdCountThreshold && C <= ColdCountThreshold.getValue(); } -template +template bool ProfileSummaryInfo::isHotOrColdCountNthPercentile(int PercentileCutoff, uint64_t C) { auto CountThreshold = computeThreshold(PercentileCutoff); @@ -337,41 +328,39 @@ return CountThreshold && C <= CountThreshold.getValue(); } -bool ProfileSummaryInfo::isHotCountNthPercentile(int PercentileCutoff, uint64_t C) { +bool ProfileSummaryInfo::isHotCountNthPercentile(int PercentileCutoff, + uint64_t C) { return isHotOrColdCountNthPercentile(PercentileCutoff, C); } -bool ProfileSummaryInfo::isColdCountNthPercentile(int PercentileCutoff, uint64_t C) { +bool ProfileSummaryInfo::isColdCountNthPercentile(int PercentileCutoff, + uint64_t C) { return isHotOrColdCountNthPercentile(PercentileCutoff, C); } uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() { - if (!HotCountThreshold) - computeThresholds(); return HotCountThreshold ? HotCountThreshold.getValue() : UINT64_MAX; } uint64_t ProfileSummaryInfo::getOrCompColdCountThreshold() { - if (!ColdCountThreshold) - computeThresholds(); return ColdCountThreshold ? ColdCountThreshold.getValue() : 0; } -bool ProfileSummaryInfo::isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI) { +bool ProfileSummaryInfo::isHotBlock(const BasicBlock *BB, + BlockFrequencyInfo *BFI) { auto Count = BFI->getBlockProfileCount(BB); return Count && isHotCount(*Count); } bool ProfileSummaryInfo::isColdBlock(const BasicBlock *BB, - BlockFrequencyInfo *BFI) { + BlockFrequencyInfo *BFI) { auto Count = BFI->getBlockProfileCount(BB); return Count && isColdCount(*Count); } -template -bool ProfileSummaryInfo::isHotOrColdBlockNthPercentile(int PercentileCutoff, - const BasicBlock *BB, - BlockFrequencyInfo *BFI) { +template +bool ProfileSummaryInfo::isHotOrColdBlockNthPercentile( + int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) { auto Count = BFI->getBlockProfileCount(BB); if (isHot) return Count && isHotCountNthPercentile(PercentileCutoff, *Count); diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -1848,10 +1848,11 @@ return false; PSI = _PSI; - if (M.getProfileSummary(/* IsCS */ false) == nullptr) + if (M.getProfileSummary(/* IsCS */ false) == nullptr) { M.setProfileSummary(Reader->getSummary().getMD(M.getContext()), ProfileSummary::PSK_Sample); - + PSI->refresh(); + } // Compute the total number of samples collected in this profile. for (const auto &I : Reader->getProfiles()) TotalCollectedSamples += I.second.getTotalSamples(); diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -1601,6 +1601,7 @@ M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()), IsCS ? ProfileSummary::PSK_CSInstr : ProfileSummary::PSK_Instr); + PSI->refresh(); std::unordered_multimap ComdatMembers; collectComdatMembers(M, ComdatMembers);