diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h --- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h +++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h @@ -40,8 +40,7 @@ private: Module &M; std::unique_ptr Summary; - bool computeSummary(); - void computeThresholds(); + // Count thresholds to answer isHotCount and isColdCount queries. Optional HotCountThreshold, ColdCountThreshold; // True if the working set size of the code is considered huge, @@ -52,34 +51,34 @@ // because the number of profile counts required to reach the hot // percentile is above a large threshold. Optional HasLargeWorkingSetSize; - // Compute the threshold for a given cutoff. - Optional computeThreshold(int PercentileCutoff); // The map that caches the threshold values. The keys are the percentile // cutoff values and the values are the corresponding threshold values. - DenseMap ThresholdCache; + mutable DenseMap ThresholdCache; public: - ProfileSummaryInfo(Module &M) : M(M) {} - ProfileSummaryInfo(ProfileSummaryInfo &&Arg) - : M(Arg.M), Summary(std::move(Arg.Summary)) {} + ProfileSummaryInfo(Module &M) : M(M) { refresh(); } + ProfileSummaryInfo(ProfileSummaryInfo &&Arg) = default; + + /// If no samples were present, attempt to refresh. + void refresh(); /// Returns true if profile summary is available. - bool hasProfileSummary() { return computeSummary(); } + bool hasProfileSummary() const { return !!Summary; } /// Returns true if module \c M has sample profile. - bool hasSampleProfile() { + bool hasSampleProfile() const { return hasProfileSummary() && Summary->getKind() == ProfileSummary::PSK_Sample; } /// Returns true if module \c M has instrumentation profile. - bool hasInstrumentationProfile() { + bool hasInstrumentationProfile() const { return hasProfileSummary() && Summary->getKind() == ProfileSummary::PSK_Instr; } /// Returns true if module \c M has context sensitive instrumentation profile. - bool hasCSInstrumentationProfile() { + bool hasCSInstrumentationProfile() const { return hasProfileSummary() && Summary->getKind() == ProfileSummary::PSK_CSInstr; } @@ -98,84 +97,91 @@ /// Returns the profile count for \p CallInst. Optional getProfileCount(const CallBase &CallInst, BlockFrequencyInfo *BFI, - bool AllowSynthetic = false); + bool AllowSynthetic = false) const; /// Returns true if module \c M has partial-profile sample profile. - bool hasPartialSampleProfile(); + bool hasPartialSampleProfile() const; /// Returns true if the working set size of the code is considered huge. - bool hasHugeWorkingSetSize(); + bool hasHugeWorkingSetSize() const; /// Returns true if the working set size of the code is considered large. - bool hasLargeWorkingSetSize(); + bool hasLargeWorkingSetSize() const; /// Returns true if \p F has hot function entry. - bool isFunctionEntryHot(const Function *F); + bool isFunctionEntryHot(const Function *F) const; /// Returns true if \p F contains hot code. - bool isFunctionHotInCallGraph(const Function *F, BlockFrequencyInfo &BFI); + bool isFunctionHotInCallGraph(const Function *F, + BlockFrequencyInfo &BFI) const; /// Returns true if \p F has cold function entry. - bool isFunctionEntryCold(const Function *F); + bool isFunctionEntryCold(const Function *F) const; /// Returns true if \p F contains only cold code. - bool isFunctionColdInCallGraph(const Function *F, BlockFrequencyInfo &BFI); + bool isFunctionColdInCallGraph(const Function *F, + BlockFrequencyInfo &BFI) const; /// Returns true if the hotness of \p F is unknown. - bool isFunctionHotnessUnknown(const Function &F); + bool isFunctionHotnessUnknown(const Function &F) const; /// Returns true if \p F contains hot code with regard to a given hot /// percentile cutoff value. bool isFunctionHotInCallGraphNthPercentile(int PercentileCutoff, const Function *F, - BlockFrequencyInfo &BFI); + BlockFrequencyInfo &BFI) const; /// Returns true if \p F contains cold code with regard to a given cold /// percentile cutoff value. bool isFunctionColdInCallGraphNthPercentile(int PercentileCutoff, const Function *F, - BlockFrequencyInfo &BFI); + BlockFrequencyInfo &BFI) const; /// Returns true if count \p C is considered hot. - bool isHotCount(uint64_t C); + bool isHotCount(uint64_t C) const; /// Returns true if count \p C is considered cold. - bool isColdCount(uint64_t C); + bool isColdCount(uint64_t C) const; /// Returns true if count \p C is considered hot with regard to a given /// hot percentile cutoff value. - bool isHotCountNthPercentile(int PercentileCutoff, uint64_t C); + bool isHotCountNthPercentile(int PercentileCutoff, uint64_t C) const; /// Returns true if count \p C is considered cold with regard to a given /// cold percentile cutoff value. - bool isColdCountNthPercentile(int PercentileCutoff, uint64_t C); + bool isColdCountNthPercentile(int PercentileCutoff, uint64_t C) const; /// Returns true if BasicBlock \p BB is considered hot. - bool isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI); + bool isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI) const; /// Returns true if BasicBlock \p BB is considered cold. - bool isColdBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI); + bool isColdBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI) const; /// Returns true if BasicBlock \p BB is considered hot with regard to a given /// hot percentile cutoff value. - bool isHotBlockNthPercentile(int PercentileCutoff, - const BasicBlock *BB, BlockFrequencyInfo *BFI); + bool isHotBlockNthPercentile(int PercentileCutoff, const BasicBlock *BB, + BlockFrequencyInfo *BFI) const; /// Returns true if BasicBlock \p BB is considered cold with regard to a given /// cold percentile cutoff value. - bool isColdBlockNthPercentile(int PercentileCutoff, - const BasicBlock *BB, BlockFrequencyInfo *BFI); + bool isColdBlockNthPercentile(int PercentileCutoff, const BasicBlock *BB, + BlockFrequencyInfo *BFI) const; /// Returns true if the call site \p CB is considered hot. - bool isHotCallSite(const CallBase &CB, BlockFrequencyInfo *BFI); + bool isHotCallSite(const CallBase &CB, BlockFrequencyInfo *BFI) const; /// Returns true if call site \p CB is considered cold. - bool isColdCallSite(const CallBase &CB, BlockFrequencyInfo *BFI); + bool isColdCallSite(const CallBase &CB, BlockFrequencyInfo *BFI) const; /// Returns HotCountThreshold if set. Recompute HotCountThreshold /// if not set. - uint64_t getOrCompHotCountThreshold(); + uint64_t getOrCompHotCountThreshold() const; /// Returns ColdCountThreshold if set. Recompute HotCountThreshold /// if not set. - uint64_t getOrCompColdCountThreshold(); + uint64_t getOrCompColdCountThreshold() const; /// Returns HotCountThreshold if set. - uint64_t getHotCountThreshold() { + uint64_t getHotCountThreshold() const { return HotCountThreshold ? HotCountThreshold.getValue() : 0; } /// Returns ColdCountThreshold if set. - uint64_t getColdCountThreshold() { + uint64_t getColdCountThreshold() const { return ColdCountThreshold ? ColdCountThreshold.getValue() : 0; } private: - template - bool isFunctionHotOrColdInCallGraphNthPercentile(int PercentileCutoff, - const Function *F, - BlockFrequencyInfo &BFI); - template - bool isHotOrColdCountNthPercentile(int PercentileCutoff, uint64_t C); - template - bool isHotOrColdBlockNthPercentile(int PercentileCutoff, const BasicBlock *BB, - BlockFrequencyInfo *BFI); + // Compute the threshold for a given cutoff. + Optional computeThreshold(int PercentileCutoff) const; + + template + bool isFunctionHotOrColdInCallGraphNthPercentile( + int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const; + + template + bool isHotOrColdCountNthPercentile(int PercentileCutoff, uint64_t C) const; + + template + bool isHotOrColdBlockNthPercentile(int PercentileCutoff, + const BasicBlock *BB, + BlockFrequencyInfo *BFI) const; }; /// An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo. diff --git a/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/llvm/lib/Analysis/ProfileSummaryInfo.cpp --- a/llvm/lib/Analysis/ProfileSummaryInfo.cpp +++ b/llvm/lib/Analysis/ProfileSummaryInfo.cpp @@ -86,28 +86,45 @@ // The profile summary metadata may be attached either by the frontend or by // any backend passes (IR level instrumentation, for example). This method // checks if the Summary is null and if so checks if the summary metadata is now -// available in the module and parses it to get the Summary object. Returns true -// if a valid Summary is available. -bool ProfileSummaryInfo::computeSummary() { - if (Summary) - return true; +// available in the module and parses it to get the Summary object. +void ProfileSummaryInfo::refresh() { + if (hasProfileSummary()) + return; // First try to get context sensitive ProfileSummary. auto *SummaryMD = M.getProfileSummary(/* IsCS */ true); if (SummaryMD) { Summary.reset(ProfileSummary::getFromMD(SummaryMD)); - return true; } - // This will actually return PSK_Instr or PSK_Sample summary. - SummaryMD = M.getProfileSummary(/* IsCS */ false); - if (!SummaryMD) - return false; - Summary.reset(ProfileSummary::getFromMD(SummaryMD)); - return true; + if (!hasProfileSummary()) { + // This will actually return PSK_Instr or PSK_Sample summary. + SummaryMD = M.getProfileSummary(/* IsCS */ false); + if (SummaryMD) + Summary.reset(ProfileSummary::getFromMD(SummaryMD)); + } + if (!hasProfileSummary()) + return; + + auto &DetailedSummary = Summary->getDetailedSummary(); + auto &HotEntry = + getEntryForPercentile(DetailedSummary, ProfileSummaryCutoffHot); + HotCountThreshold = HotEntry.MinCount; + if (ProfileSummaryHotCount.getNumOccurrences() > 0) + HotCountThreshold = ProfileSummaryHotCount; + auto &ColdEntry = + getEntryForPercentile(DetailedSummary, ProfileSummaryCutoffCold); + ColdCountThreshold = ColdEntry.MinCount; + if (ProfileSummaryColdCount.getNumOccurrences() > 0) + ColdCountThreshold = ProfileSummaryColdCount; + assert(ColdCountThreshold <= HotCountThreshold && + "Cold count threshold cannot exceed hot count threshold!"); + HasHugeWorkingSetSize = + HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold; + HasLargeWorkingSetSize = + HotEntry.NumCounts > ProfileSummaryLargeWorkingSetSizeThreshold; } -Optional ProfileSummaryInfo::getProfileCount(const CallBase &Call, - BlockFrequencyInfo *BFI, - bool AllowSynthetic) { +Optional ProfileSummaryInfo::getProfileCount( + const CallBase &Call, BlockFrequencyInfo *BFI, bool AllowSynthetic) const { assert((isa(Call) || isa(Call)) && "We can only get profile count for call/invoke instruction."); if (hasSampleProfile()) { @@ -128,8 +145,8 @@ /// Returns true if the function's entry is hot. If it returns false, it /// either means it is not hot or it is unknown whether it is hot or not (for /// example, no profile data is available). -bool ProfileSummaryInfo::isFunctionEntryHot(const Function *F) { - if (!F || !computeSummary()) +bool ProfileSummaryInfo::isFunctionEntryHot(const Function *F) const { + if (!F || !hasProfileSummary()) return false; auto FunctionCount = F->getEntryCount(); // FIXME: The heuristic used below for determining hotness is based on @@ -143,9 +160,9 @@ /// hot total call edge count. /// If it returns false, it either means it is not hot or it is unknown /// (for example, no profile data is available). -bool ProfileSummaryInfo::isFunctionHotInCallGraph(const Function *F, - BlockFrequencyInfo &BFI) { - if (!F || !computeSummary()) +bool ProfileSummaryInfo::isFunctionHotInCallGraph( + const Function *F, BlockFrequencyInfo &BFI) const { + if (!F || !hasProfileSummary()) return false; if (auto FunctionCount = F->getEntryCount()) if (isHotCount(FunctionCount.getCount())) @@ -172,9 +189,9 @@ /// the total call edge count is cold. /// If it returns false, it either means it is not cold or it is unknown /// (for example, no profile data is available). -bool ProfileSummaryInfo::isFunctionColdInCallGraph(const Function *F, - BlockFrequencyInfo &BFI) { - if (!F || !computeSummary()) +bool ProfileSummaryInfo::isFunctionColdInCallGraph( + const Function *F, BlockFrequencyInfo &BFI) const { + if (!F || !hasProfileSummary()) return false; if (auto FunctionCount = F->getEntryCount()) if (!isColdCount(FunctionCount.getCount())) @@ -196,15 +213,15 @@ return true; } -bool ProfileSummaryInfo::isFunctionHotnessUnknown(const Function &F) { +bool ProfileSummaryInfo::isFunctionHotnessUnknown(const Function &F) const { assert(hasPartialSampleProfile() && "Expect partial sample profile"); return !F.getEntryCount().hasValue(); } -template +template bool ProfileSummaryInfo::isFunctionHotOrColdInCallGraphNthPercentile( - int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) { - if (!F || !computeSummary()) + int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const { + if (!F || !hasProfileSummary()) return false; if (auto FunctionCount = F->getEntryCount()) { if (isHot && @@ -237,13 +254,13 @@ // Like isFunctionHotInCallGraph but for a given cutoff. bool ProfileSummaryInfo::isFunctionHotInCallGraphNthPercentile( - int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) { + int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const { return isFunctionHotOrColdInCallGraphNthPercentile( PercentileCutoff, F, BFI); } bool ProfileSummaryInfo::isFunctionColdInCallGraphNthPercentile( - int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) { + int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const { return isFunctionHotOrColdInCallGraphNthPercentile( PercentileCutoff, F, BFI); } @@ -251,12 +268,12 @@ /// Returns true if the function's entry is a cold. If it returns false, it /// either means it is not cold or it is unknown whether it is cold or not (for /// example, no profile data is available). -bool ProfileSummaryInfo::isFunctionEntryCold(const Function *F) { +bool ProfileSummaryInfo::isFunctionEntryCold(const Function *F) const { if (!F) return false; if (F->hasFnAttribute(Attribute::Cold)) return true; - if (!computeSummary()) + if (!hasProfileSummary()) return false; auto FunctionCount = F->getEntryCount(); // FIXME: The heuristic used below for determining coldness is based on @@ -265,31 +282,9 @@ return FunctionCount && isColdCount(FunctionCount.getCount()); } -/// Compute the hot and cold thresholds. -void ProfileSummaryInfo::computeThresholds() { - if (!computeSummary()) - return; - auto &DetailedSummary = Summary->getDetailedSummary(); - auto &HotEntry = - getEntryForPercentile(DetailedSummary, ProfileSummaryCutoffHot); - HotCountThreshold = HotEntry.MinCount; - if (ProfileSummaryHotCount.getNumOccurrences() > 0) - HotCountThreshold = ProfileSummaryHotCount; - auto &ColdEntry = - getEntryForPercentile(DetailedSummary, ProfileSummaryCutoffCold); - ColdCountThreshold = ColdEntry.MinCount; - if (ProfileSummaryColdCount.getNumOccurrences() > 0) - ColdCountThreshold = ProfileSummaryColdCount; - assert(ColdCountThreshold <= HotCountThreshold && - "Cold count threshold cannot exceed hot count threshold!"); - HasHugeWorkingSetSize = - HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold; - HasLargeWorkingSetSize = - HotEntry.NumCounts > ProfileSummaryLargeWorkingSetSizeThreshold; -} - -Optional ProfileSummaryInfo::computeThreshold(int PercentileCutoff) { - if (!computeSummary()) +Optional +ProfileSummaryInfo::computeThreshold(int PercentileCutoff) const { + if (!hasProfileSummary()) return None; auto iter = ThresholdCache.find(PercentileCutoff); if (iter != ThresholdCache.end()) { @@ -303,33 +298,25 @@ return CountThreshold; } -bool ProfileSummaryInfo::hasHugeWorkingSetSize() { - if (!HasHugeWorkingSetSize) - computeThresholds(); +bool ProfileSummaryInfo::hasHugeWorkingSetSize() const { return HasHugeWorkingSetSize && HasHugeWorkingSetSize.getValue(); } -bool ProfileSummaryInfo::hasLargeWorkingSetSize() { - if (!HasLargeWorkingSetSize) - computeThresholds(); +bool ProfileSummaryInfo::hasLargeWorkingSetSize() const { return HasLargeWorkingSetSize && HasLargeWorkingSetSize.getValue(); } -bool ProfileSummaryInfo::isHotCount(uint64_t C) { - if (!HotCountThreshold) - computeThresholds(); +bool ProfileSummaryInfo::isHotCount(uint64_t C) const { return HotCountThreshold && C >= HotCountThreshold.getValue(); } -bool ProfileSummaryInfo::isColdCount(uint64_t C) { - if (!ColdCountThreshold) - computeThresholds(); +bool ProfileSummaryInfo::isColdCount(uint64_t C) const { return ColdCountThreshold && C <= ColdCountThreshold.getValue(); } -template +template bool ProfileSummaryInfo::isHotOrColdCountNthPercentile(int PercentileCutoff, - uint64_t C) { + uint64_t C) const { auto CountThreshold = computeThreshold(PercentileCutoff); if (isHot) return CountThreshold && C >= CountThreshold.getValue(); @@ -337,41 +324,39 @@ return CountThreshold && C <= CountThreshold.getValue(); } -bool ProfileSummaryInfo::isHotCountNthPercentile(int PercentileCutoff, uint64_t C) { +bool ProfileSummaryInfo::isHotCountNthPercentile(int PercentileCutoff, + uint64_t C) const { return isHotOrColdCountNthPercentile(PercentileCutoff, C); } -bool ProfileSummaryInfo::isColdCountNthPercentile(int PercentileCutoff, uint64_t C) { +bool ProfileSummaryInfo::isColdCountNthPercentile(int PercentileCutoff, + uint64_t C) const { return isHotOrColdCountNthPercentile(PercentileCutoff, C); } -uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() { - if (!HotCountThreshold) - computeThresholds(); +uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() const { return HotCountThreshold ? HotCountThreshold.getValue() : UINT64_MAX; } -uint64_t ProfileSummaryInfo::getOrCompColdCountThreshold() { - if (!ColdCountThreshold) - computeThresholds(); +uint64_t ProfileSummaryInfo::getOrCompColdCountThreshold() const { return ColdCountThreshold ? ColdCountThreshold.getValue() : 0; } -bool ProfileSummaryInfo::isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI) { +bool ProfileSummaryInfo::isHotBlock(const BasicBlock *BB, + BlockFrequencyInfo *BFI) const { auto Count = BFI->getBlockProfileCount(BB); return Count && isHotCount(*Count); } bool ProfileSummaryInfo::isColdBlock(const BasicBlock *BB, - BlockFrequencyInfo *BFI) { + BlockFrequencyInfo *BFI) const { auto Count = BFI->getBlockProfileCount(BB); return Count && isColdCount(*Count); } -template -bool ProfileSummaryInfo::isHotOrColdBlockNthPercentile(int PercentileCutoff, - const BasicBlock *BB, - BlockFrequencyInfo *BFI) { +template +bool ProfileSummaryInfo::isHotOrColdBlockNthPercentile( + int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const { auto Count = BFI->getBlockProfileCount(BB); if (isHot) return Count && isHotCountNthPercentile(PercentileCutoff, *Count); @@ -379,26 +364,24 @@ return Count && isColdCountNthPercentile(PercentileCutoff, *Count); } -bool ProfileSummaryInfo::isHotBlockNthPercentile(int PercentileCutoff, - const BasicBlock *BB, - BlockFrequencyInfo *BFI) { +bool ProfileSummaryInfo::isHotBlockNthPercentile( + int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const { return isHotOrColdBlockNthPercentile(PercentileCutoff, BB, BFI); } -bool ProfileSummaryInfo::isColdBlockNthPercentile(int PercentileCutoff, - const BasicBlock *BB, - BlockFrequencyInfo *BFI) { +bool ProfileSummaryInfo::isColdBlockNthPercentile( + int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const { return isHotOrColdBlockNthPercentile(PercentileCutoff, BB, BFI); } bool ProfileSummaryInfo::isHotCallSite(const CallBase &CB, - BlockFrequencyInfo *BFI) { + BlockFrequencyInfo *BFI) const { auto C = getProfileCount(CB, BFI); return C && isHotCount(*C); } bool ProfileSummaryInfo::isColdCallSite(const CallBase &CB, - BlockFrequencyInfo *BFI) { + BlockFrequencyInfo *BFI) const { auto C = getProfileCount(CB, BFI); if (C) return isColdCount(*C); @@ -408,7 +391,7 @@ return hasSampleProfile() && CB.getCaller()->hasProfileData(); } -bool ProfileSummaryInfo::hasPartialSampleProfile() { +bool ProfileSummaryInfo::hasPartialSampleProfile() const { return hasProfileSummary() && Summary->getKind() == ProfileSummary::PSK_Sample && (PartialProfile || Summary->isPartialProfile()); diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -1852,6 +1852,7 @@ M.setProfileSummary(Reader->getSummary().getMD(M.getContext()), ProfileSummary::PSK_Sample); + PSI->refresh(); // Compute the total number of samples collected in this profile. for (const auto &I : Reader->getProfiles()) TotalCollectedSamples += I.second.getTotalSamples(); diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -1601,6 +1601,7 @@ M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()), IsCS ? ProfileSummary::PSK_CSInstr : ProfileSummary::PSK_Instr); + PSI->refresh(); std::unordered_multimap ComdatMembers; collectComdatMembers(M, ComdatMembers);