diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h --- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h +++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h @@ -52,6 +52,15 @@ // because the number of profile counts required to reach the hot // percentile is above a huge threshold. Optional HasHugeWorkingSetSize; + // True if the working set size of the code is considered large, + // because the number of profile counts required to reach the hot + // percentile is above a large threshold. + Optional HasLargeWorkingSetSize; + // Compute the threshold for a given cutoff. + Optional computeThreshold(int PercentileCutoff); + // The map that caches the threshold values. The keys are the percentile + // cutoff values and the values are the corresponding threshold values. + DenseMap ThresholdCache; public: ProfileSummaryInfo(Module &M) : M(M) {} @@ -96,6 +105,8 @@ bool AllowSynthetic = false); /// Returns true if the working set size of the code is considered huge. bool hasHugeWorkingSetSize(); + /// Returns true if the working set size of the code is considered large. + bool hasLargeWorkingSetSize(); /// Returns true if \p F has hot function entry. bool isFunctionEntryHot(const Function *F); /// Returns true if \p F contains hot code. @@ -104,14 +115,26 @@ bool isFunctionEntryCold(const Function *F); /// Returns true if \p F contains only cold code. bool isFunctionColdInCallGraph(const Function *F, BlockFrequencyInfo &BFI); + /// Returns true if \p F contains hot code with regard to a given hot + /// percentile cutoff value. + bool isFunctionHotInCallGraphNthPercentile(int PercentileCutoff, + const Function *F, + BlockFrequencyInfo &BFI); /// Returns true if count \p C is considered hot. bool isHotCount(uint64_t C); /// Returns true if count \p C is considered cold. bool isColdCount(uint64_t C); + /// Returns true if count \p C is considered hot with regard to a given + /// hot percentile cutoff value. + bool isHotCountNthPercentile(int PercentileCutoff, uint64_t C); /// Returns true if BasicBlock \p BB is considered hot. bool isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI); /// Returns true if BasicBlock \p BB is considered cold. bool isColdBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI); + /// Returns true if BasicBlock \p BB is considered hot with regard to a given + /// hot percentile cutoff value. + bool isHotBlockNthPercentile(int PercentileCutoff, + const BasicBlock *BB, BlockFrequencyInfo *BFI); /// Returns true if CallSite \p CS is considered hot. bool isHotCallSite(const CallSite &CS, BlockFrequencyInfo *BFI); /// Returns true if Callsite \p CS is considered cold. diff --git a/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/llvm/lib/Analysis/ProfileSummaryInfo.cpp --- a/llvm/lib/Analysis/ProfileSummaryInfo.cpp +++ b/llvm/lib/Analysis/ProfileSummaryInfo.cpp @@ -45,6 +45,13 @@ " blocks required to reach the -profile-summary-cutoff-hot" " percentile exceeds this count.")); +static cl::opt ProfileSummaryLargeWorkingSetSizeThreshold( + "profile-summary-large-working-set-size-threshold", cl::Hidden, + cl::init(12500), cl::ZeroOrMore, + cl::desc("The code working set size is considered large if the number of" + " blocks required to reach the -profile-summary-cutoff-hot" + " percentile exceeds this count.")); + // The next two options override the counts derived from summary computation and // are useful for debugging purposes. static cl::opt ProfileSummaryHotCount( @@ -186,6 +193,31 @@ return true; } +// Like isFunctionHotInCallGraph but for a given cutoff. +bool ProfileSummaryInfo::isFunctionHotInCallGraphNthPercentile( + int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) { + if (!F || !computeSummary()) + return false; + if (auto FunctionCount = F->getEntryCount()) + if (isHotCountNthPercentile(PercentileCutoff, FunctionCount.getCount())) + return true; + + if (hasSampleProfile()) { + uint64_t TotalCallCount = 0; + for (const auto &BB : *F) + for (const auto &I : BB) + if (isa(I) || isa(I)) + if (auto CallCount = getProfileCount(&I, nullptr)) + TotalCallCount += CallCount.getValue(); + if (isHotCountNthPercentile(PercentileCutoff, TotalCallCount)) + return true; + } + for (const auto &BB : *F) + if (isHotBlockNthPercentile(PercentileCutoff, &BB, &BFI)) + return true; + return false; +} + /// Returns true if the function's entry is a cold. If it returns false, it /// either means it is not cold or it is unknown whether it is cold or not (for /// example, no profile data is available). @@ -222,6 +254,23 @@ "Cold count threshold cannot exceed hot count threshold!"); HasHugeWorkingSetSize = HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold; + HasLargeWorkingSetSize = + HotEntry.NumCounts > ProfileSummaryLargeWorkingSetSizeThreshold; +} + +Optional ProfileSummaryInfo::computeThreshold(int PercentileCutoff) { + if (!computeSummary()) + return None; + auto iter = ThresholdCache.find(PercentileCutoff); + if (iter != ThresholdCache.end()) { + return iter->second; + } + auto &DetailedSummary = Summary->getDetailedSummary(); + auto &Entry = + getEntryForPercentile(DetailedSummary, PercentileCutoff); + uint64_t CountThreshold = Entry.MinCount; + ThresholdCache[PercentileCutoff] = CountThreshold; + return CountThreshold; } bool ProfileSummaryInfo::hasHugeWorkingSetSize() { @@ -230,6 +279,12 @@ return HasHugeWorkingSetSize && HasHugeWorkingSetSize.getValue(); } +bool ProfileSummaryInfo::hasLargeWorkingSetSize() { + if (!HasLargeWorkingSetSize) + computeThresholds(); + return HasLargeWorkingSetSize && HasLargeWorkingSetSize.getValue(); +} + bool ProfileSummaryInfo::isHotCount(uint64_t C) { if (!HotCountThreshold) computeThresholds(); @@ -242,6 +297,11 @@ return ColdCountThreshold && C <= ColdCountThreshold.getValue(); } +bool ProfileSummaryInfo::isHotCountNthPercentile(int PercentileCutoff, uint64_t C) { + auto CountThreshold = computeThreshold(PercentileCutoff); + return CountThreshold && C >= CountThreshold.getValue(); +} + uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() { if (!HotCountThreshold) computeThresholds(); @@ -265,6 +325,13 @@ return Count && isColdCount(*Count); } +bool ProfileSummaryInfo::isHotBlockNthPercentile(int PercentileCutoff, + const BasicBlock *BB, + BlockFrequencyInfo *BFI) { + auto Count = BFI->getBlockProfileCount(BB); + return Count && isHotCountNthPercentile(PercentileCutoff, *Count); +} + bool ProfileSummaryInfo::isHotCallSite(const CallSite &CS, BlockFrequencyInfo *BFI) { auto C = getProfileCount(CS.getInstruction(), BFI); diff --git a/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp b/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp --- a/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp +++ b/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp @@ -137,6 +137,18 @@ EXPECT_FALSE(PSI.isColdCount(100)); EXPECT_FALSE(PSI.isHotCount(100)); + EXPECT_TRUE(PSI.isHotCountNthPercentile(990000, 400)); + EXPECT_FALSE(PSI.isHotCountNthPercentile(990000, 100)); + EXPECT_FALSE(PSI.isHotCountNthPercentile(990000, 2)); + + EXPECT_TRUE(PSI.isHotCountNthPercentile(999999, 400)); + EXPECT_TRUE(PSI.isHotCountNthPercentile(999999, 100)); + EXPECT_FALSE(PSI.isHotCountNthPercentile(999999, 2)); + + EXPECT_FALSE(PSI.isHotCountNthPercentile(10000, 400)); + EXPECT_FALSE(PSI.isHotCountNthPercentile(10000, 100)); + EXPECT_FALSE(PSI.isHotCountNthPercentile(10000, 2)); + EXPECT_TRUE(PSI.isFunctionEntryHot(F)); EXPECT_FALSE(PSI.isFunctionEntryHot(G)); EXPECT_FALSE(PSI.isFunctionEntryHot(H)); @@ -160,6 +172,21 @@ EXPECT_FALSE(PSI.isHotBlock(BB2, &BFI)); EXPECT_TRUE(PSI.isHotBlock(BB3, &BFI)); + EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, &BB0, &BFI)); + EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, BB1, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB2, &BFI)); + EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, BB3, &BFI)); + + EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, &BB0, &BFI)); + EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB1, &BFI)); + EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB2, &BFI)); + EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB3, &BFI)); + + EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, &BB0, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB1, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB2, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB3, &BFI)); + CallSite CS1(BB1->getFirstNonPHI()); auto *CI2 = BB2->getFirstNonPHI(); CallSite CS2(CI2); @@ -192,6 +219,21 @@ EXPECT_FALSE(PSI.isHotBlock(BB2, &BFI)); EXPECT_TRUE(PSI.isHotBlock(BB3, &BFI)); + EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, &BB0, &BFI)); + EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, BB1, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB2, &BFI)); + EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, BB3, &BFI)); + + EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, &BB0, &BFI)); + EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB1, &BFI)); + EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB2, &BFI)); + EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB3, &BFI)); + + EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, &BB0, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB1, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB2, &BFI)); + EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB3, &BFI)); + CallSite CS1(BB1->getFirstNonPHI()); auto *CI2 = BB2->getFirstNonPHI(); // Manually attach branch weights metadata to the call instruction.