diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h --- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h +++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h @@ -52,6 +52,8 @@ // because the number of profile counts required to reach the hot // percentile is above a huge threshold. Optional HasHugeWorkingSetSize; + Optional PgsoHotCountThreshold; + Optional PgsoHasHugeWorkingSetSize; public: ProfileSummaryInfo(Module &M) : M(M) {} @@ -96,6 +98,9 @@ bool AllowSynthetic = false); /// Returns true if the working set size of the code is considered huge. bool hasHugeWorkingSetSize(); + /// Returns true if the working set size of the code is considered huge for + /// PGSO. + bool pgsoHasHugeWorkingSetSize(); /// Returns true if \p F has hot function entry. bool isFunctionEntryHot(const Function *F); /// Returns true if \p F contains hot code. @@ -104,14 +109,20 @@ bool isFunctionEntryCold(const Function *F); /// Returns true if \p F contains only cold code. bool isFunctionColdInCallGraph(const Function *F, BlockFrequencyInfo &BFI); + /// Returns true if \p F contains hot code for PGSO. + bool isFunctionPgsoHotInCallGraph(const Function *F, BlockFrequencyInfo &BFI); /// Returns true if count \p C is considered hot. bool isHotCount(uint64_t C); /// Returns true if count \p C is considered cold. bool isColdCount(uint64_t C); + /// Returns true if count \p C is considered hot for PGSO. + bool isPgsoHotCount(uint64_t C); /// Returns true if BasicBlock \p BB is considered hot. bool isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI); /// Returns true if BasicBlock \p BB is considered cold. bool isColdBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI); + /// Returns true if BasicBlock \p BB is considered hot for PGSO. + bool isPgsoHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI); /// Returns true if CallSite \p CS is considered hot. bool isHotCallSite(const CallSite &CS, BlockFrequencyInfo *BFI); /// Returns true if Callsite \p CS is considered cold. diff --git a/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/llvm/lib/Analysis/ProfileSummaryInfo.cpp --- a/llvm/lib/Analysis/ProfileSummaryInfo.cpp +++ b/llvm/lib/Analysis/ProfileSummaryInfo.cpp @@ -57,6 +57,24 @@ cl::desc("A fixed cold count that overrides the count derived from" " profile-summary-cutoff-cold")); +static cl::opt PgsoCutoffInstrProf( + "pgso-cutoff-instr-prof", cl::Hidden, cl::init(250000), cl::ZeroOrMore, + cl::desc("A count is hot for PGSO if it exceeds the minimum count " + "to reach this percentile of total counts under instrumentation " + "profile.")); + +static cl::opt PgsoCutoffSampleProf( + "pgso-cutoff-sample-prof", cl::Hidden, cl::init(800000), cl::ZeroOrMore, + cl::desc("A count is hot for PGSO if it exceeds the minimum count " + "to reach this percentile of total counts under sample profile.")); + +static cl::opt PgsoHugeWorkingSetSizeThreshold( + "pgso-huge-working-set-size-threshold", cl::Hidden, + cl::init(12500), cl::ZeroOrMore, + cl::desc("The code working set size is considered huge for PGSO if the " + "number of blocks required to reach the " + "-profile-summary-cutoff-hot percentile exceeds this count.")); + // Find the summary entry for a desired percentile of counts. static const ProfileSummaryEntry &getEntryForPercentile(SummaryEntryVector &DS, uint64_t Percentile) { @@ -186,6 +204,31 @@ return true; } +// Like isFunctionHotInCallGraph but for PGSO. +bool ProfileSummaryInfo::isFunctionPgsoHotInCallGraph(const Function *F, + BlockFrequencyInfo &BFI) { + if (!F || !computeSummary()) + return false; + if (auto FunctionCount = F->getEntryCount()) + if (isPgsoHotCount(FunctionCount.getCount())) + return true; + + if (hasSampleProfile()) { + uint64_t TotalCallCount = 0; + for (const auto &BB : *F) + for (const auto &I : BB) + if (isa(I) || isa(I)) + if (auto CallCount = getProfileCount(&I, nullptr)) + TotalCallCount += CallCount.getValue(); + if (isPgsoHotCount(TotalCallCount)) + return true; + } + for (const auto &BB : *F) + if (isPgsoHotBlock(&BB, &BFI)) + return true; + return false; +} + /// Returns true if the function's entry is a cold. If it returns false, it /// either means it is not cold or it is unknown whether it is cold or not (for /// example, no profile data is available). @@ -222,6 +265,11 @@ "Cold count threshold cannot exceed hot count threshold!"); HasHugeWorkingSetSize = HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold; + auto &PgsoHotEntry = getEntryForPercentile(DetailedSummary, + hasSampleProfile() ? PgsoCutoffSampleProf : PgsoCutoffInstrProf); + PgsoHotCountThreshold = PgsoHotEntry.MinCount; + PgsoHasHugeWorkingSetSize = + HotEntry.NumCounts > PgsoHugeWorkingSetSizeThreshold; } bool ProfileSummaryInfo::hasHugeWorkingSetSize() { @@ -230,6 +278,12 @@ return HasHugeWorkingSetSize && HasHugeWorkingSetSize.getValue(); } +bool ProfileSummaryInfo::pgsoHasHugeWorkingSetSize() { + if (!PgsoHasHugeWorkingSetSize) + computeThresholds(); + return PgsoHasHugeWorkingSetSize && PgsoHasHugeWorkingSetSize.getValue(); +} + bool ProfileSummaryInfo::isHotCount(uint64_t C) { if (!HotCountThreshold) computeThresholds(); @@ -242,6 +296,12 @@ return ColdCountThreshold && C <= ColdCountThreshold.getValue(); } +bool ProfileSummaryInfo::isPgsoHotCount(uint64_t C) { + if (!PgsoHotCountThreshold) + computeThresholds(); + return PgsoHotCountThreshold && C >= PgsoHotCountThreshold.getValue(); +} + uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() { if (!HotCountThreshold) computeThresholds(); @@ -265,6 +325,12 @@ return Count && isColdCount(*Count); } +bool ProfileSummaryInfo::isPgsoHotBlock(const BasicBlock *BB, + BlockFrequencyInfo *BFI) { + auto Count = BFI->getBlockProfileCount(BB); + return Count && isPgsoHotCount(*Count); +} + bool ProfileSummaryInfo::isHotCallSite(const CallSite &CS, BlockFrequencyInfo *BFI) { auto C = getProfileCount(CS.getInstruction(), BFI);