diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -791,7 +791,7 @@ } assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) && "Cannot have both context-sensitive and regular profile"); - assert(ProfileIsCS == (CSProfileCount > 0) && + assert((!CSProfileCount || ProfileIsCS == (CSProfileCount > 0)) && "Section flag should be consistent with actual profile"); return sampleprof_error::success; } diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -214,6 +214,11 @@ cl::desc("Use call site prioritized inlining for sample profile loader." "Currently only CSSPGO is supported.")); +static cl::opt UsePreInlinerDecision( + "sample-profile-use-preinliner", cl::Hidden, cl::ZeroOrMore, + cl::init(false), + cl::desc("Use the preinliner decisions stored in profile context.")); + static cl::opt ProfileInlineReplayFile( "sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc( @@ -1257,6 +1262,21 @@ return InlineCost::getAlways("previously inlined"); } + // With CSSPGO, the preinliner in llvm-profgen can estimate global inline + // decisions based on hotness as well as accurate function byte sizes for + // given context using function/inlinee sizes from previous build. It + // stores the decision in profile, and also adjust/merge context profile + // aiming at better context-sensitive post-inline profile quality, assming + // all inline decision estimates are going to be honored by compiler. Here + // we replay that inline decision under `sample-profile-use-preinliner`. + if (UsePreInlinerDecision) { + if (Candidate.CalleeSamples->getContext().hasAttribute( + ContextShouldBeInlined)) + return InlineCost::getAlways("preinliner"); + else + return InlineCost::getNever("preinliner"); + } + // Adjust threshold based on call site hotness, only do this for callsite // prioritized inliner because otherwise cost-benefit check is done earlier. int SampleThreshold = SampleColdCallSiteThreshold; diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp --- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp +++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp @@ -407,11 +407,15 @@ .run(); } - // Trim and merge cold context profile using cold threshold above; - SampleContextTrimmer(ProfileMap) - .trimAndMergeColdContextProfiles( - ColdCountThreshold, CSProfTrimColdContext, CSProfMergeColdContext, - CSProfMaxColdContextDepth); + // Trim and merge cold context profile using cold threshold above. By default, + // we skip such merging and trimming when preinliner is on. + if (!EnableCSPreInliner || CSProfTrimColdContext.getNumOccurrences() || + CSProfMergeColdContext.getNumOccurrences()) { + SampleContextTrimmer(ProfileMap) + .trimAndMergeColdContextProfiles( + ColdCountThreshold, CSProfTrimColdContext, CSProfMergeColdContext, + CSProfMaxColdContextDepth); + } } void CSProfileGenerator::computeSummaryAndThreshold() {