diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -1128,7 +1128,8 @@ void trimAndMergeColdContextProfiles(uint64_t ColdCountThreshold, bool TrimColdContext, bool MergeColdContext, - uint32_t ColdContextFrameLength); + uint32_t ColdContextFrameLength, + bool TrimBaseProfileOnly); // Canonicalize context profile name and attributes. void canonicalizeContextProfiles(); diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -331,7 +331,7 @@ void SampleContextTrimmer::trimAndMergeColdContextProfiles( uint64_t ColdCountThreshold, bool TrimColdContext, bool MergeColdContext, - uint32_t ColdContextFrameLength) { + uint32_t ColdContextFrameLength, bool TrimBaseProfileOnly) { if (!TrimColdContext && !MergeColdContext) return; @@ -343,10 +343,11 @@ // container std::vector> ColdProfiles; for (const auto &I : ProfileMap) { + const SampleContext &Context = I.first; const FunctionSamples &FunctionProfile = I.second; - if (FunctionProfile.getTotalSamples() >= ColdCountThreshold) - continue; - ColdProfiles.emplace_back(I.first, &I.second); + if (FunctionProfile.getTotalSamples() < ColdCountThreshold && + (!TrimBaseProfileOnly || Context.isBaseContext())) + ColdProfiles.emplace_back(Context, &I.second); } // Remove the cold profile from ProfileMap and merge them into diff --git a/llvm/test/tools/llvm-profgen/cs-preinline.test b/llvm/test/tools/llvm-profgen/cs-preinline.test --- a/llvm/test/tools/llvm-profgen/cs-preinline.test +++ b/llvm/test/tools/llvm-profgen/cs-preinline.test @@ -10,6 +10,11 @@ ; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --output=%t --csspgo-preinliner=1 -sample-profile-cold-inline-threshold=0 ; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-NO-PREINL +; Test cold profile trimming. Only base profiles should be dropped. +; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --output=%t --csspgo-preinliner=1 --csprof-trim-cold-context=1 --profile-summary-hot-count=250 + +; RUN: FileCheck %s --input-file %t --check-prefix=CHECK-TRIM + ; CHECK-DEFAULT: [main:1 @ foo]:225:0 ; CHECK-DEFAULT-NEXT: 2.1: 14 ; CHECK-DEFAULT-NEXT: 3: 15 @@ -39,3 +44,8 @@ ; CHECK-NO-PREINL-NEXT:[bar]:84:0 ; CHECK-NO-PREINL-NEXT: 1: 14 ; CHECK-NO-PREINL-NEXT: !Attributes: 1 + +; CHECK-TRIM-NOT: [foo]:225:0 +; CHECK-TRIM:[foo:3.1 @ bar]:84:0 +; CHECK-TRIM-NEXT: 1: 14 +; CHECK-TRIM-NEXT: !Attributes: 3 diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -759,7 +759,7 @@ SampleContextTrimmer(ProfileMap) .trimAndMergeColdContextProfiles( SampleProfColdThreshold, SampleTrimColdContext, - SampleMergeColdContext, SampleColdContextFrameDepth); + SampleMergeColdContext, SampleColdContextFrameDepth, false); } auto WriterOrErr = diff --git a/llvm/tools/llvm-profgen/CSPreInliner.cpp b/llvm/tools/llvm-profgen/CSPreInliner.cpp --- a/llvm/tools/llvm-profgen/CSPreInliner.cpp +++ b/llvm/tools/llvm-profgen/CSPreInliner.cpp @@ -38,6 +38,9 @@ extern cl::opt ProfileInlineGrowthLimit; extern cl::opt ProfileInlineLimitMin; extern cl::opt ProfileInlineLimitMax; +extern cl::opt CSProfMergeColdContext; +extern cl::opt CSProfTrimColdContext; +extern cl::opt CSProfMaxColdContextDepth; cl::opt EnableCSPreInliner( "csspgo-preinliner", cl::Hidden, cl::init(false), @@ -252,7 +255,7 @@ // trim out such profiles from the output. std::vector ProfilesToBeRemoved; for (auto &It : ProfileMap) { - SampleContext Context = It.second.getContext(); + SampleContext& Context = It.second.getContext(); if (!Context.isBaseContext() && !Context.hasState(InlinedContext)) { assert(Context.hasState(MergedContext) && "Not inlined context profile should be merged already"); @@ -265,7 +268,19 @@ } // Make sure ProfileMap's key is consistent with FunctionSamples' name. - SampleContextTrimmer(ProfileMap).canonicalizeContextProfiles(); + SampleContextTrimmer Trimmer(ProfileMap); + Trimmer.canonicalizeContextProfiles(); + + // Turn off the profile merger by default unless it is explicitly enabled. + if (!CSProfMergeColdContext.getNumOccurrences()) + CSProfMergeColdContext = false; + + // Trim or merge cold profiles from ProfileMap. + if (CSProfTrimColdContext || CSProfMergeColdContext) { + Trimmer.trimAndMergeColdContextProfiles( + HotCountThreshold, CSProfTrimColdContext, CSProfMergeColdContext, + CSProfMaxColdContextDepth, true); + } LLVM_DEBUG(printProfileNames(ProfileMap, false)); } diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp --- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp +++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp @@ -43,18 +43,18 @@ cl::Hidden, cl::location(llvm::sampleprof::CSProfileGenerator::MaxCompressionSize)); -static cl::opt CSProfMergeColdContext( +cl::opt CSProfMergeColdContext( "csprof-merge-cold-context", cl::init(true), cl::ZeroOrMore, cl::desc("If the total count of context profile is smaller than " "the threshold, it will be merged into context-less base " "profile.")); -static cl::opt CSProfTrimColdContext( +cl::opt CSProfTrimColdContext( "csprof-trim-cold-context", cl::init(false), cl::ZeroOrMore, cl::desc("If the total count of the profile after all merge is done " "is still smaller than threshold, it will be trimmed.")); -static cl::opt CSProfMaxColdContextDepth( +cl::opt CSProfMaxColdContextDepth( "csprof-max-cold-context-depth", cl::init(1), cl::ZeroOrMore, cl::desc("Keep the last K contexts while merging cold profile. 1 means the " "context-less base profile")); @@ -658,16 +658,15 @@ if (EnableCSPreInliner) { CSPreInliner(ProfileMap, *Binary, HotCountThreshold, ColdCountThreshold) .run(); - } - - // Trim and merge cold context profile using cold threshold above. By default, - // we skip such merging and trimming when preinliner is on. - if (!EnableCSPreInliner || CSProfTrimColdContext.getNumOccurrences() || - CSProfMergeColdContext.getNumOccurrences()) { - SampleContextTrimmer(ProfileMap) - .trimAndMergeColdContextProfiles( - HotCountThreshold, CSProfTrimColdContext, CSProfMergeColdContext, - CSProfMaxColdContextDepth); + } else { + // Trim and merge cold context profile using cold threshold above. By + // default, we skip such merging and trimming when preinliner is on. + if (CSProfTrimColdContext || CSProfMergeColdContext) { + SampleContextTrimmer(ProfileMap) + .trimAndMergeColdContextProfiles( + HotCountThreshold, CSProfTrimColdContext, CSProfMergeColdContext, + CSProfMaxColdContextDepth, false); + } } }