diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -861,6 +861,8 @@ return CallsiteSamples; } + CallsiteSampleMap &getCallsiteSamples() { return CallsiteSamples; } + /// Return the maximum of sample counts in a function body including functions /// inlined in it. uint64_t getMaxCountInside() const { @@ -1183,11 +1185,14 @@ }; // CSProfileConverter converts a full context-sensitive flat sample profile into -// a nested context-sensitive sample profile. +// a nested context-sensitive sample profile, or the other way around from a regular +// or CS nested profile to a CS flat profile. class CSProfileConverter { public: - CSProfileConverter(SampleProfileMap &Profiles); - void convertProfiles(); + CSProfileConverter(SampleProfileMap &Profiles, + std::list* Contexts = nullptr); + void convertToCSNestedProfiles(); + void convertToCSFlatProfiles(); struct FrameNode { FrameNode(StringRef FName = StringRef(), FunctionSamples *FSamples = nullptr, @@ -1209,10 +1214,17 @@ private: // Nest all children profiles into the profile of Node. - void convertProfiles(FrameNode &Node); + void convertToCSFlatProfiles(FrameNode &Node); + void convertToCSFlatProfiles(FunctionSamples &FProfile, + SampleProfileMap &NewProfileMap, + SampleContextFrameVector &Context); FrameNode *getOrCreateContextPath(const SampleContext &Context); SampleProfileMap &ProfileMap; + + // Underlying context table serves for sample profile writer when + // converting a nested profile to a CS profile. + std::list *Contexts; FrameNode RootFrame; }; diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -24,6 +24,7 @@ #include "llvm/Support/raw_ostream.h" #include #include +#include using namespace llvm; using namespace sampleprof; @@ -219,7 +220,7 @@ unsigned FunctionSamples::getOffset(const DILocation *DIL) { return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) & - 0xffff; + 0xffff; } LineLocation FunctionSamples::getCallSiteIdentifier(const DILocation *DIL, @@ -480,15 +481,9 @@ return &AllChildFrames[Hash]; } -CSProfileConverter::CSProfileConverter(SampleProfileMap &Profiles) - : ProfileMap(Profiles) { - for (auto &FuncSample : Profiles) { - FunctionSamples *FSamples = &FuncSample.second; - auto *NewNode = getOrCreateContextPath(FSamples->getContext()); - assert(!NewNode->FuncSamples && "New node cannot have sample profile"); - NewNode->FuncSamples = FSamples; - } -} +CSProfileConverter::CSProfileConverter( + SampleProfileMap &Profiles, std::list *Contexts) + : ProfileMap(Profiles), Contexts(Contexts) {} CSProfileConverter::FrameNode * CSProfileConverter::getOrCreateContextPath(const SampleContext &Context) { @@ -501,14 +496,14 @@ return Node; } -void CSProfileConverter::convertProfiles(CSProfileConverter::FrameNode &Node) { +void CSProfileConverter::convertToCSFlatProfiles(CSProfileConverter::FrameNode &Node) { // Process each child profile. Add each child profile to callsite profile map // of the current node `Node` if `Node` comes with a profile. Otherwise // promote the child profile to a standalone profile. auto *NodeProfile = Node.FuncSamples; for (auto &It : Node.AllChildFrames) { auto &ChildNode = It.second; - convertProfiles(ChildNode); + convertToCSFlatProfiles(ChildNode); auto *ChildProfile = ChildNode.FuncSamples; if (!ChildProfile) continue; @@ -547,4 +542,66 @@ } } -void CSProfileConverter::convertProfiles() { convertProfiles(RootFrame); } +void CSProfileConverter::convertToCSNestedProfiles() { + for (auto &FuncSample : ProfileMap) { + FunctionSamples *FSamples = &FuncSample.second; + auto *NewNode = getOrCreateContextPath(FSamples->getContext()); + assert(!NewNode->FuncSamples && "New node cannot have sample profile"); + NewNode->FuncSamples = FSamples; + } + convertToCSFlatProfiles(RootFrame); +} + +void CSProfileConverter::convertToCSFlatProfiles(FunctionSamples &FProfile, + SampleProfileMap &NewProfileMap, + SampleContextFrameVector &ParentContext) { + // Push current frame. + ParentContext.emplace_back(FProfile.getName(), LineLocation(0, 0)); + + // Recursively promote callee profiles to a standalone top-level profiles with + // a flattened context. + while (!FProfile.getCallsiteSamples().empty()) { + auto I = FProfile.getCallsiteSamples().begin(); + ParentContext.back().Location = I->first; + for (auto &Callee : I->second) { + auto &CalleeProfile = Callee.second; + // Add body sample. + FProfile.addBodySamples(I->first.LineOffset, I->first.Discriminator, + CalleeProfile.getEntrySamples()); + // Add callsite sample. + FProfile.addCalledTargetSamples( + I->first.LineOffset, I->first.Discriminator, CalleeProfile.getName(), + CalleeProfile.getEntrySamples()); + // Recursively convert callee profile. + convertToCSFlatProfiles(CalleeProfile, NewProfileMap, ParentContext); + } + + // Remove callee profile + FProfile.getCallsiteSamples().erase(I); + ParentContext.back().Location = LineLocation(0, 0); + } + + // Update total sample since callee samples are withdrawn. + FProfile.updateTotalSamples(); + + // Update head samples for callee profiles. + if (ParentContext.size() > 1) + FProfile.addHeadSamples(FProfile.getEntrySamples()); + + // Update current context. + Contexts->push_back(ParentContext); + SampleContext NewContext(Contexts->back()); + FProfile.setContext(NewContext); + NewProfileMap.emplace(NewContext, FProfile); + + // Pop current frame. + ParentContext.pop_back(); +} + +void CSProfileConverter::convertToCSFlatProfiles() { + SampleProfileMap NewProfileMap; + SampleContextFrameVector Context; + for (auto &FuncSample : ProfileMap) + convertToCSFlatProfiles(FuncSample.second, NewProfileMap, Context); + ProfileMap = std::move(NewProfileMap); +} diff --git a/llvm/test/tools/llvm-profdata/cs-sample-flat-profile.test b/llvm/test/tools/llvm-profdata/cs-sample-flat-profile.test new file mode 100644 --- /dev/null +++ b/llvm/test/tools/llvm-profdata/cs-sample-flat-profile.test @@ -0,0 +1,21 @@ +RUN: llvm-profdata merge --sample --text -output=%t.proftext %S/Inputs/sample-profile.proftext --gen-cs-flat-profile=1 +RUN: FileCheck %s < %t.proftext + + +; CHECK: [main]:10896:0 +; CHECK-NEXT: 4: 534 +; CHECK-NEXT: 4.2: 534 +; CHECK-NEXT: 5: 1075 +; CHECK-NEXT: 5.1: 1075 +; CHECK-NEXT: 6: 2080 +; CHECK-NEXT: 7: 534 +; CHECK-NEXT: 9: 2064 _Z3bari:1471 _Z3fooi:631 +; CHECK-NEXT: 10: 3000 inline2:2000 inline1:1000 +; CHECK-NEXT: [main:10 @ inline2]:2000:2000 +; CHECK-NEXT: 1: 2000 +; CHECK-NEXT: [_Z3bari]:1437:1437 +; CHECK-NEXT: 1: 1437 +; CHECK-NEXT: [main:10 @ inline1]:1000:1000 +; CHECK-NEXT: 1: 1000 +; CHECK-NEXT: [_Z3fooi]:610:610 +; CHECK-NEXT: 1: 610 diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -732,7 +732,7 @@ mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper, StringRef OutputFilename, ProfileFormat OutputFormat, StringRef ProfileSymbolListFile, bool CompressAllSections, - bool UseMD5, bool GenPartialProfile, bool GenCSNestedProfile, + bool UseMD5, bool GenPartialProfile, bool GenCSNestedProfile, bool GenCSFlatProfile, bool SampleMergeColdContext, bool SampleTrimColdContext, bool SampleColdContextFrameDepth, FailureMode FailMode) { using namespace sampleprof; @@ -809,10 +809,19 @@ SampleMergeColdContext, SampleColdContextFrameDepth, false); } + + // Underlying context table serves for sample profile writer when + // converting a nested profile to a CS profile. + std::list Contexts; + if (ProfileIsCSFlat && GenCSNestedProfile) { CSProfileConverter CSConverter(ProfileMap); - CSConverter.convertProfiles(); + CSConverter.convertToCSNestedProfiles(); ProfileIsCSFlat = FunctionSamples::ProfileIsCSFlat = false; + } else if (!ProfileIsCSFlat.getValue() && GenCSFlatProfile) { + CSProfileConverter CSConverter(ProfileMap, &Contexts); + CSConverter.convertToCSFlatProfiles(); + ProfileIsCSFlat = FunctionSamples::ProfileIsCSFlat = true; } auto WriterOrErr = @@ -997,6 +1006,9 @@ cl::opt GenCSNestedProfile( "gen-cs-nested-profile", cl::Hidden, cl::init(false), cl::desc("Generate nested function profiles for CSSPGO")); + cl::opt GenCSFlatProfile( + "gen-cs-flat-profile", cl::init(false), + cl::desc("Generate a CS flat profile from a nested profile.")); cl::opt DebugInfoFilename( "debug-info", cl::init(""), cl::desc("Use the provided debug info to correlate the raw profile.")); @@ -1049,7 +1061,7 @@ else mergeSampleProfile(WeightedInputs, Remapper.get(), OutputFilename, OutputFormat, ProfileSymbolListFile, CompressAllSections, - UseMD5, GenPartialProfile, GenCSNestedProfile, + UseMD5, GenPartialProfile, GenCSNestedProfile, GenCSFlatProfile, SampleMergeColdContext, SampleTrimColdContext, SampleColdContextFrameDepth, FailureMode); return 0; diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp --- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp +++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp @@ -882,7 +882,7 @@ calculateAndShowDensity(ContextLessProfiles); if (GenCSNestedProfile) { CSProfileConverter CSConverter(ProfileMap); - CSConverter.convertProfiles(); + CSConverter.convertToCSNestedProfiles(); FunctionSamples::ProfileIsCSFlat = false; FunctionSamples::ProfileIsCSNested = EnableCSPreInliner; }