diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -462,9 +462,7 @@ bool isBaseContext() const { return CallingContext.empty(); } StringRef getNameWithoutContext() const { return Name; } StringRef getCallingContext() const { return CallingContext; } - StringRef getNameWithContext(bool WithBracket = false) const { - return WithBracket ? InputContext : FullContext; - } + StringRef getNameWithContext() const { return FullContext; } private: // Give a context string, decode and populate internal states like @@ -472,7 +470,6 @@ // `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]` void setContext(StringRef ContextStr, ContextStateMask CState) { assert(!ContextStr.empty()); - InputContext = ContextStr; // Note that `[]` wrapped input indicates a full context string, otherwise // it's treated as context-less function name only. bool HasContext = ContextStr.startswith("["); @@ -504,9 +501,6 @@ } } - // Input context string including bracketed calling context and leaf function - // name - StringRef InputContext; // Full context string including calling context and leaf function name StringRef FullContext; // Function name for the associated sample profile @@ -711,7 +705,7 @@ Name = Other.getName(); if (!GUIDToFuncNameMap) GUIDToFuncNameMap = Other.GUIDToFuncNameMap; - if (Context.getNameWithContext(true).empty()) + if (Context.getNameWithContext().empty()) Context = Other.getContext(); if (FunctionHash == 0) { // Set the function hash code for the target profile. @@ -780,10 +774,8 @@ StringRef getName() const { return Name; } /// Return function name with context. - StringRef getNameWithContext(bool WithBracket = false) const { - return FunctionSamples::ProfileIsCS - ? Context.getNameWithContext(WithBracket) - : Name; + StringRef getNameWithContext() const { + return FunctionSamples::ProfileIsCS ? Context.getNameWithContext() : Name; } /// Return the original function name. @@ -986,6 +978,24 @@ SamplesWithLocList V; }; +/// SampleContextTrimmer impelements helper functions to trim, merge cold +/// context profiles. It also supports context profile canonicalization to make +/// sure ProfileMap's key is consistent with FunctionSample's name/context. +class SampleContextTrimmer { +public: + SampleContextTrimmer(StringMap &Profiles) + : ProfileMap(Profiles){}; + // Trim and merge cold context profile when requested. + void trimAndMergeColdContextProfiles(uint64_t ColdCountThreshold, + bool TrimColdContext = true, + bool MergeColdContext = true); + // Canonicalize context profile name and attributes. + void canonicalizeContextProfiles(); + +private: + StringMap &ProfileMap; +}; + /// ProfileSymbolList records the list of function symbols shown up /// in the binary used to generate the profile. It is useful to /// to discriminate a function being so cold as not to shown up diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h --- a/llvm/include/llvm/ProfileData/SampleProfWriter.h +++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h @@ -25,6 +25,7 @@ #include #include #include +#include namespace llvm { namespace sampleprof { @@ -136,13 +137,14 @@ virtual std::error_code writeHeader(const StringMap &ProfileMap) override; std::error_code writeSummary(); - std::error_code writeNameIdx(StringRef FName); + std::error_code writeNameIdx(StringRef FName, bool IsContextName = false); std::error_code writeBody(const FunctionSamples &S); inline void stablizeNameTable(std::set &V); MapVector NameTable; + std::unordered_set BracketedContextStr; - void addName(StringRef FName); + void addName(StringRef FName, bool IsContextName = false); void addNames(const FunctionSamples &S); private: diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -316,6 +316,85 @@ return sampleprof_error::success; } +void SampleContextTrimmer::trimAndMergeColdContextProfiles( + uint64_t ColdCountThreshold, bool TrimColdContext, bool MergeColdContext) { + if (!TrimColdContext && !MergeColdContext) + return; + + // Nothing to merge if sample threshold is zero + if (ColdCountThreshold == 0) + return; + + // Filter the cold profiles from ProfileMap and move them into a tmp + // container + std::vector> ColdProfiles; + for (const auto &I : ProfileMap) { + const FunctionSamples &FunctionProfile = I.second; + if (FunctionProfile.getTotalSamples() >= ColdCountThreshold) + continue; + ColdProfiles.emplace_back(I.getKey(), &I.second); + } + + // Remove the cold profile from ProfileMap and merge them into BaseProileMap + StringMap BaseProfileMap; + for (const auto &I : ColdProfiles) { + if (MergeColdContext) { + auto Ret = BaseProfileMap.try_emplace( + I.second->getContext().getNameWithoutContext(), FunctionSamples()); + FunctionSamples &BaseProfile = Ret.first->second; + BaseProfile.merge(*I.second); + } + ProfileMap.erase(I.first); + } + + // Merge the base profiles into ProfileMap; + for (const auto &I : BaseProfileMap) { + // Filter the cold base profile + if (TrimColdContext && I.second.getTotalSamples() < ColdCountThreshold && + ProfileMap.find(I.getKey()) == ProfileMap.end()) + continue; + // Merge the profile if the original profile exists, otherwise just insert + // as a new profile + auto Ret = ProfileMap.try_emplace(I.getKey(), FunctionSamples()); + if (Ret.second) { + SampleContext FContext(Ret.first->first(), RawContext); + FunctionSamples &FProfile = Ret.first->second; + FProfile.setContext(FContext); + FProfile.setName(FContext.getNameWithoutContext()); + } + FunctionSamples &OrigProfile = Ret.first->second; + OrigProfile.merge(I.second); + } +} + +void SampleContextTrimmer::canonicalizeContextProfiles() { + StringSet<> ProfilesToBeRemoved; + // Note that StringMap order is guaranteed to be top-down order, + // this makes sure we make room for promoted/merged context in the + // map, before we move profiles in the map. + for (auto &I : ProfileMap) { + FunctionSamples &FProfile = I.second; + StringRef ContextStr = FProfile.getNameWithContext(); + if (I.first() == ContextStr) + continue; + + // Use the context string from FunctionSamples to update the keys of + // ProfileMap. They can get out of sync after context profile promotion + // through pre-inliner. + auto Ret = ProfileMap.try_emplace(ContextStr, FProfile); + assert(Ret.second && "Conext conflict during canonicalization"); + FProfile = Ret.first->second; + + // Track the context profile to remove + ProfilesToBeRemoved.erase(ContextStr); + ProfilesToBeRemoved.insert(I.first()); + } + + for (auto &I : ProfilesToBeRemoved) { + ProfileMap.erase(I.first()); + } +} + std::error_code ProfileSymbolList::write(raw_ostream &OS) { // Sort the symbols before output. If doing compression. // It will make the compression much more effective. diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -46,9 +46,11 @@ // Sort the ProfileMap by total samples. typedef std::pair NameFunctionSamples; std::vector V; - for (const auto &I : ProfileMap) - V.push_back(std::make_pair(I.getKey(), &I.second)); - + for (const auto &I : ProfileMap) { + assert(I.getKey() == I.second.getNameWithContext() && + "Inconsistent profile map"); + V.push_back(std::make_pair(I.second.getNameWithContext(), &I.second)); + } llvm::stable_sort( V, [](const NameFunctionSamples &A, const NameFunctionSamples &B) { if (A.second->getTotalSamples() == B.second->getTotalSamples()) @@ -147,7 +149,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) { uint64_t Offset = OutputStream->tell(); - StringRef Name = S.getNameWithContext(true); + StringRef Name = S.getNameWithContext(); FuncOffsetTable[Name] = Offset - SecLBRProfileStart; encodeULEB128(S.getHeadSamples(), *OutputStream); return writeBody(S); @@ -160,9 +162,11 @@ encodeULEB128(FuncOffsetTable.size(), OS); // Write out FuncOffsetTable. - for (auto entry : FuncOffsetTable) { - writeNameIdx(entry.first); - encodeULEB128(entry.second, OS); + for (auto Entry : FuncOffsetTable) { + if (std::error_code EC = + writeNameIdx(Entry.first, FunctionSamples::ProfileIsCS)) + return EC; + encodeULEB128(Entry.second, OS); } FuncOffsetTable.clear(); return sampleprof_error::success; @@ -174,7 +178,9 @@ return sampleprof_error::success; auto &OS = *OutputStream; for (const auto &Entry : Profiles) { - writeNameIdx(Entry.first()); + if (std::error_code EC = writeNameIdx(Entry.second.getNameWithContext(), + FunctionSamples::ProfileIsCS)) + return EC; if (FunctionSamples::ProfileIsProbeBased) encodeULEB128(Entry.second.getFunctionHash(), OS); if (FunctionSamples::ProfileIsCS) @@ -204,7 +210,9 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection( const StringMap &ProfileMap) { for (const auto &I : ProfileMap) { - addName(I.first()); + assert(I.first() == I.second.getNameWithContext() && + "Inconsistent profile map"); + addName(I.second.getNameWithContext(), FunctionSamples::ProfileIsCS); addNames(I.second); } @@ -378,7 +386,11 @@ /// it needs to be parsed by the SampleProfileReaderText class. std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) { auto &OS = *OutputStream; - OS << S.getNameWithContext(true) << ":" << S.getTotalSamples(); + if (FunctionSamples::ProfileIsCS) + OS << "[" << S.getNameWithContext() << "]:" << S.getTotalSamples(); + else + OS << S.getName() << ":" << S.getTotalSamples(); + if (Indent == 0) OS << ":" << S.getHeadSamples(); OS << "\n"; @@ -431,15 +443,26 @@ return sampleprof_error::success; } -std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName) { - const auto &ret = NameTable.find(FName); - if (ret == NameTable.end()) +std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName, + bool IsContextName) { + std::string BracketedName; + if (IsContextName) { + BracketedName = "[" + FName.str() + "]"; + FName = StringRef(BracketedName); + } + + const auto &Ret = NameTable.find(FName); + if (Ret == NameTable.end()) return sampleprof_error::truncated_name_table; - encodeULEB128(ret->second, *OutputStream); + encodeULEB128(Ret->second, *OutputStream); return sampleprof_error::success; } -void SampleProfileWriterBinary::addName(StringRef FName) { +void SampleProfileWriterBinary::addName(StringRef FName, bool IsContextName) { + if (IsContextName) { + auto It = BracketedContextStr.insert("[" + FName.str() + "]"); + FName = StringRef(*It.first); + } NameTable.insert(std::make_pair(FName, 0)); } @@ -500,9 +523,11 @@ encodeULEB128(FuncOffsetTable.size(), OS); // Write out FuncOffsetTable. - for (auto entry : FuncOffsetTable) { - writeNameIdx(entry.first); - encodeULEB128(entry.second, OS); + for (auto Entry : FuncOffsetTable) { + if (std::error_code EC = + writeNameIdx(Entry.first, FunctionSamples::ProfileIsCS)) + return EC; + encodeULEB128(Entry.second, OS); } return sampleprof_error::success; } @@ -539,7 +564,9 @@ // Generate the name table for all the functions referenced in the profile. for (const auto &I : ProfileMap) { - addName(I.first()); + assert(I.first() == I.second.getNameWithContext() && + "Inconsistent profile map"); + addName(I.first(), FunctionSamples::ProfileIsCS); addNames(I.second); } @@ -654,7 +681,8 @@ std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) { auto &OS = *OutputStream; - if (std::error_code EC = writeNameIdx(S.getNameWithContext(true))) + if (std::error_code EC = + writeNameIdx(S.getNameWithContext(), FunctionSamples::ProfileIsCS)) return EC; encodeULEB128(S.getTotalSamples(), OS); diff --git a/llvm/test/tools/llvm-profgen/recursion-compression-noprobe.test b/llvm/test/tools/llvm-profgen/recursion-compression-noprobe.test --- a/llvm/test/tools/llvm-profgen/recursion-compression-noprobe.test +++ b/llvm/test/tools/llvm-profgen/recursion-compression-noprobe.test @@ -10,16 +10,16 @@ ; CHECK-UNCOMPRESS:[main:1 @ foo:3 @ fa]:24:0 ; CHECK-UNCOMPRESS: 1: 1 ; CHECK-UNCOMPRESS: 2: 13 fb:11 -; CHECK-UNCOMPRESS:[main:1 @ foo]:7:0 -; CHECK-UNCOMPRESS: 2: 1 -; CHECK-UNCOMPRESS: 3: 2 fa:1 ; CHECK-UNCOMPRESS:[main:1 @ foo:3 @ fa:2 @ fb:2 @ fa]:7:0 ; CHECK-UNCOMPRESS: 1: 1 ; CHECK-UNCOMPRESS: 2: 2 fb:1 -; CHECK-UNCOMPRESS:[main:1 @ foo:3 @ fa:2 @ fb:2 @ fa:2 @ fb]:2:0 -; CHECK-UNCOMPRESS: 2: 1 fa:1 +; CHECK-UNCOMPRESS:[main:1 @ foo]:7:0 +; CHECK-UNCOMPRESS: 2: 1 +; CHECK-UNCOMPRESS: 3: 2 fa:1 ; CHECK-UNCOMPRESS:[main:1 @ foo:3 @ fa:2 @ fb:2 @ fa:2 @ fb:2 @ fa]:2:0 ; CHECK-UNCOMPRESS: 4: 1 +; CHECK-UNCOMPRESS:[main:1 @ foo:3 @ fa:2 @ fb:2 @ fa:2 @ fb]:2:0 +; CHECK-UNCOMPRESS: 2: 1 fa:1 ; CHECK: [main:1 @ foo:3 @ fa:2 @ fb]:48:0 diff --git a/llvm/test/tools/llvm-profgen/recursion-compression-pseudoprobe.test b/llvm/test/tools/llvm-profgen/recursion-compression-pseudoprobe.test --- a/llvm/test/tools/llvm-profgen/recursion-compression-pseudoprobe.test +++ b/llvm/test/tools/llvm-profgen/recursion-compression-pseudoprobe.test @@ -4,11 +4,11 @@ ; RUN: llvm-profgen --perfscript=%S/Inputs/recursion-compression-pseudoprobe.perfscript --binary=%S/Inputs/recursion-compression-pseudoprobe.perfbin --output=%t --show-unwinder-output --csprof-cold-thres=0 | FileCheck %s --check-prefix=CHECK-UNWINDER ; RUN: FileCheck %s --input-file %t -; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa]:4:1 +; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb:6 @ fa]:4:1 ; CHECK-UNCOMPRESS: 1: 1 ; CHECK-UNCOMPRESS: 3: 1 -; CHECK-UNCOMPRESS: 5: 1 -; CHECK-UNCOMPRESS: 8: 1 fa:1 +; CHECK-UNCOMPRESS: 4: 1 +; CHECK-UNCOMPRESS: 7: 1 fb:1 ; CHECK-UNCOMPRESS: !CFGChecksum: 120515930909 ; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa:8 @ fa]:4:1 ; CHECK-UNCOMPRESS: 1: 1 @@ -16,28 +16,13 @@ ; CHECK-UNCOMPRESS: 4: 1 ; CHECK-UNCOMPRESS: 7: 1 fb:1 ; CHECK-UNCOMPRESS: !CFGChecksum: 120515930909 -; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb:6 @ fa]:4:1 +; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa]:4:1 ; CHECK-UNCOMPRESS: 1: 1 ; CHECK-UNCOMPRESS: 3: 1 -; CHECK-UNCOMPRESS: 4: 1 -; CHECK-UNCOMPRESS: 7: 1 fb:1 +; CHECK-UNCOMPRESS: 5: 1 +; CHECK-UNCOMPRESS: 8: 1 fa:1 ; CHECK-UNCOMPRESS: !CFGChecksum: 120515930909 -; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1 -; CHECK-UNCOMPRESS: 1: 1 -; CHECK-UNCOMPRESS: 2: 1 -; CHECK-UNCOMPRESS: 5: 1 fb:1 -; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756 -; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1 -; CHECK-UNCOMPRESS: 1: 1 -; CHECK-UNCOMPRESS: 2: 1 -; CHECK-UNCOMPRESS: 5: 1 fb:1 -; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756 -; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1 -; CHECK-UNCOMPRESS: 1: 1 -; CHECK-UNCOMPRESS: 2: 1 -; CHECK-UNCOMPRESS: 5: 1 fb:1 -; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756 -; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1 +; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb:6 @ fa:7 @ fb]:3:1 ; CHECK-UNCOMPRESS: 1: 1 ; CHECK-UNCOMPRESS: 3: 1 ; CHECK-UNCOMPRESS: 6: 1 fa:1 @@ -47,11 +32,26 @@ ; CHECK-UNCOMPRESS: 3: 1 ; CHECK-UNCOMPRESS: 6: 1 fa:1 ; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756 -; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb:6 @ fa:7 @ fb]:3:1 +; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1 ; CHECK-UNCOMPRESS: 1: 1 ; CHECK-UNCOMPRESS: 3: 1 ; CHECK-UNCOMPRESS: 6: 1 fa:1 ; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756 +; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1 +; CHECK-UNCOMPRESS: 1: 1 +; CHECK-UNCOMPRESS: 2: 1 +; CHECK-UNCOMPRESS: 5: 1 fb:1 +; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756 +; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1 +; CHECK-UNCOMPRESS: 1: 1 +; CHECK-UNCOMPRESS: 2: 1 +; CHECK-UNCOMPRESS: 5: 1 fb:1 +; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756 +; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb]:3:1 +; CHECK-UNCOMPRESS: 1: 1 +; CHECK-UNCOMPRESS: 2: 1 +; CHECK-UNCOMPRESS: 5: 1 fb:1 +; CHECK-UNCOMPRESS: !CFGChecksum: 72617220756 ; CHECK-UNCOMPRESS: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb:6 @ fa:7 @ fb:6 @ fa]:2:1 ; CHECK-UNCOMPRESS: 1: 1 ; CHECK-UNCOMPRESS: 3: 1 @@ -74,24 +74,24 @@ ; CHECK: 4: 1 ; CHECK: 7: 1 fb:1 ; CHECK: !CFGChecksum: 120515930909 -; CHECK: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:6 @ fa]:4:1 -; CHECK: 1: 1 -; CHECK: 3: 1 -; CHECK: 5: 1 -; CHECK: 8: 1 fa:1 -; CHECK: !CFGChecksum: 120515930909 ; CHECK: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:6 @ fa:8 @ fa]:4:1 ; CHECK: 1: 1 ; CHECK: 3: 1 ; CHECK: 4: 1 ; CHECK: 7: 1 fb:1 ; CHECK: !CFGChecksum: 120515930909 -; CHECK: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb]:3:1 +; CHECK: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:6 @ fa]:4:1 +; CHECK: 1: 1 +; CHECK: 3: 1 +; CHECK: 5: 1 +; CHECK: 8: 1 fa:1 +; CHECK: !CFGChecksum: 120515930909 +; CHECK: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb:6 @ fa:7 @ fb]:3:1 ; CHECK: 1: 1 ; CHECK: 3: 1 ; CHECK: 6: 1 fa:1 ; CHECK: !CFGChecksum: 72617220756 -; CHECK: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb:6 @ fa:7 @ fb]:3:1 +; CHECK: [main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb:6 @ fa:8 @ fa:7 @ fb]:3:1 ; CHECK: 1: 1 ; CHECK: 3: 1 ; CHECK: 6: 1 fa:1 @@ -99,6 +99,7 @@ + ; CHECK-UNWINDER: Binary(recursion-compression-pseudoprobe.perfbin)'s Range Counter: ; CHECK-UNWINDER: main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 ; CHECK-UNWINDER: (7a0, 7a7): 1 diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -710,7 +710,7 @@ Remapper ? remapSamples(I->second, *Remapper, Result) : FunctionSamples(); FunctionSamples &Samples = Remapper ? Remapped : I->second; - StringRef FName = Samples.getNameWithContext(true); + StringRef FName = Samples.getNameWithContext(); MergeResult(Result, ProfileMap[FName].merge(Samples, Input.Weight)); if (Result != sampleprof_error::success) { std::error_code EC = make_error_code(Result); @@ -734,7 +734,8 @@ auto Buffer = getInputFileBuf(ProfileSymbolListFile); handleExtBinaryWriter(*Writer, OutputFormat, Buffer.get(), WriterList, CompressAllSections, UseMD5, GenPartialProfile); - Writer->write(ProfileMap); + if (std::error_code EC = Writer->write(ProfileMap)) + exitWithErrorCode(std::move(EC)); } static WeightedFile parseWeightedFile(const StringRef &WeightedFilename) { diff --git a/llvm/tools/llvm-profgen/CSPreInliner.cpp b/llvm/tools/llvm-profgen/CSPreInliner.cpp --- a/llvm/tools/llvm-profgen/CSPreInliner.cpp +++ b/llvm/tools/llvm-profgen/CSPreInliner.cpp @@ -225,5 +225,8 @@ ProfileMap.erase(ContextName); } + // Make sure ProfileMap's key is consistent with FunctionSamples' name. + SampleContextTrimmer(ProfileMap).canonicalizeContextProfiles(); + LLVM_DEBUG(printProfileNames(ProfileMap, false)); } diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h --- a/llvm/tools/llvm-profgen/ProfileGenerator.h +++ b/llvm/tools/llvm-profgen/ProfileGenerator.h @@ -15,6 +15,7 @@ #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/ProfileData/SampleProfWriter.h" #include +#include using namespace llvm; using namespace sampleprof; @@ -182,9 +183,6 @@ // Post processing for profiles before writing out, such as mermining // and trimming cold profiles, running preinliner on profiles. void postProcessProfiles(); - // Merge cold context profile whose total sample is below threshold - // into base profile. - void mergeAndTrimColdProfile(StringMap &ProfileMap); void computeSummaryAndThreshold(); void write(std::unique_ptr Writer, StringMap &ProfileMap) override; @@ -192,6 +190,9 @@ // Profile summary to answer isHotCount and isColdCount queries. std::unique_ptr PSI; + // String table owning context strings created from profile generation. + std::unordered_set ContextStrings; + private: // Helper function for updating body sample for a leaf location in // FunctionProfile diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp --- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp +++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp @@ -86,7 +86,8 @@ void ProfileGenerator::write(std::unique_ptr Writer, StringMap &ProfileMap) { - Writer->write(ProfileMap); + if (std::error_code EC = Writer->write(ProfileMap)) + exitWithError(std::move(EC)); } void ProfileGenerator::write() { @@ -198,7 +199,10 @@ bool WasLeafInlined) { auto Ret = ProfileMap.try_emplace(ContextStr, FunctionSamples()); if (Ret.second) { - SampleContext FContext(Ret.first->first(), RawContext); + // Make a copy of the underlying context string in string table + // before StringRef wrapper is used for context. + auto It = ContextStrings.insert(ContextStr.str()); + SampleContext FContext(*It.first, RawContext); if (WasLeafInlined) FContext.setAttribute(ContextWasInlined); FunctionSamples &FProfile = Ret.first->second; @@ -401,84 +405,27 @@ PSI->getColdCountThreshold()) .run(); - mergeAndTrimColdProfile(ProfileMap); + // Trim and merge cold context profile using cold threshold above; + SampleContextTrimmer(ProfileMap) + .trimAndMergeColdContextProfiles( + CSProfColdThreshold, CSProfTrimColdContext, CSProfMergeColdContext); } void CSProfileGenerator::computeSummaryAndThreshold() { SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); auto Summary = Builder.computeSummaryForProfiles(ProfileMap); PSI.reset(new ProfileSummaryInfo(std::move(Summary))); -} - -void CSProfileGenerator::mergeAndTrimColdProfile( - StringMap &ProfileMap) { - if (!CSProfMergeColdContext && !CSProfTrimColdContext) - return; - - // Use threshold calculated from profile summary unless specified - uint64_t ColdThreshold = PSI->getColdCountThreshold(); - if (CSProfColdThreshold.getNumOccurrences()) { - ColdThreshold = CSProfColdThreshold; - } - // Nothing to merge if sample threshold is zero - if (ColdThreshold == 0) - return; - - // Filter the cold profiles from ProfileMap and move them into a tmp - // container - std::vector> ColdProfiles; - for (const auto &I : ProfileMap) { - const FunctionSamples &FunctionProfile = I.second; - if (FunctionProfile.getTotalSamples() >= ColdThreshold) - continue; - ColdProfiles.emplace_back(I.getKey(), &I.second); - } - - // Remove the code profile from ProfileMap and merge them into BaseProileMap - StringMap BaseProfileMap; - for (const auto &I : ColdProfiles) { - if (CSProfMergeColdContext) { - auto Ret = BaseProfileMap.try_emplace( - I.second->getContext().getNameWithoutContext(), FunctionSamples()); - FunctionSamples &BaseProfile = Ret.first->second; - BaseProfile.merge(*I.second); - } - ProfileMap.erase(I.first); - } - - // Merge the base profiles into ProfileMap; - for (const auto &I : BaseProfileMap) { - // Filter the cold base profile - if (CSProfTrimColdContext && - I.second.getTotalSamples() < CSProfColdThreshold && - ProfileMap.find(I.getKey()) == ProfileMap.end()) - continue; - // Merge the profile if the original profile exists, otherwise just insert - // as a new profile - FunctionSamples &OrigProfile = getFunctionProfileForContext(I.getKey()); - OrigProfile.merge(I.second); + // Use threshold calculated from profile summary unless specified. + if (!CSProfColdThreshold.getNumOccurrences()) { + CSProfColdThreshold = PSI->getColdCountThreshold(); } } void CSProfileGenerator::write(std::unique_ptr Writer, StringMap &ProfileMap) { - // Add bracket for context key to support different profile binary format - StringMap CxtWithBracketPMap; - for (const auto &Item : ProfileMap) { - // After CSPreInliner the key of ProfileMap is no longer accurate for - // context, use the context attached to function samples instead. - std::string ContextWithBracket = - "[" + Item.second.getNameWithContext().str() + "]"; - auto Ret = CxtWithBracketPMap.try_emplace(ContextWithBracket, Item.second); - assert(Ret.second && "Must be a unique context"); - SampleContext FContext(Ret.first->first(), RawContext); - FunctionSamples &FProfile = Ret.first->second; - FContext.setAllAttributes(FProfile.getContext().getAllAttributes()); - FProfile.setName(FContext.getNameWithoutContext()); - FProfile.setContext(FContext); - } - Writer->write(CxtWithBracketPMap); + if (std::error_code EC = Writer->write(ProfileMap)) + exitWithError(std::move(EC)); } // Helper function to extract context prefix string stack @@ -590,7 +537,7 @@ // context id to infer caller's context id to ensure they share the // same context prefix. StringRef CalleeContextId = - FunctionProfile.getContext().getNameWithContext(true); + FunctionProfile.getContext().getNameWithContext(); StringRef CallerContextId; FrameLocation &&CallerLeafFrameLoc = getCallerContext(CalleeContextId, CallerContextId);