diff --git a/llvm/include/llvm/ProfileData/ProfileCommon.h b/llvm/include/llvm/ProfileData/ProfileCommon.h --- a/llvm/include/llvm/ProfileData/ProfileCommon.h +++ b/llvm/include/llvm/ProfileData/ProfileCommon.h @@ -92,8 +92,8 @@ void addRecord(const sampleprof::FunctionSamples &FS, bool isCallsiteSample = false); - std::unique_ptr computeSummaryForProfiles( - const StringMap &Profiles); + std::unique_ptr + computeSummaryForProfiles(const sampleprof::SampleProfileMap &Profiles); std::unique_ptr getSummary(); }; diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -125,6 +125,7 @@ SecProfileSymbolList = 3, SecFuncOffsetTable = 4, SecFuncMetadata = 5, + SecCSNameTable = 6, // marker for the first type of profile. SecFuncProfileFirst = 32, SecLBRProfile = SecFuncProfileFirst @@ -144,6 +145,8 @@ return "FuncOffsetTableSection"; case SecFuncMetadata: return "FunctionMetadata"; + case SecCSNameTable: + return "CSNameTableSection"; case SecLBRProfile: return "LBRProfileSection"; } diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -242,6 +242,7 @@ #include "llvm/Support/SymbolRemappingReader.h" #include #include +#include #include #include #include @@ -381,8 +382,8 @@ /// The implementaion to read sample profiles from the associated file. virtual std::error_code readImpl() = 0; - /// Print the profile for \p FName on stream \p OS. - void dumpFunctionProfile(StringRef FName, raw_ostream &OS = dbgs()); + /// Print the profile for \p FContext on stream \p OS. + void dumpFunctionProfile(SampleContext FContext, raw_ostream &OS = dbgs()); /// Collect functions with definitions in Module M. For reader which /// support loading function profiles on demand, return true when the @@ -437,7 +438,7 @@ } /// Return all the profiles. - StringMap &getProfiles() { return Profiles; } + SampleProfileMap &getProfiles() { return Profiles; } /// Report a parse error message. void reportError(int64_t LineNumber, const Twine &Msg) const { @@ -503,7 +504,8 @@ /// The profile of every function executed at runtime is collected /// in the structure FunctionSamples. This maps function objects /// to their corresponding profiles. - StringMap Profiles; + + SampleProfileMap Profiles; /// LLVM context used to emit diagnostics. LLVMContext &Ctx; @@ -567,6 +569,20 @@ /// Return true if \p Buffer is in the format supported by this class. static bool hasFormat(const MemoryBuffer &Buffer); + +private: + /// Decode context string for a frame to get function name and location. + /// `ContextStr` is in the form of `FuncName:StartLine.Discriminator`. + void decodeContextString(StringRef ContextStr, StringRef &FName, + LineLocation &LineLoc); + + /// Create a context vector from a give context string and save it in + /// CSNameTable. + SampleContextFrames createContext(StringRef ContextStr); + + /// CSNameTable is used to save full context vectors. This serves as an + /// underlying immutable buffer for all clients. + std::list CSNameTable; }; class SampleProfileReaderBinary : public SampleProfileReader { @@ -613,7 +629,7 @@ bool at_eof() const { return Data >= End; } /// Read the next function profile instance. - std::error_code readFuncProfile(const uint8_t *Start); + std::error_code readFuncProfile(const uint8_t *Start, bool IsContextName); /// Read the contents of the given profile instance. std::error_code readProfile(FunctionSamples &FProfile); @@ -638,6 +654,7 @@ /// Read a string indirectly via the name table. virtual ErrorOr readStringFromTable(); + virtual ErrorOr readNameFromTable(bool /* IsContextName */); private: std::error_code readSummaryEntry(std::vector &Entries); @@ -695,6 +712,7 @@ std::error_code readFuncProfiles(); std::error_code readMD5NameTable(); std::error_code readNameTableSec(bool IsMD5); + std::error_code readCSNameTableSec(); std::error_code readProfileSymbolList(); virtual std::error_code readHeader() override; @@ -704,12 +722,15 @@ // placeholder for subclasses to dispatch their own section readers. virtual std::error_code readCustomSection(const SecHdrTableEntry &Entry) = 0; virtual ErrorOr readStringFromTable() override; + virtual ErrorOr + readNameFromTable(bool IsContextName = false) override; + ErrorOr readContextFromTable(); std::unique_ptr ProfSymList; - /// The table mapping from function name to the offset of its FunctionSample - /// towards file start. - DenseMap FuncOffsetTable; + /// The table mapping from function context to the offset of its + /// FunctionSample towards file start. + DenseMap FuncOffsetTable; /// The set containing the functions to use when compiling a module. DenseSet FuncsToUse; @@ -728,6 +749,10 @@ /// the lifetime of MD5StringBuf is not shorter than that of NameTable. std::unique_ptr> MD5StringBuf; + /// CSNameTable is used to save full context vectors. This serves as an + /// underlying immutable buffer for all clients. + std::unique_ptr> CSNameTable; + /// If SkipFlatProf is true, skip the sections with /// SecFlagFlat flag. bool SkipFlatProf = false; diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h --- a/llvm/include/llvm/ProfileData/SampleProfWriter.h +++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h @@ -52,7 +52,7 @@ /// Write all the sample profiles in the given map of samples. /// /// \returns status code of the file update operation. - virtual std::error_code write(const StringMap &ProfileMap); + virtual std::error_code write(const SampleProfileMap &ProfileMap); raw_ostream &getOutputStream() { return *OutputStream; } @@ -78,12 +78,10 @@ : OutputStream(std::move(OS)) {} /// Write a file header for the profile file. - virtual std::error_code - writeHeader(const StringMap &ProfileMap) = 0; + virtual std::error_code writeHeader(const SampleProfileMap &ProfileMap) = 0; // Write function profiles to the profile file. - virtual std::error_code - writeFuncProfiles(const StringMap &ProfileMap); + virtual std::error_code writeFuncProfiles(const SampleProfileMap &ProfileMap); /// Output stream where to emit the profile to. std::unique_ptr OutputStream; @@ -92,7 +90,7 @@ std::unique_ptr Summary; /// Compute summary for this profile. - void computeSummary(const StringMap &ProfileMap); + void computeSummary(const SampleProfileMap &ProfileMap); /// Profile format. SampleProfileFormat Format = SPF_None; @@ -107,8 +105,7 @@ SampleProfileWriterText(std::unique_ptr &OS) : SampleProfileWriter(OS), Indent(0) {} - std::error_code - writeHeader(const StringMap &ProfileMap) override { + std::error_code writeHeader(const SampleProfileMap &ProfileMap) override { return sampleprof_error::success; } @@ -132,19 +129,22 @@ virtual std::error_code writeSample(const FunctionSamples &S) override; protected: + virtual MapVector &getNameTable() { return NameTable; } virtual std::error_code writeMagicIdent(SampleProfileFormat Format); virtual std::error_code writeNameTable(); virtual std::error_code - writeHeader(const StringMap &ProfileMap) override; + writeHeader(const SampleProfileMap &ProfileMap) override; std::error_code writeSummary(); - std::error_code writeNameIdx(StringRef FName, bool IsContextName = false); + virtual std::error_code writeNameIdx(const SampleContext &Context); + std::error_code writeNameIdx(StringRef FName); std::error_code writeBody(const FunctionSamples &S); - inline void stablizeNameTable(std::set &V); + inline void stablizeNameTable(MapVector &NameTable, + std::set &V); MapVector NameTable; - std::unordered_set BracketedContextStr; - void addName(StringRef FName, bool IsContextName = false); + void addName(StringRef FName); + virtual void addName(const SampleContext &Context); void addNames(const FunctionSamples &S); private: @@ -168,6 +168,7 @@ // DefaultLayout SmallVector({{SecProfSummary, 0, 0, 0, 0}, {SecNameTable, 0, 0, 0, 0}, + {SecCSNameTable, 0, 0, 0, 0}, {SecFuncOffsetTable, 0, 0, 0, 0}, {SecLBRProfile, 0, 0, 0, 0}, {SecProfileSymbolList, 0, 0, 0, 0}, @@ -190,8 +191,7 @@ class SampleProfileWriterExtBinaryBase : public SampleProfileWriterBinary { using SampleProfileWriterBinary::SampleProfileWriterBinary; public: - virtual std::error_code - write(const StringMap &ProfileMap) override; + virtual std::error_code write(const SampleProfileMap &ProfileMap) override; virtual void setToCompressAllSections() override; void setToCompressSection(SecType Type); @@ -246,29 +246,31 @@ addSecFlag(SectionHdrLayout[SectionIdx], Flag); } + virtual void addName(const SampleContext &Context) override; + // placeholder for subclasses to dispatch their own section writers. virtual std::error_code writeCustomSection(SecType Type) = 0; // Verify the SecLayout is supported by the format. virtual void verifySecLayout(SectionLayout SL) = 0; // specify the order to write sections. - virtual std::error_code - writeSections(const StringMap &ProfileMap) = 0; + virtual std::error_code writeSections(const SampleProfileMap &ProfileMap) = 0; // Dispatch section writer for each section. \p LayoutIdx is the sequence // number indicating where the section is located in SectionHdrLayout. - virtual std::error_code - writeOneSection(SecType Type, uint32_t LayoutIdx, - const StringMap &ProfileMap); + virtual std::error_code writeOneSection(SecType Type, uint32_t LayoutIdx, + const SampleProfileMap &ProfileMap); // Helper function to write name table. virtual std::error_code writeNameTable() override; + virtual std::error_code writeNameIdx(const SampleContext &Context) override; + std::error_code writeCSNameIdx(const SampleContext &Context); + std::error_code writeCSNameTableSection(); - std::error_code writeFuncMetadata(const StringMap &Profiles); + std::error_code writeFuncMetadata(const SampleProfileMap &Profiles); // Functions to write various kinds of sections. - std::error_code - writeNameTableSection(const StringMap &ProfileMap); + std::error_code writeNameTableSection(const SampleProfileMap &ProfileMap); std::error_code writeFuncOffsetTable(); std::error_code writeProfileSymbolListSection(); @@ -289,7 +291,7 @@ void allocSecHdrTable(); std::error_code writeSecHdrTable(); virtual std::error_code - writeHeader(const StringMap &ProfileMap) override; + writeHeader(const SampleProfileMap &ProfileMap) override; std::error_code compressAndOutput(); // We will swap the raw_ostream held by LocalBufStream and that @@ -312,12 +314,16 @@ // be read. std::vector SecHdrTable; - // FuncOffsetTable maps function name to its profile offset in SecLBRProfile - // section. It is used to load function profile on demand. - MapVector FuncOffsetTable; + // FuncOffsetTable maps function context to its profile offset in + // SecLBRProfile section. It is used to load function profile on demand. + MapVector FuncOffsetTable; // Whether to use MD5 to represent string. bool UseMD5 = false; + /// CSNameTable maps function context to its offset in SecCSNameTable section. + /// The offset will be used everywhere where the context is referenced. + MapVector CSNameTable; + ProfileSymbolList *ProfSymList = nullptr; }; @@ -327,13 +333,11 @@ : SampleProfileWriterExtBinaryBase(OS) {} private: - std::error_code - writeDefaultLayout(const StringMap &ProfileMap); - std::error_code - writeCtxSplitLayout(const StringMap &ProfileMap); + std::error_code writeDefaultLayout(const SampleProfileMap &ProfileMap); + std::error_code writeCtxSplitLayout(const SampleProfileMap &ProfileMap); virtual std::error_code - writeSections(const StringMap &ProfileMap) override; + writeSections(const SampleProfileMap &ProfileMap) override; virtual std::error_code writeCustomSection(SecType Type) override { return sampleprof_error::success; @@ -380,8 +384,7 @@ public: virtual std::error_code writeSample(const FunctionSamples &S) override; - virtual std::error_code - write(const StringMap &ProfileMap) override; + virtual std::error_code write(const SampleProfileMap &ProfileMap) override; protected: /// The table mapping from function name to the offset of its FunctionSample @@ -392,7 +395,7 @@ uint64_t TableOffset; virtual std::error_code writeNameTable() override; virtual std::error_code - writeHeader(const StringMap &ProfileMap) override; + writeHeader(const SampleProfileMap &ProfileMap) override; std::error_code writeFuncOffsetTable(); }; diff --git a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h --- a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h +++ b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h @@ -50,7 +50,11 @@ ContextTrieNode &moveToChildContext(const LineLocation &CallSite, ContextTrieNode &&NodeToMove, +<<<<<<< HEAD uint32_t SzContextToRemove, +======= + uint32_t ContextFramesToRemove, +>>>>>>> 764e22c19250 ([CSSPGO] Split context string to deduplicate function name used in the context.) bool DeleteNode = true); void removeChildContext(const LineLocation &CallSite, StringRef CalleeName); std::map &getAllChildContext(); @@ -90,7 +94,11 @@ // calling context and the context is identified by path from root to the node. class SampleContextTracker { public: +<<<<<<< HEAD struct ProfileSorter { +======= + struct ProfileComparer { +>>>>>>> 764e22c19250 ([CSSPGO] Split context string to deduplicate function name used in the context.) bool operator()(FunctionSamples *A, FunctionSamples *B) const { // Sort function profiles by the number of total samples and their // contexts. @@ -102,7 +110,11 @@ // Keep profiles of a function sorted so that they will be processed/promoted // deterministically. +<<<<<<< HEAD using ContextSamplesTy = std::set; +======= + using ContextSamplesTy = std::set; +>>>>>>> 764e22c19250 ([CSSPGO] Split context string to deduplicate function name used in the context.) SampleContextTracker(SampleProfileMap &Profiles); // Query context profile for a specific callee with given name at a given @@ -148,10 +160,18 @@ ContextTrieNode &addTopLevelContextNode(StringRef FName); ContextTrieNode &promoteMergeContextSamplesTree(ContextTrieNode &NodeToPromo); void mergeContextNode(ContextTrieNode &FromNode, ContextTrieNode &ToNode, +<<<<<<< HEAD uint32_t SzContextToRemove); ContextTrieNode &promoteMergeContextSamplesTree(ContextTrieNode &FromNode, ContextTrieNode &ToNodeParent, uint32_t SzContextToRemove); +======= + uint32_t ContextFramesToRemove); + ContextTrieNode & + promoteMergeContextSamplesTree(ContextTrieNode &FromNode, + ContextTrieNode &ToNodeParent, + uint32_t ContextFramesToRemove); +>>>>>>> 764e22c19250 ([CSSPGO] Split context string to deduplicate function name used in the context.) // Map from function name to context profiles (excluding base profile) StringMap FuncToCtxtProfiles; diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -199,18 +199,16 @@ } void sampleprof::sortFuncProfiles( - const StringMap &ProfileMap, + const SampleProfileMap &ProfileMap, std::vector &SortedProfiles) { for (const auto &I : ProfileMap) { - assert(I.getKey() == I.second.getNameWithContext() && - "Inconsistent profile map"); - SortedProfiles.push_back( - std::make_pair(I.second.getNameWithContext(), &I.second)); + assert(I.first == I.second.getContext() && "Inconsistent profile map"); + SortedProfiles.push_back(std::make_pair(I.second.getContext(), &I.second)); } llvm::stable_sort(SortedProfiles, [](const NameFunctionSamples &A, const NameFunctionSamples &B) { if (A.second->getTotalSamples() == B.second->getTotalSamples()) - return A.first > B.first; + return A.first < B.first; return A.second->getTotalSamples() > B.second->getTotalSamples(); }); } @@ -343,23 +341,23 @@ // Filter the cold profiles from ProfileMap and move them into a tmp // container - std::vector> ColdProfiles; + std::vector> ColdProfiles; for (const auto &I : ProfileMap) { const FunctionSamples &FunctionProfile = I.second; if (FunctionProfile.getTotalSamples() >= ColdCountThreshold) continue; - ColdProfiles.emplace_back(I.getKey(), &I.second); + ColdProfiles.emplace_back(I.first, &I.second); } // Remove the cold profile from ProfileMap and merge them into // MergedProfileMap by the last K frames of context - StringMap MergedProfileMap; + SampleProfileMap MergedProfileMap; for (const auto &I : ColdProfiles) { if (MergeColdContext) { - auto Ret = MergedProfileMap.try_emplace( - I.second->getContext().getContextWithLastKFrames( - ColdContextFrameLength), - FunctionSamples()); + auto MergedContext = I.second->getContext().getContextFrames(); + if (ColdContextFrameLength < MergedContext.size()) + MergedContext = MergedContext.take_back(ColdContextFrameLength); + auto Ret = MergedProfileMap.emplace(MergedContext, FunctionSamples()); FunctionSamples &MergedProfile = Ret.first->second; MergedProfile.merge(*I.second); } @@ -370,16 +368,15 @@ for (const auto &I : MergedProfileMap) { // Filter the cold merged profile if (TrimColdContext && I.second.getTotalSamples() < ColdCountThreshold && - ProfileMap.find(I.getKey()) == ProfileMap.end()) + ProfileMap.find(I.first) == ProfileMap.end()) continue; // Merge the profile if the original profile exists, otherwise just insert // as a new profile - auto Ret = ProfileMap.try_emplace(I.getKey(), FunctionSamples()); + auto Ret = ProfileMap.emplace(I.first, FunctionSamples()); if (Ret.second) { - SampleContext FContext(Ret.first->first(), RawContext); + SampleContext FContext(Ret.first->first, RawContext); FunctionSamples &FProfile = Ret.first->second; FProfile.setContext(FContext); - FProfile.setName(FContext.getNameWithoutContext()); } FunctionSamples &OrigProfile = Ret.first->second; OrigProfile.merge(I.second); @@ -387,12 +384,12 @@ } void SampleContextTrimmer::canonicalizeContextProfiles() { - std::vector ProfilesToBeRemoved; - StringMap ProfilesToBeAdded; + std::vector ProfilesToBeRemoved; + SampleProfileMap ProfilesToBeAdded; for (auto &I : ProfileMap) { FunctionSamples &FProfile = I.second; - StringRef ContextStr = FProfile.getNameWithContext(); - if (I.first() == ContextStr) + SampleContext &Context = FProfile.getContext(); + if (I.first == Context) continue; // Use the context string from FunctionSamples to update the keys of @@ -407,10 +404,10 @@ // with different profiles) from the map can cause a conflict if they are // not handled in a right order. This can be solved by just caching the // profiles to be added. - auto Ret = ProfilesToBeAdded.try_emplace(ContextStr, FProfile); + auto Ret = ProfilesToBeAdded.emplace(Context, FProfile); (void)Ret; assert(Ret.second && "Context conflict during canonicalization"); - ProfilesToBeRemoved.push_back(I.first()); + ProfilesToBeRemoved.push_back(I.first); } for (auto &I : ProfilesToBeRemoved) { @@ -418,7 +415,7 @@ } for (auto &I : ProfilesToBeAdded) { - ProfileMap.try_emplace(I.first(), I.second); + ProfileMap.emplace(I.first, I.second); } } diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -59,9 +59,9 @@ /// /// \param FName Name of the function to print. /// \param OS Stream to emit the output to. -void SampleProfileReader::dumpFunctionProfile(StringRef FName, +void SampleProfileReader::dumpFunctionProfile(SampleContext FContext, raw_ostream &OS) { - OS << "Function: " << FName << ": " << Profiles[FName]; + OS << "Function: " << FContext.toString() << ": " << Profiles[FContext]; } /// Dump all the function profiles found on stream \p OS. @@ -233,6 +233,50 @@ return true; } +// Decode context string for a frame to get function name and location. +// `ContextStr` is in the form of `FuncName:StartLine.Discriminator`. +void SampleProfileReaderText::decodeContextString(StringRef ContextStr, + StringRef &FName, + LineLocation &LineLoc) { + // Get function name + auto EntrySplit = ContextStr.split(':'); + FName = EntrySplit.first; + + LineLoc = {0, 0}; + if (!EntrySplit.second.empty()) { + // Get line offset, use signed int for getAsInteger so string will + // be parsed as signed. + int LineOffset = 0; + auto LocSplit = EntrySplit.second.split('.'); + LocSplit.first.getAsInteger(10, LineOffset); + LineLoc.LineOffset = LineOffset; + + // Get discriminator + if (!LocSplit.second.empty()) + LocSplit.second.getAsInteger(10, LineLoc.Discriminator); + } +} + +SampleContextFrames +SampleProfileReaderText::createContext(StringRef ContextStr) { + // Remove encapsulating '[' and ']' + ContextStr = ContextStr.substr(1, ContextStr.size() - 2); + CSNameTable.emplace_back(SampleContextFrameVector()); + SampleContextFrameVector &Context = CSNameTable.back(); + StringRef ContextRemain = ContextStr; + StringRef ChildContext; + StringRef CalleeName; + while (!ContextRemain.empty()) { + auto ContextSplit = ContextRemain.split(" @ "); + ChildContext = ContextSplit.first; + ContextRemain = ContextSplit.second; + LineLocation CallSiteLoc(0, 0); + decodeContextString(ChildContext, CalleeName, CallSiteLoc); + Context.emplace_back(CalleeName, CallSiteLoc); + } + return CSNameTable.back(); +} + /// Load samples from a text file. /// /// See the documentation at the top of the file for an explanation of @@ -276,11 +320,15 @@ } SeenMetadata = false; SampleContext FContext(FName); - if (FContext.hasContext()) + if (FName.startswith("[")) { + // Note that `[]` wrapped input indicates a full context string, + // otherwise it's treated as context-less function name only. + auto Context = createContext(FName); + FContext.setContext(Context); ++CSProfileCount; + } Profiles[FContext] = FunctionSamples(); FunctionSamples &FProfile = Profiles[FContext]; - FProfile.setName(FContext.getNameWithoutContext()); FProfile.setContext(FContext); MergeResult(Result, FProfile.addTotalSamples(NumSamples)); MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples)); @@ -445,11 +493,20 @@ } ErrorOr SampleProfileReaderBinary::readStringFromTable() { - auto Idx = readStringIndex(NameTable); + auto NTable = getNameTable(); + assert(NTable && "NTable should not be nullptr"); + auto Idx = readStringIndex(*NTable); if (std::error_code EC = Idx.getError()) return EC; + return (*NTable)[*Idx]; +} - return NameTable[*Idx]; +ErrorOr +SampleProfileReaderBinary::readNameFromTable(bool IsContextName) { + auto FName(readStringFromTable()); + if (std::error_code EC = FName.getError()) + return EC; + return SampleContext(*FName); } ErrorOr SampleProfileReaderExtBinaryBase::readStringFromTable() { @@ -571,25 +628,23 @@ return sampleprof_error::success; } -std::error_code -SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) { +std::error_code SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start, + bool IsContextName) { Data = Start; auto NumHeadSamples = readNumber(); if (std::error_code EC = NumHeadSamples.getError()) return EC; - auto FName(readStringFromTable()); - if (std::error_code EC = FName.getError()) + auto FContext(readNameFromTable(IsContextName)); + if (std::error_code EC = FContext.getError()) return EC; - SampleContext FContext(*FName); - Profiles[FContext] = FunctionSamples(); - FunctionSamples &FProfile = Profiles[FContext]; - FProfile.setName(FContext.getNameWithoutContext()); - FProfile.setContext(FContext); + Profiles[*FContext] = FunctionSamples(); + FunctionSamples &FProfile = Profiles[*FContext]; + FProfile.setContext(*FContext); FProfile.addHeadSamples(*NumHeadSamples); - if (FContext.hasContext()) + if (FContext->hasContext()) CSProfileCount++; if (std::error_code EC = readProfile(FProfile)) @@ -600,13 +655,38 @@ std::error_code SampleProfileReaderBinary::readImpl() { ProfileIsFS = ProfileIsFSDisciminator; while (!at_eof()) { - if (std::error_code EC = readFuncProfile(Data)) + if (std::error_code EC = readFuncProfile(Data, false)) return EC; } return sampleprof_error::success; } +ErrorOr +SampleProfileReaderExtBinaryBase::readContextFromTable() { + auto ContextIdx = readNumber(); + if (std::error_code EC = ContextIdx.getError()) + return EC; + if (*ContextIdx >= CSNameTable->size()) + return sampleprof_error::truncated_name_table; + return (*CSNameTable)[*ContextIdx]; +} + +ErrorOr +SampleProfileReaderExtBinaryBase::readNameFromTable(bool IsContextName) { + if (IsContextName) { + auto FContext(readContextFromTable()); + if (std::error_code EC = FContext.getError()) + return EC; + return SampleContext(*FContext); + } else { + auto FName(readStringFromTable()); + if (std::error_code EC = FName.getError()) + return EC; + return SampleContext(*FName); + } +} + std::error_code SampleProfileReaderExtBinaryBase::readOneSection( const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) { Data = Start; @@ -634,6 +714,11 @@ return EC; break; } + case SecCSNameTable: { + if (std::error_code EC = readCSNameTableSec()) + return EC; + break; + } case SecLBRProfile: if (std::error_code EC = readFuncProfiles()) return EC; @@ -685,7 +770,7 @@ FuncOffsetTable.reserve(*Size); for (uint32_t I = 0; I < *Size; ++I) { - auto FName(readStringFromTable()); + auto FName(readNameFromTable(FunctionSamples::ProfileIsCS)); if (std::error_code EC = FName.getError()) return EC; @@ -711,7 +796,8 @@ const uint8_t *Start = Data; if (!LoadFuncsToBeUsed) { while (Data < End) { - if (std::error_code EC = readFuncProfile(Data)) + if (std::error_code EC = + readFuncProfile(Data, FunctionSamples::ProfileIsCS)) return EC; } assert(Data == End && "More data is read than expected"); @@ -731,20 +817,15 @@ continue; const uint8_t *FuncProfileAddr = Start + iter->second; assert(FuncProfileAddr < End && "out of LBRProfile section"); - if (std::error_code EC = readFuncProfile(FuncProfileAddr)) + if (std::error_code EC = + readFuncProfile(FuncProfileAddr, FunctionSamples::ProfileIsCS)) return EC; } } else if (FunctionSamples::ProfileIsCS) { // Compute the ordered set of names, so we can // get all context profiles under a subtree by // iterating through the ordered names. - struct Comparer { - // Ignore the closing ']' when ordering context - bool operator()(const StringRef &L, const StringRef &R) const { - return L.substr(0, L.size() - 1) < R.substr(0, R.size() - 1); - } - }; - std::set OrderedNames; + std::set OrderedNames; for (auto Name : FuncOffsetTable) { OrderedNames.insert(Name.first); } @@ -752,9 +833,8 @@ // For each function in current module, load all // context profiles for the function. for (auto NameOffset : FuncOffsetTable) { - StringRef ContextName = NameOffset.first; - SampleContext FContext(ContextName); - auto FuncName = FContext.getNameWithoutContext(); + SampleContext FContext = NameOffset.first; + auto FuncName = FContext.getName(); if (!FuncsToUse.count(FuncName) && (!Remapper || !Remapper->exist(FuncName))) continue; @@ -762,12 +842,12 @@ // For each context profile we need, try to load // all context profile in the subtree. This can // help profile guided importing for ThinLTO. - auto It = OrderedNames.find(ContextName); - while (It != OrderedNames.end() && - It->startswith(ContextName.substr(0, ContextName.size() - 1))) { + auto It = OrderedNames.find(FContext); + while (It != OrderedNames.end() && FContext.IsPrefixOf(*It)) { const uint8_t *FuncProfileAddr = Start + FuncOffsetTable[*It]; assert(FuncProfileAddr < End && "out of LBRProfile section"); - if (std::error_code EC = readFuncProfile(FuncProfileAddr)) + if (std::error_code EC = readFuncProfile( + FuncProfileAddr, FunctionSamples::ProfileIsCS)) return EC; // Remove loaded context profile so we won't // load it repeatedly. @@ -777,13 +857,14 @@ } else { for (auto NameOffset : FuncOffsetTable) { SampleContext FContext(NameOffset.first); - auto FuncName = FContext.getNameWithoutContext(); + auto FuncName = FContext.getName(); if (!FuncsToUse.count(FuncName) && (!Remapper || !Remapper->exist(FuncName))) continue; const uint8_t *FuncProfileAddr = Start + NameOffset.second; assert(FuncProfileAddr < End && "out of LBRProfile section"); - if (std::error_code EC = readFuncProfile(FuncProfileAddr)) + if (std::error_code EC = + readFuncProfile(FuncProfileAddr, FunctionSamples::ProfileIsCS)) return EC; } } @@ -908,7 +989,8 @@ const uint8_t *SavedData = Data; if (std::error_code EC = readFuncProfile( reinterpret_cast(Buffer->getBufferStart()) + - Offset)) + Offset, + false)) return EC; Data = SavedData; } @@ -985,22 +1067,61 @@ return SampleProfileReaderBinary::readNameTable(); } +std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() { + auto Size = readNumber(); + if (std::error_code EC = Size.getError()) + return EC; + + std::vector *PNameVec = + new std::vector(); + PNameVec->reserve(*Size); + for (uint32_t I = 0; I < *Size; ++I) { + SampleContextFrameVector Context; + auto ContextSize = readNumber(); + if (std::error_code EC = ContextSize.getError()) + return EC; + for (uint32_t J = 0; J < *ContextSize; ++J) { + auto FName(readStringFromTable()); + if (std::error_code EC = FName.getError()) + return EC; + auto LineOffset = readNumber(); + if (std::error_code EC = LineOffset.getError()) + return EC; + + if (!isOffsetLegal(*LineOffset)) + return std::error_code(); + + auto Discriminator = readNumber(); + if (std::error_code EC = Discriminator.getError()) + return EC; + + Context.emplace_back(SampleContextFrame( + FName.get(), LineLocation(LineOffset.get(), Discriminator.get()))); + } + + PNameVec->emplace_back(Context); + } + + // From this point the underlying object of CSNameTable should be immutable. + CSNameTable.reset(PNameVec); + return sampleprof_error::success; +} + std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) { while (Data < End) { - auto FName(readStringFromTable()); - if (std::error_code EC = FName.getError()) + auto FContext(readNameFromTable(FunctionSamples::ProfileIsCS)); + if (std::error_code EC = FContext.getError()) return EC; - SampleContext FContext(*FName); - bool ProfileInMap = Profiles.count(FContext); + bool ProfileInMap = Profiles.count(FContext.get()); if (ProfileIsProbeBased) { auto Checksum = readNumber(); if (std::error_code EC = Checksum.getError()) return EC; if (ProfileInMap) - Profiles[FContext].setFunctionHash(*Checksum); + Profiles[FContext.get()].setFunctionHash(*Checksum); } if (ProfileHasAttribute) { @@ -1008,7 +1129,7 @@ if (std::error_code EC = Attributes.getError()) return EC; if (ProfileInMap) - Profiles[FContext].getContext().setAllAttributes(*Attributes); + Profiles[FContext.get()].getContext().setAllAttributes(*Attributes); } } diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -41,8 +41,8 @@ using namespace llvm; using namespace sampleprof; -std::error_code SampleProfileWriter::writeFuncProfiles( - const StringMap &ProfileMap) { +std::error_code +SampleProfileWriter::writeFuncProfiles(const SampleProfileMap &ProfileMap) { std::vector V; sortFuncProfiles(ProfileMap, V); for (const auto &I : V) { @@ -52,8 +52,7 @@ return sampleprof_error::success; } -std::error_code -SampleProfileWriter::write(const StringMap &ProfileMap) { +std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) { if (std::error_code EC = writeHeader(ProfileMap)) return EC; @@ -117,8 +116,8 @@ return sampleprof_error::success; } -std::error_code SampleProfileWriterExtBinaryBase::write( - const StringMap &ProfileMap) { +std::error_code +SampleProfileWriterExtBinaryBase::write(const SampleProfileMap &ProfileMap) { if (std::error_code EC = writeHeader(ProfileMap)) return EC; @@ -133,11 +132,28 @@ return sampleprof_error::success; } +std::error_code +SampleProfileWriterExtBinaryBase::writeNameIdx(const SampleContext &Context) { + if (Context.hasContext()) + return writeCSNameIdx(Context); + else + return SampleProfileWriterBinary::writeNameIdx(Context.getName()); +} + +std::error_code +SampleProfileWriterExtBinaryBase::writeCSNameIdx(const SampleContext &Context) { + const auto &Ret = CSNameTable.find(Context); + if (Ret == CSNameTable.end()) + return sampleprof_error::truncated_name_table; + encodeULEB128(Ret->second, *OutputStream); + return sampleprof_error::success; +} + std::error_code SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) { uint64_t Offset = OutputStream->tell(); - StringRef Name = S.getNameWithContext(); - FuncOffsetTable[Name] = Offset - SecLBRProfileStart; + auto &Context = S.getContext(); + FuncOffsetTable[Context] = Offset - SecLBRProfileStart; encodeULEB128(S.getHeadSamples(), *OutputStream); return writeBody(S); } @@ -150,8 +166,7 @@ // Write out FuncOffsetTable. for (auto Entry : FuncOffsetTable) { - if (std::error_code EC = - writeNameIdx(Entry.first, FunctionSamples::ProfileIsCS)) + if (std::error_code EC = writeNameIdx(Entry.first)) return EC; encodeULEB128(Entry.second, OS); } @@ -160,13 +175,12 @@ } std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata( - const StringMap &Profiles) { + const SampleProfileMap &Profiles) { if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS) return sampleprof_error::success; auto &OS = *OutputStream; for (const auto &Entry : Profiles) { - if (std::error_code EC = writeNameIdx(Entry.second.getNameWithContext(), - FunctionSamples::ProfileIsCS)) + if (std::error_code EC = writeNameIdx(Entry.second.getContext())) return EC; if (FunctionSamples::ProfileIsProbeBased) encodeULEB128(Entry.second.getFunctionHash(), OS); @@ -182,7 +196,7 @@ auto &OS = *OutputStream; std::set V; - stablizeNameTable(V); + stablizeNameTable(NameTable, V); // Write out the MD5 name table. We wrote unencoded MD5 so reader can // retrieve the name using the name index without having to read the @@ -195,11 +209,10 @@ } std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection( - const StringMap &ProfileMap) { + const SampleProfileMap &ProfileMap) { for (const auto &I : ProfileMap) { - assert(I.first() == I.second.getNameWithContext() && - "Inconsistent profile map"); - addName(I.second.getNameWithContext(), FunctionSamples::ProfileIsCS); + assert(I.first == I.second.getContext() && "Inconsistent profile map"); + addName(I.second.getContext()); addNames(I.second); } @@ -218,6 +231,34 @@ return sampleprof_error::success; } +std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() { + // Sort the names to make CSNameTable deterministic. + std::set OrderedContexts; + for (const auto &I : CSNameTable) + OrderedContexts.insert(I.first); + assert(OrderedContexts.size() == CSNameTable.size() && + "Unmatched ordered and unordered contexts"); + uint64_t I = 0; + for (auto &Context : OrderedContexts) + CSNameTable[Context] = I++; + + auto &OS = *OutputStream; + encodeULEB128(OrderedContexts.size(), OS); + support::endian::Writer Writer(OS, support::little); + for (auto Context : OrderedContexts) { + auto Callsites = Context.getContextFrames(); + encodeULEB128(Callsites.size(), OS); + for (auto &Callsite : Callsites) { + if (std::error_code EC = writeNameIdx(Callsite.CallerName)) + return EC; + encodeULEB128(Callsite.Callsite.LineOffset, OS); + encodeULEB128(Callsite.Callsite.Discriminator, OS); + } + } + + return sampleprof_error::success; +} + std::error_code SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() { if (ProfSymList && ProfSymList->size() > 0) @@ -228,8 +269,7 @@ } std::error_code SampleProfileWriterExtBinaryBase::writeOneSection( - SecType Type, uint32_t LayoutIdx, - const StringMap &ProfileMap) { + SecType Type, uint32_t LayoutIdx, const SampleProfileMap &ProfileMap) { // The setting of SecFlagCompress should happen before markSectionStart. if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress()) setToCompressSection(SecProfileSymbolList); @@ -253,6 +293,10 @@ if (auto EC = writeNameTableSection(ProfileMap)) return EC; break; + case SecCSNameTable: + if (auto EC = writeCSNameTableSection()) + return EC; + break; case SecLBRProfile: SecLBRProfileStart = OutputStream->tell(); if (std::error_code EC = writeFuncProfiles(ProfileMap)) @@ -281,7 +325,7 @@ } std::error_code SampleProfileWriterExtBinary::writeDefaultLayout( - const StringMap &ProfileMap) { + const SampleProfileMap &ProfileMap) { // The const indices passed to writeOneSection below are specifying the // positions of the sections in SectionHdrLayout. Look at // initSectionHdrLayout to find out where each section is located in @@ -290,32 +334,33 @@ return EC; if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap)) return EC; - if (auto EC = writeOneSection(SecLBRProfile, 3, ProfileMap)) + if (auto EC = writeOneSection(SecCSNameTable, 2, ProfileMap)) + return EC; + if (auto EC = writeOneSection(SecLBRProfile, 4, ProfileMap)) return EC; - if (auto EC = writeOneSection(SecProfileSymbolList, 4, ProfileMap)) + if (auto EC = writeOneSection(SecProfileSymbolList, 5, ProfileMap)) return EC; - if (auto EC = writeOneSection(SecFuncOffsetTable, 2, ProfileMap)) + if (auto EC = writeOneSection(SecFuncOffsetTable, 3, ProfileMap)) return EC; - if (auto EC = writeOneSection(SecFuncMetadata, 5, ProfileMap)) + if (auto EC = writeOneSection(SecFuncMetadata, 6, ProfileMap)) return EC; return sampleprof_error::success; } -static void -splitProfileMapToTwo(const StringMap &ProfileMap, - StringMap &ContextProfileMap, - StringMap &NoContextProfileMap) { +static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap, + SampleProfileMap &ContextProfileMap, + SampleProfileMap &NoContextProfileMap) { for (const auto &I : ProfileMap) { if (I.second.getCallsiteSamples().size()) - ContextProfileMap.insert({I.first(), I.second}); + ContextProfileMap.insert({I.first, I.second}); else - NoContextProfileMap.insert({I.first(), I.second}); + NoContextProfileMap.insert({I.first, I.second}); } } std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout( - const StringMap &ProfileMap) { - StringMap ContextProfileMap, NoContextProfileMap; + const SampleProfileMap &ProfileMap) { + SampleProfileMap ContextProfileMap, NoContextProfileMap; splitProfileMapToTwo(ProfileMap, ContextProfileMap, NoContextProfileMap); if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap)) @@ -345,7 +390,7 @@ } std::error_code SampleProfileWriterExtBinary::writeSections( - const StringMap &ProfileMap) { + const SampleProfileMap &ProfileMap) { std::error_code EC; if (SecLayout == DefaultLayout) EC = writeDefaultLayout(ProfileMap); @@ -356,8 +401,8 @@ return EC; } -std::error_code SampleProfileWriterCompactBinary::write( - const StringMap &ProfileMap) { +std::error_code +SampleProfileWriterCompactBinary::write(const SampleProfileMap &ProfileMap) { if (std::error_code EC = SampleProfileWriter::write(ProfileMap)) return EC; if (std::error_code EC = writeFuncOffsetTable()) @@ -376,7 +421,7 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) { auto &OS = *OutputStream; if (FunctionSamples::ProfileIsCS) - OS << "[" << S.getNameWithContext() << "]:" << S.getTotalSamples(); + OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples(); else OS << S.getName() << ":" << S.getTotalSamples(); @@ -432,27 +477,27 @@ return sampleprof_error::success; } -std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName, - bool IsContextName) { - std::string BracketedName; - if (IsContextName) { - BracketedName = "[" + FName.str() + "]"; - FName = StringRef(BracketedName); - } +std::error_code +SampleProfileWriterBinary::writeNameIdx(const SampleContext &Context) { + return writeNameIdx(Context.getName()); +} - const auto &Ret = NameTable.find(FName); - if (Ret == NameTable.end()) +std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName) { + auto &NTable = getNameTable(); + const auto &Ret = NTable.find(FName); + if (Ret == NTable.end()) return sampleprof_error::truncated_name_table; encodeULEB128(Ret->second, *OutputStream); return sampleprof_error::success; } -void SampleProfileWriterBinary::addName(StringRef FName, bool IsContextName) { - if (IsContextName) { - auto It = BracketedContextStr.insert("[" + FName.str() + "]"); - FName = StringRef(*It.first); - } - NameTable.insert(std::make_pair(FName, 0)); +void SampleProfileWriterBinary::addName(StringRef FName) { + auto &NTable = getNameTable(); + NTable.insert(std::make_pair(FName, 0)); +} + +void SampleProfileWriterBinary::addName(const SampleContext &Context) { + addName(Context.getName()); } void SampleProfileWriterBinary::addNames(const FunctionSamples &S) { @@ -472,7 +517,18 @@ } } -void SampleProfileWriterBinary::stablizeNameTable(std::set &V) { +void SampleProfileWriterExtBinaryBase::addName(const SampleContext &Context) { + if (Context.hasContext()) { + for (auto &Callsite : Context.getContextFrames()) + SampleProfileWriterBinary::addName(Callsite.CallerName); + CSNameTable.insert(std::make_pair(Context, 0)); + } else { + SampleProfileWriterBinary::addName(Context.getName()); + } +} + +void SampleProfileWriterBinary::stablizeNameTable( + MapVector &NameTable, std::set &V) { // Sort the names to make NameTable deterministic. for (const auto &I : NameTable) V.insert(I.first); @@ -484,7 +540,7 @@ std::error_code SampleProfileWriterBinary::writeNameTable() { auto &OS = *OutputStream; std::set V; - stablizeNameTable(V); + stablizeNameTable(NameTable, V); // Write out the name table. encodeULEB128(NameTable.size(), OS); @@ -513,8 +569,7 @@ // Write out FuncOffsetTable. for (auto Entry : FuncOffsetTable) { - if (std::error_code EC = - writeNameIdx(Entry.first, FunctionSamples::ProfileIsCS)) + if (std::error_code EC = writeNameIdx(Entry.first)) return EC; encodeULEB128(Entry.second, OS); } @@ -524,7 +579,7 @@ std::error_code SampleProfileWriterCompactBinary::writeNameTable() { auto &OS = *OutputStream; std::set V; - stablizeNameTable(V); + stablizeNameTable(NameTable, V); // Write out the name table. encodeULEB128(NameTable.size(), OS); @@ -543,8 +598,8 @@ return sampleprof_error::success; } -std::error_code SampleProfileWriterBinary::writeHeader( - const StringMap &ProfileMap) { +std::error_code +SampleProfileWriterBinary::writeHeader(const SampleProfileMap &ProfileMap) { writeMagicIdent(Format); computeSummary(ProfileMap); @@ -553,9 +608,8 @@ // Generate the name table for all the functions referenced in the profile. for (const auto &I : ProfileMap) { - assert(I.first() == I.second.getNameWithContext() && - "Inconsistent profile map"); - addName(I.first(), FunctionSamples::ProfileIsCS); + assert(I.first == I.second.getContext() && "Inconsistent profile map"); + addName(I.first); addNames(I.second); } @@ -629,7 +683,7 @@ } std::error_code SampleProfileWriterExtBinaryBase::writeHeader( - const StringMap &ProfileMap) { + const SampleProfileMap &ProfileMap) { auto &OS = *OutputStream; FileStart = OS.tell(); writeMagicIdent(Format); @@ -639,7 +693,7 @@ } std::error_code SampleProfileWriterCompactBinary::writeHeader( - const StringMap &ProfileMap) { + const SampleProfileMap &ProfileMap) { support::endian::Writer Writer(*OutputStream, support::little); if (auto EC = SampleProfileWriterBinary::writeHeader(ProfileMap)) return EC; @@ -669,9 +723,7 @@ } std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) { auto &OS = *OutputStream; - - if (std::error_code EC = - writeNameIdx(S.getNameWithContext(), FunctionSamples::ProfileIsCS)) + if (std::error_code EC = writeNameIdx(S.getContext())) return EC; encodeULEB128(S.getTotalSamples(), OS); @@ -790,8 +842,7 @@ return std::move(Writer); } -void SampleProfileWriter::computeSummary( - const StringMap &ProfileMap) { +void SampleProfileWriter::computeSummary(const SampleProfileMap &ProfileMap) { SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); Summary = Builder.computeSummaryForProfiles(ProfileMap); } diff --git a/llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext b/llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext --- a/llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext +++ b/llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext @@ -14,31 +14,31 @@ 11: 23327 _Z3fibi:25228 15: 11 !Attributes: 1 +[external:12 @ main]:154:12 + 2: 12 + 3: 10 _Z5funcAi:7 + 3.1: 10 _Z5funcBi:11 + !Attributes: 0 [main]:154:0 2: 12 3: 18 _Z5funcAi:11 3.1: 18 _Z5funcBi:19 !Attributes: 0 -[external:12 @ main]:154:12 - 2: 12 - 3: 10 _Z5funcAi:7 - 3.1: 10 _Z5funcBi:11 +[external:10 @ _Z5funcBi]:120:10 + 0: 10 + 1: 10 + !Attributes: 0 +[externalA:17 @ _Z5funcBi]:120:3 + 0: 3 + 1: 3 !Attributes: 0 [main:3.1 @ _Z5funcBi]:120:19 0: 19 1: 19 _Z8funcLeafi:20 3: 12 !Attributes: 1 -[externalA:17 @ _Z5funcBi]:120:3 - 0: 3 - 1: 3 - !Attributes: 0 -[external:10 @ _Z5funcBi]:120:10 - 0: 10 - 1: 10 - !Attributes: 0 [main:3 @ _Z5funcAi]:99:11 0: 10 1: 10 _Z8funcLeafi:11 3: 24 - !Attributes: 0 + !Attributes: 0 \ No newline at end of file diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -521,9 +521,9 @@ // Find hot/warm functions in sample profile which is cold in instr profile // and adjust the profiles of those functions in the instr profile. for (const auto &PD : Reader->getProfiles()) { - StringRef FName = PD.getKey(); - const sampleprof::FunctionSamples &FS = PD.getValue(); - auto It = InstrProfileMap.find(FName); + auto &FName = PD.first; + const sampleprof::FunctionSamples &FS = PD.second; + auto It = InstrProfileMap.find(FName.toString()); if (FS.getHeadSamples() > ColdSampleThreshold && It != InstrProfileMap.end() && It->second.MaxCount <= ColdInstrThreshold && @@ -690,7 +690,7 @@ bool SampleMergeColdContext, bool SampleTrimColdContext, bool SampleColdContextFrameDepth, FailureMode FailMode) { using namespace sampleprof; - StringMap ProfileMap; + SampleProfileMap ProfileMap; SmallVector, 5> Readers; LLVMContext Context; sampleprof::ProfileSymbolList WriterList; @@ -716,7 +716,7 @@ continue; } - StringMap &Profiles = Reader->getProfiles(); + SampleProfileMap &Profiles = Reader->getProfiles(); if (ProfileIsProbeBased.hasValue() && ProfileIsProbeBased != FunctionSamples::ProfileIsProbeBased) exitWithError( @@ -725,19 +725,19 @@ if (ProfileIsCS.hasValue() && ProfileIsCS != FunctionSamples::ProfileIsCS) exitWithError("cannot merge CS profile with non-CS profile"); ProfileIsCS = FunctionSamples::ProfileIsCS; - for (StringMap::iterator I = Profiles.begin(), - E = Profiles.end(); + for (SampleProfileMap::iterator I = Profiles.begin(), E = Profiles.end(); I != E; ++I) { sampleprof_error Result = sampleprof_error::success; FunctionSamples Remapped = Remapper ? remapSamples(I->second, *Remapper, Result) : FunctionSamples(); FunctionSamples &Samples = Remapper ? Remapped : I->second; - StringRef FName = Samples.getNameWithContext(); + SampleContext FName = Samples.getContext(); MergeResult(Result, ProfileMap[FName].merge(Samples, Input.Weight)); if (Result != sampleprof_error::success) { std::error_code EC = make_error_code(Result); - handleMergeWriterError(errorCodeToError(EC), Input.Filename, FName); + handleMergeWriterError(errorCodeToError(EC), Input.Filename, + FName.toString()); } } @@ -1022,8 +1022,8 @@ namespace { struct SampleOverlapStats { - StringRef BaseName; - StringRef TestName; + SampleContext BaseName; + SampleContext TestName; // Number of overlap units uint64_t OverlapCount; // Total samples of overlap units @@ -1226,6 +1226,9 @@ /// Load profiles specified by BaseFilename and TestFilename. std::error_code loadProfiles(); + using FuncSampleStatsMap = + std::unordered_map; + private: SampleOverlapStats ProfOverlap; SampleOverlapStats HotFuncOverlap; @@ -1236,8 +1239,8 @@ std::unique_ptr TestReader; // BaseStats and TestStats hold FuncSampleStats for each function, with // function name as the key. - StringMap BaseStats; - StringMap TestStats; + FuncSampleStatsMap BaseStats; + FuncSampleStatsMap TestStats; // Low similarity threshold in floating point number double LowSimilarityThreshold; // Block samples above BaseHotThreshold or TestHotThreshold are considered hot @@ -1276,8 +1279,8 @@ void updateHotBlockOverlap(uint64_t BaseSample, uint64_t TestSample, uint64_t HotBlockCount); - void getHotFunctions(const StringMap &ProfStats, - StringMap &HotFunc, + void getHotFunctions(const FuncSampleStatsMap &ProfStats, + FuncSampleStatsMap &HotFunc, uint64_t HotThreshold) const; void computeHotFuncOverlap(); @@ -1381,26 +1384,26 @@ } void SampleOverlapAggregator::getHotFunctions( - const StringMap &ProfStats, - StringMap &HotFunc, uint64_t HotThreshold) const { + const FuncSampleStatsMap &ProfStats, FuncSampleStatsMap &HotFunc, + uint64_t HotThreshold) const { for (const auto &F : ProfStats) { if (isFunctionHot(F.second, HotThreshold)) - HotFunc.try_emplace(F.first(), F.second); + HotFunc.emplace(F.first, F.second); } } void SampleOverlapAggregator::computeHotFuncOverlap() { - StringMap BaseHotFunc; + FuncSampleStatsMap BaseHotFunc; getHotFunctions(BaseStats, BaseHotFunc, BaseHotThreshold); HotFuncOverlap.BaseCount = BaseHotFunc.size(); - StringMap TestHotFunc; + FuncSampleStatsMap TestHotFunc; getHotFunctions(TestStats, TestHotFunc, TestHotThreshold); HotFuncOverlap.TestCount = TestHotFunc.size(); HotFuncOverlap.UnionCount = HotFuncOverlap.TestCount; for (const auto &F : BaseHotFunc) { - if (TestHotFunc.count(F.first())) + if (TestHotFunc.count(F.first)) ++HotFuncOverlap.OverlapCount; else ++HotFuncOverlap.UnionCount; @@ -1612,18 +1615,19 @@ void SampleOverlapAggregator::computeSampleProfileOverlap(raw_fd_ostream &OS) { using namespace sampleprof; - StringMap BaseFuncProf; + std::unordered_map + BaseFuncProf; const auto &BaseProfiles = BaseReader->getProfiles(); for (const auto &BaseFunc : BaseProfiles) { - BaseFuncProf.try_emplace(BaseFunc.second.getNameWithContext(), - &(BaseFunc.second)); + BaseFuncProf.emplace(BaseFunc.second.getContext(), &(BaseFunc.second)); } ProfOverlap.UnionCount = BaseFuncProf.size(); const auto &TestProfiles = TestReader->getProfiles(); for (const auto &TestFunc : TestProfiles) { SampleOverlapStats FuncOverlap; - FuncOverlap.TestName = TestFunc.second.getNameWithContext(); + FuncOverlap.TestName = TestFunc.second.getContext(); assert(TestStats.count(FuncOverlap.TestName) && "TestStats should have records for all functions in test profile " "except inlinees"); @@ -1650,7 +1654,7 @@ // Two functions match with each other. Compute function-level overlap and // aggregate them into profile-level overlap. - FuncOverlap.BaseName = Match->second->getNameWithContext(); + FuncOverlap.BaseName = Match->second->getContext(); assert(BaseStats.count(FuncOverlap.BaseName) && "BaseStats should have records for all functions in base profile " "except inlinees"); @@ -1683,8 +1687,8 @@ (Match != BaseFuncProf.end() && FuncOverlap.Similarity < LowSimilarityThreshold) || (Match != BaseFuncProf.end() && !FuncFilter.NameFilter.empty() && - FuncOverlap.BaseName.find(FuncFilter.NameFilter) != - FuncOverlap.BaseName.npos)) { + FuncOverlap.BaseName.toString().find(FuncFilter.NameFilter) != + std::string::npos)) { assert(ProfOverlap.BaseSample > 0 && "Total samples in base profile should be greater than 0"); FuncOverlap.BaseWeight = @@ -1699,11 +1703,10 @@ // Traverse through functions in base profile but not in test profile. for (const auto &F : BaseFuncProf) { - assert(BaseStats.count(F.second->getNameWithContext()) && + assert(BaseStats.count(F.second->getContext()) && "BaseStats should have records for all functions in base profile " "except inlinees"); - const FuncSampleStats &FuncStats = - BaseStats[F.second->getNameWithContext()]; + const FuncSampleStats &FuncStats = BaseStats[F.second->getContext()]; ++ProfOverlap.BaseUniqueCount; ProfOverlap.BaseUniqueSample += FuncStats.SampleSum; @@ -1734,7 +1737,7 @@ FuncSampleStats FuncStats; getFuncSampleStats(I.second, FuncStats, BaseHotThreshold); ProfOverlap.BaseSample += FuncStats.SampleSum; - BaseStats.try_emplace(I.second.getNameWithContext(), FuncStats); + BaseStats.emplace(I.second.getContext(), FuncStats); } const auto &TestProf = TestReader->getProfiles(); @@ -1743,7 +1746,7 @@ FuncSampleStats FuncStats; getFuncSampleStats(I.second, FuncStats, TestHotThreshold); ProfOverlap.TestSample += FuncStats.SampleSum; - TestStats.try_emplace(I.second.getNameWithContext(), FuncStats); + TestStats.emplace(I.second.getContext(), FuncStats); } ProfOverlap.BaseName = StringRef(BaseFilename); @@ -1807,13 +1810,15 @@ FOS.PadToColumn(TestSampleCol); FOS << F.second.TestSample; FOS.PadToColumn(FuncNameCol); - FOS << F.second.TestName << "\n"; + FOS << F.second.TestName.toString() << "\n"; } } void SampleOverlapAggregator::dumpProgramSummary(raw_fd_ostream &OS) const { - OS << "Profile overlap infomation for base_profile: " << ProfOverlap.BaseName - << " and test_profile: " << ProfOverlap.TestName << "\nProgram level:\n"; + OS << "Profile overlap infomation for base_profile: " + << ProfOverlap.BaseName.toString() + << " and test_profile: " << ProfOverlap.TestName.toString() + << "\nProgram level:\n"; OS << " Whole program profile similarity: " << format("%.3f%%", ProfOverlap.Similarity * 100) << "\n"; @@ -2271,7 +2276,7 @@ namespace { struct HotFuncInfo { - StringRef FuncName; + std::string FuncName; uint64_t TotalCount; double TotalCountPercent; uint64_t MaxCount; @@ -2282,8 +2287,8 @@ EntryCount(0) {} HotFuncInfo(StringRef FN, uint64_t TS, double TSP, uint64_t MS, uint64_t ES) - : FuncName(FN), TotalCount(TS), TotalCountPercent(TSP), MaxCount(MS), - EntryCount(ES) {} + : FuncName(FN.begin(), FN.end()), TotalCount(TS), TotalCountPercent(TSP), + MaxCount(MS), EntryCount(ES) {} }; } // namespace @@ -2339,9 +2344,8 @@ } } -static int -showHotFunctionList(const StringMap &Profiles, - ProfileSummary &PS, raw_fd_ostream &OS) { +static int showHotFunctionList(const sampleprof::SampleProfileMap &Profiles, + ProfileSummary &PS, raw_fd_ostream &OS) { using namespace sampleprof; const uint32_t HotFuncCutoff = 990000; @@ -2391,8 +2395,8 @@ ? (Func.getTotalSamples() * 100.0) / ProfileTotalSample : 0; PrintValues.emplace_back(HotFuncInfo( - Func.getNameWithContext(), Func.getTotalSamples(), TotalSamplePercent, - FuncPair.second.second, Func.getEntrySamples())); + Func.getContext().toString(), Func.getTotalSamples(), + TotalSamplePercent, FuncPair.second.second, Func.getEntrySamples())); } dumpHotFunctionList(ColumnTitle, ColumnOffset, PrintValues, HotFuncCount, Profiles.size(), HotFuncSample, ProfileTotalSample, @@ -2426,7 +2430,8 @@ if (ShowAllFunctions || ShowFunction.empty()) Reader->dump(OS); else - Reader->dumpFunctionProfile(ShowFunction, OS); + // TODO: parse context string to support filtering by contexts. + Reader->dumpFunctionProfile(StringRef(ShowFunction), OS); if (ShowProfileSymbolList) { std::unique_ptr ReaderList = diff --git a/llvm/unittests/ProfileData/SampleProfTest.cpp b/llvm/unittests/ProfileData/SampleProfTest.cpp --- a/llvm/unittests/ProfileData/SampleProfTest.cpp +++ b/llvm/unittests/ProfileData/SampleProfTest.cpp @@ -193,7 +193,7 @@ BooSamples.addHeadSamples(1); BooSamples.addBodySamples(1, 0, 1232); - StringMap Profiles; + SampleProfileMap Profiles; Profiles[FooName] = std::move(FooSamples); Profiles[BarName] = std::move(BarSamples); Profiles[BazName] = std::move(BazSamples); @@ -327,7 +327,7 @@ verifyProfileSummary(Summary, M, true, true); } - void addFunctionSamples(StringMap *Smap, const char *Fname, + void addFunctionSamples(SampleProfileMap *Smap, const char *Fname, uint64_t TotalSamples, uint64_t HeadSamples) { StringRef Name(Fname); FunctionSamples FcnSamples; @@ -338,8 +338,8 @@ (*Smap)[Name] = FcnSamples; } - StringMap setupFcnSamplesForElisionTest(StringRef Policy) { - StringMap Smap; + SampleProfileMap setupFcnSamplesForElisionTest(StringRef Policy) { + SampleProfileMap Smap; addFunctionSamples(&Smap, "foo", uint64_t(20301), uint64_t(1437)); if (Policy == "" || Policy == "all") return Smap; @@ -373,7 +373,7 @@ Module M("my_module", Context); setupModuleForElisionTest(&M, Policy); - StringMap ProfMap = setupFcnSamplesForElisionTest(Policy); + SampleProfileMap ProfMap = setupFcnSamplesForElisionTest(Policy); // write profile createWriter(Format, ProfileFile.path());