Index: llvm/include/llvm/ProfileData/SampleProf.h =================================================================== --- llvm/include/llvm/ProfileData/SampleProf.h +++ llvm/include/llvm/ProfileData/SampleProf.h @@ -122,6 +122,7 @@ SecProfileSymbolList = 3, SecFuncOffsetTable = 4, SecFuncMetadata = 5, + SecCSNameTable = 6, // marker for the first type of profile. SecFuncProfileFirst = 32, SecLBRProfile = SecFuncProfileFirst @@ -141,6 +142,8 @@ return "FuncOffsetTableSection"; case SecFuncMetadata: return "FunctionMetadata"; + case SecCSNameTable: + return "CSNameTableSection"; case SecLBRProfile: return "LBRProfileSection"; } @@ -469,6 +472,33 @@ } } + static std::vector> + decodeFullContextString(StringRef ContextStr) { + // If the context string is enclosed in a pair of brackets, strip the + // brackets before decoding. + if (ContextStr.startswith("[")) { + assert(ContextStr.endswith("]") && "Unpaired [] in context string"); + ContextStr = ContextStr.substr(1, ContextStr.size() - 2); + } + + std::vector> decodeResult; + StringRef ContextRemain = ContextStr; + StringRef ChildContext; + LineLocation CallSiteLoc(0, 0); + while (!ContextRemain.empty()) { + auto ContextSplit = SampleContext::splitContextString(ContextRemain); + StringRef CurrentContext = ContextSplit.first; + ContextRemain = ContextSplit.second; + + StringRef CalleeName; + LineLocation CallSiteLoc(0, 0); + SampleContext::decodeContextString(CurrentContext, CalleeName, + CallSiteLoc); + decodeResult.emplace_back(CalleeName, CallSiteLoc); + } + return decodeResult; + } + operator StringRef() const { return FullContext; } bool hasAttribute(ContextAttributeMask A) { return Attributes & (uint32_t)A; } void setAttribute(ContextAttributeMask A) { Attributes |= (uint32_t)A; } Index: llvm/include/llvm/ProfileData/SampleProfReader.h =================================================================== --- llvm/include/llvm/ProfileData/SampleProfReader.h +++ llvm/include/llvm/ProfileData/SampleProfReader.h @@ -473,7 +473,9 @@ /// It includes all the names that have samples either in outline instance /// or inline instance. - virtual std::vector *getNameTable() { return nullptr; } + virtual std::vector *getNameTable(bool IsContextName = false) { + return nullptr; + } virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) { return false; }; /// Return whether names in the profile are all MD5 numbers. @@ -571,7 +573,10 @@ /// It includes all the names that have samples either in outline instance /// or inline instance. - virtual std::vector *getNameTable() override { return &NameTable; } + virtual std::vector * + getNameTable(bool IsContextName = false) override { + return &NameTable; + } protected: /// Read a numeric value of type T from the profile. @@ -601,7 +606,8 @@ bool at_eof() const { return Data >= End; } /// Read the next function profile instance. - std::error_code readFuncProfile(const uint8_t *Start); + std::error_code readFuncProfile(const uint8_t *Start, + bool IsContextName = false); /// Read the contents of the given profile instance. std::error_code readProfile(FunctionSamples &FProfile); @@ -625,7 +631,7 @@ std::vector NameTable; /// Read a string indirectly via the name table. - virtual ErrorOr readStringFromTable(); + virtual ErrorOr readStringFromTable(bool IsContextName = false); private: std::error_code readSummaryEntry(std::vector &Entries); @@ -683,6 +689,7 @@ std::error_code readFuncProfiles(); std::error_code readMD5NameTable(); std::error_code readNameTableSec(bool IsMD5); + std::error_code readCSNameTableSec(); std::error_code readProfileSymbolList(); virtual std::error_code readHeader() override; @@ -691,7 +698,8 @@ const SecHdrTableEntry &Entry); // placeholder for subclasses to dispatch their own section readers. virtual std::error_code readCustomSection(const SecHdrTableEntry &Entry) = 0; - virtual ErrorOr readStringFromTable() override; + virtual ErrorOr + readStringFromTable(bool IsContextName = false) override; std::unique_ptr ProfSymList; @@ -716,6 +724,11 @@ /// the lifetime of MD5StringBuf is not shorter than that of NameTable. std::unique_ptr> MD5StringBuf; + /// CSNameBuf buffer is used to save full context names. + std::unique_ptr> CSNameBuf; + /// CSNameTable contains StringRefs referring to the names in CSNameBuf. + std::unique_ptr> CSNameTable; + /// If SkipFlatProf is true, skip the sections with /// SecFlagFlat flag. bool SkipFlatProf = false; @@ -746,6 +759,13 @@ }; virtual void setSkipFlatProf(bool Skip) override { SkipFlatProf = Skip; } + + virtual std::vector * + getNameTable(bool IsContextName = false) override { + if (IsContextName && CSNameTable.get()) + return CSNameTable.get(); + return &NameTable; + } }; class SampleProfileReaderExtBinary : public SampleProfileReaderExtBinaryBase { @@ -777,7 +797,8 @@ virtual std::error_code verifySPMagic(uint64_t Magic) override; virtual std::error_code readNameTable() override; /// Read a string indirectly via the name table. - virtual ErrorOr readStringFromTable() override; + virtual ErrorOr + readStringFromTable(bool IsContextName = false) override; virtual std::error_code readHeader() override; std::error_code readFuncOffsetTable(); Index: llvm/include/llvm/ProfileData/SampleProfWriter.h =================================================================== --- llvm/include/llvm/ProfileData/SampleProfWriter.h +++ llvm/include/llvm/ProfileData/SampleProfWriter.h @@ -132,6 +132,9 @@ virtual std::error_code writeSample(const FunctionSamples &S) override; protected: + virtual MapVector &getNameTable(bool IsContextName) { + return NameTable; + } virtual std::error_code writeMagicIdent(SampleProfileFormat Format); virtual std::error_code writeNameTable(); virtual std::error_code @@ -139,7 +142,8 @@ std::error_code writeSummary(); std::error_code writeNameIdx(StringRef FName, bool IsContextName = false); std::error_code writeBody(const FunctionSamples &S); - inline void stablizeNameTable(std::set &V); + inline void stablizeNameTable(MapVector &NameTable, + std::set &V); MapVector NameTable; std::unordered_set BracketedContextStr; @@ -168,6 +172,7 @@ // DefaultLayout SmallVector({{SecProfSummary, 0, 0, 0, 0}, {SecNameTable, 0, 0, 0, 0}, + {SecCSNameTable, 0, 0, 0, 0}, {SecFuncOffsetTable, 0, 0, 0, 0}, {SecLBRProfile, 0, 0, 0, 0}, {SecProfileSymbolList, 0, 0, 0, 0}, @@ -231,6 +236,9 @@ } protected: + virtual MapVector &getNameTable(bool IsContextName) { + return IsContextName ? CSNameTable : NameTable; + } uint64_t markSectionStart(SecType Type, uint32_t LayoutIdx); std::error_code addNewSection(SecType Sec, uint32_t LayoutIdx, uint64_t SectionStart); @@ -263,6 +271,7 @@ // Helper function to write name table. virtual std::error_code writeNameTable() override; + std::error_code writeCSNameTableSection(); std::error_code writeFuncMetadata(const StringMap &Profiles); @@ -318,6 +327,9 @@ // Whether to use MD5 to represent string. bool UseMD5 = false; + // Maps context name to its index in the context name table. + MapVector CSNameTable; + ProfileSymbolList *ProfSymList = nullptr; }; Index: llvm/lib/ProfileData/SampleProfReader.cpp =================================================================== --- llvm/lib/ProfileData/SampleProfReader.cpp +++ llvm/lib/ProfileData/SampleProfReader.cpp @@ -442,17 +442,20 @@ return *Idx; } -ErrorOr SampleProfileReaderBinary::readStringFromTable() { - auto Idx = readStringIndex(NameTable); +ErrorOr +SampleProfileReaderBinary::readStringFromTable(bool IsContextName) { + auto NTable = getNameTable(IsContextName); + assert(NTable && "NTable should not be nullptr"); + auto Idx = readStringIndex(*NTable); if (std::error_code EC = Idx.getError()) return EC; - - return NameTable[*Idx]; + return (*NTable)[*Idx]; } -ErrorOr SampleProfileReaderExtBinaryBase::readStringFromTable() { +ErrorOr +SampleProfileReaderExtBinaryBase::readStringFromTable(bool IsContextName) { if (!FixedLengthMD5) - return SampleProfileReaderBinary::readStringFromTable(); + return SampleProfileReaderBinary::readStringFromTable(IsContextName); // read NameTable index. auto Idx = readStringIndex(NameTable); @@ -478,7 +481,8 @@ return SR; } -ErrorOr SampleProfileReaderCompactBinary::readStringFromTable() { +ErrorOr +SampleProfileReaderCompactBinary::readStringFromTable(bool IsContextName) { auto Idx = readStringIndex(NameTable); if (std::error_code EC = Idx.getError()) return EC; @@ -569,14 +573,14 @@ return sampleprof_error::success; } -std::error_code -SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) { +std::error_code SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start, + bool IsContextName) { Data = Start; auto NumHeadSamples = readNumber(); if (std::error_code EC = NumHeadSamples.getError()) return EC; - auto FName(readStringFromTable()); + auto FName(readStringFromTable(IsContextName)); if (std::error_code EC = FName.getError()) return EC; @@ -632,6 +636,11 @@ return EC; break; } + case SecCSNameTable: { + if (std::error_code EC = readCSNameTableSec()) + return EC; + break; + } case SecLBRProfile: if (std::error_code EC = readFuncProfiles()) return EC; @@ -683,7 +692,7 @@ FuncOffsetTable.reserve(*Size); for (uint32_t I = 0; I < *Size; ++I) { - auto FName(readStringFromTable()); + auto FName(readStringFromTable(FunctionSamples::ProfileIsCS)); if (std::error_code EC = FName.getError()) return EC; @@ -709,7 +718,8 @@ const uint8_t *Start = Data; if (!LoadFuncsToBeUsed) { while (Data < End) { - if (std::error_code EC = readFuncProfile(Data)) + if (std::error_code EC = + readFuncProfile(Data, FunctionSamples::ProfileIsCS)) return EC; } assert(Data == End && "More data is read than expected"); @@ -983,10 +993,57 @@ return SampleProfileReaderBinary::readNameTable(); } +std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() { + auto Size = readNumber(); + if (std::error_code EC = Size.getError()) + return EC; + + CSNameBuf = std::make_unique>(); + CSNameBuf->reserve(*Size + CSNameBuf->size()); + CSNameTable = std::make_unique>(); + CSNameTable->reserve(*Size + CSNameTable->size()); + for (uint32_t I = 0; I < *Size; ++I) { + std::string Callsite; + auto CallsiteSize = readNumber(); + if (std::error_code EC = CallsiteSize.getError()) + return EC; + for (uint32_t J = 0; J < *CallsiteSize; ++J) { + if (J != 0) + Callsite = Callsite + " @ "; + + auto FName(readStringFromTable()); + if (std::error_code EC = FName.getError()) + return EC; + auto LineOffset = readNumber(); + if (std::error_code EC = LineOffset.getError()) + return EC; + + if (!isOffsetLegal(*LineOffset)) + return std::error_code(); + + auto Discriminator = readNumber(); + if (std::error_code EC = Discriminator.getError()) + return EC; + + if (J == (*CallsiteSize) - 1) { + Callsite = Callsite + (*FName).str(); + continue; + } + Callsite = Callsite + (*FName).str() + ":" + std::to_string(*LineOffset) + + "." + std::to_string(*Discriminator); + } + Callsite = "[" + Callsite + "]"; + CSNameBuf->push_back(Callsite); + CSNameTable->push_back(CSNameBuf->back()); + } + + return sampleprof_error::success; +} + std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) { while (Data < End) { - auto FName(readStringFromTable()); + auto FName(readStringFromTable(FunctionSamples::ProfileIsCS)); if (std::error_code EC = FName.getError()) return EC; Index: llvm/lib/ProfileData/SampleProfWriter.cpp =================================================================== --- llvm/lib/ProfileData/SampleProfWriter.cpp +++ llvm/lib/ProfileData/SampleProfWriter.cpp @@ -195,7 +195,7 @@ auto &OS = *OutputStream; std::set V; - stablizeNameTable(V); + stablizeNameTable(NameTable, V); // Write out the MD5 name table. We wrote unencoded MD5 so reader can // retrieve the name using the name index without having to read the @@ -212,7 +212,16 @@ for (const auto &I : ProfileMap) { assert(I.first() == I.second.getNameWithContext() && "Inconsistent profile map"); - addName(I.second.getNameWithContext(), FunctionSamples::ProfileIsCS); + if (FunctionSamples::ProfileIsCS) { + StringRef CSName = I.second.getNameWithContext(); + addName(CSName, true); + std::vector> callsites = + SampleContext::decodeFullContextString(CSName); + for (const auto NameLoc : callsites) + addName(NameLoc.first, false); + } else { + addName(I.second.getName(), false); + } addNames(I.second); } @@ -231,6 +240,27 @@ return sampleprof_error::success; } +std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() { + auto &OS = *OutputStream; + std::set CSNameSet; + stablizeNameTable(CSNameTable, CSNameSet); + + encodeULEB128(CSNameSet.size(), OS); + support::endian::Writer Writer(OS, support::little); + for (auto CSName : CSNameSet) { + std::vector> callsites = + SampleContext::decodeFullContextString(CSName); + encodeULEB128(callsites.size(), OS); + for (const auto NameLoc : callsites) { + if (std::error_code EC = writeNameIdx(NameLoc.first)) + return EC; + encodeULEB128(NameLoc.second.LineOffset, OS); + encodeULEB128(NameLoc.second.Discriminator, OS); + } + } + return sampleprof_error::success; +} + std::error_code SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() { if (ProfSymList && ProfSymList->size() > 0) @@ -266,6 +296,10 @@ if (auto EC = writeNameTableSection(ProfileMap)) return EC; break; + case SecCSNameTable: + if (auto EC = writeCSNameTableSection()) + return EC; + break; case SecLBRProfile: SecLBRProfileStart = OutputStream->tell(); if (std::error_code EC = writeFuncProfiles(ProfileMap)) @@ -303,13 +337,15 @@ return EC; if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap)) return EC; - if (auto EC = writeOneSection(SecLBRProfile, 3, ProfileMap)) + if (auto EC = writeOneSection(SecCSNameTable, 2, ProfileMap)) + return EC; + if (auto EC = writeOneSection(SecLBRProfile, 4, ProfileMap)) return EC; - if (auto EC = writeOneSection(SecProfileSymbolList, 4, ProfileMap)) + if (auto EC = writeOneSection(SecProfileSymbolList, 5, ProfileMap)) return EC; - if (auto EC = writeOneSection(SecFuncOffsetTable, 2, ProfileMap)) + if (auto EC = writeOneSection(SecFuncOffsetTable, 3, ProfileMap)) return EC; - if (auto EC = writeOneSection(SecFuncMetadata, 5, ProfileMap)) + if (auto EC = writeOneSection(SecFuncMetadata, 6, ProfileMap)) return EC; return sampleprof_error::success; } @@ -453,8 +489,9 @@ FName = StringRef(BracketedName); } - const auto &Ret = NameTable.find(FName); - if (Ret == NameTable.end()) + auto &NTable = getNameTable(IsContextName); + const auto &Ret = NTable.find(FName); + if (Ret == NTable.end()) return sampleprof_error::truncated_name_table; encodeULEB128(Ret->second, *OutputStream); return sampleprof_error::success; @@ -465,7 +502,8 @@ auto It = BracketedContextStr.insert("[" + FName.str() + "]"); FName = StringRef(*It.first); } - NameTable.insert(std::make_pair(FName, 0)); + auto &NTable = getNameTable(IsContextName); + NTable.insert(std::make_pair(FName, 0)); } void SampleProfileWriterBinary::addNames(const FunctionSamples &S) { @@ -485,7 +523,8 @@ } } -void SampleProfileWriterBinary::stablizeNameTable(std::set &V) { +void SampleProfileWriterBinary::stablizeNameTable( + MapVector &NameTable, std::set &V) { // Sort the names to make NameTable deterministic. for (const auto &I : NameTable) V.insert(I.first); @@ -497,7 +536,7 @@ std::error_code SampleProfileWriterBinary::writeNameTable() { auto &OS = *OutputStream; std::set V; - stablizeNameTable(V); + stablizeNameTable(NameTable, V); // Write out the name table. encodeULEB128(NameTable.size(), OS); @@ -537,7 +576,7 @@ std::error_code SampleProfileWriterCompactBinary::writeNameTable() { auto &OS = *OutputStream; std::set V; - stablizeNameTable(V); + stablizeNameTable(NameTable, V); // Write out the name table. encodeULEB128(NameTable.size(), OS);