diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -125,6 +125,7 @@ SecFuncOffsetTable = 4, SecFuncMetadata = 5, SecCSNameTable = 6, + SecMetadataOffsetTable = 7, // marker for the first type of profile. SecFuncProfileFirst = 32, SecLBRProfile = SecFuncProfileFirst @@ -148,6 +149,8 @@ return "CSNameTableSection"; case SecLBRProfile: return "LBRProfileSection"; + case SecMetadataOffsetTable: + return "MetadtaOffsetTableSection"; default: return "UnknownSection"; } diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -241,6 +241,7 @@ #include #include #include +#include #include #include #include @@ -702,7 +703,9 @@ std::error_code readFuncMetadata(bool ProfileHasAttribute); std::error_code readFuncMetadata(bool ProfileHasAttribute, + const uint8_t *Start, FunctionSamples *FProfile); + std::error_code readMetadataOffsetTable(); std::error_code readFuncOffsetTable(); std::error_code readFuncProfiles(); std::error_code readMD5NameTable(); @@ -719,12 +722,19 @@ virtual ErrorOr readStringFromTable() override; virtual ErrorOr readSampleContextFromTable() override; ErrorOr readContextFromTable(); + std::set computeReadOffsets( + DenseMap &OffsetTable, + std::vector> *OrderedOffsets = + nullptr); std::unique_ptr ProfSymList; /// The table mapping from function context to the offset of its - /// FunctionSample towards file start. + /// FunctionSample towards section start. DenseMap FuncOffsetTable; + /// The table mapping from function context to the offset of its + /// metadata towards section start. + DenseMap MetadataOffsetTable; /// Function offset mapping ordered by contexts. std::unique_ptr>> diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h --- a/llvm/include/llvm/ProfileData/SampleProfWriter.h +++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h @@ -168,6 +168,7 @@ {SecFuncOffsetTable, 0, 0, 0, 0}, {SecLBRProfile, 0, 0, 0, 0}, {SecProfileSymbolList, 0, 0, 0, 0}, + {SecMetadataOffsetTable, 0, 0, 0, 0}, {SecFuncMetadata, 0, 0, 0, 0}}), // CtxSplitLayout SmallVector({{SecProfSummary, 0, 0, 0, 0}, @@ -181,6 +182,7 @@ {SecFuncOffsetTable, 0, 0, 0, 0}, {SecLBRProfile, 0, 0, 0, 0}, {SecProfileSymbolList, 0, 0, 0, 0}, + {SecMetadataOffsetTable, 0, 0, 0, 0}, {SecFuncMetadata, 0, 0, 0, 0}}), }; @@ -270,6 +272,7 @@ // Functions to write various kinds of sections. std::error_code writeNameTableSection(const SampleProfileMap &ProfileMap); std::error_code writeFuncOffsetTable(); + std::error_code writeMetadataOffsetTable(); std::error_code writeProfileSymbolListSection(); SectionLayout SecLayout = DefaultLayout; @@ -284,6 +287,10 @@ // start of SecLBRProfile for each Function's Profile and will keep it // in FuncOffsetTable. uint64_t SecLBRProfileStart = 0; + // Save the start of SecFuncMetadata so we can compute the offset to the + // start of SecFuncMetadata for each Function's metadata and will keep it + // in MetadataOffsetTable. + uint64_t SecMetadataStart = 0; private: void allocSecHdrTable(); @@ -315,6 +322,9 @@ // FuncOffsetTable maps function context to its profile offset in // SecLBRProfile section. It is used to load function profile on demand. MapVector FuncOffsetTable; + // MetadataOffsetTable maps function context to its metadata offset in + // SecFuncMetadata section. It is used to load function metadata on demand. + MapVector MetadataOffsetTable; // Whether to use MD5 to represent string. bool UseMD5 = false; diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -686,6 +686,10 @@ if (std::error_code EC = readFuncOffsetTable()) return EC; break; + case SecMetadataOffsetTable: + if (std::error_code EC = readMetadataOffsetTable()) + return EC; + break; case SecFuncMetadata: { ProfileIsProbeBased = hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased); @@ -752,29 +756,47 @@ return sampleprof_error::success; } -std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() { - // Collect functions used by current module if the Reader has been - // given a module. +std::error_code SampleProfileReaderExtBinaryBase::readMetadataOffsetTable() { + MetadataOffsetTable.clear(); + + auto Size = readNumber(); + if (std::error_code EC = Size.getError()) + return EC; + + MetadataOffsetTable.reserve(*Size); + + for (uint32_t I = 0; I < *Size; ++I) { + auto FContext(readSampleContextFromTable()); + if (std::error_code EC = FContext.getError()) + return EC; + + auto Offset = readNumber(); + if (std::error_code EC = Offset.getError()) + return EC; + + MetadataOffsetTable[*FContext] = *Offset; + } + + return sampleprof_error::success; +} + +std::set SampleProfileReaderExtBinaryBase::computeReadOffsets( + DenseMap &OffsetTable, + std::vector> *OrderedOffsets) { + std::set Offsets; + // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName // which will query FunctionSamples::HasUniqSuffix, so it has to be // called after FunctionSamples::HasUniqSuffix is set, i.e. after // NameTable section is read. - bool LoadFuncsToBeUsed = collectFuncsFromModule(); + if (!collectFuncsFromModule()) + return Offsets; - // When LoadFuncsToBeUsed is false, load all the function profiles. - const uint8_t *Start = Data; - if (!LoadFuncsToBeUsed) { - while (Data < End) { - if (std::error_code EC = readFuncProfile(Data)) - return EC; + // Load function profiles on demand. + if (Remapper) { + for (auto Name : FuncsToUse) { + Remapper->insert(Name); } - assert(Data == End && "More data is read than expected"); - } else { - // Load function profiles on demand. - if (Remapper) { - for (auto Name : FuncsToUse) { - Remapper->insert(Name); - } } if (ProfileIsCS) { @@ -791,10 +813,10 @@ // as if they were walked in preorder of a context trie. While // traversing the trie, a link to the highest common ancestor node is // kept so that all of its decendants will be loaded. - assert(OrderedFuncOffsets.get() && + assert(OrderedOffsets && "func offset table should always be sorted in CS profile"); const SampleContext *CommonContext = nullptr; - for (const auto &NameOffset : *OrderedFuncOffsets) { + for (const auto &NameOffset : *OrderedOffsets) { const auto &FContext = NameOffset.first; auto FName = FContext.getName(); // For function in the current module, keep its farthest ancestor @@ -811,38 +833,49 @@ (CommonContext && CommonContext->IsPrefixOf(FContext))) { // Load profile for the current context which originated from // the common ancestor. - const uint8_t *FuncProfileAddr = Start + NameOffset.second; - assert(FuncProfileAddr < End && "out of LBRProfile section"); - if (std::error_code EC = readFuncProfile(FuncProfileAddr)) - return EC; + Offsets.insert(NameOffset.second); } } } else { if (useMD5()) { for (auto Name : FuncsToUse) { auto GUID = std::to_string(MD5Hash(Name)); - auto iter = FuncOffsetTable.find(StringRef(GUID)); - if (iter == FuncOffsetTable.end()) + auto Iter = OffsetTable.find(StringRef(GUID)); + if (Iter == OffsetTable.end()) continue; - const uint8_t *FuncProfileAddr = Start + iter->second; - assert(FuncProfileAddr < End && "out of LBRProfile section"); - if (std::error_code EC = readFuncProfile(FuncProfileAddr)) - return EC; + Offsets.insert(Iter->second); } } else { - for (auto NameOffset : FuncOffsetTable) { + for (auto NameOffset : OffsetTable) { SampleContext FContext(NameOffset.first); auto FuncName = FContext.getName(); if (!FuncsToUse.count(FuncName) && (!Remapper || !Remapper->exist(FuncName))) continue; - const uint8_t *FuncProfileAddr = Start + NameOffset.second; - assert(FuncProfileAddr < End && "out of LBRProfile section"); - if (std::error_code EC = readFuncProfile(FuncProfileAddr)) - return EC; + Offsets.insert(NameOffset.second); } } } + return Offsets; +} + +std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() { + if (!M) { + // Collect functions used by current module if the Reader has been + // given a module. + while (Data < End) { + if (std::error_code EC = readFuncProfile(Data)) + return EC; + } + assert(Data == End && "More data is read than expected"); + } else { + auto Offsets = + computeReadOffsets(FuncOffsetTable, OrderedFuncOffsets.get()); + const uint8_t *Start = Data; + for (auto Offset : Offsets) { + if (std::error_code EC = readFuncProfile(Start + Offset)) + return EC; + } Data = End; } assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) && @@ -884,8 +917,7 @@ *CompressSize); char *Buffer = Allocator.Allocate(DecompressBufSize); size_t UCSize = DecompressBufSize; - llvm::Error E = - zlib::uncompress(CompressedStrings, Buffer, UCSize); + llvm::Error E = zlib::uncompress(CompressedStrings, Buffer, UCSize); if (E) return sampleprof_error::uncompress_failed; DecompressBuf = reinterpret_cast(Buffer); @@ -1087,7 +1119,9 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute, + const uint8_t *Start, FunctionSamples *FProfile) { + Data = Start; if (Data < End) { if (ProfileIsProbeBased) { auto Checksum = readNumber(); @@ -1132,7 +1166,7 @@ *Discriminator))[std::string(FContext.get().getName())]); } if (std::error_code EC = - readFuncMetadata(ProfileHasAttribute, CalleeProfile)) + readFuncMetadata(ProfileHasAttribute, Data, CalleeProfile)) return EC; } } @@ -1143,7 +1177,12 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) { - while (Data < End) { + // Collect functions used by current module if the Reader has been + // given a module. For now, we load metadata for all functions directly for CS + // flat profile since the metadata section there is as small as the metedata + // offset section. + bool LoadFuncsToBeUsed = M && !FunctionSamples::ProfileIsCS; + auto ReadMetadata = [&]() -> std::error_code { auto FContext(readSampleContextFromTable()); if (std::error_code EC = FContext.getError()) return EC; @@ -1151,12 +1190,26 @@ auto It = Profiles.find(*FContext); if (It != Profiles.end()) FProfile = &It->second; - - if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile)) + if (std::error_code EC = + readFuncMetadata(ProfileHasAttribute, Data, FProfile)) return EC; - } + return sampleprof_error::success; + }; - assert(Data == End && "More data is read than expected"); + if (!LoadFuncsToBeUsed) { + // When LoadFuncsToBeUsed is false, load all the function profiles. + while (Data < End) + ReadMetadata(); + assert(Data == End && "More data is read than expected"); + } else { + auto Offsets = computeReadOffsets(MetadataOffsetTable, nullptr); + const uint8_t *Start = Data; + for (auto Offset : Offsets) { + Data = Start + Offset; + ReadMetadata(); + } + Data = End; + } return sampleprof_error::success; } diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -191,6 +191,29 @@ return sampleprof_error::success; } +std::error_code SampleProfileWriterExtBinaryBase::writeMetadataOffsetTable() { + auto &OS = *OutputStream; + + // Write out the table size. + encodeULEB128(MetadataOffsetTable.size(), OS); + + // Write out MetadataOffsetTable. + auto WriteItem = [&](const SampleContext &Context, uint64_t Offset) { + if (std::error_code EC = writeContextIdx(Context)) + return EC; + encodeULEB128(Offset, OS); + return (std::error_code)sampleprof_error::success; + }; + + for (const auto &Entry : MetadataOffsetTable) { + if (std::error_code EC = WriteItem(Entry.first, Entry.second)) + return EC; + } + + MetadataOffsetTable.clear(); + return sampleprof_error::success; +} + std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata( const FunctionSamples &FunctionProfile) { auto &OS = *OutputStream; @@ -229,6 +252,9 @@ !FunctionSamples::ProfileIsPreInlined) return sampleprof_error::success; for (const auto &Entry : Profiles) { + uint64_t Offset = OutputStream->tell(); + auto &Context = Entry.second.getContext(); + MetadataOffsetTable[Context] = Offset - SecMetadataStart; if (std::error_code EC = writeFuncMetadata(Entry.second)) return EC; } @@ -354,7 +380,12 @@ if (auto EC = writeFuncOffsetTable()) return EC; break; + case SecMetadataOffsetTable: + if (auto EC = writeMetadataOffsetTable()) + return EC; + break; case SecFuncMetadata: + SecMetadataStart = OutputStream->tell(); if (std::error_code EC = writeFuncMetadata(ProfileMap)) return EC; break; @@ -376,7 +407,7 @@ const SampleProfileMap &ProfileMap) { // The const indices passed to writeOneSection below are specifying the // positions of the sections in SectionHdrLayout. Look at - // initSectionHdrLayout to find out where each section is located in + // ExtBinaryHdrLayoutTable to find out where each section is located in // SectionHdrLayout. if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap)) return EC; @@ -390,7 +421,9 @@ return EC; if (auto EC = writeOneSection(SecFuncOffsetTable, 3, ProfileMap)) return EC; - if (auto EC = writeOneSection(SecFuncMetadata, 6, ProfileMap)) + if (auto EC = writeOneSection(SecFuncMetadata, 7, ProfileMap)) + return EC; + if (auto EC = writeOneSection(SecMetadataOffsetTable, 6, ProfileMap)) return EC; return sampleprof_error::success; } diff --git a/llvm/test/tools/llvm-profgen/cs-preinline.test b/llvm/test/tools/llvm-profgen/cs-preinline.test --- a/llvm/test/tools/llvm-profgen/cs-preinline.test +++ b/llvm/test/tools/llvm-profgen/cs-preinline.test @@ -81,3 +81,4 @@ ; CHECK-PREINL-FLAG: ProfileSummarySection {{.*}} Flags: {{{.*}}preInlined} +; CHECK-PREINL-FLAG: MetadtaOffsetTableSection