Index: llvm/include/llvm/ProfileData/SampleProf.h =================================================================== --- llvm/include/llvm/ProfileData/SampleProf.h +++ llvm/include/llvm/ProfileData/SampleProf.h @@ -120,6 +120,7 @@ SecProfSummary = 1, SecNameTable = 2, SecProfileSymbolList = 3, + SecFuncOffsetTable = 4, // marker for the first type of profile. SecFuncProfileFirst = 32, SecLBRProfile = SecFuncProfileFirst @@ -135,6 +136,8 @@ return "NameTableSection"; case SecProfileSymbolList: return "ProfileSymbolListSection"; + case SecFuncOffsetTable: + return "FuncOffsetTableSection"; case SecLBRProfile: return "LBRProfileSection"; } Index: llvm/include/llvm/ProfileData/SampleProfReader.h =================================================================== --- llvm/include/llvm/ProfileData/SampleProfReader.h +++ llvm/include/llvm/ProfileData/SampleProfReader.h @@ -527,6 +527,16 @@ virtual std::error_code readOneSection(const uint8_t *Start, uint64_t Size, SecType Type) override; std::error_code readProfileSymbolList(uint64_t Size); + std::error_code readFuncOffsetTable(); + std::error_code readFuncProfiles(uint64_t Size); + + /// The table mapping from function name to the offset of its FunctionSample + /// towards file start. + DenseMap FuncOffsetTable; + /// The set containing the functions to use when compiling a module. + DenseSet FuncsToUse; + /// Use all functions from the input profile. + bool UseAllFuncs = true; public: SampleProfileReaderExtBinary(std::unique_ptr B, LLVMContext &C, @@ -539,6 +549,9 @@ virtual std::unique_ptr getProfileSymbolList() override { return std::move(ProfSymList); }; + + /// Collect functions to be used when compiling Module \p M. + void collectFuncsToUse(const Module &M) override; }; class SampleProfileReaderCompactBinary : public SampleProfileReaderBinary { Index: llvm/include/llvm/ProfileData/SampleProfWriter.h =================================================================== --- llvm/include/llvm/ProfileData/SampleProfWriter.h +++ llvm/include/llvm/ProfileData/SampleProfWriter.h @@ -196,6 +196,7 @@ initSectionLayout(); } + virtual std::error_code writeSample(const FunctionSamples &S) override; virtual void setProfileSymbolList(ProfileSymbolList *PSL) override { ProfSymList = PSL; }; @@ -204,12 +205,22 @@ virtual void initSectionLayout() override { SectionLayout = {{SecProfSummary, 0, 0, 0}, {SecNameTable, 0, 0, 0}, + {SecFuncOffsetTable, 0, 0, 0}, {SecLBRProfile, 0, 0, 0}, {SecProfileSymbolList, 0, 0, 0}}; }; virtual std::error_code writeSections(const StringMap &ProfileMap) override; ProfileSymbolList *ProfSymList = nullptr; + + // Save the start of SecLBRProfile so we can compute the offset to the + // start of SecLBRProfile for each Function's Profile and will keep it + // in FuncOffsetTable. + uint64_t SecLBRProfileStart; + // FuncOffsetTable maps function name to its profile offset in SecLBRProfile + // section. It is used to load function profile on demand. + MapVector FuncOffsetTable; + std::error_code writeFuncOffsetTable(); }; // CompactBinary is a compact format of binary profile which both reduces Index: llvm/lib/ProfileData/SampleProfReader.cpp =================================================================== --- llvm/lib/ProfileData/SampleProfReader.cpp +++ llvm/lib/ProfileData/SampleProfReader.cpp @@ -483,21 +483,75 @@ return EC; break; case SecLBRProfile: - while (Data < Start + Size) { - if (std::error_code EC = readFuncProfile()) - return EC; - } + if (std::error_code EC = readFuncProfiles(Size)) + return EC; break; case SecProfileSymbolList: if (std::error_code EC = readProfileSymbolList(Size)) return EC; break; + case SecFuncOffsetTable: + if (std::error_code EC = readFuncOffsetTable()) + return EC; + break; default: break; } return sampleprof_error::success; } +void SampleProfileReaderExtBinary::collectFuncsToUse(const Module &M) { + UseAllFuncs = false; + FuncsToUse.clear(); + for (auto &F : M) { + StringRef CanonName = FunctionSamples::getCanonicalFnName(F); + FuncsToUse.insert(CanonName); + } +} + +std::error_code SampleProfileReaderExtBinary::readFuncOffsetTable() { + auto Size = readNumber(); + if (std::error_code EC = Size.getError()) + return EC; + + FuncOffsetTable.reserve(*Size); + for (uint32_t I = 0; I < *Size; ++I) { + auto FName(readStringFromTable()); + if (std::error_code EC = FName.getError()) + return EC; + + auto Offset = readNumber(); + if (std::error_code EC = Offset.getError()) + return EC; + + FuncOffsetTable[*FName] = *Offset; + } + return sampleprof_error::success; +} + +std::error_code SampleProfileReaderExtBinary::readFuncProfiles(uint64_t Size) { + const uint8_t *Start = Data; + if (UseAllFuncs) { + while (Data < Start + Size) { + if (std::error_code EC = readFuncProfile()) + return EC; + } + assert(Data == Start + Size && "More data is read than expected"); + return sampleprof_error::success; + } + + for (auto Name : FuncsToUse) { + auto iter = FuncOffsetTable.find(Name); + if (iter == FuncOffsetTable.end()) + continue; + Data = Start + iter->second; + if (std::error_code EC = readFuncProfile()) + return EC; + } + Data = Start + Size; + return sampleprof_error::success; +} + std::error_code SampleProfileReaderExtBinary::readProfileSymbolList(uint64_t Size) { if (!ProfSymList) @@ -719,8 +773,16 @@ } uint64_t SampleProfileReaderExtBinaryBase::getFileSize() { - auto &LastEntry = SecHdrTable.back(); - return LastEntry.Offset + LastEntry.Size; + // Sections in SecHdrTable is not necessarily in the same order as + // sections in the profile because section like FuncOffsetTable needs + // to be written after section LBRProfile but needs to be read before + // section LBRProfile, so we cannot simply use the last entry in + // SecHdrTable to calculate the file size. + uint64_t FileSize = 0; + for (auto &Entry : SecHdrTable) { + FileSize = std::max(Entry.Offset + Entry.Size, FileSize); + } + return FileSize; } bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) { Index: llvm/lib/ProfileData/SampleProfWriter.cpp =================================================================== --- llvm/lib/ProfileData/SampleProfWriter.cpp +++ llvm/lib/ProfileData/SampleProfWriter.cpp @@ -143,6 +143,29 @@ return sampleprof_error::success; } +std::error_code +SampleProfileWriterExtBinary::writeSample(const FunctionSamples &S) { + uint64_t Offset = OutputStream->tell(); + StringRef Name = S.getName(); + FuncOffsetTable[Name] = Offset - SecLBRProfileStart; + encodeULEB128(S.getHeadSamples(), *OutputStream); + return writeBody(S); +} + +std::error_code SampleProfileWriterExtBinary::writeFuncOffsetTable() { + auto &OS = *OutputStream; + + // Write out the table size. + encodeULEB128(FuncOffsetTable.size(), OS); + + // Write out FuncOffsetTable. + for (auto entry : FuncOffsetTable) { + writeNameIdx(entry.first); + encodeULEB128(entry.second, OS); + } + return sampleprof_error::success; +} + std::error_code SampleProfileWriterExtBinary::writeSections( const StringMap &ProfileMap) { uint64_t SectionStart = markSectionStart(SecProfSummary); @@ -163,6 +186,7 @@ return EC; SectionStart = markSectionStart(SecLBRProfile); + SecLBRProfileStart = OutputStream->tell(); if (std::error_code EC = writeFuncProfiles(ProfileMap)) return EC; if (std::error_code EC = addNewSection(SecLBRProfile, SectionStart)) @@ -178,6 +202,12 @@ if (std::error_code EC = addNewSection(SecProfileSymbolList, SectionStart)) return EC; + SectionStart = markSectionStart(SecFuncOffsetTable); + if (std::error_code EC = writeFuncOffsetTable()) + return EC; + if (std::error_code EC = addNewSection(SecFuncOffsetTable, SectionStart)) + return EC; + return sampleprof_error::success; } Index: llvm/unittests/ProfileData/SampleProfTest.cpp =================================================================== --- llvm/unittests/ProfileData/SampleProfTest.cpp +++ llvm/unittests/ProfileData/SampleProfTest.cpp @@ -86,6 +86,13 @@ BarSamples.addCalledTargetSamples(1, 0, MconstructName, 1000); BarSamples.addCalledTargetSamples(1, 0, StringviewName, 437); + StringRef BazName("_Z3bazi"); + FunctionSamples BazSamples; + BazSamples.setName(BazName); + BazSamples.addTotalSamples(12557); + BazSamples.addHeadSamples(1257); + BazSamples.addBodySamples(1, 0, 12557); + Module M("my_module", Context); FunctionType *fn_type = FunctionType::get(Type::getVoidTy(Context), {}, false); @@ -95,6 +102,7 @@ StringMap Profiles; Profiles[FooName] = std::move(FooSamples); Profiles[BarName] = std::move(BarSamples); + Profiles[BazName] = std::move(BazSamples); ProfileSymbolList List; if (Format == SampleProfileFormat::SPF_Ext_Binary) { @@ -137,8 +145,6 @@ ASSERT_TRUE(NoError(EC)); } - ASSERT_EQ(2u, Reader->getProfiles().size()); - FunctionSamples *ReadFooSamples = Reader->getSamplesFor(FooName); ASSERT_TRUE(ReadFooSamples != nullptr); if (Format != SampleProfileFormat::SPF_Compact_Binary) { @@ -158,6 +164,20 @@ ReadBarSamples->findCallTargetMapAt(1, 0); ASSERT_FALSE(CTMap.getError()); + // Because _Z3bazi is not defined in module M, expect _Z3bazi's profile + // is not loaded when the profile is ExtBinary or Compact format because + // these formats support loading function profiles on demand. + FunctionSamples *ReadBazSamples = Reader->getSamplesFor(BazName); + if (Format == SampleProfileFormat::SPF_Ext_Binary || + Format == SampleProfileFormat::SPF_Compact_Binary) { + ASSERT_TRUE(ReadBazSamples == nullptr); + ASSERT_EQ(2u, Reader->getProfiles().size()); + } else { + ASSERT_TRUE(ReadBazSamples != nullptr); + ASSERT_EQ(12557u, ReadBazSamples->getTotalSamples()); + ASSERT_EQ(3u, Reader->getProfiles().size()); + } + std::string MconstructGUID; StringRef MconstructRep = getRepInFormat(MconstructName, Format, MconstructGUID); @@ -169,9 +189,9 @@ auto VerifySummary = [](ProfileSummary &Summary) mutable { ASSERT_EQ(ProfileSummary::PSK_Sample, Summary.getKind()); - ASSERT_EQ(123603u, Summary.getTotalCount()); - ASSERT_EQ(6u, Summary.getNumCounts()); - ASSERT_EQ(2u, Summary.getNumFunctions()); + ASSERT_EQ(136160u, Summary.getTotalCount()); + ASSERT_EQ(7u, Summary.getNumCounts()); + ASSERT_EQ(3u, Summary.getNumFunctions()); ASSERT_EQ(1437u, Summary.getMaxFunctionCount()); ASSERT_EQ(60351u, Summary.getMaxCount()); @@ -188,8 +208,8 @@ Cutoff = 990000; auto NinetyNinePerc = find_if(Details, Predicate); ASSERT_EQ(60000u, EightyPerc->MinCount); - ASSERT_EQ(60000u, NinetyPerc->MinCount); - ASSERT_EQ(60000u, NinetyFivePerc->MinCount); + ASSERT_EQ(12557u, NinetyPerc->MinCount); + ASSERT_EQ(12557u, NinetyFivePerc->MinCount); ASSERT_EQ(610u, NinetyNinePerc->MinCount); };