diff --git a/llvm/include/llvm/ProfileData/ProfileCommon.h b/llvm/include/llvm/ProfileData/ProfileCommon.h --- a/llvm/include/llvm/ProfileData/ProfileCommon.h +++ b/llvm/include/llvm/ProfileData/ProfileCommon.h @@ -91,6 +91,8 @@ bool isCallsiteSample = false); std::unique_ptr computeSummaryForProfiles(const sampleprof::SampleProfileMap &Profiles); + std::unique_ptr + computeSummaryForProfiles(const ArrayRef &Profiles); std::unique_ptr getSummary(); }; diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -1559,6 +1559,22 @@ } } + static void flattenProfile(const ArrayRef &InputProfiles, + SampleProfileMap &OutputProfiles, + bool ProfileIsCS = false) { + if (ProfileIsCS) { + for (const auto &I : InputProfiles) { + // Retain the profile name and clear the full context for each function + // profile. + FunctionSamples &FS = OutputProfiles.Create(I.second->getName()); + FS.merge(*I.second); + } + } else { + for (const auto &I : InputProfiles) + flattenNestedProfile(OutputProfiles, *I.second); + } + } + private: static void flattenNestedProfile(SampleProfileMap &OutputProfiles, const FunctionSamples &FS) { diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h --- a/llvm/include/llvm/ProfileData/SampleProfWriter.h +++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h @@ -88,6 +88,8 @@ /// Sample-based profile writer. Base class. class SampleProfileWriter { public: + using FunctionSamplesVector = const llvm::ArrayRef; + virtual ~SampleProfileWriter() = default; /// Write sample profiles in \p S. @@ -98,7 +100,13 @@ /// Write all the sample profiles in the given map of samples. /// /// \returns status code of the file update operation. - virtual std::error_code write(const SampleProfileMap &ProfileMap); + std::error_code write(const SampleProfileMap &ProfileMap) { + std::vector SortedProfiles; + sortFuncProfiles(ProfileMap, SortedProfiles); + return write(SortedProfiles); + } + + virtual std::error_code write(FunctionSamplesVector &ProfileMap); /// Write sample profiles up to given size limit, using the pruning strategy /// to drop some functions if necessary. @@ -135,10 +143,10 @@ : OutputStream(std::move(OS)) {} /// Write a file header for the profile file. - virtual std::error_code writeHeader(const SampleProfileMap &ProfileMap) = 0; + virtual std::error_code writeHeader(FunctionSamplesVector &ProfileMap) = 0; // Write function profiles to the profile file. - virtual std::error_code writeFuncProfiles(const SampleProfileMap &ProfileMap); + virtual std::error_code writeFuncProfiles(FunctionSamplesVector &ProfileMap); std::error_code writeWithSizeLimitInternal(SampleProfileMap &ProfileMap, size_t OutputSizeLimit, @@ -156,7 +164,7 @@ std::unique_ptr Summary; /// Compute summary for this profile. - void computeSummary(const SampleProfileMap &ProfileMap); + void computeSummary(FunctionSamplesVector &ProfileMap); /// Profile format. SampleProfileFormat Format = SPF_None; @@ -171,7 +179,7 @@ SampleProfileWriterText(std::unique_ptr &OS) : SampleProfileWriter(OS), Indent(0) {} - std::error_code writeHeader(const SampleProfileMap &ProfileMap) override { + std::error_code writeHeader(FunctionSamplesVector &ProfileMap) override { LineCount = 0; return sampleprof_error::success; } @@ -199,7 +207,7 @@ virtual MapVector &getNameTable() { return NameTable; } virtual std::error_code writeMagicIdent(SampleProfileFormat Format); virtual std::error_code writeNameTable(); - std::error_code writeHeader(const SampleProfileMap &ProfileMap) override; + std::error_code writeHeader(FunctionSamplesVector &ProfileMap) override; std::error_code writeSummary(); virtual std::error_code writeContextIdx(const SampleContext &Context); std::error_code writeNameIdx(StringRef FName); @@ -257,7 +265,7 @@ class SampleProfileWriterExtBinaryBase : public SampleProfileWriterBinary { using SampleProfileWriterBinary::SampleProfileWriterBinary; public: - std::error_code write(const SampleProfileMap &ProfileMap) override; + std::error_code write(FunctionSamplesVector &ProfileMap) override; void setToCompressAllSections() override; void setToCompressSection(SecType Type); @@ -320,12 +328,12 @@ virtual void verifySecLayout(SectionLayout SL) = 0; // specify the order to write sections. - virtual std::error_code writeSections(const SampleProfileMap &ProfileMap) = 0; + virtual std::error_code writeSections(FunctionSamplesVector &ProfileMap) = 0; // Dispatch section writer for each section. \p LayoutIdx is the sequence // number indicating where the section is located in SectionHdrLayout. virtual std::error_code writeOneSection(SecType Type, uint32_t LayoutIdx, - const SampleProfileMap &ProfileMap); + FunctionSamplesVector &ProfileMap); // Helper function to write name table. std::error_code writeNameTable() override; @@ -333,11 +341,11 @@ std::error_code writeCSNameIdx(const SampleContext &Context); std::error_code writeCSNameTableSection(); - std::error_code writeFuncMetadata(const SampleProfileMap &Profiles); + std::error_code writeFuncMetadata(FunctionSamplesVector &Profiles); std::error_code writeFuncMetadata(const FunctionSamples &Profile); // Functions to write various kinds of sections. - std::error_code writeNameTableSection(const SampleProfileMap &ProfileMap); + std::error_code writeNameTableSection(FunctionSamplesVector &ProfileMap); std::error_code writeFuncOffsetTable(); std::error_code writeProfileSymbolListSection(); @@ -357,7 +365,7 @@ private: void allocSecHdrTable(); std::error_code writeSecHdrTable(); - std::error_code writeHeader(const SampleProfileMap &ProfileMap) override; + std::error_code writeHeader(FunctionSamplesVector &ProfileMap) override; std::error_code compressAndOutput(); // We will swap the raw_ostream held by LocalBufStream and that @@ -399,10 +407,10 @@ : SampleProfileWriterExtBinaryBase(OS) {} private: - std::error_code writeDefaultLayout(const SampleProfileMap &ProfileMap); - std::error_code writeCtxSplitLayout(const SampleProfileMap &ProfileMap); + std::error_code writeDefaultLayout(FunctionSamplesVector &ProfileMap); + std::error_code writeCtxSplitLayout(FunctionSamplesVector &ProfileMap); - std::error_code writeSections(const SampleProfileMap &ProfileMap) override; + std::error_code writeSections(FunctionSamplesVector &ProfileMap) override; std::error_code writeCustomSection(SecType Type) override { return sampleprof_error::success; diff --git a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp --- a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp +++ b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp @@ -216,6 +216,26 @@ return getSummary(); } +/// Overload computeSummaryForProfiles to accept pointer to FunctionSamples. +std::unique_ptr +SampleProfileSummaryBuilder::computeSummaryForProfiles( + const ArrayRef &Profiles) { + assert(NumFunctions == 0 && + "This can only be called on an empty summary builder"); + sampleprof::SampleProfileMap ContextLessProfiles; + if (UseContextLessSummary || (sampleprof::FunctionSamples::ProfileIsCS && + !UseContextLessSummary.getNumOccurrences())) { + ProfileConverter::flattenProfile(Profiles, ContextLessProfiles, true); + for (const auto &I : ContextLessProfiles) + addRecord(I.second); + } else { + for (const auto &I : Profiles) + addRecord(*I.second); + } + + return getSummary(); +} + std::unique_ptr InstrProfSummaryBuilder::getSummary() { computeDetailedSummary(); return std::make_unique( diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -142,17 +142,15 @@ } std::error_code -SampleProfileWriter::writeFuncProfiles(const SampleProfileMap &ProfileMap) { - std::vector V; - sortFuncProfiles(ProfileMap, V); - for (const auto &I : V) { +SampleProfileWriter::writeFuncProfiles(FunctionSamplesVector &ProfileMap) { + for (const auto &I : ProfileMap) { if (std::error_code EC = writeSample(*I.second)) return EC; } return sampleprof_error::success; } -std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) { +std::error_code SampleProfileWriter::write(FunctionSamplesVector &ProfileMap) { if (std::error_code EC = writeHeader(ProfileMap)) return EC; @@ -216,7 +214,7 @@ } std::error_code -SampleProfileWriterExtBinaryBase::write(const SampleProfileMap &ProfileMap) { +SampleProfileWriterExtBinaryBase::write(FunctionSamplesVector &ProfileMap) { // When calling write on a different profile map, existing states should be // cleared. NameTable.clear(); @@ -332,12 +330,12 @@ } std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata( - const SampleProfileMap &Profiles) { + FunctionSamplesVector &Profiles) { if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS && !FunctionSamples::ProfileIsPreInlined) return sampleprof_error::success; for (const auto &Entry : Profiles) { - if (std::error_code EC = writeFuncMetadata(Entry.second)) + if (std::error_code EC = writeFuncMetadata(*Entry.second)) return EC; } return sampleprof_error::success; @@ -362,10 +360,10 @@ } std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection( - const SampleProfileMap &ProfileMap) { + FunctionSamplesVector &ProfileMap) { for (const auto &I : ProfileMap) { - addContext(I.second.getContext()); - addNames(I.second); + addContext(I.second->getContext()); + addNames(*I.second); } // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag @@ -421,7 +419,7 @@ } std::error_code SampleProfileWriterExtBinaryBase::writeOneSection( - SecType Type, uint32_t LayoutIdx, const SampleProfileMap &ProfileMap) { + SecType Type, uint32_t LayoutIdx, FunctionSamplesVector &ProfileMap) { // The setting of SecFlagCompress should happen before markSectionStart. if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress()) setToCompressSection(SecProfileSymbolList); @@ -480,7 +478,7 @@ } std::error_code SampleProfileWriterExtBinary::writeDefaultLayout( - const SampleProfileMap &ProfileMap) { + FunctionSamplesVector &ProfileMap) { // The const indices passed to writeOneSection below are specifying the // positions of the sections in SectionHdrLayout. Look at // initSectionHdrLayout to find out where each section is located in @@ -502,20 +500,25 @@ return sampleprof_error::success; } -static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap, - SampleProfileMap &ContextProfileMap, - SampleProfileMap &NoContextProfileMap) { +static void +splitProfileMapToTwo(SampleProfileWriter::FunctionSamplesVector &ProfileMap, + std::vector &ContextProfileMap, + std::vector &NoContextProfileMap) { + // ProfileMap is already sorted by total sample count, so splitting it into + // two still preserves the order. for (const auto &I : ProfileMap) { - if (I.second.getCallsiteSamples().size()) - ContextProfileMap.insert({I.first, I.second}); + if (I.second->getCallsiteSamples().size()) + ContextProfileMap.emplace_back(I.first, I.second); else - NoContextProfileMap.insert({I.first, I.second}); + NoContextProfileMap.emplace_back(I.first, I.second); } } std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout( - const SampleProfileMap &ProfileMap) { - SampleProfileMap ContextProfileMap, NoContextProfileMap; + FunctionSamplesVector &ProfileMap) { + std::vector ContextProfileMap, NoContextProfileMap; + ContextProfileMap.reserve(ProfileMap.size()); + NoContextProfileMap.reserve(ProfileMap.size()); splitProfileMapToTwo(ProfileMap, ContextProfileMap, NoContextProfileMap); if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap)) @@ -545,7 +548,7 @@ } std::error_code SampleProfileWriterExtBinary::writeSections( - const SampleProfileMap &ProfileMap) { + FunctionSamplesVector &ProfileMap) { std::error_code EC; if (SecLayout == DefaultLayout) EC = writeDefaultLayout(ProfileMap); @@ -712,7 +715,7 @@ } std::error_code -SampleProfileWriterBinary::writeHeader(const SampleProfileMap &ProfileMap) { +SampleProfileWriterBinary::writeHeader(FunctionSamplesVector &ProfileMap) { // When calling write on a different profile map, existing names should be // cleared. NameTable.clear(); @@ -725,8 +728,8 @@ // Generate the name table for all the functions referenced in the profile. for (const auto &I : ProfileMap) { - addContext(I.second.getContext()); - addNames(I.second); + addContext(I.second->getContext()); + addNames(*I.second); } writeNameTable(); @@ -793,7 +796,7 @@ } std::error_code SampleProfileWriterExtBinaryBase::writeHeader( - const SampleProfileMap &ProfileMap) { + FunctionSamplesVector &ProfileMap) { auto &OS = *OutputStream; FileStart = OS.tell(); writeMagicIdent(Format); @@ -928,7 +931,7 @@ return std::move(Writer); } -void SampleProfileWriter::computeSummary(const SampleProfileMap &ProfileMap) { +void SampleProfileWriter::computeSummary(FunctionSamplesVector &ProfileMap) { SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); Summary = Builder.computeSummaryForProfiles(ProfileMap); }