Index: include/llvm/ProfileData/InstrProf.h =================================================================== --- include/llvm/ProfileData/InstrProf.h +++ include/llvm/ProfileData/InstrProf.h @@ -674,14 +674,6 @@ uint64_t HashOffset; }; -static const uint32_t SummaryCutoffs[] = { - 10000, /* 1% */ - 100000, /* 10% */ - 200000, 300000, 400000, 500000, 600000, 500000, 600000, 700000, - 800000, 900000, 950000, 990000, 999000, 999900, 999990, 999999}; -static const uint32_t NumSummaryCutoffs = - sizeof(SummaryCutoffs) / sizeof(*SummaryCutoffs); - // Profile summary data recorded in the profile data file in indexed // format. It is introduced in version 4. The summary data follows // right after the profile file header. Index: include/llvm/ProfileData/ProfileCommon.h =================================================================== --- include/llvm/ProfileData/ProfileCommon.h +++ include/llvm/ProfileData/ProfileCommon.h @@ -24,6 +24,9 @@ namespace IndexedInstrProf { struct Summary; } +namespace sampleprof { +class FunctionSamples; +} struct InstrProfRecord; ///// Profile summary computation //// // The 'show' command displays richer summary of the profile data. The profile @@ -44,12 +47,12 @@ // We keep track of the number of times a count appears in the profile and // keep the map sorted in the descending order of counts. std::map> CountFrequencies; - std::vector DetailedSummary; std::vector DetailedSummaryCutoffs; // Sum of all counts. uint64_t TotalCount; uint64_t MaxBlockCount, MaxInternalBlockCount, MaxFunctionCount; uint32_t NumBlocks, NumFunctions; + std::vector DetailedSummary; inline void addCount(uint64_t Count, bool IsEntry); public: @@ -58,8 +61,18 @@ : DetailedSummaryCutoffs(Cutoffs), TotalCount(0), MaxBlockCount(0), MaxInternalBlockCount(0), MaxFunctionCount(0), NumBlocks(0), NumFunctions(0) {} + ProfileSummary(uint64_t TotalCount, uint64_t MaxBlockCount, + uint64_t MaxInternalBlockCount, uint64_t MaxFunctionCount, + int32_t NumBlocks, uint32_t NumFunctions, + std::vector DetailedSummary) + : TotalCount(TotalCount), MaxBlockCount(MaxBlockCount), + MaxInternalBlockCount(MaxInternalBlockCount), + MaxFunctionCount(MaxFunctionCount), NumBlocks(NumBlocks), + NumFunctions(NumFunctions), DetailedSummary(DetailedSummary) {} + ProfileSummary(const IndexedInstrProf::Summary &S); void addRecord(const InstrProfRecord &); + void addRecord(const sampleprof::FunctionSamples &FS); inline std::vector &getDetailedSummary(); void computeDetailedSummary(); uint32_t getNumBlocks() { return NumBlocks; } @@ -68,6 +81,8 @@ uint64_t getMaxFunctionCount() { return MaxFunctionCount; } uint64_t getMaxBlockCount() { return MaxBlockCount; } uint64_t getMaxInternalBlockCount() { return MaxInternalBlockCount; } + /// \brief A vector of useful cutoff values for detailed summary. + static const std::vector DefaultCutoffs; }; // This is called when a count is seen in the profile. @@ -87,5 +102,6 @@ return DetailedSummary; } + } // end namespace llvm #endif Index: include/llvm/ProfileData/SampleProf.h =================================================================== --- include/llvm/ProfileData/SampleProf.h +++ include/llvm/ProfileData/SampleProf.h @@ -74,7 +74,7 @@ uint64_t('2') << (64 - 56) | uint64_t(0xff); } -static inline uint64_t SPVersion() { return 102; } +static inline uint64_t SPVersion() { return 103; } /// Represents the relative location of an instruction. /// Index: include/llvm/ProfileData/SampleProfReader.h =================================================================== --- include/llvm/ProfileData/SampleProfReader.h +++ include/llvm/ProfileData/SampleProfReader.h @@ -129,6 +129,32 @@ // VERSION (uint32_t) // File format version number computed by SPVersion() // +// SUMMARY +// NOTE: The summary is common to instrumented and sample profiling and +// the terminology used is based on instrumented profiles. Wherever +// the summary refers to a block, we use a line instead. +// TOTAL_COUNT (uint64_t) +// Total number of samples in the profile. +// MAX_BLOCK_COUNT (uint64_t) +// Maximum number of samples on a line. +// MAX_FUNCTION_COUNT (uint64_t) +// Maximum number of head samples. +// NUM_BLOCKS (uint64_t) +// Number of lines with samples. +// NUM_FUNCTIONS (uint64_t) +// Number of functions with samples. +// NUM_DETAILED_SUMMARY_ENTRIES (size_t) +// Number of entries in detailed summary +// DETAILED_SUMMARY +// A list of detailed summary entry. Each entry consists of +// CUTOFF (uint32_t) +// Required percentile of total sample count expressed as a fraction +// multiplied by 1000000 +// MIN_BLOCK_COUNT (uint64_t) +// The minimum number of samples for the CUTOFF percentile. +// NUM_BLOCKS (uint64_t) +// Number of samples to get to the desrired percentile. +// // NAME TABLE // SIZE (uint32_t) // Number of entries in the name table. @@ -190,6 +216,7 @@ #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Function.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/ProfileData/ProfileCommon.h" #include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" @@ -270,6 +297,9 @@ static ErrorOr> create(std::unique_ptr &B, LLVMContext &C); + /// \brief Return the profile summary. + ProfileSummary &getSummary() { return *(Summary.get()); } + protected: /// \brief Map every function to its associated profile. /// @@ -283,6 +313,9 @@ /// \brief Memory buffer holding the profile file. std::unique_ptr Buffer; + + /// \brief Profile summary information. + std::unique_ptr Summary; }; class SampleProfileReaderText : public SampleProfileReader { @@ -298,6 +331,10 @@ /// \brief Return true if \p Buffer is in the format supported by this class. static bool hasFormat(const MemoryBuffer &Buffer); + +private: + /// \brief Compute summary for this profile. + void computeSummary(); }; class SampleProfileReaderBinary : public SampleProfileReader { @@ -348,6 +385,12 @@ /// Function name table. std::vector NameTable; + +private: + std::error_code readSummaryEntry(std::vector &Entries); + + /// \brief Read profile summary. + std::error_code readSummary(); }; typedef SmallVector InlineCallStack; Index: include/llvm/ProfileData/SampleProfWriter.h =================================================================== --- include/llvm/ProfileData/SampleProfWriter.h +++ include/llvm/ProfileData/SampleProfWriter.h @@ -15,6 +15,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ProfileData/ProfileCommon.h" #include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/FileSystem.h" @@ -42,7 +43,6 @@ std::error_code write(const StringMap &ProfileMap) { if (std::error_code EC = writeHeader(ProfileMap)) return EC; - for (const auto &I : ProfileMap) { StringRef FName = I.first(); const FunctionSamples &Profile = I.second; @@ -75,6 +75,12 @@ /// \brief Output stream where to emit the profile to. std::unique_ptr OutputStream; + + /// \brief Profile summary. + std::unique_ptr Summary; + + /// \brief Compute summary for this profile. + void computeSummary(const StringMap &ProfileMap); }; /// \brief Sample-based profile writer (text format). @@ -113,6 +119,7 @@ std::error_code writeHeader(const StringMap &ProfileMap) override; + std::error_code writeSummary(); std::error_code writeNameIdx(StringRef FName); std::error_code writeBody(StringRef FName, const FunctionSamples &S); Index: lib/ProfileData/InstrProfReader.cpp =================================================================== --- lib/ProfileData/InstrProfReader.cpp +++ lib/ProfileData/InstrProfReader.cpp @@ -595,9 +595,8 @@ } else { // For older version of profile data, we need to compute on the fly: using namespace IndexedInstrProf; - std::vector Cutoffs(&SummaryCutoffs[0], - &SummaryCutoffs[NumSummaryCutoffs]); - this->Summary = llvm::make_unique(Cutoffs); + this->Summary = + llvm::make_unique(ProfileSummary::DefaultCutoffs); this->Summary->computeDetailedSummary(); return Cur; } Index: lib/ProfileData/InstrProfWriter.cpp =================================================================== --- lib/ProfileData/InstrProfWriter.cpp +++ lib/ProfileData/InstrProfWriter.cpp @@ -217,9 +217,7 @@ OnDiskChainedHashTableGenerator Generator; using namespace IndexedInstrProf; - std::vector Cutoffs(&SummaryCutoffs[0], - &SummaryCutoffs[NumSummaryCutoffs]); - ProfileSummary PS(Cutoffs); + ProfileSummary PS(ProfileSummary::DefaultCutoffs); InfoObj->TheProfileSummary = &PS; // Populate the hash table generator. @@ -249,7 +247,7 @@ OS.write(0); // Reserve space to write profile summary data. - uint32_t NumEntries = Cutoffs.size(); + uint32_t NumEntries = ProfileSummary::DefaultCutoffs.size(); uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries); // Remember the summary offset. uint64_t SummaryOffset = OS.tell(); Index: lib/ProfileData/ProfileSummary.cpp =================================================================== --- lib/ProfileData/ProfileSummary.cpp +++ lib/ProfileData/ProfileSummary.cpp @@ -11,11 +11,18 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ProfileData/ProfileCommon.h" #include "llvm/ProfileData/InstrProf.h" +#include "llvm/ProfileData/ProfileCommon.h" +#include "llvm/ProfileData/SampleProf.h" using namespace llvm; +const std::vector ProfileSummary::DefaultCutoffs( + {10000, /* 1% */ + 100000, /* 10% */ + 200000, 300000, 400000, 500000, 600000, 500000, 600000, 700000, 800000, + 900000, 950000, 990000, 999000, 999900, 999990, 999999}); + void ProfileSummary::addRecord(const InstrProfRecord &R) { NumFunctions++; if (R.Counts[0] > MaxFunctionCount) @@ -25,6 +32,22 @@ addCount(R.Counts[I], (I == 0)); } +// To compute the detailed summary, we consider each line containing samples as +// equivalent to a block with a count in the instrumented profile. +void ProfileSummary::addRecord(const sampleprof::FunctionSamples &FS) { + NumFunctions++; + if (FS.getHeadSamples() > MaxFunctionCount) + MaxFunctionCount = FS.getHeadSamples(); + for (const auto &I : FS.getBodySamples()) { + const sampleprof::SampleRecord &Rec = I.second; + // addCount has a second parameter that tells whether the count (or + // samples) is for the function entry. This is used to keep track of + // internal block count. This is not meaningful for sample profile, so + // we just pass false for the second parameter. + addCount(Rec.getSamples(), false); + } +} + // The argument to this method is a vector of cutoff percentages and the return // value is a vector of (Cutoff, MinBlockCount, NumBlocks) triplets. void ProfileSummary::computeDetailedSummary() { Index: lib/ProfileData/SampleProfReader.cpp =================================================================== --- lib/ProfileData/SampleProfReader.cpp +++ lib/ProfileData/SampleProfReader.cpp @@ -22,6 +22,7 @@ #include "llvm/ProfileData/SampleProfReader.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorOr.h" @@ -220,10 +221,21 @@ } } } + if (Result == sampleprof_error::success) + computeSummary(); return Result; } +void SampleProfileReaderText::computeSummary() { + Summary.reset(new ProfileSummary(ProfileSummary::DefaultCutoffs)); + for (const auto &I : Profiles) { + const FunctionSamples &Profile = I.second; + Summary->addRecord(Profile); + } + Summary->computeDetailedSummary(); +} + bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) { bool result = false; @@ -400,6 +412,9 @@ else if (*Version != SPVersion()) return sampleprof_error::unsupported_version; + if (std::error_code EC = readSummary()) + return EC; + // Read the name table. auto Size = readNumber(); if (std::error_code EC = Size.getError()) @@ -415,6 +430,62 @@ return sampleprof_error::success; } +std::error_code SampleProfileReaderBinary::readSummaryEntry( + std::vector &Entries) { + auto Cutoff = readNumber(); + if (std::error_code EC = Cutoff.getError()) + return EC; + + auto MinBlockCount = readNumber(); + if (std::error_code EC = MinBlockCount.getError()) + return EC; + + auto NumBlocks = readNumber(); + if (std::error_code EC = NumBlocks.getError()) + return EC; + + Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks); + return sampleprof_error::success; +} + +std::error_code SampleProfileReaderBinary::readSummary() { + auto TotalCount = readNumber(); + if (std::error_code EC = TotalCount.getError()) + return EC; + + auto MaxBlockCount = readNumber(); + if (std::error_code EC = MaxBlockCount.getError()) + return EC; + + auto MaxFunctionCount = readNumber(); + if (std::error_code EC = MaxFunctionCount.getError()) + return EC; + + auto NumBlocks = readNumber(); + if (std::error_code EC = NumBlocks.getError()) + return EC; + + auto NumFunctions = readNumber(); + if (std::error_code EC = NumFunctions.getError()) + return EC; + + auto NumSummaryEntries = readNumber(); + if (std::error_code EC = NumSummaryEntries.getError()) + return EC; + + std::vector Entries; + for (unsigned i = 0; i < *NumSummaryEntries; i++) { + std::error_code EC = readSummaryEntry(Entries); + if (EC != sampleprof_error::success) + return EC; + } + Summary = llvm::make_unique(*TotalCount, *MaxBlockCount, 0, + *MaxFunctionCount, *NumBlocks, + *NumFunctions, Entries); + + return sampleprof_error::success; +} + bool SampleProfileReaderBinary::hasFormat(const MemoryBuffer &Buffer) { const uint8_t *Data = reinterpret_cast(Buffer.getBufferStart()); Index: lib/ProfileData/SampleProfWriter.cpp =================================================================== --- lib/ProfileData/SampleProfWriter.cpp +++ lib/ProfileData/SampleProfWriter.cpp @@ -120,6 +120,9 @@ encodeULEB128(SPMagic(), OS); encodeULEB128(SPVersion(), OS); + computeSummary(ProfileMap); + if (auto EC = writeSummary()) + return EC; // Generate the name table for all the functions referenced in the profile. for (const auto &I : ProfileMap) { addName(I.first()); @@ -132,10 +135,25 @@ OS << N.first; encodeULEB128(0, OS); } - return sampleprof_error::success; } +std::error_code SampleProfileWriterBinary::writeSummary() { + auto &OS = *OutputStream; + encodeULEB128(Summary->getTotalCount(), OS); + encodeULEB128(Summary->getMaxBlockCount(), OS); + encodeULEB128(Summary->getMaxFunctionCount(), OS); + encodeULEB128(Summary->getNumBlocks(), OS); + encodeULEB128(Summary->getNumFunctions(), OS); + std::vector &Entries = Summary->getDetailedSummary(); + encodeULEB128(Entries.size(), OS); + for (auto Entry : Entries) { + encodeULEB128(Entry.Cutoff, OS); + encodeULEB128(Entry.MinBlockCount, OS); + encodeULEB128(Entry.NumBlocks, OS); + } + return sampleprof_error::success; +} std::error_code SampleProfileWriterBinary::writeBody(StringRef FName, const FunctionSamples &S) { auto &OS = *OutputStream; @@ -238,3 +256,13 @@ return std::move(Writer); } + +void SampleProfileWriter::computeSummary( + const StringMap &ProfileMap) { + Summary.reset(new ProfileSummary(ProfileSummary::DefaultCutoffs)); + for (const auto &I : ProfileMap) { + const FunctionSamples &Profile = I.second; + Summary->addRecord(Profile); + } + Summary->computeDetailedSummary(); +} Index: unittests/ProfileData/SampleProfTest.cpp =================================================================== --- unittests/ProfileData/SampleProfTest.cpp +++ unittests/ProfileData/SampleProfTest.cpp @@ -55,6 +55,10 @@ FooSamples.addTotalSamples(7711); FooSamples.addHeadSamples(610); FooSamples.addBodySamples(1, 0, 610); + FooSamples.addBodySamples(2, 0, 600); + FooSamples.addBodySamples(4, 0, 60000); + FooSamples.addBodySamples(8, 0, 60351); + FooSamples.addBodySamples(10, 0, 605); StringRef BarName("_Z3bari"); FunctionSamples BarSamples; @@ -88,6 +92,32 @@ FunctionSamples &ReadBarSamples = ReadProfiles[BarName]; ASSERT_EQ(20301u, ReadBarSamples.getTotalSamples()); ASSERT_EQ(1437u, ReadBarSamples.getHeadSamples()); + + ProfileSummary &Summary = Reader->getSummary(); + ASSERT_EQ(123603u, Summary.getTotalCount()); + ASSERT_EQ(6u, Summary.getNumBlocks()); + ASSERT_EQ(2u, Summary.getNumFunctions()); + ASSERT_EQ(1437u, Summary.getMaxFunctionCount()); + ASSERT_EQ(60351u, Summary.getMaxBlockCount()); + + std::vector &Details = Summary.getDetailedSummary(); + uint32_t Cutoff = 800000; + auto Predicate = [&Cutoff](const ProfileSummaryEntry &PE) { + return PE.Cutoff == Cutoff; + }; + auto EightyPerc = std::find_if(Details.begin(), Details.end(), Predicate); + Cutoff = 900000; + auto NinetyPerc = std::find_if(Details.begin(), Details.end(), Predicate); + Cutoff = 950000; + auto NinetyFivePerc = + std::find_if(Details.begin(), Details.end(), Predicate); + Cutoff = 990000; + auto NinetyNinePerc = + std::find_if(Details.begin(), Details.end(), Predicate); + ASSERT_EQ(60000u, EightyPerc->MinBlockCount); + ASSERT_EQ(60000u, NinetyPerc->MinBlockCount); + ASSERT_EQ(60000u, NinetyFivePerc->MinBlockCount); + ASSERT_EQ(610u, NinetyNinePerc->MinBlockCount); } };