Index: include/llvm/ProfileData/InstrProf.h =================================================================== --- include/llvm/ProfileData/InstrProf.h +++ include/llvm/ProfileData/InstrProf.h @@ -750,8 +750,8 @@ void setEntry(uint32_t I, const ProfileSummaryEntry &E) { Entry &ER = getCutoffEntryBase()[I]; ER.Cutoff = E.Cutoff; - ER.MinBlockCount = E.MinBlockCount; - ER.NumBlocks = E.NumBlocks; + ER.MinBlockCount = E.MinWeight; + ER.NumBlocks = E.NumWeights; } Summary(uint32_t Size) { memset(this, 0, Size); } Index: include/llvm/ProfileData/InstrProfReader.h =================================================================== --- include/llvm/ProfileData/InstrProfReader.h +++ include/llvm/ProfileData/InstrProfReader.h @@ -353,7 +353,7 @@ /// The index into the profile data. std::unique_ptr Index; /// Profile summary data. - std::unique_ptr Summary; + std::unique_ptr Summary; IndexedInstrProfReader(const IndexedInstrProfReader &) = delete; IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete; @@ -406,7 +406,7 @@ // to be used by llvm-profdata (for dumping). Avoid using this when // the client is the compiler. InstrProfSymtab &getSymtab() override; - ProfileSummary &getSummary() { return *(Summary.get()); } + InstrumentedProfileSummary &getSummary() { return *(Summary.get()); } }; } // end namespace llvm Index: include/llvm/ProfileData/ProfileCommon.h =================================================================== --- include/llvm/ProfileData/ProfileCommon.h +++ include/llvm/ProfileData/ProfileCommon.h @@ -25,60 +25,70 @@ struct Summary; } struct InstrProfRecord; -///// Profile summary computation //// -// The 'show' command displays richer summary of the profile data. The profile -// summary is one or more (Cutoff, MinBlockCount, NumBlocks) triplets. Given a -// target execution count percentile, we compute the minimum number of blocks -// needed to reach this target and the minimum execution count of these blocks. +// The profile summary is one or more (Cutoff, MinWeight, NumWeights) triplets. +// The semantics of weights depend on the type of profile. For instrumentation +// profile, weights are block counts and for sample profile, weights are +// per-line samples. Given a target weights percentile, we compute the minimum +// number of weights needed to reach this target and the minimum among these +// weights. struct ProfileSummaryEntry { - uint32_t Cutoff; ///< The required percentile of total execution count. - uint64_t MinBlockCount; ///< The minimum execution count for this percentile. - uint64_t NumBlocks; ///< Number of blocks >= the minumum execution count. - ProfileSummaryEntry(uint32_t TheCutoff, uint64_t TheMinBlockCount, - uint64_t TheNumBlocks) - : Cutoff(TheCutoff), MinBlockCount(TheMinBlockCount), - NumBlocks(TheNumBlocks) {} + uint32_t Cutoff; ///< The required percentile of weights. + uint64_t MinWeight; ///< The minimum weight for this percentile. + uint64_t NumWeights; ///< Number of weights >= the minimum weight. + ProfileSummaryEntry(uint32_t TheCutoff, uint64_t TheMinWeight, + uint64_t TheNumWeights) + : Cutoff(TheCutoff), MinWeight(TheMinWeight), NumWeights(TheNumWeights) {} }; class ProfileSummary { - // We keep track of the number of times a count appears in the profile and - // keep the map sorted in the descending order of counts. - std::map> CountFrequencies; + // We keep track of the number of times a weight (count or samples) appears + // in the profile. The map is kept sorted in the descending order of weights. + std::map> WeightFrequencies; + +protected: std::vector DetailedSummary; std::vector DetailedSummaryCutoffs; - // Sum of all counts. - uint64_t TotalCount; - uint64_t MaxBlockCount, MaxInternalBlockCount, MaxFunctionCount; - uint32_t NumBlocks, NumFunctions; - inline void addCount(uint64_t Count, bool IsEntry); + uint64_t TotalWeight, MaxWeight; + uint32_t NumWeights; + ProfileSummary(std::vector Cutoffs) + : DetailedSummaryCutoffs(Cutoffs), TotalWeight(0), MaxWeight(0), + NumWeights(0) {} + ProfileSummary() : TotalWeight(0), MaxWeight(0), NumWeights(0) {} + inline void addWeight(uint64_t Weight); public: static const int Scale = 1000000; - ProfileSummary(std::vector Cutoffs) - : DetailedSummaryCutoffs(Cutoffs), TotalCount(0), MaxBlockCount(0), - MaxInternalBlockCount(0), MaxFunctionCount(0), NumBlocks(0), - NumFunctions(0) {} - ProfileSummary(const IndexedInstrProf::Summary &S); - void addRecord(const InstrProfRecord &); inline std::vector &getDetailedSummary(); void computeDetailedSummary(); - uint32_t getNumBlocks() { return NumBlocks; } - uint64_t getTotalCount() { return TotalCount; } +}; + +class InstrumentedProfileSummary : public ProfileSummary { + uint64_t MaxInternalBlockCount, MaxFunctionCount; + uint32_t NumFunctions; + inline void addEntryCount(uint64_t Count); + inline void addInternalCount(uint64_t Count); + +public: + InstrumentedProfileSummary(std::vector Cutoffs) + : ProfileSummary(Cutoffs), MaxInternalBlockCount(0), MaxFunctionCount(0), + NumFunctions(0) {} + InstrumentedProfileSummary(const IndexedInstrProf::Summary &S); + void addRecord(const InstrProfRecord &); + uint32_t getNumBlocks() { return NumWeights; } + uint64_t getTotalCount() { return TotalWeight; } uint32_t getNumFunctions() { return NumFunctions; } uint64_t getMaxFunctionCount() { return MaxFunctionCount; } - uint64_t getMaxBlockCount() { return MaxBlockCount; } + uint64_t getMaxBlockCount() { return MaxWeight; } uint64_t getMaxInternalBlockCount() { return MaxInternalBlockCount; } }; // This is called when a count is seen in the profile. -void ProfileSummary::addCount(uint64_t Count, bool IsEntry) { - TotalCount += Count; - if (Count > MaxBlockCount) - MaxBlockCount = Count; - if (!IsEntry && Count > MaxInternalBlockCount) - MaxInternalBlockCount = Count; - NumBlocks++; - CountFrequencies[Count]++; +void ProfileSummary::addWeight(uint64_t Weight) { + TotalWeight += Weight; + if (Weight > MaxWeight) + MaxWeight = Weight; + NumWeights++; + WeightFrequencies[Weight]++; } std::vector &ProfileSummary::getDetailedSummary() { Index: lib/ProfileData/InstrProfReader.cpp =================================================================== --- lib/ProfileData/InstrProfReader.cpp +++ lib/ProfileData/InstrProfReader.cpp @@ -589,15 +589,16 @@ for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++) Dst[I] = endian::byte_swap(Src[I]); - // initialize ProfileSummary using the SummaryData from disk. - this->Summary = llvm::make_unique(*(SummaryData.get())); + // initialize InstrumentedProfileSummary using the SummaryData from disk. + this->Summary = + llvm::make_unique(*(SummaryData.get())); return Cur + SummarySize; } else { // For older version of profile data, we need to compute on the fly: using namespace IndexedInstrProf; std::vector Cutoffs(&SummaryCutoffs[0], &SummaryCutoffs[NumSummaryCutoffs]); - this->Summary = llvm::make_unique(Cutoffs); + this->Summary = llvm::make_unique(Cutoffs); this->Summary->computeDetailedSummary(); return Cur; } Index: lib/ProfileData/InstrProfWriter.cpp =================================================================== --- lib/ProfileData/InstrProfWriter.cpp +++ lib/ProfileData/InstrProfWriter.cpp @@ -84,7 +84,7 @@ typedef uint64_t offset_type; support::endianness ValueProfDataEndianness; - ProfileSummary *TheProfileSummary; + InstrumentedProfileSummary *TheProfileSummary; InstrProfRecordWriterTrait() : ValueProfDataEndianness(support::little) {} static hash_value_type ComputeHash(key_type_ref K) { @@ -197,7 +197,7 @@ } static void setSummary(IndexedInstrProf::Summary *TheSummary, - ProfileSummary &PS) { + InstrumentedProfileSummary &PS) { using namespace IndexedInstrProf; std::vector &Res = PS.getDetailedSummary(); TheSummary->NumSummaryFields = Summary::NumKinds; @@ -219,7 +219,7 @@ using namespace IndexedInstrProf; std::vector Cutoffs(&SummaryCutoffs[0], &SummaryCutoffs[NumSummaryCutoffs]); - ProfileSummary PS(Cutoffs); + InstrumentedProfileSummary PS(Cutoffs); InfoObj->TheProfileSummary = &PS; // Populate the hash table generator. Index: lib/ProfileData/ProfileSummary.cpp =================================================================== --- lib/ProfileData/ProfileSummary.cpp +++ lib/ProfileData/ProfileSummary.cpp @@ -16,60 +16,72 @@ using namespace llvm; -void ProfileSummary::addRecord(const InstrProfRecord &R) { - NumFunctions++; - if (R.Counts[0] > MaxFunctionCount) - MaxFunctionCount = R.Counts[0]; - - for (size_t I = 0, E = R.Counts.size(); I < E; ++I) - addCount(R.Counts[I], (I == 0)); +void InstrumentedProfileSummary::addRecord(const InstrProfRecord &R) { + addEntryCount(R.Counts[0]); + for (size_t I = 1, E = R.Counts.size(); I < E; ++I) + addInternalCount(R.Counts[I]); } // The argument to this method is a vector of cutoff percentages and the return -// value is a vector of (Cutoff, MinBlockCount, NumBlocks) triplets. +// value is a vector of (Cutoff, MinWeight, NumWeights) triplets. void ProfileSummary::computeDetailedSummary() { if (DetailedSummaryCutoffs.empty()) return; - auto Iter = CountFrequencies.begin(); - auto End = CountFrequencies.end(); + auto Iter = WeightFrequencies.begin(); + auto End = WeightFrequencies.end(); std::sort(DetailedSummaryCutoffs.begin(), DetailedSummaryCutoffs.end()); uint32_t BlocksSeen = 0; - uint64_t CurrSum = 0, Count = 0; + uint64_t CurrSum = 0, Weight = 0; for (uint32_t Cutoff : DetailedSummaryCutoffs) { assert(Cutoff <= 999999); - APInt Temp(128, TotalCount); + APInt Temp(128, TotalWeight); APInt N(128, Cutoff); APInt D(128, ProfileSummary::Scale); Temp *= N; Temp = Temp.sdiv(D); - uint64_t DesiredCount = Temp.getZExtValue(); - assert(DesiredCount <= TotalCount); - while (CurrSum < DesiredCount && Iter != End) { - Count = Iter->first; + uint64_t DesiredWeight = Temp.getZExtValue(); + assert(DesiredWeight <= TotalWeight); + while (CurrSum < DesiredWeight && Iter != End) { + Weight = Iter->first; uint32_t Freq = Iter->second; - CurrSum += (Count * Freq); + CurrSum += (Weight * Freq); BlocksSeen += Freq; Iter++; } - assert(CurrSum >= DesiredCount); - ProfileSummaryEntry PSE = {Cutoff, Count, BlocksSeen}; + assert(CurrSum >= DesiredWeight); + ProfileSummaryEntry PSE = {Cutoff, Weight, BlocksSeen}; DetailedSummary.push_back(PSE); } } -ProfileSummary::ProfileSummary(const IndexedInstrProf::Summary &S) - : TotalCount(S.get(IndexedInstrProf::Summary::TotalBlockCount)), - MaxBlockCount(S.get(IndexedInstrProf::Summary::MaxBlockCount)), - MaxInternalBlockCount( - S.get(IndexedInstrProf::Summary::MaxInternalBlockCount)), +InstrumentedProfileSummary::InstrumentedProfileSummary( + const IndexedInstrProf::Summary &S) + : ProfileSummary(), MaxInternalBlockCount(S.get( + IndexedInstrProf::Summary::MaxInternalBlockCount)), MaxFunctionCount(S.get(IndexedInstrProf::Summary::MaxFunctionCount)), - NumBlocks(S.get(IndexedInstrProf::Summary::TotalNumBlocks)), NumFunctions(S.get(IndexedInstrProf::Summary::TotalNumFunctions)) { + + TotalWeight = S.get(IndexedInstrProf::Summary::TotalBlockCount); + MaxWeight = S.get(IndexedInstrProf::Summary::MaxBlockCount); + NumWeights = S.get(IndexedInstrProf::Summary::TotalNumBlocks); + for (unsigned I = 0; I < S.NumCutoffEntries; I++) { const IndexedInstrProf::Summary::Entry &Ent = S.getEntry(I); DetailedSummary.emplace_back((uint32_t)Ent.Cutoff, Ent.MinBlockCount, Ent.NumBlocks); } } +void InstrumentedProfileSummary::addEntryCount(uint64_t Count) { + addWeight(Count); + NumFunctions++; + if (Count > MaxFunctionCount) + MaxFunctionCount = Count; +} + +void InstrumentedProfileSummary::addInternalCount(uint64_t Count) { + addWeight(Count); + if (Count > MaxInternalBlockCount) + MaxInternalBlockCount = Count; +} Index: tools/llvm-profdata/llvm-profdata.cpp =================================================================== --- tools/llvm-profdata/llvm-profdata.cpp +++ tools/llvm-profdata/llvm-profdata.cpp @@ -268,7 +268,7 @@ if (ShowDetailedSummary && DetailedSummaryCutoffs.empty()) { Cutoffs = {800000, 900000, 950000, 990000, 999000, 999900, 999990}; } - ProfileSummary PS(Cutoffs); + InstrumentedProfileSummary PS(Cutoffs); if (std::error_code EC = ReaderOrErr.getError()) exitWithErrorCode(EC, Filename); @@ -352,7 +352,7 @@ OS << "Total number of blocks: " << PS.getNumBlocks() << "\n"; OS << "Total count: " << PS.getTotalCount() << "\n"; for (auto Entry : PS.getDetailedSummary()) { - OS << Entry.NumBlocks << " blocks with count >= " << Entry.MinBlockCount + OS << Entry.NumWeights << " blocks with count >= " << Entry.MinWeight << " account for " << format("%0.6g", (float)Entry.Cutoff / ProfileSummary::Scale * 100) << " percentage of the total counts.\n"; Index: unittests/ProfileData/InstrProfTest.cpp =================================================================== --- unittests/ProfileData/InstrProfTest.cpp +++ unittests/ProfileData/InstrProfTest.cpp @@ -154,7 +154,7 @@ auto Profile = Writer.writeBuffer(); readProfile(std::move(Profile)); - ProfileSummary &PS = Reader->getSummary(); + InstrumentedProfileSummary &PS = Reader->getSummary(); ASSERT_EQ(2305843009213693952U, PS.getMaxFunctionCount()); ASSERT_EQ(2305843009213693952U, PS.getMaxBlockCount()); ASSERT_EQ(10U, PS.getNumBlocks()); @@ -171,10 +171,10 @@ auto NinetyFivePerc = std::find_if(Details.begin(), Details.end(), Predicate); Cutoff = 990000; auto NinetyNinePerc = std::find_if(Details.begin(), Details.end(), Predicate); - ASSERT_EQ(576460752303423488U, EightyPerc->MinBlockCount); - ASSERT_EQ(288230376151711744U, NinetyPerc->MinBlockCount); - ASSERT_EQ(288230376151711744U, NinetyFivePerc->MinBlockCount); - ASSERT_EQ(72057594037927936U, NinetyNinePerc->MinBlockCount); + ASSERT_EQ(576460752303423488U, EightyPerc->MinWeight); + ASSERT_EQ(288230376151711744U, NinetyPerc->MinWeight); + ASSERT_EQ(288230376151711744U, NinetyFivePerc->MinWeight); + ASSERT_EQ(72057594037927936U, NinetyNinePerc->MinWeight); } TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write) {