Index: include/llvm/ProfileData/InstrProf.h =================================================================== --- include/llvm/ProfileData/InstrProf.h +++ include/llvm/ProfileData/InstrProf.h @@ -592,166 +592,168 @@ struct InstrProfRecord { StringRef Name; uint64_t Hash; - std::vector Counts; SoftInstrProfErrors SIPE; - - InstrProfRecord() = default; - InstrProfRecord(StringRef Name, uint64_t Hash, std::vector Counts) - : Name(Name), Hash(Hash), Counts(std::move(Counts)) {} - InstrProfRecord(InstrProfRecord &&) = default; - InstrProfRecord(const InstrProfRecord &RHS) - : Name(RHS.Name), Hash(RHS.Hash), Counts(RHS.Counts), SIPE(RHS.SIPE), - ValueData(RHS.ValueData - ? llvm::make_unique(*RHS.ValueData) - : nullptr) {} - InstrProfRecord &operator=(InstrProfRecord &&) = default; - InstrProfRecord &operator=(const InstrProfRecord &RHS) { - Name = RHS.Name; - Hash = RHS.Hash; - Counts = RHS.Counts; - SIPE = RHS.SIPE; - if (!RHS.ValueData) { - ValueData = nullptr; + struct ValueProfData { + std::vector IndirectCallSites; + std::vector MemOPSizes; + }; + struct CounterHolder { + std::vector Counts; + std::unique_ptr ValueData; + CounterHolder() = default; + CounterHolder(std::vector Counts) : Counts(std::move(Counts)) {} + CounterHolder(CounterHolder &&) = default; + CounterHolder(const CounterHolder &RHS) + : Counts(RHS.Counts), + ValueData(RHS.ValueData + ? llvm::make_unique(*RHS.ValueData) + : nullptr) {} + CounterHolder &operator=(CounterHolder &&) = default; + CounterHolder &operator=(const CounterHolder &RHS) { + Counts = RHS.Counts; + if (!RHS.ValueData) { + ValueData = nullptr; + return *this; + } + if (!ValueData) + ValueData = llvm::make_unique(*RHS.ValueData); + else + *ValueData = *RHS.ValueData; return *this; } - if (!ValueData) - ValueData = llvm::make_unique(*RHS.ValueData); - else - *ValueData = *RHS.ValueData; - return *this; - } - - using ValueMapType = std::vector>; - - /// Return the number of value profile kinds with non-zero number - /// of profile sites. - inline uint32_t getNumValueKinds() const; - - /// Return the number of instrumented sites for ValueKind. - inline uint32_t getNumValueSites(uint32_t ValueKind) const; - /// Return the total number of ValueData for ValueKind. - inline uint32_t getNumValueData(uint32_t ValueKind) const; - - /// Return the number of value data collected for ValueKind at profiling - /// site: Site. - inline uint32_t getNumValueDataForSite(uint32_t ValueKind, - uint32_t Site) const; - - /// Return the array of profiled values at \p Site. If \p TotalC - /// is not null, the total count of all target values at this site - /// will be stored in \c *TotalC. - inline std::unique_ptr - getValueForSite(uint32_t ValueKind, uint32_t Site, - uint64_t *TotalC = nullptr) const; - - /// Get the target value/counts of kind \p ValueKind collected at site - /// \p Site and store the result in array \p Dest. Return the total - /// counts of all target values at this site. - inline uint64_t getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind, - uint32_t Site) const; - - /// Reserve space for NumValueSites sites. - inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites); + MutableArrayRef + getValueSitesForKind(uint32_t ValueKind) { + // Cast to /add/ const (should be an implicit_cast, ideally, if that's + // ever implemented in LLVM) to call the const overload of this function, + // then cast away the constness from the result. + auto AR = const_cast(this)->getValueSitesForKind( + ValueKind); + return makeMutableArrayRef( + const_cast(AR.data()), AR.size()); + } + ArrayRef + getValueSitesForKind(uint32_t ValueKind) const { + if (!ValueData) + return None; + switch (ValueKind) { + case IPVK_IndirectCallTarget: + return ValueData->IndirectCallSites; + case IPVK_MemOPSize: + return ValueData->MemOPSizes; + default: + llvm_unreachable("Unknown value kind!"); + } + } - /// Add ValueData for ValueKind at value Site. - void addValueData(uint32_t ValueKind, uint32_t Site, - InstrProfValueData *VData, uint32_t N, - ValueMapType *ValueMap); + std::vector & + getOrCreateValueSitesForKind(uint32_t ValueKind) { + if (!ValueData) + ValueData = llvm::make_unique(); + switch (ValueKind) { + case IPVK_IndirectCallTarget: + return ValueData->IndirectCallSites; + case IPVK_MemOPSize: + return ValueData->MemOPSizes; + default: + llvm_unreachable("Unknown value kind!"); + } + } - /// Merge the counts in \p Other into this one. - /// Optionally scale merged counts by \p Weight. - void merge(InstrProfRecord &Other, uint64_t Weight = 1); + using ValueMapType = std::vector>; + + // Map indirect call target name hash to name string. + uint64_t remapValue(uint64_t Value, uint32_t ValueKind, + ValueMapType *HashKeys); + + // Merge Value Profile data from Src record to this record for ValueKind. + // Scale merged value counts by \p Weight. + void mergeValueProfData(uint32_t ValueKind, CounterHolder &Src, + uint64_t Weight); + + // Scale up value profile data count. + void scaleValueProfData(uint32_t ValueKind, uint64_t Weight); + + /// Return the number of value profile kinds with non-zero number + /// of profile sites. + inline uint32_t getNumValueKinds() const; + + /// Return the number of instrumented sites for ValueKind. + inline uint32_t getNumValueSites(uint32_t ValueKind) const; + + /// Return the total number of ValueData for ValueKind. + inline uint32_t getNumValueData(uint32_t ValueKind) const; + + /// Return the number of value data collected for ValueKind at profiling + /// site: Site. + inline uint32_t getNumValueDataForSite(uint32_t ValueKind, + uint32_t Site) const; + + /// Return the array of profiled values at \p Site. If \p TotalC + /// is not null, the total count of all target values at this site + /// will be stored in \c *TotalC. + inline std::unique_ptr + getValueForSite(uint32_t ValueKind, uint32_t Site, + uint64_t *TotalC = nullptr) const; + + /// Get the target value/counts of kind \p ValueKind collected at site + /// \p Site and store the result in array \p Dest. Return the total + /// counts of all target values at this site. + inline uint64_t getValueForSite(InstrProfValueData Dest[], + uint32_t ValueKind, uint32_t Site) const; + + /// Reserve space for NumValueSites sites. + inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites); + + /// Add ValueData for ValueKind at value Site. + void addValueData(uint32_t ValueKind, uint32_t Site, + InstrProfValueData *VData, uint32_t N, + ValueMapType *ValueMap); + + /// Merge the counts in \p Other into this one. + /// Optionally scale merged counts by \p Weight. + void merge(CounterHolder &Other, uint64_t Weight = 1); + + /// Scale up profile counts (including value profile data) by + /// \p Weight. + void scale(uint64_t Weight); + + /// Sort value profile data (per site) by count. + void sortValueData() { + for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) + for (auto &SR : getValueSitesForKind(Kind)) + SR.sortByCount(); + } - /// Scale up profile counts (including value profile data) by - /// \p Weight. - void scale(uint64_t Weight); + /// Clear value data entries and edge counters. + void Clear() { + Counts.clear(); + clearValueData(); + } - /// Sort value profile data (per site) by count. - void sortValueData() { - for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) - for (auto &SR : getValueSitesForKind(Kind)) - SR.sortByCount(); - } + /// Clear value data entries + void clearValueData() { ValueData = nullptr; } + }; + CounterHolder Counters; - /// Clear value data entries and edge counters. - void Clear() { - Counts.clear(); - clearValueData(); - } + InstrProfRecord() = default; + InstrProfRecord(StringRef Name, uint64_t Hash, std::vector Counts) + : Name(Name), Hash(Hash), Counters(std::move(Counts)) {} - /// Clear value data entries - void clearValueData() { ValueData = nullptr; } /// Get the error contained within the record's soft error counter. Error takeError() { return SIPE.takeError(); } - -private: - struct ValueProfData { - std::vector IndirectCallSites; - std::vector MemOPSizes; - }; - std::unique_ptr ValueData; - - MutableArrayRef - getValueSitesForKind(uint32_t ValueKind) { - // Cast to /add/ const (should be an implicit_cast, ideally, if that's ever - // implemented in LLVM) to call the const overload of this function, then - // cast away the constness from the result. - auto AR = const_cast(this)->getValueSitesForKind( - ValueKind); - return makeMutableArrayRef( - const_cast(AR.data()), AR.size()); - } - ArrayRef - getValueSitesForKind(uint32_t ValueKind) const { - if (!ValueData) - return None; - switch (ValueKind) { - case IPVK_IndirectCallTarget: - return ValueData->IndirectCallSites; - case IPVK_MemOPSize: - return ValueData->MemOPSizes; - default: - llvm_unreachable("Unknown value kind!"); - } - } - - std::vector & - getOrCreateValueSitesForKind(uint32_t ValueKind) { - if (!ValueData) - ValueData = llvm::make_unique(); - switch (ValueKind) { - case IPVK_IndirectCallTarget: - return ValueData->IndirectCallSites; - case IPVK_MemOPSize: - return ValueData->MemOPSizes; - default: - llvm_unreachable("Unknown value kind!"); - } - } - - // Map indirect call target name hash to name string. - uint64_t remapValue(uint64_t Value, uint32_t ValueKind, - ValueMapType *HashKeys); - - // Merge Value Profile data from Src record to this record for ValueKind. - // Scale merged value counts by \p Weight. - void mergeValueProfData(uint32_t ValueKind, InstrProfRecord &Src, - uint64_t Weight); - - // Scale up value profile data count. - void scaleValueProfData(uint32_t ValueKind, uint64_t Weight); }; -uint32_t InstrProfRecord::getNumValueKinds() const { +uint32_t InstrProfRecord::CounterHolder::getNumValueKinds() const { uint32_t NumValueKinds = 0; for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) NumValueKinds += !(getValueSitesForKind(Kind).empty()); return NumValueKinds; } -uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const { +uint32_t +InstrProfRecord::CounterHolder::getNumValueData(uint32_t ValueKind) const { uint32_t N = 0; const std::vector &SiteRecords = getValueSitesForKind(ValueKind); @@ -761,18 +763,21 @@ return N; } -uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const { +uint32_t +InstrProfRecord::CounterHolder::getNumValueSites(uint32_t ValueKind) const { return getValueSitesForKind(ValueKind).size(); } -uint32_t InstrProfRecord::getNumValueDataForSite(uint32_t ValueKind, - uint32_t Site) const { +uint32_t +InstrProfRecord::CounterHolder::getNumValueDataForSite(uint32_t ValueKind, + uint32_t Site) const { return getValueSitesForKind(ValueKind)[Site].ValueData.size(); } std::unique_ptr -InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site, - uint64_t *TotalC) const { +InstrProfRecord::CounterHolder::getValueForSite(uint32_t ValueKind, + uint32_t Site, + uint64_t *TotalC) const { uint64_t Dummy; uint64_t &TotalCount = (TotalC == nullptr ? Dummy : *TotalC); uint32_t N = getNumValueDataForSite(ValueKind, Site); @@ -787,9 +792,8 @@ return VD; } -uint64_t InstrProfRecord::getValueForSite(InstrProfValueData Dest[], - uint32_t ValueKind, - uint32_t Site) const { +uint64_t InstrProfRecord::CounterHolder::getValueForSite( + InstrProfValueData Dest[], uint32_t ValueKind, uint32_t Site) const { uint32_t I = 0; uint64_t TotalCount = 0; for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) { @@ -801,7 +805,8 @@ return TotalCount; } -void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) { +void InstrProfRecord::CounterHolder::reserveSites(uint32_t ValueKind, + uint32_t NumValueSites) { if (!NumValueSites) return; getOrCreateValueSitesForKind(ValueKind).reserve(NumValueSites); Index: include/llvm/ProfileData/InstrProfData.inc =================================================================== --- include/llvm/ProfileData/InstrProfData.inc +++ include/llvm/ProfileData/InstrProfData.inc @@ -314,8 +314,8 @@ /*! * \brief Read data from this record and save it to Record. */ - void deserializeTo(InstrProfRecord &Record, - InstrProfRecord::ValueMapType *VMap); + void deserializeTo(InstrProfRecord::CounterHolder &Record, + InstrProfRecord::CounterHolder::ValueMapType *VMap); /* * In-place byte swap: * Do byte swap for this instance. \c Old is the original order before @@ -358,12 +358,12 @@ * Return the total size in bytes of the on-disk value profile data * given the data stored in Record. */ - static uint32_t getSize(const InstrProfRecord &Record); + static uint32_t getSize(const InstrProfRecord::CounterHolder &Counters); /*! * Return a pointer to \c ValueProfData instance ready to be streamed. */ static std::unique_ptr - serializeFrom(const InstrProfRecord &Record); + serializeFrom(const InstrProfRecord::CounterHolder &Counters); /*! * Check the integrity of the record. */ @@ -392,8 +392,8 @@ /*! * Read data from this data and save it to \c Record. */ - void deserializeTo(InstrProfRecord &Record, - InstrProfRecord::ValueMapType *VMap); + void deserializeTo(InstrProfRecord::CounterHolder &Counters, + InstrProfRecord::CounterHolder::ValueMapType *VMap); void operator delete(void *ptr) { ::operator delete(ptr); } #endif } ValueProfData; @@ -412,8 +412,7 @@ * For documentation of the member methods below, refer to corresponding methods * in class InstrProfRecord. */ -typedef struct ValueProfRecordClosure { - const void *Record; +typedef struct ValueProfRecordVTable { uint32_t (*GetNumValueKinds)(const void *Record); uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); @@ -428,7 +427,7 @@ void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S); ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); -} ValueProfRecordClosure; +} ValueProfRecordVTable; INSTR_PROF_VISIBILITY ValueProfRecord * getFirstValueProfRecord(ValueProfData *VPD); @@ -528,17 +527,16 @@ * given the data stored in Record. */ INSTR_PROF_VISIBILITY uint32_t -getValueProfDataSize(ValueProfRecordClosure *Closure) { +getValueProfDataSize(const void *Record, const ValueProfRecordVTable *VTable) { uint32_t Kind; uint32_t TotalSize = sizeof(ValueProfData); - const void *Record = Closure->Record; for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { - uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); + uint32_t NumValueSites = VTable->GetNumValueSites(Record, Kind); if (!NumValueSites) continue; TotalSize += getValueProfRecordSize(NumValueSites, - Closure->GetNumValueData(Record, Kind)); + VTable->GetNumValueData(Record, Kind)); } return TotalSize; } @@ -548,19 +546,18 @@ * from the \c Closure and serialize the data into \c This record instance. */ INSTR_PROF_VISIBILITY void -serializeValueProfRecordFrom(ValueProfRecord *This, - ValueProfRecordClosure *Closure, - uint32_t ValueKind, uint32_t NumValueSites) { +serializeValueProfRecordFrom(ValueProfRecord *This, const void *Record, + const ValueProfRecordVTable *VTable, uint32_t ValueKind, + uint32_t NumValueSites) { uint32_t S; - const void *Record = Closure->Record; This->Kind = ValueKind; This->NumValueSites = NumValueSites; InstrProfValueData *DstVD = getValueProfRecordValueData(This); for (S = 0; S < NumValueSites; S++) { - uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); + uint32_t ND = VTable->GetNumValueDataForSite(Record, ValueKind, S); This->SiteCountArray[S] = ND; - Closure->GetValueForSite(Record, DstVD, ValueKind, S); + VTable->GetValueForSite(Record, DstVD, ValueKind, S); DstVD += ND; } } @@ -573,23 +570,23 @@ * in DstData. */ INSTR_PROF_VISIBILITY ValueProfData * -serializeValueProfDataFrom(ValueProfRecordClosure *Closure, +serializeValueProfDataFrom(const void *Record, const ValueProfRecordVTable *VTable, ValueProfData *DstData) { uint32_t Kind; uint32_t TotalSize = - DstData ? DstData->TotalSize : getValueProfDataSize(Closure); + DstData ? DstData->TotalSize : getValueProfDataSize(Record, VTable); ValueProfData *VPD = - DstData ? DstData : Closure->AllocValueProfData(TotalSize); + DstData ? DstData : VTable->AllocValueProfData(TotalSize); VPD->TotalSize = TotalSize; - VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); + VPD->NumValueKinds = VTable->GetNumValueKinds(Record); ValueProfRecord *VR = getFirstValueProfRecord(VPD); for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { - uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); + uint32_t NumValueSites = VTable->GetNumValueSites(Record, Kind); if (!NumValueSites) continue; - serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); + serializeValueProfRecordFrom(VR, Record, VTable, Kind, NumValueSites); VR = getValueProfRecordNext(VR); } return VPD; Index: include/llvm/ProfileData/InstrProfReader.h =================================================================== --- include/llvm/ProfileData/InstrProfReader.h +++ include/llvm/ProfileData/InstrProfReader.h @@ -144,7 +144,7 @@ line_iterator Line; bool IsIRLevelProfile = false; - Error readValueProfileData(InstrProfRecord &Record); + Error readValueProfileData(InstrProfRecord::CounterHolder &Counters); public: TextInstrProfReader(std::unique_ptr DataBuffer_) @@ -199,7 +199,7 @@ uint32_t ValueKindLast; uint32_t CurValueDataSize; - InstrProfRecord::ValueMapType FunctionPtrToNameMap; + InstrProfRecord::CounterHolder::ValueMapType FunctionPtrToNameMap; public: RawInstrProfReader(std::unique_ptr DataBuffer) @@ -245,8 +245,8 @@ Error readName(InstrProfRecord &Record); Error readFuncHash(InstrProfRecord &Record); - Error readRawCounts(InstrProfRecord &Record); - Error readValueProfilingData(InstrProfRecord &Record); + Error readRawCounts(InstrProfRecord::CounterHolder &Counters); + Error readValueProfilingData(InstrProfRecord::CounterHolder &Counters); bool atEnd() const { return Data == DataEnd; } void advanceData() { Index: include/llvm/ProfileData/InstrProfWriter.h =================================================================== --- include/llvm/ProfileData/InstrProfWriter.h +++ include/llvm/ProfileData/InstrProfWriter.h @@ -33,7 +33,8 @@ class InstrProfWriter { public: - using ProfilingData = SmallDenseMap; + using ProfilingData = + SmallDenseMap; enum ProfKind { PF_Unknown = 0, PF_FE, PF_IRLevel }; private: @@ -62,7 +63,8 @@ Error writeText(raw_fd_ostream &OS); /// Write \c Record in text format to \c OS - static void writeRecordInText(const InstrProfRecord &Record, + static void writeRecordInText(StringRef Name, uint64_t Hash, + const InstrProfRecord::CounterHolder &Counters, InstrProfSymtab &Symtab, raw_fd_ostream &OS); /// Write the profile, returning the raw data. For testing. Index: lib/ProfileData/InstrProf.cpp =================================================================== --- lib/ProfileData/InstrProf.cpp +++ lib/ProfileData/InstrProf.cpp @@ -495,13 +495,13 @@ // Merge Value Profile data from Src record to this record for ValueKind. // Scale merged value counts by \p Weight. -void InstrProfRecord::mergeValueProfData(uint32_t ValueKind, - InstrProfRecord &Src, - uint64_t Weight) { +void InstrProfRecord::CounterHolder::mergeValueProfData(uint32_t ValueKind, + CounterHolder &Src, + uint64_t Weight) { uint32_t ThisNumValueSites = getNumValueSites(ValueKind); uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind); if (ThisNumValueSites != OtherNumValueSites) { - SIPE.addError(instrprof_error::value_site_count_mismatch); + // SIPE.addError(instrprof_error::value_site_count_mismatch); return; } if (!ThisNumValueSites) @@ -510,15 +510,17 @@ getOrCreateValueSitesForKind(ValueKind); MutableArrayRef OtherSiteRecords = Src.getValueSitesForKind(ValueKind); + SoftInstrProfErrors SIPE; for (uint32_t I = 0; I < ThisNumValueSites; I++) ThisSiteRecords[I].merge(SIPE, OtherSiteRecords[I], Weight); } -void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight) { +void InstrProfRecord::CounterHolder::merge(CounterHolder &Other, + uint64_t Weight) { // If the number of counters doesn't match we either have bad data // or a hash collision. if (Counts.size() != Other.Counts.size()) { - SIPE.addError(instrprof_error::count_mismatch); + // SIPE.addError(instrprof_error::count_mismatch); return; } @@ -527,32 +529,35 @@ Counts[I] = SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed); if (Overflowed) - SIPE.addError(instrprof_error::counter_overflow); + ; // SIPE.addError(instrprof_error::counter_overflow); } for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) mergeValueProfData(Kind, Other, Weight); } -void InstrProfRecord::scaleValueProfData(uint32_t ValueKind, uint64_t Weight) { +void InstrProfRecord::CounterHolder::scaleValueProfData(uint32_t ValueKind, + uint64_t Weight) { + SoftInstrProfErrors SIPE; for (auto &R : getValueSitesForKind(ValueKind)) R.scale(SIPE, Weight); } -void InstrProfRecord::scale(uint64_t Weight) { +void InstrProfRecord::CounterHolder::scale(uint64_t Weight) { for (auto &Count : this->Counts) { bool Overflowed; Count = SaturatingMultiply(Count, Weight, &Overflowed); if (Overflowed) - SIPE.addError(instrprof_error::counter_overflow); + ; // SIPE.addError(instrprof_error::counter_overflow); } for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) scaleValueProfData(Kind, Weight); } // Map indirect call target name hash to name string. -uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind, - ValueMapType *ValueMap) { +uint64_t InstrProfRecord::CounterHolder::remapValue(uint64_t Value, + uint32_t ValueKind, + ValueMapType *ValueMap) { if (!ValueMap) return Value; switch (ValueKind) { @@ -575,9 +580,11 @@ return Value; } -void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site, - InstrProfValueData *VData, uint32_t N, - ValueMapType *ValueMap) { +void InstrProfRecord::CounterHolder::addValueData(uint32_t ValueKind, + uint32_t Site, + InstrProfValueData *VData, + uint32_t N, + ValueMapType *ValueMap) { for (uint32_t I = 0; I < N; I++) { VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap); } @@ -593,77 +600,77 @@ #include "llvm/ProfileData/InstrProfData.inc" /*! - * \brief ValueProfRecordClosure Interface implementation for InstrProfRecord - * class. These C wrappers are used as adaptors so that C++ code can be - * invoked as callbacks. + * \brief ValueProfRecordClosure Interface implementation for + * InstrProfRecord::CounterHolder class. These C wrappers are used as adaptors + * so that C++ code can be invoked as callbacks. */ -uint32_t getNumValueKindsInstrProf(const void *Record) { - return reinterpret_cast(Record)->getNumValueKinds(); +uint32_t getNumValueKindsInstrProfCounters(const void *Record) { + return reinterpret_cast(Record) + ->getNumValueKinds(); } -uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) { - return reinterpret_cast(Record) +uint32_t getNumValueSitesInstrProfCounters(const void *Record, uint32_t VKind) { + return reinterpret_cast(Record) ->getNumValueSites(VKind); } -uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) { - return reinterpret_cast(Record) +uint32_t getNumValueDataInstrProfCounters(const void *Record, uint32_t VKind) { + return reinterpret_cast(Record) ->getNumValueData(VKind); } -uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, - uint32_t S) { - return reinterpret_cast(R) +uint32_t getNumValueDataForSiteInstrProfCounters(const void *R, uint32_t VK, + uint32_t S) { + return reinterpret_cast(R) ->getNumValueDataForSite(VK, S); } -void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, - uint32_t K, uint32_t S) { - reinterpret_cast(R)->getValueForSite(Dst, K, S); +void getValueForSiteInstrProfCounters(const void *R, InstrProfValueData *Dst, + uint32_t K, uint32_t S) { + reinterpret_cast(R)->getValueForSite( + Dst, K, S); } -ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) { +ValueProfData *allocValueProfDataInstrProfCounters(size_t TotalSizeInBytes) { ValueProfData *VD = (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData()); memset(VD, 0, TotalSizeInBytes); return VD; } -static ValueProfRecordClosure InstrProfRecordClosure = { - nullptr, - getNumValueKindsInstrProf, - getNumValueSitesInstrProf, - getNumValueDataInstrProf, - getNumValueDataForSiteInstrProf, +static const ValueProfRecordVTable InstrProfRecordCountersVTable = { + getNumValueKindsInstrProfCounters, + getNumValueSitesInstrProfCounters, + getNumValueDataInstrProfCounters, + getNumValueDataForSiteInstrProfCounters, nullptr, - getValueForSiteInstrProf, - allocValueProfDataInstrProf}; + getValueForSiteInstrProfCounters, + allocValueProfDataInstrProfCounters}; // Wrapper implementation using the closure mechanism. -uint32_t ValueProfData::getSize(const InstrProfRecord &Record) { - auto Closure = InstrProfRecordClosure; - Closure.Record = &Record; - return getValueProfDataSize(&Closure); +uint32_t +ValueProfData::getSize(const InstrProfRecord::CounterHolder &Counters) { + return getValueProfDataSize(&Counters, &InstrProfRecordCountersVTable); } // Wrapper implementation using the closure mechanism. std::unique_ptr -ValueProfData::serializeFrom(const InstrProfRecord &Record) { - InstrProfRecordClosure.Record = &Record; +ValueProfData::serializeFrom(const InstrProfRecord::CounterHolder &Counters) { - std::unique_ptr VPD( - serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr)); + std::unique_ptr VPD(serializeValueProfDataFrom( + &Counters, &InstrProfRecordCountersVTable, nullptr)); return VPD; } -void ValueProfRecord::deserializeTo(InstrProfRecord &Record, - InstrProfRecord::ValueMapType *VMap) { - Record.reserveSites(Kind, NumValueSites); +void ValueProfRecord::deserializeTo( + InstrProfRecord::CounterHolder &Counters, + InstrProfRecord::CounterHolder::ValueMapType *VMap) { + Counters.reserveSites(Kind, NumValueSites); InstrProfValueData *ValueData = getValueProfRecordValueData(this); for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) { uint8_t ValueDataCount = this->SiteCountArray[VSite]; - Record.addValueData(Kind, VSite, ValueData, ValueDataCount, VMap); + Counters.addValueData(Kind, VSite, ValueData, ValueDataCount, VMap); ValueData += ValueDataCount; } } @@ -696,14 +703,15 @@ } } -void ValueProfData::deserializeTo(InstrProfRecord &Record, - InstrProfRecord::ValueMapType *VMap) { +void ValueProfData::deserializeTo( + InstrProfRecord::CounterHolder &Counters, + InstrProfRecord::CounterHolder::ValueMapType *VMap) { if (NumValueKinds == 0) return; ValueProfRecord *VR = getFirstValueProfRecord(this); for (uint32_t K = 0; K < NumValueKinds; K++) { - VR->deserializeTo(Record, VMap); + VR->deserializeTo(Counters, VMap); VR = getValueProfRecordNext(VR); } } @@ -800,16 +808,16 @@ } void annotateValueSite(Module &M, Instruction &Inst, - const InstrProfRecord &InstrProfR, + const InstrProfRecord::CounterHolder &Counters, InstrProfValueKind ValueKind, uint32_t SiteIdx, uint32_t MaxMDCount) { - uint32_t NV = InstrProfR.getNumValueDataForSite(ValueKind, SiteIdx); + uint32_t NV = Counters.getNumValueDataForSite(ValueKind, SiteIdx); if (!NV) return; uint64_t Sum = 0; std::unique_ptr VD = - InstrProfR.getValueForSite(ValueKind, SiteIdx, &Sum); + Counters.getValueForSite(ValueKind, SiteIdx, &Sum); ArrayRef VDs(VD.get(), NV); annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount); Index: lib/ProfileData/InstrProfReader.cpp =================================================================== --- lib/ProfileData/InstrProfReader.cpp +++ lib/ProfileData/InstrProfReader.cpp @@ -155,8 +155,8 @@ return success(); } -Error -TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) { +Error TextInstrProfReader::readValueProfileData( + InstrProfRecord::CounterHolder &Counters) { #define CHECK_LINE_END(Line) \ if (Line.is_at_end()) \ @@ -190,7 +190,7 @@ if (!NumValueSites) continue; - Record.reserveSites(VK, NumValueSites); + Counters.reserveSites(VK, NumValueSites); for (uint32_t S = 0; S < NumValueSites; S++) { VP_READ_ADVANCE(NumValueData); @@ -210,8 +210,8 @@ CurrentValues.push_back({Value, TakenCount}); Line++; } - Record.addValueData(ValueKind, S, CurrentValues.data(), NumValueData, - nullptr); + Counters.addValueData(ValueKind, S, CurrentValues.data(), NumValueData, + nullptr); } } return success(); @@ -252,19 +252,19 @@ return error(instrprof_error::malformed); // Read each counter and fill our internal storage with the values. - Record.Clear(); - Record.Counts.reserve(NumCounters); + Record.Counters.Clear(); + Record.Counters.Counts.reserve(NumCounters); for (uint64_t I = 0; I < NumCounters; ++I) { if (Line.is_at_end()) return error(instrprof_error::truncated); uint64_t Count; if ((Line++)->getAsInteger(10, Count)) return error(instrprof_error::malformed); - Record.Counts.push_back(Count); + Record.Counters.Counts.push_back(Count); } // Check if value profile data exists and read it if so. - if (Error E = readValueProfileData(Record)) + if (Error E = readValueProfileData(Record.Counters)) return E; // This is needed to avoid two pass parsing because llvm-profdata @@ -390,7 +390,7 @@ template Error RawInstrProfReader::readRawCounts( - InstrProfRecord &Record) { + InstrProfRecord::CounterHolder &Counters) { uint32_t NumCounters = swap(Data->NumCounters); IntPtrT CounterPtr = Data->CounterPtr; if (NumCounters == 0) @@ -405,20 +405,20 @@ return error(instrprof_error::malformed); if (ShouldSwapBytes) { - Record.Counts.clear(); - Record.Counts.reserve(RawCounts.size()); + Counters.Counts.clear(); + Counters.Counts.reserve(RawCounts.size()); for (uint64_t Count : RawCounts) - Record.Counts.push_back(swap(Count)); + Counters.Counts.push_back(swap(Count)); } else - Record.Counts = RawCounts; + Counters.Counts = RawCounts; return success(); } template Error RawInstrProfReader::readValueProfilingData( - InstrProfRecord &Record) { - Record.clearValueData(); + InstrProfRecord::CounterHolder &Counters) { + Counters.clearValueData(); CurValueDataSize = 0; // Need to match the logic in value profile dumper code in compiler-rt: uint32_t NumValueKinds = 0; @@ -439,7 +439,7 @@ // Note that besides deserialization, this also performs the conversion for // indirect call targets. The function pointers from the raw profile are // remapped into function name hashes. - VDataPtrOrErr.get()->deserializeTo(Record, &Symtab->getAddrHashMap()); + VDataPtrOrErr.get()->deserializeTo(Counters, &Symtab->getAddrHashMap()); CurValueDataSize = VDataPtrOrErr.get()->getSize(); return success(); } @@ -460,11 +460,11 @@ return E; // Read raw counts and set Record. - if (Error E = readRawCounts(Record)) + if (Error E = readRawCounts(Record.Counters)) return E; // Read value data and set Record. - if (Error E = readValueProfilingData(Record)) + if (Error E = readValueProfilingData(Record.Counters)) return E; // Iterate. @@ -495,7 +495,7 @@ if (VDataPtrOrErr.takeError()) return false; - VDataPtrOrErr.get()->deserializeTo(DataBuffer.back(), nullptr); + VDataPtrOrErr.get()->deserializeTo(DataBuffer.back().Counters, nullptr); D += VDataPtrOrErr.get()->TotalSize; return true; @@ -728,7 +728,7 @@ if (Error E = Record.takeError()) return error(std::move(E)); - Counts = Record.get().Counts; + Counts = Record.get().Counters.Counts; return success(); } Index: lib/ProfileData/InstrProfWriter.cpp =================================================================== --- lib/ProfileData/InstrProfWriter.cpp +++ lib/ProfileData/InstrProfWriter.cpp @@ -119,13 +119,13 @@ offset_type M = 0; for (const auto &ProfileData : *V) { - const InstrProfRecord &ProfRecord = ProfileData.second; + const InstrProfRecord::CounterHolder &Counters = ProfileData.second; M += sizeof(uint64_t); // The function hash M += sizeof(uint64_t); // The size of the Counts vector - M += ProfRecord.Counts.size() * sizeof(uint64_t); + M += Counters.Counts.size() * sizeof(uint64_t); // Value data - M += ValueProfData::getSize(ProfileData.second); + M += ValueProfData::getSize(Counters); } LE.write(M); @@ -136,22 +136,29 @@ Out.write(K.data(), N); } - void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) { + void EmitData(raw_ostream &Out, key_type_ref K, data_type_ref V, + offset_type) { using namespace support; endian::Writer LE(Out); for (const auto &ProfileData : *V) { - const InstrProfRecord &ProfRecord = ProfileData.second; - SummaryBuilder->addRecord(ProfRecord); + const InstrProfRecord::CounterHolder &Counters = ProfileData.second; + InstrProfRecord R; + R.Name = K; + R.Hash = ProfileData.first; + R.Counters = + ProfileData + .second; // FIXME: Avoid copying by fixing the addRecord API + SummaryBuilder->addRecord(std::move(R)); LE.write(ProfileData.first); // Function hash - LE.write(ProfRecord.Counts.size()); - for (uint64_t I : ProfRecord.Counts) + LE.write(Counters.Counts.size()); + for (uint64_t I : Counters.Counts) LE.write(I); // Write value data std::unique_ptr VDataPtr = - ValueProfData::serializeFrom(ProfileData.second); + ValueProfData::serializeFrom(Counters); uint32_t S = VDataPtr->getSize(); VDataPtr->swapBytesFromHost(ValueProfDataEndianness); Out.write((const char *)VDataPtr.get(), S); @@ -181,32 +188,35 @@ bool NewFunc; ProfilingData::iterator Where; - std::tie(Where, NewFunc) = - ProfileDataMap.insert(std::make_pair(I.Hash, InstrProfRecord())); - InstrProfRecord &Dest = Where->second; + std::tie(Where, NewFunc) = ProfileDataMap.insert( + std::make_pair(I.Hash, InstrProfRecord::CounterHolder())); + InstrProfRecord::CounterHolder &Dest = Where->second; if (NewFunc) { // We've never seen a function with this name and hash, add it. - Dest = std::move(I); - // Fix up the name to avoid dangling reference. - Dest.Name = FunctionData.find(Dest.Name)->getKey(); + Dest = std::move(I.Counters); if (Weight > 1) Dest.scale(Weight); } else { // We're updating a function we've seen before. - Dest.merge(I, Weight); + Dest.merge(I.Counters, Weight); } Dest.sortValueData(); - return Dest.takeError(); + return I.takeError(); } Error InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW) { for (auto &I : IPW.FunctionData) - for (auto &Func : I.getValue()) - if (Error E = addRecord(std::move(Func.second), 1)) + for (auto &Func : I.getValue()) { + InstrProfRecord R; + R.Name = I.getKey(); + R.Hash = Func.first; + R.Counters = std::move(Func.second); + if (Error E = addRecord(std::move(R), 1)) return E; + } return Error::success(); } @@ -214,8 +224,8 @@ if (!Sparse) return true; for (const auto &Func : PD) { - const InstrProfRecord &IPR = Func.second; - if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; })) + const InstrProfRecord::CounterHolder &CH = Func.second; + if (llvm::any_of(CH.Counts, [](uint64_t Count) { return Count > 0; })) return true; } return false; @@ -323,33 +333,35 @@ #include "llvm/ProfileData/InstrProfData.inc" }; -void InstrProfWriter::writeRecordInText(const InstrProfRecord &Func, - InstrProfSymtab &Symtab, - raw_fd_ostream &OS) { - OS << Func.Name << "\n"; - OS << "# Func Hash:\n" << Func.Hash << "\n"; - OS << "# Num Counters:\n" << Func.Counts.size() << "\n"; +void InstrProfWriter::writeRecordInText( + StringRef Name, uint64_t Hash, + const InstrProfRecord::CounterHolder &Counters, InstrProfSymtab &Symtab, + raw_fd_ostream &OS) { + OS << Name << "\n"; + OS << "# Func Hash:\n" << Hash << "\n"; + OS << "# Num Counters:\n" << Counters.Counts.size() << "\n"; OS << "# Counter Values:\n"; - for (uint64_t Count : Func.Counts) + for (uint64_t Count : Counters.Counts) OS << Count << "\n"; - uint32_t NumValueKinds = Func.getNumValueKinds(); + uint32_t NumValueKinds = Counters.getNumValueKinds(); if (!NumValueKinds) { OS << "\n"; return; } - OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n"; + OS << "# Num Value Kinds:\n" << Counters.getNumValueKinds() << "\n"; for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) { - uint32_t NS = Func.getNumValueSites(VK); + uint32_t NS = Counters.getNumValueSites(VK); if (!NS) continue; OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n"; OS << "# NumValueSites:\n" << NS << "\n"; for (uint32_t S = 0; S < NS; S++) { - uint32_t ND = Func.getNumValueDataForSite(VK, S); + uint32_t ND = Counters.getNumValueDataForSite(VK, S); OS << ND << "\n"; - std::unique_ptr VD = Func.getValueForSite(VK, S); + std::unique_ptr VD = + Counters.getValueForSite(VK, S); for (uint32_t I = 0; I < ND; I++) { if (VK == IPVK_IndirectCallTarget) OS << Symtab.getFuncName(VD[I].Value) << ":" << VD[I].Count << "\n"; @@ -375,6 +387,6 @@ for (const auto &I : FunctionData) if (shouldEncodeData(I.getValue())) for (const auto &Func : I.getValue()) - writeRecordInText(Func.second, Symtab, OS); + writeRecordInText(I.getKey(), Func.first, Func.second, Symtab, OS); return Error::success(); } Index: lib/ProfileData/ProfileSummaryBuilder.cpp =================================================================== --- lib/ProfileData/ProfileSummaryBuilder.cpp +++ lib/ProfileData/ProfileSummaryBuilder.cpp @@ -38,9 +38,9 @@ // instrumentation profiles. // Eventually MaxFunctionCount will become obsolete and this can be // removed. - addEntryCount(R.Counts[0]); - for (size_t I = 1, E = R.Counts.size(); I < E; ++I) - addInternalCount(R.Counts[I]); + addEntryCount(R.Counters.Counts[0]); + for (auto C : R.Counters.Counts) + addInternalCount(C); } // To compute the detailed summary, we consider each line containing samples as Index: tools/llvm-profdata/llvm-profdata.cpp =================================================================== --- tools/llvm-profdata/llvm-profdata.cpp +++ tools/llvm-profdata/llvm-profdata.cpp @@ -458,9 +458,9 @@ std::vector ValueSitesHistogram; } ValueSitesStats; -static void traverseAllValueSites(const InstrProfRecord &Func, uint32_t VK, - ValueSitesStats &Stats, raw_fd_ostream &OS, - InstrProfSymtab *Symtab) { +static void traverseAllValueSites(const InstrProfRecord::CounterHolder &Func, + uint32_t VK, ValueSitesStats &Stats, + raw_fd_ostream &OS, InstrProfSymtab *Symtab) { uint32_t NS = Func.getNumValueSites(VK); Stats.TotalNumValueSites += NS; for (size_t I = 0; I < NS; ++I) { @@ -528,11 +528,12 @@ if (doTextFormatDump) { InstrProfSymtab &Symtab = Reader->getSymtab(); - InstrProfWriter::writeRecordInText(Func, Symtab, OS); + InstrProfWriter::writeRecordInText(Func.Name, Func.Hash, Func.Counters, + Symtab, OS); continue; } - assert(Func.Counts.size() > 0 && "function missing entry counter"); + assert(Func.Counters.Counts.size() > 0 && "function missing entry counter"); Builder.addRecord(Func); if (Show) { @@ -544,15 +545,15 @@ OS << " " << Func.Name << ":\n" << " Hash: " << format("0x%016" PRIx64, Func.Hash) << "\n" - << " Counters: " << Func.Counts.size() << "\n"; + << " Counters: " << Func.Counters.Counts.size() << "\n"; if (!IsIRInstr) - OS << " Function count: " << Func.Counts[0] << "\n"; + OS << " Function count: " << Func.Counters.Counts[0] << "\n"; if (ShowIndirectCallTargets) OS << " Indirect Call Site Count: " - << Func.getNumValueSites(IPVK_IndirectCallTarget) << "\n"; + << Func.Counters.getNumValueSites(IPVK_IndirectCallTarget) << "\n"; - uint32_t NumMemOPCalls = Func.getNumValueSites(IPVK_MemOPSize); + uint32_t NumMemOPCalls = Func.Counters.getNumValueSites(IPVK_MemOPSize); if (ShowMemOPSizes && NumMemOPCalls > 0) OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls << "\n"; @@ -560,23 +561,23 @@ if (ShowCounts) { OS << " Block counts: ["; size_t Start = (IsIRInstr ? 0 : 1); - for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) { - OS << (I == Start ? "" : ", ") << Func.Counts[I]; + for (size_t I = Start, E = Func.Counters.Counts.size(); I < E; ++I) { + OS << (I == Start ? "" : ", ") << Func.Counters.Counts[I]; } OS << "]\n"; } if (ShowIndirectCallTargets) { OS << " Indirect Target Results:\n"; - traverseAllValueSites(Func, IPVK_IndirectCallTarget, + traverseAllValueSites(Func.Counters, IPVK_IndirectCallTarget, VPStats[IPVK_IndirectCallTarget], OS, &(Reader->getSymtab())); } if (ShowMemOPSizes && NumMemOPCalls > 0) { OS << " Memory Intrinsic Size Results:\n"; - traverseAllValueSites(Func, IPVK_MemOPSize, VPStats[IPVK_MemOPSize], OS, - nullptr); + traverseAllValueSites(Func.Counters, IPVK_MemOPSize, + VPStats[IPVK_MemOPSize], OS, nullptr); } } }