diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -318,6 +318,14 @@ raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc); +static inline uint64_t hashFuncName(StringRef F) { + // If function name is already MD5 string, do not hash again. + uint64_t Hash; + if (F.getAsInteger(10, Hash)) + Hash = MD5Hash(F); + return Hash; +} + /// Representation of a single sample record. /// /// A sample record is represented by a positive integer value, which @@ -631,8 +639,12 @@ } uint64_t getHashCode() const { - return hasContext() ? hash_value(getContextFrames()) - : hash_value(getName()); + if (hasContext()) + return hash_value(getContextFrames()); + + // For non-context function name, use its MD5 as hash value, so that it is + // consistent with the profile map's key. + return hashFuncName(getName()); } /// Set the name of the function and clear the current context. @@ -710,11 +722,6 @@ uint32_t Attributes; }; -static inline hash_code hash_value(const SampleContext &arg) { - return arg.hasContext() ? hash_value(arg.getContextFrames()) - : hash_value(arg.getName()); -} - class FunctionSamples; class SampleProfileReaderItaniumRemapper; @@ -1267,10 +1274,25 @@ raw_ostream &operator<<(raw_ostream &OS, const FunctionSamples &FS); -using SampleProfileMap = - std::unordered_map; +using SampleProfileMap = std::unordered_map; -using NameFunctionSamples = std::pair; +/// Check for MD5 collision when inserting on SampleProfileMap. Any insertion to +/// SampleProfileMap should use this function instead of emplace or operator[]. +static inline std::pair +SampleProfileMapTryEmplace(SampleProfileMap &Profiles, const SampleContext &Ctx, + const FunctionSamples &NewFS = FunctionSamples()) { + uint64_t MD5 = Ctx.getHashCode(); + auto Ret = Profiles.try_emplace(MD5, NewFS); + auto &FS = Ret.first->second; + if (!Ret.second) + if (LLVM_UNLIKELY(FS.getContext() != Ctx)) + dbgs() << "MD5 collision detected: " << FS.getContext().toString() + << " and " << Ctx.toString() << " has same hash value " << MD5 + << "\n"; + return Ret; +} + +using NameFunctionSamples = std::pair; void sortFuncProfiles(const SampleProfileMap &ProfileMap, std::vector &SortedProfiles); @@ -1316,8 +1338,6 @@ bool MergeColdContext, uint32_t ColdContextFrameLength, bool TrimBaseProfileOnly); - // Canonicalize context profile name and attributes. - void canonicalizeContextProfiles(); private: SampleProfileMap &ProfileMap; @@ -1363,12 +1383,15 @@ SampleProfileMap &OutputProfiles, bool ProfileIsCS = false) { if (ProfileIsCS) { - for (const auto &I : InputProfiles) - OutputProfiles[I.second.getName()].merge(I.second); - // Retain the profile name and clear the full context for each function - // profile. - for (auto &I : OutputProfiles) - I.second.setContext(SampleContext(I.first)); + for (const auto &I : InputProfiles) { + // Retain the profile name and clear the full context for each function + // profile. + SampleContext FName(I.second.getName()); + FunctionSamples &FS = SampleProfileMapTryEmplace(OutputProfiles, + FName).first->second; + FS.merge(I.second); + FS.setContext(FName); + } } else { for (const auto &I : InputProfiles) flattenNestedProfile(OutputProfiles, I.second); @@ -1380,8 +1403,7 @@ const FunctionSamples &FS) { // To retain the context, checksum, attributes of the original profile, make // a copy of it if no profile is found. - SampleContext &Context = FS.getContext(); - auto Ret = OutputProfiles.try_emplace(Context, FS); + auto Ret = SampleProfileMapTryEmplace(OutputProfiles, FS.getContext(), FS); FunctionSamples &Profile = Ret.first->second; if (Ret.second) { // When it's the copy of the old profile, just clear all the inlinees' diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -383,8 +383,8 @@ /// The implementaion to read sample profiles from the associated file. virtual std::error_code readImpl() = 0; - /// Print the profile for \p FContext on stream \p OS. - void dumpFunctionProfile(SampleContext FContext, raw_ostream &OS = dbgs()); + /// Print the profile for \p FunctionSamples on stream \p OS. + void dumpFunctionProfile(const FunctionSamples &FS, raw_ostream &OS = dbgs()); /// Collect functions with definitions in Module M. For reader which /// support loading function profiles on demand, return true when the @@ -410,32 +410,47 @@ /// Return the samples collected for function \p F, create empty /// FunctionSamples if it doesn't exist. FunctionSamples *getOrCreateSamplesFor(const Function &F) { - std::string FGUID; StringRef CanonName = FunctionSamples::getCanonicalFnName(F); - CanonName = getRepInFormat(CanonName, useMD5(), FGUID); - auto It = Profiles.find(CanonName); - if (It != Profiles.end()) - return &It->second; - if (!FGUID.empty()) { - assert(useMD5() && "New name should only be generated for md5 profile"); - CanonName = *MD5NameBuffer.insert(FGUID).first; + uint64_t MD5 = MD5Hash(CanonName); + auto Ret = Profiles.try_emplace(MD5, FunctionSamples()); + if (!Ret.second) { + // If MD5 collision actually happens, we need to drop the existing samples + // and pretend a new one is created, to ensure the returned samples do not + // contain unexpected contents. + SampleContext &Ctx = Ret.first->second.getContext(); + if (LLVM_UNLIKELY(Ctx.getName() != CanonName && + Ctx.getName() != std::to_string(MD5))) { + dbgs() << "MD5 collision detected: " << Ctx.toString() << " and " + << CanonName << " has same hash value " << MD5 << "\n"; + Ret.first->second = FunctionSamples(); + } } - return &Profiles[CanonName]; + return &Ret.first->second; } /// Return the samples collected for function \p F. - virtual FunctionSamples *getSamplesFor(StringRef Fname) { - std::string FGUID; - Fname = getRepInFormat(Fname, useMD5(), FGUID); - auto It = Profiles.find(Fname); - if (It != Profiles.end()) - return &It->second; + FunctionSamples *getSamplesFor(StringRef Fname) { + uint64_t MD5 = hashFuncName(Fname); + auto It = Profiles.find(MD5); + if (It != Profiles.end()) { + SampleContext &Ctx = It->second.getContext(); + // Check for MD5 collision, make sure returning an actual match. + if (LLVM_LIKELY(Ctx.getName() == Fname || + Ctx.getName() == std::to_string(MD5))) + return &It->second; + } if (Remapper) { if (auto NameInProfile = Remapper->lookUpNameInProfile(Fname)) { - auto It = Profiles.find(*NameInProfile); - if (It != Profiles.end()) - return &It->second; + // If remapper is used, function name cannot be in MD5. + MD5 = MD5Hash(*NameInProfile); + auto It = Profiles.find(MD5); + if (It != Profiles.end()) { + SampleContext &Ctx = It->second.getContext(); + if (LLVM_LIKELY(Ctx.getName() == *NameInProfile || + Ctx.getName() == std::to_string(MD5))) + return &It->second; + } } } return nullptr; @@ -654,15 +669,16 @@ /// Read the whole name table. std::error_code readNameTable(); - /// Read a string indirectly via the name table. - ErrorOr readStringFromTable(); + /// Read a string indirectly via the name table. Optionally return the index. + ErrorOr readStringFromTable(size_t *Ret = nullptr); - /// Read a context indirectly via the CSNameTable. - ErrorOr readContextFromTable(); + /// Read a context indirectly via the CSNameTable. Optionally return the + /// index. + ErrorOr readContextFromTable(size_t *Ret = nullptr); /// Read a context indirectly via the CSNameTable if the profile has context, - /// otherwise same as readStringFromTable. - ErrorOr readSampleContextFromTable(); + /// otherwise same as readStringFromTable, also return its hash value. + ErrorOr> readSampleContextFromTable(); /// Points to the current location in the buffer. const uint8_t *Data = nullptr; @@ -682,13 +698,24 @@ /// the lifetime of MD5StringBuf is not shorter than that of NameTable. std::vector MD5StringBuf; - /// The starting address of NameTable containing fixed length MD5. + /// The starting address of fixed length MD5 name table section. const uint8_t *MD5NameMemStart = nullptr; /// CSNameTable is used to save full context vectors. It is the backing buffer /// for SampleContextFrames. std::vector CSNameTable; + /// Table to cache MD5 values of sample contexts corresponding to + /// readSampleContextFromTable(), used to index into Profiles or + /// FuncOffsetTable. + std::vector MD5SampleContextTable; + + /// The starting address of the table of MD5 values of sample contexts. For + /// fixed length MD5 non-CS profile it is same as MD5NameMemStart because + /// hashes of non-CS contexts are already in the profile. Otherwise it points + /// to the start of MD5SampleContextTable. + const uint64_t *MD5SampleContextStart = nullptr; + private: std::error_code readSummaryEntry(std::vector &Entries); virtual std::error_code verifySPMagic(uint64_t Magic) = 0; @@ -762,10 +789,10 @@ std::unique_ptr ProfSymList; - /// The table mapping from function context to the offset of its + /// The table mapping from a function context's MD5 to the offset of its /// FunctionSample towards file start. /// At most one of FuncOffsetTable and FuncOffsetList is populated. - DenseMap FuncOffsetTable; + DenseMap FuncOffsetTable; /// The list version of FuncOffsetTable. This is used if every entry is /// being accessed. diff --git a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp --- a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp +++ b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp @@ -205,7 +205,11 @@ if (UseContextLessSummary || (sampleprof::FunctionSamples::ProfileIsCS && !UseContextLessSummary.getNumOccurrences())) { for (const auto &I : Profiles) { - ContextLessProfiles[I.second.getName()].merge(I.second); + auto Ret = + SampleProfileMapTryEmplace(ContextLessProfiles, I.second.getName()); + if (Ret.second) + Ret.first->second.setContext(I.second.getName()); + Ret.first->second.merge(I.second); } ProfilesToUse = &ContextLessProfiles; } diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -202,13 +202,12 @@ const SampleProfileMap &ProfileMap, std::vector &SortedProfiles) { for (const auto &I : ProfileMap) { - assert(I.first == I.second.getContext() && "Inconsistent profile map"); - SortedProfiles.push_back(std::make_pair(I.second.getContext(), &I.second)); + SortedProfiles.push_back(std::make_pair(I.first, &I.second)); } llvm::stable_sort(SortedProfiles, [](const NameFunctionSamples &A, const NameFunctionSamples &B) { if (A.second->getTotalSamples() == B.second->getTotalSamples()) - return A.first < B.first; + return A.second->getContext() < B.second->getContext(); return A.second->getTotalSamples() > B.second->getTotalSamples(); }); } @@ -357,13 +356,13 @@ // Filter the cold profiles from ProfileMap and move them into a tmp // container - std::vector> ColdProfiles; + std::vector> ColdProfiles; for (const auto &I : ProfileMap) { - const SampleContext &Context = I.first; + const SampleContext &Context = I.second.getContext(); const FunctionSamples &FunctionProfile = I.second; if (FunctionProfile.getTotalSamples() < ColdCountThreshold && (!TrimBaseProfileOnly || Context.isBaseContext())) - ColdProfiles.emplace_back(Context, &I.second); + ColdProfiles.emplace_back(I.first, &I.second); } // Remove the cold profile from ProfileMap and merge them into @@ -374,9 +373,10 @@ auto MergedContext = I.second->getContext().getContextFrames(); if (ColdContextFrameLength < MergedContext.size()) MergedContext = MergedContext.take_back(ColdContextFrameLength); - auto Ret = MergedProfileMap.emplace(MergedContext, FunctionSamples()); + auto Ret = SampleProfileMapTryEmplace(MergedProfileMap, MergedContext); FunctionSamples &MergedProfile = Ret.first->second; MergedProfile.merge(*I.second); + MergedProfile.setContext(MergedContext); } ProfileMap.erase(I.first); } @@ -388,54 +388,14 @@ ProfileMap.find(I.first) == ProfileMap.end()) continue; // Merge the profile if the original profile exists, otherwise just insert - // as a new profile - auto Ret = ProfileMap.emplace(I.first, FunctionSamples()); - if (Ret.second) { - SampleContext FContext(Ret.first->first, RawContext); - FunctionSamples &FProfile = Ret.first->second; - FProfile.setContext(FContext); - } + // as a new profile. New profiles from MergedProfileMap already have the + // right context. + auto Ret = SampleProfileMapTryEmplace(ProfileMap, I.second.getContext()); FunctionSamples &OrigProfile = Ret.first->second; OrigProfile.merge(I.second); } } -void SampleContextTrimmer::canonicalizeContextProfiles() { - std::vector ProfilesToBeRemoved; - SampleProfileMap ProfilesToBeAdded; - for (auto &I : ProfileMap) { - FunctionSamples &FProfile = I.second; - SampleContext &Context = FProfile.getContext(); - if (I.first == Context) - continue; - - // Use the context string from FunctionSamples to update the keys of - // ProfileMap. They can get out of sync after context profile promotion - // through pre-inliner. - // Duplicate the function profile for later insertion to avoid a conflict - // caused by a context both to be add and to be removed. This could happen - // when a context is promoted to another context which is also promoted to - // the third context. For example, given an original context A @ B @ C that - // is promoted to B @ C and the original context B @ C which is promoted to - // just C, adding B @ C to the profile map while removing same context (but - // with different profiles) from the map can cause a conflict if they are - // not handled in a right order. This can be solved by just caching the - // profiles to be added. - auto Ret = ProfilesToBeAdded.emplace(Context, FProfile); - (void)Ret; - assert(Ret.second && "Context conflict during canonicalization"); - ProfilesToBeRemoved.push_back(I.first); - } - - for (auto &I : ProfilesToBeRemoved) { - ProfileMap.erase(I); - } - - for (auto &I : ProfilesToBeAdded) { - ProfileMap.emplace(I.first, I.second); - } -} - std::error_code ProfileSymbolList::write(raw_ostream &OS) { // Sort the symbols before output. If doing compression. // It will make the compression much more effective. @@ -531,16 +491,18 @@ // base profiles improves the code quality for thinlto build by allowing a // profile in the prelink phase for to-be-fully-inlined functions. if (!NodeProfile) { - ProfileMap[ChildProfile->getContext()].merge(*ChildProfile); + SampleProfileMapTryEmplace(ProfileMap, ChildProfile->getContext()) + .first->second.merge(*ChildProfile); } else if (GenerateMergedBaseProfiles) { - ProfileMap[ChildProfile->getContext()].merge(*ChildProfile); + SampleProfileMapTryEmplace(ProfileMap, ChildProfile->getContext()) + .first->second.merge(*ChildProfile); auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc); SamplesMap[ChildProfile->getName().str()].getContext().setAttribute( ContextDuplicatedIntoBase); } // Remove the original child profile. - ProfileMap.erase(OrigChildContext); + ProfileMap.erase(OrigChildContext.getHashCode()); } } diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -61,9 +61,9 @@ /// /// \param FContext Name + context of the function to print. /// \param OS Stream to emit the output to. -void SampleProfileReader::dumpFunctionProfile(SampleContext FContext, +void SampleProfileReader::dumpFunctionProfile(const FunctionSamples &FS, raw_ostream &OS) { - OS << "Function: " << FContext.toString() << ": " << Profiles[FContext]; + OS << "Function: " << FS.getContext().toString() << ": " << FS; } /// Dump all the function profiles found on stream \p OS. @@ -71,7 +71,7 @@ std::vector V; sortFuncProfiles(Profiles, V); for (const auto &I : V) - dumpFunctionProfile(I.first, OS); + dumpFunctionProfile(*I.second, OS); } static void dumpFunctionProfileJson(const FunctionSamples &S, @@ -355,8 +355,8 @@ SampleContext FContext(FName, CSNameTable); if (FContext.hasContext()) ++CSProfileCount; - Profiles[FContext] = FunctionSamples(); - FunctionSamples &FProfile = Profiles[FContext]; + auto Ret = SampleProfileMapTryEmplace(Profiles, FContext); + FunctionSamples &FProfile = Ret.first->second; FProfile.setContext(FContext); MergeResult(Result, FProfile.addTotalSamples(NumSamples)); MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples)); @@ -525,7 +525,7 @@ return *Idx; } -ErrorOr SampleProfileReaderBinary::readStringFromTable() { +ErrorOr SampleProfileReaderBinary::readStringFromTable(size_t *Ret) { auto Idx = readStringIndex(NameTable); if (std::error_code EC = Idx.getError()) return EC; @@ -543,30 +543,46 @@ MD5NameMemStart + (*Idx) * sizeof(uint64_t)); SR = MD5StringBuf.emplace_back(std::to_string(FID)); } + if (Ret) + *Ret = *Idx; return SR; } -ErrorOr SampleProfileReaderBinary::readContextFromTable() { +ErrorOr +SampleProfileReaderBinary::readContextFromTable(size_t *Ret) { auto ContextIdx = readNumber(); if (std::error_code EC = ContextIdx.getError()) return EC; if (*ContextIdx >= CSNameTable.size()) return sampleprof_error::truncated_name_table; + if (Ret) + *Ret = *ContextIdx; return CSNameTable[*ContextIdx]; } -ErrorOr SampleProfileReaderBinary::readSampleContextFromTable() { +ErrorOr> +SampleProfileReaderBinary::readSampleContextFromTable() { + SampleContext Context; + size_t Idx; if (ProfileIsCS) { - auto FContext(readContextFromTable()); + auto FContext(readContextFromTable(&Idx)); if (std::error_code EC = FContext.getError()) return EC; - return SampleContext(*FContext); + Context = SampleContext(*FContext); } else { - auto FName(readStringFromTable()); + auto FName(readStringFromTable(&Idx)); if (std::error_code EC = FName.getError()) return EC; - return SampleContext(*FName); + Context = SampleContext(*FName); } + uint64_t Hash = MD5SampleContextStart[Idx]; + if (Hash == 0) { + assert(MD5SampleContextStart == MD5SampleContextTable.data()); + // Lazy computing of hash value, write back to the table. + Hash = Context.getHashCode(); + MD5SampleContextTable[Idx] = Hash; + } + return std::make_pair(Context, Hash); } std::error_code @@ -659,16 +675,25 @@ if (std::error_code EC = NumHeadSamples.getError()) return EC; - ErrorOr FContext(readSampleContextFromTable()); - if (std::error_code EC = FContext.getError()) + auto FContextHash(readSampleContextFromTable()); + if (std::error_code EC = FContextHash.getError()) return EC; - Profiles[*FContext] = FunctionSamples(); - FunctionSamples &FProfile = Profiles[*FContext]; - FProfile.setContext(*FContext); + auto &[FContext, Hash] = *FContextHash; + // Not using SampleProfileMapTryEmplace() here because we don't want to + // recalculate the hash value of the context, which is expensive. + auto Res = Profiles.try_emplace(Hash, FunctionSamples()); + FunctionSamples &FProfile = Res.first->second; + if (!Res.second) { + if (LLVM_UNLIKELY(FProfile.getContext() != FContext)) + dbgs() << "MD5 collision detected: " << FProfile.getContext().toString() + << " and " << FContext.toString() << " has same hash value " + << Res.first->first << "\n"; + } + FProfile.setContext(FContext); FProfile.addHeadSamples(*NumHeadSamples); - if (FContext->hasContext()) + if (FContext.hasContext()) CSProfileCount++; if (std::error_code EC = readProfile(FProfile)) @@ -816,18 +841,22 @@ FuncOffsetTable.reserve(*Size); for (uint64_t I = 0; I < *Size; ++I) { - auto FContext(readSampleContextFromTable()); - if (std::error_code EC = FContext.getError()) + auto FContextHash(readSampleContextFromTable()); + if (std::error_code EC = FContextHash.getError()) return EC; + auto &[FContext, Hash] = *FContextHash; auto Offset = readNumber(); if (std::error_code EC = Offset.getError()) return EC; if (UseFuncOffsetList) - FuncOffsetList.emplace_back(*FContext, *Offset); - else - FuncOffsetTable[*FContext] = *Offset; + FuncOffsetList.emplace_back(FContext, *Offset); + else { + auto Ret = FuncOffsetTable.try_emplace(Hash, *Offset); + if (LLVM_UNLIKELY(!Ret.second)) + dbgs() << "Duplicated entry or MD5 collision detected\n"; + } } return sampleprof_error::success; @@ -900,8 +929,8 @@ } else if (useMD5()) { assert(!useFuncOffsetList()); for (auto Name : FuncsToUse) { - auto GUID = std::to_string(MD5Hash(Name)); - auto iter = FuncOffsetTable.find(StringRef(GUID)); + auto GUID = MD5Hash(Name); + auto iter = FuncOffsetTable.find(GUID); if (iter == FuncOffsetTable.end()) continue; const uint8_t *FuncProfileAddr = Start + iter->second; @@ -922,7 +951,7 @@ } else { assert(!useFuncOffsetList()); for (auto Name : FuncsToUse) { - auto iter = FuncOffsetTable.find(Name); + auto iter = FuncOffsetTable.find(MD5Hash(Name)); if (iter == FuncOffsetTable.end()) continue; const uint8_t *FuncProfileAddr = Start + iter->second; @@ -1050,17 +1079,30 @@ NameTable.clear(); NameTable.reserve(*Size); + if (!ProfileIsCS) { + MD5SampleContextTable.clear(); + if (UseMD5) + MD5SampleContextTable.reserve(*Size); + else + // If we are using strings, delay MD5 computation since only a portion of + // names are used by top level functions. Use 0 to indicate MD5 value is + // to be calculated as no known string has a MD5 value of 0. + MD5SampleContextTable.resize(*Size); + } for (size_t I = 0; I < *Size; ++I) { auto Name(readString()); if (std::error_code EC = Name.getError()) return EC; if (UseMD5) { uint64_t FID = MD5Hash(*Name); + if (!ProfileIsCS) + MD5SampleContextTable.emplace_back(FID); NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(FID))); } else NameTable.push_back(*Name); } - + if (!ProfileIsCS) + MD5SampleContextStart = MD5SampleContextTable.data(); return sampleprof_error::success; } @@ -1088,6 +1130,8 @@ NameTable.clear(); NameTable.resize(*Size); MD5NameMemStart = Data; + if (!ProfileIsCS) + MD5SampleContextStart = reinterpret_cast(Data); Data = Data + (*Size) * sizeof(uint64_t); return sampleprof_error::success; } @@ -1101,12 +1145,20 @@ MD5StringBuf.reserve(MD5StringBuf.size() + *Size); NameTable.clear(); NameTable.reserve(*Size); + if (!ProfileIsCS) { + MD5SampleContextTable.clear(); + MD5SampleContextTable.reserve(*Size); + } for (size_t I = 0; I < *Size; ++I) { auto FID = readNumber(); if (std::error_code EC = FID.getError()) return EC; + if (!ProfileIsCS) + MD5SampleContextTable.emplace_back(*FID); NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(*FID))); } + if (!ProfileIsCS) + MD5SampleContextStart = MD5SampleContextTable.data(); return sampleprof_error::success; } @@ -1124,6 +1176,14 @@ CSNameTable.clear(); CSNameTable.reserve(*Size); + if (ProfileIsCS) { + // Delay MD5 computation of CS context until they are needed. Use 0 to + // indicate MD5 value is to be calculated as no known string has a MD5 + // value of 0. + MD5SampleContextTable.clear(); + MD5SampleContextTable.resize(*Size); + MD5SampleContextStart = MD5SampleContextTable.data(); + } for (size_t I = 0; I < *Size; ++I) { CSNameTable.emplace_back(SampleContextFrameVector()); auto ContextSize = readNumber(); @@ -1187,16 +1247,17 @@ if (std::error_code EC = Discriminator.getError()) return EC; - auto FContext(readSampleContextFromTable()); - if (std::error_code EC = FContext.getError()) + auto FContextHash(readSampleContextFromTable()); + if (std::error_code EC = FContextHash.getError()) return EC; + auto &[FContext, Hash] = *FContextHash; FunctionSamples *CalleeProfile = nullptr; if (FProfile) { CalleeProfile = const_cast( &FProfile->functionSamplesAt(LineLocation( *LineOffset, - *Discriminator))[std::string(FContext.get().getName())]); + *Discriminator))[std::string(FContext.getName())]); } if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, CalleeProfile)) @@ -1211,12 +1272,13 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) { while (Data < End) { - auto FContext(readSampleContextFromTable()); - if (std::error_code EC = FContext.getError()) + auto FContextHash(readSampleContextFromTable()); + if (std::error_code EC = FContextHash.getError()) return EC; + auto &[FContext, Hash] = *FContextHash; FunctionSamples *FProfile = nullptr; - auto It = Profiles.find(*FContext); - if (It != Profiles.end()) + auto It = Profiles.find(Hash); + if (It != Profiles.end() && It->second.getContext() == FContext) FProfile = &It->second; if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile)) @@ -1605,7 +1667,7 @@ // body, there will be identical replicated profiles for the // original function. In this case, we simply not bother updating // the profile of the original function. - FProfile = &Profiles[Name]; + FProfile = &SampleProfileMapTryEmplace(Profiles, Name).first->second; FProfile->addHeadSamples(HeadCount); if (FProfile->getTotalSamples() > 0) Update = false; diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -364,7 +364,6 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection( const SampleProfileMap &ProfileMap) { for (const auto &I : ProfileMap) { - assert(I.first == I.second.getContext() && "Inconsistent profile map"); addContext(I.second.getContext()); addNames(I.second); } @@ -726,8 +725,7 @@ // Generate the name table for all the functions referenced in the profile. for (const auto &I : ProfileMap) { - assert(I.first == I.second.getContext() && "Inconsistent profile map"); - addContext(I.first); + addContext(I.second.getContext()); addNames(I.second); } diff --git a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp --- a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp +++ b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp @@ -201,7 +201,7 @@ : GUIDToFuncNameMap(GUIDToFuncNameMap) { for (auto &FuncSample : Profiles) { FunctionSamples *FSamples = &FuncSample.second; - SampleContext Context = FuncSample.first; + SampleContext Context = FuncSample.second.getContext(); LLVM_DEBUG(dbgs() << "Tracking Context for function: " << Context.toString() << "\n"); ContextTrieNode *NewNode = getOrCreateContextPath(Context, true); @@ -638,7 +638,8 @@ FunctionSamples *FProfile = Node->getFunctionSamples(); // Profile's context can be empty, use ContextNode's func name. if (FProfile) - ContextLessProfiles[Node->getFuncName()].merge(*FProfile); + SampleProfileMapTryEmplace(ContextLessProfiles, Node->getFuncName()) + .first->second.merge(*FProfile); } } } // namespace llvm diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -472,9 +472,13 @@ private: FunctionSamples *getFlattenedSamplesFor(const Function &F) { StringRef CanonFName = FunctionSamples::getCanonicalFnName(F); - auto It = FlattenedProfiles.find(CanonFName); + uint64_t MD5 = MD5Hash(CanonFName); + auto It = FlattenedProfiles.find(MD5); if (It != FlattenedProfiles.end()) - return &It->second; + // Check for MD5 collision, make sure returning an actual match. + if (LLVM_LIKELY(It->second.getName() == CanonFName || + It->second.getName() == std::to_string(MD5))) + return &It->second; return nullptr; } void runOnFunction(const Function &F, const FunctionSamples &FS); diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -593,7 +593,7 @@ auto checkSampleProfileHasFUnique = [&Reader]() { for (const auto &PD : Reader->getProfiles()) { - auto &FContext = PD.first; + auto &FContext = PD.second.getContext(); if (FContext.toString().find(FunctionSamples::UniqSuffix) != std::string::npos) { return true; @@ -1031,7 +1031,9 @@ : FunctionSamples(); FunctionSamples &Samples = Remapper ? Remapped : I->second; SampleContext FContext = Samples.getContext(); - MergeResult(Result, ProfileMap[FContext].merge(Samples, Input.Weight)); + FunctionSamples &MergedSamples = + SampleProfileMapTryEmplace(ProfileMap, FContext).first->second; + MergeResult(Result, MergedSamples.merge(Samples, Input.Weight)); if (Result != sampleprof_error::success) { std::error_code EC = make_error_code(Result); handleMergeWriterError(errorCodeToError(EC), Input.Filename, @@ -2832,7 +2834,8 @@ "be printed"); // TODO: parse context string to support filtering by contexts. - Reader->dumpFunctionProfile(StringRef(ShowFunction), OS); + FunctionSamples *FS = Reader->getSamplesFor(StringRef(ShowFunction)); + Reader->dumpFunctionProfile(FS ? *FS : FunctionSamples(), OS); } if (ShowProfileSymbolList) { diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp --- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp +++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp @@ -449,7 +449,7 @@ bool ProfileGenerator::collectFunctionsFromLLVMProfile( std::unordered_set &ProfiledFunctions) { for (const auto &FS : ProfileMap) { - if (auto *Func = Binary->getBinaryFunction(FS.first.getName())) + if (auto *Func = Binary->getBinaryFunction(FS.second.getName())) ProfiledFunctions.insert(Func); } return true; @@ -468,7 +468,7 @@ FunctionSamples & ProfileGenerator::getTopLevelFunctionProfile(StringRef FuncName) { SampleContext Context(FuncName); - auto Ret = ProfileMap.emplace(Context, FunctionSamples()); + auto Ret = SampleProfileMapTryEmplace(ProfileMap, Context); if (Ret.second) { FunctionSamples &FProfile = Ret.first->second; FProfile.setContext(Context); @@ -505,14 +505,14 @@ return; // Move cold profiles into a tmp container. - std::vector ColdProfiles; + std::vector ColdProfileHashes; for (const auto &I : ProfileMap) { if (I.second.getTotalSamples() < ColdCntThreshold) - ColdProfiles.emplace_back(I.first); + ColdProfileHashes.emplace_back(I.first); } // Remove the cold profile from ProfileMap. - for (const auto &I : ColdProfiles) + for (const auto &I : ColdProfileHashes) ProfileMap.erase(I); } @@ -964,7 +964,8 @@ Context.emplace_back(Node.getFuncName(), LineLocation(0, 0)); // Save the new context for future references. SampleContextFrames NewContext = *Contexts.insert(Context).first; - auto Ret = ProfileMap.emplace(NewContext, std::move(*FProfile)); + auto Ret = SampleProfileMapTryEmplace(ProfileMap, NewContext, + std::move(*FProfile)); FunctionSamples &NewProfile = Ret.first->second; NewProfile.getContext().setContext(NewContext); Context.pop_back(); @@ -1019,7 +1020,11 @@ // Merge function samples of CS profile to calculate profile density. sampleprof::SampleProfileMap ContextLessProfiles; for (const auto &I : ProfileMap) { - ContextLessProfiles[I.second.getName()].merge(I.second); + auto Ret = SampleProfileMapTryEmplace(ContextLessProfiles, + I.second.getName()); + if (Ret.second) + Ret.first->second.setContext(I.second.getName()); + Ret.first->second.merge(I.second); } calculateAndShowDensity(ContextLessProfiles); diff --git a/llvm/unittests/ProfileData/SampleProfTest.cpp b/llvm/unittests/ProfileData/SampleProfTest.cpp --- a/llvm/unittests/ProfileData/SampleProfTest.cpp +++ b/llvm/unittests/ProfileData/SampleProfTest.cpp @@ -197,10 +197,10 @@ BooSamples.addBodySamples(1, 0, 1232); SampleProfileMap Profiles; - Profiles[FooName] = std::move(FooSamples); - Profiles[BarName] = std::move(BarSamples); - Profiles[BazName] = std::move(BazSamples); - Profiles[BooName] = std::move(BooSamples); + Profiles[MD5Hash(FooName)] = std::move(FooSamples); + Profiles[MD5Hash(BarName)] = std::move(BarSamples); + Profiles[MD5Hash(BazName)] = std::move(BazSamples); + Profiles[MD5Hash(BooName)] = std::move(BooSamples); Module M("my_module", Context); FunctionType *fn_type = @@ -337,7 +337,7 @@ FcnSamples.addTotalSamples(TotalSamples); FcnSamples.addHeadSamples(HeadSamples); FcnSamples.addBodySamples(1, 0, HeadSamples); - (*Smap)[Name] = FcnSamples; + (*Smap)[MD5Hash(Name)] = FcnSamples; } SampleProfileMap setupFcnSamplesForElisionTest(StringRef Policy) {