diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -460,6 +460,8 @@ 0x4, // Leaf of context is duplicated into the base profile }; +using FuncHash = uint64_t; + // Represents a context frame with function name and line location struct SampleContextFrame { StringRef FuncName; @@ -625,8 +627,16 @@ } uint64_t getHashCode() const { - return hasContext() ? hash_value(getContextFrames()) - : hash_value(getName()); + if (hasContext()) + return hash_value(getContextFrames()); + + // For non-context function name, use its MD5 as hash value, so that it is + // consistent with the profile map's key. + // If non-context function is already MD5 string, do not hash again. + uint64_t Hash; + if (getName().getAsInteger(10, Hash)) + Hash = MD5Hash(getName()); + return Hash; } /// Set the name of the function and clear the current context. @@ -1218,10 +1228,9 @@ raw_ostream &operator<<(raw_ostream &OS, const FunctionSamples &FS); -using SampleProfileMap = - std::unordered_map; +using SampleProfileMap = llvm::DenseMap; -using NameFunctionSamples = std::pair; +using NameFunctionSamples = std::pair; void sortFuncProfiles(const SampleProfileMap &ProfileMap, std::vector &SortedProfiles); @@ -1267,8 +1276,6 @@ bool MergeColdContext, uint32_t ColdContextFrameLength, bool TrimBaseProfileOnly); - // Canonicalize context profile name and attributes. - void canonicalizeContextProfiles(); private: SampleProfileMap &ProfileMap; diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -410,30 +410,22 @@ /// Return the samples collected for function \p F, create empty /// FunctionSamples if it doesn't exist. FunctionSamples *getOrCreateSamplesFor(const Function &F) { - std::string FGUID; StringRef CanonName = FunctionSamples::getCanonicalFnName(F); - CanonName = getRepInFormat(CanonName, useMD5(), FGUID); - auto It = Profiles.find(CanonName); - if (It != Profiles.end()) - return &It->second; - if (!FGUID.empty()) { - assert(useMD5() && "New name should only be generated for md5 profile"); - CanonName = *MD5NameBuffer.insert(FGUID).first; - } - return &Profiles[CanonName]; + FuncHash FGUID = Function::getGUID(CanonName); + return &Profiles[FGUID]; } /// Return the samples collected for function \p F. virtual FunctionSamples *getSamplesFor(StringRef Fname) { - std::string FGUID; - Fname = getRepInFormat(Fname, useMD5(), FGUID); - auto It = Profiles.find(Fname); + FuncHash FGUID = Function::getGUID(Fname); + auto It = Profiles.find(FGUID); if (It != Profiles.end()) return &It->second; if (Remapper) { if (auto NameInProfile = Remapper->lookUpNameInProfile(Fname)) { - auto It = Profiles.find(*NameInProfile); + FGUID = Function::getGUID(*NameInProfile); + auto It = Profiles.find(FGUID); if (It != Profiles.end()) return &It->second; } @@ -493,7 +485,7 @@ virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) { return false; }; /// Return whether names in the profile are all MD5 numbers. - virtual bool useMD5() { return false; } + bool useMD5() { return ProfileUsesMD5; } /// Don't read profile without context if the flag is set. This is only /// meaningful for ExtBinary format. @@ -505,6 +497,10 @@ void setModule(const Module *Mod) { M = Mod; } + /// Force the profile to use MD5 in Sample contexts, even if function names + /// are present. + virtual void setProfileUseMD5() { ProfileUsesMD5 = true; } + protected: /// Map every function to its associated profile. /// @@ -521,7 +517,7 @@ /// Extra name buffer holding names created on demand. /// This should only be needed for md5 profiles. - std::unordered_set MD5NameBuffer; + std::list MD5NameBuffer; /// Profile summary information. std::unique_ptr Summary; @@ -563,6 +559,10 @@ /// Zero out the discriminator bits higher than bit MaskedBitFrom (0 based). /// The default is to keep all the bits. uint32_t MaskedBitFrom = 31; + + /// Whether the profile uses MD5 for Sample Conexts for function names. This + /// can be one-way overriden by the user to force use MD5. + bool ProfileUsesMD5 = false; }; class SampleProfileReaderText : public SampleProfileReader { @@ -579,6 +579,10 @@ /// Return true if \p Buffer is in the format supported by this class. static bool hasFormat(const MemoryBuffer &Buffer); + void setProfileUseMD5() override { + assert(false && "Text format sample profile does not support MD5"); + } + private: /// CSNameTable is used to save full context vectors. This serves as an /// underlying immutable buffer for all clients. @@ -622,8 +626,16 @@ /// \returns the read value. ErrorOr readString(); - /// Read the string index and check whether it overflows the table. - template inline ErrorOr readStringIndex(T &Table); + /// Read an index and check whether it overflows the table. + template inline ErrorOr readIndex(T &Table); + + /// Read an index, and get the string and its hash from the name table at the + /// indexed position. + ErrorOr> readStringFromTable(); + + /// Read an index, and get the Sample Context and its hash from the name table + /// or CS name table at the indexed position. + ErrorOr> readSampleContextFromTable(); /// Return true if we've reached the end of file. bool at_eof() const { return Data >= End; } @@ -640,8 +652,40 @@ /// Read profile summary. std::error_code readSummary(); - /// Read the whole name table. - virtual std::error_code readNameTable(); + /// Read the whole name table consists of Strings. + std::error_code readNameTable(); + + /// Read the whole name table consists of ULEB128 encoded MD5 values. + std::error_code readMD5NameTable(); + + /// Whether readFuncOffsetTable() populates the sequential OrderedFuncOffsets, + /// or the map FuncOffsetTable, depends on the following criteria. + /// (a) If profile is CS, the function offset table is expected to consist of + /// sequences of contexts in pre-order layout + /// (e.g. [A, A:1 @ B, A:1 @ B:2.3 @ C] [D, D:1 @ E]), so that when a match in + /// the module is found, all prefixes of the matched function can be loaded, + /// so sequential read is required. + /// (b) If profile is not CS and not MD5, and a remapper is present, each + /// function name in the profile needs to be checked if its remapped name + /// matches a function in the module, so use the sequential container since + /// each entry is accessed. + /// (c) For other cases, the map container is used, so that each function in + /// the module can look up its samples in the profile. + /// TODO: For (c), if the cardinality of function offset table is much smaller + /// than the number of functions in the module, sequential container should be + /// faster, but we need to figure out the constant factor for cutoff. + bool useOrderedFuncOffsets() { + return profileIsCS() || (!useMD5() && getRemapper()); + } + + /// Read the whole function offset table from the profile file, and store it + /// to either OrderedFuncOffsets (sequential) or FuncOffsetTable (map). + /// Clear the previous function offset table. + std::error_code readFuncOffsetTable(); + + /// Read function profiles. If a module is present and function offset table + /// is available, only read profiles of functions present in the module. + std::error_code readFuncProfiles(); /// Points to the current location in the buffer. const uint8_t *Data = nullptr; @@ -649,12 +693,36 @@ /// Points to the end of the buffer. const uint8_t *End = nullptr; - /// Function name table. + /// MD5 hash of function name table. This is the primary key in mappings to + /// retrieve info of a function. Unused if MD5NameMemStart points to fixed + /// length MD5. + std::vector MD5NameTable; + + /// Use this to index into MD5 function names, normally points to the start of + /// MD5NameTable, but if the profile file uses fixed length MD5 names this + /// points to the start of the section and MD5NameTable is not materialized. + const FuncHash *MD5NameMemStart = nullptr; + + /// Function name table storing the actual names of functions, use lazy + /// loading by readStringFromTable(). std::vector NameTable; - /// Read a string indirectly via the name table. - virtual ErrorOr readStringFromTable(); - virtual ErrorOr readSampleContextFromTable(); + /// MD5 hash of CS name tbale. This is the primary to retrieve info of a + /// context sensitive function. + std::vector MD5CSNameTable; + + /// Backing buffer for context vectors, since SampleContext takes an ArrayRef. + std::vector CSNameTableBuffer; + + /// The set containing the functions to use when compiling a module. + DenseSet FuncsToUse; + + /// The table mapping from the MD5 of function context to the offset of its + /// FunctionSample towards file start. + DenseMap FuncOffsetTable; + + /// Function offset mapping ordered by contexts. + std::vector> OrderedFuncOffsets; private: std::error_code readSummaryEntry(std::vector &Entries); @@ -710,8 +778,7 @@ std::error_code readFuncMetadata(bool ProfileHasAttribute); std::error_code readFuncMetadata(bool ProfileHasAttribute, FunctionSamples *FProfile); - std::error_code readFuncOffsetTable(); - std::error_code readFuncProfiles(); + std::error_code readNameTableSec(bool IsMD5, bool FixedLengthMD5); std::error_code readCSNameTableSec(); std::error_code readProfileSymbolList(); @@ -722,45 +789,13 @@ const SecHdrTableEntry &Entry); // placeholder for subclasses to dispatch their own section readers. virtual std::error_code readCustomSection(const SecHdrTableEntry &Entry) = 0; - ErrorOr readStringFromTable() override; - ErrorOr readSampleContextFromTable() override; - ErrorOr readContextFromTable(); std::unique_ptr ProfSymList; - /// The table mapping from function context to the offset of its - /// FunctionSample towards file start. - DenseMap FuncOffsetTable; - - /// Function offset mapping ordered by contexts. - std::unique_ptr>> - OrderedFuncOffsets; - - /// The set containing the functions to use when compiling a module. - DenseSet FuncsToUse; - - /// The starting address of NameTable containing fixed length MD5. - const uint8_t *MD5NameMemStart = nullptr; - - /// If MD5 is used in NameTable section, the section saves uint64_t data. - /// The uint64_t data has to be converted to a string and then the string - /// will be used to initialize StringRef in NameTable. - /// Note NameTable contains StringRef so it needs another buffer to own - /// the string data. MD5StringBuf serves as the string buffer that is - /// referenced by NameTable (vector of StringRef). We make sure - /// the lifetime of MD5StringBuf is not shorter than that of NameTable. - std::unique_ptr> MD5StringBuf; - - /// CSNameTable is used to save full context vectors. This serves as an - /// underlying immutable buffer for all clients. - std::unique_ptr> CSNameTable; - /// If SkipFlatProf is true, skip the sections with /// SecFlagFlat flag. bool SkipFlatProf = false; - bool FuncOffsetsOrdered = false; - public: SampleProfileReaderExtBinaryBase(std::unique_ptr B, LLVMContext &C, SampleProfileFormat Format) @@ -779,9 +814,6 @@ /// the reader has been given a module. bool collectFuncsFromModule() override; - /// Return whether names in the profile are all MD5 numbers. - bool useMD5() override { return MD5StringBuf.get(); } - std::unique_ptr getProfileSymbolList() override { return std::move(ProfSymList); }; @@ -809,24 +841,15 @@ class SampleProfileReaderCompactBinary : public SampleProfileReaderBinary { private: - /// Function name table. - std::vector NameTable; - /// The table mapping from function name to the offset of its FunctionSample - /// towards file start. - DenseMap FuncOffsetTable; - /// The set containing the functions to use when compiling a module. - DenseSet FuncsToUse; std::error_code verifySPMagic(uint64_t Magic) override; - std::error_code readNameTable() override; - /// Read a string indirectly via the name table. - ErrorOr readStringFromTable() override; std::error_code readHeader() override; - std::error_code readFuncOffsetTable(); public: SampleProfileReaderCompactBinary(std::unique_ptr B, LLVMContext &C) - : SampleProfileReaderBinary(std::move(B), C, SPF_Compact_Binary) {} + : SampleProfileReaderBinary(std::move(B), C, SPF_Compact_Binary) { + ProfileUsesMD5 = true; + } /// \brief Return true if \p Buffer is in the format supported by this class. static bool hasFormat(const MemoryBuffer &Buffer); @@ -837,9 +860,6 @@ /// Collect functions with definitions in Module M. Return true if /// the reader has been given a module. bool collectFuncsFromModule() override; - - /// Return whether names in the profile are all MD5 numbers. - bool useMD5() override { return true; } }; using InlineCallStack = SmallVector; diff --git a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp --- a/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp +++ b/llvm/lib/ProfileData/ProfileSummaryBuilder.cpp @@ -205,7 +205,8 @@ if (UseContextLessSummary || (sampleprof::FunctionSamples::ProfileIsCS && !UseContextLessSummary.getNumOccurrences())) { for (const auto &I : Profiles) { - ContextLessProfiles[I.second.getName()].merge(I.second); + ContextLessProfiles[SampleContext(I.second.getName()).getHashCode()] + .merge(I.second); } ProfilesToUse = &ContextLessProfiles; } diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -202,13 +202,12 @@ const SampleProfileMap &ProfileMap, std::vector &SortedProfiles) { for (const auto &I : ProfileMap) { - assert(I.first == I.second.getContext() && "Inconsistent profile map"); - SortedProfiles.push_back(std::make_pair(I.second.getContext(), &I.second)); + SortedProfiles.push_back(std::make_pair(I.first, &I.second)); } llvm::stable_sort(SortedProfiles, [](const NameFunctionSamples &A, const NameFunctionSamples &B) { if (A.second->getTotalSamples() == B.second->getTotalSamples()) - return A.first < B.first; + return A.second->getContext() < B.second->getContext(); return A.second->getTotalSamples() > B.second->getTotalSamples(); }); } @@ -357,13 +356,13 @@ // Filter the cold profiles from ProfileMap and move them into a tmp // container - std::vector> ColdProfiles; + std::vector> ColdProfiles; for (const auto &I : ProfileMap) { - const SampleContext &Context = I.first; + const SampleContext &Context = I.second.getContext(); const FunctionSamples &FunctionProfile = I.second; if (FunctionProfile.getTotalSamples() < ColdCountThreshold && (!TrimBaseProfileOnly || Context.isBaseContext())) - ColdProfiles.emplace_back(Context, &I.second); + ColdProfiles.emplace_back(I.first, &I.second); } // Remove the cold profile from ProfileMap and merge them into @@ -374,7 +373,10 @@ auto MergedContext = I.second->getContext().getContextFrames(); if (ColdContextFrameLength < MergedContext.size()) MergedContext = MergedContext.take_back(ColdContextFrameLength); - auto Ret = MergedProfileMap.emplace(MergedContext, FunctionSamples()); + FunctionSamples NewFunctionSamples; + NewFunctionSamples.setContext(MergedContext); + auto Ret = MergedProfileMap.insert(std::make_pair( + SampleContext(MergedContext).getHashCode(), NewFunctionSamples)); FunctionSamples &MergedProfile = Ret.first->second; MergedProfile.merge(*I.second); } @@ -389,53 +391,12 @@ continue; // Merge the profile if the original profile exists, otherwise just insert // as a new profile - auto Ret = ProfileMap.emplace(I.first, FunctionSamples()); - if (Ret.second) { - SampleContext FContext(Ret.first->first, RawContext); - FunctionSamples &FProfile = Ret.first->second; - FProfile.setContext(FContext); - } + auto Ret = ProfileMap.insert(std::make_pair(I.first, FunctionSamples())); FunctionSamples &OrigProfile = Ret.first->second; OrigProfile.merge(I.second); } } -void SampleContextTrimmer::canonicalizeContextProfiles() { - std::vector ProfilesToBeRemoved; - SampleProfileMap ProfilesToBeAdded; - for (auto &I : ProfileMap) { - FunctionSamples &FProfile = I.second; - SampleContext &Context = FProfile.getContext(); - if (I.first == Context) - continue; - - // Use the context string from FunctionSamples to update the keys of - // ProfileMap. They can get out of sync after context profile promotion - // through pre-inliner. - // Duplicate the function profile for later insertion to avoid a conflict - // caused by a context both to be add and to be removed. This could happen - // when a context is promoted to another context which is also promoted to - // the third context. For example, given an original context A @ B @ C that - // is promoted to B @ C and the original context B @ C which is promoted to - // just C, adding B @ C to the profile map while removing same context (but - // with different profiles) from the map can cause a conflict if they are - // not handled in a right order. This can be solved by just caching the - // profiles to be added. - auto Ret = ProfilesToBeAdded.emplace(Context, FProfile); - (void)Ret; - assert(Ret.second && "Context conflict during canonicalization"); - ProfilesToBeRemoved.push_back(I.first); - } - - for (auto &I : ProfilesToBeRemoved) { - ProfileMap.erase(I); - } - - for (auto &I : ProfilesToBeAdded) { - ProfileMap.emplace(I.first, I.second); - } -} - std::error_code ProfileSymbolList::write(raw_ostream &OS) { // Sort the symbols before output. If doing compression. // It will make the compression much more effective. @@ -531,16 +492,16 @@ // base profiles improves the code quality for thinlto build by allowing a // profile in the prelink phase for to-be-fully-inlined functions. if (!NodeProfile) { - ProfileMap[ChildProfile->getContext()].merge(*ChildProfile); + ProfileMap[ChildProfile->getContext().getHashCode()].merge(*ChildProfile); } else if (GenerateMergedBaseProfiles) { - ProfileMap[ChildProfile->getContext()].merge(*ChildProfile); + ProfileMap[ChildProfile->getContext().getHashCode()].merge(*ChildProfile); auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc); SamplesMap[ChildProfile->getName().str()].getContext().setAttribute( ContextDuplicatedIntoBase); } // Remove the original child profile. - ProfileMap.erase(OrigChildContext); + ProfileMap.erase(OrigChildContext.getHashCode()); } } diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -57,13 +57,14 @@ "profile-isfs", cl::Hidden, cl::init(false), cl::desc("Profile uses flow sensitive discriminators")); -/// Dump the function profile for \p FName. +/// Dump the function profile for \p FContext. /// /// \param FContext Name + context of the function to print. /// \param OS Stream to emit the output to. void SampleProfileReader::dumpFunctionProfile(SampleContext FContext, raw_ostream &OS) { - OS << "Function: " << FContext.toString() << ": " << Profiles[FContext]; + OS << "Function: " << FContext.toString() << ": " + << Profiles[FContext.getHashCode()]; } /// Dump all the function profiles found on stream \p OS. @@ -71,7 +72,7 @@ std::vector V; sortFuncProfiles(Profiles, V); for (const auto &I : V) - dumpFunctionProfile(I.first, OS); + dumpFunctionProfile(I.second->getContext(), OS); } static void dumpFunctionProfileJson(const FunctionSamples &S, @@ -355,8 +356,7 @@ SampleContext FContext(FName, CSNameTable); if (FContext.hasContext()) ++CSProfileCount; - Profiles[FContext] = FunctionSamples(); - FunctionSamples &FProfile = Profiles[FContext]; + FunctionSamples &FProfile = Profiles[FContext.getHashCode()]; FProfile.setContext(FContext); MergeResult(Result, FProfile.addTotalSamples(NumSamples)); MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples)); @@ -515,7 +515,7 @@ } template -inline ErrorOr SampleProfileReaderBinary::readStringIndex(T &Table) { +inline ErrorOr SampleProfileReaderBinary::readIndex(T &Table) { std::error_code EC; auto Idx = readNumber(); if (std::error_code EC = Idx.getError()) @@ -525,56 +525,37 @@ return *Idx; } -ErrorOr SampleProfileReaderBinary::readStringFromTable() { - auto Idx = readStringIndex(NameTable); +ErrorOr> +SampleProfileReaderBinary::readStringFromTable() { + auto Idx = readIndex(NameTable); if (std::error_code EC = Idx.getError()) return EC; + StringRef Str = NameTable[*Idx]; + if (Str.data()) + return std::make_pair(Str, MD5NameMemStart[*Idx]); - return NameTable[*Idx]; + // Fixed length MD5 uses lazy loading, materialize MD5 as string on first + // access. + std::string &NewStr = + MD5NameBuffer.emplace_back(std::to_string(MD5NameMemStart[*Idx])); + NameTable[*Idx] = NewStr; + return std::make_pair(StringRef(NewStr), MD5NameMemStart[*Idx]); } -ErrorOr SampleProfileReaderBinary::readSampleContextFromTable() { - auto FName(readStringFromTable()); - if (std::error_code EC = FName.getError()) - return EC; - return SampleContext(*FName); -} - -ErrorOr SampleProfileReaderExtBinaryBase::readStringFromTable() { - // read NameTable index. - auto Idx = readStringIndex(NameTable); - if (std::error_code EC = Idx.getError()) - return EC; - - // Check whether the name to be accessed has been accessed before, - // if not, read it from memory directly. - StringRef &SR = NameTable[*Idx]; - if (!SR.data()) { - assert(MD5NameMemStart); - const uint8_t *SavedData = Data; - const uint8_t *SavedEnd = End; - Data = MD5NameMemStart + (*Idx) * sizeof(uint64_t); - End = (const uint8_t *)std::numeric_limits::max(); - auto FID = readUnencodedNumber(); - if (std::error_code EC = FID.getError()) +ErrorOr> +SampleProfileReaderBinary::readSampleContextFromTable() { + if (ProfileIsCS) { + auto Idx = readIndex(MD5CSNameTable); + if (std::error_code EC = Idx.getError()) return EC; - // Save the string converted from uint64_t in MD5StringBuf. All the - // references to the name are all StringRefs refering to the string - // in MD5StringBuf. - MD5StringBuf->push_back(std::to_string(*FID)); - SR = MD5StringBuf->back(); - Data = SavedData; - End = SavedEnd; + return std::make_pair(SampleContext(CSNameTableBuffer[*Idx]), + MD5CSNameTable[*Idx]); } - return SR; -} - -ErrorOr SampleProfileReaderCompactBinary::readStringFromTable() { - auto Idx = readStringIndex(NameTable); - if (std::error_code EC = Idx.getError()) + auto StrHash = readStringFromTable(); + if (std::error_code EC = StrHash.getError()) return EC; - - return StringRef(NameTable[*Idx]); + auto [Str, Hash] = *StrHash; + return std::make_pair(SampleContext(Str), Hash); } std::error_code @@ -590,7 +571,7 @@ return EC; for (uint32_t I = 0; I < *NumRecords; ++I) { - auto LineOffset = readNumber(); + auto LineOffset = readNumber(); if (std::error_code EC = LineOffset.getError()) return EC; @@ -598,7 +579,7 @@ return std::error_code(); } - auto Discriminator = readNumber(); + auto Discriminator = readNumber(); if (std::error_code EC = Discriminator.getError()) return EC; @@ -614,16 +595,17 @@ uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask(); for (uint32_t J = 0; J < *NumCalls; ++J) { - auto CalledFunction(readStringFromTable()); + auto CalledFunction = readStringFromTable(); if (std::error_code EC = CalledFunction.getError()) return EC; + auto [FName, Hash] = *CalledFunction; auto CalledFunctionSamples = readNumber(); if (std::error_code EC = CalledFunctionSamples.getError()) return EC; - FProfile.addCalledTargetSamples(*LineOffset, DiscriminatorVal, - *CalledFunction, *CalledFunctionSamples); + FProfile.addCalledTargetSamples(*LineOffset, DiscriminatorVal, FName, + *CalledFunctionSamples); } FProfile.addBodySamples(*LineOffset, DiscriminatorVal, *NumSamples); @@ -643,16 +625,17 @@ if (std::error_code EC = Discriminator.getError()) return EC; - auto FName(readStringFromTable()); - if (std::error_code EC = FName.getError()) + auto FNameHash = readStringFromTable(); + if (std::error_code EC = FNameHash.getError()) return EC; + auto [FName, Hash] = *FNameHash; // Here we handle FS discriminators: uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask(); FunctionSamples &CalleeProfile = FProfile.functionSamplesAt( - LineLocation(*LineOffset, DiscriminatorVal))[std::string(*FName)]; - CalleeProfile.setName(*FName); + LineLocation(*LineOffset, DiscriminatorVal))[FName.str()]; + CalleeProfile.setName(FName); if (std::error_code EC = readProfile(CalleeProfile)) return EC; } @@ -667,16 +650,25 @@ if (std::error_code EC = NumHeadSamples.getError()) return EC; - ErrorOr FContext(readSampleContextFromTable()); - if (std::error_code EC = FContext.getError()) + auto FContextHash = readSampleContextFromTable(); + if (std::error_code EC = FContextHash.getError()) return EC; - - Profiles[*FContext] = FunctionSamples(); - FunctionSamples &FProfile = Profiles[*FContext]; - FProfile.setContext(*FContext); + auto [FContext, Hash] = *FContextHash; + + FunctionSamples &FProfile = Profiles[Hash]; + if (FProfile.getContext().getName().data()) { + if (FProfile.getContext() != FContext) + LLVM_DEBUG(errs() << "Hash collision detected: " + << FProfile.getContext().toString() << " and " + << FContext.toString() << " has same hash value " + << Hash << "\n"); + else + assert(false && "Duplicated sample conext in profile"); + } + FProfile.setContext(FContext); FProfile.addHeadSamples(*NumHeadSamples); - if (FContext->hasContext()) + if (FContext.hasContext()) CSProfileCount++; if (std::error_code EC = readProfile(FProfile)) @@ -684,40 +676,101 @@ return sampleprof_error::success; } -std::error_code SampleProfileReaderBinary::readImpl() { - ProfileIsFS = ProfileIsFSDisciminator; - FunctionSamples::ProfileIsFS = ProfileIsFS; - while (!at_eof()) { - if (std::error_code EC = readFuncProfile(Data)) +std::error_code SampleProfileReaderBinary::readNameTable() { + auto Size = readNumber(); + if (std::error_code EC = Size.getError()) + return EC; + + MD5NameTable.clear(); + MD5NameTable.reserve(*Size); + NameTable.clear(); + NameTable.reserve(*Size); + bool UseMD5 = useMD5(); + // If forced to use MD5, function name strings are not created and lazy + // loading will handle it. + if (UseMD5) + NameTable.resize(*Size); + for (uint64_t I = 0; I < *Size; ++I) { + auto Name(readString()); + if (std::error_code EC = Name.getError()) return EC; + FuncHash MD5 = MD5Hash(*Name); + MD5NameTable.push_back(MD5); + if (!UseMD5) + NameTable.push_back(*Name); } + MD5NameMemStart = MD5NameTable.data(); return sampleprof_error::success; } -ErrorOr -SampleProfileReaderExtBinaryBase::readContextFromTable() { - auto ContextIdx = readNumber(); - if (std::error_code EC = ContextIdx.getError()) +std::error_code SampleProfileReaderBinary::readMD5NameTable() { + auto Size = readNumber(); + if (std::error_code EC = Size.getError()) return EC; - if (*ContextIdx >= CSNameTable->size()) - return sampleprof_error::truncated_name_table; - return (*CSNameTable)[*ContextIdx]; + + MD5NameTable.clear(); + MD5NameTable.reserve(*Size); + // Lazy loading, the MD5 string is created only when being accessed on the + // first time, which the StringRef has a null data pointer. + NameTable.clear(); + NameTable.resize(*Size); + for (uint64_t I = 0; I < *Size; ++I) { + auto Name(readNumber()); + if (std::error_code EC = Name.getError()) + return EC; + MD5NameTable.push_back(*Name); + } + MD5NameMemStart = MD5NameTable.data(); + + return sampleprof_error::success; } -ErrorOr -SampleProfileReaderExtBinaryBase::readSampleContextFromTable() { - if (ProfileIsCS) { - auto FContext(readContextFromTable()); - if (std::error_code EC = FContext.getError()) +std::error_code SampleProfileReaderBinary::readFuncOffsetTable() { + // If there are more than one function offset table, the profile associated + // with the previous table has to be done reading before next one + // is read. + FuncOffsetTable.clear(); + OrderedFuncOffsets.clear(); + + auto Size = readNumber(); + if (std::error_code EC = Size.getError()) + return EC; + + bool UseOrderedFuncOffsets = useOrderedFuncOffsets(); + if (UseOrderedFuncOffsets) + OrderedFuncOffsets.reserve(*Size); + else + FuncOffsetTable.reserve(*Size); + + for (uint64_t I = 0; I < *Size; ++I) { + auto FContextHash = readSampleContextFromTable(); + if (std::error_code EC = FContextHash.getError()) return EC; - return SampleContext(*FContext); - } else { - auto FName(readStringFromTable()); - if (std::error_code EC = FName.getError()) + auto [FContext, Hash] = *FContextHash; + + auto Offset = readNumber(); + if (std::error_code EC = Offset.getError()) + return EC; + + if (UseOrderedFuncOffsets) + OrderedFuncOffsets.emplace_back(FContext, *Offset); + else + FuncOffsetTable[Hash] = *Offset; + } + + return sampleprof_error::success; +} + +std::error_code SampleProfileReaderBinary::readImpl() { + ProfileIsFS = ProfileIsFSDisciminator; + FunctionSamples::ProfileIsFS = ProfileIsFS; + while (!at_eof()) { + if (std::error_code EC = readFuncProfile(Data)) return EC; - return SampleContext(*FName); } + + return sampleprof_error::success; } std::error_code SampleProfileReaderExtBinaryBase::readOneSection( @@ -759,11 +812,22 @@ if (std::error_code EC = readFuncProfiles()) return EC; break; - case SecFuncOffsetTable: - FuncOffsetsOrdered = hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered); + case SecFuncOffsetTable: { + // If a module is absent, there is no need to read function offset table + // because every profile must be loaded anyways. + if (!M) { + Data = End; + return sampleprof_error::success; + } + bool FuncOffsetsOrdered = + hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered); + if (profileIsCS() && !FuncOffsetsOrdered) + assert(false && + "func offset table should always be sorted in CS profile"); if (std::error_code EC = readFuncOffsetTable()) return EC; break; + } case SecFuncMetadata: { ProfileIsProbeBased = hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased); @@ -795,42 +859,7 @@ return true; } -std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() { - // If there are more than one FuncOffsetTable, the profile read associated - // with previous FuncOffsetTable has to be done before next FuncOffsetTable - // is read. - FuncOffsetTable.clear(); - - auto Size = readNumber(); - if (std::error_code EC = Size.getError()) - return EC; - - FuncOffsetTable.reserve(*Size); - - if (FuncOffsetsOrdered) { - OrderedFuncOffsets = - std::make_unique>>(); - OrderedFuncOffsets->reserve(*Size); - } - - for (uint64_t I = 0; I < *Size; ++I) { - auto FContext(readSampleContextFromTable()); - if (std::error_code EC = FContext.getError()) - return EC; - - auto Offset = readNumber(); - if (std::error_code EC = Offset.getError()) - return EC; - - FuncOffsetTable[*FContext] = *Offset; - if (FuncOffsetsOrdered) - OrderedFuncOffsets->emplace_back(*FContext, *Offset); - } - - return sampleprof_error::success; -} - -std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() { +std::error_code SampleProfileReaderBinary::readFuncProfiles() { // Collect functions used by current module if the Reader has been // given a module. // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName @@ -839,9 +868,16 @@ // NameTable section is read. bool LoadFuncsToBeUsed = collectFuncsFromModule(); - // When LoadFuncsToBeUsed is false, load all the function profiles. + bool UseOrderedFuncOffsets = useOrderedFuncOffsets(); + + // If function offset container is not present, we can't random access + // profiles, and if a module is not present, we don't know which profiles are + // needed, so loading all of them. const uint8_t *Start = Data; - if (!LoadFuncsToBeUsed) { + if (!LoadFuncsToBeUsed || + (UseOrderedFuncOffsets && OrderedFuncOffsets.empty()) || + (!UseOrderedFuncOffsets && FuncOffsetTable.empty())) { + while (Data < End) { if (std::error_code EC = readFuncProfile(Data)) return EC; @@ -869,10 +905,9 @@ // as if they were walked in preorder of a context trie. While // traversing the trie, a link to the highest common ancestor node is // kept so that all of its decendants will be loaded. - assert(OrderedFuncOffsets.get() && - "func offset table should always be sorted in CS profile"); + const SampleContext *CommonContext = nullptr; - for (const auto &NameOffset : *OrderedFuncOffsets) { + for (const auto &NameOffset : OrderedFuncOffsets) { const auto &FContext = NameOffset.first; auto FName = FContext.getName(); // For function in the current module, keep its farthest ancestor @@ -896,10 +931,9 @@ } } } else { - if (useMD5()) { + if (!UseOrderedFuncOffsets) { for (auto Name : FuncsToUse) { - auto GUID = std::to_string(MD5Hash(Name)); - auto iter = FuncOffsetTable.find(StringRef(GUID)); + auto iter = FuncOffsetTable.find(MD5Hash(Name)); if (iter == FuncOffsetTable.end()) continue; const uint8_t *FuncProfileAddr = Start + iter->second; @@ -908,11 +942,9 @@ return EC; } } else { - for (auto NameOffset : FuncOffsetTable) { - SampleContext FContext(NameOffset.first); - auto FuncName = FContext.getName(); - if (!FuncsToUse.count(FuncName) && - (!Remapper || !Remapper->exist(FuncName))) + for (auto NameOffset : OrderedFuncOffsets) { + auto FuncName = NameOffset.first.getName(); + if (!FuncsToUse.count(FuncName) && (!Remapper->exist(FuncName))) continue; const uint8_t *FuncProfileAddr = Start + NameOffset.second; assert(FuncProfileAddr < End && "out of LBRProfile section"); @@ -1015,37 +1047,28 @@ } std::error_code SampleProfileReaderCompactBinary::readImpl() { - // Collect functions used by current module if the Reader has been - // given a module. - bool LoadFuncsToBeUsed = collectFuncsFromModule(); - ProfileIsFS = ProfileIsFSDisciminator; - FunctionSamples::ProfileIsFS = ProfileIsFS; - std::vector OffsetsToUse; - if (!LoadFuncsToBeUsed) { - // load all the function profiles. - for (auto FuncEntry : FuncOffsetTable) { - OffsetsToUse.push_back(FuncEntry.second); - } - } else { - // load function profiles on demand. - for (auto Name : FuncsToUse) { - auto GUID = std::to_string(MD5Hash(Name)); - auto iter = FuncOffsetTable.find(StringRef(GUID)); - if (iter == FuncOffsetTable.end()) - continue; - OffsetsToUse.push_back(iter->second); - } - } + auto TableOffset = readUnencodedNumber(); + if (std::error_code EC = TableOffset.getError()) + return EC; + + const uint8_t *TableStart = + reinterpret_cast(Buffer->getBufferStart()) + + *TableOffset; - for (auto Offset : OffsetsToUse) { - const uint8_t *SavedData = Data; - if (std::error_code EC = readFuncProfile( - reinterpret_cast(Buffer->getBufferStart()) + - Offset)) + // Only read function offset table if module is present, otherwise all + // profiles must be loaded anyways. In this case offset is from the beginning + // of the file. + if (M) { + Data = TableStart; + if (std::error_code EC = readFuncOffsetTable()) return EC; - Data = SavedData; + Data = reinterpret_cast(Buffer->getBufferStart()); } - return sampleprof_error::success; + End = TableStart; + + ProfileIsFS = ProfileIsFSDisciminator; + FunctionSamples::ProfileIsFS = ProfileIsFS; + return readFuncProfiles(); } std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) { @@ -1067,61 +1090,34 @@ return sampleprof_error::bad_magic; } -std::error_code SampleProfileReaderBinary::readNameTable() { - auto Size = readNumber(); - if (std::error_code EC = Size.getError()) - return EC; - NameTable.clear(); - NameTable.reserve(*Size); - for (uint64_t I = 0; I < *Size; ++I) { - auto Name(readString()); - if (std::error_code EC = Name.getError()) - return EC; - NameTable.push_back(*Name); - } - - return sampleprof_error::success; -} - std::error_code SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5, bool FixedLengthMD5) { - if (!IsMD5) - return SampleProfileReaderBinary::readNameTable(); + if (!FixedLengthMD5) { + if (IsMD5) + return SampleProfileReaderBinary::readMD5NameTable(); + else + return SampleProfileReaderBinary::readNameTable(); + } auto Size = readNumber(); if (std::error_code EC = Size.getError()) return EC; - MD5StringBuf = std::make_unique>(); - MD5StringBuf->reserve(*Size); - if (FixedLengthMD5) { - assert(Data + (*Size) * sizeof(uint64_t) == End && - "Fixed length MD5 name table does not contain specified number of " - "entries"); - if (Data + (*Size) * sizeof(uint64_t) > End) - return sampleprof_error::truncated; - // Preallocate and initialize NameTable so we can check whether a name - // index has been read before by checking whether the element in the - // NameTable is empty, meanwhile readStringIndex can do the boundary - // check using the size of NameTable. - NameTable.resize(*Size); + assert(Data + (*Size) * sizeof(uint64_t) == End && + "Fixed length MD5 name table does not contain specified number of " + "entries"); + if (Data + (*Size) * sizeof(uint64_t) > End) + return sampleprof_error::truncated; - MD5NameMemStart = Data; - Data = Data + (*Size) * sizeof(uint64_t); - return sampleprof_error::success; - } + // Preallocate and initialize NameTable so we can check whether a name + // index has been read before by checking whether the element in the + // NameTable is empty, meanwhile readStringIndex can do the boundary + // check using the size of NameTable. NameTable.clear(); - NameTable.reserve(*Size); - for (uint64_t I = 0; I < *Size; ++I) { - auto FID = readNumber(); - if (std::error_code EC = FID.getError()) - return EC; - MD5StringBuf->push_back(std::to_string(*FID)); - // NameTable is a vector of StringRef. Here it is pushing back a - // StringRef initialized with the last string in MD5stringBuf. - NameTable.push_back(MD5StringBuf->back()); - } + NameTable.resize(*Size); + MD5NameMemStart = reinterpret_cast(Data); + Data = Data + (*Size) * sizeof(uint64_t); return sampleprof_error::success; } @@ -1134,41 +1130,44 @@ if (std::error_code EC = Size.getError()) return EC; - std::vector *PNameVec = - new std::vector(); - PNameVec->reserve(*Size); + MD5CSNameTable.clear(); + MD5CSNameTable.reserve(*Size); + CSNameTableBuffer.clear(); + CSNameTableBuffer.reserve(*Size); for (uint64_t I = 0; I < *Size; ++I) { - PNameVec->emplace_back(SampleContextFrameVector()); + auto &FrameVec = CSNameTableBuffer.emplace_back(SampleContextFrameVector()); auto ContextSize = readNumber(); if (std::error_code EC = ContextSize.getError()) return EC; + for (uint32_t J = 0; J < *ContextSize; ++J) { - auto FName(readStringFromTable()); - if (std::error_code EC = FName.getError()) + auto FNameHash = readStringFromTable(); + if (std::error_code EC = FNameHash.getError()) return EC; - auto LineOffset = readNumber(); + auto [FName, Hash] = *FNameHash; + + auto LineOffset = readNumber(); if (std::error_code EC = LineOffset.getError()) return EC; if (!isOffsetLegal(*LineOffset)) return std::error_code(); - auto Discriminator = readNumber(); + auto Discriminator = readNumber(); if (std::error_code EC = Discriminator.getError()) return EC; - PNameVec->back().emplace_back( - FName.get(), LineLocation(LineOffset.get(), Discriminator.get())); + LineLocation Loc(LineOffset.get(), Discriminator.get()); + + FrameVec.emplace_back(FName, Loc); } + MD5CSNameTable.push_back(SampleContextFrameHash()(FrameVec)); } - // From this point the underlying object of CSNameTable should be immutable. - CSNameTable.reset(PNameVec); return sampleprof_error::success; } std::error_code - SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute, FunctionSamples *FProfile) { if (Data < End) { @@ -1195,24 +1194,24 @@ return EC; for (uint32_t J = 0; J < *NumCallsites; ++J) { - auto LineOffset = readNumber(); + auto LineOffset = readNumber(); if (std::error_code EC = LineOffset.getError()) return EC; - auto Discriminator = readNumber(); + auto Discriminator = readNumber(); if (std::error_code EC = Discriminator.getError()) return EC; - auto FContext(readSampleContextFromTable()); - if (std::error_code EC = FContext.getError()) + auto FNameHash = readStringFromTable(); + if (std::error_code EC = FNameHash.getError()) return EC; + auto [FName, Hash] = *FNameHash; FunctionSamples *CalleeProfile = nullptr; if (FProfile) { - CalleeProfile = const_cast( - &FProfile->functionSamplesAt(LineLocation( - *LineOffset, - *Discriminator))[std::string(FContext.get().getName())]); + CalleeProfile = + const_cast(&FProfile->functionSamplesAt( + LineLocation(*LineOffset, *Discriminator))[FName.str()]); } if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, CalleeProfile)) @@ -1227,11 +1226,12 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) { while (Data < End) { - auto FContext(readSampleContextFromTable()); - if (std::error_code EC = FContext.getError()) + auto FContextHash = readSampleContextFromTable(); + if (std::error_code EC = FContextHash.getError()) return EC; + auto [FContext, Hash] = *FContextHash; FunctionSamples *FProfile = nullptr; - auto It = Profiles.find(*FContext); + auto It = Profiles.find(Hash); if (It != Profiles.end()) FProfile = &It->second; @@ -1243,21 +1243,6 @@ return sampleprof_error::success; } -std::error_code SampleProfileReaderCompactBinary::readNameTable() { - auto Size = readNumber(); - if (std::error_code EC = Size.getError()) - return EC; - NameTable.clear(); - NameTable.reserve(*Size); - for (uint64_t I = 0; I < *Size; ++I) { - auto FID = readNumber(); - if (std::error_code EC = FID.getError()) - return EC; - NameTable.push_back(std::to_string(*FID)); - } - return sampleprof_error::success; -} - std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint64_t Idx) { SecHdrTableEntry Entry; @@ -1310,23 +1295,22 @@ if (std::error_code EC = readSecHdrTable()) return EC; - bool HasMD5 = false; bool HasNonMD5 = false; for (auto &Entry : SecHdrTable) { if (Entry.Size && Entry.Type == SecNameTable) { bool IsMD5 = hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name) || hasSecFlag(Entry, SecNameTableFlags::SecFlagFixedLengthMD5); if (IsMD5) { - HasMD5 = true; + ProfileUsesMD5 = true; } else HasNonMD5 = true; } } - if (HasMD5 != HasNonMD5) { - assert(!(HasMD5 && HasNonMD5) && + if (ProfileUsesMD5 != HasNonMD5) { + assert(!(ProfileUsesMD5 && HasNonMD5) && "Profile contains both MD5 and non-MD5 function names, non-MD5 " "function names will be dropped"); - assert((HasMD5 || HasNonMD5) && "Profile contains no name table"); + assert((ProfileUsesMD5 || HasNonMD5) && "Profile contains no name table"); } return sampleprof_error::success; @@ -1457,41 +1441,18 @@ } std::error_code SampleProfileReaderCompactBinary::readHeader() { - SampleProfileReaderBinary::readHeader(); - if (std::error_code EC = readFuncOffsetTable()) - return EC; - return sampleprof_error::success; -} + Data = reinterpret_cast(Buffer->getBufferStart()); + End = Data + Buffer->getBufferSize(); -std::error_code SampleProfileReaderCompactBinary::readFuncOffsetTable() { - auto TableOffset = readUnencodedNumber(); - if (std::error_code EC = TableOffset.getError()) + if (std::error_code EC = readMagicIdent()) return EC; - const uint8_t *SavedData = Data; - const uint8_t *TableStart = - reinterpret_cast(Buffer->getBufferStart()) + - *TableOffset; - Data = TableStart; - - auto Size = readNumber(); - if (std::error_code EC = Size.getError()) + if (std::error_code EC = readSummary()) return EC; - FuncOffsetTable.reserve(*Size); - for (uint64_t I = 0; I < *Size; ++I) { - auto FName(readStringFromTable()); - if (std::error_code EC = FName.getError()) - return EC; - - auto Offset = readNumber(); - if (std::error_code EC = Offset.getError()) - return EC; + if (std::error_code EC = readMD5NameTable()) + return EC; - FuncOffsetTable[*FName] = *Offset; - } - End = TableStart; - Data = SavedData; return sampleprof_error::success; } @@ -1710,7 +1671,7 @@ // body, there will be identical replicated profiles for the // original function. In this case, we simply not bother updating // the profile of the original function. - FProfile = &Profiles[Name]; + FProfile = &Profiles[MD5Hash(Name)]; FProfile->addHeadSamples(HeadCount); if (FProfile->getTotalSamples() > 0) Update = false; diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -364,7 +364,6 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection( const SampleProfileMap &ProfileMap) { for (const auto &I : ProfileMap) { - assert(I.first == I.second.getContext() && "Inconsistent profile map"); addContext(I.second.getContext()); addNames(I.second); } @@ -770,8 +769,7 @@ // Generate the name table for all the functions referenced in the profile. for (const auto &I : ProfileMap) { - assert(I.first == I.second.getContext() && "Inconsistent profile map"); - addContext(I.first); + addContext(I.second.getContext()); addNames(I.second); } diff --git a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp --- a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp +++ b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp @@ -201,7 +201,7 @@ : GUIDToFuncNameMap(GUIDToFuncNameMap) { for (auto &FuncSample : Profiles) { FunctionSamples *FSamples = &FuncSample.second; - SampleContext Context = FuncSample.first; + SampleContext Context = FuncSample.second.getContext(); LLVM_DEBUG(dbgs() << "Tracking Context for function: " << Context.toString() << "\n"); ContextTrieNode *NewNode = getOrCreateContextPath(Context, true); @@ -638,7 +638,7 @@ FunctionSamples *FProfile = Node->getFunctionSamples(); // Profile's context can be empty, use ContextNode's func name. if (FProfile) - ContextLessProfiles[Node->getFuncName()].merge(*FProfile); + ContextLessProfiles[SampleContext(Node->getFuncName()).getHashCode()].merge(*FProfile); } } } // namespace llvm diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -580,7 +580,7 @@ auto checkSampleProfileHasFUnique = [&Reader]() { for (const auto &PD : Reader->getProfiles()) { - auto &FContext = PD.first; + auto &FContext = PD.second.getContext(); if (FContext.toString().find(FunctionSamples::UniqSuffix) != std::string::npos) { return true; @@ -1017,7 +1017,8 @@ : FunctionSamples(); FunctionSamples &Samples = Remapper ? Remapped : I->second; SampleContext FContext = Samples.getContext(); - MergeResult(Result, ProfileMap[FContext].merge(Samples, Input.Weight)); + MergeResult(Result, ProfileMap[FContext.getHashCode()].merge( + Samples, Input.Weight)); if (Result != sampleprof_error::success) { std::error_code EC = make_error_code(Result); handleMergeWriterError(errorCodeToError(EC), Input.Filename, diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp --- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp +++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp @@ -454,7 +454,8 @@ bool ProfileGenerator::collectFunctionsFromLLVMProfile( std::unordered_set &ProfiledFunctions) { for (const auto &FS : ProfileMap) { - if (auto *Func = Binary->getBinaryFunction(FS.first.getName())) + if (auto *Func = + Binary->getBinaryFunction(FS.second.getContext().getName())) ProfiledFunctions.insert(Func); } return true; @@ -473,7 +474,8 @@ FunctionSamples & ProfileGenerator::getTopLevelFunctionProfile(StringRef FuncName) { SampleContext Context(FuncName); - auto Ret = ProfileMap.emplace(Context, FunctionSamples()); + auto Ret = ProfileMap.insert( + std::make_pair(Context.getHashCode(), FunctionSamples())); if (Ret.second) { FunctionSamples &FProfile = Ret.first->second; FProfile.setContext(Context); @@ -510,7 +512,7 @@ return; // Move cold profiles into a tmp container. - std::vector ColdProfiles; + std::vector ColdProfiles; for (const auto &I : ProfileMap) { if (I.second.getTotalSamples() < ColdCntThreshold) ColdProfiles.emplace_back(I.first); @@ -966,7 +968,8 @@ Context.emplace_back(Node.getFuncName(), LineLocation(0, 0)); // Save the new context for future references. SampleContextFrames NewContext = *Contexts.insert(Context).first; - auto Ret = ProfileMap.emplace(NewContext, std::move(*FProfile)); + auto Ret = ProfileMap.insert(std::make_pair( + SampleContext(NewContext).getHashCode(), std::move(*FProfile))); FunctionSamples &NewProfile = Ret.first->second; NewProfile.getContext().setContext(NewContext); Context.pop_back(); @@ -1021,7 +1024,9 @@ // Merge function samples of CS profile to calculate profile density. sampleprof::SampleProfileMap ContextLessProfiles; for (const auto &I : ProfileMap) { - ContextLessProfiles[I.second.getName()].merge(I.second); + ContextLessProfiles[SampleContext(I.second.getContext().getName()) + .getHashCode()] + .merge(I.second); } calculateAndShowDensity(ContextLessProfiles); diff --git a/llvm/unittests/ProfileData/SampleProfTest.cpp b/llvm/unittests/ProfileData/SampleProfTest.cpp --- a/llvm/unittests/ProfileData/SampleProfTest.cpp +++ b/llvm/unittests/ProfileData/SampleProfTest.cpp @@ -197,10 +197,10 @@ BooSamples.addBodySamples(1, 0, 1232); SampleProfileMap Profiles; - Profiles[FooName] = std::move(FooSamples); - Profiles[BarName] = std::move(BarSamples); - Profiles[BazName] = std::move(BazSamples); - Profiles[BooName] = std::move(BooSamples); + Profiles[MD5Hash(FooName)] = std::move(FooSamples); + Profiles[MD5Hash(BarName)] = std::move(BarSamples); + Profiles[MD5Hash(BazName)] = std::move(BazSamples); + Profiles[MD5Hash(BooName)] = std::move(BooSamples); Module M("my_module", Context); FunctionType *fn_type = @@ -338,7 +338,7 @@ FcnSamples.addTotalSamples(TotalSamples); FcnSamples.addHeadSamples(HeadSamples); FcnSamples.addBodySamples(1, 0, HeadSamples); - (*Smap)[Name] = FcnSamples; + (*Smap)[MD5Hash(Name)] = FcnSamples; } SampleProfileMap setupFcnSamplesForElisionTest(StringRef Policy) {