diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -493,7 +493,7 @@ virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) { return false; }; /// Return whether names in the profile are all MD5 numbers. - bool useMD5() { return ProfileIsMD5; } + bool useMD5() const { return ProfileIsMD5; } /// Force the profile to use MD5 in Sample contexts, even if function names /// are present. @@ -668,6 +668,37 @@ /// otherwise same as readStringFromTable. ErrorOr readSampleContextFromTable(); + /// Determine which container readFuncOffsetTable() populates, the list + /// OrderedFuncOffsets or the map FuncOffsetTable, using this order: + /// (a) If profile is CS, the function offset table is expected to consist of + /// sequences of contexts in pre-order layout + /// (e.g. [A, A:1 @ B, A:1 @ B:2.3 @ C] [D, D:1 @ E]), so that when a match in + /// the module is found, all prefixes of the matched function can be loaded, + /// so the list container is needed. + /// (b) If the profile is MD5, use the map container to lookup functions in + /// the module. A remapper has no use on MD5 names. + /// (c) If a remapper is present, each function name in the profile needs to + /// be checked if its remapped name matches a function in the module, so use + /// the list container since each entry is accessed. + /// (d) For other cases, use the map container to lookup functions in the + /// module. + /// TODO: For (b) and (d), if the cardinality of the function offset table is + /// much smaller than the number of functions in the module, using the list + /// container should be faster, but we need to figure out the constant factor + /// to determine a cutoff. + bool useOrderedFuncOffsets() { + return ProfileIsCS || (!useMD5() && Remapper); + } + + /// Read the whole function offset table from the profile file, and store it + /// to either OrderedFuncOffsets (list) or FuncOffsetTable (map). Clear the + /// previous function offset table. + std::error_code readFuncOffsetTable(); + + /// Read function profiles. If a module is present and function offset table + /// is available, only read profiles of functions present in the module. + std::error_code readFuncProfiles(); + /// Points to the current location in the buffer. const uint8_t *Data = nullptr; @@ -693,6 +724,16 @@ /// for SampleContextFrames. std::vector CSNameTable; + /// The table mapping from function context to the offset of its + /// FunctionSample towards file/section start. + DenseMap FuncOffsetTable; + + /// Function offset mapping ordered by contexts. + std::vector> OrderedFuncOffsets; + + /// The set containing the functions to use when compiling a module. + DenseSet FuncsToUse; + private: std::error_code readSummaryEntry(std::vector &Entries); virtual std::error_code verifySPMagic(uint64_t Magic) = 0; @@ -747,8 +788,6 @@ std::error_code readFuncMetadata(bool ProfileHasAttribute); std::error_code readFuncMetadata(bool ProfileHasAttribute, FunctionSamples *FProfile); - std::error_code readFuncOffsetTable(); - std::error_code readFuncProfiles(); std::error_code readNameTableSec(bool IsMD5, bool FixedLengthMD5); std::error_code readCSNameTableSec(); std::error_code readProfileSymbolList(); @@ -762,17 +801,6 @@ std::unique_ptr ProfSymList; - /// The table mapping from function context to the offset of its - /// FunctionSample towards file start. - DenseMap FuncOffsetTable; - - /// Function offset mapping ordered by contexts. - std::unique_ptr>> - OrderedFuncOffsets; - - /// The set containing the functions to use when compiling a module. - DenseSet FuncsToUse; - /// If SkipFlatProf is true, skip the sections with /// SecFlagFlat flag. bool SkipFlatProf = false; @@ -824,14 +852,8 @@ class SampleProfileReaderCompactBinary : public SampleProfileReaderBinary { private: - /// The table mapping from function name to the offset of its FunctionSample - /// towards file start. - DenseMap FuncOffsetTable; - /// The set containing the functions to use when compiling a module. - DenseSet FuncsToUse; std::error_code verifySPMagic(uint64_t Magic) override; std::error_code readHeader() override; - std::error_code readFuncOffsetTable(); public: SampleProfileReaderCompactBinary(std::unique_ptr B, diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -626,6 +626,7 @@ std::error_code SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) { + assert(Start < End && "out of LBRProfile section"); Data = Start; auto NumHeadSamples = readNumber(); if (std::error_code EC = NumHeadSamples.getError()) @@ -722,7 +723,12 @@ return EC; break; case SecFuncOffsetTable: - FuncOffsetsOrdered = hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered); + // Only read function offset table if module is present, otherwise all + // profiles must be loaded anyways. + if (!M) { + Data = End; + break; + } if (std::error_code EC = readFuncOffsetTable()) return EC; break; @@ -757,26 +763,25 @@ return true; } -std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() { - // If there are more than one FuncOffsetTable, the profile read associated - // with previous FuncOffsetTable has to be done before next FuncOffsetTable +std::error_code SampleProfileReaderBinary::readFuncOffsetTable() { + // If there are more than one function offset table, the profile associated + // with the previous table has to be done reading before next one // is read. FuncOffsetTable.clear(); + OrderedFuncOffsets.clear(); auto Size = readNumber(); if (std::error_code EC = Size.getError()) return EC; - FuncOffsetTable.reserve(*Size); - - if (FuncOffsetsOrdered) { - OrderedFuncOffsets = - std::make_unique>>(); - OrderedFuncOffsets->reserve(*Size); - } + bool UseOrderedFuncOffsets = useOrderedFuncOffsets(); + if (UseOrderedFuncOffsets) + OrderedFuncOffsets.reserve(*Size); + else + FuncOffsetTable.reserve(*Size); for (uint64_t I = 0; I < *Size; ++I) { - auto FContext(readSampleContextFromTable()); + auto FContext = readSampleContextFromTable(); if (std::error_code EC = FContext.getError()) return EC; @@ -784,15 +789,16 @@ if (std::error_code EC = Offset.getError()) return EC; - FuncOffsetTable[*FContext] = *Offset; - if (FuncOffsetsOrdered) - OrderedFuncOffsets->emplace_back(*FContext, *Offset); + if (UseOrderedFuncOffsets) + OrderedFuncOffsets.emplace_back(*FContext, *Offset); + else + FuncOffsetTable[*FContext] = *Offset; } return sampleprof_error::success; } -std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() { +std::error_code SampleProfileReaderBinary::readFuncProfiles() { // Collect functions used by current module if the Reader has been // given a module. // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName @@ -801,7 +807,8 @@ // NameTable section is read. bool LoadFuncsToBeUsed = collectFuncsFromModule(); - // When LoadFuncsToBeUsed is false, load all the function profiles. + // When LoadFuncsToBeUsed is false, we are using llvm tools, need to read all + // profiles. const uint8_t *Start = Data; if (!LoadFuncsToBeUsed) { while (Data < End) { @@ -831,10 +838,8 @@ // as if they were walked in preorder of a context trie. While // traversing the trie, a link to the highest common ancestor node is // kept so that all of its decendants will be loaded. - assert(OrderedFuncOffsets.get() && - "func offset table should always be sorted in CS profile"); const SampleContext *CommonContext = nullptr; - for (const auto &NameOffset : *OrderedFuncOffsets) { + for (const auto &NameOffset : OrderedFuncOffsets) { const auto &FContext = NameOffset.first; auto FName = FContext.getName(); // For function in the current module, keep its farthest ancestor @@ -852,35 +857,38 @@ // Load profile for the current context which originated from // the common ancestor. const uint8_t *FuncProfileAddr = Start + NameOffset.second; - assert(FuncProfileAddr < End && "out of LBRProfile section"); if (std::error_code EC = readFuncProfile(FuncProfileAddr)) return EC; } } + } else if (useMD5()) { + for (auto Name : FuncsToUse) { + auto GUID = std::to_string(MD5Hash(Name)); + auto iter = FuncOffsetTable.find(StringRef(GUID)); + if (iter == FuncOffsetTable.end()) + continue; + const uint8_t *FuncProfileAddr = Start + iter->second; + if (std::error_code EC = readFuncProfile(FuncProfileAddr)) + return EC; + } + } else if (Remapper) { + for (auto NameOffset : OrderedFuncOffsets) { + SampleContext FContext(NameOffset.first); + auto FuncName = FContext.getName(); + if (!FuncsToUse.count(FuncName) && !Remapper->exist(FuncName)) + continue; + const uint8_t *FuncProfileAddr = Start + NameOffset.second; + if (std::error_code EC = readFuncProfile(FuncProfileAddr)) + return EC; + } } else { - if (useMD5()) { - for (auto Name : FuncsToUse) { - auto GUID = std::to_string(MD5Hash(Name)); - auto iter = FuncOffsetTable.find(StringRef(GUID)); - if (iter == FuncOffsetTable.end()) - continue; - const uint8_t *FuncProfileAddr = Start + iter->second; - assert(FuncProfileAddr < End && "out of LBRProfile section"); - if (std::error_code EC = readFuncProfile(FuncProfileAddr)) - return EC; - } - } else { - for (auto NameOffset : FuncOffsetTable) { - SampleContext FContext(NameOffset.first); - auto FuncName = FContext.getName(); - if (!FuncsToUse.count(FuncName) && - (!Remapper || !Remapper->exist(FuncName))) - continue; - const uint8_t *FuncProfileAddr = Start + NameOffset.second; - assert(FuncProfileAddr < End && "out of LBRProfile section"); - if (std::error_code EC = readFuncProfile(FuncProfileAddr)) - return EC; - } + for (auto Name : FuncsToUse) { + auto iter = FuncOffsetTable.find(Name); + if (iter == FuncOffsetTable.end()) + continue; + const uint8_t *FuncProfileAddr = Start + iter->second; + if (std::error_code EC = readFuncProfile(FuncProfileAddr)) + return EC; } } Data = End; @@ -977,37 +985,28 @@ } std::error_code SampleProfileReaderCompactBinary::readImpl() { - // Collect functions used by current module if the Reader has been - // given a module. - bool LoadFuncsToBeUsed = collectFuncsFromModule(); + auto TableOffset = readUnencodedNumber(); + if (std::error_code EC = TableOffset.getError()) + return EC; + + const uint8_t *TableStart = + reinterpret_cast(Buffer->getBufferStart()) + + *TableOffset; + + // Only read function offset table if a module is present, otherwise all + // profiles must be loaded anyways. In this case offset is from the beginning + // of the file. + if (M) { + Data = TableStart; + if (std::error_code EC = readFuncOffsetTable()) + return EC; + Data = reinterpret_cast(Buffer->getBufferStart()); + } + End = TableStart; + ProfileIsFS = ProfileIsFSDisciminator; FunctionSamples::ProfileIsFS = ProfileIsFS; - std::vector OffsetsToUse; - if (!LoadFuncsToBeUsed) { - // load all the function profiles. - for (auto FuncEntry : FuncOffsetTable) { - OffsetsToUse.push_back(FuncEntry.second); - } - } else { - // load function profiles on demand. - for (auto Name : FuncsToUse) { - auto GUID = std::to_string(MD5Hash(Name)); - auto iter = FuncOffsetTable.find(StringRef(GUID)); - if (iter == FuncOffsetTable.end()) - continue; - OffsetsToUse.push_back(iter->second); - } - } - - for (auto Offset : OffsetsToUse) { - const uint8_t *SavedData = Data; - if (std::error_code EC = readFuncProfile( - reinterpret_cast(Buffer->getBufferStart()) + - Offset)) - return EC; - Data = SavedData; - } - return sampleprof_error::success; + return readFuncProfiles(); } std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) { @@ -1431,40 +1430,6 @@ if (std::error_code EC = readMD5NameTable()) return EC; - if (std::error_code EC = readFuncOffsetTable()) - return EC; - return sampleprof_error::success; -} - -std::error_code SampleProfileReaderCompactBinary::readFuncOffsetTable() { - auto TableOffset = readUnencodedNumber(); - if (std::error_code EC = TableOffset.getError()) - return EC; - - const uint8_t *SavedData = Data; - const uint8_t *TableStart = - reinterpret_cast(Buffer->getBufferStart()) + - *TableOffset; - Data = TableStart; - - auto Size = readNumber(); - if (std::error_code EC = Size.getError()) - return EC; - - FuncOffsetTable.reserve(*Size); - for (uint64_t I = 0; I < *Size; ++I) { - auto FName(readStringFromTable()); - if (std::error_code EC = FName.getError()) - return EC; - - auto Offset = readNumber(); - if (std::error_code EC = Offset.getError()) - return EC; - - FuncOffsetTable[*FName] = *Offset; - } - End = TableStart; - Data = SavedData; return sampleprof_error::success; }