diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -634,15 +634,6 @@ /// \returns the read value. ErrorOr readString(); - /// Read the string index and check whether it overflows the table. - template inline ErrorOr readStringIndex(T &Table); - - /// Read the next function profile instance. - std::error_code readFuncProfile(const uint8_t *Start); - - /// Read the contents of the given profile instance. - std::error_code readProfile(FunctionSamples &FProfile); - /// Read the contents of Magic number and Version number. std::error_code readMagicIdent(); @@ -655,6 +646,9 @@ /// Read the whole name table with ULEB128 encoded MD5 values. std::error_code readMD5NameTable(); + /// Read the string index and check whether it overflows the table. + template inline ErrorOr readStringIndex(T &Table); + /// Read a string indirectly via the name table. ErrorOr readStringFromTable(); @@ -692,6 +686,12 @@ /// previous function offset table. std::error_code readFuncOffsetTable(); + /// Read the next function profile instance. + std::error_code readFuncProfile(const uint8_t *Start); + + /// Read the contents of the given profile instance. + std::error_code readProfile(FunctionSamples &FProfile); + /// Read function profiles. If a module is present and function offset table /// is available, only read profiles of functions present in the module. std::error_code readFuncProfiles(); diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -465,6 +465,27 @@ return result; } +std::error_code SampleProfileReaderBinary::readHeader() { + Data = reinterpret_cast(Buffer->getBufferStart()); + End = Data + Buffer->getBufferSize(); + + if (std::error_code EC = readMagicIdent()) + return EC; + + if (std::error_code EC = readSummary()) + return EC; + + if (std::error_code EC = readNameTable()) + return EC; + return sampleprof_error::success; +} + +std::error_code SampleProfileReaderBinary::readImpl() { + ProfileIsFS = ProfileIsFSDisciminator; + FunctionSamples::ProfileIsFS = ProfileIsFS; + return readFuncProfiles(); +} + template ErrorOr SampleProfileReaderBinary::readNumber() { unsigned NumBytesRead = 0; std::error_code EC; @@ -514,6 +535,126 @@ return Val; } +std::error_code SampleProfileReaderBinary::readMagicIdent() { + // Read and check the magic identifier. + auto Magic = readNumber(); + if (std::error_code EC = Magic.getError()) + return EC; + else if (std::error_code EC = verifySPMagic(*Magic)) + return EC; + + // Read the version number. + auto Version = readNumber(); + if (std::error_code EC = Version.getError()) + return EC; + else if (*Version != SPVersion()) + return sampleprof_error::unsupported_version; + + return sampleprof_error::success; +} + +std::error_code SampleProfileReaderBinary::readSummaryEntry( + std::vector &Entries) { + auto Cutoff = readNumber(); + if (std::error_code EC = Cutoff.getError()) + return EC; + + auto MinBlockCount = readNumber(); + if (std::error_code EC = MinBlockCount.getError()) + return EC; + + auto NumBlocks = readNumber(); + if (std::error_code EC = NumBlocks.getError()) + return EC; + + Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks); + return sampleprof_error::success; +} + +std::error_code SampleProfileReaderBinary::readSummary() { + auto TotalCount = readNumber(); + if (std::error_code EC = TotalCount.getError()) + return EC; + + auto MaxBlockCount = readNumber(); + if (std::error_code EC = MaxBlockCount.getError()) + return EC; + + auto MaxFunctionCount = readNumber(); + if (std::error_code EC = MaxFunctionCount.getError()) + return EC; + + auto NumBlocks = readNumber(); + if (std::error_code EC = NumBlocks.getError()) + return EC; + + auto NumFunctions = readNumber(); + if (std::error_code EC = NumFunctions.getError()) + return EC; + + auto NumSummaryEntries = readNumber(); + if (std::error_code EC = NumSummaryEntries.getError()) + return EC; + + std::vector Entries; + for (unsigned i = 0; i < *NumSummaryEntries; i++) { + std::error_code EC = readSummaryEntry(Entries); + if (EC != sampleprof_error::success) + return EC; + } + Summary = std::make_unique( + ProfileSummary::PSK_Sample, Entries, *TotalCount, *MaxBlockCount, 0, + *MaxFunctionCount, *NumBlocks, *NumFunctions); + + return sampleprof_error::success; +} + +std::error_code SampleProfileReaderBinary::readNameTable() { + auto Size = readNumber(); + if (std::error_code EC = Size.getError()) + return EC; + + // Normally if useMD5 is true, the name table should have MD5 values, not + // strings, however in the case that ExtBinary profile has multiple name + // tables mixing string and MD5, all of them have to be normalized to use MD5, + // because optimization passes can only handle either type. + bool UseMD5 = useMD5(); + if (UseMD5) + MD5StringBuf.reserve(MD5StringBuf.size() + *Size); + + NameTable.clear(); + NameTable.reserve(*Size); + for (size_t I = 0; I < *Size; ++I) { + auto Name(readString()); + if (std::error_code EC = Name.getError()) + return EC; + if (UseMD5) { + uint64_t MD5 = MD5Hash(*Name); + NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(MD5))); + } else + NameTable.push_back(*Name); + } + + return sampleprof_error::success; +} + +std::error_code SampleProfileReaderBinary::readMD5NameTable() { + auto Size = readNumber(); + if (std::error_code EC = Size.getError()) + return EC; + + MD5StringBuf.reserve(MD5StringBuf.size() + *Size); + NameTable.clear(); + NameTable.reserve(*Size); + for (size_t I = 0; I < *Size; ++I) { + auto FID = readNumber(); + if (std::error_code EC = FID.getError()) + return EC; + NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(*FID))); + } + return sampleprof_error::success; +} + template inline ErrorOr SampleProfileReaderBinary::readStringIndex(T &Table) { std::error_code EC; @@ -543,6 +684,64 @@ return SR; } +ErrorOr SampleProfileReaderBinary::readContextFromTable() { + auto ContextIdx = readNumber(); + if (std::error_code EC = ContextIdx.getError()) + return EC; + if (*ContextIdx >= CSNameTable.size()) + return sampleprof_error::truncated_name_table; + return CSNameTable[*ContextIdx]; +} + +ErrorOr SampleProfileReaderBinary::readSampleContextFromTable() { + if (ProfileIsCS) { + auto FContext(readContextFromTable()); + if (std::error_code EC = FContext.getError()) + return EC; + return SampleContext(*FContext); + } else { + auto FName(readStringFromTable()); + if (std::error_code EC = FName.getError()) + return EC; + return SampleContext(*FName); + } +} + +std::error_code SampleProfileReaderBinary::readFuncOffsetTable() { + // If there are more than one function offset table, the profile associated + // with the previous table has to be done reading before next one + // is read. + FuncOffsetTable.clear(); + OrderedFuncOffsets.clear(); + + auto Size = readNumber(); + if (std::error_code EC = Size.getError()) + return EC; + + bool UseOrderedFuncOffsets = useOrderedFuncOffsets(); + if (UseOrderedFuncOffsets) + OrderedFuncOffsets.reserve(*Size); + else + FuncOffsetTable.reserve(*Size); + + for (uint64_t I = 0; I < *Size; ++I) { + auto FContext = readSampleContextFromTable(); + if (std::error_code EC = FContext.getError()) + return EC; + + auto Offset = readNumber(); + if (std::error_code EC = Offset.getError()) + return EC; + + if (UseOrderedFuncOffsets) + OrderedFuncOffsets.emplace_back(*FContext, *Offset); + else + FuncOffsetTable[*FContext] = *Offset; + } + + return sampleprof_error::success; +} + std::error_code SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { auto NumSamples = readNumber(); @@ -651,154 +850,6 @@ return sampleprof_error::success; } -std::error_code SampleProfileReaderBinary::readImpl() { - ProfileIsFS = ProfileIsFSDisciminator; - FunctionSamples::ProfileIsFS = ProfileIsFS; - return readFuncProfiles(); -} - -ErrorOr SampleProfileReaderBinary::readContextFromTable() { - auto ContextIdx = readNumber(); - if (std::error_code EC = ContextIdx.getError()) - return EC; - if (*ContextIdx >= CSNameTable.size()) - return sampleprof_error::truncated_name_table; - return CSNameTable[*ContextIdx]; -} - -ErrorOr SampleProfileReaderBinary::readSampleContextFromTable() { - if (ProfileIsCS) { - auto FContext(readContextFromTable()); - if (std::error_code EC = FContext.getError()) - return EC; - return SampleContext(*FContext); - } else { - auto FName(readStringFromTable()); - if (std::error_code EC = FName.getError()) - return EC; - return SampleContext(*FName); - } -} - -std::error_code SampleProfileReaderExtBinaryBase::readOneSection( - const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) { - Data = Start; - End = Start + Size; - switch (Entry.Type) { - case SecProfSummary: - if (std::error_code EC = readSummary()) - return EC; - if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial)) - Summary->setPartialProfile(true); - if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFullContext)) - FunctionSamples::ProfileIsCS = ProfileIsCS = true; - if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagIsPreInlined)) - FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true; - if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator)) - FunctionSamples::ProfileIsFS = ProfileIsFS = true; - break; - case SecNameTable: { - /// Use fixed length MD5 instead of ULEB128 encoding so NameTable doesn't - /// need to be read in up front and can be directly accessed using index. - bool FixedLengthMD5 = - hasSecFlag(Entry, SecNameTableFlags::SecFlagFixedLengthMD5); - bool UseMD5 = - FixedLengthMD5 || hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name); - FunctionSamples::HasUniqSuffix = - hasSecFlag(Entry, SecNameTableFlags::SecFlagUniqSuffix); - if (std::error_code EC = readNameTableSec(UseMD5, FixedLengthMD5)) - return EC; - break; - } - case SecCSNameTable: { - if (std::error_code EC = readCSNameTableSec()) - return EC; - break; - } - case SecLBRProfile: - if (std::error_code EC = readFuncProfiles()) - return EC; - break; - case SecFuncOffsetTable: - // Only read function offset table if module is present, otherwise all - // profiles must be loaded anyways. - if (!M) { - Data = End; - } else { - bool FuncOffsetsOrdered = - hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered); - assert((!ProfileIsCS || FuncOffsetsOrdered) && - "func offset table should always be sorted in CS profile"); - if (std::error_code EC = readFuncOffsetTable()) - return EC; - } - break; - case SecFuncMetadata: { - ProfileIsProbeBased = - hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased); - FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased; - bool HasAttribute = - hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagHasAttribute); - if (std::error_code EC = readFuncMetadata(HasAttribute)) - return EC; - break; - } - case SecProfileSymbolList: - if (std::error_code EC = readProfileSymbolList()) - return EC; - break; - default: - if (std::error_code EC = readCustomSection(Entry)) - return EC; - break; - } - return sampleprof_error::success; -} - -bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() { - if (!M) - return false; - FuncsToUse.clear(); - for (auto &F : *M) - FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F)); - return true; -} - -std::error_code SampleProfileReaderBinary::readFuncOffsetTable() { - // If there are more than one function offset table, the profile associated - // with the previous table has to be done reading before next one - // is read. - FuncOffsetTable.clear(); - OrderedFuncOffsets.clear(); - - auto Size = readNumber(); - if (std::error_code EC = Size.getError()) - return EC; - - bool UseOrderedFuncOffsets = useOrderedFuncOffsets(); - if (UseOrderedFuncOffsets) - OrderedFuncOffsets.reserve(*Size); - else - FuncOffsetTable.reserve(*Size); - - for (uint64_t I = 0; I < *Size; ++I) { - auto FContext = readSampleContextFromTable(); - if (std::error_code EC = FContext.getError()) - return EC; - - auto Offset = readNumber(); - if (std::error_code EC = Offset.getError()) - return EC; - - if (UseOrderedFuncOffsets) - OrderedFuncOffsets.emplace_back(*FContext, *Offset); - else - FuncOffsetTable[*FContext] = *Offset; - } - - return sampleprof_error::success; -} - std::error_code SampleProfileReaderBinary::readFuncProfiles() { // Collect functions used by current module if the Reader has been // given a module. @@ -985,94 +1036,6 @@ return sampleprof_error::success; } -std::error_code SampleProfileReaderCompactBinary::readImpl() { - auto TableOffset = readUnencodedNumber(); - if (std::error_code EC = TableOffset.getError()) - return EC; - - const uint8_t *TableStart = - reinterpret_cast(Buffer->getBufferStart()) + - *TableOffset; - - // Only read function offset table if a module is present, otherwise all - // profiles must be loaded anyways. In this case offset is from the beginning - // of the file. - if (M) { - Data = TableStart; - if (std::error_code EC = readFuncOffsetTable()) - return EC; - Data = reinterpret_cast(Buffer->getBufferStart()); - } - End = TableStart; - - return SampleProfileReaderBinary::readImpl(); -} - -std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) { - if (Magic == SPMagic()) - return sampleprof_error::success; - return sampleprof_error::bad_magic; -} - -std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) { - if (Magic == SPMagic(SPF_Ext_Binary)) - return sampleprof_error::success; - return sampleprof_error::bad_magic; -} - -std::error_code -SampleProfileReaderCompactBinary::verifySPMagic(uint64_t Magic) { - if (Magic == SPMagic(SPF_Compact_Binary)) - return sampleprof_error::success; - return sampleprof_error::bad_magic; -} - -std::error_code SampleProfileReaderBinary::readNameTable() { - auto Size = readNumber(); - if (std::error_code EC = Size.getError()) - return EC; - - // Normally if useMD5 is true, the name table should have MD5 values, not - // strings, however in the case that ExtBinary profile has multiple name - // tables mixing string and MD5, all of them have to be normalized to use MD5, - // because optimization passes can only handle either type. - bool UseMD5 = useMD5(); - if (UseMD5) - MD5StringBuf.reserve(MD5StringBuf.size() + *Size); - - NameTable.clear(); - NameTable.reserve(*Size); - for (size_t I = 0; I < *Size; ++I) { - auto Name(readString()); - if (std::error_code EC = Name.getError()) - return EC; - if (UseMD5) { - uint64_t MD5 = MD5Hash(*Name); - NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(MD5))); - } else - NameTable.push_back(*Name); - } - - return sampleprof_error::success; -} - -std::error_code SampleProfileReaderBinary::readMD5NameTable() { - auto Size = readNumber(); - if (std::error_code EC = Size.getError()) - return EC; - - MD5StringBuf.reserve(MD5StringBuf.size() + *Size); - NameTable.clear(); - NameTable.reserve(*Size); - for (size_t I = 0; I < *Size; ++I) { - auto FID = readNumber(); - if (std::error_code EC = FID.getError()) - return EC; - NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(*FID))); - } - return sampleprof_error::success; -} - std::error_code SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5, bool FixedLengthMD5) { @@ -1144,7 +1107,6 @@ } std::error_code - SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute, FunctionSamples *FProfile) { if (Data < End) { @@ -1314,6 +1276,90 @@ return FileSize; } +std::error_code SampleProfileReaderExtBinaryBase::readOneSection( + const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) { + Data = Start; + End = Start + Size; + switch (Entry.Type) { + case SecProfSummary: + if (std::error_code EC = readSummary()) + return EC; + if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial)) + Summary->setPartialProfile(true); + if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFullContext)) + FunctionSamples::ProfileIsCS = ProfileIsCS = true; + if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagIsPreInlined)) + FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true; + if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator)) + FunctionSamples::ProfileIsFS = ProfileIsFS = true; + break; + case SecNameTable: { + /// Use fixed length MD5 instead of ULEB128 encoding so NameTable doesn't + /// need to be read in up front and can be directly accessed using index. + bool FixedLengthMD5 = + hasSecFlag(Entry, SecNameTableFlags::SecFlagFixedLengthMD5); + bool UseMD5 = + FixedLengthMD5 || hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name); + FunctionSamples::HasUniqSuffix = + hasSecFlag(Entry, SecNameTableFlags::SecFlagUniqSuffix); + if (std::error_code EC = readNameTableSec(UseMD5, FixedLengthMD5)) + return EC; + break; + } + case SecCSNameTable: { + if (std::error_code EC = readCSNameTableSec()) + return EC; + break; + } + case SecLBRProfile: + if (std::error_code EC = readFuncProfiles()) + return EC; + break; + case SecFuncOffsetTable: + // Only read function offset table if module is present, otherwise all + // profiles must be loaded anyways. + if (!M) { + Data = End; + } else { + bool FuncOffsetsOrdered = + hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered); + assert((!ProfileIsCS || FuncOffsetsOrdered) && + "func offset table should always be sorted in CS profile"); + if (std::error_code EC = readFuncOffsetTable()) + return EC; + } + break; + case SecFuncMetadata: { + ProfileIsProbeBased = + hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased); + FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased; + bool HasAttribute = + hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagHasAttribute); + if (std::error_code EC = readFuncMetadata(HasAttribute)) + return EC; + break; + } + case SecProfileSymbolList: + if (std::error_code EC = readProfileSymbolList()) + return EC; + break; + default: + if (std::error_code EC = readCustomSection(Entry)) + return EC; + break; + } + return sampleprof_error::success; +} + +bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() { + if (!M) + return false; + FuncsToUse.clear(); + for (auto &F : *M) + FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F)); + return true; +} + static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) { std::string Flags; if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress)) @@ -1383,39 +1429,6 @@ return true; } -std::error_code SampleProfileReaderBinary::readMagicIdent() { - // Read and check the magic identifier. - auto Magic = readNumber(); - if (std::error_code EC = Magic.getError()) - return EC; - else if (std::error_code EC = verifySPMagic(*Magic)) - return EC; - - // Read the version number. - auto Version = readNumber(); - if (std::error_code EC = Version.getError()) - return EC; - else if (*Version != SPVersion()) - return sampleprof_error::unsupported_version; - - return sampleprof_error::success; -} - -std::error_code SampleProfileReaderBinary::readHeader() { - Data = reinterpret_cast(Buffer->getBufferStart()); - End = Data + Buffer->getBufferSize(); - - if (std::error_code EC = readMagicIdent()) - return EC; - - if (std::error_code EC = readSummary()) - return EC; - - if (std::error_code EC = readNameTable()) - return EC; - return sampleprof_error::success; -} - std::error_code SampleProfileReaderCompactBinary::readHeader() { Data = reinterpret_cast(Buffer->getBufferStart()); End = Data + Buffer->getBufferSize(); @@ -1432,6 +1445,29 @@ return sampleprof_error::success; } +std::error_code SampleProfileReaderCompactBinary::readImpl() { + auto TableOffset = readUnencodedNumber(); + if (std::error_code EC = TableOffset.getError()) + return EC; + + const uint8_t *TableStart = + reinterpret_cast(Buffer->getBufferStart()) + + *TableOffset; + + // Only read function offset table if a module is present, otherwise all + // profiles must be loaded anyways. In this case offset is from the beginning + // of the file. + if (M) { + Data = TableStart; + if (std::error_code EC = readFuncOffsetTable()) + return EC; + Data = reinterpret_cast(Buffer->getBufferStart()); + } + End = TableStart; + + return SampleProfileReaderBinary::readImpl(); +} + bool SampleProfileReaderCompactBinary::collectFuncsFromModule() { if (!M) return false; @@ -1441,60 +1477,23 @@ return true; } -std::error_code SampleProfileReaderBinary::readSummaryEntry( - std::vector &Entries) { - auto Cutoff = readNumber(); - if (std::error_code EC = Cutoff.getError()) - return EC; - - auto MinBlockCount = readNumber(); - if (std::error_code EC = MinBlockCount.getError()) - return EC; - - auto NumBlocks = readNumber(); - if (std::error_code EC = NumBlocks.getError()) - return EC; - - Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks); - return sampleprof_error::success; +std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) { + if (Magic == SPMagic()) + return sampleprof_error::success; + return sampleprof_error::bad_magic; } -std::error_code SampleProfileReaderBinary::readSummary() { - auto TotalCount = readNumber(); - if (std::error_code EC = TotalCount.getError()) - return EC; +std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) { + if (Magic == SPMagic(SPF_Ext_Binary)) + return sampleprof_error::success; + return sampleprof_error::bad_magic; +} - auto MaxBlockCount = readNumber(); - if (std::error_code EC = MaxBlockCount.getError()) - return EC; - - auto MaxFunctionCount = readNumber(); - if (std::error_code EC = MaxFunctionCount.getError()) - return EC; - - auto NumBlocks = readNumber(); - if (std::error_code EC = NumBlocks.getError()) - return EC; - - auto NumFunctions = readNumber(); - if (std::error_code EC = NumFunctions.getError()) - return EC; - - auto NumSummaryEntries = readNumber(); - if (std::error_code EC = NumSummaryEntries.getError()) - return EC; - - std::vector Entries; - for (unsigned i = 0; i < *NumSummaryEntries; i++) { - std::error_code EC = readSummaryEntry(Entries); - if (EC != sampleprof_error::success) - return EC; - } - Summary = std::make_unique( - ProfileSummary::PSK_Sample, Entries, *TotalCount, *MaxBlockCount, 0, - *MaxFunctionCount, *NumBlocks, *NumFunctions); - - return sampleprof_error::success; +std::error_code +SampleProfileReaderCompactBinary::verifySPMagic(uint64_t Magic) { + if (Magic == SPMagic(SPF_Compact_Binary)) + return sampleprof_error::success; + return sampleprof_error::bad_magic; } bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) {