diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -650,7 +650,7 @@ /* Raw profile format version (start from 1). */ #define INSTR_PROF_RAW_VERSION 8 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 8 +#define INSTR_PROF_INDEX_VERSION 9 /* Coverage mapping format version (start from 0). */ #define INSTR_PROF_COVMAP_VERSION 5 diff --git a/compiler-rt/test/profile/Linux/binary-id.c b/compiler-rt/test/profile/Linux/binary-id.c --- a/compiler-rt/test/profile/Linux/binary-id.c +++ b/compiler-rt/test/profile/Linux/binary-id.c @@ -1,13 +1,13 @@ // REQUIRES: linux // RUN: %clang_profgen -Wl,--build-id=none -O2 -o %t %s // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t -// RUN: llvm-profdata show --binary-ids %t.profraw > %t.out +// RUN: llvm-profdata show --binary-ids %t.profraw > %t.out // RUN: FileCheck %s --check-prefix=NO-BINARY-ID < %t.out // RUN: llvm-profdata merge -o %t.profdata %t.profraw // RUN: %clang_profgen -Wl,--build-id -O2 -o %t %s // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t -// RUN: llvm-profdata show --binary-ids %t.profraw > %t.profraw.out +// RUN: llvm-profdata show --binary-ids %t.profraw > %t.profraw.out // RUN: FileCheck %s --check-prefix=BINARY-ID-RAW-PROF < %t.profraw.out // RUN: rm -rf %t.profdir @@ -17,6 +17,10 @@ // RUN: llvm-profdata show --binary-ids %t.profdir/default_*.profraw > %t.profraw.out // RUN: FileCheck %s --check-prefix=BINARY-ID-MERGE-PROF < %t.profraw.out +// RUN: llvm-profdata merge -o %t.profdata %t.profraw %t.profraw +// RUN: llvm-profdata show --binary-ids %t.profdata > %t.profdata.out +// RUN: FileCheck %s --check-prefix=BINARY-ID-INDEXED-PROF < %t.profraw.out + void foo() { } @@ -48,3 +52,10 @@ // BINARY-ID-MERGE-PROF-NEXT: Maximum internal block count: 0 // BINARY-ID-MERGE-PROF-NEXT: Binary IDs: // BINARY-ID-MERGE-PROF-NEXT: {{[0-9a-f]+}} + +// BINARY-ID-INDEXED-PROF: Instrumentation level: Front-end +// BINARY-ID-INDEXED-PROF-NEXT: Total functions: 3 +// BINARY-ID-INDEXED-PROF-NEXT: Maximum function count: 3 +// BINARY-ID-INDEXED-PROF-NEXT: Maximum internal block count: 0 +// BINARY-ID-INDEXED-PROF-NEXT: Binary IDs: +// BINARY-ID-INDEXED-PROF-NEXT: {{[0-9a-f]+}} diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -1050,7 +1050,9 @@ Version7 = 7, // An additional (optional) memory profile type is added. Version8 = 8, - // The current version is 8. + // Binary ids are added. + Version9 = 9, + // The current version is 9. CurrentVersion = INSTR_PROF_INDEX_VERSION }; const uint64_t Version = ProfVersion::CurrentVersion; @@ -1068,6 +1070,7 @@ uint64_t HashType; uint64_t HashOffset; uint64_t MemProfOffset; + uint64_t BinaryIdOffset; // New fields should only be added at the end to ensure that the size // computation is correct. The methods below need to be updated to ensure that // the new field is read correctly. diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -650,7 +650,7 @@ /* Raw profile format version (start from 1). */ #define INSTR_PROF_RAW_VERSION 8 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 8 +#define INSTR_PROF_INDEX_VERSION 9 /* Coverage mapping format version (start from 0). */ #define INSTR_PROF_COVMAP_VERSION 5 diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -96,6 +96,11 @@ /// Read a single record. virtual Error readNextRecord(NamedInstrProfRecord &Record) = 0; + /// Return a binary id size and data pair. + virtual std::pair getBinaryIds() { + return std::make_pair<>(0, nullptr); + }; + /// Print binary ids on stream OS. virtual Error printBinaryIds(raw_ostream &OS) { return success(); }; @@ -310,6 +315,7 @@ static bool hasFormat(const MemoryBuffer &DataBuffer); Error readHeader() override; Error readNextRecord(NamedInstrProfRecord &Record) override; + std::pair getBinaryIds() override; Error printBinaryIds(raw_ostream &OS) override; uint64_t getVersion() const override { return Version; } @@ -600,6 +606,9 @@ // Index to the current record in the record array. unsigned RecordIndex; + uint64_t BinaryIdsSize; + const uint8_t *BinaryIdsStart; + // Read the profile summary. Return a pointer pointing to one byte past the // end of the summary data if it exists or the input \c Cur. // \c UseCS indicates whether to use the context-sensitive profile summary. @@ -706,6 +715,9 @@ return *(Summary.get()); } } + + std::pair getBinaryIds() override; + Error printBinaryIds(raw_ostream &OS) override; }; } // end namespace llvm diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h --- a/llvm/include/llvm/ProfileData/InstrProfWriter.h +++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h @@ -54,6 +54,8 @@ InstrProfKind ProfileKind = InstrProfKind::Unknown; // Use raw pointer here for the incomplete type object. InstrProfRecordWriterTrait *InfoObj; + // List of binary id size and data pairs. + std::vector> BinaryIds; public: InstrProfWriter(bool Sparse = false); @@ -79,6 +81,9 @@ bool addMemProfFrame(const memprof::FrameId, const memprof::Frame &F, function_ref Warn); + // Add a binary id pair to the binary ids list. + void addBinaryIds(std::pair BinaryIds); + /// Merge existing function counts from the given writer. void mergeRecordsFromWriter(InstrProfWriter &&IPW, function_ref Warn); diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -1371,9 +1371,12 @@ // When a new field is added in the header add a case statement here to // populate it. static_assert( - IndexedInstrProf::ProfVersion::CurrentVersion == Version8, + IndexedInstrProf::ProfVersion::CurrentVersion == Version9, "Please update the reading code below if a new field has been added, " "if not add a case statement to fall through to the latest version."); + case 9ull: + H.BinaryIdOffset = read(Buffer, offsetOf(&Header::BinaryIdOffset)); + [[fallthrough]]; case 8ull: H.MemProfOffset = read(Buffer, offsetOf(&Header::MemProfOffset)); [[fallthrough]]; @@ -1390,10 +1393,12 @@ // When a new field is added to the header add a case statement here to // compute the size as offset of the new field + size of the new field. This // relies on the field being added to the end of the list. - static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version8, + static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version9, "Please update the size computation below if a new field has " "been added to the header, if not add a case statement to " "fall through to the latest version."); + case 9ull: + return offsetOf(&Header::BinaryIdOffset) + sizeof(Header::BinaryIdOffset); case 8ull: return offsetOf(&Header::MemProfOffset) + sizeof(Header::MemProfOffset); default: // Version7 (when the backwards compatible header was introduced). diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -75,6 +75,57 @@ return Reader.readHeader(); } +static size_t RoundUp(size_t size, size_t align) { + return (size + align - 1) & ~(align - 1); +} + +static Error printBinaryIdsInternal(raw_ostream &OS, + const MemoryBuffer &DataBuffer, + uint64_t BinaryIdsSize, + const uint8_t *BinaryIdsStart) { + using namespace support; + + if (BinaryIdsSize == 0) + return Error::success(); + + OS << "Binary IDs: \n"; + const uint8_t *BI = BinaryIdsStart; + const uint8_t *BIEnd = BinaryIdsStart + BinaryIdsSize; + while (BI < BIEnd) { + size_t Remaining = BIEnd - BI; + // There should be enough left to read the binary id length. + if (Remaining < sizeof(uint64_t)) + return make_error( + instrprof_error::malformed, + "not enough data to read binary id length"); + + // Read binary id length. + uint64_t BILen = endian::byte_swap( + *reinterpret_cast(BI)); + // There should be enough left to read the binary id length and its data. + if (Remaining < sizeof(BILen) + BILen) + return make_error( + instrprof_error::malformed, "not enough data to read binary id data"); + + // Increment by binary id length data type size. + BI += sizeof(BILen); + if (BI > (const uint8_t *)DataBuffer.getBufferEnd()) + return make_error( + instrprof_error::malformed, + "binary id that is read is bigger than buffer size"); + + for (uint64_t I = 0; I < BILen; I++) + OS << format("%02x", BI[I]); + OS << "\n"; + + // Increment by binary id data length, rounded to the next 8 bytes. + BI += RoundUp(BILen, sizeof(uint64_t)); + if (BI > (const uint8_t *)DataBuffer.getBufferEnd()) + return make_error(instrprof_error::malformed); + } + return Error::success(); +} + Expected> InstrProfReader::create(const Twine &Path, const InstrProfCorrelator *Correlator) { @@ -573,54 +624,17 @@ return success(); } -static size_t RoundUp(size_t size, size_t align) { - return (size + align - 1) & ~(align - 1); +template +std::pair +RawInstrProfReader::getBinaryIds() { + if (BinaryIdsSize == 0) + return std::make_pair<>(0, nullptr); + return std::make_pair<>(BinaryIdsSize, BinaryIdsStart); } template Error RawInstrProfReader::printBinaryIds(raw_ostream &OS) { - if (BinaryIdsSize == 0) - return success(); - - OS << "Binary IDs: \n"; - const uint8_t *BI = BinaryIdsStart; - const uint8_t *BIEnd = BinaryIdsStart + BinaryIdsSize; - while (BI < BIEnd) { - size_t Remaining = BIEnd - BI; - - // There should be enough left to read the binary ID size field. - if (Remaining < sizeof(uint64_t)) - return make_error( - instrprof_error::malformed, - "not enough data to read binary id length"); - - uint64_t BinaryIdLen = swap(*reinterpret_cast(BI)); - - // There should be enough left to read the binary ID size field, and the - // binary ID. - if (Remaining < sizeof(BinaryIdLen) + BinaryIdLen) - return make_error( - instrprof_error::malformed, "not enough data to read binary id data"); - - // Increment by binary id length data type size. - BI += sizeof(BinaryIdLen); - if (BI > (const uint8_t *)DataBuffer->getBufferEnd()) - return make_error( - instrprof_error::malformed, - "binary id that is read is bigger than buffer size"); - - for (uint64_t I = 0; I < BinaryIdLen; I++) - OS << format("%02x", BI[I]); - OS << "\n"; - - // Increment by binary id data length, rounded to the next 8 bytes. This - // accounts for the zero-padding after each build ID. - BI += RoundUp(BinaryIdLen, sizeof(uint64_t)); - if (BI > (const uint8_t *)DataBuffer->getBufferEnd()) - return make_error(instrprof_error::malformed); - } - - return success(); + return printBinaryIdsInternal(OS, *DataBuffer, BinaryIdsSize, BinaryIdsStart); } namespace llvm { @@ -948,9 +962,9 @@ Cur = readSummary((IndexedInstrProf::ProfVersion)Header->formatVersion(), Cur, /* UseCS */ false); if (Header->formatVersion() & VARIANT_MASK_CSIR_PROF) - Cur = readSummary((IndexedInstrProf::ProfVersion)Header->formatVersion(), Cur, - /* UseCS */ true); - + Cur = + readSummary((IndexedInstrProf::ProfVersion)Header->formatVersion(), Cur, + /* UseCS */ true); // Read the hash type and start offset. IndexedInstrProf::HashT HashType = static_cast( endian::byte_swap(Header->HashType)); @@ -963,8 +977,8 @@ auto IndexPtr = std::make_unique>( Start + HashOffset, Cur, Start, HashType, Header->formatVersion()); - // The MemProfOffset field in the header is only valid when the format version - // is higher than 8 (when it was introduced). + // The MemProfOffset field in the header is only valid when the format + // version is higher than 8 (when it was introduced). if (GET_VERSION(Header->formatVersion()) >= 8 && Header->formatVersion() & VARIANT_MASK_MEMPROF) { uint64_t MemProfOffset = @@ -974,7 +988,8 @@ // The value returned from RecordTableGenerator.Emit. const uint64_t RecordTableOffset = support::endian::readNext(Ptr); - // The offset in the stream right before invoking FrameTableGenerator.Emit. + // The offset in the stream right before invoking + // FrameTableGenerator.Emit. const uint64_t FramePayloadOffset = support::endian::readNext(Ptr); // The value returned from FrameTableGenerator.Emit. @@ -1000,11 +1015,28 @@ /*Base=*/Start, memprof::FrameLookupTrait())); } + // BinaryIdOffset field in the header is only valid when the format version + // is higher than 9 (when it was introduced). + if (GET_VERSION(Header->formatVersion()) >= 9) { + uint64_t BinaryIdOffset = + endian::byte_swap(Header->BinaryIdOffset); + const unsigned char *Ptr = Start + BinaryIdOffset; + // Read binary ids size. + BinaryIdsSize = support::endian::readNext(Ptr); + if (BinaryIdsSize % sizeof(uint64_t)) + return error(instrprof_error::bad_header); + // Set the binary ids start. + BinaryIdsStart = Ptr; + if (BinaryIdsStart > (const unsigned char *)DataBuffer->getBufferEnd()) + return make_error(instrprof_error::malformed, + "corrupted binary ids"); + } + // Load the remapping table now if requested. if (RemappingBuffer) { - Remapper = std::make_unique< - InstrProfReaderItaniumRemapper>( - std::move(RemappingBuffer), *IndexPtr); + Remapper = + std::make_unique>( + std::move(RemappingBuffer), *IndexPtr); if (Error E = Remapper->populateRemappings()) return E; } else { @@ -1136,6 +1168,17 @@ return success(); } +std::pair IndexedInstrProfReader::getBinaryIds() { + if (BinaryIdsSize == 0) + return std::make_pair<>(0, nullptr); + + return std::make_pair<>(BinaryIdsSize, BinaryIdsStart); +} + +Error IndexedInstrProfReader::printBinaryIds(raw_ostream &OS) { + return printBinaryIdsInternal(OS, *DataBuffer, BinaryIdsSize, BinaryIdsStart); +} + void InstrProfReader::accumulateCounts(CountSumOrPercent &Sum, bool IsCS) { uint64_t NumFuncs = 0; for (const auto &Func : *this) { diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -54,6 +54,7 @@ uint64_t tell() { return OS.tell(); } void write(uint64_t V) { LE.write(V); } + void writeByte(uint8_t V) { LE.write(V); } // \c patch can only be called when all data is written and flushed. // For raw_string_ostream, the patch is done on the target string @@ -280,6 +281,12 @@ return true; } +void InstrProfWriter::addBinaryIds(std::pair BI) { + if (BI.first == 0) + return; + BinaryIds.push_back(BI); +} + void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW, function_ref Warn) { for (auto &I : IPW.FunctionData) @@ -328,8 +335,13 @@ TheSummary->setEntry(I, Res[I]); } +static size_t RoundUp(size_t size, size_t align) { + return (size + align - 1) & ~(align - 1); +} + Error InstrProfWriter::writeImpl(ProfOStream &OS) { using namespace IndexedInstrProf; + using namespace support; OnDiskChainedHashTableGenerator Generator; @@ -365,11 +377,13 @@ Header.HashType = static_cast(IndexedInstrProf::HashType); Header.HashOffset = 0; Header.MemProfOffset = 0; + Header.BinaryIdOffset = 0; int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t); - // Only write out all the fields except 'HashOffset' and 'MemProfOffset'. We - // need to remember the offset of these fields to allow back patching later. - for (int I = 0; I < N - 2; I++) + // Only write out all the fields except 'HashOffset' and 'MemProfOffset' and + // 'BinaryIdOffset'. We need to remember the offset of these fields to allow + // back patching later. + for (int I = 0; I < N - 3; I++) OS.write(reinterpret_cast(&Header)[I]); // Save the location of Header.HashOffset field in \c OS. @@ -384,6 +398,13 @@ // profile contains memory profile information. OS.write(0); + // Save the location of binary ids, which is stored as uint64_t BinaryIdsSize + // and a list of uint64_t BinaryIdLength and uint8_t* BinaryIdData pairs. + uint64_t BinaryIdSectionOffset = OS.tell(); + // Reserve space for the BinaryIdOffset field to be patched later if this + // profile contains binary ids. + OS.write(0); + // Reserve space to write profile summary data. uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size(); uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries); @@ -460,6 +481,56 @@ OS.patch(PatchItems, 3); } + // Write binary ids section. + uint64_t BinaryIdSectionStart = 0; + BinaryIdSectionStart = OS.tell(); + // Calculate and write the size of binary section. + uint64_t TotalBinaryIdSizes = 0; + for (std::pair BI : BinaryIds) + TotalBinaryIdSizes += BI.first; + OS.write(TotalBinaryIdSizes); + + // Write binary id length and data. + for (std::pair BI : BinaryIds) { + uint64_t BISize = BI.first; + const uint8_t *BIData = BI.second; + const uint8_t *BIEnd = BIData + BISize; + + while (BIData < BIEnd) { + size_t Remaining = BIEnd - BIData; + // There should be enough left to read the binary id length. + if (Remaining < sizeof(uint64_t)) + return make_error( + instrprof_error::malformed, + "not enough data to read binary id length"); + // Write binary id length. + uint64_t BILen = endian::byte_swap( + *reinterpret_cast(BIData)); + OS.write(BILen); + + // There should be enough left to read the binary id length and its + if (Remaining < sizeof(BILen) + BILen) + return make_error( + instrprof_error::malformed, + "not enough data to read binary id data"); + + // Increment by binary id length data type size. + BIData += sizeof(BILen); + if (BIData > BIEnd) + return make_error( + instrprof_error::malformed, + "binary id that is read is bigger than buffer size"); + + for (size_t K = 0; K < RoundUp(BILen, sizeof(uint64_t)); K++) + OS.writeByte(BIData[K]); + + // Increment by binary id data length, rounded to the next 8 bytes. + BIData += RoundUp(BILen, sizeof(uint64_t)); + if (BIData > BIEnd) + return make_error(instrprof_error::malformed); + } + } + // Allocate space for data to be serialized out. std::unique_ptr TheSummary = IndexedInstrProf::allocSummary(SummarySize); @@ -482,8 +553,12 @@ PatchItem PatchItems[] = { // Patch the Header.HashOffset field. {HashTableStartFieldOffset, &HashTableStart, 1}, - // Patch the Header.MemProfOffset (=0 for profiles without MemProf data). + // Patch the Header.MemProfOffset (=0 for profiles without MemProf + // data). {MemProfSectionOffset, &MemProfSectionStart, 1}, + // Patch the Header.MemProfOffset (=0 for profiles without MemProf + // data). + {BinaryIdSectionOffset, &BinaryIdSectionStart, 1}, // Patch the summary data. {SummaryOffset, reinterpret_cast(TheSummary.get()), (int)(SummarySize / sizeof(uint64_t))}, diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -335,6 +335,8 @@ FuncName, firstTime); }); } + WC->Writer.addBinaryIds(Reader->getBinaryIds()); + if (Reader->hasError()) if (Error E = Reader->getError()) WC->Errors.emplace_back(std::move(E), Filename);