diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -128,9 +128,9 @@ INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) -INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) +INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) -INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -86,8 +86,8 @@ const __llvm_profile_data *__llvm_profile_end_data(void); const char *__llvm_profile_begin_names(void); const char *__llvm_profile_end_names(void); -uint64_t *__llvm_profile_begin_counters(void); -uint64_t *__llvm_profile_end_counters(void); +char *__llvm_profile_begin_counters(void); +char *__llvm_profile_end_counters(void); ValueProfNode *__llvm_profile_begin_vnodes(); ValueProfNode *__llvm_profile_end_vnodes(); uint32_t *__llvm_profile_begin_orderfile(); @@ -260,17 +260,26 @@ uint64_t __llvm_profile_get_version(void); /*! \brief Get the number of entries in the profile data section. */ +uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, + const __llvm_profile_data *End); + +/*! \brief Get the size of the profile data section in bytes. */ uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, const __llvm_profile_data *End); +/*! \brief Get the size in bytes of a single counter entry. */ +size_t __llvm_profile_counter_entry_size(void); + +/*! \brief Get the number of entries in the profile counters section. */ +uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End); + +/*! \brief Get the size of the profile counters section in bytes. */ +uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End); + /* ! \brief Given the sizes of the data and counter information, return the * number of padding bytes before and after the counters, and after the names, * in the raw profile. * - * Note: In this context, "size" means "number of entries", i.e. the first two - * arguments must be the result of __llvm_profile_get_data_size() and of - * (__llvm_profile_end_counters() - __llvm_profile_begin_counters()) resp. - * * Note: When mmap() mode is disabled, no padding bytes before/after counters * are needed. However, in mmap() mode, the counter section in the raw profile * must be page-aligned: this API computes the number of padding bytes diff --git a/compiler-rt/lib/profile/InstrProfiling.c b/compiler-rt/lib/profile/InstrProfiling.c --- a/compiler-rt/lib/profile/InstrProfiling.c +++ b/compiler-rt/lib/profile/InstrProfiling.c @@ -42,10 +42,10 @@ } COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) { - uint64_t *I = __llvm_profile_begin_counters(); - uint64_t *E = __llvm_profile_end_counters(); + char *I = __llvm_profile_begin_counters(); + char *E = __llvm_profile_end_counters(); - memset(I, 0, sizeof(uint64_t) * (E - I)); + memset(I, 0, E - I); const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -41,8 +41,8 @@ uint64_t __llvm_profile_get_size_for_buffer(void) { const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); - const uint64_t *CountersBegin = __llvm_profile_begin_counters(); - const uint64_t *CountersEnd = __llvm_profile_end_counters(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); @@ -51,13 +51,36 @@ } COMPILER_RT_VISIBILITY -uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, - const __llvm_profile_data *End) { +uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, + const __llvm_profile_data *End) { intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End; return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) / sizeof(__llvm_profile_data); } +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, + const __llvm_profile_data *End) { + return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data); +} + +COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) { + return sizeof(uint64_t); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End) { + intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End; + return ((EndI + __llvm_profile_counter_entry_size() - 1) - BeginI) / + __llvm_profile_counter_entry_size(); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End) { + return __llvm_profile_get_num_counters(Begin, End) * + __llvm_profile_counter_entry_size(); +} + /// Calculate the number of padding bytes needed to add to \p Offset in order /// for (\p Offset + Padding) to be page-aligned. static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) { @@ -89,24 +112,22 @@ // In continuous mode, the file offsets for headers and for the start of // counter sections need to be page-aligned. - uint64_t DataSizeInBytes = DataSize * sizeof(__llvm_profile_data); - uint64_t CountersSizeInBytes = CountersSize * sizeof(uint64_t); - *PaddingBytesBeforeCounters = calculateBytesNeededToPageAlign( - sizeof(__llvm_profile_header) + DataSizeInBytes); - *PaddingBytesAfterCounters = - calculateBytesNeededToPageAlign(CountersSizeInBytes); + *PaddingBytesBeforeCounters = + calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize); + *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize); *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize); } COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const uint64_t *CountersBegin, const uint64_t *CountersEnd, - const char *NamesBegin, const char *NamesEnd) { + const char *CountersBegin, const char *CountersEnd, const char *NamesBegin, + const char *NamesEnd) { /* Match logic in __llvm_profile_write_buffer(). */ const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); - uint64_t CountersSize = CountersEnd - CountersBegin; + uint64_t CountersSize = + __llvm_profile_get_counters_size(CountersBegin, CountersEnd); /* Determine how much padding is needed before/after the counters and after * the names. */ @@ -117,9 +138,8 @@ &PaddingBytesAfterCounters, &PaddingBytesAfterNames); return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + - (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters + - (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters + - NamesSize + PaddingBytesAfterNames; + DataSize + PaddingBytesBeforeCounters + CountersSize + + PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames; } COMPILER_RT_VISIBILITY @@ -136,8 +156,8 @@ COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal( char *Buffer, const __llvm_profile_data *DataBegin, - const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin, - const uint64_t *CountersEnd, const char *NamesBegin, const char *NamesEnd) { + const __llvm_profile_data *DataEnd, const char *CountersBegin, + const char *CountersEnd, const char *NamesBegin, const char *NamesEnd) { ProfDataWriter BufferWriter; initBufferWriter(&BufferWriter, Buffer); return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin, diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c --- a/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/compiler-rt/lib/profile/InstrProfilingFile.c @@ -106,13 +106,14 @@ * __llvm_profile_get_size_for_buffer(). */ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); - const uint64_t *CountersBegin = __llvm_profile_begin_counters(); - const uint64_t *CountersEnd = __llvm_profile_end_counters(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); - uint64_t CountersSize = CountersEnd - CountersBegin; + uint64_t CountersSize = + __llvm_profile_get_counters_size(CountersBegin, CountersEnd); /* Check that the counter and data sections in this image are * page-aligned. */ @@ -136,11 +137,10 @@ DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, &PaddingBytesAfterNames); - uint64_t PageAlignedCountersLength = - (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters; - uint64_t FileOffsetToCounters = - CurrentFileOffset + sizeof(__llvm_profile_header) + - (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters; + uint64_t PageAlignedCountersLength = CountersSize + PaddingBytesAfterCounters; + uint64_t FileOffsetToCounters = CurrentFileOffset + + sizeof(__llvm_profile_header) + DataSize + + PaddingBytesBeforeCounters; uint64_t *CounterMmap = (uint64_t *)mmap( (void *)CountersBegin, PageAlignedCountersLength, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToCounters); @@ -195,8 +195,8 @@ * __llvm_profile_get_size_for_buffer(). */ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); - const uint64_t *CountersBegin = __llvm_profile_begin_counters(); - const uint64_t *CountersEnd = __llvm_profile_end_counters(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); /* Get the file size. */ uint64_t FileSize = 0; @@ -211,8 +211,7 @@ return 1; } const uint64_t CountersOffsetInBiasMode = - sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + - (DataSize * sizeof(__llvm_profile_data)); + sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize; /* Update the profile fields based on the current mapping. */ INSTR_PROF_PROFILE_COUNTER_BIAS_VAR = (intptr_t)Profile - (uintptr_t)CountersBegin + CountersOffsetInBiasMode; @@ -578,9 +577,8 @@ } /* Get the sizes of counter section. */ - const uint64_t *CountersBegin = __llvm_profile_begin_counters(); - const uint64_t *CountersEnd = __llvm_profile_end_counters(); - uint64_t CountersSize = CountersEnd - CountersBegin; + uint64_t CountersSize = __llvm_profile_get_counters_size( + __llvm_profile_begin_counters(), __llvm_profile_end_counters()); int Length = getCurFilenameLength(); char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h --- a/compiler-rt/lib/profile/InstrProfilingInternal.h +++ b/compiler-rt/lib/profile/InstrProfilingInternal.h @@ -21,8 +21,8 @@ */ uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const uint64_t *CountersBegin, const uint64_t *CountersEnd, - const char *NamesBegin, const char *NamesEnd); + const char *CountersBegin, const char *CountersEnd, const char *NamesBegin, + const char *NamesEnd); /*! * \brief Write instrumentation data to the given buffer, given explicit @@ -35,8 +35,8 @@ */ int __llvm_profile_write_buffer_internal( char *Buffer, const __llvm_profile_data *DataBegin, - const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin, - const uint64_t *CountersEnd, const char *NamesBegin, const char *NamesEnd); + const __llvm_profile_data *DataEnd, const char *CountersBegin, + const char *CountersEnd, const char *NamesBegin, const char *NamesEnd); /*! * The data structure describing the data to be written by the @@ -152,8 +152,7 @@ int lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const uint64_t *CountersBegin, - const uint64_t *CountersEnd, + const char *CountersBegin, const char *CountersEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite); diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -23,18 +23,18 @@ uint64_t lprofGetLoadModuleSignature() { /* A very fast way to compute a module signature. */ uint64_t Version = __llvm_profile_get_version(); - uint64_t CounterSize = (uint64_t)(__llvm_profile_end_counters() - - __llvm_profile_begin_counters()); - uint64_t DataSize = __llvm_profile_get_data_size(__llvm_profile_begin_data(), - __llvm_profile_end_data()); + uint64_t NumCounters = __llvm_profile_get_num_counters( + __llvm_profile_begin_counters(), __llvm_profile_end_counters()); + uint64_t NumData = __llvm_profile_get_num_data(__llvm_profile_begin_data(), + __llvm_profile_end_data()); uint64_t NamesSize = (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names()); uint64_t NumVnodes = (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes()); const __llvm_profile_data *FirstD = __llvm_profile_begin_data(); - return (NamesSize << 40) + (CounterSize << 30) + (DataSize << 20) + - (NumVnodes << 10) + (DataSize > 0 ? FirstD->NameRef : 0) + Version + + return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) + + (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version + __llvm_profile_get_magic(); } @@ -48,7 +48,7 @@ SrcDataStart = (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + Header->BinaryIdsSize); - SrcDataEnd = SrcDataStart + Header->DataSize; + SrcDataEnd = SrcDataStart + Header->NumData; if (ProfileSize < sizeof(__llvm_profile_header)) return 1; @@ -56,19 +56,21 @@ /* Check the header first. */ if (Header->Magic != __llvm_profile_get_magic() || Header->Version != __llvm_profile_get_version() || - Header->DataSize != - __llvm_profile_get_data_size(__llvm_profile_begin_data(), - __llvm_profile_end_data()) || - Header->CountersSize != (uint64_t)(__llvm_profile_end_counters() - - __llvm_profile_begin_counters()) || + Header->NumData != + __llvm_profile_get_num_data(__llvm_profile_begin_data(), + __llvm_profile_end_data()) || + Header->NumCounters != + __llvm_profile_get_num_counters(__llvm_profile_begin_counters(), + __llvm_profile_end_counters()) || Header->NamesSize != (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names()) || Header->ValueKindLast != IPVK_Last) return 1; - if (ProfileSize < sizeof(__llvm_profile_header) + Header->BinaryIdsSize + - Header->DataSize * sizeof(__llvm_profile_data) + - Header->NamesSize + Header->CountersSize) + if (ProfileSize < + sizeof(__llvm_profile_header) + Header->BinaryIdsSize + + Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize + + Header->NumCounters * __llvm_profile_counter_entry_size()) return 1; for (SrcData = SrcDataStart, @@ -105,7 +107,7 @@ __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData; __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData; - uint64_t *SrcCountersStart; + char *SrcCountersStart; const char *SrcNameStart; const char *SrcValueProfDataStart, *SrcValueProfData; uintptr_t CountersDelta = Header->CountersDelta; @@ -113,13 +115,14 @@ SrcDataStart = (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + Header->BinaryIdsSize); - SrcDataEnd = SrcDataStart + Header->DataSize; - SrcCountersStart = (uint64_t *)SrcDataEnd; - SrcNameStart = (const char *)(SrcCountersStart + Header->CountersSize); + SrcDataEnd = SrcDataStart + Header->NumData; + SrcCountersStart = (char *)SrcDataEnd; + SrcNameStart = SrcCountersStart + + Header->NumCounters * __llvm_profile_counter_entry_size(); SrcValueProfDataStart = SrcNameStart + Header->NamesSize + __llvm_profile_get_num_padding_bytes(Header->NamesSize); - if (SrcNameStart < (const char *)SrcCountersStart) + if (SrcNameStart < SrcCountersStart) return 1; for (SrcData = SrcDataStart, @@ -130,8 +133,8 @@ // address of the data to the start address of the counter. On WIN64, // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign // extend CounterPtr to get the original value. - uint64_t *DstCounters = - (uint64_t *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr)); + char *DstCounters = + (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr)); unsigned NVK = 0; // SrcData is a serialized representation of the memory image. We need to @@ -141,21 +144,19 @@ // CountersDelta computes the offset into the in-buffer counter section. // // On WIN64, CountersDelta is truncated as well, so no need for signext. - uint64_t *SrcCounters = - SrcCountersStart + - ((uintptr_t)SrcData->CounterPtr - CountersDelta) / sizeof(uint64_t); + char *SrcCounters = + SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta); // CountersDelta needs to be decreased as we advance to the next data // record. CountersDelta -= sizeof(*SrcData); unsigned NC = SrcData->NumCounters; if (NC == 0) return 1; - if (SrcCounters < SrcCountersStart || - (const char *)SrcCounters >= SrcNameStart || - (const char *)(SrcCounters + NC) > SrcNameStart) + if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart || + (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart) return 1; for (unsigned I = 0; I < NC; I++) - DstCounters[I] += SrcCounters[I]; + ((uint64_t *)DstCounters)[I] += ((uint64_t *)SrcCounters)[I]; /* Now merge value profile data. */ if (!VPMergeHook) diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c b/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c --- a/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c @@ -26,11 +26,10 @@ COMPILER_RT_VISIBILITY extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME); COMPILER_RT_VISIBILITY -extern uint64_t +extern char CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME); COMPILER_RT_VISIBILITY -extern uint64_t - CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME); +extern char CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME); COMPILER_RT_VISIBILITY extern uint32_t OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME); @@ -53,9 +52,9 @@ COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) { return &NamesEnd; } COMPILER_RT_VISIBILITY -uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart; } +char *__llvm_profile_begin_counters(void) { return &CountersStart; } COMPILER_RT_VISIBILITY -uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; } +char *__llvm_profile_end_counters(void) { return &CountersEnd; } COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c b/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c --- a/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c @@ -116,13 +116,13 @@ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); - const uint64_t *CountersBegin = __llvm_profile_begin_counters(); - const uint64_t *CountersEnd = __llvm_profile_end_counters(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); const uint64_t CountersOffset = - sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + - (DataSize * sizeof(__llvm_profile_data)); - uint64_t CountersSize = CountersEnd - CountersBegin; + sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize; + uint64_t CountersSize = + __llvm_profile_get_counters_size(CountersBegin, CountersEnd); /* Don't publish a VMO if there are no counters. */ if (!CountersSize) diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c --- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -43,8 +43,8 @@ COMPILER_RT_WEAK; extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; -extern uint64_t PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; -extern uint64_t PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_NAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; @@ -65,10 +65,10 @@ COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) { return &PROF_NAME_STOP; } -COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { +COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) { return &PROF_CNTS_START; } -COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { +COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return &PROF_CNTS_STOP; } COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/compiler-rt/lib/profile/InstrProfilingPlatformOther.c --- a/compiler-rt/lib/profile/InstrProfilingPlatformOther.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -20,8 +20,8 @@ static const __llvm_profile_data *DataLast = NULL; static const char *NamesFirst = NULL; static const char *NamesLast = NULL; -static uint64_t *CountersFirst = NULL; -static uint64_t *CountersLast = NULL; +static char *CountersFirst = NULL; +static char *CountersLast = NULL; static uint32_t *OrderFileFirst = NULL; static const void *getMinAddr(const void *A1, const void *A2) { @@ -46,19 +46,21 @@ if (!DataFirst) { DataFirst = Data; DataLast = Data + 1; - CountersFirst = (uint64_t *)((uintptr_t)Data_ + Data->CounterPtr); - CountersLast = CountersFirst + Data->NumCounters; + CountersFirst = (char *)((uintptr_t)Data_ + Data->CounterPtr); + CountersLast = + CountersFirst + Data->NumCounters * __llvm_profile_counter_entry_size(); return; } DataFirst = (const __llvm_profile_data *)getMinAddr(DataFirst, Data); - CountersFirst = (uint64_t *)getMinAddr( - CountersFirst, (uint64_t *)((uintptr_t)Data_ + Data->CounterPtr)); + CountersFirst = (char *)getMinAddr( + CountersFirst, (char *)((uintptr_t)Data_ + Data->CounterPtr)); DataLast = (const __llvm_profile_data *)getMaxAddr(DataLast, Data + 1); - CountersLast = (uint64_t *)getMaxAddr( + CountersLast = (char *)getMaxAddr( CountersLast, - (uint64_t *)((uintptr_t)Data_ + Data->CounterPtr) + Data->NumCounters); + (char *)((uintptr_t)Data_ + Data->CounterPtr) + + Data->NumCounters * __llvm_profile_counter_entry_size()); } COMPILER_RT_VISIBILITY @@ -83,9 +85,9 @@ COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) { return NamesLast; } COMPILER_RT_VISIBILITY -uint64_t *__llvm_profile_begin_counters(void) { return CountersFirst; } +char *__llvm_profile_begin_counters(void) { return CountersFirst; } COMPILER_RT_VISIBILITY -uint64_t *__llvm_profile_end_counters(void) { return CountersLast; } +char *__llvm_profile_end_counters(void) { return CountersLast; } /* TODO: correctly set up OrderFileFirst. */ COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return OrderFileFirst; } diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c --- a/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -41,8 +41,8 @@ const char COMPILER_RT_SECTION(".lprfn$A") NamesStart = '\0'; const char COMPILER_RT_SECTION(".lprfn$Z") NamesEnd = '\0'; -uint64_t COMPILER_RT_SECTION(".lprfc$A") CountersStart; -uint64_t COMPILER_RT_SECTION(".lprfc$Z") CountersEnd; +char COMPILER_RT_SECTION(".lprfc$A") CountersStart; +char COMPILER_RT_SECTION(".lprfc$Z") CountersEnd; uint32_t COMPILER_RT_SECTION(".lorderfile$A") OrderFileStart; ValueProfNode COMPILER_RT_SECTION(".lprfnd$A") VNodesStart; @@ -56,8 +56,8 @@ const char *__llvm_profile_begin_names(void) { return &NamesStart + 1; } const char *__llvm_profile_end_names(void) { return &NamesEnd; } -uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart + 1; } -uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; } +char *__llvm_profile_begin_counters(void) { return &CountersStart + 1; } +char *__llvm_profile_end_counters(void) { return &CountersEnd; } uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } ValueProfNode *__llvm_profile_begin_vnodes(void) { return &VNodesStart + 1; } diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c --- a/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -244,8 +244,8 @@ /* Match logic in __llvm_profile_write_buffer(). */ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); - const uint64_t *CountersBegin = __llvm_profile_begin_counters(); - const uint64_t *CountersEnd = __llvm_profile_end_counters(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, @@ -256,7 +256,7 @@ COMPILER_RT_VISIBILITY int lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const uint64_t *CountersBegin, const uint64_t *CountersEnd, + const char *CountersBegin, const char *CountersEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite) { int DebugInfoCorrelate = @@ -265,13 +265,18 @@ /* Calculate size of sections. */ const uint64_t DataSize = DebugInfoCorrelate ? 0 : __llvm_profile_get_data_size(DataBegin, DataEnd); - const uint64_t CountersSize = CountersEnd - CountersBegin; + const uint64_t NumData = + DebugInfoCorrelate ? 0 : __llvm_profile_get_num_data(DataBegin, DataEnd); + const uint64_t CountersSize = + __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + const uint64_t NumCounters = + __llvm_profile_get_num_counters(CountersBegin, CountersEnd); const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin; /* Create the header. */ __llvm_profile_header Header; - if (!DataSize && (!DebugInfoCorrelate || !CountersSize)) + if (!NumData && (!DebugInfoCorrelate || !NumCounters)) return 0; /* Determine how much padding is needed before/after the counters and after @@ -309,10 +314,9 @@ /* Write the profile data. */ ProfDataIOVec IOVecData[] = { - {DebugInfoCorrelate ? NULL : DataBegin, sizeof(__llvm_profile_data), - DataSize, 0}, + {DebugInfoCorrelate ? NULL : DataBegin, sizeof(uint8_t), DataSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, - {CountersBegin, sizeof(uint64_t), CountersSize, 0}, + {CountersBegin, sizeof(uint8_t), CountersSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, {(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, diff --git a/compiler-rt/test/profile/instrprof-write-buffer-internal.c b/compiler-rt/test/profile/instrprof-write-buffer-internal.c --- a/compiler-rt/test/profile/instrprof-write-buffer-internal.c +++ b/compiler-rt/test/profile/instrprof-write-buffer-internal.c @@ -23,19 +23,19 @@ const void *__llvm_profile_end_data(void); const char *__llvm_profile_begin_names(void); const char *__llvm_profile_end_names(void); -uint64_t *__llvm_profile_begin_counters(void); -uint64_t *__llvm_profile_end_counters(void); +char *__llvm_profile_begin_counters(void); +char *__llvm_profile_end_counters(void); uint64_t __llvm_profile_get_size_for_buffer_internal( - const void *DataBegin, const void *DataEnd, - const uint64_t *CountersBegin, const uint64_t *CountersEnd, - const char *NamesBegin, const char *NamesEnd); + const void *DataBegin, const void *DataEnd, const char *CountersBegin, + const char *CountersEnd, const char *NamesBegin, const char *NamesEnd); -int __llvm_profile_write_buffer_internal( - char *Buffer, const void *DataBegin, - const void *DataEnd, const uint64_t *CountersBegin, - const uint64_t *CountersEnd, const char *NamesBegin, - const char *NamesEnd); +int __llvm_profile_write_buffer_internal(char *Buffer, const void *DataBegin, + const void *DataEnd, + const char *CountersBegin, + const char *CountersEnd, + const char *NamesBegin, + const char *NamesEnd); void __llvm_profile_set_dumped(void); diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -128,9 +128,9 @@ INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) -INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) +INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) -INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -233,7 +233,8 @@ uint64_t NamesDelta; const RawInstrProf::ProfileData *Data; const RawInstrProf::ProfileData *DataEnd; - const uint64_t *CountersStart; + const char *CountersStart; + const char *CountersEnd; const char *NamesStart; const char *NamesEnd; // After value profile is all read, this pointer points to @@ -310,6 +311,15 @@ bool atEnd() const { return Data == DataEnd; } void advanceData() { + // `CountersDelta` is a constant zero when using debug info correlation. + if (!Correlator) { + // The initial CountersDelta is the in-memory address difference between + // the data and counts sections: + // start(__llvm_prf_cnts) - start(__llvm_prf_data) + // As we advance to the next record, we maintain the correct CountersDelta + // with respect to the next record. + CountersDelta -= sizeof(*Data); + } Data++; ValueDataStart += CurValueDataSize; } @@ -319,20 +329,11 @@ return (const char *)ValueDataStart; } - /// Get the offset of \p CounterPtr from the start of the counters section of - /// the profile. The offset has units of "number of counters", i.e. increasing - /// the offset by 1 corresponds to an increase in the *byte offset* by 8. - ptrdiff_t getCounterOffset(IntPtrT CounterPtr) const { - return (swap(CounterPtr) - CountersDelta) / sizeof(uint64_t); - } - - const uint64_t *getCounter(ptrdiff_t Offset) const { - return CountersStart + Offset; - } - StringRef getName(uint64_t NameRef) const { return Symtab->getFuncName(swap(NameRef)); } + + int getCounterTypeSize() const { return sizeof(uint64_t); } }; using RawInstrProfReader32 = RawInstrProfReader; diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -383,22 +383,21 @@ CountersDelta = swap(Header.CountersDelta); NamesDelta = swap(Header.NamesDelta); - auto DataSize = swap(Header.DataSize); + auto NumData = swap(Header.NumData); auto PaddingBytesBeforeCounters = swap(Header.PaddingBytesBeforeCounters); - auto CountersSize = swap(Header.CountersSize); + auto CountersSize = swap(Header.NumCounters) * getCounterTypeSize(); auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters); auto NamesSize = swap(Header.NamesSize); ValueKindLast = swap(Header.ValueKindLast); - auto DataSizeInBytes = DataSize * sizeof(RawInstrProf::ProfileData); + auto DataSize = NumData * sizeof(RawInstrProf::ProfileData); auto PaddingSize = getNumPaddingBytes(NamesSize); // Profile data starts after profile header and binary ids if exist. ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdsSize; - ptrdiff_t CountersOffset = - DataOffset + DataSizeInBytes + PaddingBytesBeforeCounters; - ptrdiff_t NamesOffset = CountersOffset + (sizeof(uint64_t) * CountersSize) + - PaddingBytesAfterCounters; + ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters; + ptrdiff_t NamesOffset = + CountersOffset + CountersSize + PaddingBytesAfterCounters; ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize; auto *Start = reinterpret_cast(&Header); @@ -417,7 +416,7 @@ } else { Data = reinterpret_cast *>( Start + DataOffset); - DataEnd = Data + DataSize; + DataEnd = Data + NumData; NamesStart = Start + NamesOffset; NamesEnd = NamesStart + NamesSize; } @@ -425,7 +424,8 @@ // Binary ids start just after the header. BinaryIdsStart = reinterpret_cast(&Header) + sizeof(RawInstrProf::Header); - CountersStart = reinterpret_cast(Start + CountersOffset); + CountersStart = Start + CountersOffset; + CountersEnd = CountersStart + CountersSize; ValueDataStart = reinterpret_cast(Start + ValueDataOffset); const uint8_t *BufferEnd = (const uint8_t *)DataBuffer->getBufferEnd(); @@ -459,58 +459,36 @@ if (NumCounters == 0) return error(instrprof_error::malformed, "number of counters is zero"); - ArrayRef RawCounts; - if (Correlator) { - uint64_t CounterOffset = swap(Data->CounterPtr) / sizeof(uint64_t); - RawCounts = - makeArrayRef(CountersStart + CounterOffset, NumCounters); - } else { - IntPtrT CounterPtr = Data->CounterPtr; - ptrdiff_t CounterOffset = getCounterOffset(CounterPtr); - if (CounterOffset < 0) - return error( - instrprof_error::malformed, - ("counter offset " + Twine(CounterOffset) + " is negative").str()); - - // Check bounds. Note that the counter pointer embedded in the data record - // may itself be corrupt. - auto *NamesStartAsCounter = reinterpret_cast(NamesStart); - ptrdiff_t MaxNumCounters = NamesStartAsCounter - CountersStart; - if (MaxNumCounters < 0 || NumCounters > (uint32_t)MaxNumCounters) - return error(instrprof_error::malformed, - "counter pointer is out of bounds"); - // We need to compute the in-buffer counter offset from the in-memory - // address distance. The initial CountersDelta is the in-memory address - // difference start(__llvm_prf_cnts)-start(__llvm_prf_data), so - // SrcData->CounterPtr - CountersDelta computes the offset into the - // in-buffer counter section. - if (CounterOffset > MaxNumCounters) - return error(instrprof_error::malformed, - ("counter offset " + Twine(CounterOffset) + - " is greater than the maximum number of counters " + - Twine((uint32_t)MaxNumCounters)) - .str()); - - if (((uint32_t)CounterOffset + NumCounters) > (uint32_t)MaxNumCounters) - return error(instrprof_error::malformed, - ("number of counters " + - Twine(((uint32_t)CounterOffset + NumCounters)) + - " is greater than the maximum number of counters " + - Twine((uint32_t)MaxNumCounters)) - .str()); - // CountersDelta decreases as we advance to the next data record. - CountersDelta -= sizeof(*Data); - - RawCounts = makeArrayRef(getCounter(CounterOffset), NumCounters); - } + ptrdiff_t CounterBaseOffset = swap(Data->CounterPtr) - CountersDelta; + if (CounterBaseOffset < 0) + return error( + instrprof_error::malformed, + ("counter offset " + Twine(CounterBaseOffset) + " is negative").str()); - if (ShouldSwapBytes) { - Record.Counts.clear(); - Record.Counts.reserve(RawCounts.size()); - for (uint64_t Count : RawCounts) - Record.Counts.push_back(swap(Count)); - } else - Record.Counts = RawCounts; + if (CounterBaseOffset >= CountersEnd - CountersStart) + return error(instrprof_error::malformed, + ("counter offset " + Twine(CounterBaseOffset) + + " is greater than the maximum counter offset " + + Twine(CountersEnd - CountersStart - 1)) + .str()); + + uint64_t MaxNumCounters = + (CountersEnd - (CountersStart + CounterBaseOffset)) / + getCounterTypeSize(); + if (NumCounters > MaxNumCounters) + return error(instrprof_error::malformed, + ("number of counters " + Twine(NumCounters) + + " is greater than the maximum number of counters " + + Twine(MaxNumCounters)) + .str()); + + Record.Counts.clear(); + Record.Counts.reserve(NumCounters); + for (uint32_t I = 0; I < NumCounters; I++) { + const auto *CounterValue = reinterpret_cast( + CountersStart + CounterBaseOffset + I * getCounterTypeSize()); + Record.Counts.push_back(swap(*CounterValue)); + } return success(); } diff --git a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test --- a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test +++ b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test @@ -53,5 +53,5 @@ RUN: printf '\3\0bar\0\0\0' >> %t.profraw RUN: not llvm-profdata merge -o /dev/null %t.profraw 2>&1 | FileCheck %s -CHECK: warning: {{.+}}: malformed instrumentation profile data: number of counters 3 is greater than the maximum number of counters 2 +CHECK: warning: {{.+}}: malformed instrumentation profile data: number of counters 2 is greater than the maximum number of counters 0 CHECK: error: no profile can be merged