Index: compiler-rt/include/profile/InstrProfData.inc =================================================================== --- compiler-rt/include/profile/InstrProfData.inc +++ compiler-rt/include/profile/InstrProfData.inc @@ -76,6 +76,7 @@ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ Inc->getHash()->getZExtValue())) INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) +INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr) /* This is used to map function pointers for the indirect call targets to * function name hashes during the conversion from raw to merged profile * data. @@ -87,7 +88,9 @@ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ - ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) + ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \ +INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes)) #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ @@ -134,9 +137,13 @@ /* FIXME: A more accurate name is NumCounters */ INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) +INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes) +INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin - (uintptr_t)DataBegin) +INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, + (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER @@ -269,6 +276,9 @@ INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ INSTR_PROF_CNTS_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ + INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \ + INSTR_PROF_BITS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ INSTR_PROF_NAME_COFF, "__DATA,") @@ -648,11 +658,11 @@ /* FIXME: Please remedy the fixme in the header before bumping the version. */ /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 8 +#define INSTR_PROF_RAW_VERSION 9 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 10 +#define INSTR_PROF_INDEX_VERSION 11 /* Coverage mapping format version (start from 0). */ -#define INSTR_PROF_COVMAP_VERSION 5 +#define INSTR_PROF_COVMAP_VERSION 6 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the * version for other variants of profile. We set the lowest bit of the upper 8 @@ -689,6 +699,7 @@ #define INSTR_PROF_DATA_COMMON __llvm_prf_data #define INSTR_PROF_NAME_COMMON __llvm_prf_names #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts +#define INSTR_PROF_BITS_COMMON __llvm_prf_bits #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap @@ -700,6 +711,7 @@ #define INSTR_PROF_DATA_COFF ".lprfd$M" #define INSTR_PROF_NAME_COFF ".lprfn$M" #define INSTR_PROF_CNTS_COFF ".lprfc$M" +#define INSTR_PROF_BITS_COFF ".lprfb$M" #define INSTR_PROF_VALS_COFF ".lprfv$M" #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" @@ -711,6 +723,7 @@ #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF +#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ @@ -725,6 +738,7 @@ #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) +#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ Index: compiler-rt/lib/profile/InstrProfiling.h =================================================================== --- compiler-rt/lib/profile/InstrProfiling.h +++ compiler-rt/lib/profile/InstrProfiling.h @@ -88,6 +88,8 @@ const char *__llvm_profile_end_names(void); char *__llvm_profile_begin_counters(void); char *__llvm_profile_end_counters(void); +char *__llvm_profile_begin_bitmap(void); +char *__llvm_profile_end_bitmap(void); ValueProfNode *__llvm_profile_begin_vnodes(); ValueProfNode *__llvm_profile_end_vnodes(); uint32_t *__llvm_profile_begin_orderfile(); @@ -101,11 +103,11 @@ /*! * \brief Merge profile data from buffer. * - * Read profile data form buffer \p Profile and merge with in-process profile - * counters. The client is expected to have checked or already knows the profile - * data in the buffer matches the in-process counter structure before calling - * it. Returns 0 (success) if the profile data is valid. Upon reading - * invalid/corrupted profile data, returns 1 (failure). + * Read profile data from buffer \p Profile and merge with in-process profile + * counters and bitmaps. The client is expected to have checked or already + * know the profile data in the buffer matches the in-process counter + * structure before calling it. Returns 0 (success) if the profile data is + * valid. Upon reading invalid/corrupted profile data, returns 1 (failure). */ int __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size); @@ -113,8 +115,8 @@ * * Returns 0 (success) if the profile data in buffer \p Profile with size * \p Size was generated by the same binary and therefore matches - * structurally the in-process counters. If the profile data in buffer is - * not compatible, the interface returns 1 (failure). + * structurally the in-process counters and bitmaps. If the profile data in + * buffer is not compatible, the interface returns 1 (failure). */ int __llvm_profile_check_compatibility(const char *Profile, uint64_t Size); @@ -276,6 +278,9 @@ /*! \brief Get the size of the profile counters section in bytes. */ uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End); +/*! \brief Get the number of bytes in the profile bitmap section. */ +uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, const char *End); + /* ! \brief Given the sizes of the data and counter information, return the * number of padding bytes before and after the counters, and after the names, * in the raw profile. @@ -286,8 +291,9 @@ * needed to achieve that. */ void __llvm_profile_get_padding_sizes_for_counters( - uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, - uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, + uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters, + uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterBitmap, uint64_t *PaddingBytesAfterNames); /*! Index: compiler-rt/lib/profile/InstrProfiling.c =================================================================== --- compiler-rt/lib/profile/InstrProfiling.c +++ compiler-rt/lib/profile/InstrProfiling.c @@ -60,6 +60,10 @@ (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) ? 0xFF : 0; memset(I, ResetValue, E - I); + I = __llvm_profile_begin_bitmap(); + E = __llvm_profile_end_bitmap(); + memset(I, 0x0, E - I); + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const __llvm_profile_data *DI; Index: compiler-rt/lib/profile/InstrProfilingBuffer.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingBuffer.c +++ compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -43,11 +43,14 @@ const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); return __llvm_profile_get_size_for_buffer_internal( - DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd); + DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd, + NamesBegin, NamesEnd); } COMPILER_RT_VISIBILITY @@ -83,6 +86,11 @@ __llvm_profile_counter_entry_size(); } +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, const char *End) { + return (End - Begin); +} + /// Calculate the number of padding bytes needed to add to \p Offset in order /// for (\p Offset + Padding) to be page-aligned. static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) { @@ -102,13 +110,16 @@ COMPILER_RT_VISIBILITY void __llvm_profile_get_padding_sizes_for_counters( - uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, - uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, - uint64_t *PaddingBytesAfterNames) { + uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, + uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters, + uint64_t *PaddingBytesAfterCounters, + uint64_t *PaddingBytesAfterBitmapBytes, uint64_t *PaddingBytesAfterNames) { if (!needsCounterPadding()) { *PaddingBytesBeforeCounters = 0; *PaddingBytesAfterCounters = __llvm_profile_get_num_padding_bytes(CountersSize); + *PaddingBytesAfterBitmapBytes = + __llvm_profile_get_num_padding_bytes(NumBitmapBytes); *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize); return; } @@ -118,31 +129,38 @@ *PaddingBytesBeforeCounters = calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize); *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize); + *PaddingBytesAfterBitmapBytes = + calculateBytesNeededToPageAlign(NumBitmapBytes); *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize); } COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const char *CountersBegin, const char *CountersEnd, const char *NamesBegin, - const char *NamesEnd) { + const char *CountersBegin, const char *CountersEnd, + const char *BitmapBegin, const char *BitmapEnd, + const char *NamesBegin, const char *NamesEnd) { /* Match logic in __llvm_profile_write_buffer(). */ const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); uint64_t CountersSize = __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + const uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); /* Determine how much padding is needed before/after the counters and after * the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames; + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, - &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + DataSize, CountersSize, NumBitmapBytes, NamesSize, + &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, + &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize + PaddingBytesBeforeCounters + CountersSize + - PaddingBytesAfterCounters + NamesSize + PaddingBytesAfterNames; + PaddingBytesAfterCounters + NumBitmapBytes + + PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames; } COMPILER_RT_VISIBILITY @@ -160,9 +178,11 @@ COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal( char *Buffer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, - const char *CountersEnd, const char *NamesBegin, const char *NamesEnd) { + const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, + const char *NamesBegin, const char *NamesEnd) { ProfDataWriter BufferWriter; initBufferWriter(&BufferWriter, Buffer); return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin, - CountersEnd, 0, NamesBegin, NamesEnd, 0); + CountersEnd, BitmapBegin, BitmapEnd, 0, + NamesBegin, NamesEnd, 0); } Index: compiler-rt/lib/profile/InstrProfilingFile.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingFile.c +++ compiler-rt/lib/profile/InstrProfilingFile.c @@ -108,14 +108,18 @@ const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); uint64_t CountersSize = __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); - /* Check that the counter and data sections in this image are + /* Check that the counter, bitmap, and data sections in this image are * page-aligned. */ unsigned PageSize = getpagesize(); if ((intptr_t)CountersBegin % PageSize != 0) { @@ -123,6 +127,11 @@ CountersBegin, PageSize); return 1; } + if ((intptr_t)BitmapBegin % PageSize != 0) { + PROF_ERR("Bitmap section not page-aligned (start = %p, pagesz = %u).\n", + BitmapBegin, PageSize); + return 1; + } if ((intptr_t)DataBegin % PageSize != 0) { PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n", DataBegin, PageSize); @@ -132,10 +141,11 @@ /* Determine how much padding is needed before/after the counters and * after the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames; + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, - &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + DataSize, CountersSize, NumBitmapBytes, NamesSize, + &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, + &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); uint64_t PageAlignedCountersLength = CountersSize + PaddingBytesAfterCounters; uint64_t FileOffsetToCounters = CurrentFileOffset + @@ -155,6 +165,32 @@ FileOffsetToCounters); return 1; } + + /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap() + * will fail with EINVAL. */ + if (NumBitmapBytes == 0) + return 0; + + uint64_t PageAlignedBitmapLength = NumBitmapBytes + + PaddingBytesAfterBitmapBytes; + uint64_t FileOffsetToBitmap = CurrentFileOffset + + sizeof(__llvm_profile_header) + DataSize + + PaddingBytesBeforeCounters + CountersSize + + PaddingBytesAfterCounters; + void *BitmapMmap = mmap((void *)BitmapBegin, PageAlignedBitmapLength, + PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED, + Fileno, FileOffsetToBitmap); + if (BitmapMmap != BitmapBegin) { + PROF_ERR( + "Continuous counter sync mode is enabled, but mmap() failed (%s).\n" + " - BitmapBegin: %p\n" + " - PageAlignedBitmapLength: %" PRIu64 "\n" + " - Fileno: %d\n" + " - FileOffsetToBitmap: %" PRIu64 "\n", + strerror(errno), BitmapBegin, PageAlignedBitmapLength, Fileno, + FileOffsetToBitmap); + return 1; + } return 0; } #elif defined(__ELF__) || defined(_WIN32) @@ -197,6 +233,8 @@ const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); /* Get the file size. */ uint64_t FileSize = 0; @@ -218,6 +256,11 @@ /* Return the memory allocated for counters to OS. */ lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd); + + /* BIAS MODE not supported yet for Bitmap (MCDC). */ + + /* Return the memory allocated for counters to OS. */ + lprofReleaseMemoryPagesToOS((uintptr_t)BitmapBegin, (uintptr_t)BitmapEnd); return 0; } #else Index: compiler-rt/lib/profile/InstrProfilingInternal.h =================================================================== --- compiler-rt/lib/profile/InstrProfilingInternal.h +++ compiler-rt/lib/profile/InstrProfilingInternal.h @@ -21,7 +21,8 @@ */ uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const char *CountersBegin, const char *CountersEnd, const char *NamesBegin, + const char *CountersBegin, const char *CountersEnd, + const char *BitmapBegin, const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd); /*! @@ -36,7 +37,8 @@ int __llvm_profile_write_buffer_internal( char *Buffer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, - const char *CountersEnd, const char *NamesBegin, const char *NamesEnd); + const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, + const char *NamesBegin, const char *NamesEnd); /*! * The data structure describing the data to be written by the @@ -153,6 +155,7 @@ const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, const char *CountersEnd, + const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite); Index: compiler-rt/lib/profile/InstrProfilingMerge.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingMerge.c +++ compiler-rt/lib/profile/InstrProfilingMerge.c @@ -62,6 +62,9 @@ Header->CountersSize != __llvm_profile_get_num_counters(__llvm_profile_begin_counters(), __llvm_profile_end_counters()) || + Header->NumBitmapBytes != + __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(), + __llvm_profile_end_bitmap()) || Header->NamesSize != (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names()) || Header->ValueKindLast != IPVK_Last) @@ -70,7 +73,8 @@ if (ProfileSize < sizeof(__llvm_profile_header) + Header->BinaryIdsSize + Header->DataSize * sizeof(__llvm_profile_data) + Header->NamesSize + - Header->CountersSize * __llvm_profile_counter_entry_size()) + Header->CountersSize * __llvm_profile_counter_entry_size() + + Header->NumBitmapBytes) return 1; for (SrcData = SrcDataStart, @@ -78,7 +82,8 @@ SrcData < SrcDataEnd; ++SrcData, ++DstData) { if (SrcData->NameRef != DstData->NameRef || SrcData->FuncHash != DstData->FuncHash || - SrcData->NumCounters != DstData->NumCounters) + SrcData->NumCounters != DstData->NumCounters || + SrcData->NumBitmapBytes != DstData->NumBitmapBytes) return 1; } @@ -108,21 +113,24 @@ __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData; __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData; char *SrcCountersStart; + char *SrcBitmapStart; const char *SrcNameStart; const char *SrcValueProfDataStart, *SrcValueProfData; uintptr_t CountersDelta = Header->CountersDelta; + uintptr_t BitmapDelta = Header->BitmapDelta; SrcDataStart = (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + Header->BinaryIdsSize); SrcDataEnd = SrcDataStart + Header->DataSize; SrcCountersStart = (char *)SrcDataEnd; - SrcNameStart = SrcCountersStart + + SrcBitmapStart = SrcCountersStart + Header->CountersSize * __llvm_profile_counter_entry_size(); + SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes; SrcValueProfDataStart = SrcNameStart + Header->NamesSize + __llvm_profile_get_num_padding_bytes(Header->NamesSize); - if (SrcNameStart < SrcCountersStart) + if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart) return 1; for (SrcData = SrcDataStart, @@ -135,6 +143,8 @@ // extend CounterPtr to get the original value. char *DstCounters = (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr)); + char *DstBitmap = + (char *)((uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr)); unsigned NVK = 0; // SrcData is a serialized representation of the memory image. We need to @@ -164,6 +174,20 @@ } } + char *SrcBitmap = + SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta); + // BitmapDelta also needs to be decreased as we advance to the next data + // record. + BitmapDelta -= sizeof(*SrcData); + unsigned NB = SrcData->NumBitmapBytes; + // NumBitmapBytes may legitimately be 0. Just keep going. + if (NB != 0) { + if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart) + return 1; + for (unsigned I = 0; I < NB; I++) + DstBitmap[I] |= SrcBitmap[I]; + } + /* Now merge value profile data. */ if (!VPMergeHook) continue; Index: compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c +++ compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c @@ -31,6 +31,11 @@ COMPILER_RT_VISIBILITY extern char CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME); COMPILER_RT_VISIBILITY +extern char + BitmapStart __asm("section$start$__DATA$" INSTR_PROF_BITS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char BitmapEnd __asm("section$end$__DATA$" INSTR_PROF_BITS_SECT_NAME); +COMPILER_RT_VISIBILITY extern uint32_t OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME); @@ -56,6 +61,10 @@ COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return &CountersEnd; } COMPILER_RT_VISIBILITY +char *__llvm_profile_begin_bitmap(void) { return &BitmapStart; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; } +COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } COMPILER_RT_VISIBILITY Index: compiler-rt/lib/profile/InstrProfilingPlatformLinux.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingPlatformLinux.c +++ compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -35,6 +35,8 @@ #define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON) #define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON) #define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON) +#define PROF_BITS_START INSTR_PROF_SECT_START(INSTR_PROF_BITS_COMMON) +#define PROF_BITS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_BITS_COMMON) #define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON) #define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_COMMON) #define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_COMMON) @@ -48,6 +50,8 @@ COMPILER_RT_WEAK; extern char PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_BITS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_BITS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_NAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; @@ -74,6 +78,12 @@ COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return &PROF_CNTS_STOP; } +COMPILER_RT_VISIBILITY char *__llvm_profile_begin_bitmap(void) { + return &PROF_BITS_START; +} +COMPILER_RT_VISIBILITY char *__llvm_profile_end_bitmap(void) { + return &PROF_BITS_STOP; +} COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return &PROF_ORDERFILE_START; } Index: compiler-rt/lib/profile/InstrProfilingPlatformOther.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingPlatformOther.c +++ compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -88,6 +88,10 @@ char *__llvm_profile_begin_counters(void) { return CountersFirst; } COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return CountersLast; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_begin_bitmap(void) { return BitmapFirst; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_end_bitmap(void) { return BitmapLast; } /* TODO: correctly set up OrderFileFirst. */ COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return OrderFileFirst; } Index: compiler-rt/lib/profile/InstrProfilingPlatformWindows.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingPlatformWindows.c +++ compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -14,6 +14,7 @@ #if defined(_MSC_VER) /* Merge read-write sections into .data. */ #pragma comment(linker, "/MERGE:.lprfc=.data") +#pragma comment(linker, "/MERGE:.lprfb=.data") #pragma comment(linker, "/MERGE:.lprfd=.data") #pragma comment(linker, "/MERGE:.lprfv=.data") #pragma comment(linker, "/MERGE:.lprfnd=.data") @@ -30,6 +31,8 @@ #pragma section(".lprfd$Z", read, write) #pragma section(".lprfc$A", read, write) #pragma section(".lprfc$Z", read, write) +#pragma section(".lprfb$A", read, write) +#pragma section(".lprfb$Z", read, write) #pragma section(".lorderfile$A", read, write) #pragma section(".lprfnd$A", read, write) #pragma section(".lprfnd$Z", read, write) @@ -43,6 +46,8 @@ char COMPILER_RT_SECTION(".lprfc$A") CountersStart; char COMPILER_RT_SECTION(".lprfc$Z") CountersEnd; +char COMPILER_RT_SECTION(".lprfb$A") BitmapStart; +char COMPILER_RT_SECTION(".lprfb$Z") BitmapEnd; uint32_t COMPILER_RT_SECTION(".lorderfile$A") OrderFileStart; ValueProfNode COMPILER_RT_SECTION(".lprfnd$A") VNodesStart; @@ -58,6 +63,8 @@ char *__llvm_profile_begin_counters(void) { return &CountersStart + 1; } char *__llvm_profile_end_counters(void) { return &CountersEnd; } +char *__llvm_profile_begin_bitmap(void) { return &BitmapStart + 1; } +char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; } uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } ValueProfNode *__llvm_profile_begin_vnodes(void) { return &VNodesStart + 1; } Index: compiler-rt/lib/profile/InstrProfilingWriter.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingWriter.c +++ compiler-rt/lib/profile/InstrProfilingWriter.c @@ -246,17 +246,20 @@ const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const char *CountersBegin = __llvm_profile_begin_counters(); const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, - CountersEnd, VPDataReader, NamesBegin, NamesEnd, - SkipNameDataWrite); + CountersEnd, BitmapBegin, BitmapEnd, VPDataReader, + NamesBegin, NamesEnd, SkipNameDataWrite); } COMPILER_RT_VISIBILITY int lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, const char *CountersEnd, + const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite) { int DebugInfoCorrelate = @@ -271,6 +274,8 @@ __llvm_profile_get_counters_size(CountersBegin, CountersEnd); const uint64_t NumCounters = __llvm_profile_get_num_counters(CountersBegin, CountersEnd); + const uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin; /* Create the header. */ @@ -279,11 +284,11 @@ /* Determine how much padding is needed before/after the counters and after * the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames; + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; __llvm_profile_get_padding_sizes_for_counters( - DataSectionSize, CountersSectionSize, NamesSize, + DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, - &PaddingBytesAfterNames); + &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); { // TODO: Unfortunately the header's fields are named DataSize and @@ -300,6 +305,7 @@ * CountersDelta to match. */ #ifdef _WIN64 Header.CountersDelta = (uint32_t)Header.CountersDelta; + Header.BitmapDelta = (uint32_t)Header.BitmapDelta; #endif /* The data and names sections are omitted in lightweight mode. */ @@ -324,6 +330,8 @@ {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, + {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1}, {(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}}; Index: compiler-rt/test/profile/instrprof-write-buffer-internal.c =================================================================== --- compiler-rt/test/profile/instrprof-write-buffer-internal.c +++ compiler-rt/test/profile/instrprof-write-buffer-internal.c @@ -25,15 +25,20 @@ const char *__llvm_profile_end_names(void); char *__llvm_profile_begin_counters(void); char *__llvm_profile_end_counters(void); +char *__llvm_profile_begin_bitmap(void); +char *__llvm_profile_end_bitmap(void); uint64_t __llvm_profile_get_size_for_buffer_internal( const void *DataBegin, const void *DataEnd, const char *CountersBegin, - const char *CountersEnd, const char *NamesBegin, const char *NamesEnd); + const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, + const char *NamesBegin, const char *NamesEnd); int __llvm_profile_write_buffer_internal(char *Buffer, const void *DataBegin, const void *DataEnd, const char *CountersBegin, const char *CountersEnd, + const char *BitmapBegin, + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd); @@ -43,12 +48,14 @@ uint64_t bufsize = __llvm_profile_get_size_for_buffer_internal( __llvm_profile_begin_data(), __llvm_profile_end_data(), __llvm_profile_begin_counters(), __llvm_profile_end_counters(), + __llvm_profile_begin_bitmap(), __llvm_profile_end_bitmap(), __llvm_profile_begin_names(), __llvm_profile_end_names()); char *buf = malloc(bufsize); int ret = __llvm_profile_write_buffer_internal(buf, __llvm_profile_begin_data(), __llvm_profile_end_data(), __llvm_profile_begin_counters(), __llvm_profile_end_counters(), + __llvm_profile_begin_bitmap(), __llvm_profile_end_bitmap(), __llvm_profile_begin_names(), __llvm_profile_end_names()); if (ret != 0) { Index: llvm/docs/LangRef.rst =================================================================== --- llvm/docs/LangRef.rst +++ llvm/docs/LangRef.rst @@ -13802,6 +13802,144 @@ ``llvm.instrprof.value.profile`` intrinsic with the call to the profile runtime library with proper arguments. +'``llvm.instrprof.mcdc.parameters``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare void @llvm.instrprof.mcdc.parameters(ptr , i64 , + i32 ) + +Overview: +""""""""" + +The '``llvm.instrprof.mcdc.parameters``' intrinsic is used to initiate MC/DC +code coverage instrumentation for a function. + +Arguments: +"""""""""" + +The first argument is a pointer to a global variable containing the +name of the entity being instrumented. This should generally be the +(mangled) function name for a set of counters. + +The second argument is a hash value that can be used by the consumer +of the profile data to detect changes to the instrumented source. + +The third argument is the number of bitmap bytes required by the function to +record the number of test vectors executed for each boolean expression. + +Semantics: +"""""""""" + +This intrinsic represents basic MC/DC parameters initiating one or more MC/DC +instrumentation sequences in a function. It will cause the ``-instrprof`` pass +to generate the appropriate data structures and the code to instrument MC/DC +test vectors in a format that can be written out by a compiler runtime and +consumed via the ``llvm-profdata`` tool. + +'``llvm.instrprof.mcdc.condbitmap.update``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare void @llvm.instrprof.mcdc.condbitmap.update(ptr , i64 , + i32 , + ptr , + i1 ) + +Overview: +""""""""" + +The '``llvm.instrprof.mcdc.condbitmap.update``' intrinsic is used to track +MC/DC condition evaluation for each condition in a boolean expression. + +Arguments: +"""""""""" + +The first argument is a pointer to a global variable containing the +name of the entity being instrumented. This should generally be the +(mangled) function name for a set of counters. + +The second argument is a hash value that can be used by the consumer +of the profile data to detect changes to the instrumented source. + +The third argument is an ID of a condition to track. This value is used as a +bit index into the condition bitmap. + +The fourth argument is the address of the condition bitmap. + +The fifth argument is the boolean value representing the evaluation of the +condition (true or false) + +Semantics: +"""""""""" + +This intrinsic represents the update of a condition bitmap that is local to a +function and will cause the ``-instrprof`` pass to generate the code to +instrument the control flow around each condition in a boolean expression. The +ID of each condition corresponds to a bit index in the condition bitmap which +is set based on the evaluation of the condition. + +'``llvm.instrprof.mcdc.tvbitmap.update``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare void @llvm.instrprof.mcdc.tvbitmap.update(ptr , i64 , + i32 ) + i32 , + ptr ) + +Overview: +""""""""" + +The '``llvm.instrprof.mcdc.tvbitmap.update``' intrinsic is used to track MC/DC +test vector execution after each boolean expression has been fully executed. +The overall value of the condition bitmap, after it has been successively +updated using the '``llvm.instrprof.mcdc.condbitmap.update``' intrinsic with +the true or false evaluation of each condition, uniquely identifies an executed +MC/DC test vector and is used as a bit index into the global test vector +bitmap. + +Arguments: +"""""""""" + +The first argument is a pointer to a global variable containing the +name of the entity being instrumented. This should generally be the +(mangled) function name for a set of counters. + +The second argument is a hash value that can be used by the consumer +of the profile data to detect changes to the instrumented source. + +The third argument is the number of bitmap bytes required by the function to +record the number of test vectors executed for each boolean expression. + +The fourth argument is the byte index into the global test vector bitmap +corresponding to the function. + +The fifth argument is the address of the condition bitmap, which contains a +value representing an executed MC/DC test vector. It is loaded and used as the +bit index of the test vector bitmap. + +Semantics: +"""""""""" + +This intrinsic represents the final operation of an MC/DC instrumentation +sequence and will cause the ``-instrprof`` pass to generate the code to +instrument an update of a function's global test vector bitmap to indicate that +a test vector has been executed. The global test vector bitmap can be consumed +by the ``llvm-profdata`` and ``llvm-cov`` tools. + '``llvm.thread.pointer``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Index: llvm/include/llvm/IR/IntrinsicInst.h =================================================================== --- llvm/include/llvm/IR/IntrinsicInst.h +++ llvm/include/llvm/IR/IntrinsicInst.h @@ -1399,6 +1399,11 @@ ConstantInt *getHash() const { return cast(const_cast(getArgOperand(1))); } +}; + +/// A base class for all instrprof counter intrinsics. +class InstrProfCntrInstBase : public InstrProfInstBase { +public: // The number of counters for the instrumented function. ConstantInt *getNumCounters() const; // The index of the counter that this instruction acts on. @@ -1406,7 +1411,7 @@ }; /// This represents the llvm.instrprof.cover intrinsic. -class InstrProfCoverInst : public InstrProfInstBase { +class InstrProfCoverInst : public InstrProfCntrInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_cover; @@ -1417,7 +1422,7 @@ }; /// This represents the llvm.instrprof.increment intrinsic. -class InstrProfIncrementInst : public InstrProfInstBase { +class InstrProfIncrementInst : public InstrProfCntrInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_increment || @@ -1441,7 +1446,7 @@ }; /// This represents the llvm.instrprof.timestamp intrinsic. -class InstrProfTimestampInst : public InstrProfInstBase { +class InstrProfTimestampInst : public InstrProfCntrInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_timestamp; @@ -1452,7 +1457,7 @@ }; /// This represents the llvm.instrprof.value.profile intrinsic. -class InstrProfValueProfileInst : public InstrProfInstBase { +class InstrProfValueProfileInst : public InstrProfCntrInstBase { public: static bool classof(const IntrinsicInst *I) { return I->getIntrinsicID() == Intrinsic::instrprof_value_profile; @@ -1475,6 +1480,79 @@ } }; +/// A base class for instrprof mcdc intrinsics that require global bitmap bytes. +class InstrProfMCDCBitmapInstBase : public InstrProfInstBase { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_parameters || + I->getIntrinsicID() == Intrinsic::instrprof_mcdc_tvbitmap_update; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + + // The number of bytes used for MCDC bitmaps for the instrumented function. + ConstantInt *getNumBitmapBytes() const { + return cast(const_cast(getArgOperand(2))); + } +}; + +/// This represents the llvm.instrprof.mcdc.parameters intrinsic. +class InstrProfMCDCBitmapParameters : public InstrProfMCDCBitmapInstBase { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_parameters; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } +}; + +/// This represents the llvm.instrprof.mcdc.tvbitmap.update intrinsic. +class InstrProfMCDCTVBitmapUpdate : public InstrProfMCDCBitmapInstBase { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_tvbitmap_update; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + + // The index of the bitmap that this instruction acts on. + ConstantInt *getBitmapIndex() const { + return cast(const_cast(getArgOperand(3))); + } + + Value *getMCDCCondBitmapAddr() const { + return cast(const_cast(getArgOperand(4))); + } +}; + +/// This represents the llvm.instrprof.mcdc.condbitmap.update intrinsic. +/// It does not pertain to global bitmap updates or parameters and so doesn't +/// inherit from InstrProfMCDCBitmapInstBase. +class InstrProfMCDCCondBitmapUpdate : public InstrProfInstBase { +public: + static bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::instrprof_mcdc_condbitmap_update; + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + + ConstantInt *getCondID() const { + return cast(const_cast(getArgOperand(2))); + } + + Value *getMCDCCondBitmapAddr() const { + return cast(const_cast(getArgOperand(3))); + } + + Value *getCondBool() const { + return cast(const_cast(getArgOperand(4))); + } +}; + class PseudoProbeInst : public IntrinsicInst { public: static bool classof(const IntrinsicInst *I) { Index: llvm/include/llvm/IR/Intrinsics.td =================================================================== --- llvm/include/llvm/IR/Intrinsics.td +++ llvm/include/llvm/IR/Intrinsics.td @@ -925,6 +925,21 @@ llvm_i64_ty, llvm_i32_ty, llvm_i32_ty]>; +// A parameter configuration for instrumentation based MCDC profiling. +def int_instrprof_mcdc_parameters : Intrinsic<[], + [llvm_ptr_ty, llvm_i64_ty, + llvm_i32_ty]>; + +// A test vector bitmap update for instrumentation based MCDC profiling. +def int_instrprof_mcdc_tvbitmap_update : Intrinsic<[], + [llvm_ptr_ty, llvm_i64_ty, + llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty]>; + +// A condition bitmap value update for instrumentation based MCDC profiling. +def int_instrprof_mcdc_condbitmap_update : Intrinsic<[], + [llvm_ptr_ty, llvm_i64_ty, + llvm_i32_ty, llvm_ptr_ty, llvm_i1_ty]>; + def int_call_preallocated_setup : DefaultAttrsIntrinsic<[llvm_token_ty], [llvm_i32_ty]>; def int_call_preallocated_arg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_token_ty, llvm_i32_ty]>; def int_call_preallocated_teardown : DefaultAttrsIntrinsic<[], [llvm_token_ty]>; Index: llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h =================================================================== --- llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -1023,7 +1023,9 @@ // Compilation directory is stored separately and combined with relative // filenames to produce an absolute file path. Version6 = 5, - // The current version is Version6. + // Branch regions extended and Decision Regions added for MC/DC. + Version7 = 6, + // The current version is Version7. CurrentVersion = INSTR_PROF_COVMAP_VERSION }; Index: llvm/include/llvm/ProfileData/InstrProf.h =================================================================== --- llvm/include/llvm/ProfileData/InstrProf.h +++ llvm/include/llvm/ProfileData/InstrProf.h @@ -96,6 +96,9 @@ /// Return the name prefix of profile counter variables. inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; } +/// Return the name prefix of profile bitmap variables. +inline StringRef getInstrProfBitmapVarPrefix() { return "__profbm_"; } + /// Return the name prefix of value profile variables. inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; } @@ -326,6 +329,7 @@ invalid_prof, hash_mismatch, count_mismatch, + bitmap_mismatch, counter_overflow, value_site_count_mismatch, compress_failed, @@ -697,18 +701,23 @@ /// Profiling information for a single function. struct InstrProfRecord { std::vector Counts; + std::vector BitmapBytes; InstrProfRecord() = default; InstrProfRecord(std::vector Counts) : Counts(std::move(Counts)) {} + InstrProfRecord(std::vector Counts, + std::vector BitmapBytes) + : Counts(std::move(Counts)), BitmapBytes(std::move(BitmapBytes)) {} InstrProfRecord(InstrProfRecord &&) = default; InstrProfRecord(const InstrProfRecord &RHS) - : Counts(RHS.Counts), + : Counts(RHS.Counts), BitmapBytes(RHS.BitmapBytes), ValueData(RHS.ValueData ? std::make_unique(*RHS.ValueData) : nullptr) {} InstrProfRecord &operator=(InstrProfRecord &&) = default; InstrProfRecord &operator=(const InstrProfRecord &RHS) { Counts = RHS.Counts; + BitmapBytes = RHS.BitmapBytes; if (!RHS.ValueData) { ValueData = nullptr; return *this; @@ -887,6 +896,11 @@ NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector Counts) : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {} + NamedInstrProfRecord(StringRef Name, uint64_t Hash, + std::vector Counts, + std::vector BitmapBytes) + : InstrProfRecord(std::move(Counts), std::move(BitmapBytes)), Name(Name), + Hash(Hash) {} static bool hasCSFlagInHash(uint64_t FuncHash) { return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1); @@ -1023,6 +1037,9 @@ // An additional (optional) temporal profile traces section is added. Version10 = 10, // The current version is 10. + // An additional field is used for bitmap bytes. + Version11 = 11, + // The current version is 11. CurrentVersion = INSTR_PROF_INDEX_VERSION }; const uint64_t Version = ProfVersion::CurrentVersion; @@ -1160,6 +1177,7 @@ // Version 6: Added binary id. // Version 7: Reorder binary id and include version in signature. // Version 8: Use relative counter pointer. +// Version 9: Added relative bitmap bytes pointer and count used by MC/DC. const uint64_t Version = INSTR_PROF_RAW_VERSION; template inline uint64_t getMagic(); Index: llvm/include/llvm/ProfileData/InstrProfData.inc =================================================================== --- llvm/include/llvm/ProfileData/InstrProfData.inc +++ llvm/include/llvm/ProfileData/InstrProfData.inc @@ -76,6 +76,7 @@ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ Inc->getHash()->getZExtValue())) INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) +INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr) /* This is used to map function pointers for the indirect call targets to * function name hashes during the conversion from raw to merged profile * data. @@ -87,7 +88,9 @@ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ - ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) + ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \ +INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes)) #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ @@ -134,9 +137,13 @@ /* FIXME: A more accurate name is NumCounters */ INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) +INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes) +INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin - (uintptr_t)DataBegin) +INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, + (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER @@ -269,6 +276,9 @@ INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ INSTR_PROF_CNTS_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ + INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \ + INSTR_PROF_BITS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ INSTR_PROF_NAME_COFF, "__DATA,") @@ -648,11 +658,11 @@ /* FIXME: Please remedy the fixme in the header before bumping the version. */ /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 8 +#define INSTR_PROF_RAW_VERSION 9 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 10 +#define INSTR_PROF_INDEX_VERSION 11 /* Coverage mapping format version (start from 0). */ -#define INSTR_PROF_COVMAP_VERSION 5 +#define INSTR_PROF_COVMAP_VERSION 6 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the * version for other variants of profile. We set the lowest bit of the upper 8 @@ -689,6 +699,7 @@ #define INSTR_PROF_DATA_COMMON __llvm_prf_data #define INSTR_PROF_NAME_COMMON __llvm_prf_names #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts +#define INSTR_PROF_BITS_COMMON __llvm_prf_bits #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap @@ -700,6 +711,7 @@ #define INSTR_PROF_DATA_COFF ".lprfd$M" #define INSTR_PROF_NAME_COFF ".lprfn$M" #define INSTR_PROF_CNTS_COFF ".lprfc$M" +#define INSTR_PROF_BITS_COFF ".lprfb$M" #define INSTR_PROF_VALS_COFF ".lprfv$M" #define INSTR_PROF_VNODES_COFF ".lprfnd$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" @@ -711,6 +723,7 @@ #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF +#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ @@ -725,6 +738,7 @@ #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) +#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ Index: llvm/include/llvm/ProfileData/InstrProfReader.h =================================================================== --- llvm/include/llvm/ProfileData/InstrProfReader.h +++ llvm/include/llvm/ProfileData/InstrProfReader.h @@ -323,11 +323,14 @@ // the variant types of the profile. uint64_t Version; uint64_t CountersDelta; + uint64_t BitmapDelta; uint64_t NamesDelta; const RawInstrProf::ProfileData *Data; const RawInstrProf::ProfileData *DataEnd; const char *CountersStart; const char *CountersEnd; + const char *BitmapStart; + const char *BitmapEnd; const char *NamesStart; const char *NamesEnd; // After value profile is all read, this pointer points to @@ -429,6 +432,7 @@ Error readName(NamedInstrProfRecord &Record); Error readFuncHash(NamedInstrProfRecord &Record); Error readRawCounts(InstrProfRecord &Record); + Error readRawBitmapBytes(InstrProfRecord &Record); Error readValueProfilingData(InstrProfRecord &Record); bool atEnd() const { return Data == DataEnd; } @@ -441,6 +445,7 @@ // As we advance to the next record, we maintain the correct CountersDelta // with respect to the next record. CountersDelta -= sizeof(*Data); + BitmapDelta -= sizeof(*Data); } Data++; ValueDataStart += CurValueDataSize; @@ -729,6 +734,10 @@ Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector &Counts); + /// Fill Bitmap Bytes with the profile data for the given function name. + Error getFunctionBitmapBytes(StringRef FuncName, uint64_t FuncHash, + std::vector &BitmapBytes); + /// Return the maximum of all known function counts. /// \c UseCS indicates whether to use the context-sensitive count. uint64_t getMaximumFunctionCount(bool UseCS) { Index: llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h =================================================================== --- llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h +++ llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h @@ -50,6 +50,7 @@ uint32_t NumValueSites[IPVK_Last + 1]; GlobalVariable *RegionCounters = nullptr; GlobalVariable *DataVar = nullptr; + GlobalVariable *RegionBitmaps = nullptr; PerFunctionProfileData() { memset(NumValueSites, 0, sizeof(uint32_t) * (IPVK_Last + 1)); @@ -105,20 +106,54 @@ /// Force emitting of name vars for unused functions. void lowerCoverageData(GlobalVariable *CoverageNamesVar); + /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction + /// using the index represented by the a temp value into a bitmap. + void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins); + + /// Replace instrprof.mcdc.temp.update with a shift and or instruction using + /// the corresponding condition ID. + void lowerMCDCCondBitmapUpdate(InstrProfMCDCCondBitmapUpdate *Ins); + /// Compute the address of the counter value that this profiling instruction /// acts on. - Value *getCounterAddress(InstrProfInstBase *I); + Value *getCounterAddress(InstrProfCntrInstBase *I); /// Get the region counters for an increment, creating them if necessary. /// /// If the counter array doesn't yet exist, the profile data variables /// referring to them will also be created. - GlobalVariable *getOrCreateRegionCounters(InstrProfInstBase *Inc); + GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc); /// Create the region counters. - GlobalVariable *createRegionCounters(InstrProfInstBase *Inc, StringRef Name, + GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name, GlobalValue::LinkageTypes Linkage); + /// Compute the address of the test vector bitmap that this profiling + /// instruction acts on. + Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I); + + /// Get the region bitmaps for an increment, creating them if necessary. + /// + /// If the bitmap array doesn't yet exist, the profile data variables + /// referring to them will also be created. + GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc); + + /// Create the region bitmaps. + GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, + StringRef Name, + GlobalValue::LinkageTypes Linkage); + + /// Set Comdat property of GV, if required. + void maybeSetComdat(GlobalVariable *GV, Function *Fn, StringRef VarName); + + /// Setup the sections into which counters and bitmaps are allocated. + GlobalVariable *setupProfileSection(InstrProfInstBase *Inc, + InstrProfSectKind IPSK); + + /// Create INSTR_PROF_DATA variable for counters and bitmaps. + void createDataVariable(InstrProfCntrInstBase *Inc, + InstrProfMCDCBitmapParameters *Update); + /// Emit the section with compressed function names. void emitNameData(); Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -7137,6 +7137,12 @@ llvm_unreachable("instrprof failed to lower a timestamp"); case Intrinsic::instrprof_value_profile: llvm_unreachable("instrprof failed to lower a value profiling call"); + case Intrinsic::instrprof_mcdc_parameters: + llvm_unreachable("instrprof failed to lower mcdc parameters"); + case Intrinsic::instrprof_mcdc_tvbitmap_update: + llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update"); + case Intrinsic::instrprof_mcdc_condbitmap_update: + llvm_unreachable("instrprof failed to lower an mcdc condbitmap update"); case Intrinsic::localescape: { MachineFunction &MF = DAG.getMachineFunction(); const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); Index: llvm/lib/IR/IntrinsicInst.cpp =================================================================== --- llvm/lib/IR/IntrinsicInst.cpp +++ llvm/lib/IR/IntrinsicInst.cpp @@ -270,13 +270,13 @@ return -1; } -ConstantInt *InstrProfInstBase::getNumCounters() const { +ConstantInt *InstrProfCntrInstBase::getNumCounters() const { if (InstrProfValueProfileInst::classof(this)) llvm_unreachable("InstrProfValueProfileInst does not have counters!"); return cast(const_cast(getArgOperand(2))); } -ConstantInt *InstrProfInstBase::getIndex() const { +ConstantInt *InstrProfCntrInstBase::getIndex() const { if (InstrProfValueProfileInst::classof(this)) llvm_unreachable("Please use InstrProfValueProfileInst::getIndex()"); return cast(const_cast(getArgOperand(3))); Index: llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp =================================================================== --- llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp +++ llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp @@ -757,6 +757,7 @@ case CovMapVersion::Version4: case CovMapVersion::Version5: case CovMapVersion::Version6: + case CovMapVersion::Version7: // Decompress the name data. if (Error E = P.create(P.getNameData())) return std::move(E); @@ -775,6 +776,9 @@ else if (Version == CovMapVersion::Version6) return std::make_unique>(P, R, D, F); + else if (Version == CovMapVersion::Version7) + return std::make_unique>(P, R, D, F); } llvm_unreachable("Unsupported version"); } Index: llvm/lib/ProfileData/InstrProf.cpp =================================================================== --- llvm/lib/ProfileData/InstrProf.cpp +++ llvm/lib/ProfileData/InstrProf.cpp @@ -135,6 +135,9 @@ case instrprof_error::count_mismatch: OS << "function basic block count change detected (counter mismatch)"; break; + case instrprof_error::bitmap_mismatch: + OS << "function bitmap size change detected (bitmap size mismatch)"; + break; case instrprof_error::counter_overflow: OS << "counter overflow"; break; @@ -725,6 +728,18 @@ Warn(instrprof_error::counter_overflow); } + // If the number of bitmap bytes doesn't match we either have bad data + // or a hash collision. + if (BitmapBytes.size() != Other.BitmapBytes.size()) { + Warn(instrprof_error::bitmap_mismatch); + return; + } + + // Bitmap bytes are merged by simply ORing them together. + for (size_t I = 0, E = Other.BitmapBytes.size(); I < E; ++I) { + BitmapBytes[I] = Other.BitmapBytes[I] | BitmapBytes[I]; + } + for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) mergeValueProfData(Kind, Other, Weight, Warn); } @@ -1397,9 +1412,11 @@ // When a new field is added in the header add a case statement here to // populate it. static_assert( - IndexedInstrProf::ProfVersion::CurrentVersion == Version10, + IndexedInstrProf::ProfVersion::CurrentVersion == Version11, "Please update the reading code below if a new field has been added, " "if not add a case statement to fall through to the latest version."); + case 11ull: + [[fallthrough]]; case 10ull: H.TemporalProfTracesOffset = read(Buffer, offsetOf(&Header::TemporalProfTracesOffset)); @@ -1423,10 +1440,12 @@ // When a new field is added to the header add a case statement here to // compute the size as offset of the new field + size of the new field. This // relies on the field being added to the end of the list. - static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version10, + static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version11, "Please update the size computation below if a new field has " "been added to the header, if not add a case statement to " "fall through to the latest version."); + case 11ull: + [[fallthrough]]; case 10ull: return offsetOf(&Header::TemporalProfTracesOffset) + sizeof(Header::TemporalProfTracesOffset); Index: llvm/lib/ProfileData/InstrProfCorrelator.cpp =================================================================== --- llvm/lib/ProfileData/InstrProfCorrelator.cpp +++ llvm/lib/ProfileData/InstrProfCorrelator.cpp @@ -193,11 +193,9 @@ } template -void InstrProfCorrelatorImpl::addProbe(StringRef FunctionName, - uint64_t CFGHash, - IntPtrT CounterOffset, - IntPtrT FunctionPtr, - uint32_t NumCounters) { +void InstrProfCorrelatorImpl::addProbe( + StringRef FunctionName, uint64_t CFGHash, IntPtrT CounterOffset, + IntPtrT FunctionPtr, uint32_t NumCounters) { // Check if a probe was already added for this counter offset. if (!CounterOffsets.insert(CounterOffset).second) return; @@ -207,11 +205,15 @@ // In this mode, CounterPtr actually stores the section relative address // of the counter. maybeSwap(CounterOffset), + // TODO: MC/DC is not yet supported. + /*BitmapOffset=*/maybeSwap(0), maybeSwap(FunctionPtr), // TODO: Value profiling is not yet supported. /*ValuesPtr=*/maybeSwap(0), maybeSwap(NumCounters), /*NumValueSites=*/{maybeSwap(0), maybeSwap(0)}, + // TODO: MC/DC is not yet supported. + /*NumBitmapBytes=*/maybeSwap(0), }); NamesVec.push_back(FunctionName.str()); } Index: llvm/lib/ProfileData/InstrProfReader.cpp =================================================================== --- llvm/lib/ProfileData/InstrProfReader.cpp +++ llvm/lib/ProfileData/InstrProfReader.cpp @@ -433,6 +433,28 @@ Record.Counts.push_back(Count); } + // Bitmap byte information is indicated with special character. + if (Line->startswith("$")) { + Record.BitmapBytes.clear(); + // Read the number of bitmap bytes. + uint64_t NumBitmapBytes; + if ((Line++)->drop_front(1).trim().getAsInteger(0, NumBitmapBytes)) + return error(instrprof_error::malformed, + "number of bitmap bytes is not a valid integer"); + if (NumBitmapBytes != 0) { + // Read each bitmap and fill our internal storage with the values. + Record.BitmapBytes.reserve(NumBitmapBytes); + for (uint8_t I = 0; I < NumBitmapBytes; ++I) { + if (Line.is_at_end()) + return error(instrprof_error::truncated); + uint8_t BitmapByte; + if ((Line++)->getAsInteger(0, BitmapByte)) + return error(instrprof_error::malformed, "bitmap byte is invalid"); + Record.BitmapBytes.push_back(BitmapByte); + } + } + } + // Check if value profile data exists and read it if so. if (Error E = readValueProfileData(Record)) return error(std::move(E)); @@ -549,11 +571,14 @@ return error(instrprof_error::bad_header); CountersDelta = swap(Header.CountersDelta); + BitmapDelta = swap(Header.BitmapDelta); NamesDelta = swap(Header.NamesDelta); auto NumData = swap(Header.DataSize); auto PaddingBytesBeforeCounters = swap(Header.PaddingBytesBeforeCounters); auto CountersSize = swap(Header.CountersSize) * getCounterTypeSize(); auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters); + auto NumBitmapBytes = swap(Header.NumBitmapBytes); + auto PaddingBytesAfterBitmapBytes = swap(Header.PaddingBytesAfterBitmapBytes); auto NamesSize = swap(Header.NamesSize); ValueKindLast = swap(Header.ValueKindLast); @@ -563,8 +588,10 @@ // Profile data starts after profile header and binary ids if exist. ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdsSize; ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters; + ptrdiff_t BitmapOffset = CountersOffset + CountersSize + + PaddingBytesAfterCounters; ptrdiff_t NamesOffset = - CountersOffset + CountersSize + PaddingBytesAfterCounters; + BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes; ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize; auto *Start = reinterpret_cast(&Header); @@ -593,6 +620,8 @@ reinterpret_cast(&Header) + sizeof(RawInstrProf::Header); CountersStart = Start + CountersOffset; CountersEnd = CountersStart + CountersSize; + BitmapStart = Start + BitmapOffset; + BitmapEnd = BitmapStart + NumBitmapBytes; ValueDataStart = reinterpret_cast(Start + ValueDataOffset); const uint8_t *BufferEnd = (const uint8_t *)DataBuffer->getBufferEnd(); @@ -683,6 +712,51 @@ return success(); } +template +Error RawInstrProfReader::readRawBitmapBytes( + InstrProfRecord &Record) { + uint32_t NumBitmapBytes = swap(Data->NumBitmapBytes); + + Record.BitmapBytes.clear(); + Record.BitmapBytes.reserve(NumBitmapBytes); + + // It's possible MCDC is either not enabled or only used for some functions + // and not others. So if we record 0 bytes, just move on. + if (NumBitmapBytes == 0) { + return success(); + } + + // BitmapDelta decreases as we advance to the next data record. + ptrdiff_t BitmapOffset = swap(Data->BitmapPtr) - BitmapDelta; + if (BitmapOffset < 0) + return error( + instrprof_error::malformed, + ("bitmap offset " + Twine(BitmapOffset) + " is negative").str()); + + if (BitmapOffset >= BitmapEnd - BitmapStart) + return error(instrprof_error::malformed, + ("bitmap offset " + Twine(BitmapOffset) + + " is greater than the maximum bitmap offset " + + Twine(BitmapEnd - BitmapStart - 1)) + .str()); + + uint64_t MaxNumBitmapBytes = + (BitmapEnd - (BitmapStart + BitmapOffset)) / sizeof(uint8_t); + if (NumBitmapBytes > MaxNumBitmapBytes) + return error(instrprof_error::malformed, + ("number of bitmap bytes " + Twine(NumBitmapBytes) + + " is greater than the maximum number of bitmap bytes " + + Twine(MaxNumBitmapBytes)) + .str()); + + for (uint32_t I = 0; I < NumBitmapBytes; I++) { + const char *Ptr = BitmapStart + BitmapOffset + I; + Record.BitmapBytes.push_back(swap(*Ptr)); + } + + return success(); +} + template Error RawInstrProfReader::readValueProfilingData( InstrProfRecord &Record) { @@ -733,6 +807,10 @@ if (Error E = readRawCounts(Record)) return error(std::move(E)); + // Read raw bitmap bytes and set Record. + if (Error E = readRawBitmapBytes(Record)) + return error(std::move(E)); + // Read value data and set Record. if (Error E = readValueProfilingData(Record)) return error(std::move(E)); @@ -794,6 +872,7 @@ DataBuffer.clear(); std::vector CounterBuffer; + std::vector BitmapByteBuffer; const unsigned char *End = D + N; while (D < End) { @@ -819,7 +898,24 @@ for (uint64_t J = 0; J < CountsSize; ++J) CounterBuffer.push_back(endian::readNext(D)); - DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer)); + // Read bitmap bytes for GET_VERSION(FormatVersion) > 8. + if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version8) { + uint64_t BitmapBytes = 0; + if (D + sizeof(uint64_t) > End) + return data_type(); + BitmapBytes = endian::readNext(D); + // Read bitmap byte values. + if (D + BitmapBytes * sizeof(uint8_t) > End) + return data_type(); + BitmapByteBuffer.clear(); + BitmapByteBuffer.reserve(BitmapBytes); + for (uint64_t J = 0; J < BitmapBytes; ++J) + BitmapByteBuffer.push_back(static_cast( + endian::readNext(D))); + } + + DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer), + std::move(BitmapByteBuffer)); // Read value profiling data. if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 && @@ -1306,6 +1402,16 @@ return success(); } +Error IndexedInstrProfReader::getFunctionBitmapBytes( + StringRef FuncName, uint64_t FuncHash, std::vector &BitmapBytes) { + Expected Record = getInstrProfRecord(FuncName, FuncHash); + if (Error E = Record.takeError()) + return error(std::move(E)); + + BitmapBytes = Record.get().BitmapBytes; + return success(); +} + Error IndexedInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) { ArrayRef Data; Index: llvm/lib/ProfileData/InstrProfWriter.cpp =================================================================== --- llvm/lib/ProfileData/InstrProfWriter.cpp +++ llvm/lib/ProfileData/InstrProfWriter.cpp @@ -131,6 +131,8 @@ M += sizeof(uint64_t); // The function hash M += sizeof(uint64_t); // The size of the Counts vector M += ProfRecord.Counts.size() * sizeof(uint64_t); + M += sizeof(uint64_t); // The size of the Bitmap vector + M += ProfRecord.BitmapBytes.size() * sizeof(uint64_t); // Value data M += ValueProfData::getSize(ProfileData.second); @@ -160,6 +162,10 @@ for (uint64_t I : ProfRecord.Counts) LE.write(I); + LE.write(ProfRecord.BitmapBytes.size()); + for (uint64_t I : ProfRecord.BitmapBytes) + LE.write(I); + // Write value data std::unique_ptr VDataPtr = ValueProfData::serializeFrom(ProfileData.second); @@ -380,6 +386,8 @@ const InstrProfRecord &IPR = Func.second; if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; })) return true; + if (llvm::any_of(IPR.BitmapBytes, [](uint8_t Byte) { return Byte > 0; })) + return true; } return false; } @@ -695,6 +703,17 @@ for (uint64_t Count : Func.Counts) OS << Count << "\n"; + if (Func.BitmapBytes.size() > 0) { + OS << "# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() << "\n"; + OS << "# Bitmap Byte Values:\n"; + for (uint8_t Byte : Func.BitmapBytes) { + OS << "0x"; + OS.write_hex(Byte); + OS << "\n"; + } + OS << "\n"; + } + uint32_t NumValueKinds = Func.getNumValueKinds(); if (!NumValueKinds) { OS << "\n"; Index: llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp =================================================================== --- llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -430,6 +430,15 @@ } else if (auto *IPVP = dyn_cast(&Instr)) { lowerValueProfileInst(IPVP); MadeChange = true; + } else if (auto *IPMP = dyn_cast(&Instr)) { + IPMP->eraseFromParent(); + MadeChange = true; + } else if (auto *IPBU = dyn_cast(&Instr)) { + lowerMCDCTestVectorBitmapUpdate(IPBU); + MadeChange = true; + } else if (auto *IPTU = dyn_cast(&Instr)) { + lowerMCDCCondBitmapUpdate(IPTU); + MadeChange = true; } } } @@ -544,19 +553,33 @@ // the instrumented function. This is counting the number of instrumented // target value sites to enter it as field in the profile data variable. for (Function &F : M) { - InstrProfInstBase *FirstProfInst = nullptr; - for (BasicBlock &BB : F) - for (auto I = BB.begin(), E = BB.end(); I != E; I++) + InstrProfCntrInstBase *FirstProfInst = nullptr; + InstrProfMCDCBitmapParameters *FirstProfMCDCParams = nullptr; + for (BasicBlock &BB : F) { + for (auto I = BB.begin(), E = BB.end(); I != E; I++) { if (auto *Ind = dyn_cast(I)) computeNumValueSiteCounts(Ind); - else if (FirstProfInst == nullptr && - (isa(I) || isa(I))) - FirstProfInst = dyn_cast(I); + else { + if (FirstProfInst == nullptr && + (isa(I) || isa(I))) + FirstProfInst = dyn_cast(I); + if (FirstProfMCDCParams == nullptr) + FirstProfMCDCParams = dyn_cast(I); + } + } + } + + // If the MCDCBitmapParameters intrinsic was seen, create the bitmaps. + if (FirstProfMCDCParams != nullptr) { + static_cast(getOrCreateRegionBitmaps(FirstProfMCDCParams)); + } - // Value profiling intrinsic lowering requires per-function profile data - // variable to be created first. - if (FirstProfInst != nullptr) + // Use a profile intrinsic was seen to create the region counters and data + // variable. Also create the data variable based on the MCDCParams. + if (FirstProfInst != nullptr) { static_cast(getOrCreateRegionCounters(FirstProfInst)); + createDataVariable(FirstProfInst, FirstProfMCDCParams); + } } for (Function &F : M) @@ -670,7 +693,7 @@ Ind->eraseFromParent(); } -Value *InstrProfiling::getCounterAddress(InstrProfInstBase *I) { +Value *InstrProfiling::getCounterAddress(InstrProfCntrInstBase *I) { auto *Counters = getOrCreateRegionCounters(I); IRBuilder<> Builder(I); @@ -710,6 +733,19 @@ return Builder.CreateIntToPtr(Add, Addr->getType()); } +Value *InstrProfiling::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) { + auto *Bitmaps = getOrCreateRegionBitmaps(I); + IRBuilder<> Builder(I); + + auto *Addr = + Builder.CreateConstInBoundsGEP2_32(Bitmaps->getValueType(), Bitmaps, 0, + I->getBitmapIndex()->getZExtValue()); + + // TODO: May need to support runtime counter relocation for bitmaps prior to + // upstreaming. See getCounterAddress(). + return Addr; +} + void InstrProfiling::lowerCover(InstrProfCoverInst *CoverInstruction) { auto *Addr = getCounterAddress(CoverInstruction); IRBuilder<> Builder(CoverInstruction); @@ -769,6 +805,86 @@ CoverageNamesVar->eraseFromParent(); } +void InstrProfiling::lowerMCDCTestVectorBitmapUpdate( + InstrProfMCDCTVBitmapUpdate *Update) { + IRBuilder<> Builder(Update); + auto *Int8Ty = Type::getInt8Ty(M->getContext()); + auto *Int8PtrTy = Type::getInt8PtrTy(M->getContext()); + auto *Int32Ty = Type::getInt32Ty(M->getContext()); + auto *Int64Ty = Type::getInt64Ty(M->getContext()); + auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr(); + auto *BitmapAddr = getBitmapAddress(Update); + + // Load Temp Val. + // %mcdc.temp = load i32, ptr %mcdc.addr, align 4 + auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp"); + + // Calculate byte offset using div8. + // %1 = lshr i32 %mcdc.temp, 3 + auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3); + + // Add byte offset to section base byte address. + // %2 = zext i32 %1 to i64 + // %3 = add i64 ptrtoint (ptr @__profbm_test to i64), %2 + auto *BitmapByteAddr = Builder.CreateAdd( + Builder.CreatePtrToInt(BitmapAddr, Int64Ty), + Builder.CreateZExtOrBitCast(BitmapByteOffset, Int64Ty)); + + // Convert to a pointer. + // %4 = inttoptr i32 %3 to ptr + BitmapByteAddr = Builder.CreateIntToPtr(BitmapByteAddr, Int8PtrTy); + + // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8) + // %5 = and i32 %mcdc.temp, 7 + // %6 = trunc i32 %5 to i8 + auto *BitToSet = Builder.CreateTrunc(Builder.CreateAnd(Temp, 0x7), Int8Ty); + + // Shift bit offset left to form a bitmap. + // %7 = shl i8 1, %6 + auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet); + + // Load profile bitmap byte. + // %mcdc.bits = load i8, ptr %4, align 1 + auto *Bitmap = Builder.CreateLoad(Int8Ty, BitmapByteAddr, "mcdc.bits"); + + // Perform logical OR of profile bitmap byte and shifted bit offset. + // %8 = or i8 %mcdc.bits, %7 + auto *Result = Builder.CreateOr(Bitmap, ShiftedVal); + + // Store the updated profile bitmap byte. + // store i8 %8, ptr %3, align 1 + Builder.CreateStore(Result, BitmapByteAddr); + Update->eraseFromParent(); +} + +void InstrProfiling::lowerMCDCCondBitmapUpdate( + InstrProfMCDCCondBitmapUpdate *Update) { + IRBuilder<> Builder(Update); + auto *Int32Ty = Type::getInt32Ty(M->getContext()); + auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr(); + + // Load the MCDC temporary value from the stack. + // %mcdc.temp = load i32, ptr %mcdc.addr, align 4 + auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp"); + + // Zero-extend the evaluated condition boolean value (0 or 1) by 32bits. + // %1 = zext i1 %tobool to i32 + auto *CondV_32 = Builder.CreateZExt(Update->getCondBool(), Int32Ty); + + // Shift the boolean value left (by the condition's ID) to form a bitmap. + // %2 = shl i32 %1, getCondID()> + auto *ShiftedVal = Builder.CreateShl(CondV_32, Update->getCondID()); + + // Perform logical OR of the bitmap against the loaded MCDC temporary value. + // %3 = or i32 %mcdc.temp, %2 + auto *Result = Builder.CreateOr(Temp, ShiftedVal); + + // Store the updated temporary value back to the stack. + // store i32 %3, ptr %mcdc.addr, align 4 + Builder.CreateStore(Result, MCDCCondBitmapAddr); + Update->eraseFromParent(); +} + /// Get the name of a profiling variable for a particular function. static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed) { @@ -924,37 +1040,30 @@ return true; } -GlobalVariable * -InstrProfiling::createRegionCounters(InstrProfInstBase *Inc, StringRef Name, - GlobalValue::LinkageTypes Linkage) { - uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); - auto &Ctx = M->getContext(); - GlobalVariable *GV; - if (isa(Inc)) { - auto *CounterTy = Type::getInt8Ty(Ctx); - auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters); - // TODO: `Constant::getAllOnesValue()` does not yet accept an array type. - std::vector InitialValues(NumCounters, - Constant::getAllOnesValue(CounterTy)); - GV = new GlobalVariable(*M, CounterArrTy, false, Linkage, - ConstantArray::get(CounterArrTy, InitialValues), - Name); - GV->setAlignment(Align(1)); - } else { - auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); - GV = new GlobalVariable(*M, CounterTy, false, Linkage, - Constant::getNullValue(CounterTy), Name); - GV->setAlignment(Align(8)); +void InstrProfiling::maybeSetComdat(GlobalVariable *GV, Function *Fn, + StringRef VarName) { + bool DataReferencedByCode = profDataReferencedByCode(*M); + bool NeedComdat = needsComdatForCounter(*Fn, *M); + bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); + if (UseComdat) { + StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode + ? GV->getName() + : VarName; + Comdat *C = M->getOrInsertComdat(GroupName); + if (!NeedComdat) + C->setSelectionKind(Comdat::NoDeduplicate); + GV->setComdat(C); + // COFF doesn't allow the comdat group leader to have private linkage, so + // upgrade private linkage to internal linkage to produce a symbol table + // entry. + if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage()) + GV->setLinkage(GlobalValue::InternalLinkage); } - return GV; } -GlobalVariable * -InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) { +GlobalVariable *InstrProfiling::setupProfileSection(InstrProfInstBase *Inc, + InstrProfSectKind IPSK) { GlobalVariable *NamePtr = Inc->getName(); - auto &PD = ProfileDataMap[NamePtr]; - if (PD.RegionCounters) - return PD.RegionCounters; // Match the linkage and visibility of the name global. Function *Fn = Inc->getParent()->getParent(); @@ -964,9 +1073,9 @@ // Use internal rather than private linkage so the counter variable shows up // in the symbol table when using debug info for correlation. if (DebugInfoCorrelate && TT.isOSBinFormatMachO() && - Linkage == GlobalValue::PrivateLinkage) + Linkage == GlobalValue::PrivateLinkage) { Linkage = GlobalValue::InternalLinkage; - + } // Due to the limitation of binder as of 2021/09/28, the duplicate weak // symbols in the same csect won't be discarded. When there are duplicate weak // symbols, we can NOT guarantee that the relocations get resolved to the @@ -993,42 +1102,96 @@ // nodeduplicate COMDAT which is lowered to a zero-flag section group. This // allows -z start-stop-gc to discard the entire group when the function is // discarded. - bool DataReferencedByCode = profDataReferencedByCode(*M); - bool NeedComdat = needsComdatForCounter(*Fn, *M); bool Renamed; - std::string CntsVarName = - getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed); - std::string DataVarName = - getVarName(Inc, getInstrProfDataVarPrefix(), Renamed); - auto MaybeSetComdat = [&](GlobalVariable *GV) { - bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); - if (UseComdat) { - StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode - ? GV->getName() - : CntsVarName; - Comdat *C = M->getOrInsertComdat(GroupName); - if (!NeedComdat) - C->setSelectionKind(Comdat::NoDeduplicate); - GV->setComdat(C); - // COFF doesn't allow the comdat group leader to have private linkage, so - // upgrade private linkage to internal linkage to produce a symbol table - // entry. - if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage()) - GV->setLinkage(GlobalValue::InternalLinkage); - } - }; + GlobalVariable *Ptr; + StringRef VarPrefix; + std::string VarName; + if (IPSK == IPSK_cnts) { + VarPrefix = getInstrProfCountersVarPrefix(); + VarName = getVarName(Inc, VarPrefix, Renamed); + InstrProfCntrInstBase *CntrIncrement = dyn_cast(Inc); + Ptr = createRegionCounters(CntrIncrement, VarName, Linkage); + } else if (IPSK == IPSK_bitmap) { + VarPrefix = getInstrProfBitmapVarPrefix(); + VarName = getVarName(Inc, VarPrefix, Renamed); + InstrProfMCDCBitmapInstBase *BitmapUpdate = + dyn_cast(Inc); + Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage); + } else { + llvm_unreachable("Profile Section must be for Counters or Bitmaps"); + } + + Ptr->setVisibility(Visibility); + // Put the counters and bitmaps in their own sections so linkers can + // remove unneeded sections. + Ptr->setSection(getInstrProfSectionName(IPSK, TT.getObjectFormat())); + Ptr->setLinkage(Linkage); + maybeSetComdat(Ptr, Fn, VarName); + return Ptr; +} + +GlobalVariable * +InstrProfiling::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, + StringRef Name, + GlobalValue::LinkageTypes Linkage) { + uint64_t NumBytes = Inc->getNumBitmapBytes()->getZExtValue(); + auto *BitmapTy = ArrayType::get(Type::getInt8Ty(M->getContext()), NumBytes); + auto GV = new GlobalVariable(*M, BitmapTy, false, Linkage, + Constant::getNullValue(BitmapTy), Name); + GV->setAlignment(Align(1)); + return GV; +} + +GlobalVariable * +InstrProfiling::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) { + GlobalVariable *NamePtr = Inc->getName(); + auto &PD = ProfileDataMap[NamePtr]; + if (PD.RegionBitmaps) + return PD.RegionBitmaps; + auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap); + PD.RegionBitmaps = BitmapPtr; + return PD.RegionBitmaps; +} + +GlobalVariable * +InstrProfiling::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name, + GlobalValue::LinkageTypes Linkage) { uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); - LLVMContext &Ctx = M->getContext(); + auto &Ctx = M->getContext(); + GlobalVariable *GV; + if (isa(Inc)) { + auto *CounterTy = Type::getInt8Ty(Ctx); + auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters); + // TODO: `Constant::getAllOnesValue()` does not yet accept an array type. + std::vector InitialValues(NumCounters, + Constant::getAllOnesValue(CounterTy)); + GV = new GlobalVariable(*M, CounterArrTy, false, Linkage, + ConstantArray::get(CounterArrTy, InitialValues), + Name); + GV->setAlignment(Align(1)); + } else { + auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); + GV = new GlobalVariable(*M, CounterTy, false, Linkage, + Constant::getNullValue(CounterTy), Name); + GV->setAlignment(Align(8)); + } + return GV; +} - auto *CounterPtr = createRegionCounters(Inc, CntsVarName, Linkage); - CounterPtr->setVisibility(Visibility); - CounterPtr->setSection( - getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat())); - CounterPtr->setLinkage(Linkage); - MaybeSetComdat(CounterPtr); +GlobalVariable * +InstrProfiling::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) { + GlobalVariable *NamePtr = Inc->getName(); + auto &PD = ProfileDataMap[NamePtr]; + if (PD.RegionCounters) + return PD.RegionCounters; + + auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts); PD.RegionCounters = CounterPtr; + if (DebugInfoCorrelate) { + LLVMContext &Ctx = M->getContext(); + Function *Fn = Inc->getParent()->getParent(); if (auto *SP = Fn->getSubprogram()) { DIBuilder DB(*M, true, SP->getUnit()); Metadata *FunctionNameAnnotation[] = { @@ -1063,7 +1226,53 @@ Ctx.diagnose( DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning)); } + + // Mark the counter variable as used so that it isn't optimized out. + CompilerUsedVars.push_back(PD.RegionCounters); + } + + return PD.RegionCounters; +} + +void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc, + InstrProfMCDCBitmapParameters *Params) { + if (DebugInfoCorrelate) + return; + + GlobalVariable *NamePtr = Inc->getName(); + auto &PD = ProfileDataMap[NamePtr]; + + LLVMContext &Ctx = M->getContext(); + + Function *Fn = Inc->getParent()->getParent(); + GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage(); + GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility(); + + // Use internal rather than private linkage so the counter variable shows up + // in the symbol table when using debug info for correlation. + if (DebugInfoCorrelate && TT.isOSBinFormatMachO() && + Linkage == GlobalValue::PrivateLinkage) { + Linkage = GlobalValue::InternalLinkage; } + // Due to the limitation of binder as of 2021/09/28, the duplicate weak + // symbols in the same csect won't be discarded. When there are duplicate weak + // symbols, we can NOT guarantee that the relocations get resolved to the + // intended weak symbol, so we can not ensure the correctness of the relative + // CounterPtr, so we have to use private linkage for counter and data symbols. + if (TT.isOSBinFormatXCOFF()) { + Linkage = GlobalValue::PrivateLinkage; + Visibility = GlobalValue::DefaultVisibility; + } + + bool DataReferencedByCode = profDataReferencedByCode(*M); + bool NeedComdat = needsComdatForCounter(*Fn, *M); + bool Renamed; + + // The Data Variable section is anchored to profile counters. + std::string CntsVarName = + getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed); + std::string DataVarName = + getVarName(Inc, getInstrProfDataVarPrefix(), Renamed); auto *Int8PtrTy = Type::getInt8PtrTy(Ctx); // Allocate statically the array of pointers to value profile nodes for @@ -1082,16 +1291,17 @@ ValuesVar->setSection( getInstrProfSectionName(IPSK_vals, TT.getObjectFormat())); ValuesVar->setAlignment(Align(8)); - MaybeSetComdat(ValuesVar); + maybeSetComdat(ValuesVar, Fn, CntsVarName); ValuesPtrExpr = ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx)); } - if (DebugInfoCorrelate) { - // Mark the counter variable as used so that it isn't optimized out. - CompilerUsedVars.push_back(PD.RegionCounters); - return PD.RegionCounters; - } + uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); + auto *CounterPtr = PD.RegionCounters; + + uint64_t NumBitmapBytes = 0; + if (Params != nullptr) + NumBitmapBytes = Params->getNumBitmapBytes()->getZExtValue(); // Create data variable. auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext()); @@ -1134,6 +1344,16 @@ ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy), ConstantExpr::getPtrToInt(Data, IntPtrTy)); + // Bitmaps are relative to the same data variable as profile counters. + GlobalVariable *BitmapPtr = PD.RegionBitmaps; + Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0); + + if (BitmapPtr != nullptr) { + RelativeBitmapPtr = + ConstantExpr::getSub(ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy), + ConstantExpr::getPtrToInt(Data, IntPtrTy)); + } + Constant *DataVals[] = { #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init, #include "llvm/ProfileData/InstrProfData.inc" @@ -1143,7 +1363,7 @@ Data->setVisibility(Visibility); Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat())); Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT)); - MaybeSetComdat(Data); + maybeSetComdat(Data, Fn, CntsVarName); PD.DataVar = Data; @@ -1155,8 +1375,6 @@ NamePtr->setLinkage(GlobalValue::PrivateLinkage); // Collect the referenced names to be used by emitNameData. ReferencedNames.push_back(NamePtr); - - return PD.RegionCounters; } void InstrProfiling::emitVNodes() { Index: llvm/test/Instrumentation/InstrProfiling/mcdc.ll =================================================================== --- /dev/null +++ llvm/test/Instrumentation/InstrProfiling/mcdc.ll @@ -0,0 +1,50 @@ +; Check that MC/DC intrinsics are properly lowered +; RUN: opt < %s -passes=instrprof -S | FileCheck %s + +target triple = "x86_64-unknown-linux-gnu" + +@__profn_test = private constant [4 x i8] c"test" + +; CHECK: @__profbm_test = private global [1 x i8] zeroinitializer, section "__llvm_prf_bits", comdat, align 1 + +define dso_local void @test(i32 noundef %A) { +entry: + %A.addr = alloca i32, align 4 + %mcdc.addr = alloca i32, align 4 + call void @llvm.instrprof.cover(ptr @__profn_test, i64 99278, i32 5, i32 0) + ; CHECK: store i8 0, ptr @__profc_test, align 1 + + call void @llvm.instrprof.mcdc.parameters(ptr @__profn_test, i64 99278, i32 1) + store i32 0, ptr %mcdc.addr, align 4 + %0 = load i32, ptr %A.addr, align 4 + %tobool = icmp ne i32 %0, 0 + + call void @llvm.instrprof.mcdc.condbitmap.update(ptr @__profn_test, i64 99278, i32 0, ptr %mcdc.addr, i1 %tobool) + ; CHECK: %mcdc.temp = load i32, ptr %mcdc.addr, align 4 + ; CHECK-NEXT: %1 = zext i1 %tobool to i32 + ; CHECK-NEXT: %2 = shl i32 %1, 0 + ; CHECK-NEXT: %3 = or i32 %mcdc.temp, %2 + ; CHECK-NEXT: store i32 %3, ptr %mcdc.addr, align 4 + + call void @llvm.instrprof.mcdc.tvbitmap.update(ptr @__profn_test, i64 99278, i32 1, i32 0, ptr %mcdc.addr) + ; CHECK: %mcdc.temp1 = load i32, ptr %mcdc.addr, align 4 + ; CHECK-NEXT: %4 = lshr i32 %mcdc.temp1, 3 + ; CHECK-NEXT: %5 = zext i32 %4 to i64 + ; CHECK-NEXT: %6 = add i64 ptrtoint (ptr @__profbm_test to i64), %5 + ; CHECK-NEXT: %7 = inttoptr i64 %6 to ptr + ; CHECK-NEXT: %8 = and i32 %mcdc.temp1, 7 + ; CHECK-NEXT: %9 = trunc i32 %8 to i8 + ; CHECK-NEXT: %10 = shl i8 1, %9 + ; CHECK-NEXT: %mcdc.bits = load i8, ptr %7, align 1 + ; CHECK-NEXT: %11 = or i8 %mcdc.bits, %10 + ; CHECK-NEXT: store i8 %11, ptr %7, align 1 + ret void +} + +declare void @llvm.instrprof.cover(ptr, i64, i32, i32) + +declare void @llvm.instrprof.mcdc.parameters(ptr, i64, i32) + +declare void @llvm.instrprof.mcdc.condbitmap.update(ptr, i64, i32, ptr, i1) + +declare void @llvm.instrprof.mcdc.tvbitmap.update(ptr, i64, i32, i32, ptr) Index: llvm/test/Transforms/PGOProfile/comdat_internal.ll =================================================================== --- llvm/test/Transforms/PGOProfile/comdat_internal.ll +++ llvm/test/Transforms/PGOProfile/comdat_internal.ll @@ -13,9 +13,9 @@ ; CHECK: @__llvm_profile_raw_version = hidden constant i64 {{[0-9]+}}, comdat ; CHECK-NOT: __profn__stdin__foo ; CHECK: @__profc__stdin__foo.[[#FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8 -; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, ptr, ptr, i32, [2 x i16] } { i64 -5640069336071256030, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), ptr null +; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 -5640069336071256030, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), i64 0, ptr null ; CHECK-NOT: @foo -; CHECK-SAME: , ptr null, i32 1, [2 x i16] zeroinitializer }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8 +; CHECK-SAME: , ptr null, i32 1, [2 x i16] zeroinitializer, i32 0 }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8 ; CHECK: @__llvm_prf_nm ; CHECK: @llvm.compiler.used Index: llvm/test/tools/llvm-profdata/Inputs/mcdc-1.proftext =================================================================== --- /dev/null +++ llvm/test/tools/llvm-profdata/Inputs/mcdc-1.proftext @@ -0,0 +1,16 @@ +main +# Func Hash: +702755447896 +# Num Counters: +4 +# Counter Values: +1 +0 +1 +0 +# Num Bitmask Bytes: +$1 +# Bitmask Byte Values: +2 + + Index: llvm/test/tools/llvm-profdata/Inputs/mcdc-2.proftext =================================================================== --- /dev/null +++ llvm/test/tools/llvm-profdata/Inputs/mcdc-2.proftext @@ -0,0 +1,35 @@ +main +# Func Hash: +702755447896 +# Num Counters: +4 +# Counter Values: +1 +1 +1 +1 +# Num Bitmask Bytes: +$1 +# Bitmask Byte Values: +8 + + +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$0x2 +# Bitmask Byte Values: +0x29 +0x0 + + Index: llvm/test/tools/llvm-profdata/Inputs/mcdc-3.proftext =================================================================== --- /dev/null +++ llvm/test/tools/llvm-profdata/Inputs/mcdc-3.proftext @@ -0,0 +1,23 @@ +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$8 +# Bitmask Byte Values: +0x0 +0x2 +0x3 +0xf +0xf +0xa +0xc +0x2 Index: llvm/test/tools/llvm-profdata/Inputs/mcdc-4.proftext =================================================================== --- /dev/null +++ llvm/test/tools/llvm-profdata/Inputs/mcdc-4.proftext @@ -0,0 +1,23 @@ +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$ 8 +# Bitmask Byte Values: +1 +2 +3 +4 +5 +6 +7 +8 Index: llvm/test/tools/llvm-profdata/Inputs/mcdc-err0.proftext =================================================================== --- /dev/null +++ llvm/test/tools/llvm-profdata/Inputs/mcdc-err0.proftext @@ -0,0 +1,23 @@ +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$8.9 +# Bitmask Byte Values: +1 +2 +3 +4 +5 +6 +7 +8 Index: llvm/test/tools/llvm-profdata/Inputs/mcdc-err1.proftext =================================================================== --- /dev/null +++ llvm/test/tools/llvm-profdata/Inputs/mcdc-err1.proftext @@ -0,0 +1,23 @@ +test3 +# Func Hash: +15288018065 +# Num Counters: +6 +# Counter Values: +4 +2 +1 +0 +0 +2 +# Num Bitmask Bytes: +$8 +# Bitmask Byte Values: +1 +2 +3 +4 +5.4 +6 +7 +8 Index: llvm/test/tools/llvm-profdata/binary-ids-padding.test =================================================================== --- llvm/test/tools/llvm-profdata/binary-ids-padding.test +++ llvm/test/tools/llvm-profdata/binary-ids-padding.test @@ -5,13 +5,15 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw // There will be 2 20-byte binary IDs, so the total Binary IDs size will be 64 bytes. // 2 * 8 binary ID sizes // + 2 * 20 binary IDs (of size 20) @@ -23,8 +25,11 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\3\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\20\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -51,14 +56,18 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\067\265\035\031\112\165\023\344' >> %t.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw -RUN: printf '\xd8\xff\3\0\1\0\0\0' >> %t.profraw +RUN: printf '\xc8\xff\3\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\067\0\0\0\0\0\0\0' >> %t.profraw Index: llvm/test/tools/llvm-profdata/large-binary-id-size.test =================================================================== --- llvm/test/tools/llvm-profdata/large-binary-id-size.test +++ llvm/test/tools/llvm-profdata/large-binary-id-size.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\40\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -9,6 +9,9 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Check for a corrupted size being too large past the end of the file. RUN: printf '\7\7\7\7\7\7\7\7' >> %t.profraw Index: llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test =================================================================== --- llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test +++ llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test @@ -5,20 +5,25 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -35,7 +40,9 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\3\0foo\0\0\0' >> %t.profraw Index: llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test =================================================================== --- llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test +++ llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test @@ -5,20 +5,26 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -35,8 +41,10 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Make NumCounters = 0 so that we get "number of counters is zero" error message RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\3\0foo\0\0\0' >> %t.profraw Index: llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test =================================================================== --- llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test +++ llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test @@ -5,20 +5,25 @@ // INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) // INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumBitmaskBytes, NumBitmaskBytes) // INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\2\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\6\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\6\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -38,10 +43,12 @@ // Octal '\11' is 9 in decimal: this should push CounterOffset to 1. As there are two counters, // the profile reader should error out. RUN: printf '\11\0\6\0\1\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Counter Section Index: llvm/test/tools/llvm-profdata/mcdc-bitmap.test =================================================================== --- /dev/null +++ llvm/test/tools/llvm-profdata/mcdc-bitmap.test @@ -0,0 +1,54 @@ +# Test MC/DC bitmap reading and merging. + +# Merge as profdata. +RUN: llvm-profdata merge %p/Inputs/mcdc-1.proftext %p/Inputs/mcdc-2.proftext -o %t.profdata +RUN: llvm-profdata show %t.profdata --text -all-functions | FileCheck %s --check-prefix=MCDC +# Merge as proftext. +RUN: llvm-profdata merge %p/Inputs/mcdc-1.proftext %p/Inputs/mcdc-2.proftext -o %t.proftext +RUN: llvm-profdata show %t.proftext --text -all-functions | FileCheck %s --check-prefix=MCDC + +MCDC: # Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: # Bitmap Byte Values: +MCDC-NEXT: a +MCDC: # Num Bitmap Bytes: +MCDC-NEXT: $2 +MCDC-NEXT: # Bitmap Byte Values: +MCDC-NEXT: 0x29 +MCDC-NEXT: 0x0 + +# Merge as profdata. +RUN: llvm-profdata merge %p/Inputs/mcdc-3.proftext %p/Inputs/mcdc-4.proftext -o %t.profdata +RUN: llvm-profdata show %t.profdata --text -all-functions | FileCheck %s --check-prefix=MCDC2 +# Merge as proftext. +RUN: llvm-profdata merge %p/Inputs/mcdc-3.proftext %p/Inputs/mcdc-4.proftext -o %t.proftext +RUN: llvm-profdata show %t.proftext --text -all-functions | FileCheck %s --check-prefix=MCDC2 + +MCDC2: # Num Bitmap Bytes: +MCDC2-NEXT: $8 +MCDC2-NEXT: # Bitmap Byte Values: +MCDC2-NEXT: 0x1 +MCDC2-NEXT: 0x2 +MCDC2-NEXT: 0x3 +MCDC2-NEXT: 0xf +MCDC2-NEXT: 0xf +MCDC2-NEXT: 0xe +MCDC2-NEXT: 0xf +MCDC2-NEXT: 0xa + +# Incompatible size mismatch. +RUN: llvm-profdata merge %p/Inputs/mcdc-2.proftext %p/Inputs/mcdc-4.proftext -o %t.profdata 2>&1 | FileCheck %s --check-prefix=MCDC3 +# Merge as proftext +RUN: llvm-profdata merge %p/Inputs/mcdc-2.proftext %p/Inputs/mcdc-4.proftext -o %t.proftext 2>&1 | FileCheck %s --check-prefix=MCDC3 + +MCDC3: function bitmap size change detected (bitmap size mismatch) + +# Invalid number of bitmap bytes. +RUN: not llvm-profdata merge %p/Inputs/mcdc-3.proftext %p/Inputs/mcdc-err0.proftext -o %t.proftext 2>&1 | FileCheck %s --check-prefix=MCDC4 + +MCDC4: malformed instrumentation profile data: number of bitmap bytes is not a valid integer + +# Invalid bitmap byte. +RUN: not llvm-profdata merge %p/Inputs/mcdc-3.proftext %p/Inputs/mcdc-err1.proftext -o %t.proftext 2>&1 | FileCheck %s --check-prefix=MCDC5 + +MCDC5: malformed instrumentation profile data: bitmap byte is invalid Index: llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test =================================================================== --- llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test +++ llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw // We should fail on this because the binary IDs is not a multiple of 8 bytes. RUN: printf '\77\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw Index: llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test =================================================================== --- llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test +++ llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test @@ -8,6 +8,9 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\20' >> %t RUN: printf '\0\0\0\1\0\4\0\0' >> %t RUN: printf '\0\0\0\2\0\4\0\0' >> %t Index: llvm/test/tools/llvm-profdata/raw-32-bits-be.test =================================================================== --- llvm/test/tools/llvm-profdata/raw-32-bits-be.test +++ llvm/test/tools/llvm-profdata/raw-32-bits-be.test @@ -1,37 +1,46 @@ RUN: printf '\377lprofR\201' > %t -RUN: printf '\0\0\0\0\0\0\0\10' >> %t +RUN: printf '\0\0\0\0\0\0\0\11' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\4' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\20' >> %t RUN: printf '\0\0\0\0\1\0\0\0' >> %t +RUN: printf '\0\0\0\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\2\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\134\370\302\114\333\030\275\254' >> %t RUN: printf '\0\0\0\0\0\0\0\1' >> %t RUN: printf '\1\0\0\0' >> %t +RUN: printf '\3\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\1' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\3' >> %t +RUN: printf '\0\0\0\0' >> %t RUN: printf '\344\023\165\112\031\035\265\067' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t -RUN: printf '\0\xff\xff\xe0' >> %t +RUN: printf '\0\xff\xff\xd8' >> %t +RUN: printf '\2\xff\xff\xd3' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\2' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\1' >> %t +RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\023' >> %t RUN: printf '\0\0\0\0\0\0\0\067' >> %t RUN: printf '\0\0\0\0\0\0\0\101' >> %t +RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s +RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -48,3 +57,14 @@ CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 + +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $3 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 0x2a Index: llvm/test/tools/llvm-profdata/raw-32-bits-le.test =================================================================== --- llvm/test/tools/llvm-profdata/raw-32-bits-le.test +++ llvm/test/tools/llvm-profdata/raw-32-bits-le.test @@ -1,37 +1,46 @@ RUN: printf '\201Rforpl\377' > %t -RUN: printf '\10\0\0\0\0\0\0\0' >> %t +RUN: printf '\11\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\2\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\3\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\4\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\20\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1\0\0\0\0' >> %t +RUN: printf '\0\0\0\3\0\0\0\0' >> %t RUN: printf '\0\0\0\2\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\254\275\030\333\114\302\370\134' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1' >> %t +RUN: printf '\0\0\0\3' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\1\0\0\0' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\3\0\0\0' >> %t +RUN: printf '\0\0\0\0' >> %t RUN: printf '\067\265\035\031\112\165\023\344' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t -RUN: printf '\xe0\xff\xff\0' >> %t +RUN: printf '\xd8\xff\xff\0' >> %t +RUN: printf '\xd3\xff\xff\2' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\2\0\0\0' >> %t -RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\1\0\0\0' >> %t +RUN: printf '\0\0\0\0' >> %t RUN: printf '\023\0\0\0\0\0\0\0' >> %t RUN: printf '\067\0\0\0\0\0\0\0' >> %t RUN: printf '\101\0\0\0\0\0\0\0' >> %t +RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s +RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -48,3 +57,14 @@ CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 + +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $3 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 0x2a Index: llvm/test/tools/llvm-profdata/raw-64-bits-be.test =================================================================== --- llvm/test/tools/llvm-profdata/raw-64-bits-be.test +++ llvm/test/tools/llvm-profdata/raw-64-bits-be.test @@ -1,35 +1,44 @@ RUN: printf '\377lprofr\201' > %t -RUN: printf '\0\0\0\0\0\0\0\10' >> %t +RUN: printf '\0\0\0\0\0\0\0\11' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\4' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\20' >> %t RUN: printf '\0\0\0\1\0\4\0\0' >> %t +RUN: printf '\0\0\0\3\0\4\0\0' >> %t RUN: printf '\0\0\0\2\0\4\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\134\370\302\114\333\030\275\254' >> %t RUN: printf '\0\0\0\0\0\0\0\1' >> %t RUN: printf '\0\0\0\1\0\4\0\0' >> %t +RUN: printf '\0\0\0\3\0\4\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1\0\0\0\0' >> %t +RUN: printf '\0\0\0\3\0\0\0\0' >> %t RUN: printf '\344\023\165\112\031\035\265\067' >> %t RUN: printf '\0\0\0\0\0\0\0\02' >> %t -RUN: printf '\0\0\0\1\0\3\xff\xd8' >> %t +RUN: printf '\0\0\0\1\0\3\xff\xc8' >> %t +RUN: printf '\0\0\0\3\0\3\xff\xc3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\02\0\0\0\0' >> %t +RUN: printf '\0\0\0\1\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\023' >> %t RUN: printf '\0\0\0\0\0\0\0\067' >> %t RUN: printf '\0\0\0\0\0\0\0\101' >> %t +RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s +RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -46,3 +55,14 @@ CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 + +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $3 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 0x2a Index: llvm/test/tools/llvm-profdata/raw-64-bits-le.test =================================================================== --- llvm/test/tools/llvm-profdata/raw-64-bits-le.test +++ llvm/test/tools/llvm-profdata/raw-64-bits-le.test @@ -1,35 +1,44 @@ RUN: printf '\201rforpl\377' > %t -RUN: printf '\10\0\0\0\0\0\0\0' >> %t +RUN: printf '\11\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\2\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\3\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\4\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\20\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\4\0\1\0\0\0' >> %t +RUN: printf '\0\0\4\0\3\0\0\0' >> %t RUN: printf '\0\0\4\0\2\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\254\275\030\333\114\302\370\134' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\4\0\1\0\0\0' >> %t +RUN: printf '\0\0\4\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t +RUN: printf '\3\0\0\0\0\0\0\0' >> %t RUN: printf '\067\265\035\031\112\165\023\344' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t -RUN: printf '\xd8\xff\3\0\1\0\0\0' >> %t +RUN: printf '\xc8\xff\3\0\1\0\0\0' >> %t +RUN: printf '\xc3\xff\3\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t +RUN: printf '\1\0\0\0\0\0\0\0' >> %t RUN: printf '\023\0\0\0\0\0\0\0' >> %t RUN: printf '\067\0\0\0\0\0\0\0' >> %t RUN: printf '\101\0\0\0\0\0\0\0' >> %t +RUN: printf '\125\125\125\052' >> %t RUN: printf '\7\0foo\1bar\0\0\0\0\0\0\0' >> %t RUN: llvm-profdata show %t -all-functions -counts | FileCheck %s +RUN: llvm-profdata show %t -all-functions -text | FileCheck %s -check-prefix=MCDC CHECK: Counters: CHECK: foo: @@ -46,3 +55,14 @@ CHECK: Total functions: 2 CHECK: Maximum function count: 55 CHECK: Maximum internal block count: 65 + +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $3 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC-NEXT: 55 +MCDC: Num Bitmap Bytes: +MCDC-NEXT: $1 +MCDC-NEXT: Bitmap Byte Values: +MCDC-NEXT: 0x2a Index: llvm/test/tools/llvm-profdata/raw-two-profiles.test =================================================================== --- llvm/test/tools/llvm-profdata/raw-two-profiles.test +++ llvm/test/tools/llvm-profdata/raw-two-profiles.test @@ -1,12 +1,15 @@ RUN: printf '\201rforpl\377' > %t-foo.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw @@ -15,20 +18,25 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\023\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\3\0foo\0\0\0' >> %t-foo.profraw RUN: printf '\201rforpl\377' > %t-bar.profraw -RUN: printf '\10\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\11\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\2\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\10\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\6\0\2\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw @@ -37,7 +45,9 @@ RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\067\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\101\0\0\0\0\0\0\0' >> %t-bar.profraw