Index: lib/profile/InstrProfiling.h =================================================================== --- lib/profile/InstrProfiling.h +++ lib/profile/InstrProfiling.h @@ -78,15 +78,16 @@ /*! * \brief Prepares the value profiling data for output. * - * Prepares a single __llvm_profile_value_data array out of the many - * ValueProfNode trees (one per instrumented function). + * Returns an array of pointers to value profile data. */ -uint64_t __llvm_profile_gather_value_data(uint8_t **DataArray); +struct ValueProfData; +struct ValueProfData **__llvm_profile_gather_value_data(uint64_t *Size); /*! * \brief Write instrumentation data to the current file. * - * Writes to the file with the last name given to \a __llvm_profile_set_filename(), + * Writes to the file with the last name given to \a * + * __llvm_profile_set_filename(), * or if it hasn't been called, the \c LLVM_PROFILE_FILE environment variable, * or if that's not set, the last name given to * \a __llvm_profile_override_default_filename(), or if that's not set, Index: lib/profile/InstrProfilingFile.c =================================================================== --- lib/profile/InstrProfilingFile.c +++ lib/profile/InstrProfilingFile.c @@ -31,12 +31,16 @@ } static int writeFile(FILE *File) { - uint8_t *ValueDataBegin = NULL; - const uint64_t ValueDataSize = - __llvm_profile_gather_value_data(&ValueDataBegin); - int r = llvmWriteProfData(fileWriter, File, ValueDataBegin, ValueDataSize); - free(ValueDataBegin); - return r; + const char *BufferSzStr = 0; + uint64_t ValueDataSize = 0; + struct ValueProfData **ValueDataArray = + __llvm_profile_gather_value_data(&ValueDataSize); + FreeHook = &free; + CallocHook = &calloc; + BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE"); + if (BufferSzStr && BufferSzStr[0]) + VPBufferSize = atoi(BufferSzStr); + return llvmWriteProfData(fileWriter, File, ValueDataArray, ValueDataSize); } static int writeFileWithName(const char *OutputName) { Index: lib/profile/InstrProfilingInternal.h =================================================================== --- lib/profile/InstrProfilingInternal.h +++ lib/profile/InstrProfilingInternal.h @@ -52,17 +52,20 @@ uint32_t llvmBufferWriter(ProfDataIOVec *IOVecs, uint32_t NumIOVecs, void **WriterCtx); int llvmWriteProfData(WriterCallback Writer, void *WriterCtx, - const uint8_t *ValueDataBegin, + struct ValueProfData **ValueDataArray, const uint64_t ValueDataSize); int llvmWriteProfDataImpl(WriterCallback Writer, void *WriterCtx, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin, const uint64_t *CountersEnd, - const uint8_t *ValueDataBegin, + struct ValueProfData **ValueDataBeginArray, const uint64_t ValueDataSize, const char *NamesBegin, const char *NamesEnd); extern char *(*GetEnvHook)(const char *); +extern void (*FreeHook)(void *); +extern void* (*CallocHook)(size_t, size_t); +extern uint32_t VPBufferSize; #endif Index: lib/profile/InstrProfilingValue.c =================================================================== --- lib/profile/InstrProfilingValue.c +++ lib/profile/InstrProfilingValue.c @@ -21,7 +21,8 @@ #define PROF_OOM_RETURN(Msg) \ { \ PROF_OOM(Msg) \ - return 0; \ + free(ValueDataArray); \ + return NULL; \ } #if COMPILER_RT_HAS_ATOMICS != 1 @@ -78,24 +79,6 @@ return 1; } -static void deallocateValueProfileCounters(__llvm_profile_data *Data) { - uint64_t NumVSites = 0, I; - uint32_t VKI; - if (!Data->Values) - return; - for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI) - NumVSites += Data->NumValueSites[VKI]; - for (I = 0; I < NumVSites; I++) { - ValueProfNode *Node = ((ValueProfNode **)Data->Values)[I]; - while (Node) { - ValueProfNode *Next = Node->Next; - free(Node); - Node = Next; - } - } - free(Data->Values); -} - COMPILER_RT_VISIBILITY void __llvm_profile_instrument_target(uint64_t TargetValue, void *Data, uint32_t CounterIndex) { @@ -147,92 +130,48 @@ } } -/* For multi-threaded programs, while the profile is being dumped, other - threads may still be updating the value profile data and creating new - value entries. To accommadate this, we need to add extra bytes to the - data buffer. The size of the extra space is controlled by an environment - variable. */ -static unsigned getVprofExtraBytes() { - const char *ExtraStr = - GetEnvHook ? GetEnvHook("LLVM_VALUE_PROF_BUFFER_EXTRA") : 0; - if (!ExtraStr || !ExtraStr[0]) - return 1024; - return (unsigned)atoi(ExtraStr); -} - -/* Extract the value profile data info from the runtime. */ -#define DEF_VALUE_RECORD(R, NS, V) \ - ValueProfRuntimeRecord R; \ - if (initializeValueProfRuntimeRecord(&R, NS, V)) \ - PROF_OOM_RETURN("Failed to write value profile data "); - -#define DTOR_VALUE_RECORD(R) finalizeValueProfRuntimeRecord(&R); - -COMPILER_RT_VISIBILITY uint64_t -__llvm_profile_gather_value_data(uint8_t **VDataArray) { - size_t S = 0, RealSize = 0, BufferCapacity = 0, Extra = 0; +COMPILER_RT_VISIBILITY ValueProfData ** +__llvm_profile_gather_value_data(uint64_t *ValueDataSize) { + size_t S = 0; __llvm_profile_data *I; - if (!VDataArray) - PROF_OOM_RETURN("Failed to write value profile data "); + ValueProfData **ValueDataArray; const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + ValueDataArray = + (ValueProfData **)calloc(DataEnd - DataBegin, sizeof(void *)); + if (!ValueDataArray) + PROF_OOM_RETURN("Failed to write value profile data "); + /* * Compute the total Size of the buffer to hold ValueProfData * structures for functions with value profile data. */ for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) { - - DEF_VALUE_RECORD(R, I->NumValueSites, I->Values); + ValueProfRuntimeRecord R; + if (initializeValueProfRuntimeRecord(&R, I->NumValueSites, I->Values)) + PROF_OOM_RETURN("Failed to write value profile data "); /* Compute the size of ValueProfData from this runtime record. */ - if (getNumValueKindsRT(&R) != 0) - S += getValueProfDataSizeRT(&R); - - DTOR_VALUE_RECORD(R); - } - /* No value sites or no value profile data is collected. */ - if (!S) - return 0; - - Extra = getVprofExtraBytes(); - BufferCapacity = S + Extra; - *VDataArray = calloc(BufferCapacity, sizeof(uint8_t)); - if (!*VDataArray) - PROF_OOM_RETURN("Failed to write value profile data "); - - ValueProfData *VD = (ValueProfData *)(*VDataArray); - /* - * Extract value profile data and write into ValueProfData structure - * one by one. Note that new value profile data added to any value - * site (from another thread) after the ValueProfRuntimeRecord is - * initialized (when the profile data snapshot is taken) won't be - * collected. This is not a problem as those dropped value will have - * very low taken count. - */ - for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) { - DEF_VALUE_RECORD(R, I->NumValueSites, I->Values); - if (getNumValueKindsRT(&R) == 0) - continue; - - /* Record R has taken a snapshot of the VP data at this point. Newly - added VP data for this function will be dropped. */ - /* Check if there is enough space. */ - if (BufferCapacity - RealSize < getValueProfDataSizeRT(&R)) { - PROF_ERR("Value profile data is dropped :%s \n", - "Out of buffer space. Use environment " - " LLVM_VALUE_PROF_BUFFER_EXTRA to allocate more"); - I->Values = 0; + if (getNumValueKindsRT(&R) != 0) { + ValueProfData *VD = NULL; + uint32_t VS = getValueProfDataSizeRT(&R); + VD = (ValueProfData *)calloc(VS, sizeof(uint8_t)); + if (!VD) + PROF_OOM_RETURN("Failed to write value profile data "); + serializeValueProfDataFromRT(&R, VD); + ValueDataArray[I - DataBegin] = VD; + S += VS; } + finalizeValueProfRuntimeRecord(&R); + } - serializeValueProfDataFromRT(&R, VD); - deallocateValueProfileCounters(I); - I->Values = VD; - RealSize += VD->TotalSize; - VD = (ValueProfData *)((char *)VD + VD->TotalSize); - DTOR_VALUE_RECORD(R); + if (!S) { + free(ValueDataArray); + ValueDataArray = NULL; } - return RealSize; + *ValueDataSize = S; + return ValueDataArray; } Index: lib/profile/InstrProfilingWriter.c =================================================================== --- lib/profile/InstrProfilingWriter.c +++ lib/profile/InstrProfilingWriter.c @@ -11,6 +11,12 @@ #include "InstrProfilingInternal.h" #include +#define INSTR_PROF_VALUE_PROF_DATA +#include "InstrProfData.inc" +void (*FreeHook)(void *) = NULL; +void* (*CallocHook)(size_t, size_t) = NULL; +uint32_t VPBufferSize = 0; + /* The buffer writer is reponsponsible in keeping writer state * across the call. */ @@ -29,7 +35,7 @@ COMPILER_RT_VISIBILITY int llvmWriteProfData(WriterCallback Writer, void *WriterCtx, - const uint8_t *ValueDataBegin, + ValueProfData **ValueDataArray, const uint64_t ValueDataSize) { /* Match logic in __llvm_profile_write_buffer(). */ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); @@ -39,15 +45,80 @@ const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); return llvmWriteProfDataImpl(Writer, WriterCtx, DataBegin, DataEnd, - CountersBegin, CountersEnd, ValueDataBegin, + CountersBegin, CountersEnd, ValueDataArray, ValueDataSize, NamesBegin, NamesEnd); } +#define VP_BUFFER_SIZE 8 * 1024 +static int writeValueProfData(WriterCallback Writer, void *WriterCtx, + ValueProfData **ValueDataBegin, + uint64_t NumVData) { + ValueProfData **ValueDataArray = ValueDataBegin; + char *BufferStart = 0, *Buffer; + ValueProfData *CurVData; + uint32_t I = 0, BufferSz; + + if (!ValueDataBegin) + return 0; + + BufferSz = VPBufferSize ? VPBufferSize : VP_BUFFER_SIZE; + BufferStart = (char *)CallocHook(BufferSz, 1); + if (!BufferStart) + return -1; + + uint32_t WriteSize = 0; + Buffer = BufferStart; + do { + CurVData = ValueDataArray[I]; + if (!CurVData) { + I++; + continue; + } + + /* Buffer is full or not large enough, it is time to flush. */ + if (CurVData->TotalSize + WriteSize > BufferSz) { + if (WriteSize) { + ProfDataIOVec IO[] = {{BufferStart, sizeof(char), WriteSize}}; + if (Writer(IO, 1, &WriterCtx)) + return -1; + WriteSize = 0; + Buffer = BufferStart; + } + /* Special case, bypass the buffer completely. */ + if (CurVData->TotalSize > BufferSz) { + ProfDataIOVec IO[] = {{CurVData, sizeof(char), CurVData->TotalSize}}; + if (Writer(IO, 1, &WriterCtx)) + return -1; + FreeHook(ValueDataArray[I]); + I++; + } + } else { + /* Write the data to buffer */ + ProfDataIOVec IO[] = {{CurVData, sizeof(char), CurVData->TotalSize}}; + llvmBufferWriter(IO, 1, (void **)&Buffer); + WriteSize += CurVData->TotalSize; + FreeHook(ValueDataArray[I]); + I++; + } + } while (I < NumVData); + + /* Final flush. */ + if (WriteSize) { + ProfDataIOVec IO[] = {{BufferStart, sizeof(char), WriteSize}}; + if (Writer(IO, 1, &WriterCtx)) + return -1; + } + + FreeHook(ValueDataBegin); + FreeHook(BufferStart); + return 0; +} + COMPILER_RT_VISIBILITY int llvmWriteProfDataImpl( WriterCallback Writer, void *WriterCtx, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin, const uint64_t *CountersEnd, - const uint8_t *ValueDataBegin, const uint64_t ValueDataSize, + ValueProfData **ValueDataBegin, const uint64_t ValueDataSize, const char *NamesBegin, const char *NamesEnd) { /* Calculate size of sections. */ @@ -70,18 +141,13 @@ #include "InstrProfData.inc" /* Write the data. */ - ProfDataIOVec IOVec[] = { - {&Header, sizeof(__llvm_profile_header), 1}, - {DataBegin, sizeof(__llvm_profile_data), DataSize}, - {CountersBegin, sizeof(uint64_t), CountersSize}, - {NamesBegin, sizeof(char), NamesSize}, - {Zeroes, sizeof(char), Padding}}; + ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1}, + {DataBegin, sizeof(__llvm_profile_data), DataSize}, + {CountersBegin, sizeof(uint64_t), CountersSize}, + {NamesBegin, sizeof(char), NamesSize}, + {Zeroes, sizeof(char), Padding}}; if (Writer(IOVec, sizeof(IOVec) / sizeof(*IOVec), &WriterCtx)) return -1; - if (ValueDataBegin) { - ProfDataIOVec IOVec2[] = {{ValueDataBegin, sizeof(char), ValueDataSize}}; - if (Writer(IOVec2, sizeof(IOVec2) / sizeof(*IOVec2), &WriterCtx)) - return -1; - } - return 0; + + return writeValueProfData(Writer, WriterCtx, ValueDataBegin, DataSize); } Index: test/profile/instrprof-value-prof.c =================================================================== --- test/profile/instrprof-value-prof.c +++ test/profile/instrprof-value-prof.c @@ -9,6 +9,22 @@ // value profile merging current do sorting based on target values -- this will destroy the order of the target // in the list leading to comparison problem. For now just check a small subset of output. // RUN: llvm-profdata show --all-functions -ic-targets %t-merged.profdata | FileCheck %s -check-prefix=MERGE +// +// RUN: env LLVM_PROFILE_FILE=%t-3.profraw LLVM_VP_BUFFER_SIZE=1 %run %t 1 +// RUN: env LLVM_PROFILE_FILE=%t-4.profraw LLVM_VP_BUFFER_SIZE=8 %run %t 1 +// RUN: env LLVM_PROFILE_FILE=%t-5.profraw LLVM_VP_BUFFER_SIZE=128 %run %t 1 +// RUN: env LLVM_PROFILE_FILE=%t-6.profraw LLVM_VP_BUFFER_SIZE=1024 %run %t 1 +// RUN: env LLVM_PROFILE_FILE=%t-7.profraw LLVM_VP_BUFFER_SIZE=102400 %run %t 1 +// RUN: llvm-profdata merge -o %t-3.profdata %t-3.profraw +// RUN: llvm-profdata merge -o %t-4.profdata %t-4.profraw +// RUN: llvm-profdata merge -o %t-5.profdata %t-5.profraw +// RUN: llvm-profdata merge -o %t-6.profdata %t-6.profraw +// RUN: llvm-profdata merge -o %t-7.profdata %t-7.profraw +// RUN: llvm-profdata show --all-functions -ic-targets %t-3.profdata | FileCheck %s +// RUN: llvm-profdata show --all-functions -ic-targets %t-4.profdata | FileCheck %s +// RUN: llvm-profdata show --all-functions -ic-targets %t-5.profdata | FileCheck %s +// RUN: llvm-profdata show --all-functions -ic-targets %t-6.profdata | FileCheck %s +// RUN: llvm-profdata show --all-functions -ic-targets %t-7.profdata | FileCheck %s #include #include