diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -194,7 +194,8 @@ void __llvm_profile_set_filename(const char *Name); /*! - * \brief Set the FILE object for writing instrumentation data. + * \brief Set the FILE object for writing instrumentation data. Return 0 if set + * successfully or return 1 if failed. * * Sets the FILE object to be used for subsequent calls to * \a __llvm_profile_write_file(). The profile file name set by environment @@ -213,13 +214,10 @@ * instrumented image/DSO). This API only modifies the file object within the * copy of the runtime available to the calling image. * - * Warning: This is a no-op if continuous mode (\ref - * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is - * that in continuous mode, profile counters are mmap()'d to the profile at - * program initialization time. Support for transferring the mmap'd profile - * counts to a new file has not been implemented. + * It requires no other threads are active at the time of the call in continuous + * mode. */ -void __llvm_profile_set_file_object(FILE *File, int EnableMerge); +int __llvm_profile_set_file_object(FILE *File, int EnableMerge); /*! \brief Register to write instrumentation data to file at exit. */ int __llvm_profile_register_write_file_atexit(void); diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c --- a/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/compiler-rt/lib/profile/InstrProfilingFile.c @@ -97,10 +97,56 @@ static const int ContinuousModeSupported = 1; static const int UseBiasVar = 0; static const char *FileOpenMode = "a+b"; -static intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; static void *BiasAddr = NULL; static void *BiasDefaultAddr = NULL; -static int MmapFlags = MAP_FIXED | MAP_SHARED; +static int +mmapForContinuousMode(const __llvm_profile_data *DataBegin, uint64_t DataSize, + const uint64_t *CountersBegin, + const uint64_t *CountersEnd, const uint64_t CountersSize, + const uint64_t NamesSize, uint64_t CurrentFileOffset, + FILE *File, uint64_t *MappedLength) { + /* Check that the counter and data sections in this image are + * page-aligned. */ + unsigned PageSize = getpagesize(); + if ((intptr_t)CountersBegin % PageSize != 0) { + PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n", + CountersBegin, PageSize); + return 1; + } + if ((intptr_t)DataBegin % PageSize != 0) { + PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n", + DataBegin, PageSize); + return 1; + } + int Fileno = fileno(File); + /* Determine how much padding is needed before/after the counters and + * after the names. */ + uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, + PaddingBytesAfterNames; + __llvm_profile_get_padding_sizes_for_counters( + DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + + *MappedLength = (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters; + uint64_t FileOffsetToCounters = + CurrentFileOffset + sizeof(__llvm_profile_header) + + (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters; + uint64_t *CounterMmap = (uint64_t *)mmap( + (void *)CountersBegin, *MappedLength, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToCounters); + if (CounterMmap != CountersBegin) { + PROF_ERR( + "Continuous counter sync mode is enabled, but mmap() failed (%s).\n" + " - CountersBegin: %p\n" + " - PageAlignedCountersLength: %" PRIu64 "\n" + " - Fileno: %d\n" + " - FileOffsetToCounters: %" PRIu64 "\n", + strerror(errno), CountersBegin, *MappedLength, Fileno, + FileOffsetToCounters); + return 1; + } + return 0; +} #elif defined(__ELF__) || defined(_WIN32) #define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \ @@ -134,15 +180,44 @@ * used and runtime provides a weak alias so we can check if it's defined. */ static void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; static void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR; -static int MmapFlags = MAP_SHARED; +static int +mmapForContinuousMode(const __llvm_profile_data *DataBegin, uint64_t DataSize, + const uint64_t *CountersBegin, + const uint64_t *CountersEnd, const uint64_t CountersSize, + const uint64_t NamesSize, uint64_t CurrentFileOffset, + FILE *File, uint64_t *MappedLength) { + /* Get the file size. */ + uint64_t FileSize = ftell(File); + + /* Map the profile. */ + char *Profile = (char *)mmap(NULL, FileSize, PROT_READ | PROT_WRITE, + MAP_SHARED, fileno(File), 0); + if (Profile == MAP_FAILED) { + PROF_ERR("Unable to mmap profile: %s\n", strerror(errno)); + return 1; + } + const uint64_t CountersOffsetInBiasMode = + sizeof(__llvm_profile_header) + (DataSize * sizeof(__llvm_profile_data)); + /* Update the profile fields based on the current mapping. */ + INSTR_PROF_PROFILE_COUNTER_BIAS_VAR = + (intptr_t)Profile - (uintptr_t)CountersBegin + CountersOffsetInBiasMode; + + /* Return the memory allocated for counters to OS. */ + lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd); + return 0; +} #else static const int ContinuousModeSupported = 0; static const int UseBiasVar = 0; static const char *FileOpenMode = "a+b"; -static intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; static void *BiasAddr = NULL; static void *BiasDefaultAddr = NULL; -static int MmapFlags = MAP_SHARED; +static int +mmapForContinuousMode(const __llvm_profile_data *DataBegin, uint64_t DataSize, + const uint64_t *CountersBegin, + const uint64_t *CountersEnd, const uint64_t CountersSize, + const uint64_t NamesSize, uint64_t CurrentFileOffset, + FILE *File, uint64_t *MappedLength) { return 0; } #endif static int isProfileMergeRequested() { return ProfileMergeRequested; } @@ -154,23 +229,120 @@ static FILE *getProfileFile() { return ProfileFile; } static void setProfileFile(FILE *File) { ProfileFile = File; } -COMPILER_RT_VISIBILITY void __llvm_profile_set_file_object(FILE *File, - int EnableMerge) { +static int getCurFilenameLength(); +static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf); + +static unsigned doMerging() { + return lprofCurFilename.MergePoolSize || isProfileMergeRequested(); +} + +static int doProfileMerging(FILE *ProfileFile, int *MergeDone); + +#if defined(__APPLE__) +COMPILER_RT_VISIBILITY int __llvm_profile_set_file_object(FILE *File, + int EnableMerge) { + if (__llvm_profile_is_continuous_mode_enabled()) { + if (doMerging() != (unsigned)EnableMerge) { + PROF_ERR("Inconsistent merging operation in continuous mode, EnableMerge " + "is set to: %d\n", + EnableMerge); + return 1; + } + int NewFileno = fileno(File); + int Length = getCurFilenameLength(); + char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + const char *OldFilename = getCurFilename(FilenameBuf, 0); + FILE *OldFile = fopen(OldFilename, "rb"); + if (!OldFile) { + PROF_ERR("can't open old profile with mode rb: %s\n", OldFilename); + return 1; + } + int OldFileno = fileno(OldFile); + if (fseek(OldFile, 0L, SEEK_END) == -1) { + PROF_ERR("Unable to set file object, unable to get size: %s\n", + strerror(errno)); + return 1; + } + size_t OldFileSize = ftell(OldFile); + + /* Get the sizes of various profile data sections. Taken from + * __llvm_profile_get_size_for_buffer(). */ + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const uint64_t *CountersBegin = __llvm_profile_begin_counters(); + const uint64_t *CountersEnd = __llvm_profile_end_counters(); + const char *NamesBegin = __llvm_profile_begin_names(); + const char *NamesEnd = __llvm_profile_end_names(); + const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); + uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + uint64_t CountersSize = CountersEnd - CountersBegin; + + /* Get the new file size. */ + if (fseek(File, 0L, SEEK_END) == -1) { + PROF_ERR("Unable to merge profile data, unable to get size: %s\n", + strerror(errno)); + return 1; + } + int NewFileOffset = ftello(File); + + if (EnableMerge) { + /* Merge new profile into memory counter before copy. */ + int MergeDone = 0; + int rc = doProfileMerging(File, &MergeDone); + if (rc) { + PROF_ERR("Profile Merging of file failed: %s\n", strerror(errno)); + return 1; + } + /* Reset offset to 0 to copy to the beginning of the new file. */ + NewFileOffset = 0; + } + + /* Append old profile to the end of the new profile. */ + (void)COMPILER_RT_FTRUNCATE(File, NewFileOffset + OldFileSize); + uint64_t *OldFileMmap = + mmap(NULL, OldFileSize, PROT_READ, MAP_PRIVATE, OldFileno, 0); + if (OldFileMmap == MAP_FAILED) { + PROF_ERR("Unable to mmap profile: %s\n", strerror(errno)); + return 1; + } + uint64_t *FileMmap = mmap(NULL, OldFileSize, PROT_READ | PROT_WRITE, + MAP_SHARED, NewFileno, NewFileOffset); + if (FileMmap == MAP_FAILED) { + PROF_ERR("Unable to mmap profile: %s\n", strerror(errno)); + return 1; + } + memcpy(FileMmap, OldFileMmap, OldFileSize); + munmap(FileMmap, OldFileSize); + munmap(OldFileMmap, OldFileSize); + + /* Determine how much padding is needed before/after the counters and after + * the names. */ + uint64_t PageAlignedCountersLength; + mmapForContinuousMode(DataBegin, DataSize, CountersBegin, CountersEnd, + CountersSize, NamesSize, NewFileOffset, File, + &PageAlignedCountersLength); + /* Unmap old profile. */ + munmap((void *)CountersBegin, PageAlignedCountersLength); + } else { + setProfileFile(File); + setProfileMergeRequested(EnableMerge); + } + return 0; +} +#else +COMPILER_RT_VISIBILITY int __llvm_profile_set_file_object(FILE *File, + int EnableMerge) { if (__llvm_profile_is_continuous_mode_enabled()) { PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported, because " "continuous sync mode (%%c) is enabled", fileno(File)); - return; + return 1; } setProfileFile(File); setProfileMergeRequested(EnableMerge); + return 0; } - -static int getCurFilenameLength(); -static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf); -static unsigned doMerging() { - return lprofCurFilename.MergePoolSize || isProfileMergeRequested(); -} +#endif /* Return 1 if there is an error, otherwise return 0. */ static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, @@ -578,73 +750,10 @@ /* mmap() the profile counters so long as there is at least one counter. * If there aren't any counters, mmap() would fail with EINVAL. */ if (CountersSize > 0) { - int Fileno = fileno(File); - if (UseBiasVar) { - /* Get the file size. */ - uint64_t FileSize = ftell(File); - - /* Map the profile. */ - char *Profile = (char *)mmap(NULL, FileSize, PROT_READ | PROT_WRITE, - MmapFlags, Fileno, 0); - if (Profile == MAP_FAILED) { - PROF_ERR("Unable to mmap profile: %s\n", strerror(errno)); - return; - } - const uint64_t CountersOffsetInBiasMode = - sizeof(__llvm_profile_header) + - (DataSize * sizeof(__llvm_profile_data)); - /* Update the profile fields based on the current mapping. */ - INSTR_PROF_PROFILE_COUNTER_BIAS_VAR = (intptr_t)Profile - - (uintptr_t)CountersBegin + - CountersOffsetInBiasMode; - - /* Return the memory allocated for counters to OS. */ - lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, - (uintptr_t)CountersEnd); - } else { - /* Check that the counter and data sections in this image are - * page-aligned. */ - unsigned PageSize = getpagesize(); - if ((intptr_t)CountersBegin % PageSize != 0) { - PROF_ERR( - "Counters section not page-aligned (start = %p, pagesz = %u).\n", - CountersBegin, PageSize); - return; - } - if ((intptr_t)DataBegin % PageSize != 0) { - PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n", - DataBegin, PageSize); - return; - } - /* Determine how much padding is needed before/after the counters and - * after the names. */ - uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames; - __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, - &PaddingBytesAfterCounters, &PaddingBytesAfterNames); - - uint64_t PageAlignedCountersLength = - (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters; - uint64_t FileOffsetToCounters = - CurrentFileOffset + sizeof(__llvm_profile_header) + - (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters; - - uint64_t *CounterMmap = - (uint64_t *)mmap((void *)CountersBegin, PageAlignedCountersLength, - PROT_READ | PROT_WRITE, MmapFlags, - Fileno, FileOffsetToCounters); - if (CounterMmap != CountersBegin) { - PROF_ERR( - "Continuous counter sync mode is enabled, but mmap() failed (%s).\n" - " - CountersBegin: %p\n" - " - PageAlignedCountersLength: %" PRIu64 "\n" - " - Fileno: %d\n" - " - FileOffsetToCounters: %" PRIu64 "\n", - strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno, - FileOffsetToCounters); - } - } + uint64_t MappedLength; + mmapForContinuousMode(DataBegin, DataSize, CountersBegin, CountersEnd, + CountersSize, NamesSize, CurrentFileOffset, File, + &MappedLength); } if (doMerging()) { diff --git a/compiler-rt/test/profile/ContinuousSyncMode/darwin-proof-of-concept.c b/compiler-rt/test/profile/ContinuousSyncMode/darwin-proof-of-concept.c --- a/compiler-rt/test/profile/ContinuousSyncMode/darwin-proof-of-concept.c +++ b/compiler-rt/test/profile/ContinuousSyncMode/darwin-proof-of-concept.c @@ -15,6 +15,12 @@ // RUN: %run %t create %t.tmpfile // RUN: %run %t validate %t.tmpfile +// Create a 'profile' using mmap, reset profile file via copying and munmap, +// and validate new profile. +// RUN: %run %t create %t.tmpfile_round2_old +// RUN: %run %t reset %t.tmpfile_round2_old %t.tmpfile_round2_new +// RUN: %run %t validate %t.tmpfile_round2_new + #include #include #include @@ -122,6 +128,64 @@ return EXIT_SUCCESS; } +int reset_tmpfile(char *old_path, char *new_path) { + int old_fd = open(old_path, O_RDONLY); + int new_fd = open(new_path, O_RDWR | O_TRUNC | O_CREAT, 0666); + if (old_fd == -1 || new_fd == -1) { + perror("open"); + return EXIT_FAILURE; + } + + // Unmmap old profile. + if (munmap(&cnts_start, 0x4000) == -1) { + perror("munmap"); + return EXIT_FAILURE; + } + + // Grow the new file to hold data and counters. + if (0 != ftruncate(new_fd, cnts_len + data_len)) { + perror("ftruncate"); + return EXIT_FAILURE; + } + + // Get file size of old profile. + size_t size = lseek(old_fd, 0, SEEK_END); + if (size == -1) { + perror("lseek"); + return EXIT_FAILURE; + } + + // Copy from old profile to new profile. + int *old_file_mmap = (int *)mmap(NULL, size, PROT_READ, MAP_PRIVATE, old_fd, + 0); + int *new_file_mmap = (int *)mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_SHARED, new_fd, 0); + if (old_file_mmap == MAP_FAILED || new_file_mmap == MAP_FAILED) { + perror("mmap"); + return EXIT_FAILURE; + } + + memcpy(new_file_mmap, old_file_mmap, size); + if (munmap(new_file_mmap, size) == -1 || munmap(old_file_mmap, size) == -1) { + perror("munmap"); + return EXIT_FAILURE; + } + + // Map the counters into the file, before the data. + // + // Requirements (on Darwin): + // - &cnts_start must be page-aligned. + // - The length and offset-into-fd must be page-aligned. + int *counter_map = (int *)mmap(&cnts_start, 0x4000, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_SHARED, new_fd, 0); + if (counter_map != &cnts_start) { + perror("mmap"); + return EXIT_FAILURE; + } + // Intentionally do not msync(), munmap(), or close(). + return EXIT_SUCCESS; +} + int main(int argc, char **argv) { intptr_t cnts_start_int = (intptr_t)&cnts_start; intptr_t data_start_int = (intptr_t)&data_start; @@ -146,6 +210,8 @@ return create_tmpfile(path); else if (0 == strcmp(action, "validate")) return validate_tmpfile(path); + else if (0 == strcmp(action, "reset")) + return reset_tmpfile(path, argv[3]); else return EXIT_FAILURE; } diff --git a/compiler-rt/test/profile/ContinuousSyncMode/set-file-object.c b/compiler-rt/test/profile/ContinuousSyncMode/set-file-object.c --- a/compiler-rt/test/profile/ContinuousSyncMode/set-file-object.c +++ b/compiler-rt/test/profile/ContinuousSyncMode/set-file-object.c @@ -1,34 +1,75 @@ // REQUIRES: darwin -// RUN: %clang_pgogen -o %t.exe %s -// RUN: env LLVM_PROFILE_FILE="%c%t.profraw" %run %t.exe %t.bad 2>&1 | FileCheck %s +// Test using __llvm_profile_set_file_object in continuous mode (%c). +// Create & cd into a temporary directory. +// RUN: rm -rf %t.dir && mkdir -p %t.dir && cd %t.dir +// RUN: %clang_pgogen -o main.exe %s -mllvm -instrprof-atomic-counter-update-all=1 -// CHECK: __llvm_profile_set_file_object(fd={{[0-9]+}}) not supported -// CHECK: Profile data not written to file: already written. +// Test continuous mode with __llvm_profile_set_file_object on empty file. +// RUN: env LLVM_PROFILE_FILE="%t.dir/profdir/%cprofraw.old" %run %t.dir/main.exe nomerge profraw.new +// RUN: diff %t.dir/profdir/profraw.old profraw.new + +// Test continuous mode without %m and __llvm_profile_set_file_object on +// non-empty file. +// This should append the old profile to the end of new profile. +// RUN: env LLVM_PROFILE_FILE="%t.dir/profdir/%cprofraw.old" %run %t.dir/main.exe nomerge profraw.new +// RUN: llvm-profdata show --all-functions --counts profraw.new | FileCheck %s -check-prefix=APPEND + +// APPEND-LABEL: Counters: +// APPEND-NEXT: main: +// APPEND-NEXT: Hash: 0x{{.*}} +// APPEND-NEXT: Counters: +// APPEND-NEXT: Block counts: [[COUNTS:.*]] +// APPEND-NEXT: main: +// APPEND-NEXT: Hash: 0x{{.*}} +// APPEND-NEXT: Counters: +// APPEND-NEXT: Block counts: [[COUNTS]] +// APPEND-NEXT: Instrumentation level: IR entry_first = 0 +// APPEND-NEXT: Functions shown: 2 +// APPEND-NEXT: Total functions: 2 +// APPEND-NEXT: Maximum function count: 0 +// APPEND-NEXT: Maximum internal block count: 1 + +// Test continuous mode with %m and __llvm_profile_set_file_object(.., 1) on +// non-empty file. +// This should merge old profile with the new profile. +// RUN: rm profraw.new +// RUN: env LLVM_PROFILE_FILE="%t.dir/profdir/%c%mprofraw.old" %run %t.dir/main.exe merge profraw.new +// RUN: env LLVM_PROFILE_FILE="%t.dir/profdir/%c%mprofraw.old" %run %t.dir/main.exe merge profraw.new +// RUN: llvm-profdata merge profraw.new -o profdata +// RUN: llvm-profdata show --all-functions --counts profdata | FileCheck %s -check-prefix=MERGE + +// MERGE-LABEL: Counters: +// MERGE-NEXT: main: +// MERGE-NEXT: Hash: 0x{{.*}} +// MERGE-NEXT: Counters: +// MERGE-NEXT: Block counts: {{.*}} +// MERGE-NEXT: Instrumentation level: IR entry_first = 0 +// MERGE-NEXT: Functions shown: 1 +// MERGE-NEXT: Total functions: 1 +// MERGE-NEXT: Maximum function count: 0 +// MERGE-NEXT: Maximum internal block count: 2 #include +#include extern int __llvm_profile_is_continuous_mode_enabled(void); extern void __llvm_profile_set_file_object(FILE *, int); -extern int __llvm_profile_write_file(void); int main(int argc, char **argv) { - if (!__llvm_profile_is_continuous_mode_enabled()) + if (argc < 3 || !__llvm_profile_is_continuous_mode_enabled()) return 1; - - FILE *f = fopen(argv[1], "a+b"); - if (!f) + int merge; + if (strcmp(argv[1], "nomerge") == 0) + merge = 0; + else if (strcmp(argv[1], "merge") == 0) + merge = 1; + else return 1; - __llvm_profile_set_file_object(f, 0); // Try to set the file to "%t.bad". - - if (__llvm_profile_write_file() != 0) - return 1; - - f = fopen(argv[1], "r"); - if (!f) - return 1; + char *file_name = argv[2]; - fseek(f, 0, SEEK_END); - return ftell(f); // Check that the "%t.bad" is empty. + FILE *file = fopen(file_name, "a+b"); + __llvm_profile_set_file_object(file, merge); + return 0; }