Index: lib/profile/InstrProfilingFile.c =================================================================== --- lib/profile/InstrProfilingFile.c +++ lib/profile/InstrProfilingFile.c @@ -18,7 +18,20 @@ /* For _alloca. */ #include #endif +#if defined(_WIN32) +#include "WindowsMMap.h" +/* For _chsize_s */ +#include +#else +#include +#include +#include +#if defined(__linux__) +#include +#endif +#endif +#define PROFILE_POOL_MAX_SIZE 9 #define MAX_PID_SIZE 16 /* Data structure holding the result of parsed filename pattern. */ typedef struct lprofFilename { @@ -28,13 +41,22 @@ char Hostname[COMPILER_RT_MAX_HOSTLEN]; unsigned NumPids; unsigned NumHosts; + /* When in-process merging is enabled, this parameter specifies + * the total number of profile data files shared by all the processes + * spawned from the same binary. By default the value is 1. If merging + * is not enabled, its value should be 0. This parameter is specified + * by the %[0-9]m specifier. For instance %2m enables merging using + * 2 profile data files. %1m is equivalent to %m. Also %m specifier + * can only appear once at the end of the name pattern. */ + unsigned MergePoolSize; } lprofFilename; -lprofFilename lprofCurFilename = {0, {0}, {0}, 0, 0}; +lprofFilename lprofCurFilename = {0, {0}, {0}, 0, 0, 0}; int getpid(void); static int getCurFilenameLength(); static const char *getCurFilename(char *FilenameBuf); +static unsigned doMerging() { return lprofCurFilename.MergePoolSize; } /* Return 1 if there is an error, otherwise return 0. */ static uint32_t fileWriter(ProfDataIOVec *IOVecs, uint32_t NumIOVecs, @@ -66,13 +88,84 @@ } } +/* Read profile data in \c ProfileFile and merge with in-memory + profile counters. Returns -1 if there is fatal error, otheriwse + 0 is returned. +*/ +static int doProfileMerging(FILE *ProfileFile) { + uint64_t ProfileFileSize; + char *ProfileBuffer; + + fseek(ProfileFile, 0L, SEEK_END); + ProfileFileSize = ftell(ProfileFile); + /* Restore file offset. */ + fseek(ProfileFile, 0L, SEEK_SET); + if (ProfileFileSize < sizeof(__llvm_profile_header)) { + if (ProfileFileSize) + PROF_WARN("Unable to merge profile data: %s\n", + "source profile file is too small."); + return 0; + } + + ProfileBuffer = mmap(0, ProfileFileSize, PROT_READ, MAP_SHARED | MAP_FILE, + fileno(ProfileFile), 0); + if (ProfileBuffer == (void *)-1) { + PROF_ERR("Unable to merge profile data, mmap failed: %s\n", + strerror(errno)); + return -1; + } + + if (__llvm_profile_check_compatibility(ProfileBuffer, ProfileFileSize)) { + (void)munmap(ProfileBuffer, ProfileFileSize); + PROF_WARN("Unable to merge profile data: %s\n", + "source profile file is not compatible."); + return 0; + } + + /* Now start merging */ + __llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize); + (void)munmap(ProfileBuffer, ProfileFileSize); + + return 0; +} + +/* Open the profile data for merging. It opens the file in r+b mode with + * file locking. If the file has content which is compatible with the + * current process, it also reads in the profile data in the file and merge + * it with in-memory counters. After the profile data is merged in memory, + * the original profile data is truncated and gets ready for the profile + * dumper. With profile merging enabled, each executable as well as any of + * its instrumented shared libraries dump profile data into their own data file. +*/ +static FILE *openFileForMerging(const char *ProfileFileName) { + FILE *ProfileFile; + int rc; + + ProfileFile = lprofOpenFileEx(ProfileFileName); + if (!ProfileFile) + return NULL; + + rc = doProfileMerging(ProfileFile); + if (rc || COMPILER_RT_FTRUNCATE(ProfileFile, 0L)) { + PROF_ERR("Profile Merging of file %s failed: %s\n", ProfileFileName, + strerror(errno)); + fclose(ProfileFile); + return NULL; + } + fseek(ProfileFile, 0L, SEEK_SET); + return ProfileFile; +} + /* Write profile data to file \c OutputName. */ static int writeFile(const char *OutputName) { int RetVal; FILE *OutputFile; - /* Append to the file to support profiling multiple shared objects. */ - OutputFile = fopen(OutputName, "ab"); + if (!doMerging()) + OutputFile = fopen(OutputName, "ab"); + else + OutputFile = openFileForMerging(OutputName); + if (!OutputFile) return -1; @@ -122,6 +215,7 @@ int NumPids = 0, NumHosts = 0, I; char *PidChars = &lprofCurFilename.PidChars[0]; char *Hostname = &lprofCurFilename.Hostname[0]; + int MergingEnabled = 0; lprofCurFilename.FilenamePat = FilenamePat; /* Check the filename for "%p", which indicates a pid-substitution. */ @@ -144,6 +238,21 @@ FilenamePat); return -1; } + } else if (FilenamePat[I] == 'm' || + (FilenamePat[I] >= '1' && FilenamePat[I] <= '9' && + FilenamePat[I + 1] == 'm')) { + if (MergingEnabled || (FilenamePat[I] == 'm' && FilenamePat[I + 1]) || + (FilenamePat[I + 1] == 'm' && FilenamePat[I + 2])) { + PROF_WARN( + "%%m specifier can only be specified once at the end of %s.\n", + FilenamePat); + return -1; + } + MergingEnabled = 1; + if (FilenamePat[I] == 'm') + lprofCurFilename.MergePoolSize = 1; + else + lprofCurFilename.MergePoolSize = FilenamePat[I] - '0'; } } @@ -162,22 +271,28 @@ NewFile = !OldFilenamePat || (strcmp(OldFilenamePat, lprofCurFilename.FilenamePat)); - if (NewFile) + if (NewFile && !lprofCurFilename.MergePoolSize) truncateCurrentFile(); } /* Return buffer length that is required to store the current profile * filename with PID and hostname substitutions. */ +#define SIGLEN 24 static int getCurFilenameLength() { + int Len; if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0]) return 0; - if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts)) + if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts || + lprofCurFilename.MergePoolSize)) return strlen(lprofCurFilename.FilenamePat); - return strlen(lprofCurFilename.FilenamePat) + - lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars) - 2) + - lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2); + Len = strlen(lprofCurFilename.FilenamePat) + + lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars) - 2) + + lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2); + if (lprofCurFilename.MergePoolSize) + Len += SIGLEN; + return Len; } /* Return the pointer to the current profile file name (after substituting @@ -191,7 +306,8 @@ if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0]) return 0; - if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts)) + if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts || + lprofCurFilename.MergePoolSize)) return lprofCurFilename.FilenamePat; PidLength = strlen(lprofCurFilename.PidChars); @@ -205,6 +321,15 @@ } else if (FilenamePat[I] == 'h') { memcpy(FilenameBuf + J, lprofCurFilename.Hostname, HostNameLength); J += HostNameLength; + } else if (FilenamePat[I] == 'm' || + (FilenamePat[I] >= '1' && FilenamePat[I] <= '9' && + FilenamePat[I + 1] == 'm')) { + char LoadModuleSignature[SIGLEN]; + int ProfilePoolId = getpid() % lprofCurFilename.MergePoolSize; + snprintf(LoadModuleSignature, SIGLEN, "%" PRIu64 "_%d", + lprofGetLoadModuleSignature(), ProfilePoolId); + memcpy(FilenameBuf + J, LoadModuleSignature, SIGLEN); + J += SIGLEN; } /* Drop any unknown substitutions. */ } else Index: lib/profile/InstrProfilingInternal.h =================================================================== --- lib/profile/InstrProfilingInternal.h +++ lib/profile/InstrProfilingInternal.h @@ -156,6 +156,13 @@ void lprofSetMaxValsPerSite(uint32_t MaxVals); void lprofSetupValueProfiler(); +/* Return the profile header 'siguature' value associated with the current + * executation or shared library. The signature value can be used to for + * a profile name that is unique to this load module so that it does not + * collide with profiles from other binaries. It also allows shared libraries + * to dump merged profile data into its own profile file. */ +uint64_t lprofGetLoadModuleSignature(); + COMPILER_RT_VISIBILITY extern char *(*GetEnvHook)(const char *); COMPILER_RT_VISIBILITY extern void (*FreeHook)(void *); COMPILER_RT_VISIBILITY extern uint8_t *DynamicBufferIOBuffer; Index: lib/profile/InstrProfilingMerge.c =================================================================== --- lib/profile/InstrProfilingMerge.c +++ lib/profile/InstrProfilingMerge.c @@ -19,6 +19,22 @@ COMPILER_RT_WEAK void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *) = NULL; +COMPILER_RT_VISIBILITY +uint64_t lprofGetLoadModuleSignature() { + /* A very fast way to compute the module signature. */ + uint64_t CounterSize = (uint64_t)(__llvm_profile_end_counters() - + __llvm_profile_begin_counters()); + uint64_t DataSize = __llvm_profile_get_data_size(__llvm_profile_begin_data(), + __llvm_profile_end_data()); + uint64_t NamesSize = + (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names()); + uint64_t NumVnodes = + (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes()); + const __llvm_profile_data *FirstD = __llvm_profile_begin_data(); + + return (NamesSize << 40) + (CounterSize << 30) + (DataSize << 20) + + (NumVnodes << 10) + (DataSize > 0 ? FirstD->NameRef : 0); +} /* Returns 1 if profile is not structurally compatible. */ COMPILER_RT_VISIBILITY Index: lib/profile/InstrProfilingPort.h =================================================================== --- lib/profile/InstrProfilingPort.h +++ lib/profile/InstrProfilingPort.h @@ -14,12 +14,16 @@ #define COMPILER_RT_ALIGNAS(x) __declspec(align(x)) #define COMPILER_RT_VISIBILITY #define COMPILER_RT_WEAK __declspec(selectany) +/* Need to include */ #define COMPILER_RT_ALLOCA _alloca +/* Need to include and */ +#define COMPILER_RT_FTRUNCATE(f,l) _chsize(_fileno(f),l) #elif __GNUC__ #define COMPILER_RT_ALIGNAS(x) __attribute__((aligned(x))) #define COMPILER_RT_VISIBILITY __attribute__((visibility("hidden"))) #define COMPILER_RT_WEAK __attribute__((weak)) #define COMPILER_RT_ALLOCA __builtin_alloca +#define COMPILER_RT_FTRUNCATE(f,l) ftruncate(fileno(f),l) #endif #if defined(__APPLE__) Index: test/profile/instrprof-basic.c =================================================================== --- test/profile/instrprof-basic.c +++ test/profile/instrprof-basic.c @@ -1,17 +1,30 @@ // RUN: %clang_profgen -o %t -O3 %s // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t // RUN: llvm-profdata merge -o %t.profdata %t.profraw -// RUN: %clang_profuse=%t.profdata -o - -S -emit-llvm %s | FileCheck %s +// RUN: %clang_profuse=%t.profdata -o - -S -emit-llvm %s | FileCheck %s --check-prefix=COMMON --check-prefix=ORIG +// +// RUN: rm -f %t.profraw_e_* +// RUN: env LLVM_PROFILE_FILE=%t.profraw_e_%1m %run %t +// RUN: env LLVM_PROFILE_FILE=%t.profraw_e_%1m %run %t +// RUN: llvm-profdata merge -o %t.em.profdata %t.profraw_e_* +// RUN: %clang_profuse=%t.em.profdata -o - -S -emit-llvm %s | FileCheck %s --check-prefix=COMMON --check-prefix=MERGE +// +// RUN: %clang -o %t.merge -fprofile-instr-generate=%t.profraw_%m -O3 %s +// RUN: rm -f %t.profraw_* +// RUN: %run %t.merge +// RUN: %run %t.merge +// RUN: llvm-profdata merge -o %t.m.profdata %t.profraw_* +// RUN: %clang_profuse=%t.m.profdata -o - -S -emit-llvm %s | FileCheck %s --check-prefix=COMMON --check-prefix=MERGE int begin(int i) { - // CHECK: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}, !prof ![[PD1:[0-9]+]] + // COMMON: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}, !prof ![[PD1:[0-9]+]] if (i) return 0; return 1; } int end(int i) { - // CHECK: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}, !prof ![[PD2:[0-9]+]] + // COMMON: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}, !prof ![[PD2:[0-9]+]] if (i) return 0; return 1; @@ -21,11 +34,13 @@ begin(0); end(1); - // CHECK: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}, !prof ![[PD2:[0-9]+]] + // COMMON: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}, !prof ![[PD2:[0-9]+]] if (argc) return 0; return 1; } -// CHECK: ![[PD1]] = !{!"branch_weights", i32 1, i32 2} -// CHECK: ![[PD2]] = !{!"branch_weights", i32 2, i32 1} +// ORIG: ![[PD1]] = !{!"branch_weights", i32 1, i32 2} +// ORIG: ![[PD2]] = !{!"branch_weights", i32 2, i32 1} +// MERGE: ![[PD1]] = !{!"branch_weights", i32 1, i32 3} +// MERGE: ![[PD2]] = !{!"branch_weights", i32 3, i32 1}