diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -664,6 +664,7 @@ #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime +#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias /* The variable that holds the name of the profile data * specified via command line. */ diff --git a/compiler-rt/lib/profile/CMakeLists.txt b/compiler-rt/lib/profile/CMakeLists.txt --- a/compiler-rt/lib/profile/CMakeLists.txt +++ b/compiler-rt/lib/profile/CMakeLists.txt @@ -53,7 +53,6 @@ InstrProfiling.c InstrProfilingInternal.c InstrProfilingValue.c - InstrProfilingBiasVar.c InstrProfilingBuffer.c InstrProfilingFile.c InstrProfilingMerge.c diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -319,11 +319,4 @@ */ extern char INSTR_PROF_PROFILE_NAME_VAR[1]; /* __llvm_profile_filename. */ -/*! - * This variable is a weak symbol defined in InstrProfilingBiasVar.c. It - * allows compiler instrumentation to provide overriding definition with - * value from compiler command line. This variable has hidden visibility. - */ -COMPILER_RT_VISIBILITY extern intptr_t __llvm_profile_counter_bias; - #endif /* PROFILE_INSTRPROFILING_H_ */ diff --git a/compiler-rt/lib/profile/InstrProfilingBiasVar.c b/compiler-rt/lib/profile/InstrProfilingBiasVar.c deleted file mode 100644 --- a/compiler-rt/lib/profile/InstrProfilingBiasVar.c +++ /dev/null @@ -1,15 +0,0 @@ -/*===- InstrProfilingBiasVar.c - profile counter bias variable setup ------===*\ -|* -|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -|* See https://llvm.org/LICENSE.txt for license information. -|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -|* -\*===----------------------------------------------------------------------===*/ - -#include "InstrProfiling.h" - -/* The runtime should only provide its own definition of this symbol when the - * user has not specified one. Set this up by moving the runtime's copy of this - * symbol to an object file within the archive. - */ -COMPILER_RT_WEAK intptr_t __llvm_profile_counter_bias = -1; diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -67,13 +67,20 @@ return 0; } +static int needsCounterPadding(void) { +#if defined(__APPLE__) + return __llvm_profile_is_continuous_mode_enabled(); +#else + return 0; +#endif +} + COMPILER_RT_VISIBILITY void __llvm_profile_get_padding_sizes_for_counters( uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterNames) { - if (!__llvm_profile_is_continuous_mode_enabled() || - lprofRuntimeCounterRelocation()) { + if (!needsCounterPadding()) { *PaddingBytesBeforeCounters = 0; *PaddingBytesAfterCounters = 0; *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize); diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c --- a/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/compiler-rt/lib/profile/InstrProfilingFile.c @@ -426,33 +426,6 @@ fclose(File); } -#if !defined(__Fuchsia__) && !defined(_WIN32) -static void assertIsZero(int *i) { - if (*i) - PROF_WARN("Expected flag to be 0, but got: %d\n", *i); -} - -/* Write a partial profile to \p Filename, which is required to be backed by - * the open file object \p File. */ -static int writeProfileWithFileObject(const char *Filename, FILE *File) { - setProfileFile(File); - int rc = writeFile(Filename); - if (rc) - PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); - setProfileFile(NULL); - return rc; -} - -/* Unlock the profile \p File and clear the unlock flag. */ -static void unlockProfile(int *ProfileRequiresUnlock, FILE *File) { - if (!*ProfileRequiresUnlock) { - PROF_WARN("%s", "Expected to require profile unlock\n"); - } - lprofUnlockFileHandle(File); - *ProfileRequiresUnlock = 0; -} -#endif // !defined(__Fuchsia__) && !defined(_WIN32) - static int writeMMappedFile(FILE *OutputFile, char **Profile) { if (!OutputFile) return -1; @@ -481,83 +454,38 @@ return 0; } -static void relocateCounters(void) { - if (!__llvm_profile_is_continuous_mode_enabled() || - !lprofRuntimeCounterRelocation()) - return; - - /* Get the sizes of various profile data sections. Taken from - * __llvm_profile_get_size_for_buffer(). */ - const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); - const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); - const uint64_t *CountersBegin = __llvm_profile_begin_counters(); - const uint64_t *CountersEnd = __llvm_profile_end_counters(); - uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); - const uint64_t CountersOffset = sizeof(__llvm_profile_header) + - (DataSize * sizeof(__llvm_profile_data)); - - int Length = getCurFilenameLength(); - char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); - const char *Filename = getCurFilename(FilenameBuf, 0); - if (!Filename) - return; - - FILE *File = NULL; - char *Profile = NULL; - - if (!doMerging()) { - File = fopen(Filename, "w+b"); - if (!File) - return; - - if (writeMMappedFile(File, &Profile) == -1) { - fclose(File); - return; - } - } else { - File = lprofOpenFileEx(Filename); - if (!File) - return; - - uint64_t ProfileFileSize = 0; - if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) { - lprofUnlockFileHandle(File); - fclose(File); - return; - } +// TODO: Move these functions into InstrProfilingPlatform* files. +#if defined(__APPLE__) +static void assertIsZero(int *i) { + if (*i) + PROF_WARN("Expected flag to be 0, but got: %d\n", *i); +} - if (!ProfileFileSize) { - if (writeMMappedFile(File, &Profile) == -1) { - fclose(File); - return; - } - } else { - /* The merged profile has a non-zero length. Check that it is compatible - * with the data in this process. */ - if (mmapProfileForMerging(File, ProfileFileSize, &Profile) == -1) { - fclose(File); - return; - } - } +/* Write a partial profile to \p Filename, which is required to be backed by + * the open file object \p File. */ +static int writeProfileWithFileObject(const char *Filename, FILE *File) { + setProfileFile(File); + int rc = writeFile(Filename); + if (rc) + PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); + setProfileFile(NULL); + return rc; +} - lprofUnlockFileHandle(File); +/* Unlock the profile \p File and clear the unlock flag. */ +static void unlockProfile(int *ProfileRequiresUnlock, FILE *File) { + if (!*ProfileRequiresUnlock) { + PROF_WARN("%s", "Expected to require profile unlock\n"); } - /* Update the profile fields based on the current mapping. */ - __llvm_profile_counter_bias = - (intptr_t)Profile - (uintptr_t)CountersBegin + CountersOffset; - - /* Return the memory allocated for counters to OS. */ - lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd); + lprofUnlockFileHandle(File); + *ProfileRequiresUnlock = 0; } static void initializeProfileForContinuousMode(void) { if (!__llvm_profile_is_continuous_mode_enabled()) return; -#if defined(__Fuchsia__) || defined(_WIN32) - PROF_ERR("%s\n", "Continuous mode not yet supported on Fuchsia or Windows."); -#else // defined(__Fuchsia__) || defined(_WIN32) /* Get the sizes of various profile data sections. Taken from * __llvm_profile_get_size_for_buffer(). */ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); @@ -683,8 +611,109 @@ if (ProfileRequiresUnlock) unlockProfile(&ProfileRequiresUnlock, File); -#endif // defined(__Fuchsia__) || defined(_WIN32) } +#elif defined(__ELF__) || defined(_WIN32) + +#define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \ + INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default) +intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0; + +/* This variable is a weak external reference which could be used to detect + * whether or not the compiler defined this symbol. */ +#if defined(_WIN32) +COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; +#pragma comment(linker, "/alternatename:" \ + INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" \ + INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR)) +#else +COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR + __attribute__((weak, alias(INSTR_PROF_QUOTE( + INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR)))); +#endif + +static void initializeProfileForContinuousMode(void) { + if (!__llvm_profile_is_continuous_mode_enabled()) + return; + + /* This symbol is defined by the compiler when runtime counter relocation is + * used and runtime provides a weak alias so we can check if it's defined. */ + void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; + void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR; + if (BiasAddr == BiasDefaultAddr) { + PROF_ERR("%s\n", "__llvm_profile_counter_bias is undefined"); + return; + } + + /* Get the sizes of various profile data sections. Taken from + * __llvm_profile_get_size_for_buffer(). */ + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const uint64_t *CountersBegin = __llvm_profile_begin_counters(); + const uint64_t *CountersEnd = __llvm_profile_end_counters(); + uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + const uint64_t CountersOffset = + sizeof(__llvm_profile_header) + (DataSize * sizeof(__llvm_profile_data)); + + int Length = getCurFilenameLength(); + char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + const char *Filename = getCurFilename(FilenameBuf, 0); + if (!Filename) + return; + + FILE *File = NULL; + char *Profile = NULL; + + if (!doMerging()) { + File = fopen(Filename, "w+b"); + if (!File) + return; + + if (writeMMappedFile(File, &Profile) == -1) { + fclose(File); + return; + } + } else { + File = lprofOpenFileEx(Filename); + if (!File) + return; + + uint64_t ProfileFileSize = 0; + if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) { + lprofUnlockFileHandle(File); + fclose(File); + return; + } + + if (!ProfileFileSize) { + if (writeMMappedFile(File, &Profile) == -1) { + fclose(File); + return; + } + } else { + /* The merged profile has a non-zero length. Check that it is compatible + * with the data in this process. */ + if (mmapProfileForMerging(File, ProfileFileSize, &Profile) == -1) { + fclose(File); + return; + } + } + + lprofUnlockFileHandle(File); + } + + /* Update the profile fields based on the current mapping. */ + INSTR_PROF_PROFILE_COUNTER_BIAS_VAR = + (intptr_t)Profile - (uintptr_t)CountersBegin + + CountersOffset; + + /* Return the memory allocated for counters to OS. */ + lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd); +} +#else +static void initializeProfileForContinuousMode(void) { + PROF_ERR("%s\n", "continuous mode is unsupported on this platform"); +} +#endif static const char *DefaultProfileName = "default.profraw"; static void resetFilenameToDefault(void) { @@ -784,9 +813,14 @@ FilenamePat); return -1; } - +#if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32) __llvm_profile_set_page_size(getpagesize()); __llvm_profile_enable_continuous_mode(); +#else + PROF_WARN("%s", "Continous mode is currently only supported for Mach-O," + " ELF and COFF formats."); + return -1; +#endif } else { unsigned MergePoolSize = getMergePoolSize(FilenamePat, &I); if (!MergePoolSize) @@ -843,12 +877,8 @@ } truncateCurrentFile(); - if (__llvm_profile_is_continuous_mode_enabled()) { - if (lprofRuntimeCounterRelocation()) - relocateCounters(); - else - initializeProfileForContinuousMode(); - } + if (__llvm_profile_is_continuous_mode_enabled()) + initializeProfileForContinuousMode(); } /* Return buffer length that is required to store the current profile @@ -1004,9 +1034,6 @@ ProfileNameSpecifier PNS = PNS_unknown; int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0); - if (__llvm_profile_counter_bias != -1) - lprofSetRuntimeCounterRelocation(1); - EnvFilenamePat = getFilenamePatFromEnv(); if (EnvFilenamePat) { /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h --- a/compiler-rt/lib/profile/InstrProfilingInternal.h +++ b/compiler-rt/lib/profile/InstrProfilingInternal.h @@ -184,10 +184,6 @@ unsigned lprofProfileDumped(void); void lprofSetProfileDumped(unsigned); -/* Return non zero value if counters are being relocated at runtime. */ -unsigned lprofRuntimeCounterRelocation(void); -void lprofSetRuntimeCounterRelocation(unsigned); - COMPILER_RT_VISIBILITY extern void (*FreeHook)(void *); COMPILER_RT_VISIBILITY extern uint8_t *DynamicBufferIOBuffer; COMPILER_RT_VISIBILITY extern uint32_t VPBufferSize; diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.c b/compiler-rt/lib/profile/InstrProfilingInternal.c --- a/compiler-rt/lib/profile/InstrProfilingInternal.c +++ b/compiler-rt/lib/profile/InstrProfilingInternal.c @@ -23,14 +23,4 @@ ProfileDumped = Value; } -static unsigned RuntimeCounterRelocation = 0; - -COMPILER_RT_VISIBILITY unsigned lprofRuntimeCounterRelocation(void) { - return RuntimeCounterRelocation; -} - -COMPILER_RT_VISIBILITY void lprofSetRuntimeCounterRelocation(unsigned Value) { - RuntimeCounterRelocation = Value; -} - #endif diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c b/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c --- a/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c @@ -34,16 +34,14 @@ #include "InstrProfilingInternal.h" #include "InstrProfilingUtil.h" +/* This variable is an external reference to symbol defined by the compiler. */ +COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; + COMPILER_RT_VISIBILITY unsigned lprofProfileDumped() { return 1; } COMPILER_RT_VISIBILITY void lprofSetProfileDumped(unsigned Value) {} -COMPILER_RT_VISIBILITY unsigned lprofRuntimeCounterRelocation(void) { - return 1; -} -COMPILER_RT_VISIBILITY void lprofSetRuntimeCounterRelocation(unsigned Value) {} - static const char ProfileSinkName[] = "llvm-profile"; static inline void lprofWrite(const char *fmt, ...) { @@ -116,14 +114,6 @@ return; } - /* This symbol is defined as weak and initialized to -1 by the runtimer, but - * compiler will generate a strong definition initialized to 0 when runtime - * counter relocation is used. */ - if (__llvm_profile_counter_bias == -1) { - lprofWrite("LLVM Profile: counter relocation at runtime is required\n"); - return; - } - const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); const uint64_t *CountersBegin = __llvm_profile_begin_counters(); @@ -192,7 +182,7 @@ lprofWrite("LLVM Profile: {{{dumpfile:%s:%s}}}\n", ProfileSinkName, VmoName); /* Update the profile fields based on the current mapping. */ - __llvm_profile_counter_bias = + INSTR_PROF_PROFILE_COUNTER_BIAS_VAR = (intptr_t)Mapping - (uintptr_t)CountersBegin + CountersOffset; /* Return the memory allocated for counters to OS. */ diff --git a/compiler-rt/lib/profile/InstrProfilingUtil.h b/compiler-rt/lib/profile/InstrProfilingUtil.h --- a/compiler-rt/lib/profile/InstrProfilingUtil.h +++ b/compiler-rt/lib/profile/InstrProfilingUtil.h @@ -74,11 +74,11 @@ /* Restore previously suspended SIGKILL. */ void lprofRestoreSigKill(); -inline size_t lprofRoundUpTo(size_t x, size_t boundary) { +static inline size_t lprofRoundUpTo(size_t x, size_t boundary) { return (x + boundary - 1) & ~(boundary - 1); } -inline size_t lprofRoundDownTo(size_t x, size_t boundary) { +static inline size_t lprofRoundDownTo(size_t x, size_t boundary) { return x & ~(boundary - 1); } diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -156,7 +156,7 @@ } inline StringRef getInstrProfCounterBiasVarName() { - return "__llvm_profile_counter_bias"; + return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR); } /// Return the marker used to separate PGO names during serialization. diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -664,6 +664,7 @@ #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime +#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias /* The variable that holds the name of the profile data * specified via command line. */ diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -467,9 +467,14 @@ } bool InstrProfiling::isRuntimeCounterRelocationEnabled() const { + // Mach-O don't support weak external references. + if (TT.isOSBinFormatMachO()) + return false; + if (RuntimeCounterRelocation.getNumOccurrences() > 0) return RuntimeCounterRelocation; + // Fuchsia uses runtime counter relocation by default. return TT.isOSFuchsia(); } @@ -690,10 +695,19 @@ Type *Int64Ty = Type::getInt64Ty(M->getContext()); GlobalVariable *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName()); if (!Bias) { + // Compiler must define this variable when runtime counter relocation + // is being used. Runtime has a weak external reference that is used + // to check whether that's the case or not. Bias = new GlobalVariable(*M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage, Constant::getNullValue(Int64Ty), getInstrProfCounterBiasVarName()); Bias->setVisibility(GlobalVariable::HiddenVisibility); + // A definition that's weak (linkonce_odr) without being in a COMDAT + // section wouldn't lead to link errors, but it would lead to a dead + // data word from every TU but one. Putting it in COMDAT ensures there + // will be exactly one data slot in the link. + if (TT.supportsCOMDAT()) + Bias->setComdat(M->getOrInsertComdat(Bias->getName())); } LI = Builder.CreateLoad(Int64Ty, Bias); } diff --git a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll --- a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll +++ b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll @@ -4,7 +4,8 @@ target triple = "x86_64-unknown-linux-gnu" @__profn_foo = private constant [3 x i8] c"foo" -; RELOC: @__llvm_profile_counter_bias = linkonce_odr hidden global i64 0 +; RELOC: $__llvm_profile_counter_bias = comdat any +; RELOC: @__llvm_profile_counter_bias = linkonce_odr hidden global i64 0, comdat ; CHECK-LABEL: define void @foo ; CHECK-NEXT: %pgocount = load i64, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc_foo, i64 0, i64 0)