Index: compiler-rt/lib/profile/CMakeLists.txt =================================================================== --- compiler-rt/lib/profile/CMakeLists.txt +++ compiler-rt/lib/profile/CMakeLists.txt @@ -52,6 +52,7 @@ GCDAProfiling.c InstrProfiling.c InstrProfilingValue.c + InstrProfilingBiasVar.c InstrProfilingBuffer.c InstrProfilingFile.c InstrProfilingMerge.c Index: compiler-rt/lib/profile/InstrProfiling.h =================================================================== --- compiler-rt/lib/profile/InstrProfiling.h +++ compiler-rt/lib/profile/InstrProfiling.h @@ -261,4 +261,11 @@ */ extern char INSTR_PROF_PROFILE_NAME_VAR[1]; /* __llvm_profile_filename. */ +/*! + * This variable is a weak symbol defined in InstrProfilingBiasVar.c. It + * allows compiler instrumentation to provide overriding definition with + * value from compiler command line. This variable has hidden visibility. + */ +COMPILER_RT_VISIBILITY extern intptr_t __llvm_profile_counter_bias; + #endif /* PROFILE_INSTRPROFILING_H_ */ Index: compiler-rt/lib/profile/InstrProfilingBiasVar.c =================================================================== --- /dev/null +++ compiler-rt/lib/profile/InstrProfilingBiasVar.c @@ -0,0 +1,15 @@ +/*===- InstrProfilingBiasVar.c - profile counter bias variable setup ------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" + +/* The runtime should only provide its own definition of this symbol when the + * user has not specified one. Set this up by moving the runtime's copy of this + * symbol to an object file within the archive. + */ +COMPILER_RT_WEAK intptr_t __llvm_profile_counter_bias = -1; Index: compiler-rt/lib/profile/InstrProfilingFile.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingFile.c +++ compiler-rt/lib/profile/InstrProfilingFile.c @@ -94,12 +94,15 @@ ProfileMergeRequested = EnableMerge; } +static int RuntimeCounterRelocation = 0; static FILE *ProfileFile = NULL; static FILE *getProfileFile() { return ProfileFile; } static void setProfileFile(FILE *File) { ProfileFile = File; } COMPILER_RT_VISIBILITY void __llvm_profile_set_file_object(FILE *File, int EnableMerge) { + if (RuntimeCounterRelocation) + return; setProfileFile(File); setProfileMergeRequested(EnableMerge); } @@ -164,52 +167,83 @@ } } -/* Read profile data in \c ProfileFile and merge with in-memory - profile counters. Returns -1 if there is fatal error, otheriwse - 0 is returned. Returning 0 does not mean merge is actually - performed. If merge is actually done, *MergeDone is set to 1. -*/ -static int doProfileMerging(FILE *ProfileFile, int *MergeDone) { - uint64_t ProfileFileSize; - char *ProfileBuffer; - +/// Get the size of the profile file. If there are any errors, print the +/// message under the assumption that the profile is being read for merging +/// purposes, and return -1. Otherwise return the file size in the inout param +/// \p ProfileFileSize. +static int getProfileFileSizeForMerging(FILE *ProfileFile, + uint64_t *ProfileFileSize) { if (fseek(ProfileFile, 0L, SEEK_END) == -1) { PROF_ERR("Unable to merge profile data, unable to get size: %s\n", strerror(errno)); return -1; } - ProfileFileSize = ftell(ProfileFile); + *ProfileFileSize = ftell(ProfileFile); - /* Restore file offset. */ if (fseek(ProfileFile, 0L, SEEK_SET) == -1) { PROF_ERR("Unable to merge profile data, unable to rewind: %s\n", strerror(errno)); return -1; } - /* Nothing to merge. */ - if (ProfileFileSize < sizeof(__llvm_profile_header)) { - if (ProfileFileSize) - PROF_WARN("Unable to merge profile data: %s\n", - "source profile file is too small."); - return 0; + if (*ProfileFileSize > 0 && + *ProfileFileSize < sizeof(__llvm_profile_header)) { + + PROF_WARN("Unable to merge profile data: %s\n", + "source profile file is too small."); + return -1; } - ProfileBuffer = mmap(NULL, ProfileFileSize, PROT_READ, MAP_SHARED | MAP_FILE, - fileno(ProfileFile), 0); - if (ProfileBuffer == MAP_FAILED) { + return 0; +} + +/// mmap() \p ProfileFile for profile merging purposes, assuming that an +/// exclusive lock is held on the file and that \p ProfileFileSize is the +/// length of the file. Return the mmap'd buffer in the inout variable +/// \p ProfileBuffer. Returns -1 on failure. On success, the caller is +/// responsible for unmapping the mmap'd buffer in \p ProfileBuffer. +static int mmapProfileForMerging(FILE *ProfileFile, uint64_t ProfileFileSize, + char **ProfileBuffer) { + *ProfileBuffer = mmap(NULL, ProfileFileSize, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FILE, fileno(ProfileFile), 0); + if (*ProfileBuffer == MAP_FAILED) { PROF_ERR("Unable to merge profile data, mmap failed: %s\n", strerror(errno)); return -1; } - if (__llvm_profile_check_compatibility(ProfileBuffer, ProfileFileSize)) { + if (__llvm_profile_check_compatibility(*ProfileBuffer, ProfileFileSize)) { (void)munmap(ProfileBuffer, ProfileFileSize); PROF_WARN("Unable to merge profile data: %s\n", "source profile file is not compatible."); - return 0; + return -1; } + return 0; +} + +/* Read profile data in \c ProfileFile and merge with in-memory + profile counters. Returns -1 if there is fatal error, otheriwse + 0 is returned. Returning 0 does not mean merge is actually + performed. If merge is actually done, *MergeDone is set to 1. +*/ +static int doProfileMerging(FILE *ProfileFile, int *MergeDone) { + uint64_t ProfileFileSize; + char *ProfileBuffer; + + /* Get the size of the profile on disk. */ + if (getProfileFileSizeForMerging(ProfileFile, &ProfileFileSize) == -1) + return -1; + + /* Nothing to merge. */ + if (!ProfileFileSize) + return 0; + + /* mmap() the profile and check that it is compatible with the data in + * the current image. */ + if (mmapProfileForMerging(ProfileFile, ProfileFileSize, &ProfileBuffer) == -1) + return -1; + /* Now start merging */ __llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize); @@ -356,6 +390,98 @@ fclose(File); } +static int writeMMappedFile(FILE *OutputFile, char **Profile) { + if (!OutputFile) + return -1; + + /* Write the data into a file. */ + setupIOBuffer(); + ProfDataWriter fileWriter; + initFileWriter(&fileWriter, OutputFile); + if (lprofWriteData(&fileWriter, NULL, 0)) { + PROF_ERR("Failed to write profile: %s\n", strerror(errno)); + return -1; + } + fflush(OutputFile); + + /* Get the file size. */ + uint64_t FileSize = ftell(OutputFile); + + /* Map the profile. */ + *Profile = (char *)mmap( + NULL, FileSize, PROT_READ | PROT_WRITE, MAP_SHARED, fileno(OutputFile), 0); + if (*Profile == MAP_FAILED) { + PROF_ERR("Unable to mmap profile: %s\n", strerror(errno)); + return -1; + } + + return 0; +} + +static void initializeProfileForContinuousMode(void) { + if (!RuntimeCounterRelocation) + return; + + /* Get the sizes of various profile data sections. Taken from + * __llvm_profile_get_size_for_buffer(). */ + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + const uint64_t CountersOffset = sizeof(__llvm_profile_header) + + (DataSize * sizeof(__llvm_profile_data)); + + int Length = getCurFilenameLength(); + char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + const char *Filename = getCurFilename(FilenameBuf, 0); + if (!Filename) + return; + + FILE *File = NULL; + char *Profile = NULL; + + if (!doMerging()) { + File = fopen(Filename, "w+b"); + if (!File) + return; + + if (writeMMappedFile(File, &Profile) == -1) { + fclose(File); + return; + } + } else { + File = lprofOpenFileEx(Filename); + if (!File) + return; + + uint64_t ProfileFileSize = 0; + if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) { + lprofUnlockFileHandle(File); + fclose(File); + return; + } + + if (!ProfileFileSize) { + if (writeMMappedFile(File, &Profile) == -1) { + fclose(File); + return; + } + } else { + /* The merged profile has a non-zero length. Check that it is compatible + * with the data in this process. */ + if (mmapProfileForMerging(File, ProfileFileSize, &Profile) == -1) { + fclose(File); + return; + } + } + + lprofUnlockFileHandle(File); + } + + /* Update the profile fields based on the current mapping. */ + __llvm_profile_counter_bias = (intptr_t)Profile - + (uintptr_t)__llvm_profile_begin_counters() + CountersOffset; +} + static const char *DefaultProfileName = "default.profraw"; static void resetFilenameToDefault(void) { if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) { @@ -447,6 +573,7 @@ const char *OldFilenamePat = lprofCurFilename.FilenamePat; ProfileNameSpecifier OldPNS = lprofCurFilename.PNS; + /* The old profile name specifier takes precedence over the old one. */ if (PNS < OldPNS) return; @@ -475,6 +602,9 @@ } truncateCurrentFile(); + + if (RuntimeCounterRelocation) + initializeProfileForContinuousMode(); } /* Return buffer length that is required to store the current profile @@ -623,6 +753,9 @@ ProfileNameSpecifier PNS = PNS_unknown; int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0); + if (__llvm_profile_counter_bias != -1) + RuntimeCounterRelocation = 1; + EnvFilenamePat = getFilenamePatFromEnv(); if (EnvFilenamePat) { /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid @@ -646,6 +779,8 @@ */ COMPILER_RT_VISIBILITY void __llvm_profile_set_filename(const char *FilenamePat) { + if (RuntimeCounterRelocation) + return; parseAndSetFilename(FilenamePat, PNS_runtime_api, 1); } @@ -660,6 +795,10 @@ char *FilenameBuf; int PDeathSig = 0; + if (RuntimeCounterRelocation) { + return 0; + } + if (lprofProfileDumped()) { PROF_NOTE("Profile data not written to file: %s.\n", "already written"); return 0; @@ -776,6 +915,9 @@ if (HasBeenRegistered) return 0; + if (RuntimeCounterRelocation) + return 0; + lprofSetupValueProfiler(); HasBeenRegistered = 1; Index: llvm/include/llvm/ProfileData/InstrProf.h =================================================================== --- llvm/include/llvm/ProfileData/InstrProf.h +++ llvm/include/llvm/ProfileData/InstrProf.h @@ -153,6 +153,10 @@ return "__llvm_profile_runtime_user"; } +inline StringRef getInstrProfCounterBiasVarName() { + return "__llvm_profile_counter_bias"; +} + /// Return the marker used to separate PGO names during serialization. inline StringRef getInstrProfNameSeparator() { return "\01"; } Index: llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp =================================================================== --- llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -82,6 +82,11 @@ cl::desc("Rename counter variable of a comdat function based on cfg hash"), cl::init(true)); +cl::opt RuntimeCounterRelocation( + "runtime-counter-relocation", + cl::desc("Enable relocating counters at runtime."), + cl::init(false)); + cl::opt ValueProfileStaticAlloc( "vp-static-alloc", cl::desc("Do static counter allocation for value profiler"), @@ -407,6 +412,20 @@ } bool InstrProfiling::lowerIntrinsics(Function *F) { + Value *Load = nullptr; + if (RuntimeCounterRelocation) { + if (!F->empty()) { + IRBuilder<> Builder(&F->getEntryBlock().front()); + Type *Int64Ty = Type::getInt64Ty(M->getContext()); + GlobalVariable *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName()); + if (!Bias) + Bias = new GlobalVariable(*M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage, + Constant::getNullValue(Int64Ty), + getInstrProfCounterBiasVarName()); + Load = Builder.CreateLoad(Int64Ty, Bias); + } + } + bool MadeChange = false; PromotionCandidates.clear(); for (BasicBlock &BB : *F) { @@ -423,8 +442,11 @@ } } - if (!MadeChange) + if (!MadeChange) { + if (RuntimeCounterRelocation && Load) + Load->deleteValue(); return false; + } promoteCounterLoadStores(F); return true; @@ -640,6 +662,17 @@ Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(), Counters, 0, Index); + if (RuntimeCounterRelocation) { + Type *Int64Ty = Type::getInt64Ty(M->getContext()); + Type *Int64PtrTy = Type::getInt64PtrTy(M->getContext()); + Function *Fn = Inc->getParent()->getParent(); + Instruction &I = Fn->getEntryBlock().front(); + if (LoadInst *LI = dyn_cast(&I)) { + auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), LI); + Addr = Builder.CreateIntToPtr(Add, Int64PtrTy); + } + } + if (Options.Atomic || AtomicCounterUpdateAll) { Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(), AtomicOrdering::Monotonic);