Index: compiler-rt/lib/profile/InstrProfData.inc =================================================================== --- compiler-rt/lib/profile/InstrProfData.inc +++ compiler-rt/lib/profile/InstrProfData.inc @@ -75,7 +75,7 @@ INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ Inc->getHash()->getZExtValue())) -INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ +INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ ConstantExpr::getBitCast(CounterPtr, \ llvm::Type::getInt64PtrTy(Ctx))) /* This is used to map function pointers for the indirect call targets to Index: compiler-rt/lib/profile/InstrProfilingFile.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingFile.c +++ compiler-rt/lib/profile/InstrProfilingFile.c @@ -88,6 +88,7 @@ static lprofFilename lprofCurFilename = {0, 0, 0, {0}, {0}, 0, 0, 0, PNS_unknown}; +static int RuntimeAllocatedCounters = 0; static int ProfileMergeRequested = 0; static int isProfileMergeRequested() { return ProfileMergeRequested; } static void setProfileMergeRequested(int EnableMerge) { @@ -100,6 +101,8 @@ COMPILER_RT_VISIBILITY void __llvm_profile_set_file_object(FILE *File, int EnableMerge) { + if (RuntimeAllocatedCounters) + return; setProfileFile(File); setProfileMergeRequested(EnableMerge); } @@ -164,52 +167,82 @@ } } -/* Read profile data in \c ProfileFile and merge with in-memory - profile counters. Returns -1 if there is fatal error, otheriwse - 0 is returned. Returning 0 does not mean merge is actually - performed. If merge is actually done, *MergeDone is set to 1. -*/ -static int doProfileMerging(FILE *ProfileFile, int *MergeDone) { - uint64_t ProfileFileSize; - char *ProfileBuffer; - +/// Get the size of the profile file. If there are any errors, print the +/// message under the assumption that the profile is being read for merging +/// purposes, and return -1. Otherwise return the file size in the inout param +/// \p ProfileFileSize. +static int getProfileFileSizeForMerging(FILE *ProfileFile, + uint64_t *ProfileFileSize) { if (fseek(ProfileFile, 0L, SEEK_END) == -1) { PROF_ERR("Unable to merge profile data, unable to get size: %s\n", strerror(errno)); return -1; } - ProfileFileSize = ftell(ProfileFile); + *ProfileFileSize = ftell(ProfileFile); - /* Restore file offset. */ if (fseek(ProfileFile, 0L, SEEK_SET) == -1) { PROF_ERR("Unable to merge profile data, unable to rewind: %s\n", strerror(errno)); return -1; } - /* Nothing to merge. */ - if (ProfileFileSize < sizeof(__llvm_profile_header)) { - if (ProfileFileSize) - PROF_WARN("Unable to merge profile data: %s\n", - "source profile file is too small."); - return 0; + if (*ProfileFileSize > 0 && + *ProfileFileSize < sizeof(__llvm_profile_header)) { + PROF_WARN("Unable to merge profile data: %s\n", + "source profile file is too small."); + return -1; } - ProfileBuffer = mmap(NULL, ProfileFileSize, PROT_READ, MAP_SHARED | MAP_FILE, - fileno(ProfileFile), 0); - if (ProfileBuffer == MAP_FAILED) { + return 0; +} + +/// mmap() \p ProfileFile for profile merging purposes, assuming that an +/// exclusive lock is held on the file and that \p ProfileFileSize is the +/// length of the file. Return the mmap'd buffer in the inout variable +/// \p ProfileBuffer. Returns -1 on failure. On success, the caller is +/// responsible for unmapping the mmap'd buffer in \p ProfileBuffer. +static int mmapProfileForMerging(FILE *ProfileFile, uint64_t ProfileFileSize, + char **ProfileBuffer) { + *ProfileBuffer = mmap(NULL, ProfileFileSize, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FILE, fileno(ProfileFile), 0); + if (*ProfileBuffer == MAP_FAILED) { PROF_ERR("Unable to merge profile data, mmap failed: %s\n", strerror(errno)); return -1; } - if (__llvm_profile_check_compatibility(ProfileBuffer, ProfileFileSize)) { + if (__llvm_profile_check_compatibility(*ProfileBuffer, ProfileFileSize)) { (void)munmap(ProfileBuffer, ProfileFileSize); PROF_WARN("Unable to merge profile data: %s\n", "source profile file is not compatible."); - return 0; + return -1; } + return 0; +} + +/* Read profile data in \c ProfileFile and merge with in-memory + profile counters. Returns -1 if there is fatal error, otheriwse + 0 is returned. Returning 0 does not mean merge is actually + performed. If merge is actually done, *MergeDone is set to 1. +*/ +static int doProfileMerging(FILE *ProfileFile, int *MergeDone) { + uint64_t ProfileFileSize; + char *ProfileBuffer; + + /* Get the size of the profile on disk. */ + if (getProfileFileSizeForMerging(ProfileFile, &ProfileFileSize) == -1) + return -1; + + /* Nothing to merge. */ + if (!ProfileFileSize) + return 0; + + /* mmap() the profile and check that it is compatible with the data in + * the current image. */ + if (mmapProfileForMerging(ProfileFile, ProfileFileSize, &ProfileBuffer) == -1) + return -1; + /* Now start merging */ __llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize); @@ -356,6 +389,105 @@ fclose(File); } +static int createProfileWithAllocatedCounters(FILE *OutputFile) { + /* Get the sizes of various profile data sections. Taken from + * __llvm_profile_get_size_for_buffer(). */ + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + + setupIOBuffer(); + ProfDataWriter fileWriter; + initFileWriter(&fileWriter, OutputFile); + if (lprofWriteData(&fileWriter, NULL, 0)) { + PROF_ERR("Failed to write profile: %s\n", strerror(errno)); + return -1; + } + fflush(OutputFile); + + uint64_t FileSize = ftell(OutputFile); + + char *ProfileBuffer = mmap(NULL, FileSize, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FILE, fileno(OutputFile), 0); + if (ProfileBuffer == MAP_FAILED) { + PROF_ERR("Unable to mmap profile: %s\n", strerror(errno)); + return -1; + } + + /* Update the profile fields based on the current mapping. */ + __llvm_profile_header *Header = (__llvm_profile_header *)ProfileBuffer; + __llvm_profile_data *Data = (__llvm_profile_data *)( + ProfileBuffer + sizeof(__llvm_profile_header)); + uint64_t *Counters = (uint64_t *)(ProfileBuffer + + sizeof(__llvm_profile_header) + sizeof(__llvm_profile_data) * DataSize); + Header->CountersDelta = (uint64_t)Counters; + for (const __llvm_profile_data *DI = DataBegin; DI != DataEnd; DI++) { + ((__llvm_profile_data *)DI)->CounterPtr = Counters; + Data->CounterPtr = Counters; + Data++; + Counters += DI->NumCounters; + } + + return 0; +} + +static void allocateCounters(void) { + /* Get the sizes of various profile data sections. Taken from + * __llvm_profile_get_size_for_buffer(). */ + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + + /* Open the raw profile in append mode. */ + int Length = getCurFilenameLength(); + char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + const char *Filename = getCurFilename(FilenameBuf, 0); + if (!Filename) + return; + + FILE *File = NULL; + char *ProfileBuffer = NULL; + + if (!doMerging()) { + File = lprofOpenFile(Filename); + if (!File) + return; + + createProfileWithAllocatedCounters(File); + } else { + File = lprofOpenFileEx(Filename); + if (!File) + return; + + uint64_t ProfileFileSize = 0; + if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) { + lprofUnlockFileHandle(File); + return; + } + + if (!ProfileFileSize) { + createProfileWithAllocatedCounters(File); + } else { + /* The merged profile has a non-zero length. Check that it is compatible + * with the data in this process. */ + if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1) { + lprofUnlockFileHandle(File); + return; + } + + /* Update the profile fields based on the current mapping. */ + uint64_t *Counters = (uint64_t *)(ProfileBuffer + + sizeof(__llvm_profile_header) + sizeof(__llvm_profile_data) * DataSize); + for (const __llvm_profile_data *DI = DataBegin; DI != DataEnd; DI++) { + ((__llvm_profile_data *)DI)->CounterPtr = Counters; + Counters += DI->NumCounters; + } + } + + lprofUnlockFileHandle(File); + } +} + static const char *DefaultProfileName = "default.profraw"; static void resetFilenameToDefault(void) { if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) { @@ -447,6 +579,7 @@ const char *OldFilenamePat = lprofCurFilename.FilenamePat; ProfileNameSpecifier OldPNS = lprofCurFilename.PNS; + /* The old profile name specifier takes precedence over the old one. */ if (PNS < OldPNS) return; @@ -475,6 +608,9 @@ } truncateCurrentFile(); + + if (RuntimeAllocatedCounters) + allocateCounters(); } /* Return buffer length that is required to store the current profile @@ -623,6 +759,12 @@ ProfileNameSpecifier PNS = PNS_unknown; int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0); + // TODO: This is potentially error prone, maybe we should extend the + // header and include a field that indicates whether to allocate counters. + if (__llvm_profile_begin_data() != __llvm_profile_end_data() && + __llvm_profile_begin_counters() == __llvm_profile_end_counters()) + RuntimeAllocatedCounters = 1; + EnvFilenamePat = getFilenamePatFromEnv(); if (EnvFilenamePat) { /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid @@ -646,6 +788,8 @@ */ COMPILER_RT_VISIBILITY void __llvm_profile_set_filename(const char *FilenamePat) { + if (RuntimeAllocatedCounters) + return; parseAndSetFilename(FilenamePat, PNS_runtime_api, 1); } @@ -660,7 +804,7 @@ char *FilenameBuf; int PDeathSig = 0; - if (lprofProfileDumped()) { + if (lprofProfileDumped() || RuntimeAllocatedCounters) { PROF_NOTE("Profile data not written to file: %s.\n", "already written"); return 0; } @@ -776,6 +920,9 @@ if (HasBeenRegistered) return 0; + if (RuntimeAllocatedCounters) + return 0; + lprofSetupValueProfiler(); HasBeenRegistered = 1; Index: compiler-rt/lib/profile/InstrProfilingMerge.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingMerge.c +++ compiler-rt/lib/profile/InstrProfilingMerge.c @@ -56,13 +56,16 @@ Header->DataSize != __llvm_profile_get_data_size(__llvm_profile_begin_data(), __llvm_profile_end_data()) || - Header->CountersSize != (uint64_t)(__llvm_profile_end_counters() - - __llvm_profile_begin_counters()) || Header->NamesSize != (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names()) || Header->ValueKindLast != IPVK_Last) return 1; + if (__llvm_profile_begin_counters() != __llvm_profile_end_counters() && + Header->CountersSize != (uint64_t)(__llvm_profile_end_counters() - + __llvm_profile_begin_counters())) + return 1; + if (ProfileSize < sizeof(__llvm_profile_header) + Header->DataSize * sizeof(__llvm_profile_data) + Header->NamesSize + Header->CountersSize) Index: compiler-rt/lib/profile/InstrProfilingRuntime.cpp =================================================================== --- compiler-rt/lib/profile/InstrProfilingRuntime.cpp +++ compiler-rt/lib/profile/InstrProfilingRuntime.cpp @@ -19,11 +19,11 @@ class RegisterRuntime { public: RegisterRuntime() { - __llvm_profile_register_write_file_atexit(); __llvm_profile_initialize_file(); + __llvm_profile_register_write_file_atexit(); } }; -RegisterRuntime Registration; +RegisterRuntime Registration __attribute__((init_priority(101))); } Index: compiler-rt/lib/profile/InstrProfilingWriter.c =================================================================== --- compiler-rt/lib/profile/InstrProfilingWriter.c +++ compiler-rt/lib/profile/InstrProfilingWriter.c @@ -252,16 +252,20 @@ const uint64_t *CountersBegin, const uint64_t *CountersEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, const char *NamesEnd, int SkipNameDataWrite) { - /* Calculate size of sections. */ const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); - const uint64_t CountersSize = CountersEnd - CountersBegin; + uint64_t CountersSize = 0; + int SkipCounterDataWrite = 0; + if (CountersBegin != CountersEnd) { + CountersSize = CountersEnd - CountersBegin; + } else { + for (const __llvm_profile_data *Data = DataBegin; Data != DataEnd; Data++) + CountersSize += Data->NumCounters; + SkipCounterDataWrite = 1; + } const uint64_t NamesSize = NamesEnd - NamesBegin; const uint64_t Padding = __llvm_profile_get_num_padding_bytes(NamesSize); - /* Enough zeroes for padding. */ - const char Zeroes[sizeof(uint64_t)] = {0}; - /* Create the header. */ __llvm_profile_header Header; @@ -272,15 +276,22 @@ #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init; #include "InstrProfData.inc" + /* Enough zeroes for padding. */ + const char Zeroes[sizeof(uint64_t)] = {0}; + /* Write the data. */ ProfDataIOVec IOVec[] = { {&Header, sizeof(__llvm_profile_header), 1}, {DataBegin, sizeof(__llvm_profile_data), DataSize}, - {CountersBegin, sizeof(uint64_t), CountersSize}, + {SkipCounterDataWrite ? NULL : CountersBegin, sizeof(uint64_t), CountersSize}, {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize}, {Zeroes, sizeof(uint8_t), Padding}}; if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec))) return -1; + /* Value profiling is not yet supported when writing counters directly. */ + if (SkipCounterDataWrite) + return 0; + return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd); } Index: compiler-rt/test/profile/instrprof-basic.c =================================================================== --- compiler-rt/test/profile/instrprof-basic.c +++ compiler-rt/test/profile/instrprof-basic.c @@ -41,6 +41,12 @@ // RUN: rm -f %t.dir4/merge4 // RUN: llvm-profdata merge -o %t.m4.profdata ./ // RUN: %clang_profuse=%t.m4.profdata -O0 -o - -S -emit-llvm %s | FileCheck %s --check-prefix=COMMON --check-prefix=PGOMERGE +// +// Test runtime allocated counters +// RUN: %clang_profgen -o %t -O3 -mllvm -counter-static-alloc=false %s +// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t +// RUN: llvm-profdata merge -o %t.profdata %t.profraw +// RUN: %clang_profuse=%t.profdata -o - -S -emit-llvm %s | FileCheck %s --check-prefix=COMMON --check-prefix=RUNTIME int begin(int i) { // COMMON: br i1 %{{.*}}, label %{{.*}}, label %{{.*}}, !prof ![[PD1:[0-9]+]] Index: llvm/include/llvm/ProfileData/InstrProfData.inc =================================================================== --- llvm/include/llvm/ProfileData/InstrProfData.inc +++ llvm/include/llvm/ProfileData/InstrProfData.inc @@ -75,9 +75,8 @@ INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ Inc->getHash()->getZExtValue())) -INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ - ConstantExpr::getBitCast(CounterPtr, \ - llvm::Type::getInt64PtrTy(Ctx))) +INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ + CounterPtr) /* This is used to map function pointers for the indirect call targets to * function name hashes during the conversion from raw to merged profile * data. Index: llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h =================================================================== --- llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h +++ llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h @@ -49,7 +49,7 @@ std::function GetTLI; struct PerFunctionProfileData { uint32_t NumValueSites[IPVK_Last + 1]; - GlobalVariable *RegionCounters = nullptr; + Value *RegionCounters = nullptr; GlobalVariable *DataVar = nullptr; PerFunctionProfileData() { @@ -101,7 +101,7 @@ /// /// If the counter array doesn't yet exist, the profile data variables /// referring to them will also be created. - GlobalVariable *getOrCreateRegionCounters(InstrProfIncrementInst *Inc); + Value *getOrCreateRegionCounters(InstrProfIncrementInst *Inc); /// Emit the section with compressed function names. void emitNameData(); Index: llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp =================================================================== --- llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -82,6 +82,11 @@ cl::desc("Rename counter variable of a comdat function based on cfg hash"), cl::init(true)); +cl::opt CounterStaticAlloc( + "counter-static-alloc", + cl::desc("Do static counter allocation."), + cl::init(true)); + cl::opt ValueProfileStaticAlloc( "vp-static-alloc", cl::desc("Do static counter allocation for value profiler"), @@ -633,12 +638,13 @@ } void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) { - GlobalVariable *Counters = getOrCreateRegionCounters(Inc); + Value *Counters = getOrCreateRegionCounters(Inc); IRBuilder<> Builder(Inc); + LLVMContext &Ctx = M->getContext(); uint64_t Index = Inc->getIndex()->getZExtValue(); - Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(), - Counters, 0, Index); + Value *Addr = Builder.CreateConstInBoundsGEP1_64(Type::getInt64Ty(Ctx), + Counters, Index); if (Options.Atomic || AtomicCounterUpdateAll) { Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(), @@ -729,7 +735,7 @@ return true; } -GlobalVariable * +Value * InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) { GlobalVariable *NamePtr = Inc->getName(); auto It = ProfileDataMap.find(NamePtr); @@ -775,19 +781,25 @@ uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); LLVMContext &Ctx = M->getContext(); - ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); + auto *Int64PtrTy = Type::getInt64PtrTy(Ctx); // Create the counters variable. - auto *CounterPtr = - new GlobalVariable(*M, CounterTy, false, Linkage, - Constant::getNullValue(CounterTy), - getVarName(Inc, getInstrProfCountersVarPrefix())); - CounterPtr->setVisibility(Visibility); - CounterPtr->setSection( - getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat())); - CounterPtr->setAlignment(Align(8)); - MaybeSetComdat(CounterPtr); - CounterPtr->setLinkage(Linkage); + Constant *CounterPtr = ConstantPointerNull::get(Int64PtrTy); + if (CounterStaticAlloc) { + ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); + auto *CounterVar = + new GlobalVariable(*M, CounterTy, false, Linkage, + Constant::getNullValue(CounterTy), + getVarName(Inc, getInstrProfCountersVarPrefix())); + CounterVar->setVisibility(Visibility); + CounterVar->setSection( + getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat())); + CounterVar->setAlignment(Align(8)); + MaybeSetComdat(CounterVar); + CounterVar->setLinkage(Linkage); + CounterPtr = + ConstantExpr::getBitCast(CounterVar, Type::getInt64PtrTy(Ctx)); + } auto *Int8PtrTy = Type::getInt8PtrTy(Ctx); // Allocate statically the array of pointers to value profile nodes for @@ -844,7 +856,14 @@ MaybeSetComdat(Data); Data->setLinkage(Linkage); - PD.RegionCounters = CounterPtr; + IRBuilder<> Builder(Inc); + + if (CounterStaticAlloc) { + PD.RegionCounters = CounterPtr; + } else { + auto *Addr = Builder.CreateConstInBoundsGEP2_32(DataTy, Data, 0, 2); + PD.RegionCounters = Builder.CreateLoad(llvm::Type::getInt64PtrTy(Ctx), Addr); + } PD.DataVar = Data; ProfileDataMap[NamePtr] = PD;