Index: compiler-rt/lib/xray/CMakeLists.txt =================================================================== --- compiler-rt/lib/xray/CMakeLists.txt +++ compiler-rt/lib/xray/CMakeLists.txt @@ -18,6 +18,7 @@ set(XRAY_PROFILER_MODE_SOURCES xray_profile_collector.cc + xray_profiler.cc xray_profiler_flags.cc) # Implementation files for all XRay architectures. @@ -195,13 +196,14 @@ DEFS ${XRAY_COMMON_DEFINITIONS} OBJECT_LIBS RTXrayBASIC PARENT_TARGET xray) - add_compiler_rt_runtime(clang_rt.xray-profiler - STATIC - ARCHS ${arch} - CFLAGS ${XRAY_CFLAGS} - DEFS ${XRAY_COMMON_DEFINITIONS} - OBJECT_LIBS RTXrayPROFILER - PARENT_TARGET xray) + # Profiler Mode runtime + add_compiler_rt_runtime(clang_rt.xray-profiler + STATIC + ARCHS ${arch} + CFLAGS ${XRAY_CFLAGS} + DEFS ${XRAY_COMMON_DEFINITIONS} + OBJECT_LIBS RTXrayPROFILER + PARENT_TARGET xray) endforeach() endif() # not Apple Index: compiler-rt/lib/xray/xray_profiler.cc =================================================================== --- /dev/null +++ compiler-rt/lib/xray/xray_profiler.cc @@ -0,0 +1,296 @@ +//===-- xray_profiling.cc --------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of XRay, a dynamic runtime instrumentation system. +// +// This is the implementation of a profiling handler. +// +//===----------------------------------------------------------------------===// +#include + +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_flags.h" +#include "xray/xray_interface.h" +#include "xray/xray_log_interface.h" + +#include "xray_flags.h" +#include "xray_profile_collector.h" +#include "xray_profiler_flags.h" +#include "xray_tsc.h" +#include "xray_utils.h" +#include + +namespace __xray { + +namespace { + +__sanitizer::atomic_sint32_t ProfilerLogFlushStatus = { + XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING}; + +__sanitizer::atomic_sint32_t ProfilerLogStatus = { + XRayLogInitStatus::XRAY_LOG_UNINITIALIZED}; + +__sanitizer::SpinMutex ProfilerOptionsMutex; + +struct alignas(64) ProfilingTLD { + FunctionCallTrie::Allocators *Allocators = nullptr; + FunctionCallTrie *FCT = nullptr; +}; + +static pthread_key_t ProfilingKey; + +ProfilingTLD &getThreadLocalData() XRAY_NEVER_INSTRUMENT { + thread_local ProfilingTLD TLD; + thread_local bool UNUSED Once = [] { + pthread_setspecific(ProfilingKey, &TLD); + return false; + }(); + + // We need to check whether the global flag to finalizing/finalized has been + // switched. If it is, then we ought to not actually initialise the data. + auto Status = __sanitizer::atomic_load(&ProfilerLogStatus, + __sanitizer::memory_order_acquire); + if (Status == XRayLogInitStatus::XRAY_LOG_FINALIZING || + Status == XRayLogInitStatus::XRAY_LOG_FINALIZED) + return TLD; + + // If we're live, then we re-initialize TLD if the pointers are not null. + if (UNLIKELY(TLD.Allocators == nullptr && TLD.FCT == nullptr)) { + TLD.Allocators = reinterpret_cast( + InternalAlloc(sizeof(FunctionCallTrie::Allocators))); + new (TLD.Allocators) FunctionCallTrie::Allocators(); + *TLD.Allocators = FunctionCallTrie::InitAllocators(); + TLD.FCT = reinterpret_cast( + InternalAlloc(sizeof(FunctionCallTrie))); + new (TLD.FCT) FunctionCallTrie(*TLD.Allocators); + } + + return TLD; +} + +} // namespace + +const char *profilerCompilerDefinedFlags() XRAY_NEVER_INSTRUMENT { +#ifdef XRAY_PROFILER_DEFAULT_OPTIONS + return SANITIZER_STRINGIFY(XRAY_PROFILER_DEFAULT_OPTIONS); +#else + return ""; +#endif +} + +__sanitizer::atomic_sint32_t ProfileFlushStatus = { + XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING}; + +XRayLogFlushStatus profilingFlush() XRAY_NEVER_INSTRUMENT { + // When flushing, all we really do is reset the global state, and only when + // the log has already been finalized. + if (__sanitizer::atomic_load(&ProfilerLogStatus, + __sanitizer::memory_order_acquire) != + XRayLogInitStatus::XRAY_LOG_FINALIZED) { + if (__sanitizer::Verbosity()) + Report("Not flushing profiles, profiler not been finalized.\n"); + return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; + } + + s32 Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; + if (!__sanitizer::atomic_compare_exchange_strong( + &ProfilerLogFlushStatus, &Result, + XRayLogFlushStatus::XRAY_LOG_FLUSHING, + __sanitizer::memory_order_release)) { + if (__sanitizer::Verbosity()) + Report("Not flushing profiles, implementation still finalizing.\n"); + } + + profileCollectorService::reset(); + + __sanitizer::atomic_store(&ProfilerLogStatus, + XRayLogFlushStatus::XRAY_LOG_FLUSHED, + __sanitizer::memory_order_release); + + return XRayLogFlushStatus::XRAY_LOG_FLUSHED; +} + +namespace { + +thread_local volatile bool ReentranceGuard = false; + +void postCurrentThreadFCT(ProfilingTLD &TLD) { + if (TLD.Allocators == nullptr || TLD.FCT == nullptr) + return; + + profileCollectorService::post(*TLD.FCT, GetTid()); + TLD.FCT->~FunctionCallTrie(); + TLD.Allocators->~Allocators(); + InternalFree(TLD.FCT); + InternalFree(TLD.Allocators); + TLD.FCT = nullptr; + TLD.Allocators = nullptr; +} + +} // namespace + +void profilingHandleArg0(int32_t FuncId, + XRayEntryType Entry) XRAY_NEVER_INSTRUMENT { + unsigned char CPU; + auto TSC = readTSC(CPU); + if (ReentranceGuard) + return; + ReentranceGuard = true; + + auto Status = __sanitizer::atomic_load(&ProfilerLogStatus, + __sanitizer::memory_order_acquire); + auto &TLD = getThreadLocalData(); + if (UNLIKELY(Status == XRayLogInitStatus::XRAY_LOG_FINALIZED || + Status == XRayLogInitStatus::XRAY_LOG_FINALIZING)) { + postCurrentThreadFCT(TLD); + ReentranceGuard = false; + return; + } + + switch (Entry) { + case XRayEntryType::ENTRY: + case XRayEntryType::LOG_ARGS_ENTRY: + TLD.FCT->enterFunction(FuncId, TSC); + break; + case XRayEntryType::EXIT: + case XRayEntryType::TAIL: + TLD.FCT->exitFunction(FuncId, TSC); + break; + default: + // FIXME: Handle bugs. + break; + } + + ReentranceGuard = false; +} + +void profilingHandleArg1(int32_t FuncId, XRayEntryType Entry, + uint64_t) XRAY_NEVER_INSTRUMENT { + return profilingHandleArg0(FuncId, Entry); +} + +XRayLogInitStatus profilingFinalize() XRAY_NEVER_INSTRUMENT { + s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_INITIALIZED; + if (!__sanitizer::atomic_compare_exchange_strong( + &ProfilerLogStatus, &CurrentStatus, + XRayLogInitStatus::XRAY_LOG_FINALIZING, + __sanitizer::memory_order_release)) { + if (__sanitizer::Verbosity()) + Report("Cannot finalize profile, the profiler is not initialized.\n"); + return static_cast(CurrentStatus); + } + + // Wait a grace period to allow threads to see that we're finalizing. + __sanitizer::SleepForMillis(profilerFlags()->xray_profiling_grace_period_ms); + + // We also want to make sure that the current thread's data is cleaned up, if + // we have any. + auto &TLD = getThreadLocalData(); + postCurrentThreadFCT(TLD); + + // Then we force serialize the log data. + profileCollectorService::serialize(); + + __sanitizer::atomic_store(&ProfilerLogStatus, + XRayLogInitStatus::XRAY_LOG_FINALIZED, + __sanitizer::memory_order_release); + return XRayLogInitStatus::XRAY_LOG_FINALIZED; +} + +XRayLogInitStatus +profilingLoggingInit(size_t BufferSize, size_t BufferMax, void *Options, + size_t OptionsSize) XRAY_NEVER_INSTRUMENT { + s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; + if (!__sanitizer::atomic_compare_exchange_strong( + &ProfilerLogStatus, &CurrentStatus, + XRayLogInitStatus::XRAY_LOG_INITIALIZING, + __sanitizer::memory_order_release)) { + if (__sanitizer::Verbosity()) + Report( + "Cannot initialize already initialised profiling implementation.\n"); + return static_cast(CurrentStatus); + } + + // Here we use the sanitizer flag parsing mechanism. When PR36790 is fixed, + // migrate to using a different API for configuration. + { + SpinMutexLock Lock(&ProfilerOptionsMutex); + FlagParser ConfigParser; + auto *F = profilerFlags(); + F->setDefaults(); + registerProfilerFlags(&ConfigParser, F); + const char *ProfilerCompileFlags = profilerCompilerDefinedFlags(); + ConfigParser.ParseString(ProfilerCompileFlags); + ConfigParser.ParseString(static_cast(Options)); + if (Verbosity()) + ReportUnrecognizedFlags(); + } + + // We need to reset the profile data collection implementation now. + profileCollectorService::reset(); + + // We need to set up the at-thread-exit handler. + static bool UNUSED Once = [] { + pthread_key_create(&ProfilingKey, +[](void *) { + // This is the thread-exit handler. + auto &TLD = getThreadLocalData(); + if (TLD.Allocators == nullptr && TLD.FCT == nullptr) + return; + + postCurrentThreadFCT(TLD); + }); + return false; + }(); + + __xray_log_set_buffer_iterator(profileCollectorService::nextBuffer); + __xray_set_handler(profilingHandleArg0); + __xray_set_handler_arg1(profilingHandleArg1); + + __sanitizer::atomic_store(&ProfilerLogStatus, + XRayLogInitStatus::XRAY_LOG_INITIALIZED, + __sanitizer::memory_order_release); + if (__sanitizer::Verbosity()) + Report("XRay Profiling init successful.\n"); + + return XRayLogInitStatus::XRAY_LOG_INITIALIZED; +} + +bool profilingDynamicInitializer() XRAY_NEVER_INSTRUMENT { + // Set up the flag defaults from the static defaults and the compiler-provided + // defaults. + { + SpinMutexLock Lock(&ProfilerOptionsMutex); + auto *F = profilerFlags(); + F->setDefaults(); + FlagParser ProfilingParser; + registerProfilerFlags(&ProfilingParser, F); + const char *ProfilerCompileFlags = profilerCompilerDefinedFlags(); + ProfilingParser.ParseString(ProfilerCompileFlags); + } + + XRayLogImpl Impl{ + profilingLoggingInit, + profilingFinalize, + profilingHandleArg0, + profilingFlush, + }; + auto RegistrationResult = __xray_log_register_mode("xray-profiling", Impl); + if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK && + __sanitizer::Verbosity()) + Report( + "Cannot register XRay Profiling mode to 'xray-profiling'; error = %d\n", + RegistrationResult); + if (!__sanitizer::internal_strcmp(flags()->xray_mode, "xray-profiling")) + __xray_set_log_impl(Impl); + return true; +} + +} // namespace __xray + +static auto UNUSED Unused = __xray::profilingDynamicInitializer(); Index: compiler-rt/test/xray/TestCases/Posix/profiling-multi-threaded.cc =================================================================== --- /dev/null +++ compiler-rt/test/xray/TestCases/Posix/profiling-multi-threaded.cc @@ -0,0 +1,53 @@ +// Check that we can get a profile from a single-threaded application, on +// demand through the XRay logging implementation API. +// +// FIXME: Make -fxray-modes=xray-profiling part of the default? +// RUN: %clangxx_xray -std=c++11 %s -o %t -fxray-modes=xray-profiler +// RUN: %run %t +// +// UNSUPPORTED: target-is-mips64,target-is-mips64el + +#include "xray/xray_interface.h" +#include "xray/xray_log_interface.h" +#include +#include +#include +#include + +#define XRAY_ALWAYS_INSTRUMENT [[clang::xray_always_instrument]] +#define XRAY_NEVER_INSTRUMENT [[clang::xray_never_instrument]] + +XRAY_ALWAYS_INSTRUMENT void f2() { return; } +XRAY_ALWAYS_INSTRUMENT void f1() { f2(); } +XRAY_ALWAYS_INSTRUMENT void f0() { f1(); } + +using namespace std; + +volatile int buffer_counter = 0; + +XRAY_NEVER_INSTRUMENT void process_buffer(const char *, XRayBuffer) { + // FIXME: Actually assert the contents of the buffer. + ++buffer_counter; +} + +XRAY_ALWAYS_INSTRUMENT int main(int, char **) { + assert(__xray_log_select_mode("xray-profiling") == + XRayLogRegisterStatus::XRAY_REGISTRATION_OK); + assert(__xray_log_get_current_mode() != nullptr); + std::string current_mode = __xray_log_get_current_mode(); + assert(current_mode == "xray-profiling"); + assert(__xray_patch() == XRayPatchingStatus::SUCCESS); + assert(__xray_log_init(0, 0, nullptr, 0) == + XRayLogInitStatus::XRAY_LOG_INITIALIZED); + std::thread t0([] { f0(); }); + std::thread t1([] { f0(); }); + f0(); + t0.join(); + t1.join(); + assert(__xray_log_finalize() == XRayLogInitStatus::XRAY_LOG_FINALIZED); + assert(__xray_log_process_buffers(process_buffer) == + XRayLogFlushStatus::XRAY_LOG_FLUSHED); + // We're running three threds, so we expect three buffers. + assert(buffer_counter == 3); + assert(__xray_log_flushLog() == XRayLogFlushStatus::XRAY_LOG_FLUSHED); +} Index: compiler-rt/test/xray/TestCases/Posix/profiling-single-threaded.cc =================================================================== --- /dev/null +++ compiler-rt/test/xray/TestCases/Posix/profiling-single-threaded.cc @@ -0,0 +1,48 @@ +// Check that we can get a profile from a single-threaded application, on +// demand through the XRay logging implementation API. +// +// FIXME: Make -fxray-modes=xray-profiling part of the default? +// RUN: %clangxx_xray -std=c++11 %s -o %t -fxray-modes=xray-profiler +// RUN: %run %t +// +// UNSUPPORTED: target-is-mips64,target-is-mips64el + +#include "xray/xray_interface.h" +#include "xray/xray_log_interface.h" +#include +#include +#include + +#define ALWAYS_INSTRUMENT [[clang::xray_always_instrument]] +#define NEVER_INSTRUMENT [[clang::xray_never_instrument]] + +ALWAYS_INSTRUMENT void f2() { return; } +ALWAYS_INSTRUMENT void f1() { f2(); } +ALWAYS_INSTRUMENT void f0() { f1(); } + +using namespace std; + +volatile int buffer_counter = 0; + +NEVER_INSTRUMENT void process_buffer(const char *, XRayBuffer) { + // FIXME: Actually assert the contents of the buffer. + ++buffer_counter; +} + +ALWAYS_INSTRUMENT int main(int, char **) { + assert(__xray_log_select_mode("xray-profiling") == + XRayLogRegisterStatus::XRAY_REGISTRATION_OK); + assert(__xray_log_get_current_mode() != nullptr); + std::string current_mode = __xray_log_get_current_mode(); + assert(current_mode == "xray-profiling"); + assert(__xray_patch() == XRayPatchingStatus::SUCCESS); + assert(__xray_log_init(0, 0, nullptr, 0) == + XRayLogInitStatus::XRAY_LOG_INITIALIZED); + f0(); + assert(__xray_log_finalize() == XRayLogInitStatus::XRAY_LOG_FINALIZED); + f0(); + assert(__xray_log_process_buffers(process_buffer) == + XRayLogFlushStatus::XRAY_LOG_FLUSHED); + assert(buffer_counter == 1); + assert(__xray_log_flushLog() == XRayLogFlushStatus::XRAY_LOG_FLUSHED); +}