Index: lib/xray/tests/CMakeLists.txt =================================================================== --- lib/xray/tests/CMakeLists.txt +++ lib/xray/tests/CMakeLists.txt @@ -49,7 +49,7 @@ -lstdc++ -lm ${CMAKE_THREAD_LIBS_INIT} -lpthread -L${COMPILER_RT_LIBRARY_OUTPUT_DIR} -lclang_rt.xray-${arch} - -latomic -ldl -lrt) + -ldl -lrt) endif() # FIXME: Figure out how to run even just the unit tests on APPLE. endforeach() Index: lib/xray/xray_buffer_queue.h =================================================================== --- lib/xray/xray_buffer_queue.h +++ lib/xray/xray_buffer_queue.h @@ -15,10 +15,9 @@ #ifndef XRAY_BUFFER_QUEUE_H #define XRAY_BUFFER_QUEUE_H -#include -#include +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_mutex.h" #include -#include #include #include @@ -42,9 +41,9 @@ // We use a bool to indicate whether the Buffer has been used in this // freelist implementation. std::deque> Buffers; - std::mutex Mutex; + __sanitizer::BlockingMutex Mutex; std::unordered_set OwnedBuffers; - std::atomic Finalizing; + __sanitizer::atomic_uint8_t Finalizing; public: enum class ErrorCode : unsigned { @@ -94,7 +93,10 @@ /// - ... ErrorCode releaseBuffer(Buffer &Buf); - bool finalizing() const { return Finalizing.load(std::memory_order_acquire); } + bool finalizing() const { + return __sanitizer::atomic_load(&Finalizing, + __sanitizer::memory_order_acquire); + } /// Sets the state of the BufferQueue to finalizing, which ensures that: /// @@ -109,7 +111,7 @@ /// Buffer is marked 'used' (i.e. has been the result of getBuffer(...) and a /// releaseBuffer(...) operation. template void apply(F Fn) { - std::lock_guard G(Mutex); + __sanitizer::BlockingMutexLock G(&Mutex); for (const auto &T : Buffers) { if (std::get<1>(T)) Fn(std::get<0>(T)); Index: lib/xray/xray_buffer_queue.cc =================================================================== --- lib/xray/xray_buffer_queue.cc +++ lib/xray/xray_buffer_queue.cc @@ -23,7 +23,7 @@ using namespace __sanitizer; BufferQueue::BufferQueue(std::size_t B, std::size_t N, bool &Success) - : BufferSize(B), Buffers(N), Mutex(), OwnedBuffers(), Finalizing(false) { + : BufferSize(B), Buffers(N), Mutex(), OwnedBuffers(), Finalizing{0} { for (auto &T : Buffers) { void *Tmp = malloc(BufferSize); if (Tmp == nullptr) { @@ -40,9 +40,9 @@ } BufferQueue::ErrorCode BufferQueue::getBuffer(Buffer &Buf) { - if (Finalizing.load(std::memory_order_acquire)) + if (__sanitizer::atomic_load(&Finalizing, __sanitizer::memory_order_acquire)) return ErrorCode::QueueFinalizing; - std::lock_guard Guard(Mutex); + __sanitizer::BlockingMutexLock Guard(&Mutex); if (Buffers.empty()) return ErrorCode::NotEnoughMemory; auto &T = Buffers.front(); @@ -57,7 +57,7 @@ BufferQueue::ErrorCode BufferQueue::releaseBuffer(Buffer &Buf) { if (OwnedBuffers.count(Buf.Buffer) == 0) return ErrorCode::UnrecognizedBuffer; - std::lock_guard Guard(Mutex); + __sanitizer::BlockingMutexLock Guard(&Mutex); // Now that the buffer has been released, we mark it as "used". Buffers.emplace(Buffers.end(), Buf, true /* used */); @@ -67,7 +67,8 @@ } BufferQueue::ErrorCode BufferQueue::finalize() { - if (Finalizing.exchange(true, std::memory_order_acq_rel)) + if (__sanitizer::atomic_exchange(&Finalizing, 1, + __sanitizer::memory_order_acq_rel)) return ErrorCode::QueueFinalizing; return ErrorCode::Ok; } Index: lib/xray/xray_fdr_logging.cc =================================================================== --- lib/xray/xray_fdr_logging.cc +++ lib/xray/xray_fdr_logging.cc @@ -17,15 +17,14 @@ #include "xray_fdr_logging.h" #include #include -#include #include -#include #include #include #include #include #include +#include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" #include "xray/xray_interface.h" #include "xray/xray_records.h" @@ -41,10 +40,10 @@ // Global BufferQueue. std::shared_ptr BQ; -std::atomic LoggingStatus{ +__sanitizer::atomic_sint32_t LoggingStatus = { XRayLogInitStatus::XRAY_LOG_UNINITIALIZED}; -std::atomic LogFlushStatus{ +__sanitizer::atomic_sint32_t LogFlushStatus = { XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING}; std::unique_ptr FDROptions; @@ -53,11 +52,12 @@ void *Options, size_t OptionsSize) XRAY_NEVER_INSTRUMENT { assert(OptionsSize == sizeof(FDRLoggingOptions)); - XRayLogInitStatus CurrentStatus = XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; - if (!LoggingStatus.compare_exchange_strong( - CurrentStatus, XRayLogInitStatus::XRAY_LOG_INITIALIZING, - std::memory_order_release, std::memory_order_relaxed)) - return CurrentStatus; + s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; + if (__sanitizer::atomic_compare_exchange_strong( + &LoggingStatus, &CurrentStatus, + XRayLogInitStatus::XRAY_LOG_INITIALIZING, + __sanitizer::memory_order_release)) + return static_cast(CurrentStatus); FDROptions.reset(new FDRLoggingOptions()); *FDROptions = *reinterpret_cast(Options); @@ -74,22 +74,24 @@ // Install the actual handleArg0 handler after initialising the buffers. __xray_set_handler(fdrLoggingHandleArg0); - LoggingStatus.store(XRayLogInitStatus::XRAY_LOG_INITIALIZED, - std::memory_order_release); + __sanitizer::atomic_store(&LoggingStatus, + XRayLogInitStatus::XRAY_LOG_INITIALIZED, + __sanitizer::memory_order_release); return XRayLogInitStatus::XRAY_LOG_INITIALIZED; } // Must finalize before flushing. XRayLogFlushStatus fdrLoggingFlush() XRAY_NEVER_INSTRUMENT { - if (LoggingStatus.load(std::memory_order_acquire) != + if (__sanitizer::atomic_load(&LoggingStatus, + __sanitizer::memory_order_acquire) != XRayLogInitStatus::XRAY_LOG_FINALIZED) return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; - XRayLogFlushStatus Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; - if (!LogFlushStatus.compare_exchange_strong( - Result, XRayLogFlushStatus::XRAY_LOG_FLUSHING, - std::memory_order_release, std::memory_order_relaxed)) - return Result; + s32 Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; + if (__sanitizer::atomic_compare_exchange_strong( + &LogFlushStatus, &Result, XRayLogFlushStatus::XRAY_LOG_FLUSHING, + __sanitizer::memory_order_release)) + return static_cast(Result); // Make a copy of the BufferQueue pointer to prevent other threads that may be // resetting it from blowing away the queue prematurely while we're dealing @@ -110,7 +112,8 @@ Fd = getLogFD(); if (Fd == -1) { auto Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; - LogFlushStatus.store(Result, std::memory_order_release); + __sanitizer::atomic_store(&LogFlushStatus, Result, + __sanitizer::memory_order_release); return Result; } @@ -129,43 +132,48 @@ retryingWriteAll(Fd, reinterpret_cast(B.Buffer), reinterpret_cast(B.Buffer) + B.Size); }); - LogFlushStatus.store(XRayLogFlushStatus::XRAY_LOG_FLUSHED, - std::memory_order_release); + __sanitizer::atomic_store(&LogFlushStatus, + XRayLogFlushStatus::XRAY_LOG_FLUSHED, + __sanitizer::memory_order_release); return XRayLogFlushStatus::XRAY_LOG_FLUSHED; } XRayLogInitStatus fdrLoggingFinalize() XRAY_NEVER_INSTRUMENT { - XRayLogInitStatus CurrentStatus = XRayLogInitStatus::XRAY_LOG_INITIALIZED; - if (!LoggingStatus.compare_exchange_strong( - CurrentStatus, XRayLogInitStatus::XRAY_LOG_FINALIZING, - std::memory_order_release, std::memory_order_relaxed)) - return CurrentStatus; + s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_INITIALIZED; + if (__sanitizer::atomic_compare_exchange_strong( + &LoggingStatus, &CurrentStatus, + XRayLogInitStatus::XRAY_LOG_FINALIZING, + __sanitizer::memory_order_release)) + return static_cast(CurrentStatus); // Do special things to make the log finalize itself, and not allow any more // operations to be performed until re-initialized. BQ->finalize(); - LoggingStatus.store(XRayLogInitStatus::XRAY_LOG_FINALIZED, - std::memory_order_release); + __sanitizer::atomic_store(&LoggingStatus, + XRayLogInitStatus::XRAY_LOG_FINALIZED, + __sanitizer::memory_order_release); return XRayLogInitStatus::XRAY_LOG_FINALIZED; } XRayLogInitStatus fdrLoggingReset() XRAY_NEVER_INSTRUMENT { - XRayLogInitStatus CurrentStatus = XRayLogInitStatus::XRAY_LOG_FINALIZED; - if (!LoggingStatus.compare_exchange_strong( - CurrentStatus, XRayLogInitStatus::XRAY_LOG_UNINITIALIZED, - std::memory_order_release, std::memory_order_relaxed)) - return CurrentStatus; + s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_FINALIZED; + if (__sanitizer::atomic_compare_exchange_strong( + &LoggingStatus, &CurrentStatus, + XRayLogInitStatus::XRAY_LOG_INITIALIZED, + __sanitizer::memory_order_release)) + return static_cast(CurrentStatus); // Release the in-memory buffer queue. BQ.reset(); // Spin until the flushing status is flushed. - XRayLogFlushStatus CurrentFlushingStatus = + s32 CurrentFlushingStatus = XRayLogFlushStatus::XRAY_LOG_FLUSHED; - while (!LogFlushStatus.compare_exchange_weak( - CurrentFlushingStatus, XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING, - std::memory_order_release, std::memory_order_relaxed)) { + while (__sanitizer::atomic_compare_exchange_weak( + &LogFlushStatus, &CurrentFlushingStatus, + XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING, + __sanitizer::memory_order_release)) { if (CurrentFlushingStatus == XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING) break; CurrentFlushingStatus = XRayLogFlushStatus::XRAY_LOG_FLUSHED; Index: lib/xray/xray_fdr_logging_impl.h =================================================================== --- lib/xray/xray_fdr_logging_impl.h +++ lib/xray/xray_fdr_logging_impl.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -91,12 +92,12 @@ /// walk backward through its buffer and erase trivial functions to avoid /// polluting the log and may use the buffer queue to obtain or release a /// buffer. -static void -processFunctionHook(int32_t FuncId, XRayEntryType Entry, uint64_t TSC, - unsigned char CPU, - int (*wall_clock_reader)(clockid_t, struct timespec *), - const std::atomic &LoggingStatus, - const std::shared_ptr &BQ); +static void processFunctionHook(int32_t FuncId, XRayEntryType Entry, + uint64_t TSC, unsigned char CPU, + int (*wall_clock_reader)(clockid_t, + struct timespec *), + __sanitizer::atomic_sint32_t &LoggingStatus, + const std::shared_ptr &BQ); //-----------------------------------------------------------------------------| // The rest of the file is implementation. | @@ -166,8 +167,9 @@ }; static inline bool loggingInitialized( - const std::atomic &LoggingStatus) XRAY_NEVER_INSTRUMENT { - return LoggingStatus.load(std::memory_order_acquire) == + const __sanitizer::atomic_sint32_t &LoggingStatus) XRAY_NEVER_INSTRUMENT { + return __sanitizer::atomic_load(&LoggingStatus, + __sanitizer::memory_order_acquire) == XRayLogInitStatus::XRAY_LOG_INITIALIZED; } @@ -305,10 +307,11 @@ static inline void processFunctionHook( int32_t FuncId, XRayEntryType Entry, uint64_t TSC, unsigned char CPU, int (*wall_clock_reader)(clockid_t, struct timespec *), - const std::atomic &LoggingStatus, + __sanitizer::atomic_sint32_t &LoggingStatus, const std::shared_ptr &BQ) XRAY_NEVER_INSTRUMENT { // Bail out right away if logging is not initialized yet. - if (LoggingStatus.load(std::memory_order_acquire) != + if (__sanitizer::atomic_load(&LoggingStatus, + __sanitizer::memory_order_acquire) != XRayLogInitStatus::XRAY_LOG_INITIALIZED) return; @@ -352,7 +355,8 @@ if (Buffer.Buffer == nullptr) { auto EC = LocalBQ->getBuffer(Buffer); if (EC != BufferQueue::ErrorCode::Ok) { - auto LS = LoggingStatus.load(std::memory_order_acquire); + auto LS = __sanitizer::atomic_load(&LoggingStatus, + __sanitizer::memory_order_acquire); if (LS != XRayLogInitStatus::XRAY_LOG_FINALIZING && LS != XRayLogInitStatus::XRAY_LOG_FINALIZED) Report("Failed to acquire a buffer; error=%s\n", Index: lib/xray/xray_init.cc =================================================================== --- lib/xray/xray_init.cc +++ lib/xray/xray_init.cc @@ -12,7 +12,6 @@ // XRay initialisation logic. //===----------------------------------------------------------------------===// -#include #include #include #include @@ -28,7 +27,6 @@ extern const XRaySledEntry __stop_xray_instr_map[] __attribute__((weak)); } -using namespace __sanitizer; using namespace __xray; // When set to 'true' this means the XRay runtime has been initialised. We use @@ -38,10 +36,11 @@ // // FIXME: Support DSO instrumentation maps too. The current solution only works // for statically linked executables. -std::atomic XRayInitialized{false}; +__sanitizer::atomic_uint8_t XRayInitialized{0}; // This should always be updated before XRayInitialized is updated. -std::atomic<__xray::XRaySledMap> XRayInstrMap{}; +__sanitizer::SpinMutex XRayInstrMapMutex; +XRaySledMap XRayInstrMap; // __xray_init() will do the actual loading of the current process' memory map // and then proceed to look for the .xray_instr_map section/segment. @@ -52,15 +51,13 @@ return; } - // Now initialize the XRayInstrMap global struct with the address of the - // entries, reinterpreted as an array of XRaySledEntry objects. We use the - // virtual pointer we have from the section to provide us the correct - // information. - __xray::XRaySledMap SledMap{}; - SledMap.Sleds = __start_xray_instr_map; - SledMap.Entries = __stop_xray_instr_map - __start_xray_instr_map; - XRayInstrMap.store(SledMap, std::memory_order_release); - XRayInitialized.store(true, std::memory_order_release); + { + __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); + XRayInstrMap.Sleds = __start_xray_instr_map; + XRayInstrMap.Entries = __stop_xray_instr_map - __start_xray_instr_map; + } + __sanitizer::atomic_store(&XRayInitialized, true, + __sanitizer::memory_order_release); if (flags()->patch_premain) __xray_patch(); Index: lib/xray/xray_interface.cc =================================================================== --- lib/xray/xray_interface.cc +++ lib/xray/xray_interface.cc @@ -15,7 +15,6 @@ #include "xray_interface_internal.h" -#include #include #include #include @@ -46,10 +45,10 @@ #endif /* CPU architecture */ // This is the function to call when we encounter the entry or exit sleds. -std::atomic XRayPatchedFunction{nullptr}; +__sanitizer::atomic_uintptr_t XRayPatchedFunction{0}; // This is the function to call from the arg1-enabled sleds/trampolines. -std::atomic XRayArgLogger{nullptr}; +__sanitizer::atomic_uintptr_t XRayArgLogger{0}; // MProtectHelper is an RAII wrapper for calls to mprotect(...) that will undo // any successful mprotect(...) changes. This is used to make a page writeable @@ -88,13 +87,18 @@ } // namespace __xray -extern std::atomic XRayInitialized; -extern std::atomic<__xray::XRaySledMap> XRayInstrMap; +extern __sanitizer::SpinMutex XRayInstrMapMutex; +extern __sanitizer::atomic_uint8_t XRayInitialized; +extern __xray::XRaySledMap XRayInstrMap; int __xray_set_handler(void (*entry)(int32_t, XRayEntryType)) XRAY_NEVER_INSTRUMENT { - if (XRayInitialized.load(std::memory_order_acquire)) { - __xray::XRayPatchedFunction.store(entry, std::memory_order_release); + if (__sanitizer::atomic_load(&XRayInitialized, + __sanitizer::memory_order_acquire)) { + + __sanitizer::atomic_store(&__xray::XRayPatchedFunction, + reinterpret_cast(entry), + __sanitizer::memory_order_release); return 1; } return 0; @@ -104,7 +108,7 @@ return __xray_set_handler(nullptr); } -std::atomic XRayPatching{false}; +__sanitizer::atomic_uint8_t XRayPatching{0}; using namespace __xray; @@ -132,26 +136,29 @@ // implementation. |Enable| defines whether we're enabling or disabling the // runtime XRay instrumentation. XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT { - if (!XRayInitialized.load(std::memory_order_acquire)) + if (!__sanitizer::atomic_load(&XRayInitialized, + __sanitizer::memory_order_acquire)) return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized. - static bool NotPatching = false; - if (!XRayPatching.compare_exchange_strong(NotPatching, true, - std::memory_order_acq_rel, - std::memory_order_acquire)) { + uint8_t NotPatching = false; + if (!__sanitizer::atomic_compare_exchange_strong( + &XRayPatching, &NotPatching, true, __sanitizer::memory_order_acq_rel)) return XRayPatchingStatus::ONGOING; // Already patching. - } - bool PatchingSuccess = false; + uint8_t PatchingSuccess = false; auto XRayPatchingStatusResetter = scopeCleanup([&PatchingSuccess] { - if (!PatchingSuccess) { - XRayPatching.store(false, std::memory_order_release); - } + if (!PatchingSuccess) + __sanitizer::atomic_store(&XRayPatching, false, + __sanitizer::memory_order_release); }); // Step 1: Compute the function id, as a unique identifier per function in the // instrumentation map. - XRaySledMap InstrMap = XRayInstrMap.load(std::memory_order_acquire); + XRaySledMap InstrMap; + { + __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); + InstrMap = XRayInstrMap; + } if (InstrMap.Entries == 0) return XRayPatchingStatus::NOT_INITIALIZED; @@ -205,7 +212,8 @@ } (void)Success; } - XRayPatching.store(false, std::memory_order_release); + __sanitizer::atomic_store(&XRayPatching, false, + __sanitizer::memory_order_release); PatchingSuccess = true; return XRayPatchingStatus::SUCCESS; } @@ -218,15 +226,16 @@ return controlPatching(false); } -int __xray_set_handler_arg1(void (*Handler)(int32_t, XRayEntryType, uint64_t)) -{ - if (!XRayInitialized.load(std::memory_order_acquire)) { +int __xray_set_handler_arg1(void (*Handler)(int32_t, XRayEntryType, uint64_t)) { + if (!__sanitizer::atomic_load(&XRayInitialized, + __sanitizer::memory_order_acquire)) return 0; - } + // A relaxed write might not be visible even if the current thread gets // scheduled on a different CPU/NUMA node. We need to wait for everyone to // have this handler installed for consistency of collected data across CPUs. - XRayArgLogger.store(Handler, std::memory_order_release); + __sanitizer::atomic_store(&XRayArgLogger, reinterpret_cast(Handler), + __sanitizer::memory_order_release); return 1; } int __xray_remove_handler_arg1() { return __xray_set_handler_arg1(nullptr); } Index: lib/xray/xray_log_interface.cc =================================================================== --- lib/xray/xray_log_interface.cc +++ lib/xray/xray_log_interface.cc @@ -12,45 +12,46 @@ //===----------------------------------------------------------------------===// #include "xray/xray_log_interface.h" +#include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_mutex.h" #include "xray/xray_interface.h" #include "xray_defs.h" #include -#include -std::mutex XRayImplMutex; +__sanitizer::SpinMutex XRayImplMutex; std::unique_ptr GlobalXRayImpl; void __xray_set_log_impl(XRayLogImpl Impl) XRAY_NEVER_INSTRUMENT { if (Impl.log_init == nullptr || Impl.log_finalize == nullptr || Impl.handle_arg0 == nullptr || Impl.flush_log == nullptr) { - std::lock_guard Guard(XRayImplMutex); + __sanitizer::SpinMutexLock Guard(&XRayImplMutex); GlobalXRayImpl.reset(); return; } - std::lock_guard Guard(XRayImplMutex); + __sanitizer::SpinMutexLock Guard(&XRayImplMutex); GlobalXRayImpl.reset(new XRayLogImpl); *GlobalXRayImpl = Impl; } XRayLogInitStatus __xray_init(size_t BufferSize, size_t MaxBuffers, void *Args, size_t ArgsSize) XRAY_NEVER_INSTRUMENT { - std::lock_guard Guard(XRayImplMutex); + __sanitizer::SpinMutexLock Guard(&XRayImplMutex); if (!GlobalXRayImpl) return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; return GlobalXRayImpl->log_init(BufferSize, MaxBuffers, Args, ArgsSize); } XRayLogInitStatus __xray_log_finalize() XRAY_NEVER_INSTRUMENT { - std::lock_guard Guard(XRayImplMutex); + __sanitizer::SpinMutexLock Guard(&XRayImplMutex); if (!GlobalXRayImpl) return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED; return GlobalXRayImpl->log_finalize(); } XRayLogFlushStatus __xray_log_flushLog() XRAY_NEVER_INSTRUMENT { - std::lock_guard Guard(XRayImplMutex); + __sanitizer::SpinMutexLock Guard(&XRayImplMutex); if (!GlobalXRayImpl) return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING; return GlobalXRayImpl->flush_log();