diff --git a/libcxxabi/src/cxa_guard_impl.h b/libcxxabi/src/cxa_guard_impl.h --- a/libcxxabi/src/cxa_guard_impl.h +++ b/libcxxabi/src/cxa_guard_impl.h @@ -196,94 +196,42 @@ AtomicInt guard_byte; }; -//===----------------------------------------------------------------------===// -// GuardBase -//===----------------------------------------------------------------------===// - -enum class AcquireResult { - INIT_IS_DONE, - INIT_IS_PENDING, -}; -constexpr AcquireResult INIT_IS_DONE = AcquireResult::INIT_IS_DONE; -constexpr AcquireResult INIT_IS_PENDING = AcquireResult::INIT_IS_PENDING; - -template -struct GuardObject { - GuardObject() = delete; - GuardObject(GuardObject const&) = delete; - GuardObject& operator=(GuardObject const&) = delete; - -private: - GuardByte guard_byte; - -public: - /// ARM Constructor - explicit GuardObject(uint32_t* g) - : base_address(g), init_byte_address(reinterpret_cast(g) + 1), thread_id_address(nullptr), - guard_byte(reinterpret_cast(raw_guard_object)) {} - - /// Itanium Constructor - explicit GuardObject(uint64_t* g) - : base_address(g), init_byte_address(reinterpret_cast(g) + 1), - thread_id_address(reinterpret_cast(g) + 1), - guard_byte(reinterpret_cast(raw_guard_object)) {} - - /// Implements __cxa_guard_acquire. - AcquireResult cxa_guard_acquire() { - // use short-circuit evalutation to avoid calling init_byte.acquire when - // guard_byte.acquire returns true. (i.e. don't call it when we know from - // the guard byte that initialization has already been completed) - if (guard_byte.acquire()) - return INIT_IS_DONE; - return derived()->acquire_init_byte(); - } - - /// Implements __cxa_guard_release. - void cxa_guard_release() { - // Update guard byte first, so if somebody is woken up by init_byte.release - // and comes all the way back around to __cxa_guard_acquire again, they see - // it as having completed initialization. - guard_byte.release(); - derived()->release_init_byte(); - } - - /// Implements __cxa_guard_abort. - void cxa_guard_abort() { - guard_byte.abort(); - derived()->abort_init_byte(); - } - -public: - /// base_address - the address of the original guard object. - void* const base_address; - /// The address of the byte used by the implementation during initialization. - uint8_t* const init_byte_address; - /// An optional address storing an identifier for the thread performing initialization. - /// It's used to detect recursive initialization. - uint32_t* const thread_id_address; - -private: - Derived* derived() { return static_cast(this); } -}; - //===----------------------------------------------------------------------===// // Single Threaded Implementation //===----------------------------------------------------------------------===// -struct InitByteNoThreads : GuardObject { - using GuardObject::GuardObject; +/// InitByteNoThreads - Doesn't use any inter-thread synchronization when +/// managing reads and writes to the init byte. +struct InitByteNoThreads { + InitByteNoThreads() = delete; + InitByteNoThreads(InitByteNoThreads const&) = delete; + InitByteNoThreads& operator=(InitByteNoThreads const&) = delete; + + explicit InitByteNoThreads(uint8_t* _init_byte_address, uint32_t*) : init_byte_address(_init_byte_address) {} - AcquireResult acquire_init_byte() { + /// The init byte portion of cxa_guard_acquire. Returns true if + /// initialization has already been completed. + /// Note: On completion, we haven't 'acquired' ownership of anything mutex + /// related. The name is simply referring to __cxa_guard_acquire. + bool acquire() { if (*init_byte_address == COMPLETE_BIT) - return INIT_IS_DONE; + return true; if (*init_byte_address & PENDING_BIT) ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization"); *init_byte_address = PENDING_BIT; - return INIT_IS_PENDING; + return false; } - void release_init_byte() { *init_byte_address = COMPLETE_BIT; } - void abort_init_byte() { *init_byte_address = UNSET; } + /// The init byte portion of cxa_guard_release. + /// Note: On completion, we haven't 'released' ownership of anything mutex + /// related. The name is simply referring to __cxa_guard_release. + void release() { *init_byte_address = COMPLETE_BIT; } + /// The init byte portion of cxa_guard_abort. + void abort() { *init_byte_address = UNSET; } + +private: + /// The address of the byte used during initialization. + uint8_t* const init_byte_address; }; //===----------------------------------------------------------------------===// @@ -323,18 +271,22 @@ struct LibcppCondVar {}; #endif // !defined(_LIBCXXABI_HAS_NO_THREADS) +/// InitByteGlobalMutex - Uses a global mutex and condition variable (common to +/// all static local variables) to manage reads and writes to the init byte. template -struct InitByteGlobalMutex : GuardObject> { +struct InitByteGlobalMutex { - using BaseT = typename InitByteGlobalMutex::GuardObject; - using BaseT::BaseT; - - explicit InitByteGlobalMutex(uint32_t* g) : BaseT(g), has_thread_id_support(false) {} - explicit InitByteGlobalMutex(uint64_t* g) : BaseT(g), has_thread_id_support(GetThreadID != nullptr) {} + explicit InitByteGlobalMutex(uint8_t* _init_byte_address, uint32_t* _thread_id_address) + : init_byte_address(_init_byte_address), thread_id_address(_thread_id_address), + has_thread_id_support(_thread_id_address != nullptr && GetThreadID != nullptr) {} public: - AcquireResult acquire_init_byte() { + /// The init byte portion of cxa_guard_acquire. Returns true if + /// initialization has already been completed. + /// Note: On completion, we haven't 'acquired' ownership of anything mutex + /// related. The name is simply referring to __cxa_guard_acquire. + bool acquire() { LockGuard g("__cxa_guard_acquire"); // Check for possible recursive initialization. if (has_thread_id_support && (*init_byte_address & PENDING_BIT)) { @@ -349,16 +301,19 @@ } if (*init_byte_address == COMPLETE_BIT) - return INIT_IS_DONE; + return true; if (has_thread_id_support) *thread_id_address = current_thread_id.get(); *init_byte_address = PENDING_BIT; - return INIT_IS_PENDING; + return false; } - void release_init_byte() { + /// The init byte portion of cxa_guard_release. + /// Note: On completion, we haven't 'released' ownership of anything mutex + /// related. The name is simply referring to __cxa_guard_release. + void release() { bool has_waiting; { LockGuard g("__cxa_guard_release"); @@ -372,7 +327,8 @@ } } - void abort_init_byte() { + /// The init byte portion of cxa_guard_abort. + void abort() { bool has_waiting; { LockGuard g("__cxa_guard_abort"); @@ -389,8 +345,12 @@ } private: - using BaseT::init_byte_address; - using BaseT::thread_id_address; + /// The address of the byte used during initialization. + uint8_t* const init_byte_address; + /// An optional address storing an identifier for the thread performing initialization. + /// It's used to detect recursive initialization. + uint32_t* const thread_id_address; + const bool has_thread_id_support; LazyValue current_thread_id; @@ -437,38 +397,34 @@ constexpr bool PlatformSupportsFutex() { return +PlatformFutexWait != nullptr; } -/// InitByteFutex - Manages initialization using atomics and the futex syscall -/// for waiting and waking. +/// InitByteFutex - Uses a futex to manage reads and writes to the init byte. template -struct InitByteFutex : GuardObject> { - using BaseT = typename InitByteFutex::GuardObject; - - /// ARM Constructor - explicit InitByteFutex(uint32_t* g) - : BaseT(g), init_byte(this->init_byte_address), - has_thread_id_support(this->thread_id_address && GetThreadIDArg != nullptr), - thread_id(this->thread_id_address) {} +struct InitByteFutex { - /// Itanium Constructor - explicit InitByteFutex(uint64_t* g) - : BaseT(g), init_byte(this->init_byte_address), - has_thread_id_support(this->thread_id_address && GetThreadIDArg != nullptr), - thread_id(this->thread_id_address) {} + explicit InitByteFutex(uint8_t* _init_byte_address, uint32_t* _thread_id_address) + : init_byte(_init_byte_address), + has_thread_id_support(_thread_id_address != nullptr && GetThreadIDArg != nullptr), + thread_id(_thread_id_address), base_address(reinterpret_cast( + /*_init_byte_address & ~0x3*/ _init_byte_address - 1)) {} public: - AcquireResult acquire_init_byte() { + /// The init byte portion of cxa_guard_acquire. Returns true if + /// initialization has already been completed. + /// Note: On completion, we haven't 'acquired' ownership of anything mutex + /// related. The name is simply referring to __cxa_guard_acquire. + bool acquire() { while (true) { uint8_t last_val = UNSET; if (init_byte.compare_exchange(&last_val, PENDING_BIT, std::_AO_Acq_Rel, std::_AO_Acquire)) { if (has_thread_id_support) { thread_id.store(current_thread_id.get(), std::_AO_Relaxed); } - return INIT_IS_PENDING; + return false; } if (last_val == COMPLETE_BIT) - return INIT_IS_DONE; + return true; if (last_val & PENDING_BIT) { @@ -485,7 +441,7 @@ if (!init_byte.compare_exchange(&last_val, PENDING_BIT | WAITING_BIT, std::_AO_Acq_Rel, std::_AO_Release)) { // (1) success, via someone else's work! if (last_val == COMPLETE_BIT) - return INIT_IS_DONE; + return true; // (3) someone else, bailed on doing the work, retry from the start! if (last_val == UNSET) @@ -499,13 +455,17 @@ } } - void release_init_byte() { + /// The init byte portion of cxa_guard_release. + /// Note: On completion, we haven't 'released' ownership of anything mutex + /// related. The name is simply referring to __cxa_guard_release. + void release() { uint8_t old = init_byte.exchange(COMPLETE_BIT, std::_AO_Acq_Rel); if (old & WAITING_BIT) wake_all(); } - void abort_init_byte() { + /// The init byte portion of cxa_guard_abort. + void abort() { if (has_thread_id_support) thread_id.store(0, std::_AO_Relaxed); @@ -516,12 +476,11 @@ private: /// Use the futex to wait on the current guard variable. Futex expects a - /// 32-bit 4-byte aligned address as the first argument, so we have to use use - /// the base address of the guard variable (not the init byte). - void wait_on_initialization() { - Wait(static_cast(this->base_address), expected_value_for_futex(PENDING_BIT | WAITING_BIT)); - } - void wake_all() { Wake(static_cast(this->base_address)); } + /// 32-bit 4-byte aligned address as the first argument, so we use the 4-byte + /// aligned address that encompasses the init byte (i.e. the address of the + /// raw guard object that was passed to __cxa_guard_acquire/release/abort). + void wait_on_initialization() { Wait(base_address, expected_value_for_futex(PENDING_BIT | WAITING_BIT)); } + void wake_all() { Wake(base_address); } private: AtomicInt init_byte; @@ -531,6 +490,10 @@ AtomicInt thread_id; LazyValue current_thread_id; + /// the 4-byte-aligned address that encompasses the init byte (i.e. the + /// address of the raw guard object). + int* const base_address; + /// Create the expected integer value for futex `wait(int* addr, int expected)`. /// We pass the base address as the first argument, So this function creates /// an zero-initialized integer with `b` copied at the correct offset. @@ -543,6 +506,66 @@ static_assert(Wait != nullptr && Wake != nullptr, ""); }; +//===----------------------------------------------------------------------===// +// GuardObject +//===----------------------------------------------------------------------===// + +enum class AcquireResult { + INIT_IS_DONE, + INIT_IS_PENDING, +}; +constexpr AcquireResult INIT_IS_DONE = AcquireResult::INIT_IS_DONE; +constexpr AcquireResult INIT_IS_PENDING = AcquireResult::INIT_IS_PENDING; + +/// Co-ordinates between GuardByte and InitByte. +template +struct GuardObject { + GuardObject() = delete; + GuardObject(GuardObject const&) = delete; + GuardObject& operator=(GuardObject const&) = delete; + +private: + GuardByte guard_byte; + InitByteT init_byte; + +public: + /// ARM Constructor + explicit GuardObject(uint32_t* raw_guard_object) + : guard_byte(reinterpret_cast(raw_guard_object)), + init_byte(reinterpret_cast(raw_guard_object) + 1, nullptr) {} + + /// Itanium Constructor + explicit GuardObject(uint64_t* raw_guard_object) + : guard_byte(reinterpret_cast(raw_guard_object)), + init_byte(reinterpret_cast(raw_guard_object) + 1, reinterpret_cast(raw_guard_object) + 1) { + } + + /// Implements __cxa_guard_acquire. + AcquireResult cxa_guard_acquire() { + // use short-circuit evalutation to avoid calling init_byte.acquire when + // guard_byte.acquire returns true. (i.e. don't call it when we know from + // the guard byte that initialization has already been completed) + if (guard_byte.acquire() || init_byte.acquire()) + return INIT_IS_DONE; + return INIT_IS_PENDING; + } + + /// Implements __cxa_guard_release. + void cxa_guard_release() { + // Update guard byte first, so if somebody is woken up by init_byte.release + // and comes all the way back around to __cxa_guard_acquire again, they see + // it as having completed initialization. + guard_byte.release(); + init_byte.release(); + } + + /// Implements __cxa_guard_abort. + void cxa_guard_abort() { + guard_byte.abort(); + init_byte.abort(); + } +}; + //===----------------------------------------------------------------------===// // //===----------------------------------------------------------------------===// @@ -559,20 +582,23 @@ template struct SelectImplementation; +/// Manage initialization without performing any inter-thread synchronization. template <> struct SelectImplementation { - using type = InitByteNoThreads; + using type = GuardObject; }; +/// Manage initialization using a global mutex and condition variable. template <> struct SelectImplementation { - using type = InitByteGlobalMutex::instance, - GlobalStatic::instance, PlatformThreadID>; + using type = GuardObject::instance, + GlobalStatic::instance, PlatformThreadID>>; }; +/// Manage initialization using atomics and the futex syscall for waiting and waking. template <> struct SelectImplementation { - using type = InitByteFutex; + using type = GuardObject>; }; // TODO(EricWF): We should prefer the futex implementation when available. But diff --git a/libcxxabi/test/guard_test_basic.pass.cpp b/libcxxabi/test/guard_test_basic.pass.cpp --- a/libcxxabi/test/guard_test_basic.pass.cpp +++ b/libcxxabi/test/guard_test_basic.pass.cpp @@ -119,16 +119,13 @@ { #if defined(_LIBCXXABI_HAS_NO_THREADS) static_assert(CurrentImplementation == Implementation::NoThreads, ""); - static_assert( - std::is_same::value, ""); + static_assert(std::is_same>::value, ""); #else static_assert(CurrentImplementation == Implementation::GlobalMutex, ""); static_assert( - std::is_same< - SelectedImplementation, - InitByteGlobalMutex::instance, - GlobalStatic::instance>>::value, + std::is_same::instance, + GlobalStatic::instance>>>::value, ""); #endif } @@ -142,19 +139,17 @@ } } { - Tests::test(); - Tests::test(); + Tests>::test(); + Tests>::test(); } { using MutexImpl = - InitByteGlobalMutex; + GuardObject>; Tests::test(); Tests::test(); } { - using FutexImpl = - InitByteFutex<&NopFutexWait, &NopFutexWake, &MockGetThreadID>; + using FutexImpl = GuardObject>; Tests::test(); Tests::test(); }