Index: libcxx/include/atomic =================================================================== --- libcxx/include/atomic +++ libcxx/include/atomic @@ -642,20 +642,22 @@ } // namespace __gcc_atomic +#define __cxx_atomic_is_lock_free(x) __atomic_is_lock_free(x, 0) + template -static inline +static _LIBCPP_INLINE_VISIBILITY inline typename enable_if< __gcc_atomic::__can_assign::value>::type -__c11_atomic_init(volatile _Atomic(_Tp)* __a, _Tp __val) { +__cxx_atomic_init(volatile _Atomic(_Tp)* __a, _Tp __val) { __a->__a_value = __val; } template -static inline +static _LIBCPP_INLINE_VISIBILITY inline typename enable_if< !__gcc_atomic::__can_assign::value && __gcc_atomic::__can_assign< _Atomic(_Tp)*, _Tp>::value>::type -__c11_atomic_init(volatile _Atomic(_Tp)* __a, _Tp __val) { +__cxx_atomic_init(volatile _Atomic(_Tp)* __a, _Tp __val) { // [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because // the default operator= in an object is not volatile, a byte-by-byte copy // is required. @@ -668,34 +670,34 @@ } template -static inline void __c11_atomic_init(_Atomic(_Tp)* __a, _Tp __val) { +static _LIBCPP_INLINE_VISIBILITY inline void __cxx_atomic_init(_Atomic(_Tp)* __a, _Tp __val) { __a->__a_value = __val; } -static inline void __c11_atomic_thread_fence(memory_order __order) { +static _LIBCPP_INLINE_VISIBILITY inline void __cxx_atomic_thread_fence(memory_order __order) { __atomic_thread_fence(__gcc_atomic::__to_gcc_order(__order)); } -static inline void __c11_atomic_signal_fence(memory_order __order) { +static _LIBCPP_INLINE_VISIBILITY inline void __cxx_atomic_signal_fence(memory_order __order) { __atomic_signal_fence(__gcc_atomic::__to_gcc_order(__order)); } template -static inline void __c11_atomic_store(volatile _Atomic(_Tp)* __a, _Tp __val, +static _LIBCPP_INLINE_VISIBILITY inline void __cxx_atomic_store(volatile _Atomic(_Tp)* __a, _Tp __val, memory_order __order) { - return __atomic_store(&__a->__a_value, &__val, - __gcc_atomic::__to_gcc_order(__order)); + __atomic_store(&__a->__a_value, &__val, + __gcc_atomic::__to_gcc_order(__order)); } template -static inline void __c11_atomic_store(_Atomic(_Tp)* __a, _Tp __val, +static _LIBCPP_INLINE_VISIBILITY inline void __cxx_atomic_store(_Atomic(_Tp)* __a, _Tp __val, memory_order __order) { __atomic_store(&__a->__a_value, &__val, __gcc_atomic::__to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_load(const volatile _Atomic(_Tp)* __a, +static _LIBCPP_INLINE_VISIBILITY inline _Tp __cxx_atomic_load(const volatile _Atomic(_Tp)* __a, memory_order __order) { _Tp __ret; __atomic_load(&__a->__a_value, &__ret, @@ -704,7 +706,7 @@ } template -static inline _Tp __c11_atomic_load(const _Atomic(_Tp)* __a, memory_order __order) { +static _LIBCPP_INLINE_VISIBILITY inline _Tp __cxx_atomic_load(const _Atomic(_Tp)* __a, memory_order __order) { _Tp __ret; __atomic_load(&__a->__a_value, &__ret, __gcc_atomic::__to_gcc_order(__order)); @@ -712,7 +714,7 @@ } template -static inline _Tp __c11_atomic_exchange(volatile _Atomic(_Tp)* __a, +static _LIBCPP_INLINE_VISIBILITY inline _Tp __cxx_atomic_exchange(volatile _Atomic(_Tp)* __a, _Tp __value, memory_order __order) { _Tp __ret; __atomic_exchange(&__a->__a_value, &__value, &__ret, @@ -721,7 +723,7 @@ } template -static inline _Tp __c11_atomic_exchange(_Atomic(_Tp)* __a, _Tp __value, +static _LIBCPP_INLINE_VISIBILITY inline _Tp __cxx_atomic_exchange(_Atomic(_Tp)* __a, _Tp __value, memory_order __order) { _Tp __ret; __atomic_exchange(&__a->__a_value, &__value, &__ret, @@ -730,7 +732,7 @@ } template -static inline bool __c11_atomic_compare_exchange_strong( +static _LIBCPP_INLINE_VISIBILITY inline bool __cxx_atomic_compare_exchange_strong( volatile _Atomic(_Tp)* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, @@ -740,7 +742,7 @@ } template -static inline bool __c11_atomic_compare_exchange_strong( +static _LIBCPP_INLINE_VISIBILITY inline bool __cxx_atomic_compare_exchange_strong( _Atomic(_Tp)* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, @@ -750,7 +752,7 @@ } template -static inline bool __c11_atomic_compare_exchange_weak( +static _LIBCPP_INLINE_VISIBILITY inline bool __cxx_atomic_compare_exchange_weak( volatile _Atomic(_Tp)* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, @@ -760,7 +762,7 @@ } template -static inline bool __c11_atomic_compare_exchange_weak( +static _LIBCPP_INLINE_VISIBILITY inline bool __cxx_atomic_compare_exchange_weak( _Atomic(_Tp)* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, @@ -783,74 +785,162 @@ struct __skip_amt<_Tp[n]> { }; template -static inline _Tp __c11_atomic_fetch_add(volatile _Atomic(_Tp)* __a, - _Td __delta, memory_order __order) { +static _LIBCPP_INLINE_VISIBILITY inline _Tp __cxx_atomic_fetch_add(volatile _Atomic(_Tp)* __a, + _Td __delta, memory_order __order) { return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, __gcc_atomic::__to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_add(_Atomic(_Tp)* __a, _Td __delta, - memory_order __order) { +static _LIBCPP_INLINE_VISIBILITY inline _Tp __cxx_atomic_fetch_add(_Atomic(_Tp)* __a, _Td __delta, + memory_order __order) { return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, __gcc_atomic::__to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_sub(volatile _Atomic(_Tp)* __a, - _Td __delta, memory_order __order) { +static _LIBCPP_INLINE_VISIBILITY inline _Tp __cxx_atomic_fetch_sub(volatile _Atomic(_Tp)* __a, + _Td __delta, memory_order __order) { return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, __gcc_atomic::__to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_sub(_Atomic(_Tp)* __a, _Td __delta, - memory_order __order) { +static _LIBCPP_INLINE_VISIBILITY inline _Tp __cxx_atomic_fetch_sub(_Atomic(_Tp)* __a, _Td __delta, + memory_order __order) { return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, __gcc_atomic::__to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_and(volatile _Atomic(_Tp)* __a, - _Tp __pattern, memory_order __order) { +static _LIBCPP_INLINE_VISIBILITY inline _Tp __cxx_atomic_fetch_and(volatile _Atomic(_Tp)* __a, + _Tp __pattern, memory_order __order) { return __atomic_fetch_and(&__a->__a_value, __pattern, __gcc_atomic::__to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_and(_Atomic(_Tp)* __a, - _Tp __pattern, memory_order __order) { +static _LIBCPP_INLINE_VISIBILITY inline _Tp __cxx_atomic_fetch_and(_Atomic(_Tp)* __a, + _Tp __pattern, memory_order __order) { return __atomic_fetch_and(&__a->__a_value, __pattern, __gcc_atomic::__to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_or(volatile _Atomic(_Tp)* __a, - _Tp __pattern, memory_order __order) { +static _LIBCPP_INLINE_VISIBILITY inline _Tp __cxx_atomic_fetch_or(volatile _Atomic(_Tp)* __a, + _Tp __pattern, memory_order __order) { return __atomic_fetch_or(&__a->__a_value, __pattern, __gcc_atomic::__to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_or(_Atomic(_Tp)* __a, _Tp __pattern, - memory_order __order) { +static _LIBCPP_INLINE_VISIBILITY inline _Tp __cxx_atomic_fetch_or(_Atomic(_Tp)* __a, _Tp __pattern, + memory_order __order) { return __atomic_fetch_or(&__a->__a_value, __pattern, __gcc_atomic::__to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_xor(volatile _Atomic(_Tp)* __a, - _Tp __pattern, memory_order __order) { +static _LIBCPP_INLINE_VISIBILITY inline _Tp __cxx_atomic_fetch_xor(volatile _Atomic(_Tp)* __a, + _Tp __pattern, memory_order __order) { return __atomic_fetch_xor(&__a->__a_value, __pattern, __gcc_atomic::__to_gcc_order(__order)); } template -static inline _Tp __c11_atomic_fetch_xor(_Atomic(_Tp)* __a, _Tp __pattern, - memory_order __order) { +static _LIBCPP_INLINE_VISIBILITY inline _Tp __cxx_atomic_fetch_xor(_Atomic(_Tp)* __a, _Tp __pattern, + memory_order __order) { return __atomic_fetch_xor(&__a->__a_value, __pattern, __gcc_atomic::__to_gcc_order(__order)); } + +#else //_LIBCPP_HAS_GCC_ATOMIC_IMP + +#define __cxx_atomic_is_lock_free(x) __c11_atomic_is_lock_free(x) + +template +static _LIBCPP_INLINE_VISIBILITY inline +void __cxx_atomic_init(_Ap __a, _Tp __val) { + __c11_atomic_init(__a, __val); +} + +static _LIBCPP_INLINE_VISIBILITY inline +void __cxx_atomic_thread_fence(memory_order __order) { + __c11_atomic_thread_fence(__order); +} + +static _LIBCPP_INLINE_VISIBILITY inline +void __cxx_atomic_signal_fence(memory_order __order) { + __c11_atomic_signal_fence(__order); +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +void __cxx_atomic_store(_Ap __a, _Tp __val, memory_order __order) { + __c11_atomic_store(__a, __val, __order); +} + +template ::type>::type> +static _LIBCPP_INLINE_VISIBILITY inline +auto __cxx_atomic_load(_Ap __a, memory_order __order) + -> decltype(__c11_atomic_load((_Ap)0, memory_order_relaxed)) { + return __c11_atomic_load(const_cast<_Tp*>(__a), __order); +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_exchange(_Ap __a, _Tp __value, memory_order __order) { + return __c11_atomic_exchange(__a, __value, __order); +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +bool __cxx_atomic_compare_exchange_strong(_Ap __a, _Tp* __expected, _Tp __value, + memory_order __success, memory_order __failure) { + return __c11_atomic_compare_exchange_strong(__a, __expected, __value, + __success, __failure); +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +bool __cxx_atomic_compare_exchange_weak(_Ap __a, _Tp* __expected, _Tp __value, + memory_order __success, memory_order __failure) { + return __c11_atomic_compare_exchange_weak(__a, __expected, __value, + __success, __failure); +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +auto __cxx_atomic_fetch_add(_Ap __a, _Td __delta, memory_order __order) + -> decltype(__c11_atomic_load((_Ap)0, memory_order_relaxed)) { + return __c11_atomic_fetch_add(__a, __delta, __order); +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +auto __cxx_atomic_fetch_sub(_Ap __a, _Td __delta, memory_order __order) + -> decltype(__c11_atomic_load((_Ap)0, memory_order_relaxed)) { + return __c11_atomic_fetch_sub(__a, __delta, __order); +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_fetch_and(_Ap __a, _Tp __pattern, memory_order __order) { + return __c11_atomic_fetch_and(__a, __pattern, __order); +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_fetch_or(_Ap __a, _Tp __pattern, memory_order __order) { + return __c11_atomic_fetch_or(__a, __pattern, __order); +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_fetch_xor(_Ap __a, _Tp __pattern, memory_order __order) { + return __c11_atomic_fetch_xor(__a, __pattern, __order); +} + #endif // _LIBCPP_HAS_GCC_ATOMIC_IMP template @@ -885,91 +975,402 @@ # define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE #endif +namespace __cxx_detail { +#if defined(__cpp_lib_atomic_is_always_lock_free) + template struct __is_always_lock_free { + enum { __value = __atomic_always_lock_free(sizeof(_Tp), 0) }; }; +#else + template struct __is_always_lock_free { enum { __value = false }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_BOOL_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_CHAR_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_CHAR16_T_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_CHAR32_T_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_WCHAR_T_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_SHORT_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_INT_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_LONG_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_LLONG_LOCK_FREE }; }; + template struct __is_always_lock_free<_Tp*> { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; + template<> struct __is_always_lock_free { enum { __value = 2 == ATOMIC_POINTER_LOCK_FREE }; }; +#endif +} + +#ifdef LIBCXX_FREESTANDING + +template +struct __cxx_locked_atomic_type { + +#if _GNUC_VER >= 501 + static_assert(is_trivially_copyable<_Tp>::value, + "std::atomic requires that 'Tp' be a trivially copyable type"); +#endif + + _LIBCPP_INLINE_VISIBILITY + __cxx_locked_atomic_type() _NOEXCEPT + : __a_value(), __a_lock(0) {} + _LIBCPP_CONSTEXPR explicit __cxx_locked_atomic_type(_Tp value) _NOEXCEPT + : __a_value(value), __a_lock(0) {} + + _Tp __a_value; + mutable _Atomic(int) __a_lock; + + template + _LIBCPP_INLINE_VISIBILITY inline void reader_section(_Function && __f) const volatile { + while(1 == __cxx_atomic_exchange(&__a_lock, 1, memory_order_acquire)); + __f(__a_value); + __cxx_atomic_store(&__a_lock, 0, memory_order_release); + } + template + _LIBCPP_INLINE_VISIBILITY inline void reader_section(_Function && __f) const { + while(1 == __cxx_atomic_exchange(&__a_lock, 1, memory_order_acquire)); + __f(__a_value); + __cxx_atomic_store(&__a_lock, 0, memory_order_release); + } + template + _LIBCPP_INLINE_VISIBILITY inline void writer_section(_Function && __f) volatile { + while(1 == __cxx_atomic_exchange(&__a_lock, 1, memory_order_acquire)); + __f(__a_value); + __cxx_atomic_store(&__a_lock, 0, memory_order_release); + } + template + _LIBCPP_INLINE_VISIBILITY inline void writer_section(_Function && __f) { + while(1 == __cxx_atomic_exchange(&__a_lock, 1, memory_order_acquire)); + __f(__a_value); + __cxx_atomic_store(&__a_lock, 0, memory_order_release); + } +}; + +template +static _LIBCPP_INLINE_VISIBILITY inline +void __cxx_atomic_init(volatile __cxx_locked_atomic_type<_Tp>* __a, _Tp __val) { + __a->__a_value = __val; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +void __cxx_atomic_init(__cxx_locked_atomic_type<_Tp>* __a, _Tp __val) { + __a->__a_value = __val; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +void __cxx_atomic_store(volatile __cxx_locked_atomic_type<_Tp>* __a, _Tp __val, + memory_order) { + __a->writer_section([&](_Tp volatile& __a_value){ __a_value = __val; }); +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +void __cxx_atomic_store(__cxx_locked_atomic_type<_Tp>* __a, _Tp __val, + memory_order) { + __a->writer_section([&](_Tp & __a_value){ __a_value = __val; }); +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_load(const volatile __cxx_locked_atomic_type<_Tp>* __a, + memory_order) { + _Tp __val; + __a->reader_section([&](_Tp const volatile& __a_value){ __val = __a_value; }); + return __val; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_load(const __cxx_locked_atomic_type<_Tp>* __a, memory_order) { + _Tp __val; + __a->reader_section([&](_Tp const& __a_value){ __val = __a_value; }); + return __val; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_exchange(volatile __cxx_locked_atomic_type<_Tp>* __a, + _Tp __value, memory_order) { + __a->writer_section([&](_Tp volatile& __a_value){ + _Tp __temp = __a_value; + __a_value = __value; + __value = __temp; + }); + return __value; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_exchange(__cxx_locked_atomic_type<_Tp>* __a, _Tp __value, + memory_order) { + __a->writer_section([&](_Tp & __a_value){ + _Tp __temp = __a_value; + __a_value = __value; + __value = __temp; + }); + return __value; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +bool __cxx_atomic_compare_exchange_strong(volatile __cxx_locked_atomic_type<_Tp>* __a, + _Tp* __expected, _Tp __value, + memory_order, memory_order) { + bool __ret; + __a->writer_section([&](_Tp volatile& __a_value){ + __ret = __a_value == *__expected; + if(__ret) + __a_value = __value; + else + *__expected = __a_value; }); + return __ret; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +bool __cxx_atomic_compare_exchange_strong(__cxx_locked_atomic_type<_Tp>* __a, + _Tp* __expected, _Tp __value, + memory_order, memory_order) { + bool __ret; + __a->writer_section([&](_Tp & __a_value){ + __ret = __a_value == *__expected; + if(__ret) + __a_value = __value; + else + *__expected = __a_value; }); + return __ret; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +bool __cxx_atomic_compare_exchange_weak(volatile __cxx_locked_atomic_type<_Tp>* __a, + _Tp* __expected, _Tp __value, + memory_order, memory_order) { + bool __ret; + __a->writer_section([&](_Tp volatile& __a_value){ + __ret = __a_value == *__expected; + if(__ret) + __a_value = __value; + else + *__expected = __a_value; }); + return __ret; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +bool __cxx_atomic_compare_exchange_weak(__cxx_locked_atomic_type<_Tp>* __a, + _Tp* __expected, _Tp __value, + memory_order, memory_order) { + bool __ret; + __a->writer_section([&](_Tp & __a_value){ + __ret = __a_value == *__expected; + if(__ret) + __a_value = __value; + else + *__expected = __a_value; }); + return __ret; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_fetch_add(volatile __cxx_locked_atomic_type<_Tp>* __a, + _Td __delta, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp volatile& __a_value){ + __old = __a_value; + __a_value += __delta; }); + return __old; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_fetch_add(__cxx_locked_atomic_type<_Tp>* __a, + _Td __delta, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp & __a_value){ + __old = __a_value; + __a_value += __delta; }); + return __old; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_fetch_sub(volatile __cxx_locked_atomic_type<_Tp>* __a, + _Td __delta, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp volatile& __a_value){ + __old = __a_value; + __a_value -= __delta; }); + return __old; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_fetch_sub(__cxx_locked_atomic_type<_Tp>* __a, + _Td __delta, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp & __a_value){ + __old = __a_value; + __a_value -= __delta; }); + return __old; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_fetch_and(volatile __cxx_locked_atomic_type<_Tp>* __a, + _Tp __pattern, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp volatile& __a_value){ + __old = __a_value; + __a_value &= __pattern; }); + return __old; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_fetch_and(__cxx_locked_atomic_type<_Tp>* __a, + _Tp __pattern, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp & __a_value){ + __old = __a_value; + __a_value &= __pattern; }); + return __old; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_fetch_or(volatile __cxx_locked_atomic_type<_Tp>* __a, + _Tp __pattern, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp volatile& __a_value){ + __old = __a_value; + __a_value |= __pattern; }); + return __old; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_fetch_or(__cxx_locked_atomic_type<_Tp>* __a, + _Tp __pattern, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp & __a_value){ + __old = __a_value; + __a_value |= __pattern; }); + return __old; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_fetch_xor(volatile __cxx_locked_atomic_type<_Tp>* __a, + _Tp __pattern, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp volatile& __a_value){ + __old = __a_value; + __a_value ^= __pattern; }); + return __old; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __cxx_atomic_fetch_xor(__cxx_locked_atomic_type<_Tp>* __a, + _Tp __pattern, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp & __a_value){ + __old = __a_value; + __a_value ^= __pattern; }); + return __old; +} + +template +using __atomic_type = typename conditional<__cxx_detail::__is_always_lock_free<_Tp>::__value, + _Atomic(_Tp), + __cxx_locked_atomic_type<_Tp>>::type; +#else +template +using __atomic_type = _Atomic(_Tp); +#endif //LIBCXX_FREESTANDING + // general atomic template ::value && !is_same<_Tp, bool>::value> struct __atomic_base // false { - mutable _Atomic(_Tp) __a_; + mutable __atomic_type<_Tp> __a_; #if defined(__cpp_lib_atomic_is_always_lock_free) - static _LIBCPP_CONSTEXPR bool is_always_lock_free = __atomic_always_lock_free(sizeof(__a_), 0); + static _LIBCPP_CONSTEXPR bool is_always_lock_free = __cxx_detail::__is_always_lock_free<_Tp>::__value; #endif _LIBCPP_INLINE_VISIBILITY bool is_lock_free() const volatile _NOEXCEPT - { -#if defined(_LIBCPP_HAS_C_ATOMIC_IMP) - return __c11_atomic_is_lock_free(sizeof(_Tp)); -#else - return __atomic_is_lock_free(sizeof(_Tp), 0); -#endif - } + {return __cxx_atomic_is_lock_free(sizeof(_Tp));} _LIBCPP_INLINE_VISIBILITY bool is_lock_free() const _NOEXCEPT {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();} _LIBCPP_INLINE_VISIBILITY void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) - {__c11_atomic_store(&__a_, __d, __m);} + {__cxx_atomic_store(&__a_, __d, __m);} _LIBCPP_INLINE_VISIBILITY void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) - {__c11_atomic_store(&__a_, __d, __m);} + {__cxx_atomic_store(&__a_, __d, __m);} _LIBCPP_INLINE_VISIBILITY _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) - {return __c11_atomic_load(&__a_, __m);} + {return __cxx_atomic_load(&__a_, __m);} _LIBCPP_INLINE_VISIBILITY _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) - {return __c11_atomic_load(&__a_, __m);} + {return __cxx_atomic_load(&__a_, __m);} _LIBCPP_INLINE_VISIBILITY operator _Tp() const volatile _NOEXCEPT {return load();} _LIBCPP_INLINE_VISIBILITY operator _Tp() const _NOEXCEPT {return load();} _LIBCPP_INLINE_VISIBILITY _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_exchange(&__a_, __d, __m);} + {return __cxx_atomic_exchange(&__a_, __d, __m);} _LIBCPP_INLINE_VISIBILITY _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_exchange(&__a_, __d, __m);} + {return __cxx_atomic_exchange(&__a_, __d, __m);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) - {return __c11_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} + {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) - {return __c11_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} + {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __s, __f);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) - {return __c11_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} + {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) - {return __c11_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} + {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __s, __f);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} + {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} + {return __cxx_atomic_compare_exchange_weak(&__a_, &__e, __d, __m, __m);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} + {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} _LIBCPP_INLINE_VISIBILITY bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} + {return __cxx_atomic_compare_exchange_strong(&__a_, &__e, __d, __m, __m);} _LIBCPP_INLINE_VISIBILITY #ifndef _LIBCPP_CXX03_LANG @@ -1005,40 +1406,44 @@ { typedef __atomic_base<_Tp, false> __base; _LIBCPP_INLINE_VISIBILITY - __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT +#ifndef _LIBCPP_CXX03_LANG + __atomic_base() _NOEXCEPT = default; +#else + __atomic_base() _NOEXCEPT : __base() {} +#endif // _LIBCPP_CXX03_LANG _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} _LIBCPP_INLINE_VISIBILITY _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_fetch_add(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_fetch_add(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_fetch_sub(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_fetch_sub(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_fetch_and(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_fetch_and(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_and(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_fetch_or(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_fetch_or(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_or(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_fetch_xor(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_fetch_xor(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_xor(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp operator++(int) volatile _NOEXCEPT {return fetch_add(_Tp(1));} @@ -1086,7 +1491,11 @@ { typedef __atomic_base<_Tp> __base; _LIBCPP_INLINE_VISIBILITY - atomic() _NOEXCEPT _LIBCPP_DEFAULT +#ifndef _LIBCPP_CXX03_LANG + atomic() _NOEXCEPT = default; +#else + atomic() _NOEXCEPT : __base() {} +#endif // _LIBCPP_CXX03_LANG _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} @@ -1106,7 +1515,11 @@ { typedef __atomic_base<_Tp*> __base; _LIBCPP_INLINE_VISIBILITY - atomic() _NOEXCEPT _LIBCPP_DEFAULT +#ifndef _LIBCPP_CXX03_LANG + atomic() _NOEXCEPT = default; +#else + atomic() _NOEXCEPT : __base() {} +#endif // _LIBCPP_CXX03_LANG _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} @@ -1120,17 +1533,17 @@ _LIBCPP_INLINE_VISIBILITY _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_fetch_add(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_fetch_add(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_add(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_fetch_sub(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_fetch_sub(&this->__a_, __op, __m);} + {return __cxx_atomic_fetch_sub(&this->__a_, __op, __m);} _LIBCPP_INLINE_VISIBILITY _Tp* operator++(int) volatile _NOEXCEPT {return fetch_add(1);} @@ -1183,7 +1596,7 @@ void atomic_init(volatile atomic<_Tp>* __o, _Tp __d) _NOEXCEPT { - __c11_atomic_init(&__o->__a_, __d); + __cxx_atomic_init(&__o->__a_, __d); } template @@ -1191,7 +1604,7 @@ void atomic_init(atomic<_Tp>* __o, _Tp __d) _NOEXCEPT { - __c11_atomic_init(&__o->__a_, __d); + __cxx_atomic_init(&__o->__a_, __d); } // atomic_store @@ -1723,16 +2136,16 @@ _LIBCPP_INLINE_VISIBILITY bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_exchange(&__a_, true, __m);} + {return __cxx_atomic_exchange(&__a_, true, __m);} _LIBCPP_INLINE_VISIBILITY bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_exchange(&__a_, true, __m);} + {return __cxx_atomic_exchange(&__a_, true, __m);} _LIBCPP_INLINE_VISIBILITY void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {__c11_atomic_store(&__a_, false, __m);} + {__cxx_atomic_store(&__a_, false, __m);} _LIBCPP_INLINE_VISIBILITY void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT - {__c11_atomic_store(&__a_, false, __m);} + {__cxx_atomic_store(&__a_, false, __m);} _LIBCPP_INLINE_VISIBILITY #ifndef _LIBCPP_CXX03_LANG @@ -1818,14 +2231,14 @@ void atomic_thread_fence(memory_order __m) _NOEXCEPT { - __c11_atomic_thread_fence(__m); + __cxx_atomic_thread_fence(__m); } inline _LIBCPP_INLINE_VISIBILITY void atomic_signal_fence(memory_order __m) _NOEXCEPT { - __c11_atomic_signal_fence(__m); + __cxx_atomic_signal_fence(__m); } // Atomics for standard typedef types