Index: libcxx/include/atomic =================================================================== --- libcxx/include/atomic +++ libcxx/include/atomic @@ -682,8 +682,8 @@ template static inline void __c11_atomic_store(volatile _Atomic(_Tp)* __a, _Tp __val, memory_order __order) { - return __atomic_store(&__a->__a_value, &__val, - __gcc_atomic::__to_gcc_order(__order)); + __atomic_store(&__a->__a_value, &__val, + __gcc_atomic::__to_gcc_order(__order)); } template @@ -884,12 +884,185 @@ # define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE #endif +#ifndef _Atomic_s +#define _Atomic_s(_Tp, _Sco) _Atomic(_Tp) +#endif + +#if defined(_LIBCPP_FREESTANDING) && defined(__cpp_lib_atomic_is_always_lock_free) + +template +struct __cxx_locked_atomic_type { + +#if _GNUC_VER >= 501 + static_assert(is_trivially_copyable<_Tp>::value, + "std::atomic requires that 'Tp' be a trivially copyable type"); +#endif + + _LIBCPP_INLINE_VISIBILITY + __cxx_locked_atomic_type() _NOEXCEPT + : __a_value(), __a_lock(0) {} + _LIBCPP_INLINE_VISIBILITY + _LIBCPP_CONSTEXPR explicit __cxx_locked_atomic_type(_Tp value) _NOEXCEPT + : __a_value(value), __a_lock(0) {} + + _Tp __a_value; + mutable _Atomic_s(int, _Sco) __a_lock; + + template + _LIBCPP_INLINE_VISIBILITY inline void reader_section(_Function && __f) const volatile { + while(1 == __c11_atomic_exchange(&__a_lock, 1, memory_order_acquire)); + __f(__a_value); + __c11_atomic_store(&__a_lock, 0, memory_order_release); + } + template + _LIBCPP_INLINE_VISIBILITY inline void writer_section(_Function && __f) volatile { + while(1 == __c11_atomic_exchange(&__a_lock, 1, memory_order_acquire)); + __f(__a_value); + __c11_atomic_store(&__a_lock, 0, memory_order_release); + } +}; + +template +static _LIBCPP_INLINE_VISIBILITY inline +void __c11_atomic_init(volatile __cxx_locked_atomic_type<_Tp, _Sco>* __a, _Tp __val) { + __a->__a_value = __val; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +void __c11_atomic_store(volatile __cxx_locked_atomic_type<_Tp, _Sco>* __a, _Tp __val, + memory_order) { + __a->writer_section([&](_Tp volatile& __a_value){ __a_value = __val; }); +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __c11_atomic_load(const volatile __cxx_locked_atomic_type<_Tp, _Sco>* __a, + memory_order) { + _Tp __val; + __a->reader_section([&](_Tp const volatile& __a_value){ __val = __a_value; }); + return __val; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __c11_atomic_exchange(volatile __cxx_locked_atomic_type<_Tp, _Sco>* __a, + _Tp __value, memory_order) { + __a->writer_section([&](_Tp volatile& __a_value){ + _Tp __temp = __a_value; + __a_value = __value; + __value = __temp; + }); + return __value; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +bool __c11_atomic_compare_exchange_strong(volatile __cxx_locked_atomic_type<_Tp, _Sco>* __a, + _Tp* __expected, _Tp __value, + memory_order, memory_order) { + bool __ret; + __a->writer_section([&](_Tp volatile& __a_value){ + __ret = __a_value == *__expected; + if(__ret) + __a_value = __value; + else + *__expected = __a_value; }); + return __ret; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +bool __c11_atomic_compare_exchange_weak(volatile __cxx_locked_atomic_type<_Tp, _Sco>* __a, + _Tp* __expected, _Tp __value, + memory_order, memory_order) { + bool __ret; + __a->writer_section([&](_Tp volatile& __a_value){ + __ret = __a_value == *__expected; + if(__ret) + __a_value = __value; + else + *__expected = __a_value; }); + return __ret; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __c11_atomic_fetch_add(volatile __cxx_locked_atomic_type<_Tp, _Sco>* __a, + _Td __delta, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp volatile& __a_value){ + __old = __a_value; + __a_value += __delta; }); + return __old; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __c11_atomic_fetch_sub(volatile __cxx_locked_atomic_type<_Tp, _Sco>* __a, + _Td __delta, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp volatile& __a_value){ + __old = __a_value; + __a_value -= __delta; }); + return __old; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __c11_atomic_fetch_and(volatile __cxx_locked_atomic_type<_Tp, _Sco>* __a, + _Tp __pattern, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp volatile& __a_value){ + __old = __a_value; + __a_value &= __pattern; }); + return __old; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __c11_atomic_fetch_or(volatile __cxx_locked_atomic_type<_Tp, _Sco>* __a, + _Tp __pattern, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp volatile& __a_value){ + __old = __a_value; + __a_value |= __pattern; }); + return __old; +} + +template +static _LIBCPP_INLINE_VISIBILITY inline +_Tp __c11_atomic_fetch_xor(volatile __cxx_locked_atomic_type<_Tp, _Sco>* __a, + _Tp __pattern, memory_order) { + _Tp __old; + __a->writer_section([&](_Tp volatile& __a_value){ + __old = __a_value; + __a_value ^= __pattern; }); + return __old; +} + +template +using __atomic_type = typename conditional<__atomic_always_lock_free(sizeof(_Tp), 0), + _Atomic_s(_Tp, _Sco), + __cxx_locked_atomic_type<_Tp, _Sco>>::type; + +#else + +template +using __atomic_type = _Atomic_s(_Tp, _Sco); + +#endif //_LIBCPP_FREESTANDING && __cpp_lib_atomic_is_always_lock_free + // general atomic -template ::value && !is_same<_Tp, bool>::value> +#ifndef _LIBCPP_ATOMIC_SCOPE_DEFAULT +#define _LIBCPP_ATOMIC_SCOPE_DEFAULT int +#endif + +template ::value && !is_same<_Tp, bool>::value> struct __atomic_base // false { - mutable _Atomic(_Tp) __a_; + mutable __atomic_type<_Tp, _Sco> __a_; #if defined(__cpp_lib_atomic_is_always_lock_free) static _LIBCPP_CONSTEXPR bool is_always_lock_free = __atomic_always_lock_free(sizeof(__a_), 0); @@ -992,19 +1165,23 @@ }; #if defined(__cpp_lib_atomic_is_always_lock_free) -template -_LIBCPP_CONSTEXPR bool __atomic_base<_Tp, __b>::is_always_lock_free; +template +_LIBCPP_CONSTEXPR bool __atomic_base<_Tp, _Sco, __b>::is_always_lock_free; #endif // atomic -template -struct __atomic_base<_Tp, true> - : public __atomic_base<_Tp, false> +template +struct __atomic_base<_Tp, _Sco, true> + : public __atomic_base<_Tp, _Sco, false> { - typedef __atomic_base<_Tp, false> __base; + typedef __atomic_base<_Tp, _Sco, false> __base; _LIBCPP_INLINE_VISIBILITY - __atomic_base() _NOEXCEPT _LIBCPP_DEFAULT +#ifndef _LIBCPP_CXX03_LANG + __atomic_base() _NOEXCEPT = default; +#else + __atomic_base() _NOEXCEPT : __base() {} +#endif // _LIBCPP_CXX03_LANG _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {} @@ -1085,7 +1262,11 @@ { typedef __atomic_base<_Tp> __base; _LIBCPP_INLINE_VISIBILITY - atomic() _NOEXCEPT _LIBCPP_DEFAULT +#ifndef _LIBCPP_CXX03_LANG + atomic() _NOEXCEPT = default; +#else + atomic() _NOEXCEPT : __base() {} +#endif // _LIBCPP_CXX03_LANG _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {} @@ -1105,7 +1286,11 @@ { typedef __atomic_base<_Tp*> __base; _LIBCPP_INLINE_VISIBILITY - atomic() _NOEXCEPT _LIBCPP_DEFAULT +#ifndef _LIBCPP_CXX03_LANG + atomic() _NOEXCEPT = default; +#else + atomic() _NOEXCEPT : __base() {} +#endif // _LIBCPP_CXX03_LANG _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {} @@ -1716,22 +1901,26 @@ // flag type and operations +#ifndef _LIBCPP_ATOMIC_FLAG_TYPE +#define _LIBCPP_ATOMIC_FLAG_TYPE bool +#endif + typedef struct atomic_flag { - _Atomic(bool) __a_; + _Atomic(_LIBCPP_ATOMIC_FLAG_TYPE) __a_; _LIBCPP_INLINE_VISIBILITY bool test_and_set(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {return __c11_atomic_exchange(&__a_, true, __m);} + {return __c11_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} _LIBCPP_INLINE_VISIBILITY bool test_and_set(memory_order __m = memory_order_seq_cst) _NOEXCEPT - {return __c11_atomic_exchange(&__a_, true, __m);} + {return __c11_atomic_exchange(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(true), __m);} _LIBCPP_INLINE_VISIBILITY void clear(memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT - {__c11_atomic_store(&__a_, false, __m);} + {__c11_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} _LIBCPP_INLINE_VISIBILITY void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT - {__c11_atomic_store(&__a_, false, __m);} + {__c11_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);} _LIBCPP_INLINE_VISIBILITY #ifndef _LIBCPP_CXX03_LANG