Index: libcxx/include/__atomic_support =================================================================== --- /dev/null +++ libcxx/include/__atomic_support @@ -0,0 +1,157 @@ +//===----------------------------------------------------------------------===//// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===//// + +#ifndef ATOMIC_SUPPORT_H +#define ATOMIC_SUPPORT_H + +#include <__config> + +#if defined(__clang__) && __has_builtin(__atomic_load_n) \ + && __has_builtin(__atomic_store_n) \ + && __has_builtin(__atomic_add_fetch) \ + && __has_builtin(__atomic_compare_exchange_n) \ + && defined(__ATOMIC_RELAXED) \ + && defined(__ATOMIC_CONSUME) \ + && defined(__ATOMIC_ACQUIRE) \ + && defined(__ATOMIC_RELEASE) \ + && defined(__ATOMIC_ACQ_REL) \ + && defined(__ATOMIC_SEQ_CST) +# define _LIBCPP_HAS_ATOMIC_BUILTINS +#elif !defined(__clang__) && defined(_GNUC_VER) && _GNUC_VER >= 407 +# define _LIBCPP_HAS_ATOMIC_BUILTINS +#endif + +#if !defined(_LIBCPP_HAS_ATOMIC_BUILTINS) && !defined(_LIBCPP_HAS_NO_THREADS) +# if defined(_MSC_VER) && !defined(__clang__) + _LIBCPP_WARNING("Building libc++ without __atomic builtins is unsupported") +# else +# warning Building libc++ without __atomic builtins is unsupported +# endif +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +namespace { + +#if defined(_LIBCPP_HAS_ATOMIC_BUILTINS) && !defined(_LIBCPP_HAS_NO_THREADS) + +enum __libcpp_atomic_order { + _AO_Relaxed = __ATOMIC_RELAXED, + _AO_Consume = __ATOMIC_CONSUME, + _AO_Acquire = __ATOMIC_ACQUIRE, + _AO_Release = __ATOMIC_RELEASE, + _AO_Acq_Rel = __ATOMIC_ACQ_REL, + _AO_Seq = __ATOMIC_SEQ_CST +}; + +template +inline _LIBCPP_INLINE_VISIBILITY +void __libcpp_atomic_store(_ValueType* __dest, _FromType __val, + int __order = _AO_Seq) +{ + __atomic_store_n(__dest, __val, __order); +} + +template +inline _LIBCPP_INLINE_VISIBILITY +void __libcpp_relaxed_store(_ValueType* __dest, _FromType __val) +{ + __atomic_store_n(__dest, __val, _AO_Relaxed); +} + +template +inline _LIBCPP_INLINE_VISIBILITY +_ValueType __libcpp_atomic_load(_ValueType const* __val, + int __order = _AO_Seq) +{ + return __atomic_load_n(__val, __order); +} + +template +inline _LIBCPP_INLINE_VISIBILITY +_ValueType __libcpp_atomic_add(_ValueType* __val, _AddType __a, + int __order = _AO_Seq) +{ + return __atomic_add_fetch(__val, __a, __order); +} + +template +inline _LIBCPP_INLINE_VISIBILITY +bool __libcpp_atomic_compare_exchange(_ValueType* __val, + _ValueType* __expected, _ValueType __after, + int __success_order = _AO_Seq, + int __fail_order = _AO_Seq) +{ + return __atomic_compare_exchange_n(__val, __expected, __after, true, + __success_order, __fail_order); +} + +#else // _LIBCPP_HAS_NO_THREADS + +enum __libcpp_atomic_order { + _AO_Relaxed, + _AO_Consume, + _AO_Acquire, + _AO_Release, + _AO_Acq_Rel, + _AO_Seq +}; + +template +inline _LIBCPP_INLINE_VISIBILITY +void __libcpp_atomic_store(_ValueType* __dest, _FromType __val, + int = 0) +{ + *__dest = __val; +} + +template +inline _LIBCPP_INLINE_VISIBILITY +void __libcpp_relaxed_store(_ValueType* __dest, _FromType __val) +{ + *__dest = __val; +} + +template +inline _LIBCPP_INLINE_VISIBILITY +_ValueType __libcpp_atomic_load(_ValueType const* __val, + int = 0) +{ + return *__val; +} + +template +inline _LIBCPP_INLINE_VISIBILITY +_ValueType __libcpp_atomic_add(_ValueType* __val, _AddType __a, + int = 0) +{ + return *__val += __a; +} + +template +inline _LIBCPP_INLINE_VISIBILITY +bool __libcpp_atomic_compare_exchange(_ValueType* __val, + _ValueType* __expected, _ValueType __after, + int = 0, int = 0) +{ + if (*__val == *__expected) { + *__val = __after; + return true; + } + *__expected = *__val; + return false; +} + +#endif // _LIBCPP_HAS_NO_THREADS + +} // end namespace + +_LIBCPP_END_NAMESPACE_STD + +#endif // ATOMIC_SUPPORT_H Index: libcxx/include/memory =================================================================== --- libcxx/include/memory +++ libcxx/include/memory @@ -638,6 +638,8 @@ #include #include +#include <__atomic_support> + #if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER) # include #endif @@ -3788,12 +3790,34 @@ #endif // _LIBCPP_STD_VER > 14 +namespace +{ +// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively) +// should be sufficient for thread safety. +// See https://llvm.org/bugs/show_bug.cgi?id=22803 +template __attribute__((always_inline)) +inline _LIBCPP_INLINE_VISIBILITY T +increment(T& t) _NOEXCEPT +{ + return __libcpp_atomic_add(&t, 1, _AO_Relaxed); +} + +template __attribute__((always_inline)) +inline _LIBCPP_INLINE_VISIBILITY T +decrement(T& t) _NOEXCEPT +{ + return __libcpp_atomic_add(&t, -1, _AO_Acq_Rel); +} +} // namespace + class _LIBCPP_EXCEPTION_ABI bad_weak_ptr : public std::exception { public: virtual ~bad_weak_ptr() _NOEXCEPT; - virtual const char* what() const _NOEXCEPT; + virtual const char* what() const _NOEXCEPT { + return "bad_weak_ptr"; + } }; _LIBCPP_NORETURN inline _LIBCPP_ALWAYS_INLINE @@ -3824,8 +3848,18 @@ explicit __shared_count(long __refs = 0) _NOEXCEPT : __shared_owners_(__refs) {} - void __add_shared() _NOEXCEPT; - bool __release_shared() _NOEXCEPT; + inline _LIBCPP_INLINE_VISIBILITY + void __add_shared() _NOEXCEPT { + increment(__shared_owners_); + } + inline _LIBCPP_INLINE_VISIBILITY + bool __release_shared() _NOEXCEPT { + if (decrement(__shared_owners_) == -1) { + __on_zero_shared(); + return true; + } + return false; + } _LIBCPP_INLINE_VISIBILITY long use_count() const _NOEXCEPT { return __libcpp_relaxed_load(&__shared_owners_) + 1; @@ -3846,9 +3880,19 @@ virtual ~__shared_weak_count(); public: - void __add_shared() _NOEXCEPT; - void __add_weak() _NOEXCEPT; - void __release_shared() _NOEXCEPT; + inline _LIBCPP_INLINE_VISIBILITY + void __add_shared() _NOEXCEPT { + __shared_count::__add_shared(); + } + inline _LIBCPP_INLINE_VISIBILITY + void __add_weak() _NOEXCEPT { + increment(__shared_weak_owners_); + } + inline _LIBCPP_INLINE_VISIBILITY + void __release_shared() _NOEXCEPT { + if (__shared_count::__release_shared()) + __release_weak(); + } void __release_weak() _NOEXCEPT; _LIBCPP_INLINE_VISIBILITY long use_count() const _NOEXCEPT {return __shared_count::use_count();} Index: libcxx/src/include/atomic_support.h =================================================================== --- libcxx/src/include/atomic_support.h +++ /dev/null @@ -1,158 +0,0 @@ -//===----------------------------------------------------------------------===//// -// -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===//// - -#ifndef ATOMIC_SUPPORT_H -#define ATOMIC_SUPPORT_H - -#include "__config" -#include "memory" // for __libcpp_relaxed_load - -#if defined(__clang__) && __has_builtin(__atomic_load_n) \ - && __has_builtin(__atomic_store_n) \ - && __has_builtin(__atomic_add_fetch) \ - && __has_builtin(__atomic_compare_exchange_n) \ - && defined(__ATOMIC_RELAXED) \ - && defined(__ATOMIC_CONSUME) \ - && defined(__ATOMIC_ACQUIRE) \ - && defined(__ATOMIC_RELEASE) \ - && defined(__ATOMIC_ACQ_REL) \ - && defined(__ATOMIC_SEQ_CST) -# define _LIBCPP_HAS_ATOMIC_BUILTINS -#elif !defined(__clang__) && defined(_GNUC_VER) && _GNUC_VER >= 407 -# define _LIBCPP_HAS_ATOMIC_BUILTINS -#endif - -#if !defined(_LIBCPP_HAS_ATOMIC_BUILTINS) && !defined(_LIBCPP_HAS_NO_THREADS) -# if defined(_MSC_VER) && !defined(__clang__) - _LIBCPP_WARNING("Building libc++ without __atomic builtins is unsupported") -# else -# warning Building libc++ without __atomic builtins is unsupported -# endif -#endif - -_LIBCPP_BEGIN_NAMESPACE_STD - -namespace { - -#if defined(_LIBCPP_HAS_ATOMIC_BUILTINS) && !defined(_LIBCPP_HAS_NO_THREADS) - -enum __libcpp_atomic_order { - _AO_Relaxed = __ATOMIC_RELAXED, - _AO_Consume = __ATOMIC_CONSUME, - _AO_Acquire = __ATOMIC_ACQUIRE, - _AO_Release = __ATOMIC_RELEASE, - _AO_Acq_Rel = __ATOMIC_ACQ_REL, - _AO_Seq = __ATOMIC_SEQ_CST -}; - -template -inline _LIBCPP_INLINE_VISIBILITY -void __libcpp_atomic_store(_ValueType* __dest, _FromType __val, - int __order = _AO_Seq) -{ - __atomic_store_n(__dest, __val, __order); -} - -template -inline _LIBCPP_INLINE_VISIBILITY -void __libcpp_relaxed_store(_ValueType* __dest, _FromType __val) -{ - __atomic_store_n(__dest, __val, _AO_Relaxed); -} - -template -inline _LIBCPP_INLINE_VISIBILITY -_ValueType __libcpp_atomic_load(_ValueType const* __val, - int __order = _AO_Seq) -{ - return __atomic_load_n(__val, __order); -} - -template -inline _LIBCPP_INLINE_VISIBILITY -_ValueType __libcpp_atomic_add(_ValueType* __val, _AddType __a, - int __order = _AO_Seq) -{ - return __atomic_add_fetch(__val, __a, __order); -} - -template -inline _LIBCPP_INLINE_VISIBILITY -bool __libcpp_atomic_compare_exchange(_ValueType* __val, - _ValueType* __expected, _ValueType __after, - int __success_order = _AO_Seq, - int __fail_order = _AO_Seq) -{ - return __atomic_compare_exchange_n(__val, __expected, __after, true, - __success_order, __fail_order); -} - -#else // _LIBCPP_HAS_NO_THREADS - -enum __libcpp_atomic_order { - _AO_Relaxed, - _AO_Consume, - _AO_Acquire, - _AO_Release, - _AO_Acq_Rel, - _AO_Seq -}; - -template -inline _LIBCPP_INLINE_VISIBILITY -void __libcpp_atomic_store(_ValueType* __dest, _FromType __val, - int = 0) -{ - *__dest = __val; -} - -template -inline _LIBCPP_INLINE_VISIBILITY -void __libcpp_relaxed_store(_ValueType* __dest, _FromType __val) -{ - *__dest = __val; -} - -template -inline _LIBCPP_INLINE_VISIBILITY -_ValueType __libcpp_atomic_load(_ValueType const* __val, - int = 0) -{ - return *__val; -} - -template -inline _LIBCPP_INLINE_VISIBILITY -_ValueType __libcpp_atomic_add(_ValueType* __val, _AddType __a, - int = 0) -{ - return *__val += __a; -} - -template -inline _LIBCPP_INLINE_VISIBILITY -bool __libcpp_atomic_compare_exchange(_ValueType* __val, - _ValueType* __expected, _ValueType __after, - int = 0, int = 0) -{ - if (*__val == *__expected) { - *__val = __after; - return true; - } - *__expected = *__val; - return false; -} - -#endif // _LIBCPP_HAS_NO_THREADS - -} // end namespace - -_LIBCPP_END_NAMESPACE_STD - -#endif // ATOMIC_SUPPORT_H Index: libcxx/src/memory.cpp =================================================================== --- libcxx/src/memory.cpp +++ libcxx/src/memory.cpp @@ -13,85 +13,17 @@ #include "mutex" #include "thread" #endif -#include "include/atomic_support.h" +#include "__atomic_support" _LIBCPP_BEGIN_NAMESPACE_STD -namespace -{ - -// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively) -// should be sufficient for thread safety. -// See https://llvm.org/bugs/show_bug.cgi?id=22803 -template -inline T -increment(T& t) _NOEXCEPT -{ - return __libcpp_atomic_add(&t, 1, _AO_Relaxed); -} - -template -inline T -decrement(T& t) _NOEXCEPT -{ - return __libcpp_atomic_add(&t, -1, _AO_Acq_Rel); -} - -} // namespace - const allocator_arg_t allocator_arg = allocator_arg_t(); bad_weak_ptr::~bad_weak_ptr() _NOEXCEPT {} -const char* -bad_weak_ptr::what() const _NOEXCEPT -{ - return "bad_weak_ptr"; -} +__shared_count::~__shared_count() {} -__shared_count::~__shared_count() -{ -} - -void -__shared_count::__add_shared() _NOEXCEPT -{ - increment(__shared_owners_); -} - -bool -__shared_count::__release_shared() _NOEXCEPT -{ - if (decrement(__shared_owners_) == -1) - { - __on_zero_shared(); - return true; - } - return false; -} - -__shared_weak_count::~__shared_weak_count() -{ -} - -void -__shared_weak_count::__add_shared() _NOEXCEPT -{ - __shared_count::__add_shared(); -} - -void -__shared_weak_count::__add_weak() _NOEXCEPT -{ - increment(__shared_weak_owners_); -} - -void -__shared_weak_count::__release_shared() _NOEXCEPT -{ - if (__shared_count::__release_shared()) - __release_weak(); -} +__shared_weak_count::~__shared_weak_count() {} void __shared_weak_count::__release_weak() _NOEXCEPT Index: libcxx/src/mutex.cpp =================================================================== --- libcxx/src/mutex.cpp +++ libcxx/src/mutex.cpp @@ -12,7 +12,7 @@ #include "limits" #include "system_error" #include "cassert" -#include "include/atomic_support.h" +#include "__atomic_support" _LIBCPP_BEGIN_NAMESPACE_STD #ifndef _LIBCPP_HAS_NO_THREADS