Index: include/__mutex_base =================================================================== --- include/__mutex_base +++ include/__mutex_base @@ -11,6 +11,7 @@ #ifndef _LIBCPP___MUTEX_BASE #define _LIBCPP___MUTEX_BASE +#include #include <__config> #include #include @@ -288,17 +289,16 @@ class _LIBCPP_TYPE_VIS condition_variable { #ifndef _LIBCPP_CXX03_LANG - __libcpp_condvar_t __cv_ = _LIBCPP_CONDVAR_INITIALIZER; + atomic_uint __state{0}; #else - __libcpp_condvar_t __cv_; + atomic_uint __state; #endif - public: _LIBCPP_INLINE_VISIBILITY #ifndef _LIBCPP_CXX03_LANG constexpr condition_variable() _NOEXCEPT = default; #else - condition_variable() _NOEXCEPT {__cv_ = (__libcpp_condvar_t)_LIBCPP_CONDVAR_INITIALIZER;} + condition_variable() _NOEXCEPT {atomic_init(&__state, (unsigned int)0);} #endif ~condition_variable(); @@ -315,11 +315,23 @@ _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS void wait(unique_lock& __lk, _Predicate __pred); - template + template _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS cv_status wait_until(unique_lock& __lk, - const chrono::time_point<_Clock, _Duration>& __t); + const chrono::time_point& __t); + + template + _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS + cv_status + wait_until(unique_lock& __lk, + const chrono::time_point& __t); + + template + _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS + cv_status + wait_until(unique_lock& __lk, + const chrono::time_point<_Clock, _Duration>& __t); template _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS @@ -341,31 +353,20 @@ const chrono::duration<_Rep, _Period>& __d, _Predicate __pred); - typedef __libcpp_condvar_t* native_handle_type; - _LIBCPP_INLINE_VISIBILITY native_handle_type native_handle() {return &__cv_;} + typedef atomic_uint* native_handle_type; + _LIBCPP_INLINE_VISIBILITY native_handle_type native_handle() { + return reinterpret_cast(&__state); + } private: - void __do_timed_wait(unique_lock& __lk, - chrono::time_point) _NOEXCEPT; + int __wait(unique_lock& __lk, chrono::nanoseconds __t, + bool __rel_timeout, bool __realtime) _NOEXCEPT; + void __pulse(int thread_count) _NOEXCEPT; + int __futex(volatile void* ftx, int op, int value, + const struct timespec* timeout, int bitset) _NOEXCEPT; }; #endif // !_LIBCPP_HAS_NO_THREADS -template -inline _LIBCPP_INLINE_VISIBILITY -typename enable_if -< - chrono::__is_duration<_To>::value, - _To ->::type -__ceil(chrono::duration<_Rep, _Period> __d) -{ - using namespace chrono; - _To __r = duration_cast<_To>(__d); - if (__r < __d) - ++__r; - return __r; -} - #ifndef _LIBCPP_HAS_NO_THREADS template void @@ -375,6 +376,28 @@ wait(__lk); } +template +cv_status +condition_variable::wait_until(unique_lock& __lk, + const chrono::time_point& __t) +{ + auto __tp = chrono::time_point_cast(__t); + auto duration = __tp.time_since_epoch(); + return __wait(__lk, duration, false, true) == -ETIMEDOUT ? + cv_status::timeout : cv_status::no_timeout; +} + +template +cv_status +condition_variable::wait_until(unique_lock& __lk, + const chrono::time_point& __t) +{ + auto __tp = chrono::time_point_cast(__t); + auto duration = __tp.time_since_epoch(); + return __wait(__lk, duration, false, false) == -ETIMEDOUT ? + cv_status::timeout : cv_status::no_timeout; +} + template cv_status condition_variable::wait_until(unique_lock& __lk, @@ -404,20 +427,14 @@ condition_variable::wait_for(unique_lock& __lk, const chrono::duration<_Rep, _Period>& __d) { - using namespace chrono; if (__d <= __d.zero()) return cv_status::timeout; - typedef time_point > __sys_tpf; - typedef time_point __sys_tpi; - __sys_tpf _Max = __sys_tpi::max(); - steady_clock::time_point __c_now = steady_clock::now(); - system_clock::time_point __s_now = system_clock::now(); - if (_Max - __d > __s_now) - __do_timed_wait(__lk, __s_now + __ceil(__d)); - else - __do_timed_wait(__lk, __sys_tpi::max()); - return steady_clock::now() - __c_now < __d ? cv_status::no_timeout : - cv_status::timeout; + + auto duration_ns = chrono::duration_cast(__d); + if (duration_ns < __d) duration_ns++; + + return __wait(__lk, duration_ns, true, false) == -ETIMEDOUT ? + cv_status::timeout : cv_status::no_timeout; } template Index: src/condition_variable.cpp =================================================================== --- src/condition_variable.cpp +++ src/condition_variable.cpp @@ -11,6 +11,10 @@ #ifndef _LIBCPP_HAS_NO_THREADS +#include +#include +#include + #include "condition_variable" #include "thread" #include "system_error" @@ -20,60 +24,95 @@ condition_variable::~condition_variable() { - __libcpp_condvar_destroy(&__cv_); + atomic_store_explicit(&__state, 0xdeadc04d, memory_order_relaxed); +} + +void +condition_variable::__pulse(int thread_count) _NOEXCEPT +{ + _LIBCPP_CONSTEXPR unsigned int kCondCounterStep = 0x0004; + atomic_fetch_add_explicit(&__state, kCondCounterStep, memory_order_relaxed); + __futex(&__state, FUTEX_WAKE_PRIVATE, thread_count, nullptr, 0); } void condition_variable::notify_one() _NOEXCEPT { - __libcpp_condvar_signal(&__cv_); + __pulse(1); } void condition_variable::notify_all() _NOEXCEPT { - __libcpp_condvar_broadcast(&__cv_); + __pulse(numeric_limits::max()); } void condition_variable::wait(unique_lock& lk) _NOEXCEPT { - if (!lk.owns_lock()) - __throw_system_error(EPERM, - "condition_variable::wait: mutex not locked"); - int ec = __libcpp_condvar_wait(&__cv_, lk.mutex()->native_handle()); - if (ec) - __throw_system_error(ec, "condition_variable wait failed"); + __wait(lk, chrono::nanoseconds(0), false, false); } -void -condition_variable::__do_timed_wait(unique_lock& lk, - chrono::time_point tp) _NOEXCEPT +int +condition_variable::__wait(unique_lock& __lk, chrono::nanoseconds __t, + bool __rel_timeout, bool __realtime) _NOEXCEPT { + int op; + int val3; + + if (__t.count() == 0 || __rel_timeout) { + op = FUTEX_WAIT_PRIVATE; + val3 = 0; + } else if (__realtime) { + op = FUTEX_WAIT_BITSET_PRIVATE | FUTEX_CLOCK_REALTIME; + val3 = FUTEX_BITSET_MATCH_ANY; + } else { + op = FUTEX_WAIT_BITSET_PRIVATE; + val3 = FUTEX_BITSET_MATCH_ANY; + } + using namespace chrono; - if (!lk.owns_lock()) - __throw_system_error(EPERM, - "condition_variable::timed wait: mutex not locked"); - nanoseconds d = tp.time_since_epoch(); - if (d > nanoseconds(0x59682F000000E941)) - d = nanoseconds(0x59682F000000E941); + if (__t > nanoseconds(0x59682F000000E941)) + __t = nanoseconds(0x59682F000000E941); timespec ts; - seconds s = duration_cast(d); - typedef decltype(ts.tv_sec) ts_sec; - _LIBCPP_CONSTEXPR ts_sec ts_sec_max = numeric_limits::max(); - if (s.count() < ts_sec_max) - { - ts.tv_sec = static_cast(s.count()); - ts.tv_nsec = static_cast((d - s).count()); + timespec* pts = nullptr; + + if (__t.count() > 0) { + seconds s = duration_cast(__t); + typedef decltype(ts.tv_sec) ts_sec; + _LIBCPP_CONSTEXPR ts_sec ts_sec_max = numeric_limits::max(); + if (s.count() < ts_sec_max) + { + ts.tv_sec = static_cast(s.count()); + ts.tv_nsec = static_cast((__t - s).count()); + } + else + { + ts.tv_sec = ts_sec_max; + ts.tv_nsec = giga::num - 1; + } + pts = &ts; } - else - { - ts.tv_sec = ts_sec_max; - ts.tv_nsec = giga::num - 1; + + unsigned int old_state = atomic_load_explicit(&__state, memory_order_relaxed); + __lk.unlock(); + int ec = __futex(&__state, op, old_state, pts, val3); + __lk.lock(); + + return ec; +} + +int condition_variable::__futex(volatile void* ftx, int op, int value, + const struct timespec* timeout, + int bitset) _NOEXCEPT { + // Our generated syscall assembler sets errno, but our callers don't want to. + int saved_errno = errno; + int result = syscall(__NR_futex, ftx, op, value, timeout, NULL, bitset); + if (result == -1) { + result = -errno; + errno = saved_errno; } - int ec = __libcpp_condvar_timedwait(&__cv_, lk.mutex()->native_handle(), &ts); - if (ec != 0 && ec != ETIMEDOUT) - __throw_system_error(ec, "condition_variable timed_wait failed"); + return result; } void