diff --git a/compiler-rt/lib/scudo/standalone/condition_variable_linux.cpp b/compiler-rt/lib/scudo/standalone/condition_variable_linux.cpp --- a/compiler-rt/lib/scudo/standalone/condition_variable_linux.cpp +++ b/compiler-rt/lib/scudo/standalone/condition_variable_linux.cpp @@ -19,6 +19,8 @@ #include #include +#include + namespace scudo { void ConditionVariableLinux::notifyAllImpl(UNUSED HybridMutex &M) { @@ -28,19 +30,26 @@ // `CurState` to futex waiting queue `M` so that the awoken threads won't be // blocked again due to locked `M` by current thread. if (V == State::Waiting) { - syscall(SYS_futex, reinterpret_cast(&CurState), FUTEX_WAKE_PRIVATE, - INT_MAX, nullptr, nullptr, 0); + long NumMoved = syscall(SYS_futex, reinterpret_cast(&CurState), + FUTEX_CMP_REQUEUE_PRIVATE, /* Threads to wake */ 0, + /* Threads to requeue */ INT_MAX, + reinterpret_cast(&M.M), State::AllAwoken); + DCHECK_NE(NumMoved, -1L); + atomic_fetch_add(&M.NumWaiters, static_cast(NumMoved), + memory_order_release); } } void ConditionVariableLinux::waitImpl(HybridMutex &M) { atomic_exchange(&CurState, State::Waiting, memory_order_acq_rel); + atomic_fetch_add(&M.NumWaiters, 1U, memory_order_release); // TODO: Use ScopedUnlock when it's supported. M.unlock(); syscall(SYS_futex, reinterpret_cast(&CurState), FUTEX_WAIT_PRIVATE, State::Waiting, nullptr, nullptr, 0); M.lock(); + atomic_fetch_sub(&M.NumWaiters, 1U, memory_order_release); } } // namespace scudo diff --git a/compiler-rt/lib/scudo/standalone/linux.cpp b/compiler-rt/lib/scudo/standalone/linux.cpp --- a/compiler-rt/lib/scudo/standalone/linux.cpp +++ b/compiler-rt/lib/scudo/standalone/linux.cpp @@ -122,8 +122,15 @@ } void HybridMutex::unlock() { + bool DoAwake = false; + // Note that this has to be done before releasing the lock. + if (atomic_load(&NumWaiters, memory_order_acquire) > 0) + DoAwake = true; if (atomic_fetch_sub(&M, 1U, memory_order_release) != Locked) { + DoAwake = true; atomic_store(&M, Unlocked, memory_order_release); + } + if (DoAwake) { syscall(SYS_futex, reinterpret_cast(&M), FUTEX_WAKE_PRIVATE, 1, nullptr, nullptr, 0); } diff --git a/compiler-rt/lib/scudo/standalone/mutex.h b/compiler-rt/lib/scudo/standalone/mutex.h --- a/compiler-rt/lib/scudo/standalone/mutex.h +++ b/compiler-rt/lib/scudo/standalone/mutex.h @@ -59,6 +59,9 @@ static constexpr u8 NumberOfYields = 8U; #if SCUDO_LINUX + friend class ConditionVariableLinux; + // TODO: Maybe this can be non-atomic variable + atomic_u32 NumWaiters = {}; atomic_u32 M = {}; #elif SCUDO_FUCHSIA sync_mutex_t M = {};