[scudo] Make HybridMutex be aware of waiters from ConditionVariable

[Chia-hungDuan]

Locking mutex doesn't need to increase the NumWaiters

compiler-rt/lib/scudo/standalone/condition_variable_linux.cpp

 
 #include "atomic_helpers.h"
 
 #include <limits.h>
 #include <linux/futex.h>
 #include <sys/syscall.h>
 #include <unistd.h>
 
+#include <errno.h>
+
 namespace scudo {
 
 void ConditionVariableLinux::notifyAllImpl(UNUSED HybridMutex &M) {
   u32 V = atomic_exchange(&CurState, State::AllAwoken, memory_order_acq_rel);
 
   // TODO(chiahungduan): Move the waiters from the futex waiting queue
   // `CurState` to futex waiting queue `M` so that the awoken threads won't be
   // blocked again due to locked `M` by current thread.
   if (V == State::Waiting) {
-    syscall(SYS_futex, reinterpret_cast<uptr>(&CurState), FUTEX_WAKE_PRIVATE,
-            INT_MAX, nullptr, nullptr, 0);
+    long NumMoved = syscall(SYS_futex, reinterpret_cast<uptr>(&CurState),
+                            FUTEX_CMP_REQUEUE_PRIVATE, /* Threads to wake */ 0,
+                            /* Threads to requeue */ INT_MAX,
+                            reinterpret_cast<uptr>(&M.M), State::AllAwoken);
+    DCHECK_NE(NumMoved, -1L);
+    atomic_fetch_add(&M.NumWaiters, static_cast<u32>(NumMoved),
+                     memory_order_release);
   }
 }
 
 void ConditionVariableLinux::waitImpl(HybridMutex &M) {
   atomic_exchange(&CurState, State::Waiting, memory_order_acq_rel);
+  atomic_fetch_add(&M.NumWaiters, 1U, memory_order_release);
 
   // TODO: Use ScopedUnlock when it's supported.
   M.unlock();
   syscall(SYS_futex, reinterpret_cast<uptr>(&CurState), FUTEX_WAIT_PRIVATE,
           State::Waiting, nullptr, nullptr, 0);
   M.lock();
+  atomic_fetch_sub(&M.NumWaiters, 1U, memory_order_release);
 }
 
 } // namespace scudo
 
 #endif // SCUDO_LINUX

compiler-rt/lib/scudo/standalone/linux.cpp

void HybridMutex::lockSlow() {
   while (V != Unlocked) {
     syscall(SYS_futex, reinterpret_cast<uptr>(&M), FUTEX_WAIT_PRIVATE, Sleeping,
             nullptr, nullptr, 0);
     V = atomic_exchange(&M, Sleeping, memory_order_acquire);
   }
 }
 
 void HybridMutex::unlock() {
+  bool DoAwake = false;
+  // Note that this has to be done before releasing the lock.
+  if (atomic_load(&NumWaiters, memory_order_acquire) > 0)
+    DoAwake = true;
   if (atomic_fetch_sub(&M, 1U, memory_order_release) != Locked) {
+    DoAwake = true;
     atomic_store(&M, Unlocked, memory_order_release);
+  }
+  if (DoAwake) {
     syscall(SYS_futex, reinterpret_cast<uptr>(&M), FUTEX_WAKE_PRIVATE, 1,
             nullptr, nullptr, 0);
   }
 }
 
 void HybridMutex::assertHeldImpl() {
   CHECK(atomic_load(&M, memory_order_acquire) != Unlocked);
 }

compiler-rt/lib/scudo/standalone/mutex.h

 
 private:
   void assertHeldImpl();
 
   static constexpr u8 NumberOfTries = 8U;
   static constexpr u8 NumberOfYields = 8U;
 
 #if SCUDO_LINUX
+  friend class ConditionVariableLinux;
+  // TODO: Maybe this can be non-atomic variable
+  atomic_u32 NumWaiters = {};
   atomic_u32 M = {};
 #elif SCUDO_FUCHSIA
   sync_mutex_t M = {};
 #endif
 
   void lockSlow() ACQUIRE();
 };