diff --git a/compiler-rt/lib/scudo/standalone/linux.cpp b/compiler-rt/lib/scudo/standalone/linux.cpp --- a/compiler-rt/lib/scudo/standalone/linux.cpp +++ b/compiler-rt/lib/scudo/standalone/linux.cpp @@ -104,33 +104,56 @@ } bool HybridMutex::tryLock() { - return atomic_compare_exchange(&M, Unlocked, Locked) == Unlocked; + constexpr u32 kHeldMask = 1UL; + const u32 V = atomic_load(&M, memory_order_acquire); + return (V & kHeldMask) == Unlocked && + atomic_compare_exchange(&M, V, (V | Locked)) == V; } // The following is based on https://akkadia.org/drepper/futex.pdf. void HybridMutex::lockSlow() { - u32 V = atomic_compare_exchange(&M, Unlocked, Locked); - if (V == Unlocked) - return; - if (V != Sleeping) - V = atomic_exchange(&M, Sleeping, memory_order_acquire); - while (V != Unlocked) { - syscall(SYS_futex, reinterpret_cast(&M), FUTEX_WAIT_PRIVATE, Sleeping, - nullptr, nullptr, 0); - V = atomic_exchange(&M, Sleeping, memory_order_acquire); - } + constexpr u32 kIncWaiter = 1UL << 1; + constexpr u32 kHeldMask = 1UL; + + do { + u32 V = atomic_load(&M, memory_order_acquire); + if ((V & kHeldMask) == Unlocked) { + if (atomic_compare_exchange(&M, V, (V | Locked)) == V) + return; + } else { + atomic_fetch_add(&M, kIncWaiter, memory_order_release); + V += kIncWaiter; + do { + if (syscall(SYS_futex, reinterpret_cast(&M), FUTEX_WAIT_PRIVATE, + V, nullptr, nullptr, 0) != 0) { + V = atomic_load(&M, memory_order_acquire); + } else { + break; + } + } while ((V & kHeldMask) != Unlocked); + atomic_fetch_sub(&M, kIncWaiter, memory_order_release); + } + } while (true); } void HybridMutex::unlock() { - if (atomic_fetch_sub(&M, 1U, memory_order_release) != Locked) { - atomic_store(&M, Unlocked, memory_order_release); - syscall(SYS_futex, reinterpret_cast(&M), FUTEX_WAKE_PRIVATE, 1, - nullptr, nullptr, 0); - } + constexpr u32 kHeldMask = 1UL; + + do { + const u32 V = atomic_load(&M, memory_order_acquire); + if (atomic_compare_exchange(&M, V, (V & ~kHeldMask)) == V) { + if ((V & ~kHeldMask) != 0) { + syscall(SYS_futex, reinterpret_cast(&M), FUTEX_WAKE_PRIVATE, 1, + nullptr, nullptr, 0); + } + return; + } + } while (true); } void HybridMutex::assertHeldImpl() { - CHECK(atomic_load(&M, memory_order_acquire) != Unlocked); + constexpr u32 kHeldMask = 1UL; + CHECK((atomic_load(&M, memory_order_acquire) & kHeldMask) != Unlocked); } u64 getMonotonicTime() { diff --git a/compiler-rt/lib/scudo/standalone/mutex.h b/compiler-rt/lib/scudo/standalone/mutex.h --- a/compiler-rt/lib/scudo/standalone/mutex.h +++ b/compiler-rt/lib/scudo/standalone/mutex.h @@ -55,8 +55,8 @@ private: void assertHeldImpl(); - static constexpr u8 NumberOfTries = 8U; - static constexpr u8 NumberOfYields = 8U; + static constexpr u8 NumberOfTries = 0U; + static constexpr u8 NumberOfYields = 0U; #if SCUDO_LINUX atomic_u32 M = {};