tsan: don't use spinning in __cxa_guard_acquire/pthread_once

[dvyukov]

Summary

Currently we use passive spinning with internal_sched_yield to wait
in __cxa_guard_acquire/pthread_once. Passive spinning tends to degrade
ungracefully under high load. Use FutexWait/Wake instead.

Depends on D107359.

[vitalybuka]

compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp

853

static?

873

looks like only two atomic_* lines are related to the description
the rest looks like a separate NFC patch

[dvyukov]

compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp

853

Why?
https://reviews.llvm.org/D106953#inline-1018509

[vitalybuka]

compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp

853

Thanks.

[dvyukov]

split introduction of named constants into separate commit

[dvyukov]

compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp

873

Moved introduction of named consts into https://reviews.llvm.org/D107445

compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp

if (in_symbolizer()) {
     *memptr = p;
     return 0;
   }
   SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz);
   return user_posix_memalign(thr, pc, memptr, align, sz);
 }
 #endif
 
+// Both __cxa_guard_acquire and pthread_once 0-initialize
+// the object initially. pthread_once does not have any
+// other ABI requirements. __cxa_guard_acquire assumes
+// that any non-0 value in the first byte means that
+// initialization is completed. Contents of the remaining
+// bytes are up to us.
+constexpr u32 kGuardInit = 0;
+constexpr u32 kGuardDone = 1;
+constexpr u32 kGuardRunning = 1 << 16;
+constexpr u32 kGuardWaiter = 1 << 17;
+
 static int guard_acquire(ThreadState *thr, uptr pc, atomic_uint32_t *g) {
   OnPotentiallyBlockingRegionBegin();
   auto on_exit = at_scope_exit(&OnPotentiallyBlockingRegionEnd);
   for (;;) {
     u32 cmp = atomic_load(g, memory_order_acquire);
-    if (cmp == 0) {
-      if (atomic_compare_exchange_strong(g, &cmp, 1 << 16,
+    if (cmp == kGuardInit) {
+      if (atomic_compare_exchange_strong(g, &cmp, kGuardRunning,
                                          memory_order_relaxed))
         return 1;
-    } else if (cmp == 1) {
+    } else if (cmp == kGuardDone) {
       if (!thr->in_ignored_lib)
         Acquire(thr, pc, (uptr)g);
       return 0;
     } else {
-      internal_sched_yield();
+      if ((cmp & kGuardWaiter) ||
+          atomic_compare_exchange_strong(g, &cmp, cmp | kGuardWaiter,
+                                         memory_order_relaxed))
+        FutexWait(g, cmp | kGuardWaiter);
     }
   }
 }
 
 static void guard_release(ThreadState *thr, uptr pc, atomic_uint32_t *g) {
   if (!thr->in_ignored_lib)
     Release(thr, pc, (uptr)g);
-  atomic_store(g, 1, memory_order_release);
+  u32 old = atomic_exchange(g, kGuardDone, memory_order_release);
+  if (old & kGuardWaiter)
+    FutexWake(g, 1 << 30);
 }
 
 // __cxa_guard_acquire and friends need to be intercepted in a special way -
 // regular interceptors will break statically-linked libstdc++. Linux
 // interceptors are especially defined as weak functions (so that they don't
 // cause link errors when user defines them as well). So they silently
 // auto-disable themselves when such symbol is already present in the binary. If
 // we link libstdc++ statically, it will bring own __cxa_guard_acquire which

 
 STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) {
   SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g);
   guard_release(thr, pc, g);
 }
 
 STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) {
   SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g);
-  atomic_store(g, 0, memory_order_relaxed);
+  atomic_store(g, kGuardInit, memory_order_relaxed);
 }
 
 namespace __tsan {
 void DestroyThreadState() {
   ThreadState *thr = cur_thread();
   Processor *proc = thr->proc();
   ThreadFinish(thr);
   ProcUnwire(proc, thr);