tsan: optimize test-only barrier

[dvyukov]

Summary

The updated lots_of_threads.c test with 300 threads
started running for too long on machines with low
hardware parallelism (e.g. taskset -c 0-1).
On lots of CPUs it finishes in ~2 secs. But with
taskset -c 0-1 it runs for hundreds of seconds
effectively spinning in the barrier in the sleep loop.

We now have the handy futex API in sanitizer_common.
Use it instead of the passive spin loop.
It makes the test run only faster with taskset -c 0-1,
it runs for ~1.5 secs, while with full parallelism
it still runs for ~2 secs (but consumes less CPU time).

Depends on D107131.

compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp

 // 2. Then we switched to atomics+usleep(10). But usleep produced parasitic
 //    "as-if synchronized via sleep" messages in reports which failed some
 //    output tests.
 // 3. Then we switched to atomics+sched_yield. But this produced tons of tsan-
 //    visible events, which lead to "failed to restore stack trace" failures.
 // Note that no_sanitize_thread attribute does not turn off atomic interception
 // so attaching it to the function defined in user code does not help.
 // That's why we now have what we have.
-constexpr uptr kBarrierThreadBits = 10;
-constexpr uptr kBarrierThreads = 1 << kBarrierThreadBits;
+constexpr u32 kBarrierThreadBits = 10;
+constexpr u32 kBarrierThreads = 1 << kBarrierThreadBits;
 
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_testonly_barrier_init(u64 *barrier, u32 count) {
-  if (count >= kBarrierThreads) {
-    Printf("barrier_init: count is too large (%d)\n", count);
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init(
+    atomic_uint32_t *barrier, u32 num_threads) {
+  if (num_threads >= kBarrierThreads) {
+    Printf("barrier_init: count is too large (%d)\n", num_threads);
     Die();
   }
   // kBarrierThreadBits lsb is thread count,
   // the remaining are count of entered threads.
-  *barrier = count;
+  atomic_store(barrier, num_threads, memory_order_relaxed);
 }
 
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_testonly_barrier_wait(u64 *barrier) {
-  constexpr uptr kThreadMask = kBarrierThreads - 1;
-  unsigned old = __atomic_fetch_add(barrier, kBarrierThreads, __ATOMIC_RELAXED);
-  unsigned old_epoch = (old >> kBarrierThreadBits) / (old & kThreadMask);
+static u32 barrier_epoch(u32 value) {
+  return (value >> kBarrierThreadBits) / (value & (kBarrierThreads - 1));
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait(
+    atomic_uint32_t *barrier) {
+  u32 old = atomic_fetch_add(barrier, kBarrierThreads, memory_order_relaxed);
+  u32 old_epoch = barrier_epoch(old);
+  if (barrier_epoch(old + kBarrierThreads) != old_epoch) {
+    FutexWake(barrier, (1 << 30));
+    return;
+  }
   for (;;) {
-    unsigned cur = __atomic_load_n(barrier, __ATOMIC_RELAXED);
-    unsigned cur_epoch = (cur >> kBarrierThreadBits) / (cur & kThreadMask);
-    if (cur_epoch != old_epoch)
-      break;
-    internal_usleep(100);
+    u32 cur = atomic_load(barrier, memory_order_relaxed);
+    if (barrier_epoch(cur) != old_epoch)
+      return;
+    FutexWait(barrier, cur);
   }
 }

compiler-rt/test/tsan/test.h

 
 #ifdef __APPLE__
 #include <mach/mach_time.h>
 #endif
 
 // TSan-invisible barrier.
 // Tests use it to establish necessary execution order in a way that does not
 // interfere with tsan (does not establish synchronization between threads).
-typedef unsigned long long invisible_barrier_t;
+typedef unsigned invisible_barrier_t;
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 void __tsan_testonly_barrier_init(invisible_barrier_t *barrier,
     unsigned count);
 void __tsan_testonly_barrier_wait(invisible_barrier_t *barrier);
 unsigned long __tsan_testonly_shadow_stack_current_size();