Defer StartBackgroundThread() and StopBackgroundThread() in TSan

[krytarowski]

Summary

NetBSD cannot spawn new POSIX thread entities in early
libc and libpthread initialization stage. Defer this to the point
of intercepting the first pthread_create(3) call.

This is the last change that makes Thread Sanitizer functional
on NetBSD/amd64 without downstream patches.

---

Testing Time: 64.91s

---

Failing Tests (5):

  ThreadSanitizer-x86_64 :: dtls.c
  ThreadSanitizer-x86_64 :: ignore_lib5.cc
  ThreadSanitizer-x86_64 :: ignored-interceptors-mmap.cc
  ThreadSanitizer-x86_64 :: mutex_lock_destroyed.cc
  ThreadSanitizer-x86_64 :: vfork.cc

Expected Passes    : 290
Expected Failures  : 1
Unsupported Tests  : 83
Unexpected Failures: 5

Sponsored by <The NetBSD Foundation>

[krytarowski]

Try to make the code prettier.

[dvyukov]

LGTM with a nit

lib/tsan/rtl/tsan_rtl.cc

423

atomic_exchange will be slightly simpler than CAS:

static atomic_uint32_t bg_thread = {};
if (atomic_load(&bg_thread, memory_order_relaxed) == 0 &&
    atomic_exchange(&bg_thread, 1, memory_order_relaxed) == 0) {
  StartBackgroundThread();
  SetSandboxingCallback(StopBackgroundThread);
}

[dvyukov]

lib/tsan/rtl/tsan_rtl.h

703

Another nit: please don't prefix function in tsan with tsan. Everything here is tsan-something. It does not add any value to add tsan everywhere.

[dvyukov]

lib/tsan/rtl/tsan_rtl.h

703

And this all is already in __tsan namespace. So this is actually tsan's __tsan::TSan...

[krytarowski]

• rename TSanMaybeSpawnBackgroundThread() to MaybeSpawnBackgroundThread ()
• simplify atomic check in MaybeSpawnBackgroundThread()

[krytarowski]

Thank you again for the great support!

Next MSan.

Status:

MSan - 1/3 tests pass, but needs more work
Scudo - I got it to build and pass some tests locally, however a lot of them are designed for <malloc.h> specific internals from GLIBC
XRay - it does not look to be far away, waiting for Darwin patches to get merged
LSan - still no functional StopTheWorld(),
KAsan - frontend/llvm part is done, but kernel part needs porting
KUBsan - same as KAsan
ESan - not started, appears to require more work to handle FILE*.

[eugenis]

Interesting. Did you see the same problem with ASan spawning its background thread (the one that does RSS limiting) too early as well? If not, then what's different between that and TSan?

[krytarowski]

In D40583#939467, @eugenis wrote:

Interesting. Did you see the same problem with ASan spawning its background thread (the one that does RSS limiting) too early as well? If not, then what's different between that and TSan?

Do you mean the code in sanitizer_common_libcdep.cc?

void MaybeStartBackgroudThread() {
#if SANITIZER_LINUX && \
    !SANITIZER_GO  // Need to implement/test on other platforms.
  // Start the background thread if one of the rss limits is given.
  if (!common_flags()->hard_rss_limit_mb &&
      !common_flags()->soft_rss_limit_mb &&
      !common_flags()->heap_profile) return;
  if (!&real_pthread_create) return;  // Can't spawn the thread anyway.
  internal_start_thread(BackgroundThread, nullptr);
#endif
}

It's Linux-only as of today.

[eugenis]

Yes, that's it. Thanks.

lib/tsan/rtl/tsan_interceptors.cc

#endif
   volatile int foo = 42;
   foo++;
   return res;
 }
 
 TSAN_INTERCEPTOR(int, pthread_create,
     void *th, void *attr, void *(*callback)(void*), void * param) {
   SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param);
+
+  MaybeSpawnBackgroundThread();
+
   if (ctx->after_multithreaded_fork) {
     if (flags()->die_after_fork) {
       Report("ThreadSanitizer: starting new threads after multi-threaded "
           "fork is not supported. Dying (set die_after_fork=0 to override)\n");
       Die();
     } else {
       VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded "
           "fork is not supported (pid %d). Continuing because of "

lib/tsan/rtl/tsan_rtl.h

 #endif
 
 u32 CurrentStackId(ThreadState *thr, uptr pc);
 ReportStack *SymbolizeStackId(u32 stack_id);
 void PrintCurrentStack(ThreadState *thr, uptr pc);
 void PrintCurrentStackSlow(uptr pc);  // uses libunwind
 
 void Initialize(ThreadState *thr);
+void MaybeSpawnBackgroundThread();
 int Finalize(ThreadState *thr);
 
 void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
 void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
 
 void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
     int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic);
 void MemoryAccessImpl(ThreadState *thr, uptr addr,

lib/tsan/rtl/tsan_rtl.cc

#if !SANITIZER_GO
   InstallDeadlySignalHandlers(TsanOnDeadlySignal);
 #endif
   // Setup correct file descriptor for error reports.
   __sanitizer_set_report_path(common_flags()->log_path);
   InitializeSuppressions();
 #if !SANITIZER_GO
   InitializeLibIgnore();
   Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
-  // On MIPS, TSan initialization is run before
-  // __pthread_initialize_minimal_internal() is finished, so we can not spawn
-  // new threads.
-#ifndef __mips__
-  StartBackgroundThread();
-  SetSandboxingCallback(StopBackgroundThread);
-#endif
 #endif
 
   VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n",
           (int)internal_getpid());
 
   // Initialize thread 0.
   int tid = ThreadCreate(thr, 0, 0, true);
   CHECK_EQ(tid, 0);
Printf("ThreadSanitizer is suspended at startup (pid %d)."
            " Call __tsan_resume().\n",
            (int)internal_getpid());
     while (__tsan_resumed == 0) {}
   }
 
   OnInitialize();
 }
 
+void MaybeSpawnBackgroundThread() {
+  // On MIPS, TSan initialization is run before
+  // __pthread_initialize_minimal_internal() is finished, so we can not spawn
+  // new threads.
+#if !SANITIZER_GO && !defined(__mips__)
+  static atomic_uint32_t bg_thread = {};
+  if (atomic_load(&bg_thread, memory_order_relaxed) == 0 &&
+      atomic_exchange(&bg_thread, 1, memory_order_relaxed) == 0) {
+    StartBackgroundThread();
+    SetSandboxingCallback(StopBackgroundThread);
+  }
+#endif
+}
+
+
 int Finalize(ThreadState *thr) {
   bool failed = false;
 
   if (common_flags()->print_module_map == 1) PrintModuleMap();
 
   if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)
     SleepForMillis(flags()->atexit_sleep_ms);