[ASan][Darwin] Ensure we always register GCD worker threads

[yln]

Summary

In rare cases, our interceptors for dispatch_async (and friends) are
not enough to reliably observe GCD worker thread creation. When we
later find a bug (via interceptors) on a thread that hasn't been
registered we crash during report generation because we never set the
current thread context (via SetCurrentThread()).

This change adds a "pthread introspection hook", to guarantee ASan
initialization for GCD worker threads. This code is similiar to what we
are already doing for TSan, so I tried to factor out the common code.

[wrotki]

LVGTM - very DRY :)

[delcypher]

Really nice refactor. LGTM.

[delcypher]

compiler-rt/lib/asan/asan_mac.cpp

167

Sidenote: Should we eventually move (i.e. not in this patch) to this mechanism to register all threads (i.e. not just GCD worker threads)? It might be nice if there was only one way we registered threads rather than two.

[yln]

compiler-rt/lib/asan/asan_mac.cpp

167

I thought about this as well. There is 2 questions: "Should we do it?" and "Is it worth it?".

For the first question, I think the main argument for it is:

• Consistency: it would be nice to have one place/code path that does it

But my feeling is the the argument against it is stronger:

• Minimize differences with other platforms. When other things/places in the sanitizer make assumptions then "mostly default + Apple corner cases" is probably a better situation then "Apple uses different mechanism".

Both options aren't optimal for maintenance, but I feel like the 2nd one is lower risk.

compiler-rt/lib/asan/asan_mac.cpp

 //===-- asan_mac.cpp ------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //

 // from <crt_externs.h>, but we don't have that file on iOS
 extern "C" {
   extern char ***_NSGetArgv(void);
   extern char ***_NSGetEnviron(void);
 }
 
 namespace __asan {
 
-void InitializePlatformInterceptors() {}
+static void ThreadCreateCallback(uptr thread, bool gcd_worker);
+void InitializePlatformInterceptors() {
+  ThreadEventCallbacks callbacks = {
+      .create = ThreadCreateCallback,
+  };
+  InstallPthreadIntrospectionHook(callbacks);
+}
 void InitializePlatformExceptionHandlers() {}
 bool IsSystemHeapAddress (uptr addr) { return false; }
 
 // No-op. Mac does not support static linkage anyway.
 void *AsanDoesNotSupportStaticLinkage() {
   return 0;
 }
 
t = AsanThread::Create(/* start_routine */ nullptr, /* arg */ nullptr,
                            parent_tid, stack, /* detached */ true);
     t->Init();
     asanThreadRegistry().StartThread(t->tid(), GetTid(), ThreadType::Worker,
                                      nullptr);
     SetCurrentThread(t);
   }
 }
 
+// Make sure we observe the creation of all GCD worker threads, even those that
+// run blocks that aren't scheduled via the intercepted APIs below.
+static void ThreadCreateCallback(uptr thread, bool gcd_worker) {
+  if (gcd_worker) {
+    GET_STACK_TRACE_THREAD;
+    asan_register_worker_thread(/*parent_tid=*/0, &stack);
+  }
+}
+
 // For use by only those functions that allocated the context via
 // alloc_asan_context().
 extern "C"
 void asan_dispatch_call_block_and_release(void *block) {
   GET_STACK_TRACE_THREAD;
   asan_block_context_t *context = (asan_block_context_t*)block;
   VReport(2,
           "asan_dispatch_call_block_and_release(): "

compiler-rt/lib/sanitizer_common/sanitizer_mac.h

 //===-- sanitizer_mac.h -----------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //

 
 MacosVersion GetMacosAlignedVersion();
 DarwinKernelVersion GetDarwinKernelVersion();
 
 char **GetEnviron();
 
 void RestrictMemoryToMaxAddress(uptr max_address);
 
+using ThreadEventCallback = void (*)(uptr thread);
+using ThreadCreateEventCallback = void (*)(uptr thread, bool gcd_worker);
+struct ThreadEventCallbacks {
+  ThreadCreateEventCallback create;
+  ThreadEventCallback start;
+  ThreadEventCallback terminate;
+  ThreadEventCallback destroy;
+};
+
+void InstallPthreadIntrospectionHook(const ThreadEventCallbacks &callbacks);
+
 }  // namespace __sanitizer
 
 #endif  // SANITIZER_MAC
 #endif  // SANITIZER_MAC_H

compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp

 //===-- sanitizer_mac.cpp -------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //

 #include <libkern/OSAtomic.h>
 #include <mach-o/dyld.h>
 #include <mach/mach.h>
 #include <mach/mach_time.h>
 #include <mach/vm_statistics.h>
 #include <malloc/malloc.h>
 #include <os/log.h>
 #include <pthread.h>
+#include <pthread/introspection.h>
 #include <sched.h>
 #include <signal.h>
 #include <spawn.h>
 #include <stdlib.h>
 #include <sys/ioctl.h>
 #include <sys/mman.h>
 #include <sys/resource.h>
 #include <sys/stat.h>

 }
 
 u32 GetNumberOfCPUs() {
   return (u32)sysconf(_SC_NPROCESSORS_ONLN);
 }
 
 void InitializePlatformCommonFlags(CommonFlags *cf) {}
 
+// Pthread introspection hook
+//
+// * GCD worker threads are created without a call to pthread_create(), but we
+//   still need to register these threads (with ThreadCreate/Start()).
+// * We use the "pthread introspection hook" below to observe the creation of
+//   such threads.
+// * GCD worker threads don't have parent threads and the CREATE event is
+//   delivered in the context of the thread itself.  CREATE events for regular
+//   threads, are delivered on the parent.  We use this to tell apart which
+//   threads are GCD workers with `thread == pthread_self()`.
+//
+static pthread_introspection_hook_t prev_pthread_introspection_hook;
+static ThreadEventCallbacks thread_event_callbacks;
+
+static void sanitizer_pthread_introspection_hook(unsigned int event,
+                                                 pthread_t thread, void *addr,
+                                                 size_t size) {
+  // create -> start -> terminate -> destroy
+  // * create/destroy are usually (not guaranteed) delivered on the parent and
+  //   track resource allocation/reclamation
+  // * start/terminate are guaranteed to be delivered in the context of the
+  //   thread and give hooks into "just after (before) thread starts (stops)
+  //   executing"
+  DCHECK(event >= PTHREAD_INTROSPECTION_THREAD_CREATE &&
+         event <= PTHREAD_INTROSPECTION_THREAD_DESTROY);
+
+  if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) {
+    bool gcd_worker = thread == pthread_self();
+    if (thread_event_callbacks.create)
+      thread_event_callbacks.create((uptr)thread, gcd_worker);
+  } else if (event == PTHREAD_INTROSPECTION_THREAD_START) {
+    CHECK_EQ(thread, pthread_self());
+    if (thread_event_callbacks.start)
+      thread_event_callbacks.start((uptr)thread);
+  }
+
+  if (prev_pthread_introspection_hook)
+    prev_pthread_introspection_hook(event, thread, addr, size);
+
+  if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) {
+    CHECK_EQ(thread, pthread_self());
+    if (thread_event_callbacks.terminate)
+      thread_event_callbacks.terminate((uptr)thread);
+  } else if (event == PTHREAD_INTROSPECTION_THREAD_DESTROY) {
+    if (thread_event_callbacks.destroy)
+      thread_event_callbacks.destroy((uptr)thread);
+  }
+}
+
+void InstallPthreadIntrospectionHook(const ThreadEventCallbacks &callbacks) {
+  prev_pthread_introspection_hook =
+      pthread_introspection_hook_install(&sanitizer_pthread_introspection_hook);
+  thread_event_callbacks = callbacks;
+}
+
 }  // namespace __sanitizer
 
 #endif  // SANITIZER_MAC

compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp

 //===-- tsan_platform_mac.cpp ---------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
#  else  // !SANITIZER_GO
       LoAppMemBeg(), LoAppMemEnd(), app_res / 1024, app_dirty / 1024,
 #  endif
       stacks.n_uniq_ids, stacks.allocated / 1024, nthread, nlive);
 }
 
 #  if !SANITIZER_GO
 void InitializeShadowMemoryPlatform() { }
 
-// On OS X, GCD worker threads are created without a call to pthread_create. We
-// need to properly register these threads with ThreadCreate and ThreadStart.
-// These threads don't have a parent thread, as they are created "spuriously".
-// We're using a libpthread API that notifies us about a newly created thread.
-// The `thread == pthread_self()` check indicates this is actually a worker
-// thread. If it's just a regular thread, this hook is called on the parent
-// thread.
-typedef void (*pthread_introspection_hook_t)(unsigned int event,
-                                             pthread_t thread, void *addr,
-                                             size_t size);
-extern "C" pthread_introspection_hook_t pthread_introspection_hook_install(
-    pthread_introspection_hook_t hook);
-static const uptr PTHREAD_INTROSPECTION_THREAD_CREATE = 1;
-static const uptr PTHREAD_INTROSPECTION_THREAD_TERMINATE = 3;
-static pthread_introspection_hook_t prev_pthread_introspection_hook;
-static void my_pthread_introspection_hook(unsigned int event, pthread_t thread,
-                                          void *addr, size_t size) {
-  if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) {
-    if (thread == pthread_self()) {
-      // The current thread is a newly created GCD worker thread.
-      ThreadState *thr = cur_thread();
-      Processor *proc = ProcCreate();
-      ProcWire(proc, thr);
-      ThreadState *parent_thread_state = nullptr;  // No parent.
-      Tid tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true);
-      CHECK_NE(tid, kMainTid);
-      ThreadStart(thr, tid, GetTid(), ThreadType::Worker);
-    }
-  } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) {
-    CHECK_EQ(thread, pthread_self());
-    ThreadState *thr = cur_thread();
-    if (thr->tctx) {
-      DestroyThreadState();
-    }
-  }
-
-  if (prev_pthread_introspection_hook != nullptr)
-    prev_pthread_introspection_hook(event, thread, addr, size);
-}
+// Register GCD worker threads, which are created without an observable call to
+// pthread_create().
+static void ThreadCreateCallback(uptr thread, bool gcd_worker) {
+  if (gcd_worker) {
+}
+
+// Destroy thread state for *all* threads.
+static void ThreadTerminateCallback(uptr thread) {
 #endif
 
 void InitializePlatformEarly() {
 #  if !SANITIZER_GO && SANITIZER_IOS
   uptr max_vm = GetMaxUserVirtualAddress() + 1;
   if (max_vm != HiAppMemEnd()) {
     Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
            (void *)max_vm, (void *)HiAppMemEnd());
     Die();
   }
 #endif
 }
 
 static uptr longjmp_xor_key = 0;
 
 void InitializePlatform() {
   DisableCoreDumperIfNecessary();
 #if !SANITIZER_GO
   CheckAndProtect();
 
   InitializeThreadStateStorage();
 
-  prev_pthread_introspection_hook =
-      pthread_introspection_hook_install(&my_pthread_introspection_hook);
+  ThreadEventCallbacks callbacks = {
+      .create = ThreadCreateCallback,
+      .terminate = ThreadTerminateCallback,
+  };
+  InstallPthreadIntrospectionHook(callbacks);
 #endif
 
   if (GetMacosAlignedVersion() >= MacosVersion(10, 14)) {
     // Libsystem currently uses a process-global key; this might change.
     const unsigned kTLSLongjmpXorKeySlot = 0x7;
     longjmp_xor_key = (uptr)pthread_getspecific(kTLSLongjmpXorKeySlot);
   }
 }