Correct detection of a thread termination

[krytarowski]

Summary

Stop using the Linux solution with pthread_key_create(3).
This approach does not work on NetBSD, because calling
the thread destructor is not the latest operation on a POSIX
thread entity. NetBSD's libpthread still calls at least
pthread_mutex_lock and pthread_mutex_unlock.

Detect _lwp_exit(2) call as it is really the latest operation
called from a detaching POSIX thread.

This resolves one set of crashes observed in
the Thread Sanitizer execution.

Sponsored by <The NetBSD Foundation>

[krytarowski]

For the reference: late fixup committed in SVN http://llvm.org/viewvc/llvm-project?rev=317369&view=rev

Index: lib/tsan/rtl/tsan_interceptors.cc
===================================================================
--- lib/tsan/rtl/tsan_interceptors.cc   (wersja 317367)
+++ lib/tsan/rtl/tsan_interceptors.cc   (kopia robocza)
@@ -2451,8 +2451,8 @@
 #if SANITIZER_NETBSD
 TSAN_INTERCEPTOR(void, _lwp_exit) {
   SCOPED_TSAN_INTERCEPTOR(_lwp_exit);
+  DestroyThreadState();
   REAL(_lwp_exit)();
-  DestroyThreadState();
 }
 #define TSAN_MAYBE_INTERCEPT__LWP_EXIT TSAN_INTERCEPT(_lwp_exit)
 #else

lib/tsan/rtl/tsan_interceptors.cc

static ThreadSignalContext *SigCtx(ThreadState *thr) {
   if (ctx == 0 && !thr->is_dead) {
     ctx = (ThreadSignalContext*)MmapOrDie(sizeof(*ctx), "ThreadSignalContext");
     MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx));
     thr->signal_ctx = ctx;
   }
   return ctx;
 }
 
-#if !SANITIZER_MAC
+#if !SANITIZER_MAC && !SANITIZER_NETBSD
 static unsigned g_thread_finalize_key;
 #endif
 
 ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname,
                                      uptr pc)
     : thr_(thr), pc_(pc), in_ignored_lib_(false), ignoring_(false) {
   Initialize(thr);
   if (!thr_->is_inited) return;
if (sctx) {
     thr->signal_ctx = 0;
     UnmapOrDie(sctx, sizeof(*sctx));
   }
   DTLS_Destroy();
   cur_thread_finalize();
 }
 }  // namespace __tsan
 
-#if !SANITIZER_MAC
+#if !SANITIZER_MAC && !SANITIZER_NETBSD
 static void thread_finalize(void *v) {
   uptr iter = (uptr)v;
   if (iter > 1) {
     if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) {
       Printf("ThreadSanitizer: failed to set thread key\n");
       Die();
     }
     return;
extern "C" void *__tsan_thread_start_func(void *arg) {
   ThreadParam *p = (ThreadParam*)arg;
   void* (*callback)(void *arg) = p->callback;
   void *param = p->param;
   int tid = 0;
   {
     ThreadState *thr = cur_thread();
     // Thread-local state is not initialized yet.
     ScopedIgnoreInterceptors ignore;
-#if !SANITIZER_MAC
+#if !SANITIZER_MAC && !SANITIZER_NETBSD
     ThreadIgnoreBegin(thr, 0);
     if (pthread_setspecific(g_thread_finalize_key,
                             (void *)GetPthreadDestructorIterations())) {
       Printf("ThreadSanitizer: failed to set thread key\n");
       Die();
     }
     ThreadIgnoreEnd(thr, 0);
 #endif
TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) {
   if (!dtv)
     return res;
   // New DTLS block has been allocated.
   MemoryResetRange(thr, 0, dtv->beg, dtv->size);
   return res;
 }
 #endif
 
+#if SANITIZER_NETBSD
+TSAN_INTERCEPTOR(void, _lwp_exit) {
+  SCOPED_TSAN_INTERCEPTOR(_lwp_exit);
+  REAL(_lwp_exit)();
+  DestroyThreadState();
+}
+#define TSAN_MAYBE_INTERCEPT__LWP_EXIT TSAN_INTERCEPT(_lwp_exit)
+#else
+#define TSAN_MAYBE_INTERCEPT__LWP_EXIT
+#endif
+
 namespace __tsan {
 
 static void finalize(void *arg) {
   ThreadState *thr = cur_thread();
   int status = Finalize(thr);
   // Make sure the output is not lost.
   FlushStreams();
   if (status)
#endif
   TSAN_INTERCEPT(on_exit);
   TSAN_INTERCEPT(__cxa_atexit);
   TSAN_INTERCEPT(_exit);
 
 #ifdef NEED_TLS_GET_ADDR
   TSAN_INTERCEPT(__tls_get_addr);
 #endif
 
+  TSAN_MAYBE_INTERCEPT__LWP_EXIT;
+
 #if !SANITIZER_MAC && !SANITIZER_ANDROID
   // Need to setup it, because interceptors check that the function is resolved.
   // But atexit is emitted directly into the module, so can't be resolved.
   REAL(atexit) = (int(*)(void(*)()))unreachable;
 #endif
 
   if (REAL(__cxa_atexit)(&finalize, 0, 0)) {
     Printf("ThreadSanitizer: failed to setup atexit callback\n");
     Die();
   }
 
-#if !SANITIZER_MAC
+#if !SANITIZER_MAC && !SANITIZER_NETBSD
   if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) {
     Printf("ThreadSanitizer: failed to create thread key\n");
     Die();
   }
 #endif
 
   FdInit();
 }