[tsan] Add a DCHECK to verify __tsan_read* and __tsan_write function aren't called from ScopedInterceptor

[kubamracek]

Summary

Interceptors using ScopedInteceptor should never call into user's code before the ScopedInterceptor is out of scope (and its destructor is called). Let's add a DCHECK to enforce that.

[kubamracek]

This currently fails several GCD tests (fix is in http://reviews.llvm.org/D15419), plus the sigsuspend.cc test. This fails because a signal handler (do_nothing_signal_handler) is called via this backtrace:

frame #0: __sanitizer::CheckFailed "((thr->in_interceptor_count)) == ((0))" at sanitizer_common.cc:158
frame #1:__tsan_write1(addr=0x0000000100001080) + 146 at tsan_interface_inl.h:39
frame #2: sigsuspend.cc.tmp`::do_nothing_signal_handler() + 47 at sigsuspend.cc:17 [opt]
frame #3: libsystem_platform.dylib`_sigtramp + 26
frame #4: libsystem_kernel.dylib`__sigsuspend + 11
frame #5: wrap_sigsuspend(mask=0x00007fff5fbff9cc) + 142 at tsan_interceptors.cc:2104
frame #6: sigsuspend.cc.tmp`main + 157 at sigsuspend.cc:35 [opt]

...and the wrap_sigsuspend interceptor calls REAL(sigsuspend) while ScopedInterceptor is still in scope:

TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) {
  SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask);
  return REAL(sigsuspend)(mask);
}

[kubamracek]

Updating patch.

[kubamracek]

There are some failures on the Linux buildbot, http://lab.llvm.org:8011/builders/sanitizer-x86_64-linux-autoconf/builds/13046/steps/test%20tsan%20in%20debug%20compiler-rt%20build/logs/stdio:

Failing Tests (3):
ThreadSanitizer :: dl_iterate_phdr.cc
ThreadSanitizer :: signal_sync.cc
ThreadSanitizer :: signal_thread.cc

The "dl_iterate_phdr.cc" one should be easy to fix, we just need to annotate the callback from the interceptor into user code. However, I don't know yet what's the right solution for the signal tests.

compiler-rt/trunk/lib/tsan/rtl/tsan_interceptors.cc

if (!thr_->ignore_interceptors) {
     FuncEntry(thr, pc);
   }
   DPrintf("#%d: intercept %s()\n", thr_->tid, fname);
   if (!thr_->in_ignored_lib && libignore()->IsIgnored(pc)) {
     in_ignored_lib_ = true;
     thr_->in_ignored_lib = true;
     ThreadIgnoreBegin(thr_, pc_);
   }
+#if SANITIZER_DEBUG
+  thr_->in_interceptor_count++;
+#endif
 }
 
 ScopedInterceptor::~ScopedInterceptor() {
+#if SANITIZER_DEBUG
+  thr_->in_interceptor_count--;
+#endif
   if (in_ignored_lib_) {
     thr_->in_ignored_lib = false;
     ThreadIgnoreEnd(thr_, pc_);
   }
   if (!thr_->ignore_interceptors) {
     ProcessPendingSignals(thr_);
     FuncExit(thr_);
     CheckNoLocks(thr_);
   }
 }
 
 void ScopedInterceptor::UserCallbackStart() {
+#if SANITIZER_DEBUG
+  thr_->in_interceptor_count--;
+#endif
   if (in_ignored_lib_) {
     thr_->in_ignored_lib = false;
     ThreadIgnoreEnd(thr_, pc_);
   }
 }
 
 void ScopedInterceptor::UserCallbackEnd() {
   if (in_ignored_lib_) {
     thr_->in_ignored_lib = true;
     ThreadIgnoreBegin(thr_, pc_);
   }
+#if SANITIZER_DEBUG
+  thr_->in_interceptor_count++;
+#endif
 }
 
 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
 #if SANITIZER_FREEBSD
 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
 #else
 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
 #endif
TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) {
   sigaction_t old;
   int res = sigaction(sig, &act, &old);
   if (res)
     return SIG_ERR;
   return old.sa_handler;
 }
 
 TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) {
+  {
     SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask);
+  }
   return REAL(sigsuspend)(mask);
 }
 
 TSAN_INTERCEPTOR(int, raise, int sig) {
   SCOPED_TSAN_INTERCEPTOR(raise, sig);
   ThreadSignalContext *sctx = SigCtx(thr);
   CHECK_NE(sctx, 0);
   int prev = sctx->int_signal_send;

compiler-rt/trunk/lib/tsan/rtl/tsan_interface_inl.h

 
 #include "tsan_interface.h"
 #include "tsan_rtl.h"
 
 #define CALLERPC ((uptr)__builtin_return_address(0))
 
 using namespace __tsan;  // NOLINT
 
-void __tsan_read1(void *addr) {
-  MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog1);
-}
-
-void __tsan_read2(void *addr) {
-  MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog2);
-}
-
-void __tsan_read4(void *addr) {
-  MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog4);
-}
-
-void __tsan_read8(void *addr) {
-  MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
-}
-
-void __tsan_write1(void *addr) {
-  MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog1);
-}
-
-void __tsan_write2(void *addr) {
-  MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog2);
-}
-
-void __tsan_write4(void *addr) {
-  MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog4);
-}
-
-void __tsan_write8(void *addr) {
-  MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
-}
-
-void __tsan_read1_pc(void *addr, void *pc) {
-  MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog1);
-}
-
-void __tsan_read2_pc(void *addr, void *pc) {
-  MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog2);
-}
-
-void __tsan_read4_pc(void *addr, void *pc) {
-  MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog4);
-}
-
-void __tsan_read8_pc(void *addr, void *pc) {
-  MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog8);
-}
-
-void __tsan_write1_pc(void *addr, void *pc) {
-  MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog1);
-}
-
-void __tsan_write2_pc(void *addr, void *pc) {
-  MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog2);
-}
-
-void __tsan_write4_pc(void *addr, void *pc) {
-  MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog4);
-}
-
-void __tsan_write8_pc(void *addr, void *pc) {
-  MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog8);
-}
+#define TSAN_MEM_ACCESS_FUNC(type, func, size) \
+  void __tsan_##type(void *addr) {             \
+    ThreadState *thr = cur_thread();           \
+    DCHECK_EQ(thr->in_interceptor_count, 0);   \
+    func(thr, CALLERPC, (uptr)addr, size);     \
+  }
+
+#define TSAN_MEM_ACCESS_FUNC_PC(type, func, size) \
+  void __tsan_##type(void *addr, void *pc) {      \
+    ThreadState *thr = cur_thread();              \
+    DCHECK_EQ(thr->in_interceptor_count, 0);      \
+    func(thr, (uptr)pc, (uptr)addr, size);        \
+  }
+
+TSAN_MEM_ACCESS_FUNC(read1, MemoryRead, kSizeLog1)
+TSAN_MEM_ACCESS_FUNC(read2, MemoryRead, kSizeLog2)
+TSAN_MEM_ACCESS_FUNC(read4, MemoryRead, kSizeLog4)
+TSAN_MEM_ACCESS_FUNC(read8, MemoryRead, kSizeLog8)
+TSAN_MEM_ACCESS_FUNC(write1, MemoryWrite, kSizeLog1)
+TSAN_MEM_ACCESS_FUNC(write2, MemoryWrite, kSizeLog2)
+TSAN_MEM_ACCESS_FUNC(write4, MemoryWrite, kSizeLog4)
+TSAN_MEM_ACCESS_FUNC(write8, MemoryWrite, kSizeLog8)
+TSAN_MEM_ACCESS_FUNC_PC(read1_pc, MemoryRead, kSizeLog1)
+TSAN_MEM_ACCESS_FUNC_PC(read2_pc, MemoryRead, kSizeLog2)
+TSAN_MEM_ACCESS_FUNC_PC(read4_pc, MemoryRead, kSizeLog4)
+TSAN_MEM_ACCESS_FUNC_PC(read8_pc, MemoryRead, kSizeLog8)
+TSAN_MEM_ACCESS_FUNC_PC(write1_pc, MemoryWrite, kSizeLog1)
+TSAN_MEM_ACCESS_FUNC_PC(write2_pc, MemoryWrite, kSizeLog2)
+TSAN_MEM_ACCESS_FUNC_PC(write4_pc, MemoryWrite, kSizeLog4)
+TSAN_MEM_ACCESS_FUNC_PC(write8_pc, MemoryWrite, kSizeLog8)
 
 void __tsan_vptr_update(void **vptr_p, void *new_val) {
   CHECK_EQ(sizeof(vptr_p), 8);
   if (*vptr_p != new_val) {
     ThreadState *thr = cur_thread();
     thr->is_vptr_access = true;
     MemoryWrite(thr, CALLERPC, (uptr)vptr_p, kSizeLog8);
     thr->is_vptr_access = false;

compiler-rt/trunk/lib/tsan/rtl/tsan_rtl.h

 #endif
 #if TSAN_COLLECT_STATS
   u64 stat[StatCnt];
 #endif
   const int tid;
   const int unique_id;
   bool in_symbolizer;
   bool in_ignored_lib;
+#if SANITIZER_DEBUG
+  int in_interceptor_count;
+#endif
   bool is_inited;
   bool is_dead;
   bool is_freeing;
   bool is_vptr_access;
   const uptr stk_addr;
   const uptr stk_size;
   const uptr tls_addr;
   const uptr tls_size;