tsan: split thread into logical and physical state

[dvyukov]

Summary

Currently ThreadState holds both logical state (required for race-detection algorithm, user-visible)
and physical state (various caches, most notably malloc cache). Move physical state in a new
Process entity. Besides just being the right thing from abstraction point of view, this solves several
problems:

1. Cache everything on P level in Go. Currently we cache on a mix of goroutine and OS thread levels.

This unnecessary increases memory consumption.

2. Properly handle free operations in Go. Frees are issue by GC which don't have goroutine context.

As the result we could not do anything more than just clearing shadow. For example, we leaked
sync objects and heap block descriptors.

3. This will allow to get rid of libc malloc in Go (now we have Processor context for internal allocator cache).

This in turn will allow to get rid of dependency on libc entirely.

4. Potentially we can make Processor per-CPU in C++ mode instead of per-thread, which will

reduce resource consumption.
The distinction between Thread and Processor is currently used only by Go, C++ creates Processor per OS thread,
which is equivalent to the current scheme.

[dvyukov]

This broke a bunch of bots:
http://lab.llvm.org:8080/green//job/clang-stage1-cmake-RA-globalisel_check/288/consoleFull#808796896a1ca8a51-895e-46c6-af87-ce24fa4cd561
http://lab.llvm.org:8080/green//job/clang-stage1-cmake-RA-globalisel_check/288/consoleFull#808796896a1ca8a51-895e-46c6-af87-ce24fa4cd561
http://lab.llvm.org:8011/builders/clang-cmake-aarch64-42vma/builds/6274

[dvyukov]

... and racecheck_unittest:
http://lab.llvm.org:8011/builders/sanitizer-x86_64-linux-autoconf/builds/15922/steps/tsan%20racecheck_unittest/logs/stdio

[dvyukov]

powerpc as well:
http://lab.llvm.org:8011/builders/sanitizer-ppc64be-linux/builds/784

[dvyukov]

Kuba, Renato, Bill,

I would like to ask one of you to try to figure out why it crashes on darwin/arm64/powerpc. I don't have accesses to such machines, and it works on linux/amd64. There is probably some initialization order issue, maybe common for all these platforms. This blocks some important changes for Go.

Thanks in advance

[rengolin]

+Adhemerval

[zatrazz]

This is indeed an initialization order issue where 'ProcCreate' is trying to access the ThreadState object through internal_alloc without it has been proper initialized (on ThreadStart function). I think it would be better to use plain allocation for 'Process' objects. I check this fix on top the patch on both aarch64 and powerpc64le and it shows no regression:

diff --git a/lib/tsan/rtl/tsan_rtl_proc.cc b/lib/tsan/rtl/tsan_rtl_proc.cc
index 710a3bf..c396350 100644
--- a/lib/tsan/rtl/tsan_rtl_proc.cc
+++ b/lib/tsan/rtl/tsan_rtl_proc.cc
@@ -19,7 +19,7 @@
 namespace __tsan {

 Processor *ProcCreate() {
-  void *mem = internal_alloc(MBlockProcessor, sizeof(Processor));
+  void *mem = InternalAlloc(sizeof(Processor));
   internal_memset(mem, 0, sizeof(Processor));
   Processor *proc = new(mem) Processor;
   proc->thr = nullptr;
@@ -41,7 +41,7 @@ void ProcDestroy(Processor *proc) {
   if (common_flags()->detect_deadlocks)
      ctx->dd->DestroyPhysicalThread(proc->dd_pt);
   proc->~Processor();
-  internal_free(proc);
+  InternalFree(proc);
 }

 void ProcWire(Processor *proc, ThreadState *thr) {

[dvyukov]

Thank you very much!
I will try to resubmit with your change tomorrow.

[dvyukov]

It was working on integration with Go runtime but then was sidetracked. I want to shake out things on Go side and ensure that it all works end-to-end, and only then submit the llvm part.

lib/tsan/CMakeLists.txt

set(TSAN_SOURCES
   rtl/tsan_md5.cc
   rtl/tsan_mman.cc
   rtl/tsan_mutex.cc
   rtl/tsan_mutexset.cc
   rtl/tsan_report.cc
   rtl/tsan_rtl.cc
   rtl/tsan_rtl_mutex.cc
   rtl/tsan_rtl_report.cc
+  rtl/tsan_rtl_proc.cc
   rtl/tsan_rtl_thread.cc
   rtl/tsan_stack_trace.cc
   rtl/tsan_stat.cc
   rtl/tsan_suppressions.cc
   rtl/tsan_symbolize.cc
   rtl/tsan_sync.cc)
 
 set(TSAN_CXX_SOURCES

lib/tsan/go/build.bat

-type tsan_go.cc ..\rtl\tsan_interface_atomic.cc ..\rtl\tsan_clock.cc ..\rtl\tsan_flags.cc ..\rtl\tsan_md5.cc ..\rtl\tsan_mutex.cc ..\rtl\tsan_report.cc ..\rtl\tsan_rtl.cc ..\rtl\tsan_rtl_mutex.cc ..\rtl\tsan_rtl_report.cc ..\rtl\tsan_rtl_thread.cc ..\rtl\tsan_stat.cc ..\rtl\tsan_suppressions.cc ..\rtl\tsan_sync.cc ..\rtl\tsan_stack_trace.cc ..\..\sanitizer_common\sanitizer_allocator.cc ..\..\sanitizer_common\sanitizer_common.cc ..\..\sanitizer_common\sanitizer_flags.cc ..\..\sanitizer_common\sanitizer_stacktrace.cc ..\..\sanitizer_common\sanitizer_libc.cc ..\..\sanitizer_common\sanitizer_printf.cc ..\..\sanitizer_common\sanitizer_suppressions.cc ..\..\sanitizer_common\sanitizer_thread_registry.cc ..\rtl\tsan_platform_windows.cc ..\..\sanitizer_common\sanitizer_win.cc ..\..\sanitizer_common\sanitizer_deadlock_detector1.cc ..\..\sanitizer_common\sanitizer_stackdepot.cc ..\..\sanitizer_common\sanitizer_persistent_allocator.cc ..\..\sanitizer_common\sanitizer_flag_parser.cc ..\..\sanitizer_common\sanitizer_symbolizer.cc > gotsan.cc
+type tsan_go.cc ..\rtl\tsan_interface_atomic.cc ..\rtl\tsan_clock.cc ..\rtl\tsan_flags.cc ..\rtl\tsan_md5.cc ..\rtl\tsan_mutex.cc ..\rtl\tsan_report.cc ..\rtl\tsan_rtl.cc ..\rtl\tsan_rtl_mutex.cc ..\rtl\tsan_rtl_report.cc ..\rtl\tsan_rtl_thread.cc ..\rtl\tsan_rtl_proc.cc ..\rtl\tsan_stat.cc ..\rtl\tsan_suppressions.cc ..\rtl\tsan_sync.cc ..\rtl\tsan_stack_trace.cc ..\..\sanitizer_common\sanitizer_allocator.cc ..\..\sanitizer_common\sanitizer_common.cc ..\..\sanitizer_common\sanitizer_flags.cc ..\..\sanitizer_common\sanitizer_stacktrace.cc ..\..\sanitizer_common\sanitizer_libc.cc ..\..\sanitizer_common\sanitizer_printf.cc ..\..\sanitizer_common\sanitizer_suppressions.cc ..\..\sanitizer_common\sanitizer_thread_registry.cc ..\rtl\tsan_platform_windows.cc ..\..\sanitizer_common\sanitizer_win.cc ..\..\sanitizer_common\sanitizer_deadlock_detector1.cc ..\..\sanitizer_common\sanitizer_stackdepot.cc ..\..\sanitizer_common\sanitizer_persistent_allocator.cc ..\..\sanitizer_common\sanitizer_flag_parser.cc ..\..\sanitizer_common\sanitizer_symbolizer.cc > gotsan.cc
 
 gcc -c -o race_windows_amd64.syso gotsan.cc -I..\rtl -I..\.. -I..\..\sanitizer_common -I..\..\..\include -m64 -Wall -fno-exceptions -fno-rtti -DSANITIZER_GO -Wno-error=attributes -Wno-attributes -Wno-format -Wno-maybe-uninitialized -DSANITIZER_DEBUG=0 -O3 -fomit-frame-pointer -std=c++11
 

lib/tsan/go/buildgo.sh

 #!/bin/sh
 
 set -e
 
 SRCS="
   tsan_go.cc
   ../rtl/tsan_clock.cc
   ../rtl/tsan_flags.cc
   ../rtl/tsan_interface_atomic.cc
   ../rtl/tsan_md5.cc
   ../rtl/tsan_mutex.cc
   ../rtl/tsan_report.cc
   ../rtl/tsan_rtl.cc
   ../rtl/tsan_rtl_mutex.cc
   ../rtl/tsan_rtl_report.cc
   ../rtl/tsan_rtl_thread.cc
+  ../rtl/tsan_rtl_proc.cc
   ../rtl/tsan_stack_trace.cc
   ../rtl/tsan_stat.cc
   ../rtl/tsan_suppressions.cc
   ../rtl/tsan_sync.cc
   ../../sanitizer_common/sanitizer_allocator.cc
   ../../sanitizer_common/sanitizer_common.cc
   ../../sanitizer_common/sanitizer_common_libcdep.cc
   ../../sanitizer_common/sanitizer_deadlock_detector2.cc
else
   FLAGS="$FLAGS -DSANITIZER_DEBUG=1 -g"
 fi
 
 if [ "$SILENT" != "1" ]; then
   echo $CC gotsan.cc -c -o $DIR/race_$SUFFIX.syso $FLAGS $CFLAGS
 fi
 $CC $DIR/gotsan.cc -c -o $DIR/race_$SUFFIX.syso $FLAGS $CFLAGS
 
-$CC test.c $DIR/race_$SUFFIX.syso -m64 -o $DIR/test $OSLDFLAGS
+$CC test.c $DIR/race_$SUFFIX.syso -m64 -g -o $DIR/test $OSLDFLAGS
 
 export GORACE="exitcode=0 atexit_sleep_ms=0"
 if [ "$SILENT" != "1" ]; then
   $DIR/test
 else
   $DIR/test 2>/dev/null
 fi

lib/tsan/go/test.c

 //===-- test.c ------------------------------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // Sanity test for Go runtime.
 //
 //===----------------------------------------------------------------------===//
 
 #include <stdio.h>
 
-void __tsan_init(void **thr, void (*cb)(void*));
+void __tsan_init(void **thr, void **proc, void (*cb)(void*));
 void __tsan_fini();
 void __tsan_map_shadow(void *addr, unsigned long size);
 void __tsan_go_start(void *thr, void **chthr, void *pc);
 void __tsan_go_end(void *thr);
+void __tsan_proc_create(void **pproc);
+void __tsan_proc_destroy(void *proc);
+void __tsan_proc_wire(void *proc, void *thr);
+void __tsan_proc_unwire(void *proc, void *thr);
 void __tsan_read(void *thr, void *addr, void *pc);
 void __tsan_write(void *thr, void *addr, void *pc);
 void __tsan_func_enter(void *thr, void *pc);
 void __tsan_func_exit(void *thr);
-void __tsan_malloc(void *p, unsigned long sz);
+void __tsan_malloc(void *thr, void *pc, void *p, unsigned long sz);
+void __tsan_free(void *proc, void *p, unsigned long sz);
 void __tsan_acquire(void *thr, void *addr);
 void __tsan_release(void *thr, void *addr);
 void __tsan_release_merge(void *thr, void *addr);
 
 void symbolize_cb(void *ctx) {}
 
 char buf0[100<<10];
 
 void foobar() {}
 void barfoo() {}
 
 int main(void) {
   void *thr0 = 0;
+  void *proc0 = 0;
+  __tsan_init(&thr0, &proc0, symbolize_cb);
   char *buf = (char*)((unsigned long)buf0 + (64<<10) - 1 & ~((64<<10) - 1));
-  __tsan_malloc(buf, 10);
-  __tsan_init(&thr0, symbolize_cb);
   __tsan_map_shadow(buf, 4096);
+  __tsan_malloc(thr0, 0, buf, 10);
+  __tsan_free(proc0, buf, 10);
   __tsan_func_enter(thr0, (char*)&main + 1);
-  __tsan_malloc(buf, 10);
+  __tsan_malloc(thr0, 0, buf, 10);
   __tsan_release(thr0, buf);
   __tsan_release_merge(thr0, buf);
   void *thr1 = 0;
   __tsan_go_start(thr0, &thr1, (char*)&barfoo + 1);
   void *thr2 = 0;
   __tsan_go_start(thr0, &thr2, (char*)&barfoo + 1);
+  __tsan_func_exit(thr0);
+  __tsan_proc_unwire(proc0, thr0);
+  __tsan_proc_wire(proc0, thr1);
   __tsan_func_enter(thr1, (char*)&foobar + 1);
   __tsan_func_enter(thr1, (char*)&foobar + 1);
   __tsan_write(thr1, buf, (char*)&barfoo + 1);
   __tsan_acquire(thr1, buf);
   __tsan_func_exit(thr1);
   __tsan_func_exit(thr1);
   __tsan_go_end(thr1);
+  void *proc1 = 0;
+  __tsan_proc_create(&proc1);
+  __tsan_proc_wire(proc1, thr2);
   __tsan_func_enter(thr2, (char*)&foobar + 1);
   __tsan_read(thr2, buf, (char*)&barfoo + 1);
+  __tsan_free(proc1, buf, 10);
   __tsan_func_exit(thr2);
   __tsan_go_end(thr2);
-  __tsan_func_exit(thr0);
+  __tsan_proc_destroy(proc0);
+  __tsan_proc_destroy(proc1);
   __tsan_fini();
   return 0;
 }

lib/tsan/go/tsan_go.cc

 
 static ThreadState *AllocGoroutine() {
   ThreadState *thr = (ThreadState*)internal_alloc(MBlockThreadContex,
       sizeof(ThreadState));
   internal_memset(thr, 0, sizeof(*thr));
   return thr;
 }
 
-void __tsan_init(ThreadState **thrp, void (*cb)(SymbolizeContext *cb)) {
+void __tsan_init(ThreadState **thrp,
+                 Processor **procp,
+                 void (*cb)(SymbolizeContext *cb)) {
   symbolize_cb = cb;
   ThreadState *thr = AllocGoroutine();
   main_thr = *thrp = thr;
   Initialize(thr);
+  *procp = thr->proc;
   inited = true;
 }
 
 void __tsan_fini() {
   // FIXME: Not necessary thread 0.
   ThreadState *thr = main_thr;
   int res = Finalize(thr);
   exit(res);

 void __tsan_func_enter(ThreadState *thr, void *pc) {
   FuncEntry(thr, (uptr)pc);
 }
 
 void __tsan_func_exit(ThreadState *thr) {
   FuncExit(thr);
 }
 
-void __tsan_malloc(void *p, uptr sz) {
-  if (!inited)
-    return;
-  MemoryResetRange(0, 0, (uptr)p, sz);
+void __tsan_malloc(ThreadState *thr, uptr pc, uptr p, uptr sz) {
+  CHECK(inited);
+  if (thr && thr->proc)
+    ctx->metamap.AllocBlock(thr, pc, p, sz);
+  MemoryResetRange(thr, 0, p, sz);
+}
+
+void __tsan_free(Processor *proc, uptr p, uptr sz) {
+  ctx->metamap.FreeRange(proc, p, sz);
 }
 
 void __tsan_go_start(ThreadState *parent, ThreadState **pthr, void *pc) {
   ThreadState *thr = AllocGoroutine();
   *pthr = thr;
   int goid = ThreadCreate(parent, (uptr)pc, 0, true);
+  Processor *proc = parent->proc;
+  ProcUnwire(proc, parent);
+  ProcWire(proc, thr);
   ThreadStart(thr, goid, 0);
+  ProcUnwire(proc, thr);
+  ProcWire(proc, parent);
 }
 
 void __tsan_go_end(ThreadState *thr) {
+  Processor *proc = thr->proc;
   ThreadFinish(thr);
+  ProcUnwire(proc, thr);
   internal_free(thr);
 }
 
+void __tsan_proc_create(Processor **pproc) {
+  *pproc = ProcCreate();
+}
+
+void __tsan_proc_destroy(Processor *proc) {
+  ProcDestroy(proc);
+}
+
+void __tsan_proc_wire(Processor *proc, ThreadState *thr) {
+  ProcWire(proc, thr);
+}
+
+void __tsan_proc_unwire(Processor *proc, ThreadState *thr) {
+  ProcUnwire(proc, thr);
+}
+
 void __tsan_acquire(ThreadState *thr, void *addr) {
   Acquire(thr, 0, (uptr)addr);
 }
 
 void __tsan_release(ThreadState *thr, void *addr) {
   ReleaseStore(thr, 0, (uptr)addr);
 }
 

lib/tsan/rtl/tsan_defs.h

 
 struct MD5Hash {
   u64 hash[2];
   bool operator==(const MD5Hash &other) const;
 };
 
 MD5Hash md5_hash(const void *data, uptr size);
 
+struct Processor;
 struct ThreadState;
 class ThreadContext;
 struct Context;
 struct ReportStack;
 class ReportDesc;
 class RegionAlloc;
 
 // Descriptor of user's memory block.

lib/tsan/rtl/tsan_interceptors.cc

 #define TSAN_MAYBE_INTERCEPT_MMAP64
 #endif
 
 TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) {
   SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz);
   if (sz != 0) {
     // If sz == 0, munmap will return EINVAL and don't unmap any memory.
     DontNeedShadowFor((uptr)addr, sz);
-    ctx->metamap.ResetRange(thr, pc, (uptr)addr, (uptr)sz);
+    ctx->metamap.ResetRange(thr->proc, (uptr)addr, (uptr)sz);
   }
   int res = REAL(munmap)(addr, sz);
   return res;
 }
 
 #if SANITIZER_LINUX
 TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
   SCOPED_INTERCEPTOR_RAW(memalign, align, sz);

 STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) {
   SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g);
   atomic_store(g, 0, memory_order_relaxed);
 }
 
 namespace __tsan {
 void DestroyThreadState() {
   ThreadState *thr = cur_thread();
+  Processor *proc = thr->proc;
   ThreadFinish(thr);
+  ProcUnwire(proc, thr);
+  ProcDestroy(proc);
   ThreadSignalContext *sctx = thr->signal_ctx;
   if (sctx) {
     thr->signal_ctx = 0;
     UnmapOrDie(sctx, sizeof(*sctx));
   }
   cur_thread_finalize();
 }
 }  // namespace __tsan
if (pthread_setspecific(g_thread_finalize_key,
                             (void *)GetPthreadDestructorIterations())) {
       Printf("ThreadSanitizer: failed to set thread key\n");
       Die();
     }
     ThreadIgnoreEnd(thr, 0);
 #endif
     while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0)
       internal_sched_yield();
+    Processor *proc = ProcCreate();
+    ProcWire(proc, thr);
     ThreadStart(thr, tid, GetTid());
     atomic_store(&p->tid, 0, memory_order_release);
   }
   void *res = callback(param);
   // Prevent the callback from being tail called,
   // it mixes up stack traces.
   volatile int foo = 42;
   foo++;

lib/tsan/rtl/tsan_interface_java.cc

void __tsan_java_free(jptr ptr, jptr size) {
   DPrintf("#%d: java_free(%p, %p)\n", thr->tid, ptr, size);
   CHECK_NE(jctx, 0);
   CHECK_NE(size, 0);
   CHECK_EQ(ptr % kHeapAlignment, 0);
   CHECK_EQ(size % kHeapAlignment, 0);
   CHECK_GE(ptr, jctx->heap_begin);
   CHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
 
-  ctx->metamap.FreeRange(thr, pc, ptr, size);
+  ctx->metamap.FreeRange(thr->proc, ptr, size);
 }
 
 void __tsan_java_move(jptr src, jptr dst, jptr size) {
   SCOPED_JAVA_FUNC(__tsan_java_move);
   DPrintf("#%d: java_move(%p, %p, %p)\n", thr->tid, src, dst, size);
   CHECK_NE(jctx, 0);
   CHECK_NE(size, 0);
   CHECK_EQ(src % kHeapAlignment, 0);

lib/tsan/rtl/tsan_mman.h

 #include "tsan_defs.h"
 
 namespace __tsan {
 
 const uptr kDefaultAlignment = 16;
 
 void InitializeAllocator();
 void ReplaceSystemMalloc();
-void AllocatorThreadStart(ThreadState *thr);
-void AllocatorThreadFinish(ThreadState *thr);
+void AllocatorProcStart(Processor *proc);
+void AllocatorProcFinish(Processor *proc);
 void AllocatorPrintStats();
 
 // For user allocations.
 void *user_alloc(ThreadState *thr, uptr pc, uptr sz,
                  uptr align = kDefaultAlignment, bool signal = true);
 void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n);
 // Does not accept NULL.
 void user_free(ThreadState *thr, uptr pc, void *p, bool signal = true);
 void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz);
 void *user_alloc_aligned(ThreadState *thr, uptr pc, uptr sz, uptr align);
 uptr user_alloc_usable_size(const void *p);
 
 // Invoking malloc/free hooks that may be installed by the user.
 void invoke_malloc_hook(void *ptr, uptr size);
 void invoke_free_hook(void *ptr);
 
 enum MBlockType {
   MBlockScopedBuf,
   MBlockString,
   MBlockStackTrace,
   MBlockShadowStack,
   MBlockSync,
   MBlockClock,
+  MBlockProcessor,
   MBlockThreadContex,
   MBlockDeadInfo,
   MBlockRacyStacks,
   MBlockRacyAddresses,
   MBlockAtExit,
   MBlockFlag,
   MBlockReport,
   MBlockReportMop,

lib/tsan/rtl/tsan_mman.cc

 Allocator *allocator() {
   return reinterpret_cast<Allocator*>(&allocator_placeholder);
 }
 
 void InitializeAllocator() {
   allocator()->Init(common_flags()->allocator_may_return_null);
 }
 
-void AllocatorThreadStart(ThreadState *thr) {
-  allocator()->InitCache(&thr->alloc_cache);
-  internal_allocator()->InitCache(&thr->internal_alloc_cache);
+void AllocatorProcStart(Processor *proc) {
+  allocator()->InitCache(&proc->alloc_cache);
+  internal_allocator()->InitCache(&proc->internal_alloc_cache);
 }
 
-void AllocatorThreadFinish(ThreadState *thr) {
-  allocator()->DestroyCache(&thr->alloc_cache);
-  internal_allocator()->DestroyCache(&thr->internal_alloc_cache);
+void AllocatorProcFinish(Processor *proc) {
+  allocator()->DestroyCache(&proc->alloc_cache);
+  internal_allocator()->DestroyCache(&proc->internal_alloc_cache);
 }
 
 void AllocatorPrintStats() {
   allocator()->PrintStats();
 }
 
 static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
   if (atomic_load_relaxed(&thr->in_signal_handler) == 0 ||
       !flags()->report_signal_unsafe)
     return;
   VarSizeStackTrace stack;
   ObtainCurrentStack(thr, pc, &stack);
   if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack))
     return;
   ThreadRegistryLock l(ctx->thread_registry);
   ScopedReport rep(ReportTypeSignalUnsafe);
   rep.AddStack(stack, true);
   OutputReport(thr, rep);
 }
 
 void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) {
   if ((sz >= (1ull << 40)) || (align >= (1ull << 40)))
     return allocator()->ReturnNullOrDie();
-  void *p = allocator()->Allocate(&thr->alloc_cache, sz, align);
+  void *p = allocator()->Allocate(&thr->proc->alloc_cache, sz, align);
   if (p == 0)
     return 0;
   if (ctx && ctx->initialized)
     OnUserAlloc(thr, pc, (uptr)p, sz, true);
   if (signal)
     SignalUnsafeCall(thr, pc);
   return p;
 }
 
 void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) {
   if (CallocShouldReturnNullDueToOverflow(size, n))
     return allocator()->ReturnNullOrDie();
   void *p = user_alloc(thr, pc, n * size);
   if (p)
     internal_memset(p, 0, n * size);
   return p;
 }
 
 void user_free(ThreadState *thr, uptr pc, void *p, bool signal) {
   if (ctx && ctx->initialized)
     OnUserFree(thr, pc, (uptr)p, true);
-  allocator()->Deallocate(&thr->alloc_cache, p);
+  allocator()->Deallocate(&thr->proc->alloc_cache, p);
   if (signal)
     SignalUnsafeCall(thr, pc);
 }
 
 void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {
   DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p);
   ctx->metamap.AllocBlock(thr, pc, p, sz);
   if (write && thr->ignore_reads_and_writes == 0)
     MemoryRangeImitateWrite(thr, pc, (uptr)p, sz);
   else
     MemoryResetRange(thr, pc, (uptr)p, sz);
 }
 
 void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) {
   CHECK_NE(p, (void*)0);
-  uptr sz = ctx->metamap.FreeBlock(thr, pc, p);
+  uptr sz = ctx->metamap.FreeBlock(thr->proc, p);
   DPrintf("#%d: free(%p, %zu)\n", thr->tid, p, sz);
   if (write && thr->ignore_reads_and_writes == 0)
     MemoryRangeFreed(thr, pc, (uptr)p, sz);
 }
 
 void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) {
   void *p2 = 0;
   // FIXME: Handle "shrinking" more efficiently,

 }
 
 void *internal_alloc(MBlockType typ, uptr sz) {
   ThreadState *thr = cur_thread();
   if (thr->nomalloc) {
     thr->nomalloc = 0;  // CHECK calls internal_malloc().
     CHECK(0);
   }
-  return InternalAlloc(sz, &thr->internal_alloc_cache);
+  return InternalAlloc(sz, &thr->proc->internal_alloc_cache);
 }
 
 void internal_free(void *p) {
   ThreadState *thr = cur_thread();
   if (thr->nomalloc) {
     thr->nomalloc = 0;  // CHECK calls internal_malloc().
     CHECK(0);
   }
-  InternalFree(p, &thr->internal_alloc_cache);
+  InternalFree(p, &thr->proc->internal_alloc_cache);
 }
 
 }  // namespace __tsan
 
 using namespace __tsan;
 
 extern "C" {
 uptr __sanitizer_get_current_allocated_bytes() {

 }
 
 uptr __sanitizer_get_allocated_size(const void *p) {
   return user_alloc_usable_size(p);
 }
 
 void __tsan_on_thread_idle() {
   ThreadState *thr = cur_thread();
-  allocator()->SwallowCache(&thr->alloc_cache);
-  internal_allocator()->SwallowCache(&thr->internal_alloc_cache);
-  ctx->metamap.OnThreadIdle(thr);
+  allocator()->SwallowCache(&thr->proc->alloc_cache);
+  internal_allocator()->SwallowCache(&thr->proc->internal_alloc_cache);
+  ctx->metamap.OnProcIdle(thr->proc);
 }
 }  // extern "C"

lib/tsan/rtl/tsan_rtl.h

struct JmpBuf {
   uptr sp;
   uptr mangled_sp;
   int int_signal_send;
   bool in_blocking_func;
   uptr in_signal_handler;
   uptr *shadow_stack_pos;
 };
 
+// A Processor represents a physical thread, or a P for Go.
+// It is used to store internal resources like allocate cache, and does not
+// participate in race-detection logic (invisible to end user).
+// In C++ it is tied to an OS thread just like ThreadState, however ideally
+// it should be tied to a CPU (this way we will have fewer allocator caches).
+// In Go it is tied to a P, so there are significantly fewer Processor's than
+// ThreadState's (which are tied to Gs).
+// A ThreadState must be wired with a Processor to handle events.
+struct Processor {
+  ThreadState *thr; // currently wired thread, or nullptr
+#ifndef SANITIZER_GO
+  AllocatorCache alloc_cache;
+  InternalAllocatorCache internal_alloc_cache;
+#endif
+  DenseSlabAllocCache block_cache;
+  DenseSlabAllocCache sync_cache;
+  DenseSlabAllocCache clock_cache;
+  DDPhysicalThread *dd_pt;
+};
+
 // This struct is stored in TLS.
 struct ThreadState {
   FastState fast_state;
   // Synch epoch represents the threads's epoch before the last synchronization
   // action. It allows to reduce number of shadow state updates.
   // For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
   // if we are processing write to X from the same thread at epoch=200,
   // we do nothing, because both writes happen in the same 'synch epoch'.
#endif
   uptr *shadow_stack;
   uptr *shadow_stack_end;
   uptr *shadow_stack_pos;
   u64 *racy_shadow_addr;
   u64 racy_state[2];
   MutexSet mset;
   ThreadClock clock;
 #ifndef SANITIZER_GO
-  AllocatorCache alloc_cache;
-  InternalAllocatorCache internal_alloc_cache;
   Vector<JmpBuf> jmp_bufs;
   int ignore_interceptors;
 #endif
 #if TSAN_COLLECT_STATS
   u64 stat[StatCnt];
 #endif
   const int tid;
   const int unique_id;
   bool in_symbolizer;
   bool in_ignored_lib;
   bool is_inited;
   bool is_dead;
   bool is_freeing;
   bool is_vptr_access;
   const uptr stk_addr;
   const uptr stk_size;
   const uptr tls_addr;
   const uptr tls_size;
   ThreadContext *tctx;
 
 #if SANITIZER_DEBUG && !SANITIZER_GO
   InternalDeadlockDetector internal_deadlock_detector;
 #endif
-  DDPhysicalThread *dd_pt;
   DDLogicalThread *dd_lt;
 
+  // Current wired Processor, or nullptr. Required to handle any events.
+  Processor *proc;
+
   atomic_uintptr_t in_signal_handler;
   ThreadSignalContext *signal_ctx;
 
-  DenseSlabAllocCache block_cache;
-  DenseSlabAllocCache sync_cache;
-  DenseSlabAllocCache clock_cache;
-
 #ifndef SANITIZER_GO
   u32 last_sleep_stack_id;
   ThreadClock last_sleep_clock;
 #endif
 
   // Set in regions of runtime that must be signal-safe and fork-safe.
   // If set, malloc must not be called.
   int nomalloc;

 int ThreadTid(ThreadState *thr, uptr pc, uptr uid);
 void ThreadJoin(ThreadState *thr, uptr pc, int tid);
 void ThreadDetach(ThreadState *thr, uptr pc, int tid);
 void ThreadFinalize(ThreadState *thr);
 void ThreadSetName(ThreadState *thr, const char *name);
 int ThreadCount(ThreadState *thr);
 void ProcessPendingSignals(ThreadState *thr);
 
+Processor *ProcCreate();
+void ProcDestroy(Processor *proc);
+void ProcWire(Processor *proc, ThreadState *thr);
+void ProcUnwire(Processor *proc, ThreadState *thr);
+
 void MutexCreate(ThreadState *thr, uptr pc, uptr addr,
                  bool rw, bool recursive, bool linker_init);
 void MutexDestroy(ThreadState *thr, uptr pc, uptr addr);
 void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec = 1,
                bool try_lock = false);
 int  MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all = false);
 void MutexReadLock(ThreadState *thr, uptr pc, uptr addr, bool try_lock = false);
 void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr);

lib/tsan/rtl/tsan_rtl.cc

void Initialize(ThreadState *thr) {
   InitializePlatformEarly();
 #ifndef SANITIZER_GO
   // Re-exec ourselves if we need to set additional env or command line args.
   MaybeReexec();
 
   InitializeAllocator();
   ReplaceSystemMalloc();
 #endif
+  if (common_flags()->detect_deadlocks)
+    ctx->dd = DDetector::Create(flags());
+  Processor *proc = ProcCreate();
+  ProcWire(proc, thr);
   InitializeInterceptors();
   CheckShadowMapping();
   InitializePlatform();
   InitializeMutex();
   InitializeDynamicAnnotations();
 #ifndef SANITIZER_GO
   InitializeShadowMemory();
 #endif
   // Setup correct file descriptor for error reports.
   __sanitizer_set_report_path(common_flags()->log_path);
   InitializeSuppressions();
 #ifndef SANITIZER_GO
   InitializeLibIgnore();
   Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
   // On MIPS, TSan initialization is run before
   // __pthread_initialize_minimal_internal() is finished, so we can not spawn
   // new threads.
 #ifndef __mips__
   StartBackgroundThread();
   SetSandboxingCallback(StopBackgroundThread);
 #endif
 #endif
-  if (common_flags()->detect_deadlocks)
-    ctx->dd = DDetector::Create(flags());
 
   VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n",
           (int)internal_getpid());
 
   // Initialize thread 0.
   int tid = ThreadCreate(thr, 0, 0, true);
   CHECK_EQ(tid, 0);
   ThreadStart(thr, tid, internal_getpid());

lib/tsan/rtl/tsan_rtl_mutex.cc

 
 struct Callback : DDCallback {
   ThreadState *thr;
   uptr pc;
 
   Callback(ThreadState *thr, uptr pc)
       : thr(thr)
       , pc(pc) {
-    DDCallback::pt = thr->dd_pt;
+    DDCallback::pt = thr->proc->dd_pt;
     DDCallback::lt = thr->dd_lt;
   }
 
   u32 Unwind() override { return CurrentStackId(thr, pc); }
   int UniqueTid() override { return thr->unique_id; }
 };
 
 void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s) {
if (flags()->report_destroy_locked
       && s->owner_tid != SyncVar::kInvalidTid
       && !s->is_broken) {
     s->is_broken = true;
     unlock_locked = true;
   }
   u64 mid = s->GetId();
   u32 last_lock = s->last_lock;
   if (!unlock_locked)
-    s->Reset(thr);  // must not reset it before the report is printed
+    s->Reset(thr->proc);  // must not reset it before the report is printed
   s->mtx.Unlock();
   if (unlock_locked) {
     ThreadRegistryLock l(ctx->thread_registry);
     ScopedReport rep(ReportTypeMutexDestroyLocked);
     rep.AddMutex(mid);
     VarSizeStackTrace trace;
     ObtainCurrentStack(thr, pc, &trace);
     rep.AddStack(trace);
     FastState last(last_lock);
     RestoreStack(last.tid(), last.epoch(), &trace, 0);
     rep.AddStack(trace, true);
     rep.AddLocation(addr, 1);
     OutputReport(thr, rep);
   }
   if (unlock_locked) {
     SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr);
     if (s != 0) {
-      s->Reset(thr);
+      s->Reset(thr->proc);
       s->mtx.Unlock();
     }
   }
   thr->mset.Remove(mid);
   // s will be destroyed and freed in MetaMap::FreeBlock.
 }
 
 void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec, bool try_lock) {
ctx->thread_registry->RunCallbackForEachThreadLocked(
       UpdateSleepClockCallback, thr);
 }
 #endif
 
 void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) {
   if (thr->ignore_sync)
     return;
   thr->clock.set(thr->fast_state.epoch());
-  thr->clock.acquire(&thr->clock_cache, c);
+  thr->clock.acquire(&thr->proc->clock_cache, c);
   StatInc(thr, StatSyncAcquire);
 }
 
 void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
   if (thr->ignore_sync)
     return;
   thr->clock.set(thr->fast_state.epoch());
   thr->fast_synch_epoch = thr->fast_state.epoch();
-  thr->clock.release(&thr->clock_cache, c);
+  thr->clock.release(&thr->proc->clock_cache, c);
   StatInc(thr, StatSyncRelease);
 }
 
 void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) {
   if (thr->ignore_sync)
     return;
   thr->clock.set(thr->fast_state.epoch());
   thr->fast_synch_epoch = thr->fast_state.epoch();
-  thr->clock.ReleaseStore(&thr->clock_cache, c);
+  thr->clock.ReleaseStore(&thr->proc->clock_cache, c);
   StatInc(thr, StatSyncRelease);
 }
 
 void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
   if (thr->ignore_sync)
     return;
   thr->clock.set(thr->fast_state.epoch());
   thr->fast_synch_epoch = thr->fast_state.epoch();
-  thr->clock.acq_rel(&thr->clock_cache, c);
+  thr->clock.acq_rel(&thr->proc->clock_cache, c);
   StatInc(thr, StatSyncAcquire);
   StatInc(thr, StatSyncRelease);
 }
 
 void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) {
   if (r == 0)
     return;
   ThreadRegistryLock l(ctx->thread_registry);

lib/tsan/rtl/tsan_rtl_proc.cc

+//===-- tsan_rtl_proc.cc ------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "tsan_rtl.h"
+#include "tsan_mman.h"
+#include "tsan_flags.h"
+
+namespace __tsan {
+
+Processor *ProcCreate() {
+  void *mem = internal_alloc(MBlockProcessor, sizeof(Processor));
+  internal_memset(mem, 0, sizeof(Processor));
+  Processor *proc = new(mem) Processor;
+  proc->thr = nullptr;
+#ifndef SANITIZER_GO
+  AllocatorProcStart(proc);
+#endif
+  if (common_flags()->detect_deadlocks)
+    proc->dd_pt = ctx->dd->CreatePhysicalThread();
+  return proc;
+}
+
+void ProcDestroy(Processor *proc) {
+  CHECK_EQ(proc->thr, nullptr);
+#ifndef SANITIZER_GO
+  AllocatorProcFinish(proc);
+#endif
+  ctx->clock_alloc.FlushCache(&proc->clock_cache);
+  ctx->metamap.OnProcIdle(proc);
+  if (common_flags()->detect_deadlocks)
+     ctx->dd->DestroyPhysicalThread(proc->dd_pt);
+  proc->~Processor();
+  internal_free(proc);
+}
+
+void ProcWire(Processor *proc, ThreadState *thr) {
+  CHECK_EQ(thr->proc, nullptr);
+  CHECK_EQ(proc->thr, nullptr);
+  thr->proc = proc;
+  proc->thr = thr;
+}
+
+void ProcUnwire(Processor *proc, ThreadState *thr) {
+  CHECK_EQ(thr->proc, proc);
+  CHECK_EQ(proc->thr, thr);
+  thr->proc = nullptr;
+  proc->thr = nullptr;
+}
+
+}  // namespace __tsan

lib/tsan/rtl/tsan_rtl_thread.cc

 
 void ThreadContext::OnDead() {
   CHECK_EQ(sync.size(), 0);
 }
 
 void ThreadContext::OnJoined(void *arg) {
   ThreadState *caller_thr = static_cast<ThreadState *>(arg);
   AcquireImpl(caller_thr, 0, &sync);
-  sync.Reset(&caller_thr->clock_cache);
+  sync.Reset(&caller_thr->proc->clock_cache);
 }
 
 struct OnCreatedArgs {
   ThreadState *thr;
   uptr pc;
 };
 
 void ThreadContext::OnCreated(void *arg) {

 void ThreadContext::OnReset() {
   CHECK_EQ(sync.size(), 0);
   FlushUnneededShadowMemory(GetThreadTrace(tid), TraceSize() * sizeof(Event));
   //!!! FlushUnneededShadowMemory(GetThreadTraceHeader(tid), sizeof(Trace));
 }
 
 void ThreadContext::OnDetached(void *arg) {
   ThreadState *thr1 = static_cast<ThreadState*>(arg);
-  sync.Reset(&thr1->clock_cache);
+  sync.Reset(&thr1->proc->clock_cache);
 }
 
 struct OnStartedArgs {
   ThreadState *thr;
   uptr stk_addr;
   uptr stk_size;
   uptr tls_addr;
   uptr tls_size;

 #else
   // Setup dynamic shadow stack.
   const int kInitStackSize = 8;
   thr->shadow_stack = (uptr*)internal_alloc(MBlockShadowStack,
       kInitStackSize * sizeof(uptr));
   thr->shadow_stack_pos = thr->shadow_stack;
   thr->shadow_stack_end = thr->shadow_stack + kInitStackSize;
 #endif
-#ifndef SANITIZER_GO
-  AllocatorThreadStart(thr);
-#endif
-  if (common_flags()->detect_deadlocks) {
-    thr->dd_pt = ctx->dd->CreatePhysicalThread();
+  if (common_flags()->detect_deadlocks)
     thr->dd_lt = ctx->dd->CreateLogicalThread(unique_id);
-  }
   thr->fast_state.SetHistorySize(flags()->history_size);
   // Commit switch to the new part of the trace.
   // TraceAddEvent will reset stack0/mset0 in the new part for us.
   TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
 
   thr->fast_synch_epoch = epoch0;
   AcquireImpl(thr, 0, &sync);
   StatInc(thr, StatSyncAcquire);
-  sync.Reset(&thr->clock_cache);
+  sync.Reset(&thr->proc->clock_cache);
   thr->is_inited = true;
   DPrintf("#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx "
           "tls_addr=%zx tls_size=%zx\n",
           tid, (uptr)epoch0, args->stk_addr, args->stk_size,
           args->tls_addr, args->tls_size);
 }
 
 void ThreadContext::OnFinished() {
   if (!detached) {
     thr->fast_state.IncrementEpoch();
     // Can't increment epoch w/o writing to the trace as well.
     TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
     ReleaseImpl(thr, 0, &sync);
   }
   epoch1 = thr->fast_state.epoch();
 
-  if (common_flags()->detect_deadlocks) {
-    ctx->dd->DestroyPhysicalThread(thr->dd_pt);
+  if (common_flags()->detect_deadlocks)
     ctx->dd->DestroyLogicalThread(thr->dd_lt);
-  }
-  ctx->clock_alloc.FlushCache(&thr->clock_cache);
-  ctx->metamap.OnThreadIdle(thr);
-#ifndef SANITIZER_GO
-  AllocatorThreadFinish(thr);
-#endif
   thr->~ThreadState();
 #if TSAN_COLLECT_STATS
   StatAggregate(ctx->stat, thr->stat);
 #endif
   thr = 0;
 }
 
 #ifndef SANITIZER_GO

lib/tsan/rtl/tsan_sync.h

struct SyncVar {
   u32 next;  // in MetaMap
   DDMutex dd;
   SyncClock read_clock;  // Used for rw mutexes only.
   // The clock is placed last, so that it is situated on a different cache line
   // with the mtx. This reduces contention for hot sync objects.
   SyncClock clock;
 
   void Init(ThreadState *thr, uptr pc, uptr addr, u64 uid);
-  void Reset(ThreadState *thr);
+  void Reset(Processor *proc);
 
   u64 GetId() const {
     // 47 lsb is addr, then 14 bits is low part of uid, then 3 zero bits.
     return GetLsb((u64)addr | (uid << 47), 61);
   }
   bool CheckId(u64 uid) const {
     CHECK_EQ(uid, GetLsb(uid, 14));
     return GetLsb(this->uid, 14) == uid;
   }
   static uptr SplitId(u64 id, u64 *uid) {
     *uid = id >> 47;
     return (uptr)GetLsb(id, 47);
   }
 };
 
 /* MetaMap allows to map arbitrary user pointers onto various descriptors.
    Currently it maps pointers to heap block descriptors and sync var descs.
    It uses 1/2 direct shadow, see tsan_platform.h.
 */
 class MetaMap {
  public:
   MetaMap();
 
   void AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz);
-  uptr FreeBlock(ThreadState *thr, uptr pc, uptr p);
-  bool FreeRange(ThreadState *thr, uptr pc, uptr p, uptr sz);
-  void ResetRange(ThreadState *thr, uptr pc, uptr p, uptr sz);
+  uptr FreeBlock(Processor *proc, uptr p);
+  bool FreeRange(Processor *proc, uptr p, uptr sz);
+  void ResetRange(Processor *proc, uptr p, uptr sz);
   MBlock* GetBlock(uptr p);
 
   SyncVar* GetOrCreateAndLock(ThreadState *thr, uptr pc,
                               uptr addr, bool write_lock);
   SyncVar* GetIfExistsAndLock(uptr addr);
 
   void MoveMemory(uptr src, uptr dst, uptr sz);
 
-  void OnThreadIdle(ThreadState *thr);
+  void OnProcIdle(Processor *proc);
 
  private:
   static const u32 kFlagMask  = 3u << 30;
   static const u32 kFlagBlock = 1u << 30;
   static const u32 kFlagSync  = 2u << 30;
   typedef DenseSlabAlloc<MBlock, 1<<16, 1<<12> BlockAlloc;
   typedef DenseSlabAlloc<SyncVar, 1<<16, 1<<10> SyncAlloc;
   BlockAlloc block_alloc_;

lib/tsan/rtl/tsan_sync.cc

void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid) {
 
   creation_stack_id = 0;
   if (kCppMode)  // Go does not use them
     creation_stack_id = CurrentStackId(thr, pc);
   if (common_flags()->detect_deadlocks)
     DDMutexInit(thr, pc, this);
 }
 
-void SyncVar::Reset(ThreadState *thr) {
+void SyncVar::Reset(Processor *proc) {
   uid = 0;
   creation_stack_id = 0;
   owner_tid = kInvalidTid;
   last_lock = 0;
   recursion = 0;
   is_rw = 0;
   is_recursive = 0;
   is_broken = 0;
   is_linker_init = 0;
 
-  if (thr == 0) {
+  if (proc == 0) {
     CHECK_EQ(clock.size(), 0);
     CHECK_EQ(read_clock.size(), 0);
   } else {
-    clock.Reset(&thr->clock_cache);
-    read_clock.Reset(&thr->clock_cache);
+    clock.Reset(&proc->clock_cache);
+    read_clock.Reset(&proc->clock_cache);
   }
 }
 
 MetaMap::MetaMap() {
   atomic_store(&uid_gen_, 0, memory_order_relaxed);
 }
 
 void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {
-  u32 idx = block_alloc_.Alloc(&thr->block_cache);
+  u32 idx = block_alloc_.Alloc(&thr->proc->block_cache);
   MBlock *b = block_alloc_.Map(idx);
   b->siz = sz;
   b->tid = thr->tid;
   b->stk = CurrentStackId(thr, pc);
   u32 *meta = MemToMeta(p);
   DCHECK_EQ(*meta, 0);
   *meta = idx | kFlagBlock;
 }
 
-uptr MetaMap::FreeBlock(ThreadState *thr, uptr pc, uptr p) {
+uptr MetaMap::FreeBlock(Processor *proc, uptr p) {
   MBlock* b = GetBlock(p);
   if (b == 0)
     return 0;
   uptr sz = RoundUpTo(b->siz, kMetaShadowCell);
-  FreeRange(thr, pc, p, sz);
+  FreeRange(proc, p, sz);
   return sz;
 }
 
-bool MetaMap::FreeRange(ThreadState *thr, uptr pc, uptr p, uptr sz) {
+bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {
   bool has_something = false;
   u32 *meta = MemToMeta(p);
   u32 *end = MemToMeta(p + sz);
   if (end == meta)
     end++;
   for (; meta < end; meta++) {
     u32 idx = *meta;
     if (idx == 0) {
       // Note: don't write to meta in this case -- the block can be huge.
       continue;
     }
     *meta = 0;
     has_something = true;
     while (idx != 0) {
       if (idx & kFlagBlock) {
-        block_alloc_.Free(&thr->block_cache, idx & ~kFlagMask);
+        block_alloc_.Free(&proc->block_cache, idx & ~kFlagMask);
         break;
       } else if (idx & kFlagSync) {
         DCHECK(idx & kFlagSync);
         SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);
         u32 next = s->next;
-        s->Reset(thr);
-        sync_alloc_.Free(&thr->sync_cache, idx & ~kFlagMask);
+        s->Reset(proc);
+        sync_alloc_.Free(&proc->sync_cache, idx & ~kFlagMask);
         idx = next;
       } else {
         CHECK(0);
       }
     }
   }
   return has_something;
 }
 
 // ResetRange removes all meta objects from the range.
 // It is called for large mmap-ed regions. The function is best-effort wrt
 // freeing of meta objects, because we don't want to page in the whole range
 // which can be huge. The function probes pages one-by-one until it finds a page
 // without meta objects, at this point it stops freeing meta objects. Because
 // thread stacks grow top-down, we do the same starting from end as well.
-void MetaMap::ResetRange(ThreadState *thr, uptr pc, uptr p, uptr sz) {
+void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
   const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;
   const uptr kPageSize = GetPageSizeCached() * kMetaRatio;
   if (sz <= 4 * kPageSize) {
     // If the range is small, just do the normal free procedure.
-    FreeRange(thr, pc, p, sz);
+    FreeRange(proc, p, sz);
     return;
   }
   // First, round both ends of the range to page size.
   uptr diff = RoundUp(p, kPageSize) - p;
   if (diff != 0) {
-    FreeRange(thr, pc, p, diff);
+    FreeRange(proc, p, diff);
     p += diff;
     sz -= diff;
   }
   diff = p + sz - RoundDown(p + sz, kPageSize);
   if (diff != 0) {
-    FreeRange(thr, pc, p + sz - diff, diff);
+    FreeRange(proc, p + sz - diff, diff);
     sz -= diff;
   }
   // Now we must have a non-empty page-aligned range.
   CHECK_GT(sz, 0);
   CHECK_EQ(p, RoundUp(p, kPageSize));
   CHECK_EQ(sz, RoundUp(sz, kPageSize));
   const uptr p0 = p;
   const uptr sz0 = sz;
   // Probe start of the range.
   while (sz > 0) {
-    bool has_something = FreeRange(thr, pc, p, kPageSize);
+    bool has_something = FreeRange(proc, p, kPageSize);
     p += kPageSize;
     sz -= kPageSize;
     if (!has_something)
       break;
   }
   // Probe end of the range.
   while (sz > 0) {
-    bool has_something = FreeRange(thr, pc, p - kPageSize, kPageSize);
+    bool has_something = FreeRange(proc, p - kPageSize, kPageSize);
     sz -= kPageSize;
     if (!has_something)
       break;
   }
   // Finally, page out the whole range (including the parts that we've just
   // freed). Note: we can't simply madvise, because we need to leave a zeroed
   // range (otherwise __tsan_java_move can crash if it encounters a left-over
   // meta objects in java heap).
for (;;) {
       if (idx == 0)
         break;
       if (idx & kFlagBlock)
         break;
       DCHECK(idx & kFlagSync);
       SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);
       if (s->addr == addr) {
         if (myidx != 0) {
-          mys->Reset(thr);
-          sync_alloc_.Free(&thr->sync_cache, myidx);
+          mys->Reset(thr->proc);
+          sync_alloc_.Free(&thr->proc->sync_cache, myidx);
         }
         if (write_lock)
           s->mtx.Lock();
         else
           s->mtx.ReadLock();
         return s;
       }
       idx = s->next;
     }
     if (!create)
       return 0;
     if (*meta != idx0) {
       idx0 = *meta;
       continue;
     }
 
     if (myidx == 0) {
       const u64 uid = atomic_fetch_add(&uid_gen_, 1, memory_order_relaxed);
-      myidx = sync_alloc_.Alloc(&thr->sync_cache);
+      myidx = sync_alloc_.Alloc(&thr->proc->sync_cache);
       mys = sync_alloc_.Map(myidx);
       mys->Init(thr, pc, addr, uid);
     }
     mys->next = idx0;
     if (atomic_compare_exchange_strong((atomic_uint32_t*)meta, &idx0,
         myidx | kFlagSync, memory_order_release)) {
       if (write_lock)
         mys->mtx.Lock();
while (idx != 0) {
       CHECK(idx & kFlagSync);
       SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);
       s->addr += diff;
       idx = s->next;
     }
   }
 }
 
-void MetaMap::OnThreadIdle(ThreadState *thr) {
-  block_alloc_.FlushCache(&thr->block_cache);
-  sync_alloc_.FlushCache(&thr->sync_cache);
+void MetaMap::OnProcIdle(Processor *proc) {
+  block_alloc_.FlushCache(&proc->block_cache);
+  sync_alloc_.FlushCache(&proc->sync_cache);
 }
 
 }  // namespace __tsan

lib/tsan/tests/unit/tsan_sync_test.cc

TEST(MetaMap, Basic) {
   ThreadState *thr = cur_thread();
   MetaMap *m = &ctx->metamap;
   u64 block[1] = {};  // fake malloc block
   m->AllocBlock(thr, 0, (uptr)&block[0], 1 * sizeof(u64));
   MBlock *mb = m->GetBlock((uptr)&block[0]);
   EXPECT_NE(mb, (MBlock*)0);
   EXPECT_EQ(mb->siz, 1 * sizeof(u64));
   EXPECT_EQ(mb->tid, thr->tid);
-  uptr sz = m->FreeBlock(thr, 0, (uptr)&block[0]);
+  uptr sz = m->FreeBlock(thr->proc, (uptr)&block[0]);
   EXPECT_EQ(sz, 1 * sizeof(u64));
   mb = m->GetBlock((uptr)&block[0]);
   EXPECT_EQ(mb, (MBlock*)0);
 }
 
 TEST(MetaMap, FreeRange) {
   ThreadState *thr = cur_thread();
   MetaMap *m = &ctx->metamap;
   u64 block[4] = {};  // fake malloc block
   m->AllocBlock(thr, 0, (uptr)&block[0], 1 * sizeof(u64));
   m->AllocBlock(thr, 0, (uptr)&block[1], 3 * sizeof(u64));
   MBlock *mb1 = m->GetBlock((uptr)&block[0]);
   EXPECT_EQ(mb1->siz, 1 * sizeof(u64));
   MBlock *mb2 = m->GetBlock((uptr)&block[1]);
   EXPECT_EQ(mb2->siz, 3 * sizeof(u64));
-  m->FreeRange(thr, 0, (uptr)&block[0], 4 * sizeof(u64));
+  m->FreeRange(thr->proc, (uptr)&block[0], 4 * sizeof(u64));
   mb1 = m->GetBlock((uptr)&block[0]);
   EXPECT_EQ(mb1, (MBlock*)0);
   mb2 = m->GetBlock((uptr)&block[1]);
   EXPECT_EQ(mb2, (MBlock*)0);
 }
 
 TEST(MetaMap, Sync) {
   ThreadState *thr = cur_thread();
   MetaMap *m = &ctx->metamap;
   u64 block[4] = {};  // fake malloc block
   m->AllocBlock(thr, 0, (uptr)&block[0], 4 * sizeof(u64));
   SyncVar *s1 = m->GetIfExistsAndLock((uptr)&block[0]);
   EXPECT_EQ(s1, (SyncVar*)0);
   s1 = m->GetOrCreateAndLock(thr, 0, (uptr)&block[0], true);
   EXPECT_NE(s1, (SyncVar*)0);
   EXPECT_EQ(s1->addr, (uptr)&block[0]);
   s1->mtx.Unlock();
   SyncVar *s2 = m->GetOrCreateAndLock(thr, 0, (uptr)&block[1], false);
   EXPECT_NE(s2, (SyncVar*)0);
   EXPECT_EQ(s2->addr, (uptr)&block[1]);
   s2->mtx.ReadUnlock();
-  m->FreeBlock(thr, 0, (uptr)&block[0]);
+  m->FreeBlock(thr->proc, (uptr)&block[0]);
   s1 = m->GetIfExistsAndLock((uptr)&block[0]);
   EXPECT_EQ(s1, (SyncVar*)0);
   s2 = m->GetIfExistsAndLock((uptr)&block[1]);
   EXPECT_EQ(s2, (SyncVar*)0);
-  m->OnThreadIdle(thr);
+  m->OnProcIdle(thr->proc);
 }
 
 TEST(MetaMap, MoveMemory) {
   ThreadState *thr = cur_thread();
   MetaMap *m = &ctx->metamap;
   u64 block1[4] = {};  // fake malloc block
   u64 block2[4] = {};  // fake malloc block
   m->AllocBlock(thr, 0, (uptr)&block1[0], 3 * sizeof(u64));
TEST(MetaMap, MoveMemory) {
   s1 = m->GetIfExistsAndLock((uptr)&block2[0]);
   EXPECT_NE(s1, (SyncVar*)0);
   EXPECT_EQ(s1->addr, (uptr)&block2[0]);
   s1->mtx.Unlock();
   s2 = m->GetIfExistsAndLock((uptr)&block2[1]);
   EXPECT_NE(s2, (SyncVar*)0);
   EXPECT_EQ(s2->addr, (uptr)&block2[1]);
   s2->mtx.Unlock();
-  m->FreeRange(thr, 0, (uptr)&block2[0], 4 * sizeof(u64));
+  m->FreeRange(thr->proc, (uptr)&block2[0], 4 * sizeof(u64));
 }
 
 TEST(MetaMap, ResetSync) {
   ThreadState *thr = cur_thread();
   MetaMap *m = &ctx->metamap;
   u64 block[1] = {};  // fake malloc block
   m->AllocBlock(thr, 0, (uptr)&block[0], 1 * sizeof(u64));
   SyncVar *s = m->GetOrCreateAndLock(thr, 0, (uptr)&block[0], true);
-  s->Reset(thr);
+  s->Reset(thr->proc);
   s->mtx.Unlock();
-  uptr sz = m->FreeBlock(thr, 0, (uptr)&block[0]);
+  uptr sz = m->FreeBlock(thr->proc, (uptr)&block[0]);
   EXPECT_EQ(sz, 1 * sizeof(u64));
 }
 
 }  // namespace __tsan