tsan: include internal allocator into deadlock detection

[dvyukov]

Summary

Now that we lock the internal allocator around fork,
it's possible it will create additional deadlocks.
Add a fake mutex that substitutes the internal allocator
for the purposes of deadlock detection.

Depends on D114531.

[melver]

Nice!

compiler-rt/lib/tsan/rtl/tsan_defs.h

enum MutexType {
   MutexTypeTrace = MutexLastCommon,
   MutexTypeReport,
   MutexTypeSyncVar,
   MutexTypeAnnotations,
   MutexTypeAtExit,
   MutexTypeFired,
   MutexTypeRacy,
   MutexTypeGlobalProc,
+  MutexTypeInternalAlloc,
 };
 
 }  // namespace __tsan
 
 #endif  // TSAN_DEFS_H

compiler-rt/lib/tsan/rtl/tsan_mman.cpp

 static char allocator_placeholder[sizeof(Allocator)] ALIGNED(64);
 Allocator *allocator() {
   return reinterpret_cast<Allocator*>(&allocator_placeholder);
 }
 
 struct GlobalProc {
   Mutex mtx;
   Processor *proc;
-
-  GlobalProc() : mtx(MutexTypeGlobalProc), proc(ProcCreate()) {}
+  // This mutex represents the internal allocator combined for
+  // the purposes of deadlock detection. The internal allocator
+  // uses multiple mutexes, moreover they are locked only occasionally
+  // and they are spin mutexes which don't support deadlock detection.
+  // So we use this fake mutex to serve as a substitute for these mutexes.
+  CheckedMutex internal_alloc_mtx;
+
+  GlobalProc()
+      : mtx(MutexTypeGlobalProc),
+        proc(ProcCreate()),
+        internal_alloc_mtx(MutexTypeInternalAlloc) {}
 };
 
 static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64);
 GlobalProc *global_proc() {
   return reinterpret_cast<GlobalProc*>(&global_proc_placeholder);
 }
 
+static void InternalAllocAccess() {
+  global_proc()->internal_alloc_mtx.Lock();
+  global_proc()->internal_alloc_mtx.Unlock();
+}
+
 ScopedGlobalProcessor::ScopedGlobalProcessor() {
   GlobalProc *gp = global_proc();
   ThreadState *thr = cur_thread();
   if (thr->proc())
     return;
   // If we don't have a proc, use the global one.
   // There are currently only two known case where this path is triggered:
   //   __interceptor_free
ScopedGlobalProcessor::~ScopedGlobalProcessor() {
   if (thr->proc() != gp->proc)
     return;
   ProcUnwire(gp->proc, thr);
   gp->mtx.Unlock();
 }
 
 void AllocatorLock() NO_THREAD_SAFETY_ANALYSIS {
   global_proc()->mtx.Lock();
+  global_proc()->internal_alloc_mtx.Lock();
   InternalAllocatorLock();
 }
 
 void AllocatorUnlock() NO_THREAD_SAFETY_ANALYSIS {
   InternalAllocatorUnlock();
+  global_proc()->internal_alloc_mtx.Unlock();
   global_proc()->mtx.Unlock();
 }
 
 static constexpr uptr kMaxAllowedMallocSize = 1ull << 40;
 static uptr max_user_defined_malloc_size;
 
 void InitializeAllocator() {
   SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);

 }
 
 void *Alloc(uptr sz) {
   ThreadState *thr = cur_thread();
   if (thr->nomalloc) {
     thr->nomalloc = 0;  // CHECK calls internal_malloc().
     CHECK(0);
   }
+  InternalAllocAccess();
   return InternalAlloc(sz, &thr->proc()->internal_alloc_cache);
 }
 
 void FreeImpl(void *p) {
   ThreadState *thr = cur_thread();
   if (thr->nomalloc) {
     thr->nomalloc = 0;  // CHECK calls internal_malloc().
     CHECK(0);
   }
+  InternalAllocAccess();
   InternalFree(p, &thr->proc()->internal_alloc_cache);
 }
 
 }  // namespace __tsan
 
 using namespace __tsan;
 
 extern "C" {

compiler-rt/lib/tsan/rtl/tsan_rtl.cpp

 }  // namespace __tsan
 
 #if SANITIZER_CHECK_DEADLOCKS
 namespace __sanitizer {
 using namespace __tsan;
 MutexMeta mutex_meta[] = {
     {MutexInvalid, "Invalid", {}},
     {MutexThreadRegistry, "ThreadRegistry", {}},
-    {MutexTypeTrace, "Trace", {MutexLeaf}},
-    {MutexTypeReport, "Report", {MutexTypeSyncVar, MutexTypeGlobalProc}},
-    {MutexTypeSyncVar, "SyncVar", {}},
+    {MutexTypeTrace, "Trace", {}},
+    {MutexTypeReport,
+     "Report",
+     {MutexTypeSyncVar, MutexTypeGlobalProc, MutexTypeTrace}},
+    {MutexTypeSyncVar, "SyncVar", {MutexTypeTrace}},
     {MutexTypeAnnotations, "Annotations", {}},
     {MutexTypeAtExit, "AtExit", {MutexTypeSyncVar}},
     {MutexTypeFired, "Fired", {MutexLeaf}},
     {MutexTypeRacy, "Racy", {MutexLeaf}},
     {MutexTypeGlobalProc, "GlobalProc", {}},
+    {MutexTypeInternalAlloc, "InternalAlloc", {MutexLeaf}},
     {},
 };
 
 void PrintMutexPC(uptr pc) { StackTrace(&pc, 1).Print(); }
 }  // namespace __sanitizer
 #endif