tsan: include MBlock/SyncObj stats into mem profile

[dvyukov]

Summary

Include info about MBlock/SyncObj memory consumption in the memory profile.

Depends on D110148.

[vitalybuka]

compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h

52

we don't need explicit?

88

don't need to lock if pos == nullptr

compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp

131

how can you overflow here this?

[melver]

compiler-rt/lib/tsan/rtl/tsan_sync.h

130

Could add a 'struct MetaMap::MemoryStats' to contain the 2 variables and return that, in case you think it'd be cleaner.

[dvyukov]

rebased and addressed comments

[dvyukov]

compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp

131

"mem_block_mem + sync_obj_mem + stacks->allocated" (and internal allocator memory in the next patch) should come from the mmap region. Thus we subtract it to not double account.
But we obtain these numbers from different sources, so I am not 100% sure it can't underflow. I think it can in some corner cases. One example I can think of is is some of that additional memory is not committed yet, so it won't show up in mem[MemMmap] because it counts real RSS (from /proc/self/maps). In this case mem[MemMmap] can theoretically go negative and I would rather print 0 than 17592186044416MB.

compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h

static_assert((kL2Size & (kL2Size - 1)) == 0,
                 "kL2Size must be a power-of-two");
   static_assert((kL1Size * kL2Size) <= (1ull << (sizeof(IndexT) * 8)),
                 "kL1Size/kL2Size are too large");
   static_assert(((kL1Size * kL2Size - 1) & kReserved) == 0,
                 "reserved bits don't fit");
   static_assert(sizeof(T) > sizeof(IndexT),
                 "it doesn't make sense to use dense alloc");
 
-  explicit DenseSlabAlloc(LinkerInitialized, const char *name) {
-    freelist_ = 0;
-    fillpos_ = 0;
-    name_ = name;
-  }
+  explicit DenseSlabAlloc(LinkerInitialized, const char *name) : name_(name) {}
 
   explicit DenseSlabAlloc(const char *name)
       : DenseSlabAlloc(LINKER_INITIALIZED, name) {
     // It can be very large.
     // Don't page it in for linker initialized objects.
     internal_memset(map_, 0, sizeof(map_));
   }
 
public:
 
   T *Map(IndexT idx) {
     DCHECK_NE(idx, 0);
     DCHECK_LE(idx, kL1Size * kL2Size);
     return &map_[idx / kL2Size][idx % kL2Size];
   }
 
   void FlushCache(Cache *c) {
     SpinMutexLock lock(&mtx_);
     while (c->pos) {
       IndexT idx = c->cache[--c->pos];
       *(IndexT*)Map(idx) = freelist_;
       freelist_ = idx;
     }
   }
 
   void InitCache(Cache *c) {
     c->pos = 0;
     internal_memset(c->cache, 0, sizeof(c->cache));
   }
 
+  uptr AllocatedMemory() const {
+    return atomic_load_relaxed(&fillpos_) * kL2Size * sizeof(T);
+  }
+
  private:
   T *map_[kL1Size];
   SpinMutex mtx_;
-  IndexT freelist_;
-  uptr fillpos_;
-  const char *name_;
+  IndexT freelist_ = {0};
+  atomic_uintptr_t fillpos_ = {0};
+  const char *const name_;
 
   void Refill(Cache *c) {
     SpinMutexLock lock(&mtx_);
     if (freelist_ == 0) {
-      if (fillpos_ == kL1Size) {
+      uptr fillpos = atomic_load_relaxed(&fillpos_);
+      if (fillpos == kL1Size) {
         Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n",
             name_, kL1Size, kL2Size);
         Die();
       }
-      VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n",
-          name_, fillpos_, kL1Size, kL2Size);
+      VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n", name_,
+              fillpos, kL1Size, kL2Size);
       T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), name_);
       // Reserve 0 as invalid index.
-      IndexT start = fillpos_ == 0 ? 1 : 0;
+      IndexT start = fillpos == 0 ? 1 : 0;
       for (IndexT i = start; i < kL2Size; i++) {
         new(batch + i) T;
-        *(IndexT*)(batch + i) = i + 1 + fillpos_ * kL2Size;
+        *(IndexT *)(batch + i) = i + 1 + fillpos * kL2Size;
       }
       *(IndexT*)(batch + kL2Size - 1) = 0;
-      freelist_ = fillpos_ * kL2Size + start;
-      map_[fillpos_++] = batch;
+      freelist_ = fillpos * kL2Size + start;
+      map_[fillpos] = batch;
+      atomic_store_relaxed(&fillpos_, fillpos + 1);
     }
     for (uptr i = 0; i < Cache::kSize / 2 && freelist_ != 0; i++) {
       IndexT idx = freelist_;
       c->cache[c->pos++] = idx;
       freelist_ = *(IndexT*)Map(idx);
     }
   }
 

compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp

void FillProfileCallback(uptr p, uptr rss, bool file,
   else if (p >= TraceMemBeg() && p < TraceMemEnd())
     mem[MemTrace] += rss;
   else
     mem[MemOther] += rss;
 }
 
 void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
   uptr mem[MemCount];
-  internal_memset(mem, 0, sizeof(mem[0]) * MemCount);
+  internal_memset(mem, 0, sizeof(mem));
   __sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7);
+  uptr mem_block_mem, sync_obj_mem;
+  ctx->metamap.GetMemoryStats(&mem_block_mem, &sync_obj_mem);
   StackDepotStats *stacks = StackDepotGetStats();
+  // All these are allocated from the common mmap region.
+  mem[MemMmap] -= mem_block_mem + sync_obj_mem + stacks->allocated;
+  if (s64(mem[MemMmap]) < 0)
+    mem[MemMmap] = 0;
   internal_snprintf(buf, buf_size,
                     "RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd"
-      " trace:%zd heap:%zd other:%zd stacks=%zd[%zd] nthr=%zd/%zd\n",
-      mem[MemTotal] >> 20, mem[MemShadow] >> 20, mem[MemMeta] >> 20,
-      mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemTrace] >> 20,
-      mem[MemHeap] >> 20, mem[MemOther] >> 20,
-      stacks->allocated >> 20, stacks->n_uniq_ids,
-      nlive, nthread);
+                    " trace:%zd heap:%zd other:%zd memblocks:%zd syncobj:%zu"
+                    " stacks=%zd[%zd] nthr=%zd/%zd\n",
+                    mem[MemTotal] >> 20, mem[MemShadow] >> 20,
+                    mem[MemMeta] >> 20, mem[MemFile] >> 20, mem[MemMmap] >> 20,
+                    mem[MemTrace] >> 20, mem[MemHeap] >> 20,
+                    mem[MemOther] >> 20, mem_block_mem >> 20,
+                    sync_obj_mem >> 20, stacks->allocated >> 20,
+                    stacks->n_uniq_ids, nlive, nthread);
 }
 
 #if SANITIZER_LINUX
 void FlushShadowMemoryCallback(
     const SuspendedThreadsList &suspended_threads_list,
     void *argument) {
   ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd());
 }

compiler-rt/lib/tsan/rtl/tsan_sync.h

public:
   SyncVar *GetSyncIfExists(uptr addr) {
     return GetSync(nullptr, 0, addr, false, false);
   }
 
   void MoveMemory(uptr src, uptr dst, uptr sz);
 
   void OnProcIdle(Processor *proc);
 
+  void GetMemoryStats(uptr *mem_block_mem, uptr *sync_obj_mem) const;
+
  private:
   static const u32 kFlagMask  = 3u << 30;
   static const u32 kFlagBlock = 1u << 30;
   static const u32 kFlagSync  = 2u << 30;
   typedef DenseSlabAlloc<MBlock, 1 << 18, 1 << 12, kFlagMask> BlockAlloc;
   typedef DenseSlabAlloc<SyncVar, 1 << 20, 1 << 10, kFlagMask> SyncAlloc;
   BlockAlloc block_alloc_;
   SyncAlloc sync_alloc_;

compiler-rt/lib/tsan/rtl/tsan_sync.cpp

void MetaMap::MoveMemory(uptr src, uptr dst, uptr sz) {
   }
 }
 
 void MetaMap::OnProcIdle(Processor *proc) {
   block_alloc_.FlushCache(&proc->block_cache);
   sync_alloc_.FlushCache(&proc->sync_cache);
 }
 
+void MetaMap::GetMemoryStats(uptr *mem_block_mem, uptr *sync_obj_mem) const {
+  *mem_block_mem = block_alloc_.AllocatedMemory();
+  *sync_obj_mem = sync_alloc_.AllocatedMemory();
+}
+
 }  // namespace __tsan