[hwasan] Madvise away thread aux data

[eugenis]

Summary

Release memory pages for thread data (allocator cache, stack allocations
ring buffer, etc) when a thread exits. We can not simply munmap them
because this memory is custom allocated within a limited address range,
and it needs to stay "reserved".

[eugenis]

This one is hard to test reliably for RSS change, because it only affect thread exit, and, generally, we don't know how much memory the platform thread support would release in that event, and it might even be a different amount for different threads (some kind of stack caching, etc).

We don't rely on ReleaseMemoryPagesToOS clearing memory here, so this an almost side-effect-free optimization.

PTAL.

[pcc]

compiler-rt/lib/hwasan/hwasan_thread_list.h

186 ↗

(On Diff #181389)

Would it help to release memory starting from the original value of free_space_ here? It seems that you would be able to probabilistically free a fraction of a page more that way. Or am I missing something?

[eugenis]

madvise the entire allocation range

[eugenis]

ping

[pcc]

LGTM

compiler-rt/lib/hwasan/hwasan_thread_list.h

146 ↗

(On Diff #183194)

Remove FIXME.

compiler-rt/trunk/lib/hwasan/hwasan_thread_list.h

 struct ThreadStats {
   uptr n_live_threads;
   uptr total_stack_size;
 };
 
 class HwasanThreadList {
  public:
   HwasanThreadList(uptr storage, uptr size)
-      : free_space_(storage),
-        free_space_end_(storage + size),
-        ring_buffer_size_(RingBufferSize()) {}
+      : free_space_(storage), free_space_end_(storage + size) {
+    // [storage, storage + size) is used as a vector of
+    // thread_alloc_size_-sized, ring_buffer_size_*2-aligned elements.
+    // Each element contains
+    // * a ring buffer at offset 0,
+    // * a Thread object at offset ring_buffer_size_.
+    ring_buffer_size_ = RingBufferSize();
+    thread_alloc_size_ =
+        RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2);
+  }
 
   Thread *CreateCurrentThread() {
     Thread *t;
     {
       SpinMutexLock l(&list_mutex_);
       t = free_list_.Pop();
-      if (t)
-        internal_memset((void *)t, 0, sizeof(Thread) + ring_buffer_size_);
-      else
+      if (t) {
+        uptr start = (uptr)t - ring_buffer_size_;
+        internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread));
+      } else {
         t = AllocThread();
+      }
       live_list_.Push(t);
     }
-    t->Init((uptr)(t + 1), ring_buffer_size_);
+    t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_);
     AddThreadStats(t);
     return t;
   }
 
+  void DontNeedThread(Thread *t) {
+    uptr start = (uptr)t - ring_buffer_size_;
+    ReleaseMemoryPagesToOS(start, start + thread_alloc_size_);
+  }
+
   void ReleaseThread(Thread *t) {
-    // FIXME: madvise away the ring buffer?
     RemoveThreadStats(t);
     t->Destroy();
     SpinMutexLock l(&list_mutex_);
     live_list_.Remove(t);
     free_list_.Push(t);
+    DontNeedThread(t);
   }
 
   Thread *GetThreadByBufferAddress(uptr p) {
-    uptr align = ring_buffer_size_ * 2;
-    return (Thread *)(RoundDownTo(p, align) - sizeof(Thread));
+    return (Thread *)(RoundDownTo(p, ring_buffer_size_ * 2) +
+                      ring_buffer_size_);
   }
 
   uptr MemoryUsedPerThread() {
     uptr res = sizeof(Thread) + ring_buffer_size_;
     if (auto sz = flags()->heap_history_size)
       res += HeapAllocationsRingBuffer::SizeInBytes(sz);
     return res;
   }
public:
   ThreadStats GetThreadStats() {
     SpinMutexLock l(&stats_mutex_);
     return stats_;
   }
 
  private:
   Thread *AllocThread() {
     uptr align = ring_buffer_size_ * 2;
-    uptr ring_buffer_start = RoundUpTo(free_space_ + sizeof(Thread), align);
-    free_space_ = ring_buffer_start + ring_buffer_size_;
+    CHECK(IsAligned(free_space_, align));
+    Thread *t = (Thread *)(free_space_ + ring_buffer_size_);
+    free_space_ += thread_alloc_size_;
     CHECK(free_space_ <= free_space_end_ && "out of thread memory");
-    return (Thread *)(ring_buffer_start - sizeof(Thread));
+    return t;
   }
 
   uptr free_space_;
   uptr free_space_end_;
   uptr ring_buffer_size_;
+  uptr thread_alloc_size_;
 
   ThreadListHead free_list_;
   ThreadListHead live_list_;
   SpinMutex list_mutex_;
 
   ThreadStats stats_;
   SpinMutex stats_mutex_;
 };
 
 void InitThreadList(uptr storage, uptr size);
 HwasanThreadList &hwasanThreadList();
 
 } // namespace