Expose thread local quarantine size as ASAN option.

[alekseyshl]

Summary

Make thread local quarantine size an option so it can be turned off to save memory.

[eugenis]

test/asan/TestCases/Linux/thread_local_quarantine_size_kb.cc

27 ↗

(On Diff #82253)

background thread?

27 ↗

(On Diff #82253)

ah right, rss limit

[eugenis]

Could you change the test to use __sanitizer_get_heap_size? That would avoid the need to sleep in the test (which can be flaky under load). Also, that way you can measure incremental memory use, instead of hardcoding a total rss limit which includes all kinds of unrelated allocations.

[alekseyshl]

• Verify the actual heap size, not the total RSS.

[alekseyshl]

Done

[eugenis]

LGTM

compiler-rt/trunk/lib/asan/asan_allocator.h

enum AllocType {
   FROM_NEW = 2,     // Memory block came from operator new.
   FROM_NEW_BR = 3   // Memory block came from operator new [ ]
 };
 
 struct AsanChunk;
 
 struct AllocatorOptions {
   u32 quarantine_size_mb;
+  u32 thread_local_quarantine_size_kb;
   u16 min_redzone;
   u16 max_redzone;
   u8 may_return_null;
   u8 alloc_dealloc_mismatch;
   s32 release_to_os_interval_ms;
 
   void SetFrom(const Flags *f, const CommonFlags *cf);
   void CopyTo(Flags *f, CommonFlags *cf);

compiler-rt/trunk/lib/asan/asan_allocator.cc

 QuarantineCache *GetQuarantineCache(AsanThreadLocalMallocStorage *ms) {
   CHECK(ms);
   CHECK_LE(sizeof(QuarantineCache), sizeof(ms->quarantine_cache));
   return reinterpret_cast<QuarantineCache *>(ms->quarantine_cache);
 }
 
 void AllocatorOptions::SetFrom(const Flags *f, const CommonFlags *cf) {
   quarantine_size_mb = f->quarantine_size_mb;
+  thread_local_quarantine_size_kb = f->thread_local_quarantine_size_kb;
   min_redzone = f->redzone;
   max_redzone = f->max_redzone;
   may_return_null = cf->allocator_may_return_null;
   alloc_dealloc_mismatch = f->alloc_dealloc_mismatch;
   release_to_os_interval_ms = cf->allocator_release_to_os_interval_ms;
 }
 
 void AllocatorOptions::CopyTo(Flags *f, CommonFlags *cf) {
   f->quarantine_size_mb = quarantine_size_mb;
+  f->thread_local_quarantine_size_kb = thread_local_quarantine_size_kb;
   f->redzone = min_redzone;
   f->max_redzone = max_redzone;
   cf->allocator_may_return_null = may_return_null;
   f->alloc_dealloc_mismatch = alloc_dealloc_mismatch;
   cf->allocator_release_to_os_interval_ms = release_to_os_interval_ms;
 }
 
 struct Allocator {
   static const uptr kMaxAllowedMallocSize =
       FIRST_32_SECOND_64(3UL << 30, 1ULL << 40);
-  static const uptr kMaxThreadLocalQuarantine =
-      // It is not advised to go lower than 64Kb, otherwise quarantine batches
-      // pushed from thread local quarantine to global one will create too much
-      // overhead. One quarantine batch size is 8Kb and it  holds up to 1021
-      // chunk, which amounts to 1/8 memory overhead per batch when thread local
-      // quarantine is set to 64Kb.
-      (ASAN_LOW_MEMORY) ? 1 << 16 : FIRST_32_SECOND_64(1 << 18, 1 << 20);
 
   AsanAllocator allocator;
   AsanQuarantine quarantine;
   StaticSpinMutex fallback_mutex;
   AllocatorCache fallback_allocator_cache;
   QuarantineCache fallback_quarantine_cache;
 
   // ------------------- Options --------------------------
void CheckOptions(const AllocatorOptions &options) const {
     CHECK_LE(options.max_redzone, 2048);
     CHECK(IsPowerOfTwo(options.min_redzone));
     CHECK(IsPowerOfTwo(options.max_redzone));
   }
 
   void SharedInitCode(const AllocatorOptions &options) {
     CheckOptions(options);
     quarantine.Init((uptr)options.quarantine_size_mb << 20,
-                    kMaxThreadLocalQuarantine);
+                    (uptr)options.thread_local_quarantine_size_kb << 10);
     atomic_store(&alloc_dealloc_mismatch, options.alloc_dealloc_mismatch,
                  memory_order_release);
     atomic_store(&min_redzone, options.min_redzone, memory_order_release);
     atomic_store(&max_redzone, options.max_redzone, memory_order_release);
   }
 
   void Initialize(const AllocatorOptions &options) {
     allocator.Init(options.may_return_null, options.release_to_os_interval_ms);
if (CanPoisonMemory()) {
           },
           this);
       allocator.ForceUnlock();
     }
   }
 
   void GetOptions(AllocatorOptions *options) const {
     options->quarantine_size_mb = quarantine.GetSize() >> 20;
+    options->thread_local_quarantine_size_kb = quarantine.GetCacheSize() >> 10;
     options->min_redzone = atomic_load(&min_redzone, memory_order_acquire);
     options->max_redzone = atomic_load(&max_redzone, memory_order_acquire);
     options->may_return_null = allocator.MayReturnNull();
     options->alloc_dealloc_mismatch =
         atomic_load(&alloc_dealloc_mismatch, memory_order_acquire);
     options->release_to_os_interval_ms = allocator.ReleaseToOSIntervalMs();
   }
 

compiler-rt/trunk/lib/asan/asan_flags.cc

#endif
   }
   if (f->quarantine_size >= 0)
     f->quarantine_size_mb = f->quarantine_size >> 20;
   if (f->quarantine_size_mb < 0) {
     const int kDefaultQuarantineSizeMb =
         (ASAN_LOW_MEMORY) ? 1UL << 4 : 1UL << 8;
     f->quarantine_size_mb = kDefaultQuarantineSizeMb;
   }
+  if (f->thread_local_quarantine_size_kb < 0) {
+    const u32 kDefaultThreadLocalQuarantineSizeKb =
+        // It is not advised to go lower than 64Kb, otherwise quarantine batches
+        // pushed from thread local quarantine to global one will create too
+        // much overhead. One quarantine batch size is 8Kb and it  holds up to
+        // 1021 chunk, which amounts to 1/8 memory overhead per batch when
+        // thread local quarantine is set to 64Kb.
+        (ASAN_LOW_MEMORY) ? 1 << 6 : FIRST_32_SECOND_64(1 << 8, 1 << 10);
+    f->thread_local_quarantine_size_kb = kDefaultThreadLocalQuarantineSizeKb;
+  }
   if (!f->replace_str && common_flags()->intercept_strlen) {
     Report("WARNING: strlen interceptor is enabled even though replace_str=0. "
            "Use intercept_strlen=0 to disable it.");
   }
   if (!f->replace_str && common_flags()->intercept_strchr) {
     Report("WARNING: strchr* interceptors are enabled even though "
            "replace_str=0. Use intercept_strchr=0 to disable them.");
   }

compiler-rt/trunk/lib/asan/asan_flags.inc

 // See COMMON_FLAG in sanitizer_flags.inc for more details.
 
 ASAN_FLAG(int, quarantine_size, -1,
             "Deprecated, please use quarantine_size_mb.")
 ASAN_FLAG(int, quarantine_size_mb, -1,
           "Size (in Mb) of quarantine used to detect use-after-free "
           "errors. Lower value may reduce memory usage but increase the "
           "chance of false negatives.")
+ASAN_FLAG(int, thread_local_quarantine_size_kb, -1,
+          "Size (in Kb) of thread local quarantine used to detect "
+          "use-after-free errors. Lower value may reduce memory usage but "
+          "increase the chance of false negatives. It is not advised to go "
+          "lower than 64Kb, otherwise frequent transfers to global quarantine "
+          "might affect performance.")
 ASAN_FLAG(int, redzone, 16,
           "Minimal size (in bytes) of redzones around heap objects. "
           "Requirement: redzone >= 16, is a power of two.")
 ASAN_FLAG(int, max_redzone, 2048,
           "Maximal size (in bytes) of redzones around heap objects.")
 ASAN_FLAG(
     bool, debug, false,
     "If set, prints some debugging information and does additional checks.")

compiler-rt/trunk/lib/asan/asan_rtl.cc

if (kMidMemBeg) {
     Printf(" %p %p",
            (void*)MEM_TO_SHADOW(kMidShadowBeg),
            (void*)MEM_TO_SHADOW(kMidShadowEnd));
   }
   Printf("\n");
   Printf("redzone=%zu\n", (uptr)flags()->redzone);
   Printf("max_redzone=%zu\n", (uptr)flags()->max_redzone);
   Printf("quarantine_size_mb=%zuM\n", (uptr)flags()->quarantine_size_mb);
+  Printf("thread_local_quarantine_size_kb=%zuK\n",
+         (uptr)flags()->thread_local_quarantine_size_kb);
   Printf("malloc_context_size=%zu\n",
          (uptr)common_flags()->malloc_context_size);
 
   Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE);
   Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY);
   Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET);
   CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
   if (kMidMemBeg)

compiler-rt/trunk/lib/sanitizer_common/sanitizer_quarantine.h

public:
   void Init(uptr size, uptr cache_size) {
     atomic_store(&max_size_, size, memory_order_release);
     atomic_store(&min_size_, size / 10 * 9,
                  memory_order_release); // 90% of max size.
     max_cache_size_ = cache_size;
   }
 
   uptr GetSize() const { return atomic_load(&max_size_, memory_order_acquire); }
+  uptr GetCacheSize() const { return max_cache_size_; }
 
   void Put(Cache *c, Callback cb, Node *ptr, uptr size) {
     c->Enqueue(cb, ptr, size);
     if (c->Size() > max_cache_size_)
       Drain(c, cb);
   }
 
   void NOINLINE Drain(Cache *c, Callback cb) {

compiler-rt/trunk/test/asan/TestCases/Linux/thread_local_quarantine_size_kb.cc

+// Test thread_local_quarantine_size_kb
+
+// RUN: %clangxx_asan  %s -o %t
+// RUN: %env_asan_opts=thread_local_quarantine_size_kb=256:verbosity=1 %run %t 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=CHECK-VALUE
+// RUN: %env_asan_opts=thread_local_quarantine_size_kb=64:quarantine_size_mb=64 %run %t 2>&1 | \
+// RUN:   FileCheck %s --allow-empty --check-prefix=CHECK-SMALL-LOCAL-CACHE-SMALL-OVERHEAD
+// RUN: %env_asan_opts=thread_local_quarantine_size_kb=0:quarantine_size_mb=64 %run %t 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=CHECK-NO-LOCAL-CACHE-HUGE-OVERHEAD
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sanitizer/allocator_interface.h>
+
+// The idea is allocate a lot of small blocks, totaling 5Mb of user memory
+// total, and verify that quarantine does not incur too much memory overhead.
+// There's always an overhead for red zones, shadow memory and such, but
+// quarantine accounting should not significantly contribute to that.
+static const int kNumAllocs = 20000;
+static const int kAllocSize = 256;
+static const size_t kHeapSizeLimit = 12 << 20;
+
+int main() {
+  size_t old_heap_size = __sanitizer_get_heap_size();
+  for (int i = 0; i < kNumAllocs; i++) {
+    char *g = new char[kAllocSize];
+    memset(g, -1, kAllocSize);
+    delete [] (g);
+  }
+  size_t new_heap_size = __sanitizer_get_heap_size();
+  fprintf(stderr, "heap size: new: %zd old: %zd\n", new_heap_size,
+          old_heap_size);
+  if (new_heap_size - old_heap_size > kHeapSizeLimit)
+    fprintf(stderr, "Heap size limit exceeded");
+}
+
+// CHECK-VALUE: thread_local_quarantine_size_kb=256K
+// CHECK-SMALL-LOCAL-CACHE-SMALL-OVERHEAD-NOT: Heap size limit exceeded
+// CHECK-NO-LOCAL-CACHE-HUGE-OVERHEAD: Heap size limit exceeded