[sanitizer_common] Create max_allocation_size_mb flag.

[morehouse]

Summary

The flag allows the user to specify a maximum allocation size that the
sanitizers will honor. Any larger allocations will return nullptr or
crash depending on allocator_may_return_null.

[eugenis]

compiler-rt/lib/asan/asan_allocator.cpp

288

why not in SharedInitCode ?
why not use Min(, kMaxAllowedMallocSize) same as you do in lsan and msan allocators?

444

Do we need this check as well?
needed_size > max_user_defined_malloc_size

[morehouse]

compiler-rt/lib/asan/asan_allocator.cpp

288

why not in SharedInitCode ?

SharedInitCode is also called from ReInitialize, which I think is redundant, since the common_flags wouldn't have changed (right?).

why not use Min(, kMaxAllowedMallocSize) same as you do in lsan and msan allocators?

I'm not sure we want to include ASan metadata in this new limit, while we do include it for the check against kMaxAllowedMallocSize.

444

I *think* we only want to check size here, but I'm not sure. @kcc Does Mozilla care about ASan metadata and alignment padding for this limit?

[eugenis]

LGTM

compiler-rt/lib/asan/asan_allocator.cpp

444

Makes sense.

[kcc]

LGTM, thanks!

compiler-rt/lib/asan/asan_allocator.cpp

static const uptr kMaxAllowedMallocSize =
       FIRST_32_SECOND_64(3UL << 30, 1ULL << 40);
 
   AsanAllocator allocator;
   AsanQuarantine quarantine;
   StaticSpinMutex fallback_mutex;
   AllocatorCache fallback_allocator_cache;
   QuarantineCache fallback_quarantine_cache;
 
+  uptr max_user_defined_malloc_size;
   atomic_uint8_t rss_limit_exceeded;
 
   // ------------------- Options --------------------------
   atomic_uint16_t min_redzone;
   atomic_uint16_t max_redzone;
   atomic_uint8_t alloc_dealloc_mismatch;
 
   // ------------------- Initialization ------------------------
void SharedInitCode(const AllocatorOptions &options) {
     atomic_store(&min_redzone, options.min_redzone, memory_order_release);
     atomic_store(&max_redzone, options.max_redzone, memory_order_release);
   }
 
   void InitLinkerInitialized(const AllocatorOptions &options) {
     SetAllocatorMayReturnNull(options.may_return_null);
     allocator.InitLinkerInitialized(options.release_to_os_interval_ms);
     SharedInitCode(options);
+    max_user_defined_malloc_size = common_flags()->max_allocation_size_mb
+                                       ? common_flags()->max_allocation_size_mb
+                                             << 20
+                                       : kMaxAllowedMallocSize;
   }
 
   bool RssLimitExceeded() {
     return atomic_load(&rss_limit_exceeded, memory_order_relaxed);
   }
 
   void SetRssLimitExceeded(bool limit_exceeded) {
     atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);
   }
void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack,
     bool using_primary_allocator = true;
     // If we are allocating from the secondary allocator, there will be no
     // automatic right redzone, so add the right redzone manually.
     if (!PrimaryAllocator::CanAllocate(needed_size, alignment)) {
       needed_size += rz_size;
       using_primary_allocator = false;
     }
     CHECK(IsAligned(needed_size, min_alignment));
-    if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize) {
+    if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize ||
+        size > max_user_defined_malloc_size) {
       if (AllocatorMayReturnNull()) {
         Report("WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n",
                (void*)size);
         return nullptr;
       }
-      ReportAllocationSizeTooBig(size, needed_size, kMaxAllowedMallocSize,
-                                 stack);
+      uptr malloc_limit =
+          Min(kMaxAllowedMallocSize, max_user_defined_malloc_size);
+      ReportAllocationSizeTooBig(size, needed_size, malloc_limit, stack);
     }
 
     AsanThread *t = GetCurrentThread();
     void *allocated;
     if (t) {
       AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
       allocated = allocator.Allocate(cache, needed_size, 8);
     } else {

compiler-rt/lib/lsan/lsan_allocator.cpp

 #elif defined(__mips64) || defined(__aarch64__)
 static const uptr kMaxAllowedMallocSize = 4UL << 30;
 #else
 static const uptr kMaxAllowedMallocSize = 8UL << 30;
 #endif
 
 static Allocator allocator;
 
+static uptr max_malloc_size;
+
 void InitializeAllocator() {
   SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
   allocator.InitLinkerInitialized(
       common_flags()->allocator_release_to_os_interval_ms);
+  if (common_flags()->max_allocation_size_mb)
+    max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20,
+                          kMaxAllowedMallocSize);
+  else
+    max_malloc_size = kMaxAllowedMallocSize;
 }
 
 void AllocatorThreadFinish() {
   allocator.SwallowCache(GetAllocatorCache());
 }
 
 static ChunkMetadata *Metadata(const void *p) {
   return reinterpret_cast<ChunkMetadata *>(allocator.GetMetaData(p));
static void RegisterDeallocation(void *p) {
   atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 0, memory_order_relaxed);
 }
 
 static void *ReportAllocationSizeTooBig(uptr size, const StackTrace &stack) {
   if (AllocatorMayReturnNull()) {
     Report("WARNING: LeakSanitizer failed to allocate 0x%zx bytes\n", size);
     return nullptr;
   }
-  ReportAllocationSizeTooBig(size, kMaxAllowedMallocSize, &stack);
+  ReportAllocationSizeTooBig(size, max_malloc_size, &stack);
 }
 
 void *Allocate(const StackTrace &stack, uptr size, uptr alignment,
                bool cleared) {
   if (size == 0)
     size = 1;
-  if (size > kMaxAllowedMallocSize)
+  if (size > max_malloc_size)
     return ReportAllocationSizeTooBig(size, stack);
   void *p = allocator.Allocate(GetAllocatorCache(), size, alignment);
   if (UNLIKELY(!p)) {
     SetAllocatorOutOfMemory();
     if (AllocatorMayReturnNull())
       return nullptr;
     ReportOutOfMemory(size, &stack);
   }
void Deallocate(void *p) {
   RunFreeHooks(p);
   RegisterDeallocation(p);
   allocator.Deallocate(GetAllocatorCache(), p);
 }
 
 void *Reallocate(const StackTrace &stack, void *p, uptr new_size,
                  uptr alignment) {
   RegisterDeallocation(p);
-  if (new_size > kMaxAllowedMallocSize) {
+  if (new_size > max_malloc_size) {
     allocator.Deallocate(GetAllocatorCache(), p);
     return ReportAllocationSizeTooBig(new_size, stack);
   }
   p = allocator.Reallocate(GetAllocatorCache(), p, new_size, alignment);
   RegisterAllocation(stack, p, new_size);
   return p;
 }
 

compiler-rt/lib/msan/msan_allocator.cpp

 #endif
 typedef CombinedAllocator<PrimaryAllocator> Allocator;
 typedef Allocator::AllocatorCache AllocatorCache;
 
 static Allocator allocator;
 static AllocatorCache fallback_allocator_cache;
 static StaticSpinMutex fallback_mutex;
 
+static uptr max_malloc_size;
+
 void MsanAllocatorInit() {
   SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
   allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
+  if (common_flags()->max_allocation_size_mb)
+    max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20,
+                          kMaxAllowedMallocSize);
+  else
+    max_malloc_size = kMaxAllowedMallocSize;
 }
 
 AllocatorCache *GetAllocatorCache(MsanThreadLocalMallocStorage *ms) {
   CHECK(ms);
   CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
   return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
 }
 
 void MsanThreadLocalMallocStorage::CommitBack() {
   allocator.SwallowCache(GetAllocatorCache(this));
 }
 
 static void *MsanAllocate(StackTrace *stack, uptr size, uptr alignment,
                           bool zeroise) {
-  if (size > kMaxAllowedMallocSize) {
+  if (size > max_malloc_size) {
     if (AllocatorMayReturnNull()) {
       Report("WARNING: MemorySanitizer failed to allocate 0x%zx bytes\n", size);
       return nullptr;
     }
-    ReportAllocationSizeTooBig(size, kMaxAllowedMallocSize, stack);
+    ReportAllocationSizeTooBig(size, max_malloc_size, stack);
   }
   MsanThread *t = GetCurrentThread();
   void *allocated;
   if (t) {
     AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
     allocated = allocator.Allocate(cache, size, alignment);
   } else {
     SpinMutexLock l(&fallback_mutex);

compiler-rt/lib/sanitizer_common/sanitizer_flags.inc

COMMON_FLAG(uptr, soft_rss_limit_mb, 0,
             "Soft RSS limit in Mb."
             " If non-zero, a background thread is spawned at startup"
             " which periodically reads RSS. If the limit is reached"
             " all subsequent malloc/new calls will fail or return NULL"
             " (depending on the value of allocator_may_return_null)"
             " until the RSS goes below the soft limit."
             " This limit does not affect memory allocations other than"
             " malloc/new.")
+COMMON_FLAG(uptr, max_allocation_size_mb, 0,
+            "If non-zero, malloc/new calls larger than this size will return "
+            "nullptr (or crash if allocator_may_return_null=false).")
 COMMON_FLAG(bool, heap_profile, false, "Experimental heap profiler, asan-only")
 COMMON_FLAG(s32, allocator_release_to_os_interval_ms,
             ((bool)SANITIZER_FUCHSIA || (bool)SANITIZER_WINDOWS) ? -1 : 5000,
             "Only affects a 64-bit allocator. If set, tries to release unused "
             "memory to the OS, but not more often than this interval (in "
             "milliseconds). Negative values mean do not attempt to release "
             "memory to the OS.\n")
 COMMON_FLAG(bool, can_use_proc_maps_statm, true,

compiler-rt/lib/tsan/rtl/tsan_mman.cpp

ScopedGlobalProcessor::~ScopedGlobalProcessor() {
   GlobalProc *gp = global_proc();
   ThreadState *thr = cur_thread();
   if (thr->proc() != gp->proc)
     return;
   ProcUnwire(gp->proc, thr);
   gp->mtx.Unlock();
 }
 
+static constexpr uptr kMaxAllowedMallocSize = 1ull << 40;
+static uptr max_user_defined_malloc_size;
+
 void InitializeAllocator() {
   SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
   allocator()->Init(common_flags()->allocator_release_to_os_interval_ms);
+  max_user_defined_malloc_size = common_flags()->max_allocation_size_mb
+                                     ? common_flags()->max_allocation_size_mb
+                                           << 20
+                                     : kMaxAllowedMallocSize;
 }
 
 void InitializeAllocatorLate() {
   new(global_proc()) GlobalProc();
 }
 
 void AllocatorProcStart(Processor *proc) {
   allocator()->InitCache(&proc->alloc_cache);
static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
   if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack))
     return;
   ThreadRegistryLock l(ctx->thread_registry);
   ScopedReport rep(ReportTypeSignalUnsafe);
   rep.AddStack(stack, true);
   OutputReport(thr, rep);
 }
 
-static constexpr uptr kMaxAllowedMallocSize = 1ull << 40;
 
 void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz, uptr align,
                           bool signal) {
-  if (sz >= kMaxAllowedMallocSize || align >= kMaxAllowedMallocSize) {
+  if (sz >= kMaxAllowedMallocSize || align >= kMaxAllowedMallocSize ||
+      sz > max_user_defined_malloc_size) {
     if (AllocatorMayReturnNull())
       return nullptr;
+    uptr malloc_limit =
+        Min(kMaxAllowedMallocSize, max_user_defined_malloc_size);
     GET_STACK_TRACE_FATAL(thr, pc);
-    ReportAllocationSizeTooBig(sz, kMaxAllowedMallocSize, &stack);
+    ReportAllocationSizeTooBig(sz, malloc_limit, &stack);
   }
   void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align);
   if (UNLIKELY(!p)) {
     SetAllocatorOutOfMemory();
     if (AllocatorMayReturnNull())
       return nullptr;
     GET_STACK_TRACE_FATAL(thr, pc);
     ReportOutOfMemory(sz, &stack);

compiler-rt/test/sanitizer_common/TestCases/max_allocation_size.cpp

+// Test the behavior of malloc/calloc/realloc/new when the allocation size
+// exceeds the configured max_allocation_size_mb flag.
+// By default (allocator_may_return_null=0) the process should crash. With
+// allocator_may_return_null=1 the allocator should return nullptr and set errno
+// to the appropriate error code.
+//
+// RUN: %clangxx -O0 %s -o %t
+// RUN: %run %t malloc 2>&1 | FileCheck %s --check-prefix=CHECK-NOTNULL
+// RUN: %env_tool_opts=max_allocation_size_mb=3 %run %t malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-NOTNULL
+// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=0 \
+// RUN:   not %run %t malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mCRASH
+// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=1 \
+// RUN:   %run %t malloc 2>&1 | FileCheck %s --check-prefix=CHECK-NULL
+// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=0 \
+// RUN:   not %run %t calloc 2>&1 | FileCheck %s --check-prefix=CHECK-cCRASH
+// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=1 \
+// RUN:   %run %t calloc 2>&1 | FileCheck %s --check-prefix=CHECK-NULL
+// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=0 \
+// RUN:   not %run %t realloc 2>&1 | FileCheck %s --check-prefix=CHECK-rCRASH
+// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=1 \
+// RUN:   %run %t realloc 2>&1 | FileCheck %s --check-prefix=CHECK-NULL
+// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=0 \
+// RUN:   not %run %t realloc-after-malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-mrCRASH
+// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=1 \
+// RUN:   %run %t realloc-after-malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-NULL
+// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=0 \
+// RUN:   not %run %t new 2>&1 | FileCheck %s --check-prefix=CHECK-nCRASH
+// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=1 \
+// RUN:   not %run %t new 2>&1 | FileCheck %s --check-prefix=CHECK-nCRASH-OOM
+// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=0 \
+// RUN:   not %run %t new-nothrow 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-nnCRASH
+// RUN: %env_tool_opts=max_allocation_size_mb=2:allocator_may_return_null=1 \
+// RUN:   %run %t new-nothrow 2>&1 | FileCheck %s --check-prefix=CHECK-NULL
+
+// win32 is disabled due to failing errno tests.
+// UNSUPPORTED: ubsan, windows-msvc
+
+#include <assert.h>
+#include <errno.h>
+#include <limits>
+#include <new>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static void *allocate(const char *Action, size_t Size) {
+  if (!strcmp(Action, "malloc"))
+    return malloc(Size);
+  if (!strcmp(Action, "calloc"))
+    return calloc((Size + 3) / 4, 4);
+  if (!strcmp(Action, "realloc"))
+    return realloc(nullptr, Size);
+  if (!strcmp(Action, "realloc-after-malloc")) {
+    void *P = malloc(100);
+    if (void *Ret = realloc(P, Size))
+      return Ret;
+    free(P);
+    return nullptr;
+  }
+  if (!strcmp(Action, "new"))
+    return ::operator new(Size);
+  if (!strcmp(Action, "new-nothrow"))
+    return ::operator new(Size, std::nothrow);
+  assert(0);
+}
+
+static void deallocate(const char *Action, void *Ptr) {
+  if (!strcmp(Action, "malloc") || !strcmp(Action, "calloc") ||
+      !strcmp(Action, "realloc") || !strcmp(Action, "realloc-after-malloc"))
+    return free(Ptr);
+  if (!strcmp(Action, "new"))
+    return ::operator delete(Ptr);
+  if (!strcmp(Action, "new-nothrow"))
+    return ::operator delete(Ptr, std::nothrow);
+  assert(0);
+}
+
+int main(int Argc, char **Argv) {
+  assert(Argc == 2);
+  const char *Action = Argv[1];
+  fprintf(stderr, "%s:\n", Action);
+
+  constexpr size_t MaxAllocationSize = size_t{2} << 20;
+
+  // Should succeed when max_allocation_size_mb is set.
+  void *volatile P = allocate(Action, MaxAllocationSize);
+  assert(P);
+  deallocate(Action, P);
+
+  // Should fail when max_allocation_size_mb is set.
+  P = allocate(Action, MaxAllocationSize + 1);
+  // The NULL pointer is printed differently on different systems, while (long)0
+  // is always the same.
+  fprintf(stderr, "errno: %d, P: %lx\n", errno, (long)P);
+  deallocate(Action, P);
+
+  // Should succeed when max_allocation_size_mb is set.
+  P = allocate(Action, MaxAllocationSize);
+  assert(P);
+  deallocate(Action, P);
+
+  return 0;
+}
+
+// CHECK-mCRASH: malloc:
+// CHECK-mCRASH: {{SUMMARY: .*Sanitizer: allocation-size-too-big}}
+// CHECK-cCRASH: calloc:
+// CHECK-cCRASH: {{SUMMARY: .*Sanitizer: allocation-size-too-big}}
+// CHECK-rCRASH: realloc:
+// CHECK-rCRASH: {{SUMMARY: .*Sanitizer: allocation-size-too-big}}
+// CHECK-mrCRASH: realloc-after-malloc:
+// CHECK-mrCRASH: {{SUMMARY: .*Sanitizer: allocation-size-too-big}}
+// CHECK-nCRASH: new:
+// CHECK-nCRASH: {{SUMMARY: .*Sanitizer: allocation-size-too-big}}
+// CHECK-nCRASH-OOM: new:
+// CHECK-nCRASH-OOM: {{SUMMARY: .*Sanitizer: out-of-memory}}
+// CHECK-nnCRASH: new-nothrow:
+// CHECK-nnCRASH: {{SUMMARY: .*Sanitizer: allocation-size-too-big}}
+
+// CHECK-NULL: {{malloc|calloc|calloc-overflow|realloc|realloc-after-malloc|new-nothrow}}
+// CHECK-NULL: errno: 12, P: 0
+//
+// CHECK-NOTNULL-NOT: P: 0