[LSan] Make LSan allocator allocator_may_return_null compliant

[alekseyshl]

Summary

LSan allocator used to always return nullptr on too big allocation requests
(the definition of "too big" depends on platform and bitness), now it
follows policy configured by allocator_may_return_null flag.

[eugenis]

lib/lsan/lsan_allocator.cc

137

Could you rename this to something like CallocShouldFailDueToOverflow or even CallocWouldOverflow ?

[alekseyshl]

lib/lsan/lsan_allocator.cc

137

Will do as a separate patch.

lib/lsan/lsan_allocator.cc

 }
 
 void *Allocate(const StackTrace &stack, uptr size, uptr alignment,
                bool cleared) {
   if (size == 0)
     size = 1;
   if (size > kMaxAllowedMallocSize) {
     Report("WARNING: LeakSanitizer failed to allocate %zu bytes\n", size);
-    return nullptr;
+    return Allocator::FailureHandler::OnBadRequest();
   }
   void *p = allocator.Allocate(GetAllocatorCache(), size, alignment);
   // Do not rely on the allocator to clear the memory (it's slow).
   if (cleared && allocator.FromPrimary(p))
     memset(p, 0, size);
   RegisterAllocation(stack, p, size);
   if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(p, size);
   RunMallocHooks(p, size);
   return p;
 }
 
 void Deallocate(void *p) {
   if (&__sanitizer_free_hook) __sanitizer_free_hook(p);
   RunFreeHooks(p);
   RegisterDeallocation(p);
   allocator.Deallocate(GetAllocatorCache(), p);
 }
 
 void *Reallocate(const StackTrace &stack, void *p, uptr new_size,
                  uptr alignment) {
   RegisterDeallocation(p);
   if (new_size > kMaxAllowedMallocSize) {
     Report("WARNING: LeakSanitizer failed to allocate %zu bytes\n", new_size);
     allocator.Deallocate(GetAllocatorCache(), p);
-    return nullptr;
+    return Allocator::FailureHandler::OnBadRequest();
   }
   p = allocator.Reallocate(GetAllocatorCache(), p, new_size, alignment);
   RegisterAllocation(stack, p, new_size);
   return p;
 }
 
 void GetAllocatorCacheRange(uptr *begin, uptr *end) {
   *begin = (uptr)GetAllocatorCache();
void lsan_free(void *p) {
   Deallocate(p);
 }
 
 void *lsan_realloc(void *p, uptr size, const StackTrace &stack) {
   return Reallocate(stack, p, size, 1);
 }
 
 void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack) {
+  if (CallocShouldReturnNullDueToOverflow(size, nmemb))
+    return Allocator::FailureHandler::OnBadRequest();
   size *= nmemb;
   return Allocate(stack, size, 1, true);
 }
 
 void *lsan_valloc(uptr size, const StackTrace &stack) {
   if (size == 0)
     size = GetPageSizeCached();
   return Allocate(stack, size, GetPageSizeCached(), kAlwaysClearMemory);

lib/lsan/lsan_interceptors.cc

if (lsan_init_is_running) {
     static uptr calloc_memory_for_dlsym[kCallocPoolSize];
     static uptr allocated;
     uptr size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
     void *mem = (void*)&calloc_memory_for_dlsym[allocated];
     allocated += size_in_words;
     CHECK(allocated < kCallocPoolSize);
     return mem;
   }
-  if (CallocShouldReturnNullDueToOverflow(size, nmemb)) return nullptr;
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
   return lsan_calloc(nmemb, size, stack);
 }
 
 INTERCEPTOR(void*, realloc, void *q, uptr size) {
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;

test/lsan/TestCases/allocator_returns_null.cc

+// Test the behavior of malloc/calloc/realloc/new when the allocation size is
+// more than LSan allocator's max allowed one.
+// By default (allocator_may_return_null=0) the process should crash.
+// With allocator_may_return_null=1 the allocator should return 0, except the
+// operator new(), which should crash anyway (operator new(std::nothrow) should
+// return nullptr, indeed).
+//
+// RUN: %clangxx_lsan -O0 %s -o %t
+// RUN: not %run %t malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-mCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=1     %run %t malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-mNULL
+// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t calloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-cCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=1     %run %t calloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-cNULL
+// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t calloc-overflow 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-coCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=1     %run %t calloc-overflow 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-coNULL
+// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t realloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-rCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=1     %run %t realloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-rNULL
+// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t realloc-after-malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-mrCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=1     %run %t realloc-after-malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-mrNULL
+// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t new 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-nCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=1 not %run %t new 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-nCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t new-nothrow 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-nnCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=1     %run %t new-nothrow 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-nnNULL
+
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits>
+#include <new>
+
+int main(int argc, char **argv) {
+  // Disable stderr buffering. Needed on Windows.
+  setvbuf(stderr, NULL, _IONBF, 0);
+
+  assert(argc == 2);
+  const char *action = argv[1];
+  fprintf(stderr, "%s:\n", action);
+
+  // Use max of ASan and LSan allocator limits to cover both "lsan" and
+  // "lsan + asan" configs.
+  static const size_t kMaxAllowedMallocSizePlusOne =
+#if __LP64__ || defined(_WIN64)
+      (1ULL << 40) + 1;
+#else
+      (3UL << 30) + 1;
+#endif
+
+  void *x = 0;
+  if (!strcmp(action, "malloc")) {
+    x = malloc(kMaxAllowedMallocSizePlusOne);
+  } else if (!strcmp(action, "calloc")) {
+    x = calloc((kMaxAllowedMallocSizePlusOne / 4) + 1, 4);
+  } else if (!strcmp(action, "calloc-overflow")) {
+    volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max();
+    size_t kArraySize = 4096;
+    volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10;
+    x = calloc(kArraySize, kArraySize2);
+  } else if (!strcmp(action, "realloc")) {
+    x = realloc(0, kMaxAllowedMallocSizePlusOne);
+  } else if (!strcmp(action, "realloc-after-malloc")) {
+    char *t = (char*)malloc(100);
+    *t = 42;
+    x = realloc(t, kMaxAllowedMallocSizePlusOne);
+    assert(*t == 42);
+    free(t);
+  } else if (!strcmp(action, "new")) {
+    x = operator new(kMaxAllowedMallocSizePlusOne);
+  } else if (!strcmp(action, "new-nothrow")) {
+    x = operator new(kMaxAllowedMallocSizePlusOne, std::nothrow);
+  } else {
+    assert(0);
+  }
+
+  // The NULL pointer is printed differently on different systems, while (long)0
+  // is always the same.
+  fprintf(stderr, "x: %zu\n", (size_t)x);
+  free(x);
+
+  return x != 0;
+}
+
+// CHECK-mCRASH: malloc:
+// CHECK-mCRASH: Sanitizer's allocator is terminating the process
+// CHECK-cCRASH: calloc:
+// CHECK-cCRASH: Sanitizer's allocator is terminating the process
+// CHECK-coCRASH: calloc-overflow:
+// CHECK-coCRASH: Sanitizer's allocator is terminating the process
+// CHECK-rCRASH: realloc:
+// CHECK-rCRASH: Sanitizer's allocator is terminating the process
+// CHECK-mrCRASH: realloc-after-malloc:
+// CHECK-mrCRASH: Sanitizer's allocator is terminating the process
+// CHECK-nCRASH: new:
+// CHECK-nCRASH: Sanitizer's allocator is terminating the process
+// CHECK-nnCRASH: new-nothrow:
+// CHECK-nnCRASH: Sanitizer's allocator is terminating the process
+
+// CHECK-mNULL: malloc:
+// CHECK-mNULL: x: 0
+// CHECK-cNULL: calloc:
+// CHECK-cNULL: x: 0
+// CHECK-coNULL: calloc-overflow:
+// CHECK-coNULL: x: 0
+// CHECK-rNULL: realloc:
+// CHECK-rNULL: x: 0
+// CHECK-mrNULL: realloc-after-malloc:
+// CHECK-mrNULL: x: 0
+// CHECK-nnNULL: new-nothrow:
+// CHECK-nnNULL: x: 0