Index: compiler-rt/trunk/lib/lsan/lsan_allocator.cc =================================================================== --- compiler-rt/trunk/lib/lsan/lsan_allocator.cc +++ compiler-rt/trunk/lib/lsan/lsan_allocator.cc @@ -74,7 +74,7 @@ size = 1; if (size > kMaxAllowedMallocSize) { Report("WARNING: LeakSanitizer failed to allocate %zu bytes\n", size); - return nullptr; + return Allocator::FailureHandler::OnBadRequest(); } void *p = allocator.Allocate(GetAllocatorCache(), size, alignment); // Do not rely on the allocator to clear the memory (it's slow). @@ -99,7 +99,7 @@ if (new_size > kMaxAllowedMallocSize) { Report("WARNING: LeakSanitizer failed to allocate %zu bytes\n", new_size); allocator.Deallocate(GetAllocatorCache(), p); - return nullptr; + return Allocator::FailureHandler::OnBadRequest(); } p = allocator.Reallocate(GetAllocatorCache(), p, new_size, alignment); RegisterAllocation(stack, p, new_size); @@ -134,6 +134,8 @@ } void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack) { + if (CheckForCallocOverflow(size, nmemb)) + return Allocator::FailureHandler::OnBadRequest(); size *= nmemb; return Allocate(stack, size, 1, true); } Index: compiler-rt/trunk/lib/lsan/lsan_interceptors.cc =================================================================== --- compiler-rt/trunk/lib/lsan/lsan_interceptors.cc +++ compiler-rt/trunk/lib/lsan/lsan_interceptors.cc @@ -70,7 +70,6 @@ CHECK(allocated < kCallocPoolSize); return mem; } - if (CheckForCallocOverflow(size, nmemb)) return nullptr; ENSURE_LSAN_INITED; GET_STACK_TRACE_MALLOC; return lsan_calloc(nmemb, size, stack); @@ -199,6 +198,7 @@ } #endif // SANITIZER_INTERCEPT_MCHECK_MPROBE + // TODO(alekseys): throw std::bad_alloc instead of dying on OOM. #define OPERATOR_NEW_BODY(nothrow) \ ENSURE_LSAN_INITED; \ @@ -207,22 +207,28 @@ if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\ return res; +#define OPERATOR_DELETE_BODY \ + ENSURE_LSAN_INITED; \ + Deallocate(ptr); + +// On OS X it's not enough to just provide our own 'operator new' and +// 'operator delete' implementations, because they're going to be in the runtime +// dylib, and the main executable will depend on both the runtime dylib and +// libstdc++, each of has its implementation of new and delete. +// To make sure that C++ allocation/deallocation operators are overridden on +// OS X we need to intercept them using their mangled names. +#if !SANITIZER_MAC + INTERCEPTOR_ATTRIBUTE void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); } INTERCEPTOR_ATTRIBUTE void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); } INTERCEPTOR_ATTRIBUTE -void *operator new(size_t size, std::nothrow_t const&) { - OPERATOR_NEW_BODY(true /*nothrow*/); -} +void *operator new(size_t size, std::nothrow_t const&) +{ OPERATOR_NEW_BODY(true /*nothrow*/); } INTERCEPTOR_ATTRIBUTE -void *operator new[](size_t size, std::nothrow_t const&) { - OPERATOR_NEW_BODY(true /*nothrow*/); -} - -#define OPERATOR_DELETE_BODY \ - ENSURE_LSAN_INITED; \ - Deallocate(ptr); +void *operator new[](size_t size, std::nothrow_t const&) +{ OPERATOR_NEW_BODY(true /*nothrow*/); } INTERCEPTOR_ATTRIBUTE void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } @@ -231,9 +237,31 @@ INTERCEPTOR_ATTRIBUTE void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } INTERCEPTOR_ATTRIBUTE -void operator delete[](void *ptr, std::nothrow_t const &) { - OPERATOR_DELETE_BODY; -} +void operator delete[](void *ptr, std::nothrow_t const &) +{ OPERATOR_DELETE_BODY; } + +#else // SANITIZER_MAC + +INTERCEPTOR(void *, _Znwm, size_t size) +{ OPERATOR_NEW_BODY(false /*nothrow*/); } +INTERCEPTOR(void *, _Znam, size_t size) +{ OPERATOR_NEW_BODY(false /*nothrow*/); } +INTERCEPTOR(void *, _ZnwmRKSt9nothrow_t, size_t size, std::nothrow_t const&) +{ OPERATOR_NEW_BODY(true /*nothrow*/); } +INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) +{ OPERATOR_NEW_BODY(true /*nothrow*/); } + +INTERCEPTOR(void, _ZdlPv, void *ptr) +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR(void, _ZdaPv, void *ptr) +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY; } +INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) +{ OPERATOR_DELETE_BODY; } + +#endif // !SANITIZER_MAC + ///// Thread initialization and finalization. ///// Index: compiler-rt/trunk/test/lsan/TestCases/allocator_returns_null.cc =================================================================== --- compiler-rt/trunk/test/lsan/TestCases/allocator_returns_null.cc +++ compiler-rt/trunk/test/lsan/TestCases/allocator_returns_null.cc @@ -0,0 +1,123 @@ +// Test the behavior of malloc/calloc/realloc/new when the allocation size is +// more than LSan allocator's max allowed one. +// By default (allocator_may_return_null=0) the process should crash. +// With allocator_may_return_null=1 the allocator should return 0, except the +// operator new(), which should crash anyway (operator new(std::nothrow) should +// return nullptr, indeed). +// +// RUN: %clangxx_lsan -O0 %s -o %t +// RUN: not %run %t malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mCRASH +// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t malloc 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-mCRASH +// RUN: %env_lsan_opts=allocator_may_return_null=1 %run %t malloc 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-mNULL +// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t calloc 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-cCRASH +// RUN: %env_lsan_opts=allocator_may_return_null=1 %run %t calloc 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-cNULL +// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t calloc-overflow 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-coCRASH +// RUN: %env_lsan_opts=allocator_may_return_null=1 %run %t calloc-overflow 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-coNULL +// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t realloc 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-rCRASH +// RUN: %env_lsan_opts=allocator_may_return_null=1 %run %t realloc 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-rNULL +// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t realloc-after-malloc 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-mrCRASH +// RUN: %env_lsan_opts=allocator_may_return_null=1 %run %t realloc-after-malloc 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-mrNULL +// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t new 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-nCRASH +// RUN: %env_lsan_opts=allocator_may_return_null=1 not %run %t new 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-nCRASH +// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t new-nothrow 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-nnCRASH +// RUN: %env_lsan_opts=allocator_may_return_null=1 %run %t new-nothrow 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECK-nnNULL + +#include +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + // Disable stderr buffering. Needed on Windows. + setvbuf(stderr, NULL, _IONBF, 0); + + assert(argc == 2); + const char *action = argv[1]; + fprintf(stderr, "%s:\n", action); + + // Use max of ASan and LSan allocator limits to cover both "lsan" and + // "lsan + asan" configs. + static const size_t kMaxAllowedMallocSizePlusOne = +#if __LP64__ || defined(_WIN64) + (1ULL << 40) + 1; +#else + (3UL << 30) + 1; +#endif + + void *x = 0; + if (!strcmp(action, "malloc")) { + x = malloc(kMaxAllowedMallocSizePlusOne); + } else if (!strcmp(action, "calloc")) { + x = calloc((kMaxAllowedMallocSizePlusOne / 4) + 1, 4); + } else if (!strcmp(action, "calloc-overflow")) { + volatile size_t kMaxSizeT = std::numeric_limits::max(); + size_t kArraySize = 4096; + volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10; + x = calloc(kArraySize, kArraySize2); + } else if (!strcmp(action, "realloc")) { + x = realloc(0, kMaxAllowedMallocSizePlusOne); + } else if (!strcmp(action, "realloc-after-malloc")) { + char *t = (char*)malloc(100); + *t = 42; + x = realloc(t, kMaxAllowedMallocSizePlusOne); + assert(*t == 42); + free(t); + } else if (!strcmp(action, "new")) { + x = operator new(kMaxAllowedMallocSizePlusOne); + } else if (!strcmp(action, "new-nothrow")) { + x = operator new(kMaxAllowedMallocSizePlusOne, std::nothrow); + } else { + assert(0); + } + + // The NULL pointer is printed differently on different systems, while (long)0 + // is always the same. + fprintf(stderr, "x: %zu\n", (size_t)x); + free(x); + + return x != 0; +} + +// CHECK-mCRASH: malloc: +// CHECK-mCRASH: Sanitizer's allocator is terminating the process +// CHECK-cCRASH: calloc: +// CHECK-cCRASH: Sanitizer's allocator is terminating the process +// CHECK-coCRASH: calloc-overflow: +// CHECK-coCRASH: Sanitizer's allocator is terminating the process +// CHECK-rCRASH: realloc: +// CHECK-rCRASH: Sanitizer's allocator is terminating the process +// CHECK-mrCRASH: realloc-after-malloc: +// CHECK-mrCRASH: Sanitizer's allocator is terminating the process +// CHECK-nCRASH: new: +// CHECK-nCRASH: Sanitizer's allocator is terminating the process +// CHECK-nnCRASH: new-nothrow: +// CHECK-nnCRASH: Sanitizer's allocator is terminating the process + +// CHECK-mNULL: malloc: +// CHECK-mNULL: x: 0 +// CHECK-cNULL: calloc: +// CHECK-cNULL: x: 0 +// CHECK-coNULL: calloc-overflow: +// CHECK-coNULL: x: 0 +// CHECK-rNULL: realloc: +// CHECK-rNULL: x: 0 +// CHECK-mrNULL: realloc-after-malloc: +// CHECK-mrNULL: x: 0 +// CHECK-nnNULL: new-nothrow: +// CHECK-nnNULL: x: 0