Index: compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_secondary.h =================================================================== --- compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_secondary.h +++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_secondary.h @@ -36,9 +36,12 @@ if (alignment > page_size_) map_size += alignment; // Overflow. - if (map_size < size) return ReturnNullOrDieOnBadRequest(); + if (map_size < size) + return ReturnNullOrDieOnBadRequest(); uptr map_beg = reinterpret_cast( - MmapOrDie(map_size, "LargeMmapAllocator")); + MmapOrDieOnFatalError(map_size, "LargeMmapAllocator")); + if (!map_beg) + return ReturnNullOrDieOnOOM(); CHECK(IsAligned(map_beg, page_size_)); MapUnmapCallback().OnMap(map_beg, map_size); uptr map_end = map_beg + map_size; Index: compiler-rt/trunk/lib/sanitizer_common/sanitizer_common.h =================================================================== --- compiler-rt/trunk/lib/sanitizer_common/sanitizer_common.h +++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_common.h @@ -85,6 +85,9 @@ return MmapOrDie(size, mem_type, /*raw_report*/ true); } void UnmapOrDie(void *addr, uptr size); +// Behaves just like MmapOrDie, but tolerates out of memory condition, in that +// case returns nullptr. +void *MmapOrDieOnFatalError(uptr size, const char *mem_type); void *MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name = nullptr); void *MmapNoReserveOrDie(uptr size, const char *mem_type); Index: compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix.cc =================================================================== --- compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix.cc +++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix.cc @@ -22,6 +22,7 @@ #include "sanitizer_procmaps.h" #include "sanitizer_stacktrace.h" +#include #include #include #include @@ -145,6 +146,21 @@ DecreaseTotalMmap(size); } +void *MmapOrDieOnFatalError(uptr size, const char *mem_type) { + size = RoundUpTo(size, GetPageSizeCached()); + uptr res = internal_mmap(nullptr, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + int reserrno; + if (internal_iserror(res, &reserrno)) { + if (reserrno == ENOMEM) + return nullptr; + ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno); + } + IncreaseTotalMmap(size); + return (void *)res; +} + // We want to map a chunk of address space aligned to 'alignment'. // We do it by maping a bit more and then unmaping redundant pieces. // We probably can do it with fewer syscalls in some OS-dependent way. Index: compiler-rt/trunk/lib/sanitizer_common/sanitizer_win.cc =================================================================== --- compiler-rt/trunk/lib/sanitizer_common/sanitizer_win.cc +++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_win.cc @@ -131,6 +131,16 @@ } } +void *MmapOrDieOnFatalError(uptr size, const char *mem_type) { + void *rv = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + if (rv == 0) { + error_t last_error = GetLastError(); + if (last_error != ERROR_NOT_ENOUGH_MEMORY) + ReportMmapFailureAndDie(size, mem_type, "allocate", last_error); + } + return rv; +} + // We want to map a chunk of address space aligned to 'alignment'. void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) { CHECK(IsPowerOfTwo(size)); Index: compiler-rt/trunk/test/asan/TestCases/Linux/allocator_oom_test.cc =================================================================== --- compiler-rt/trunk/test/asan/TestCases/Linux/allocator_oom_test.cc +++ compiler-rt/trunk/test/asan/TestCases/Linux/allocator_oom_test.cc @@ -0,0 +1,82 @@ +// Test the behavior of malloc/calloc/realloc when the allocation causes OOM +// in the secondary allocator. +// By default (allocator_may_return_null=0) the process should crash. +// With allocator_may_return_null=1 the allocator should return 0. +// Set the limit to 20.5T on 64 bits to account for ASan shadow memory, +// allocator buffers etc. so that the test allocation of ~1T will trigger OOM. +// Limit this test to Linux since we're relying on allocator internal +// limits (shadow memory size, allocation limits etc.) + +// RUN: %clangxx_asan -O0 %s -o %t +// RUN: ulimit -v 22024290304 +// RUN: not %run %t malloc 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=CHECK-MALLOC,CHECK-CRASH +// RUN: %env_asan_opts=allocator_may_return_null=0 not %run %t malloc 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=CHECK-MALLOC,CHECK-CRASH +// RUN: %env_asan_opts=allocator_may_return_null=1 %run %t malloc 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=CHECK-MALLOC,CHECK-NULL +// RUN: %env_asan_opts=allocator_may_return_null=0 not %run %t calloc 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=CHECK-CALLOC,CHECK-CRASH +// RUN: %env_asan_opts=allocator_may_return_null=1 %run %t calloc 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=CHECK-CALLOC,CHECK-NULL +// RUN: %env_asan_opts=allocator_may_return_null=0 not %run %t realloc 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=CHECK-REALLOC,CHECK-CRASH +// RUN: %env_asan_opts=allocator_may_return_null=1 %run %t realloc 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=CHECK-REALLOC,CHECK-NULL +// RUN: %env_asan_opts=allocator_may_return_null=0 not %run %t realloc-after-malloc 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=CHECK-MALLOC-REALLOC,CHECK-CRASH +// RUN: %env_asan_opts=allocator_may_return_null=1 %run %t realloc-after-malloc 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=CHECK-MALLOC-REALLOC,CHECK-NULL + +#include +#include +#include +#include + +int main(int argc, char **argv) { + assert(argc == 2); + const char *action = argv[1]; + fprintf(stderr, "%s:\n", action); + + // Allocate just a bit less than max allocation size enforced by ASan's + // allocator (currently 1T and 3G). + const size_t size = +#if __LP64__ + (1ULL << 40) - (1ULL << 30); +#else + (3ULL << 30) - (1ULL << 20); +#endif + + void *x = 0; + + if (!strcmp(action, "malloc")) { + x = malloc(size); + } else if (!strcmp(action, "calloc")) { + x = calloc(size / 4, 4); + } else if (!strcmp(action, "realloc")) { + x = realloc(0, size); + } else if (!strcmp(action, "realloc-after-malloc")) { + char *t = (char*)malloc(100); + *t = 42; + x = realloc(t, size); + assert(*t == 42); + free(t); + } else { + assert(0); + } + + // The NULL pointer is printed differently on different systems, while (long)0 + // is always the same. + fprintf(stderr, "x: %lx\n", (long)x); + free(x); + + return x != 0; +} + +// CHECK-MALLOC: malloc: +// CHECK-CALLOC: calloc: +// CHECK-REALLOC: realloc: +// CHECK-MALLOC-REALLOC: realloc-after-malloc: + +// CHECK-CRASH: AddressSanitizer's allocator is terminating the process +// CHECK-NULL: x: 0