[Sanitizers] Secondary allocator respects allocator_may_return_null=1.

[alekseyshl]

Summary

Context: https://github.com/google/sanitizers/issues/740.

Making secondary allocator to respect allocator_may_return_null=1 flag
and return nullptr when "out of memory" happens.

More changes in primary allocator and operator new will follow.

[kcc]

Code LGTM. What about a test?

[alekseyshl]

In D34243#781428, @kcc wrote:

Code LGTM. What about a test?

I have not figured out a reliable way to test mmap's out of memory condition, especially for 64 bits.

[cryptoad]

Do you want me to change that for the Scudo secondary as well, or can you bundle it with this?

[cryptoad]

In D34243#781514, @cryptoad wrote:

Do you want me to change that for the Scudo secondary as well, or can you bundle it with this?

Forget that, apparently we are already doing it!

[alekseyshl]

• Adding OOM test.

[eugenis]

test/asan/TestCases/Linux/allocator_oom_test.cc

45 ↗

(On Diff #102738)

This is fragile, but I don't really see a better way. Maybe add a while (x) loop around the allocation in case one is not enough?

Btw, why does allocating (1<<40) - (1ULL << 30) fail?

[alekseyshl]

test/asan/TestCases/Linux/allocator_oom_test.cc

45 ↗

(On Diff #102738)

Because of "ulimit -v 22024290304" (there's quite a lot less than 1T memory left after all internal ASan allocations).

[alekseyshl]

Are we ok with this change?

[kcc]

LGTM

[rnk]

Can you update the OOM message in the windows tests? http://lab.llvm.org:8011/builders/sanitizer-windows/builds/12993

[alekseyshl]

In D34243#782636, @rnk wrote:

Can you update the OOM message in the windows tests? http://lab.llvm.org:8011/builders/sanitizer-windows/builds/12993

D34292

[uweigand]

The new test case fails on SystemZ, making all our build bots red.

The problem seems to be that the size of the shadow areas is bigger on SystemZ, which means after the ulimit -v, already the ASAN initialization fails since it can no longer allocate the shadow area:

==63256==ERROR: AddressSanitizer failed to allocate 0x2000000001000 (562949953425408) bytes at address ffffffffff000 (errno: 12)
==63256==ReserveShadowMemoryRange failed while trying to map 0x2000000001000 bytes. Perhaps you're using ulimit -v

Program received signal SIGABRT, Aborted.
0x000003fffd83d4f8 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
54        return INLINE_SYSCALL (tgkill, 3, pid, selftid, sig);
(gdb) bt
#0  0x000003fffd83d4f8 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
#1  0x000003fffd83edee in __GI_abort () at abort.c:89
#2  0x0000000001104d8a in __sanitizer::Abort () at /home/uweigand/llvm/llvm-head/projects/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc:146
#3  0x00000000010e2918 in __asan::ReserveShadowMemoryRange (beg=4503599627366400, end=<optimized out>, name=0x1114c72 "low shadow")
    at /home/uweigand/llvm/llvm-head/projects/compiler-rt/lib/asan/asan_rtl.cc:99
#4  0x00000000010e5302 in __asan::InitializeShadowMemory () at /home/uweigand/llvm/llvm-head/projects/compiler-rt/lib/asan/asan_rtl.cc:472
#5  __asan::AsanInitInternal () at /home/uweigand/llvm/llvm-head/projects/compiler-rt/lib/asan/asan_rtl.cc:555
#6  0x000003fffdf91a44 in _dl_init (main_map=0x3fffdfa70d8, argc=<optimized out>, argv=0x3fffffff2c8, env=0x3fffffff2d8) at dl-init.c:105
#7  0x000003fffdf80fbc in _dl_start_user () from /lib/ld64.so.1

[alekseyshl]

In D34243#785631, @uweigand wrote:

The new test case fails on SystemZ, making all our build bots red.

The problem seems to be that the size of the shadow areas is bigger on SystemZ, which means after the ulimit -v, already the ASAN initialization fails since it can no longer allocate the shadow area:

==63256==ERROR: AddressSanitizer failed to allocate 0x2000000001000 (562949953425408) bytes at address ffffffffff000 (errno: 12)
==63256==ReserveShadowMemoryRange failed while trying to map 0x2000000001000 bytes. Perhaps you're using ulimit -v

Program received signal SIGABRT, Aborted.
0x000003fffd83d4f8 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
54        return INLINE_SYSCALL (tgkill, 3, pid, selftid, sig);
(gdb) bt
#0  0x000003fffd83d4f8 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54
#1  0x000003fffd83edee in __GI_abort () at abort.c:89
#2  0x0000000001104d8a in __sanitizer::Abort () at /home/uweigand/llvm/llvm-head/projects/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc:146
#3  0x00000000010e2918 in __asan::ReserveShadowMemoryRange (beg=4503599627366400, end=<optimized out>, name=0x1114c72 "low shadow")
    at /home/uweigand/llvm/llvm-head/projects/compiler-rt/lib/asan/asan_rtl.cc:99
#4  0x00000000010e5302 in __asan::InitializeShadowMemory () at /home/uweigand/llvm/llvm-head/projects/compiler-rt/lib/asan/asan_rtl.cc:472
#5  __asan::AsanInitInternal () at /home/uweigand/llvm/llvm-head/projects/compiler-rt/lib/asan/asan_rtl.cc:555
#6  0x000003fffdf91a44 in _dl_init (main_map=0x3fffdfa70d8, argc=<optimized out>, argv=0x3fffffff2c8, env=0x3fffffff2d8) at dl-init.c:105
#7  0x000003fffdf80fbc in _dl_start_user () from /lib/ld64.so.1

D34414 should resolve it and sorry for the trouble!

compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_secondary.h

public:
   }
 
   void *Allocate(AllocatorStats *stat, uptr size, uptr alignment) {
     CHECK(IsPowerOfTwo(alignment));
     uptr map_size = RoundUpMapSize(size);
     if (alignment > page_size_)
       map_size += alignment;
     // Overflow.
-    if (map_size < size) return ReturnNullOrDieOnBadRequest();
+    if (map_size < size)
+      return ReturnNullOrDieOnBadRequest();
     uptr map_beg = reinterpret_cast<uptr>(
-        MmapOrDie(map_size, "LargeMmapAllocator"));
+        MmapOrDieOnFatalError(map_size, "LargeMmapAllocator"));
+    if (!map_beg)
+      return ReturnNullOrDieOnOOM();
     CHECK(IsAligned(map_beg, page_size_));
     MapUnmapCallback().OnMap(map_beg, map_size);
     uptr map_end = map_beg + map_size;
     uptr res = map_beg + page_size_;
     if (res & (alignment - 1))  // Align.
       res += alignment - (res & (alignment - 1));
     CHECK(IsAligned(res, alignment));
     CHECK(IsAligned(res, page_size_));

compiler-rt/trunk/lib/sanitizer_common/sanitizer_common.h

void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
                           uptr *tls_addr, uptr *tls_size);
 
 // Memory management
 void *MmapOrDie(uptr size, const char *mem_type, bool raw_report = false);
 INLINE void *MmapOrDieQuietly(uptr size, const char *mem_type) {
   return MmapOrDie(size, mem_type, /*raw_report*/ true);
 }
 void UnmapOrDie(void *addr, uptr size);
+// Behaves just like MmapOrDie, but tolerates out of memory condition, in that
+// case returns nullptr.
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type);
 void *MmapFixedNoReserve(uptr fixed_addr, uptr size,
                          const char *name = nullptr);
 void *MmapNoReserveOrDie(uptr size, const char *mem_type);
 void *MmapFixedOrDie(uptr fixed_addr, uptr size);
 void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name = nullptr);
 void *MmapNoAccess(uptr size);
 // Map aligned chunk of address space; size and alignment are powers of two.
 void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type);

compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix.cc

 #if SANITIZER_POSIX
 
 #include "sanitizer_common.h"
 #include "sanitizer_libc.h"
 #include "sanitizer_posix.h"
 #include "sanitizer_procmaps.h"
 #include "sanitizer_stacktrace.h"
 
+#include <errno.h>
 #include <fcntl.h>
 #include <signal.h>
 #include <sys/mman.h>
 
 #if SANITIZER_LINUX
 #include <sys/utsname.h>
 #endif
 
void UnmapOrDie(void *addr, uptr size) {
   if (internal_iserror(res)) {
     Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n",
            SanitizerToolName, size, size, addr);
     CHECK("unable to unmap" && 0);
   }
   DecreaseTotalMmap(size);
 }
 
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
+  size = RoundUpTo(size, GetPageSizeCached());
+  uptr res = internal_mmap(nullptr, size,
+                           PROT_READ | PROT_WRITE,
+                           MAP_PRIVATE | MAP_ANON, -1, 0);
+  int reserrno;
+  if (internal_iserror(res, &reserrno)) {
+    if (reserrno == ENOMEM)
+      return nullptr;
+    ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno);
+  }
+  IncreaseTotalMmap(size);
+  return (void *)res;
+}
+
 // We want to map a chunk of address space aligned to 'alignment'.
 // We do it by maping a bit more and then unmaping redundant pieces.
 // We probably can do it with fewer syscalls in some OS-dependent way.
 void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) {
   CHECK(IsPowerOfTwo(size));
   CHECK(IsPowerOfTwo(alignment));
   uptr map_size = size + alignment;
   uptr map_res = (uptr)MmapOrDie(map_size, mem_type);

compiler-rt/trunk/lib/sanitizer_common/sanitizer_win.cc

if (VirtualFree(addr, size, MEM_DECOMMIT) == 0) {
       Report("ERROR: %s failed to "
              "deallocate 0x%zx (%zd) bytes at address %p (error code: %d)\n",
              SanitizerToolName, size, size, addr, GetLastError());
       CHECK("unable to unmap" && 0);
     }
   }
 }
 
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
+  void *rv = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+  if (rv == 0) {
+    error_t last_error = GetLastError();
+    if (last_error != ERROR_NOT_ENOUGH_MEMORY)
+      ReportMmapFailureAndDie(size, mem_type, "allocate", last_error);
+  }
+  return rv;
+}
+
 // We want to map a chunk of address space aligned to 'alignment'.
 void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) {
   CHECK(IsPowerOfTwo(size));
   CHECK(IsPowerOfTwo(alignment));
 
   // Windows will align our allocations to at least 64K.
   alignment = Max(alignment, GetMmapGranularity());
 

compiler-rt/trunk/test/asan/TestCases/Linux/allocator_oom_test.cc

+// Test the behavior of malloc/calloc/realloc when the allocation causes OOM
+// in the secondary allocator.
+// By default (allocator_may_return_null=0) the process should crash.
+// With allocator_may_return_null=1 the allocator should return 0.
+// Set the limit to 20.5T on 64 bits to account for ASan shadow memory,
+// allocator buffers etc. so that the test allocation of ~1T will trigger OOM.
+// Limit this test to Linux since we're relying on allocator internal
+// limits (shadow memory size, allocation limits etc.)
+
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: ulimit -v 22024290304
+// RUN: not %run %t malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefixes=CHECK-MALLOC,CHECK-CRASH
+// RUN: %env_asan_opts=allocator_may_return_null=0 not %run %t malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefixes=CHECK-MALLOC,CHECK-CRASH
+// RUN: %env_asan_opts=allocator_may_return_null=1     %run %t malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefixes=CHECK-MALLOC,CHECK-NULL
+// RUN: %env_asan_opts=allocator_may_return_null=0 not %run %t calloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefixes=CHECK-CALLOC,CHECK-CRASH
+// RUN: %env_asan_opts=allocator_may_return_null=1     %run %t calloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefixes=CHECK-CALLOC,CHECK-NULL
+// RUN: %env_asan_opts=allocator_may_return_null=0 not %run %t realloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefixes=CHECK-REALLOC,CHECK-CRASH
+// RUN: %env_asan_opts=allocator_may_return_null=1     %run %t realloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefixes=CHECK-REALLOC,CHECK-NULL
+// RUN: %env_asan_opts=allocator_may_return_null=0 not %run %t realloc-after-malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefixes=CHECK-MALLOC-REALLOC,CHECK-CRASH
+// RUN: %env_asan_opts=allocator_may_return_null=1     %run %t realloc-after-malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefixes=CHECK-MALLOC-REALLOC,CHECK-NULL
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <assert.h>
+
+int main(int argc, char **argv) {
+  assert(argc == 2);
+  const char *action = argv[1];
+  fprintf(stderr, "%s:\n", action);
+
+  // Allocate just a bit less than max allocation size enforced by ASan's
+  // allocator (currently 1T and 3G).
+  const size_t size =
+#if __LP64__
+      (1ULL << 40) - (1ULL << 30);
+#else
+      (3ULL << 30) - (1ULL << 20);
+#endif
+
+  void *x = 0;
+
+  if (!strcmp(action, "malloc")) {
+    x = malloc(size);
+  } else if (!strcmp(action, "calloc")) {
+    x = calloc(size / 4, 4);
+  } else if (!strcmp(action, "realloc")) {
+    x = realloc(0, size);
+  } else if (!strcmp(action, "realloc-after-malloc")) {
+    char *t = (char*)malloc(100);
+    *t = 42;
+    x = realloc(t, size);
+    assert(*t == 42);
+    free(t);
+  } else {
+    assert(0);
+  }
+
+  // The NULL pointer is printed differently on different systems, while (long)0
+  // is always the same.
+  fprintf(stderr, "x: %lx\n", (long)x);
+  free(x);
+
+  return x != 0;
+}
+
+// CHECK-MALLOC: malloc:
+// CHECK-CALLOC: calloc:
+// CHECK-REALLOC: realloc:
+// CHECK-MALLOC-REALLOC: realloc-after-malloc:
+
+// CHECK-CRASH: AddressSanitizer's allocator is terminating the process
+// CHECK-NULL: x: 0