[Sanitizers] ASan and LSan allocator set errno on failure.

[alekseyshl]

Summary

Set proper errno code on alloction failures and change some
implementations to satisfy their man-specified requirements:

LSan: valloc and memalign
ASan: pvalloc, memalign and posix_memalign

Changing both allocators in one patch since LSan depends on ASan allocator in some configurations.

lib/asan/asan_allocator.cc

 
 #include "asan_allocator.h"
 #include "asan_mapping.h"
 #include "asan_poisoning.h"
 #include "asan_report.h"
 #include "asan_stack.h"
 #include "asan_thread.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_list.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 #include "sanitizer_common/sanitizer_quarantine.h"
 #include "lsan/lsan_common.h"
 
 namespace __asan {

 void AsanThreadLocalMallocStorage::CommitBack() {
   instance.CommitBack(this);
 }
 
 void PrintInternalAllocatorStats() {
   instance.PrintStats();
 }
 
-void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack,
-                    AllocType alloc_type) {
-  return instance.Allocate(size, alignment, stack, alloc_type, true);
-}
-
 void asan_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) {
   instance.Deallocate(ptr, 0, stack, alloc_type);
 }
 
 void asan_sized_free(void *ptr, uptr size, BufferedStackTrace *stack,
                      AllocType alloc_type) {
   instance.Deallocate(ptr, size, stack, alloc_type);
 }
 
+inline void *check_ptr(void *ptr) {
+  if (UNLIKELY(!ptr))
+    errno = errno_ENOMEM;
+  return ptr;
+}
+
 void *asan_malloc(uptr size, BufferedStackTrace *stack) {
-  return instance.Allocate(size, 8, stack, FROM_MALLOC, true);
+  return check_ptr(instance.Allocate(size, 8, stack, FROM_MALLOC, true));
 }
 
 void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
-  return instance.Calloc(nmemb, size, stack);
+  return check_ptr(instance.Calloc(nmemb, size, stack));
 }
 
 void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) {
   if (!p)
-    return instance.Allocate(size, 8, stack, FROM_MALLOC, true);
+    return check_ptr(instance.Allocate(size, 8, stack, FROM_MALLOC, true));
   if (size == 0) {
     if (flags()->allocator_frees_and_returns_null_on_realloc_zero) {
       instance.Deallocate(p, 0, stack, FROM_MALLOC);
       return nullptr;
     }
     // Allocate a size of 1 if we shouldn't free() on Realloc to 0
     size = 1;
   }
-  return instance.Reallocate(p, size, stack);
+  return check_ptr(instance.Reallocate(p, size, stack));
 }
 
 void *asan_valloc(uptr size, BufferedStackTrace *stack) {
-  return instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC, true);
+  return check_ptr(
+      instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC, true));
 }
 
 void *asan_pvalloc(uptr size, BufferedStackTrace *stack) {
   uptr PageSize = GetPageSizeCached();
-  size = RoundUpTo(size, PageSize);
-  if (size == 0) {
-    // pvalloc(0) should allocate one page.
-    size = PageSize;
-  }
-  return instance.Allocate(size, PageSize, stack, FROM_MALLOC, true);
+  size = size ? RoundUpTo(size, PageSize) : PageSize;
+  return check_ptr(instance.Allocate(size, PageSize, stack, FROM_MALLOC, true));
+
+void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack,
+                    AllocType alloc_type) {
+  if (UNLIKELY(!IsPowerOfTwo(alignment))) {
+    errno = errno_EINVAL;
+    return AsanAllocator::FailureHandler::OnBadRequest();
+  }
+  return check_ptr(
+      instance.Allocate(size, alignment, stack, alloc_type, true));
 }
 
 int asan_posix_memalign(void **memptr, uptr alignment, uptr size,
                         BufferedStackTrace *stack) {
+  if (UNLIKELY(!IsPowerOfTwo(alignment) ||
+               (alignment % sizeof(void *)) != 0)) {  // NOLINT
+    AsanAllocator::FailureHandler::OnBadRequest();
+    return errno_EINVAL;
+  }
   void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC, true);
+  if (!ptr)
+    return errno_ENOMEM;
   CHECK(IsAligned((uptr)ptr, alignment));
   *memptr = ptr;
   return 0;
 }
 
 uptr asan_malloc_usable_size(const void *ptr, uptr pc, uptr bp) {
   if (!ptr) return 0;
   uptr usable_size = instance.AllocationSize(reinterpret_cast<uptr>(ptr));

lib/asan/tests/asan_test.cc

 //===-- asan_test.cc ------------------------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of AddressSanitizer, an address sanity checker.
 //
 //===----------------------------------------------------------------------===//
 #include "asan_test_utils.h"
 
+#include <errno.h>
+
 NOINLINE void *malloc_fff(size_t size) {
   void *res = malloc/**/(size); break_optimization(0); return res;}
 NOINLINE void *malloc_eee(size_t size) {
   void *res = malloc_fff(size); break_optimization(0); return res;}
 NOINLINE void *malloc_ddd(size_t size) {
   void *res = malloc_eee(size); break_optimization(0); return res;}
 NOINLINE void *malloc_ccc(size_t size) {
   void *res = malloc_ddd(size); break_optimization(0); return res;}
TEST(AddressSanitizer, VariousMallocsTest) {
   b[50] = 0;
   delete [] b;
 
   int *c = new int;
   *c = 0;
   delete c;
 
 #if SANITIZER_TEST_HAS_POSIX_MEMALIGN
-  int *pm;
-  int pm_res = posix_memalign((void**)&pm, kPageSize, kPageSize);
+  void *pm = 0;
+  // Valid allocation.
+  int pm_res = posix_memalign(&pm, kPageSize, kPageSize);
   EXPECT_EQ(0, pm_res);
+  EXPECT_NE(nullptr, pm);
   free(pm);
+
+  // Alignment is not a power of 2.
+  EXPECT_DEATH(posix_memalign(&pm, 3, kPageSize),
+               "allocator is terminating the process instead of returning 0");
+  // Alignment is a power of 2, but not a multiple of size(void *).
+  EXPECT_DEATH(posix_memalign(&pm, 2, kPageSize),
+               "allocator is terminating the process instead of returning 0");
 #endif  // SANITIZER_TEST_HAS_POSIX_MEMALIGN
 
 #if SANITIZER_TEST_HAS_MEMALIGN
   int *ma = (int*)memalign(kPageSize, kPageSize);
   EXPECT_EQ(0U, (uintptr_t)ma % kPageSize);
   ma[123] = 0;
   free(ma);
+
+  EXPECT_DEATH(memalign(3, kPageSize),
+               "allocator is terminating the process instead of returning 0");
 #endif  // SANITIZER_TEST_HAS_MEMALIGN
 }
 
 TEST(AddressSanitizer, CallocTest) {
   int *a = (int*)calloc(100, sizeof(int));
   EXPECT_EQ(0, a[10]);
   free(a);
 }

lib/lsan/lsan_allocator.cc

 // See lsan_allocator.h for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "lsan_allocator.h"
 
 #include "sanitizer_common/sanitizer_allocator.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 #include "sanitizer_common/sanitizer_stacktrace.h"
 #include "lsan_common.h"
 
 extern "C" void *memset(void *ptr, int value, uptr num);
 
 namespace __lsan {
void *Allocate(const StackTrace &stack, uptr size, uptr alignment,
   if (cleared && allocator.FromPrimary(p))
     memset(p, 0, size);
   RegisterAllocation(stack, p, size);
   if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(p, size);
   RunMallocHooks(p, size);
   return p;
 }
 
+static void *Calloc(uptr nmemb, uptr size, const StackTrace &stack) {
+  if (UNLIKELY(CheckForCallocOverflow(size, nmemb)))
+    return Allocator::FailureHandler::OnBadRequest();
+  size *= nmemb;
+  return Allocate(stack, size, 1, true);
+}
+
 void Deallocate(void *p) {
   if (&__sanitizer_free_hook) __sanitizer_free_hook(p);
   RunFreeHooks(p);
   RegisterDeallocation(p);
   allocator.Deallocate(GetAllocatorCache(), p);
 }
 
 void *Reallocate(const StackTrace &stack, void *p, uptr new_size,

 }
 
 uptr GetMallocUsableSize(const void *p) {
   ChunkMetadata *m = Metadata(p);
   if (!m) return 0;
   return m->requested_size;
 }
 
+inline void *check_ptr(void *ptr) {
+  if (UNLIKELY(!ptr))
+    errno = errno_ENOMEM;
+  return ptr;
+}
+
 void *lsan_memalign(uptr alignment, uptr size, const StackTrace &stack) {
-  return Allocate(stack, size, alignment, kAlwaysClearMemory);
+  if (UNLIKELY(!IsPowerOfTwo(alignment))) {
+    errno = errno_EINVAL;
+    return Allocator::FailureHandler::OnBadRequest();
+  }
+  return check_ptr(Allocate(stack, size, alignment, kAlwaysClearMemory));
 }
 
 void *lsan_malloc(uptr size, const StackTrace &stack) {
-  return Allocate(stack, size, 1, kAlwaysClearMemory);
+  return check_ptr(Allocate(stack, size, 1, kAlwaysClearMemory));
 }
 
 void lsan_free(void *p) {
   Deallocate(p);
 }
 
 void *lsan_realloc(void *p, uptr size, const StackTrace &stack) {
-  return Reallocate(stack, p, size, 1);
+  return check_ptr(Reallocate(stack, p, size, 1));
 }
 
 void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack) {
-  if (CheckForCallocOverflow(size, nmemb))
-    return Allocator::FailureHandler::OnBadRequest();
-  size *= nmemb;
-  return Allocate(stack, size, 1, true);
+  return check_ptr(Calloc(nmemb, size, stack));
 }
 
 void *lsan_valloc(uptr size, const StackTrace &stack) {
-  if (size == 0)
-    size = GetPageSizeCached();
-  return Allocate(stack, size, GetPageSizeCached(), kAlwaysClearMemory);
+  return check_ptr(
+      Allocate(stack, size, GetPageSizeCached(), kAlwaysClearMemory));
 }
 
 uptr lsan_mz_size(const void *p) {
   return GetMallocUsableSize(p);
 }
 
 ///// Interface to the common LSan module. /////
 

test/asan/TestCases/allocator_returns_null.cc

 // RUN: %env_asan_opts=allocator_may_return_null=1 not %run %t new 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=CHECK-nCRASH
 // RUN: %env_asan_opts=allocator_may_return_null=0 not %run %t new-nothrow 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=CHECK-nnCRASH
 // RUN: %env_asan_opts=allocator_may_return_null=1     %run %t new-nothrow 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=CHECK-nnNULL
 
 #include <assert.h>
-#include <string.h>
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <limits>
 #include <new>
 
 int main(int argc, char **argv) {
   // Disable stderr buffering. Needed on Windows.
   setvbuf(stderr, NULL, _IONBF, 0);
 
   assert(argc == 2);
#endif
   } else if (!strcmp(action, "new")) {
     x = operator new(kMaxAllowedMallocSizePlusOne);
   } else if (!strcmp(action, "new-nothrow")) {
     x = operator new(kMaxAllowedMallocSizePlusOne, std::nothrow);
   } else {
     assert(0);
   }
 
+  fprintf(stderr, "errno: %d\n", errno);
+
   // The NULL pointer is printed differently on different systems, while (long)0
   // is always the same.
   fprintf(stderr, "x: %lx\n", (long)x);
   free(x);
 
   return x != 0;
 }
 
 // CHECK-mCRASH: malloc:
 // CHECK-mCRASH: AddressSanitizer's allocator is terminating the process
 // CHECK-cCRASH: calloc:
 // CHECK-cCRASH: AddressSanitizer's allocator is terminating the process
 // CHECK-coCRASH: calloc-overflow:
 // CHECK-coCRASH: AddressSanitizer's allocator is terminating the process
 // CHECK-rCRASH: realloc:
 // CHECK-rCRASH: AddressSanitizer's allocator is terminating the process
 // CHECK-mrCRASH: realloc-after-malloc:
 // CHECK-mrCRASH: AddressSanitizer's allocator is terminating the process
 // CHECK-nCRASH: new:
 // CHECK-nCRASH: AddressSanitizer's allocator is terminating the process
 // CHECK-nnCRASH: new-nothrow:
 // CHECK-nnCRASH: AddressSanitizer's allocator is terminating the process
 
 // CHECK-mNULL: malloc:
+// CHECK-mNULL: errno: 12
 // CHECK-mNULL: x: 0
 // CHECK-cNULL: calloc:
+// CHECK-cNULL: errno: 12
 // CHECK-cNULL: x: 0
 // CHECK-coNULL: calloc-overflow:
+// CHECK-coNULL: errno: 12
 // CHECK-coNULL: x: 0
 // CHECK-rNULL: realloc:
+// CHECK-rNULL: errno: 12
 // CHECK-rNULL: x: 0
 // CHECK-mrNULL: realloc-after-malloc:
+// CHECK-mrNULL: errno: 12
 // CHECK-mrNULL: x: 0
 // CHECK-nnNULL: new-nothrow:
 // CHECK-nnNULL: x: 0

test/lsan/TestCases/allocator_returns_null.cc

 // RUN: %env_lsan_opts=allocator_may_return_null=1 not %run %t new 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=CHECK-nCRASH
 // RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t new-nothrow 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=CHECK-nnCRASH
 // RUN: %env_lsan_opts=allocator_may_return_null=1     %run %t new-nothrow 2>&1 \
 // RUN:   | FileCheck %s --check-prefix=CHECK-nnNULL
 
 #include <assert.h>
-#include <string.h>
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <limits>
 #include <new>
 
 int main(int argc, char **argv) {
   // Disable stderr buffering. Needed on Windows.
   setvbuf(stderr, NULL, _IONBF, 0);
 
   assert(argc == 2);
#endif
   } else if (!strcmp(action, "new")) {
     x = operator new(kMaxAllowedMallocSizePlusOne);
   } else if (!strcmp(action, "new-nothrow")) {
     x = operator new(kMaxAllowedMallocSizePlusOne, std::nothrow);
   } else {
     assert(0);
   }
 
+  fprintf(stderr, "errno: %d\n", errno);
+
   // The NULL pointer is printed differently on different systems, while (long)0
   // is always the same.
   fprintf(stderr, "x: %zu\n", (size_t)x);
   free(x);
 
   return x != 0;
 }
 
 // CHECK-mCRASH: malloc:
 // CHECK-mCRASH: Sanitizer's allocator is terminating the process
 // CHECK-cCRASH: calloc:
 // CHECK-cCRASH: Sanitizer's allocator is terminating the process
 // CHECK-coCRASH: calloc-overflow:
 // CHECK-coCRASH: Sanitizer's allocator is terminating the process
 // CHECK-rCRASH: realloc:
 // CHECK-rCRASH: Sanitizer's allocator is terminating the process
 // CHECK-mrCRASH: realloc-after-malloc:
 // CHECK-mrCRASH: Sanitizer's allocator is terminating the process
 // CHECK-nCRASH: new:
 // CHECK-nCRASH: Sanitizer's allocator is terminating the process
 // CHECK-nnCRASH: new-nothrow:
 // CHECK-nnCRASH: Sanitizer's allocator is terminating the process
 
 // CHECK-mNULL: malloc:
+// CHECK-mNULL: errno: 12
 // CHECK-mNULL: x: 0
 // CHECK-cNULL: calloc:
+// CHECK-cNULL: errno: 12
 // CHECK-cNULL: x: 0
 // CHECK-coNULL: calloc-overflow:
+// CHECK-coNULL: errno: 12
 // CHECK-coNULL: x: 0
 // CHECK-rNULL: realloc:
+// CHECK-rNULL: errno: 12
 // CHECK-rNULL: x: 0
 // CHECK-mrNULL: realloc-after-malloc:
+// CHECK-mrNULL: errno: 12
 // CHECK-mrNULL: x: 0
 // CHECK-nnNULL: new-nothrow:
 // CHECK-nnNULL: x: 0