[sanitizer] Support IsRssLimitExceeded in all sanitizers

[kstoimenov]

LTGM

compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp

86

I am little worried that someone might try to use this variable to check if the tread has been initialized. Instead maybe make this a void* and aways return nullptr?

[vitalybuka]

compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp

86

you can check void* as well

maybe something like:

class ThreadGlobalConstructor() {
  ThreadGlobalConstructor() {
     ....
  }
} thread_global_constructor UNUSED;

[kstoimenov]

compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp

86–102

Can you try this? It might be a little too convoluted as it is using the comma operator. The lambda is void in this case and the expression is always nullptr. Probably your class solution is better though...

[vitalybuka]

update

compiler-rt/lib/asan/asan_rtl.cpp

void PrintAddressSpaceLayout() {
   Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET);
   CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
   if (kMidMemBeg)
     CHECK(kMidShadowBeg > kLowShadowEnd &&
           kMidMemBeg > kMidShadowEnd &&
           kHighShadowBeg > kMidMemEnd);
 }
 
-static bool UNUSED __local_asan_dyninit = [] {
-  MaybeStartBackgroudThread();
-  return false;
-}();
-
 static void AsanInitInternal() {
   if (LIKELY(asan_inited)) return;
   SanitizerToolName = "AddressSanitizer";
   CHECK(!asan_init_is_running && "ASan init calls itself!");
   asan_init_is_running = true;
 
   CacheBinaryName();
 

compiler-rt/lib/dfsan/dfsan_allocator.cpp

if (size > max_malloc_size) {
     if (AllocatorMayReturnNull()) {
       Report("WARNING: DataflowSanitizer failed to allocate 0x%zx bytes\n",
              size);
       return nullptr;
     }
     BufferedStackTrace stack;
     ReportAllocationSizeTooBig(size, max_malloc_size, &stack);
   }
+  if (UNLIKELY(IsRssLimitExceeded())) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    BufferedStackTrace stack;
+    ReportRssLimitExceeded(&stack);
+  }
   DFsanThread *t = GetCurrentThread();
   void *allocated;
   if (t) {
     AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
     allocated = allocator.Allocate(cache, size, alignment);
   } else {
     SpinMutexLock l(&fallback_mutex);
     AllocatorCache *cache = &fallback_allocator_cache;

compiler-rt/lib/hwasan/hwasan_allocator.cpp

static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment,
   if (orig_size > kMaxAllowedMallocSize) {
     if (AllocatorMayReturnNull()) {
       Report("WARNING: HWAddressSanitizer failed to allocate 0x%zx bytes\n",
              orig_size);
       return nullptr;
     }
     ReportAllocationSizeTooBig(orig_size, kMaxAllowedMallocSize, stack);
   }
+  if (UNLIKELY(IsRssLimitExceeded())) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    ReportRssLimitExceeded(stack);
+  }
 
   alignment = Max(alignment, kShadowAlignment);
   uptr size = TaggedSize(orig_size);
   Thread *t = GetCurrentThread();
   void *allocated;
   if (t) {
     allocated = allocator.Allocate(t->allocator_cache(), size, alignment);
   } else {

compiler-rt/lib/lsan/lsan_allocator.cpp

 }
 
 void *Allocate(const StackTrace &stack, uptr size, uptr alignment,
                bool cleared) {
   if (size == 0)
     size = 1;
   if (size > max_malloc_size)
     return ReportAllocationSizeTooBig(size, stack);
+  if (UNLIKELY(IsRssLimitExceeded())) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    ReportRssLimitExceeded(&stack);
+  }
   void *p = allocator.Allocate(GetAllocatorCache(), size, alignment);
   if (UNLIKELY(!p)) {
     SetAllocatorOutOfMemory();
     if (AllocatorMayReturnNull())
       return nullptr;
     ReportOutOfMemory(size, &stack);
   }
   // Do not rely on the allocator to clear the memory (it's slow).

compiler-rt/lib/memprof/memprof_rtl.cpp

Printf("malloc_context_size=%zu\n",
          (uptr)common_flags()->malloc_context_size);
 
   Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE);
   Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY);
   Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET);
   CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
 }
 
-static bool UNUSED __local_memprof_dyninit = [] {
-  MaybeStartBackgroudThread();
-  return false;
-}();
-
 static void MemprofInitInternal() {
   if (LIKELY(memprof_inited))
     return;
   SanitizerToolName = "MemProfiler";
   CHECK(!memprof_init_is_running && "MemProf init calls itself!");
   memprof_init_is_running = true;
 
   CacheBinaryName();

compiler-rt/lib/msan/msan_allocator.cpp

static void *MsanAllocate(StackTrace *stack, uptr size, uptr alignment,
                           bool zeroise) {
   if (size > max_malloc_size) {
     if (AllocatorMayReturnNull()) {
       Report("WARNING: MemorySanitizer failed to allocate 0x%zx bytes\n", size);
       return nullptr;
     }
     ReportAllocationSizeTooBig(size, max_malloc_size, stack);
   }
+  if (UNLIKELY(IsRssLimitExceeded())) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    ReportRssLimitExceeded(stack);
+  }
   MsanThread *t = GetCurrentThread();
   void *allocated;
   if (t) {
     AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
     allocated = allocator.Allocate(cache, size, alignment);
   } else {
     SpinMutexLock l(&fallback_mutex);
     AllocatorCache *cache = &fallback_allocator_cache;

compiler-rt/lib/sanitizer_common/sanitizer_common.h

 #else
 // Unused on Windows.
   return 0;
 #endif
 }
 
 void *internal_start_thread(void *(*func)(void*), void *arg);
 void internal_join_thread(void *th);
-void MaybeStartBackgroudThread();
 
 // Make the compiler think that something is going on there.
 // Use this inside a loop that looks like memset/memcpy/etc to prevent the
 // compiler from recognising it and turning it into an actual call to
 // memset/memcpy/etc.
 static inline void SanitizerBreakOptimization(void *arg) {
 #if defined(_MSC_VER) && !defined(__clang__)
   _ReadWriteBarrier();

compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp

while (true) {
     if (heap_profile &&
         current_rss_mb > rss_during_last_reported_profile * 1.1) {
       Printf("\n\nHEAP PROFILE at RSS %zdMb\n", current_rss_mb);
       __sanitizer_print_memory_profile(90, 20);
       rss_during_last_reported_profile = current_rss_mb;
     }
   }
 }
-#endif
 
-void MaybeStartBackgroudThread() {
-#if (SANITIZER_LINUX || SANITIZER_NETBSD) && \
-    !SANITIZER_GO  // Need to implement/test on other platforms.
+static void MaybeStartBackgroudThread() {
+  // Need to implement/test on other platforms.
   // Start the background thread if one of the rss limits is given.
   if (!common_flags()->hard_rss_limit_mb &&
       !common_flags()->soft_rss_limit_mb &&
       !common_flags()->heap_profile) return;
   if (!&real_pthread_create) {
     VPrintf(1, "%s: real_pthread_create undefined\n", SanitizerToolName);
     return;  // Can't spawn the thread anyway.
   }
 
   static bool started = false;
   if (!started) {
     started = true;
     internal_start_thread(BackgroundThread, nullptr);
   }
-#endif
 }
 
+#  pragma clang diagnostic push
+// We avoid global-constructors to be sure that globals are ready when
+// sanitizers need them. This can happend before global constructors executed.
+// Here we don't mind if thread is started on later stages.
+#  pragma clang diagnostic ignored "-Wglobal-constructors"
+static struct BackgroudThreadStarted {
+  BackgroudThreadStarted() { MaybeStartBackgroudThread(); }
+} background_thread_strarter UNUSED;
+#  pragma clang diagnostic pop
+#endif
+
 void WriteToSyslog(const char *msg) {
   InternalScopedString msg_copy;
   msg_copy.append("%s", msg);
   const char *p = msg_copy.data();
 
   // Print one line at a time.
   // syslog, at least on Android, has an implicit message length limit.
   while (char* q = internal_strchr(p, '\n')) {

compiler-rt/lib/tsan/rtl/tsan_mman.cpp

if (sz >= kMaxAllowedMallocSize || align >= kMaxAllowedMallocSize ||
       sz > max_user_defined_malloc_size) {
     if (AllocatorMayReturnNull())
       return nullptr;
     uptr malloc_limit =
         Min(kMaxAllowedMallocSize, max_user_defined_malloc_size);
     GET_STACK_TRACE_FATAL(thr, pc);
     ReportAllocationSizeTooBig(sz, malloc_limit, &stack);
   }
+  if (UNLIKELY(IsRssLimitExceeded())) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    GET_STACK_TRACE_FATAL(thr, pc);
+    ReportRssLimitExceeded(&stack);
+  }
   void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align);
   if (UNLIKELY(!p)) {
     SetAllocatorOutOfMemory();
     if (AllocatorMayReturnNull())
       return nullptr;
     GET_STACK_TRACE_FATAL(thr, pc);
     ReportOutOfMemory(sz, &stack);
   }

compiler-rt/test/sanitizer_common/TestCases/Linux/soft_rss_limit_mb_test.cpp

 // Check soft_rss_limit_mb. Not all sanitizers implement it yet.
 // RUN: %clangxx -O2 %s -o %t
 //
 // Run with limit should fail:
 // RUN: %env_tool_opts=soft_rss_limit_mb=220:quarantine_size=1:allocator_may_return_null=1     %run %t 2>&1 | FileCheck %s -check-prefix=CHECK_MAY_RETURN_1
-// RUN: %env_tool_opts=soft_rss_limit_mb=220:quarantine_size=1:allocator_may_return_null=0 not %run %t 2>&1 | FileCheck %s -check-prefix=CHECK_MAY_RETURN_0
+// RUN: %env_tool_opts=soft_rss_limit_mb=220:quarantine_size=1:allocator_may_return_null=0 not %run %t 2>&1 | FileCheck %s -check-prefix=CHECK_MAY_RETURN_0 --implicit-check-not="returned null"
 
 // This run uses getrusage. We can only test getrusage when allocator_may_return_null=0
 // because getrusage gives us max-rss, not current-rss.
-// RUN: %env_tool_opts=soft_rss_limit_mb=220:quarantine_size=1:allocator_may_return_null=0:can_use_proc_maps_statm=0 not %run %t 2>&1 | FileCheck %s -check-prefix=CHECK_MAY_RETURN_0
+// RUN: %env_tool_opts=soft_rss_limit_mb=220:quarantine_size=1:allocator_may_return_null=0:can_use_proc_maps_statm=0 not %run %t 2>&1 | FileCheck %s -check-prefix=CHECK_MAY_RETURN_0 --implicit-check-not="returned null"
 // REQUIRES: stable-runtime
 
-// FIXME: make it work for other sanitizers.
-// XFAIL: lsan
-// XFAIL: tsan
-// XFAIL: msan
+// Ubsan does not intercept pthread_create.
 // XFAIL: ubsan
 
 // https://github.com/google/sanitizers/issues/981
 // UNSUPPORTED: android-26
 
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 
 static const int kMaxNumAllocs = 1 << 9;
 static const int kAllocSize = 1 << 20;  // Large enough to go via mmap.
 
 static char *allocs[kMaxNumAllocs];
 
 int main() {
-  int num_allocs = kMaxNumAllocs / 4;
-  for (int i = 0; i < 3; i++, num_allocs *= 2) {
+  int num_allocs = kMaxNumAllocs / 16;
+  for (int i = 0; num_allocs <= kMaxNumAllocs; i++, num_allocs *= 2) {
     fprintf(stderr, "[%d] allocating %d times\n", i, num_allocs);
     int zero_results = 0;
     for (int j = 0; j < num_allocs; j++) {
       if ((j % (num_allocs / 8)) == 0) {
         usleep(100000);
         fprintf(stderr, "  [%d]\n", j);
       }
       allocs[j] = (char*)malloc(kAllocSize);
if (zero_results != num_allocs)
       fprintf(stderr, "Some of the malloc calls returned non-null: %d\n",
               num_allocs - zero_results);
     for (int j = 0; j < num_allocs; j++) {
       free(allocs[j]);
     }
   }
 }
 
-// CHECK_MAY_RETURN_1: allocating 128 times
-// CHECK_MAY_RETURN_1: Some of the malloc calls returned non-null: 128
+// CHECK_MAY_RETURN_1: allocating 32 times
+// CHECK_MAY_RETURN_1: Some of the malloc calls returned non-null:
 // CHECK_MAY_RETURN_1: allocating 256 times
 // CHECK_MAY_RETURN_1: Some of the malloc calls returned null:
 // CHECK_MAY_RETURN_1: Some of the malloc calls returned non-null:
 // CHECK_MAY_RETURN_1: allocating 512 times
 // CHECK_MAY_RETURN_1: Some of the malloc calls returned null:
 // CHECK_MAY_RETURN_1: Some of the malloc calls returned non-null:
 
-// CHECK_MAY_RETURN_0: allocating 128 times
-// CHECK_MAY_RETURN_0: Some of the malloc calls returned non-null: 128
-// CHECK_MAY_RETURN_0: allocating 256 times
+// CHECK_MAY_RETURN_0: allocating 32 times
+// CHECK_MAY_RETURN_0: Some of the malloc calls returned non-null:
 // CHECK_MAY_RETURN_0: {{SUMMARY: .*Sanitizer: rss-limit-exceeded}}

compiler-rt/test/sanitizer_common/TestCases/hard_rss_limit_mb_test.cpp

 // Check hard_rss_limit_mb. Not all sanitizers implement it yet.
 // RUN: %clangxx -O2 %s -o %t
 //
 // Run with limit should fail:
 // RUN: %env_tool_opts=hard_rss_limit_mb=100                           not %run %t 2>&1 | FileCheck %s
 // This run uses getrusage:
 // RUN: %env_tool_opts=hard_rss_limit_mb=100:can_use_proc_maps_statm=0 not %run %t 2>&1 | FileCheck %s
 //
 // Run w/o limit or with a large enough limit should pass:
-// RUN: %env_tool_opts=hard_rss_limit_mb=1000 %run %t
+// RUN: %env_tool_opts=hard_rss_limit_mb=4000 %run %t
 // RUN: %run %t
 //
-// FIXME: make it work for other sanitizers.
-// XFAIL: lsan
-// XFAIL: tsan
-// XFAIL: msan
+// Ubsan does not intercept pthread_create.
 // XFAIL: ubsan
-
 // UNSUPPORTED: freebsd, solaris, darwin
 
 #include <string.h>
 #include <stdio.h>
 #include <unistd.h>
 
 const int kNumAllocs = 200 * 1000;
 const int kAllocSize = 1000;