Add support for background thread on NetBSD in ASan

[krytarowski]

Summary

Change the point of calling MaybeStartBackgroudThread() from AsanInitInternal()
that is too early on NetBSD to a constructor (with aid of C++11 lambda construct).

Enable the code for background thread as is for NetBSD.

Rename test/sanitizer_common/TestCases/Linux/hard_rss_limit_mb_test.cc
to test/sanitizer_common/TestCases/hard_rss_limit_mb_test.cc and allow runs
on NetBSD. This tests passes correctly.

[krytarowski]

Ping?

[eugenis]

LGTM

This should also fix https://github.com/google/sanitizers/issues/981 on Android O. Would you mind removing the UNSUPPORTED line from the test?

[krytarowski]

In D55887#1342091, @eugenis wrote:

LGTM

This should also fix https://github.com/google/sanitizers/issues/981 on Android O. Would you mind removing the UNSUPPORTED line from the test?

I will do it!

lib/asan/asan_rtl.cc

void PrintAddressSpaceLayout() {
   Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET);
   CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
   if (kMidMemBeg)
     CHECK(kMidShadowBeg > kLowShadowEnd &&
           kMidMemBeg > kMidShadowEnd &&
           kHighShadowBeg > kMidMemEnd);
 }
 
+static bool UNUSED __local_asan_dyninit = [] {
+  MaybeStartBackgroudThread();
+  SetSoftRssLimitExceededCallback(AsanSoftRssLimitExceededCallback);
+
+  return false;
+}();
+
 static void AsanInitInternal() {
   if (LIKELY(asan_inited)) return;
   SanitizerToolName = "AddressSanitizer";
   CHECK(!asan_init_is_running && "ASan init calls itself!");
   asan_init_is_running = true;
 
   CacheBinaryName();
   CheckASLR();
static void AsanInitInternal() {
 
   AsanTSDInit(PlatformTSDDtor);
   InstallDeadlySignalHandlers(AsanOnDeadlySignal);
 
   AllocatorOptions allocator_options;
   allocator_options.SetFrom(flags(), common_flags());
   InitializeAllocator(allocator_options);
 
-  MaybeStartBackgroudThread();
-  SetSoftRssLimitExceededCallback(AsanSoftRssLimitExceededCallback);
-
   // On Linux AsanThread::ThreadStart() calls malloc() that's why asan_inited
   // should be set to 1 prior to initializing the threads.
   asan_inited = 1;
   asan_init_is_running = false;
 
   if (flags()->atexit)
     Atexit(asan_atexit);
 

lib/sanitizer_common/sanitizer_common_libcdep.cc

 namespace __sanitizer {
 
 static void (*SoftRssLimitExceededCallback)(bool exceeded);
 void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) {
   CHECK_EQ(SoftRssLimitExceededCallback, nullptr);
   SoftRssLimitExceededCallback = Callback;
 }
 
-#if SANITIZER_LINUX && !SANITIZER_GO
+#if (SANITIZER_LINUX || SANITIZER_NETBSD) && !SANITIZER_GO
 // Weak default implementation for when sanitizer_stackdepot is not linked in.
 SANITIZER_WEAK_ATTRIBUTE StackDepotStats *StackDepotGetStats() {
   return nullptr;
 }
 
 void BackgroundThread(void *arg) {
   const uptr hard_rss_limit_mb = common_flags()->hard_rss_limit_mb;
   const uptr soft_rss_limit_mb = common_flags()->soft_rss_limit_mb;
void WriteToSyslog(const char *msg) {
   // Note that this will add an extra newline at the end.
   // FIXME: buffer extra output. This would need a thread-local buffer, which
   // on Android requires plugging into the tools (ex. ASan's) Thread class.
   if (*p)
     WriteOneLineToSyslog(p);
 }
 
 void MaybeStartBackgroudThread() {
-#if SANITIZER_LINUX && \
+#if (SANITIZER_LINUX || SANITIZER_NETBSD) && \
     !SANITIZER_GO  // Need to implement/test on other platforms.
   // Start the background thread if one of the rss limits is given.
   if (!common_flags()->hard_rss_limit_mb &&
       !common_flags()->soft_rss_limit_mb &&
       !common_flags()->heap_profile) return;
   if (!&real_pthread_create) return;  // Can't spawn the thread anyway.
   internal_start_thread(BackgroundThread, nullptr);
 #endif

test/sanitizer_common/TestCases/Linux/hard_rss_limit_mb_test.cc

-// Check hard_rss_limit_mb. Not all sanitizers implement it yet.
-// RUN: %clangxx -O2 %s -o %t
-//
-// Run with limit should fail:
-// RUN: %env_tool_opts=hard_rss_limit_mb=100                           not %run %t 2>&1 | FileCheck %s
-// This run uses getrusage:
-// RUN: %env_tool_opts=hard_rss_limit_mb=100:can_use_proc_maps_statm=0 not %run %t 2>&1 | FileCheck %s
-//
-// Run w/o limit or with a large enough limit should pass:
-// RUN: %env_tool_opts=hard_rss_limit_mb=1000 %run %t
-// RUN: %run %t
-//
-// FIXME: make it work for other sanitizers.
-// XFAIL: lsan
-// XFAIL: tsan
-// XFAIL: msan
-// XFAIL: ubsan
-
-// https://github.com/google/sanitizers/issues/981
-// UNSUPPORTED: android-26
-
-#include <string.h>
-#include <stdio.h>
-#include <unistd.h>
-
-const int kNumAllocs = 200 * 1000;
-const int kAllocSize = 1000;
-volatile char *sink[kNumAllocs];
-
-int main(int argc, char **argv) {
-  for (int i = 0; i < kNumAllocs; i++) {
-    if ((i % 1000) == 0) {
-      // Don't write to stderr! Doing that triggers a kernel race condition
-      // between this thread and the rss-limit thread, and may lose part of the
-      // output. See https://lkml.org/lkml/2014/2/17/324.
-      printf("[%d]\n", i);
-    }
-    char *x = new char[kAllocSize];
-    memset(x, 0, kAllocSize);
-    sink[i] = x;
-  }
-  sleep(1);  // Make sure the background thread has time to kill the process.
-// CHECK: hard rss limit exhausted
-}

test/sanitizer_common/TestCases/hard_rss_limit_mb_test.cc

+// Check hard_rss_limit_mb. Not all sanitizers implement it yet.
+// RUN: %clangxx -O2 %s -o %t
+//
+// Run with limit should fail:
+// RUN: %env_tool_opts=hard_rss_limit_mb=100                           not %run %t 2>&1 | FileCheck %s
+// This run uses getrusage:
+// RUN: %env_tool_opts=hard_rss_limit_mb=100:can_use_proc_maps_statm=0 not %run %t 2>&1 | FileCheck %s
+//
+// Run w/o limit or with a large enough limit should pass:
+// RUN: %env_tool_opts=hard_rss_limit_mb=1000 %run %t
+// RUN: %run %t
+//
+// FIXME: make it work for other sanitizers.
+// XFAIL: lsan
+// XFAIL: tsan
+// XFAIL: msan
+// XFAIL: ubsan
+
+// https://github.com/google/sanitizers/issues/981
+// UNSUPPORTED: android-26
+
+// UNSUPPORTED: freebsd, solaris, darwin
+
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+
+const int kNumAllocs = 200 * 1000;
+const int kAllocSize = 1000;
+volatile char *sink[kNumAllocs];
+
+int main(int argc, char **argv) {
+  for (int i = 0; i < kNumAllocs; i++) {
+    if ((i % 1000) == 0) {
+      // Don't write to stderr! Doing that triggers a kernel race condition
+      // between this thread and the rss-limit thread, and may lose part of the
+      // output. See https://lkml.org/lkml/2014/2/17/324.
+      printf("[%d]\n", i);
+    }
+    char *x = new char[kAllocSize];
+    memset(x, 0, kAllocSize);
+    sink[i] = x;
+  }
+  sleep(1);  // Make sure the background thread has time to kill the process.
+// CHECK: hard rss limit exhausted
+}