[LSan] New experimental flag for background leak checking before exit.

[m.kashkarov]

Summary

New sanitizer option - leak_check_interval_s:
spawns a background thread at startup which periodically do recoverable leak check with given interval.
Report file names (if log_path option != stderr|stdout) are appended with prefix "in-progress" and for the DoLeakCheck() call at exit - are restores back.

This can be used for leak sanitization long-running processes that never exits.

[m.kashkarov]

Add unimplemented in lsan mac version, update standalone lsan.

[m.ostapenko]

Adding maintainers + some comments.

lib/lsan/lsan_common.cc

767

Why do you need all this stuff? Can't you use separate file descriptor from main report_fd? Because changing report_fd on the fly looks like a bad idea: what will happen if some another thread is reporting concurrently (corrupted reports, I guess)?

lib/lsan/lsan_common.h

155

You don't need to explicitly initialize global variables with 0.

lib/lsan/lsan_interceptors.cc

34

Please don't. Why do you need this at all?

414

Likewise, why do you need this?

[m.kashkarov]

lib/lsan/lsan_common.cc

767

If it's ok to create another file only for leak detected in the background thread - i can change the current logging behaviour.

lib/lsan/lsan_common.h

155

There is build error error: uninitialized const ‘__lsan::kLeaksIntervalNotSet’ [-fpermissive] when not set to 0

lib/lsan/lsan_interceptors.cc

34

Oh, didn't notice that it was already included here, tested on gcc without #include "sanitizer_common/sanitizer_posix.h".

414

This is for the real_pthread_create used in internal_start_thread.

[m.kashkarov]

Fixed typo error, update.

[alekseyshl]

lib/asan/asan_rtl.cc

463

FYI, this won't work on Android (see D27003, for example). Same for LSan.

[alekseyshl]

Can it be implemented with timer_create() instead of the thread?

[m.kashkarov]

Back to this feature, draft with signals impl.

[m.kashkarov]

Updated.

[m.kashkarov]

^ping.

[m.kashkarov]

^^ping.

[kcc]

I'd prefer to not have this functionality in lsan -- it already has __lsan_do_recoverable_leak_check which should be sufficient for a user to implement this functionality on their side.
(If not, that may need to be fixed).

lib/asan/asan_rtl.cc

static void AsanInitInternal() {
   // Create main thread.
   AsanThread *main_thread = CreateMainThread();
   CHECK_EQ(0, main_thread->tid());
   force_interface_symbols();  // no-op.
   SanitizerInitializeUnwinder();
 
   if (CAN_SANITIZE_LEAKS) {
     __lsan::InitCommonLsan();
-    if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) {
+    if (common_flags()->detect_leaks) {
+      __lsan::MaybeStartBackgroundLeakCheckingThread();
+      if (common_flags()->leak_check_at_exit) {
         if (flags()->halt_on_error)
           Atexit(__lsan::DoLeakCheck);
         else
           Atexit(__lsan::DoRecoverableLeakCheckVoid);
       }
     }
+  }
 
 #if CAN_SANITIZE_UB
   __ubsan::InitAsPlugin();
 #endif
 
   InitializeSuppressions();
 
   if (CAN_SANITIZE_LEAKS) {

lib/lsan/lsan.cc

extern "C" void __lsan_init() {
   InitializeInterceptors();
   InitializeThreadRegistry();
   InstallDeadlySignalHandlers(LsanOnDeadlySignal);
   u32 tid = ThreadCreate(0, 0, true);
   CHECK_EQ(tid, 0);
   ThreadStart(tid, GetTid());
   SetCurrentThread(tid);
 
-  if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit)
-    Atexit(DoLeakCheck);
+  if (common_flags()->detect_leaks) {
+    MaybeStartBackgroundLeakCheckingThread();
+    if (common_flags()->leak_check_at_exit) Atexit(DoLeakCheck);
+  }
 
   InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
 
   lsan_inited = true;
   lsan_init_is_running = false;
 }
 
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 void __sanitizer_print_stack_trace() {
   GET_STACK_TRACE_FATAL;
   stack.Print();
 }

lib/lsan/lsan_common.h

 
 // Functions called from the parent tool.
 const char *MaybeCallLsanDefaultOptions();
 void InitCommonLsan();
 void DoLeakCheck();
 void DoRecoverableLeakCheckVoid();
 void DisableCounterUnderflow();
 bool DisabledInThisThread();
+void MaybeRestoreLogPath();
+void MaybeAppendToLogPath(const char *suffix);
+
+static const uptr kLeaksIntervalNotSet = 0;
+void MaybeStartBackgroundLeakCheckingThread();
 
 // Used to implement __lsan::ScopedDisabler.
 void DisableInThisThread();
 void EnableInThisThread();
 // Can be used to ignore memory allocated by an intercepted
 // function.
 struct ScopedInterceptorDisabler {
   ScopedInterceptorDisabler() { DisableInThisThread(); }

lib/lsan/lsan_common.cc

Report(
         "HINT: For debugging, try setting environment variable "
         "LSAN_OPTIONS=verbosity=1:log_threads=1\n");
     Report(
         "HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc)\n");
     Die();
   }
   param.leak_report.ApplySuppressions();
   uptr unsuppressed_count = param.leak_report.UnsuppressedLeakCount();
+  ScopedErrorReportLock l;
   if (unsuppressed_count > 0) {
     Decorator d;
     Printf("\n"
            "================================================================="
            "\n");
     Printf("%s", d.Error());
     Report("ERROR: LeakSanitizer: detected memory leaks\n");
     Printf("%s", d.Default());

 static bool has_reported_leaks = false;
 bool HasReportedLeaks() { return has_reported_leaks; }
 
 void DoLeakCheck() {
   BlockingMutexLock l(&global_mutex);
   static bool already_done;
   if (already_done) return;
   already_done = true;
+  MaybeRestoreLogPath();
   has_reported_leaks = CheckForLeaks();
   if (has_reported_leaks) HandleLeaks();
 }
 
 static int DoRecoverableLeakCheck() {
   BlockingMutexLock l(&global_mutex);
   bool have_leaks = CheckForLeaks();
   return have_leaks ? 1 : 0;

 
 uptr LeakReport::UnsuppressedLeakCount() {
   uptr result = 0;
   for (uptr i = 0; i < leaks_.size(); i++)
     if (!leaks_[i].is_suppressed) result++;
   return result;
 }
 
+void MaybeRestoreLogPath() {
+  if (common_flags()->leak_check_interval_s == kLeaksIntervalNotSet) return;
+  __sanitizer_set_report_path(common_flags()->log_path);
+}
+
+void MaybeAppendToLogPath(const char *suffix) {
+  if (common_flags()->leak_check_interval_s == kLeaksIntervalNotSet) return;
+  const char *log_path = common_flags()->log_path;
+  if (internal_strcmp(log_path, "stderr") == 0 ||
+      internal_strcmp(log_path, "stdout") == 0 || !log_path)
+    return;
+  static char new_log_path[kMaxPathLength] = {'\0'};
+  if (UNLIKELY(new_log_path[0] == '\0')) {
+    internal_snprintf(new_log_path, kMaxPathLength, "%s.%s", log_path, suffix);
+    __sanitizer_set_report_path(new_log_path);
+  }
+}
+
 } // namespace __lsan
 #else // CAN_SANITIZE_LEAKS
 namespace __lsan {
 void InitCommonLsan() { }
 void DoLeakCheck() { }
 void DoRecoverableLeakCheckVoid() { }
 void DisableInThisThread() { }
 void EnableInThisThread() { }

lib/lsan/lsan_common_linux.cc

 
 #include "sanitizer_common/sanitizer_platform.h"
 #include "lsan_common.h"
 
 #if CAN_SANITIZE_LEAKS && SANITIZER_LINUX
 #include <link.h>
 
 #include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_file.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_getauxval.h"
 #include "sanitizer_common/sanitizer_linux.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 
 namespace __lsan {
 
 static const char kLinkerName[] = "ld";

 // while holding the libdl lock in the parent thread, we can safely reenter it
 // in the tracer. The solution is to run stoptheworld from a dl_iterate_phdr()
 // callback in the parent thread.
 void DoStopTheWorld(StopTheWorldCallback callback, void *argument) {
   DoStopTheWorldParam param = {callback, argument};
   dl_iterate_phdr(DoStopTheWorldCallback, &param);
 }
 
+void BackgroundLeakCheckingThread(void *arg) {
+  ScopedInterceptorDisabler disabler;
+  uptr interval_s = common_flags()->leak_check_interval_s;
+  MaybeAppendToLogPath("in-progress");
+  while (true) {
+    SleepForSeconds(interval_s);
+    // Do not save previous reports
+    TruncateReportFile();
+    DoRecoverableLeakCheckVoid();
+  }
+  MaybeRestoreLogPath();
+}
+
+void MaybeStartBackgroundLeakCheckingThread() {
+  if (common_flags()->leak_check_interval_s == kLeaksIntervalNotSet) return;
+  internal_start_thread(BackgroundLeakCheckingThread, nullptr);
+}
+
 } // namespace __lsan
 
 #endif // CAN_SANITIZE_LEAKS && SANITIZER_LINUX

lib/lsan/lsan_common_mac.cc

 // if required at that point. Calling Die() here is undefined behavior and
 // causes rare race conditions.
 void HandleLeaks() {}
 
 void DoStopTheWorld(StopTheWorldCallback callback, void *argument) {
   StopTheWorld(callback, argument);
 }
 
+void MaybeStartBackgroundLeakCheckingThread() {
+  UNIMPLEMENTED();
+}
+
 } // namespace __lsan
 
 #endif // CAN_SANITIZE_LEAKS && SANITIZER_MAC

lib/lsan/lsan_interceptors.cc

 #include "sanitizer_common/sanitizer_tls_get_addr.h"
 #include "lsan.h"
 #include "lsan_allocator.h"
 #include "lsan_common.h"
 #include "lsan_thread.h"
 
 #include <stddef.h>
 
 using namespace __lsan;
 
 extern "C" {
 int pthread_attr_init(void *attr);
 int pthread_attr_destroy(void *attr);
 int pthread_attr_getdetachstate(void *attr, int *v);
 int pthread_key_create(unsigned *key, void (*destructor)(void* v));
 int pthread_setspecific(unsigned key, const void *v);
 }
INTERCEPTOR(int, pthread_join, void *th, void **ret) {
   ENSURE_LSAN_INITED;
   int tid = ThreadTid((uptr)th);
   int res = REAL(pthread_join)(th, ret);
   if (res == 0)
     ThreadJoin(tid);
   return res;
 }
 
+DEFINE_REAL_PTHREAD_FUNCTIONS
+
 INTERCEPTOR(void, _exit, int status) {
   if (status == 0 && HasReportedLeaks()) status = common_flags()->exitcode;
   REAL(_exit)(status);
 }
 
 #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
 #include "sanitizer_common/sanitizer_signal_interceptors.inc"
 

lib/sanitizer_common/sanitizer_common.h

 typedef void (*LowLevelAllocateCallback)(uptr ptr, uptr size);
 // Allows to register tool-specific callbacks for LowLevelAllocator.
 // Passing NULL removes the callback.
 void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback);
 
 // IO
 void CatastrophicErrorWrite(const char *buffer, uptr length);
 void RawWrite(const char *buffer);
+void TruncateReportFile();
 bool ColorizeReports();
 void RemoveANSIEscapeSequencesFromString(char *buffer);
 void Printf(const char *format, ...);
 void Report(const char *format, ...);
 void SetPrintfAndReportCallback(void (*callback)(const char *));
 #define VReport(level, ...)                                              \
   do {                                                                   \
     if ((uptr)Verbosity() >= (level)) Report(__VA_ARGS__); \

lib/sanitizer_common/sanitizer_file.h

 #include "sanitizer_mutex.h"
 
 namespace __sanitizer {
 
 struct ReportFile {
   void Write(const char *buffer, uptr length);
   bool SupportsColors();
   void SetReportPath(const char *path);
+  void Truncate();
 
   // Don't use fields directly. They are only declared public to allow
   // aggregate initialization.
 
   // Protects fields below.
   StaticSpinMutex *mu;
   // Opened file descriptor. Defaults to stderr. It may be equal to
   // kInvalidFd, in which case new file will be opened when necessary.

  private:
   void ReopenIfNecessary();
 };
 extern ReportFile report_file;
 
 enum FileAccessMode {
   RdOnly,
   WrOnly,
-  RdWr
+  RdWr,
+  WrTrunc
 };
 
 // Returns kInvalidFd on error.
 fd_t OpenFile(const char *filename, FileAccessMode mode,
               error_t *errno_p = nullptr);
 void CloseFile(fd_t);
 
 // Return true on success, false on error.

lib/sanitizer_common/sanitizer_file.cc

 
 StaticSpinMutex report_file_mu;
 ReportFile report_file = {&report_file_mu, kStderrFd, "", "", 0};
 
 void RawWrite(const char *buffer) {
   report_file.Write(buffer, internal_strlen(buffer));
 }
 
+void TruncateReportFile() {
+  report_file.Truncate();
+}
+
 void ReportFile::ReopenIfNecessary() {
   mu->CheckLocked();
   if (fd == kStdoutFd || fd == kStderrFd) return;
 
   uptr pid = internal_getpid();
   // If in tracer, use the parent's file.
   if (pid == stoptheworld_tracer_pid)
     pid = stoptheworld_tracer_ppid;
if (internal_strcmp(path, "stdout") == 0) {
     fd = kStdoutFd;
   } else if (internal_strcmp(path, "stderr") == 0) {
     fd = kStderrFd;
   } else {
     internal_snprintf(path_prefix, kMaxPathLength, "%s", path);
   }
 }
 
+void ReportFile::Truncate() {
+  if (fd == kStdoutFd || fd == kStderrFd) return;
+  if (!full_path) return;
+
+  SpinMutexLock l(mu);
+  if (fd != kInvalidFd) CloseFile(fd);
+
+  fd = OpenFile(full_path, WrTrunc);
+}
+
 bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
                       uptr *read_len, uptr max_len, error_t *errno_p) {
   uptr PageSize = GetPageSizeCached();
   uptr kMinFileLen = PageSize;
   *buff = nullptr;
   *buff_size = 0;
   *read_len = 0;
   // The files we usually open are not seekable, so try different buffer sizes.

lib/sanitizer_common/sanitizer_flags.inc

COMMON_FLAG(bool, strip_env, 1,
             "Whether to remove the sanitizer from DYLD_INSERT_LIBRARIES to "
             "avoid passing it to children. Default is true.")
 COMMON_FLAG(bool, detect_leaks, !SANITIZER_MAC, "Enable memory leak detection.")
 COMMON_FLAG(
     bool, leak_check_at_exit, true,
     "Invoke leak checking in an atexit handler. Has no effect if "
     "detect_leaks=false, or if __lsan_do_leak_check() is called before the "
     "handler has a chance to run.")
+COMMON_FLAG(uptr, leak_check_interval_s, 0,
+            "Experimental. If set, creates background thread that performs "
+            "recoverable leak checking every leak_check_interval_s seconds. "
+            "Disabled if zero or detect_leaks=false.\n")
 COMMON_FLAG(bool, allocator_may_return_null, false,
             "If false, the allocator will crash instead of returning 0 on "
             "out-of-memory.")
 COMMON_FLAG(bool, print_summary, true,
             "If false, disable printing error summaries in addition to error "
             "reports.")
 COMMON_FLAG(int, print_module_map, 0,
             "OS X only (0 - don't print, 1 - print only once before process "

lib/sanitizer_common/sanitizer_posix.cc

 }
 
 fd_t OpenFile(const char *filename, FileAccessMode mode, error_t *errno_p) {
   int flags;
   switch (mode) {
     case RdOnly: flags = O_RDONLY; break;
     case WrOnly: flags = O_WRONLY | O_CREAT; break;
     case RdWr: flags = O_RDWR | O_CREAT; break;
+    case WrTrunc: flags = O_WRONLY | O_CREAT | O_TRUNC; break;
   }
   fd_t res = internal_open(filename, flags, 0660);
   if (internal_iserror(res, errno_p))
     return kInvalidFd;
   return res;
 }
 
 void CloseFile(fd_t fd) {

test/lsan/TestCases/Linux/leak_check_interval.cc

+// Test for LSan background leak checking during application runtime every 1 second.
+// RUN: LSAN_BASE="use_stacks=0:use_registers=0"
+// RUN: %clangxx_lsan %s -o %t
+// RUN: %env_lsan_opts=$LSAN_BASE":leak_check_interval_s=1:leak_check_at_exit=1" not %run %t 2>&1 | FileCheck %s
+// RUN: %env_lsan_opts=$LSAN_BASE":leak_check_interval_s=1:leak_check_at_exit=0" %run %t 2>&1 | FileCheck %s
+// RUN: %env_lsan_opts=$LSAN_BASE":leak_check_interval_s=0:leak_check_at_exit=1" not %run %t 2>&1 | FileCheck %s
+// RUN: %env_lsan_opts=$LSAN_BASE":leak_check_interval_s=0:leak_check_at_exit=0" %run %t 2>&1
+
+#include <stdlib.h>
+#include <unistd.h>
+
+int f() {
+  volatile int *a = (int *)malloc(20);
+  a[0] = 1;
+  return a[0];
+}
+
+int main() {
+  f();
+  sleep(2);
+  return 0;
+}
+
+// CHECK: LeakSanitizer: detected memory leaks