Move MallocFreeTracer to FuzzerMonitor

[aarongreen]

Summary

This change separates MallocFreeTracer from Fuzzer to facilitate it also being included in remote processes being fuzzed.

This is change 10 of (at least) 18 for cross-process fuzzing support.

[aarongreen]

Rebase

[aarongreen]

Multiprocess fuzzing will not be supported by the libFuzzer maintainers. Fuchsia has implemented a new approach with their Component Fuzzing Framework (RFC-117).

compiler-rt/lib/fuzzer/CMakeLists.txt

 set(LIBFUZZER_SOURCES
   FuzzerCrossOver.cpp
   FuzzerDataFlowTrace.cpp
   FuzzerDriver.cpp
   FuzzerDSORelative.cpp
   FuzzerExtFunctionsDlsym.cpp
   FuzzerExtFunctionsWeak.cpp
   FuzzerExtFunctionsWindows.cpp
   FuzzerExtraCounters.cpp
   FuzzerFork.cpp
   FuzzerIO.cpp
   FuzzerIOPosix.cpp
   FuzzerIOWindows.cpp
   FuzzerLoop.cpp
   FuzzerMerge.cpp
+  FuzzerMonitor.cpp
   FuzzerMutate.cpp
   FuzzerSHA1.cpp
   FuzzerTracePC.cpp
   FuzzerUtil.cpp
   FuzzerUtilDarwin.cpp
   FuzzerUtilFuchsia.cpp
   FuzzerUtilLinux.cpp
   FuzzerUtilPosix.cpp

compiler-rt/lib/fuzzer/FuzzerLoop.cpp

 
 thread_local bool Fuzzer::IsMyThread;
 
 bool RunningUserCallback = false;
 
 // Only one Fuzzer per process.
 static Fuzzer *F;
 
-// Leak detection is expensive, so we first check if there were more mallocs
-// than frees (using the sanitizer malloc hooks) and only then try to call lsan.
-struct MallocFreeTracer {
-  void Start(int TraceLevel) {
-    this->TraceLevel = TraceLevel;
-    if (TraceLevel)
-      Printf("MallocFreeTracer: START\n");
-    Mallocs = 0;
-    Frees = 0;
-  }
-  // Returns true if there were more mallocs than frees.
-  bool Stop() {
-    if (TraceLevel)
-      Printf("MallocFreeTracer: STOP %zd %zd (%s)\n", Mallocs.load(),
-             Frees.load(), Mallocs == Frees ? "same" : "DIFFERENT");
-    bool Result = Mallocs > Frees;
-    Mallocs = 0;
-    Frees = 0;
-    TraceLevel = 0;
-    return Result;
-  }
-  std::atomic<size_t> Mallocs;
-  std::atomic<size_t> Frees;
-  int TraceLevel = 0;
-
-  std::recursive_mutex TraceMutex;
-  bool TraceDisabled = false;
-};
-
-static MallocFreeTracer AllocTracer;
-
-// Locks printing and avoids nested hooks triggered from mallocs/frees in
-// sanitizer.
-class TraceLock {
-public:
-  TraceLock() : Lock(AllocTracer.TraceMutex) {
-    AllocTracer.TraceDisabled = !AllocTracer.TraceDisabled;
-  }
-  ~TraceLock() { AllocTracer.TraceDisabled = !AllocTracer.TraceDisabled; }
-
-  bool IsDisabled() const {
-    // This is already inverted value.
-    return !AllocTracer.TraceDisabled;
-  }
-
-private:
-  std::lock_guard<std::recursive_mutex> Lock;
-};
-
-ATTRIBUTE_NO_SANITIZE_MEMORY
-void MallocHook(const volatile void *ptr, size_t size) {
-  size_t N = AllocTracer.Mallocs++;
-  F->HandleMalloc(size);
-  if (int TraceLevel = AllocTracer.TraceLevel) {
-    TraceLock Lock;
-    if (Lock.IsDisabled())
-      return;
-    Printf("MALLOC[%zd] %p %zd\n", N, ptr, size);
-    if (TraceLevel >= 2 && EF)
-      PrintStackTrace();
-  }
-}
-
-ATTRIBUTE_NO_SANITIZE_MEMORY
-void FreeHook(const volatile void *ptr) {
-  size_t N = AllocTracer.Frees++;
-  if (int TraceLevel = AllocTracer.TraceLevel) {
-    TraceLock Lock;
-    if (Lock.IsDisabled())
-      return;
-    Printf("FREE[%zd]   %p\n", N, ptr);
-    if (TraceLevel >= 2 && EF)
-      PrintStackTrace();
-  }
-}
-
-// Crash on a single malloc that exceeds the rss limit.
-void Fuzzer::HandleMalloc(size_t Size) {
-  if (!Options.MallocLimitMb || (Size >> 20) < (size_t)Options.MallocLimitMb)
-    return;
-  Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(),
-         Size);
-  Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
-  PrintStackTrace();
-  DumpCurrentUnit("oom-");
-  Printf("SUMMARY: libFuzzer: out-of-memory\n");
-  PrintFinalStats();
-  _Exit(Options.OOMExitCode); // Stop right now.
-}
-
 Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
                FuzzingOptions Options)
     : CB(CB), Corpus(Corpus), MD(MD), Options(Options) {
   if (EF->__sanitizer_set_death_callback)
     EF->__sanitizer_set_death_callback(FuzzerDeathCallback);
   assert(!F);
   F = this;
   TPC.ResetMaps();
   IsMyThread = true;
-  if (Options.DetectLeaks && EF->__sanitizer_install_malloc_and_free_hooks)
-    EF->__sanitizer_install_malloc_and_free_hooks(MallocHook, FreeHook);
+  AllocMonitorConfigure(Options);
   TPC.SetUseCounters(Options.UseCounters);
   TPC.SetUseValueProfileMask(Options.UseValueProfile);
   if (Options.Verbosity)
     TPC.PrintModuleInfo();
   if (!Options.OutputCorpus.empty() && Options.ReloadIntervalSec)
     EpochOfLastReadOfOutputCorpus = GetEpoch(Options.OutputCorpus);
   MaxInputLen = MaxMutationLen = Options.MaxLen;
   TmpMaxMutationLen = 0;  // Will be set once we load the corpus.

 void Fuzzer::OutOfMemoryCallback(unsigned long PID, size_t LastMalloc) {
   if (EF->__sanitizer_acquire_crash_state &&
       !EF->__sanitizer_acquire_crash_state())
     return;
   Printf("==%lu== ERROR: libFuzzer: out-of-memory ", GetPid(PID));
   if (LastMalloc) {
     Printf("(malloc(%zd))\n", LastMalloc);
     Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
-    PrintStackTrace();
+    PrintStackTrace(PID);
   } else {
     Printf("(used: %zdMb; limit: %zdMb)\n", GetPeakRSSMb(), Options.RssLimitMb);
     Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
     PrintMemoryProfile(PID);
   }
   DumpCurrentUnit("oom-");
   Printf("SUMMARY: libFuzzer: out-of-memory\n");
   PrintFinalStats();
if (EF->__msan_unpoison)
     EF->__msan_unpoison(DataCopy, Size);
   if (EF->__msan_unpoison_param)
     EF->__msan_unpoison_param(2);
   if (CurrentUnitData && CurrentUnitData != Data)
     memcpy(CurrentUnitData, Data, Size);
   CurrentUnitSize = Size;
   {
     ScopedEnableMsanInterceptorChecks S;
-    AllocTracer.Start(Options.TraceMalloc);
+    AllocMonitorStartTracing(Options.TraceMalloc);
     UnitStartTime = system_clock::now();
     TPC.ResetMaps();
     RunningUserCallback = true;
     int Res = CB(DataCopy, Size);
     RunningUserCallback = false;
     UnitStopTime = system_clock::now();
     (void)Res;
     assert(Res == 0);
-    HasMoreMallocsThanFrees = AllocTracer.Stop();
+    HasMoreMallocsThanFrees = AllocMonitorStopTracing();
   }
   if (!LooseMemeq(DataCopy, Data, Size))
     CrashOnOverwrittenData();
   CurrentUnitSize = 0;
   delete[] DataCopy;
 }
 
 std::string Fuzzer::WriteToOutputCorpus(const Unit &U) {
if (Options.LenControl) {
       }
     } else {
       TmpMaxMutationLen = MaxMutationLen;
     }
 
     // Perform several mutations and runs.
     MutateAndTestOne();
 
-    PurgeAllocator();
+    AllocMonitorPurge();
   }
 
   PrintStats("DONE  ", "\n");
   MD.PrintRecommendedDictionary();
 }
 
 void Fuzzer::MinimizeCrashLoop(const Unit &U) {
   if (U.size() <= 1)

compiler-rt/lib/fuzzer/FuzzerMonitor.h

 //===- FuzzerMonitor.h ------------------------------------------*- C++ -* ===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Interfaces for trace mallocs/frees and calling back on errors.
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_FUZZER_MONITOR_H
 #define LLVM_FUZZER_MONITOR_H
 
+#include "FuzzerOptions.h"
+#include "FuzzerPlatform.h"
+
+namespace fuzzer {
+
+void AllocMonitorConfigure(const FuzzingOptions &Options);
+void AllocMonitorStartTracing(int TraceLevel);
+bool AllocMonitorStopTracing();
+void AllocMonitorPurge();
+
+} // namespace fuzzer
+
 extern "C" {
 
 ATTRIBUTE_INTERFACE void FuzzerAlarmCallback();
 ATTRIBUTE_INTERFACE void FuzzerCrashSignalCallback(unsigned long PID);
 ATTRIBUTE_INTERFACE void FuzzerDeathCallback();
 ATTRIBUTE_INTERFACE void FuzzerExitCallback(unsigned long PID);
 ATTRIBUTE_INTERFACE void FuzzerFileSizeExceedCallback();
 ATTRIBUTE_INTERFACE void FuzzerGracefulExitCallback();

compiler-rt/lib/fuzzer/FuzzerMonitor.cpp

+//===- FuzzerMonitor.cpp ----------------------------------------*- C++ -* ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Defines an interface for tracing mallocs and frees.
+//===----------------------------------------------------------------------===//
+
+#include "FuzzerMonitor.h"
+#include "FuzzerExtFunctions.h"
+#include "FuzzerIO.h"
+#include "FuzzerPlatform.h"
+#include "FuzzerUtil.h"
+#include <atomic>
+#include <chrono>
+#include <mutex>
+
+namespace fuzzer {
+namespace {
+
+void MallocHook(const volatile void *ptr, size_t size);
+void FreeHook(const volatile void *ptr);
+
+// Leak detection is expensive, so we first check if there were more mallocs
+// than frees (using the sanitizer malloc hooks) and only then try to call lsan.
+static struct {
+  void Configure(const FuzzingOptions &Options) {
+    MallocLimit = Options.MallocLimitMb << 20;
+    TraceLevel = Options.TraceMalloc;
+    if (Options.DetectLeaks && EF->__sanitizer_install_malloc_and_free_hooks)
+      EF->__sanitizer_install_malloc_and_free_hooks(MallocHook, FreeHook);
+    if (Options.RssLimitMb < 0)
+      RssLimitMb = 0;
+    else
+      RssLimitMb = static_cast<size_t>(Options.RssLimitMb) / 2;
+    if (!EF->__sanitizer_purge_allocator)
+      PurgeIntervalSec = -1;
+    else
+      PurgeIntervalSec = Options.PurgeAllocatorIntervalSec;
+  }
+
+  void StartTracing(int TraceLevel) {
+    this->TraceLevel = TraceLevel;
+    if (TraceLevel)
+      Printf("MallocFreeTracer: START\n");
+    Mallocs = 0;
+    Frees = 0;
+  }
+
+  // Returns true if there were more mallocs than frees.
+  bool StopTracing() {
+    if (TraceLevel)
+      Printf("MallocFreeTracer: STOP %zd %zd (%s)\n", Mallocs.load(),
+             Frees.load(), Mallocs == Frees ? "same" : "DIFFERENT");
+    bool Result = Mallocs > Frees;
+    Mallocs = 0;
+    Frees = 0;
+    TraceLevel = 0;
+    return Result;
+  }
+
+  void Purge() {
+    if (PurgeIntervalSec < 0)
+      return;
+    auto Now = std::chrono::system_clock::now();
+    auto Elapsed = std::chrono::duration_cast<std::chrono::seconds>(
+        Now - LastPurgeAttempt);
+    if (Elapsed.count() >= PurgeIntervalSec && GetPeakRSSMb() > RssLimitMb)
+      EF->__sanitizer_purge_allocator();
+    LastPurgeAttempt = Now;
+  }
+
+  size_t MallocLimit;
+  std::atomic<size_t> Mallocs;
+  std::atomic<size_t> Frees;
+  int TraceLevel = 0;
+
+  std::recursive_mutex TraceMutex;
+  bool TraceDisabled;
+
+  size_t RssLimitMb;
+  int PurgeIntervalSec;
+  std::chrono::system_clock::time_point LastPurgeAttempt =
+      std::chrono::system_clock::now();
+
+} AllocMonitor;
+
+// Locks printing and avoids nested hooks triggered from mallocs/frees in
+// sanitizer.
+class TraceLock {
+public:
+  TraceLock() : Lock(AllocMonitor.TraceMutex) {
+    AllocMonitor.TraceDisabled = !AllocMonitor.TraceDisabled;
+  }
+  ~TraceLock() { AllocMonitor.TraceDisabled = !AllocMonitor.TraceDisabled; }
+
+  bool IsDisabled() const {
+    // This is already inverted value.
+    return !AllocMonitor.TraceDisabled;
+  }
+
+private:
+  std::lock_guard<std::recursive_mutex> Lock;
+};
+
+ATTRIBUTE_NO_SANITIZE_MEMORY
+void MallocHook(const volatile void *ptr, size_t size) {
+  size_t N = AllocMonitor.Mallocs++;
+  if (AllocMonitor.MallocLimit && size >= AllocMonitor.MallocLimit)
+    FuzzerMallocLimitCallback(0, size);
+  if (int TraceLevel = AllocMonitor.TraceLevel) {
+    TraceLock Lock;
+    if (Lock.IsDisabled())
+      return;
+    Printf("MALLOC[%zd] %p %zd\n", N, ptr, size);
+    if (TraceLevel >= 2 && EF)
+      PrintStackTrace();
+  }
+}
+
+ATTRIBUTE_NO_SANITIZE_MEMORY
+void FreeHook(const volatile void *ptr) {
+  size_t N = AllocMonitor.Frees++;
+  if (int TraceLevel = AllocMonitor.TraceLevel) {
+    TraceLock Lock;
+    if (Lock.IsDisabled())
+      return;
+    Printf("FREE[%zd]   %p\n", N, ptr);
+    if (TraceLevel >= 2 && EF)
+      PrintStackTrace();
+  }
+}
+
+} // namespace
+
+void AllocMonitorConfigure(const FuzzingOptions &Options) {
+  AllocMonitor.Configure(Options);
+}
+
+void AllocMonitorStartTracing(int TraceLevel) {
+  AllocMonitor.StartTracing(TraceLevel);
+}
+
+bool AllocMonitorStopTracing() { return AllocMonitor.StopTracing(); }
+
+void AllocMonitorPurge() { AllocMonitor.Purge(); }
+
+} // namespace fuzzer