[libFuzzer] Periodically purge allocator's quarantine to prolong fuzzing sessions.

[alekseyshl]

Summary

Fuzzing targets that allocate/deallocate a lot of memory tend to consume
a lot of RSS when ASan quarantine is enabled. Purging quarantine between
iterations and returning memory to OS keeps RSS down and should not
reduce the quarantine effectiveness provided the fuzz target does not
preserve state between iterations (in this case this feature can be turned off).

Based on D39153.

[vitalybuka]

Did you consider to avoid interval option at all?
e.g. closer you get to rss limit -> smaller number of fuzz iteration without purge?

lib/fuzzer/FuzzerLoop.cpp

604 ↗

(On Diff #119739)

for could you put following together?

EF->__sanitizer_purge_allocator();
LastAllocatorPurgeAttemptTime = system_clock::now();

e.g.

void Fuzzer::PurgeAllocator() {
  if (some condition)
    return;
  if (some condition)
    return;
  if (some condition)
    return;
  if (some condition)
    return;
 
  EF->__sanitizer_purge_allocator();
  LastAllocatorPurgeAttemptTime = system_clock::now();
}

or 

void Fuzzer::PurgeAllocator() {
  if (some condition ||
      some condition ||
      some condition ||
      some condition) {
     return;
  } 
 
  EF->__sanitizer_purge_allocator();
  LastAllocatorPurgeAttemptTime = system_clock::now();
}

[alekseyshl]

lib/fuzzer/FuzzerLoop.cpp

604 ↗

(On Diff #119739)

No, that's not what I wanted to achieve here, I want to record LastAllocatorPurgeAttemptTime regadless of the result of the last check, so that we do not call GetPeakRSSMb() too often.

[kcc]

compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp

603

Style nit pick.
libFuzzer tries to follow the LLVM code style exactly, and in that style we don't use {} in statements like this. (here, and below in this function)

compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp

int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   Options.UseValueProfile = Flags.use_value_profile;
   Options.Shrink = Flags.shrink;
   Options.ReduceInputs = Flags.reduce_inputs;
   Options.ShuffleAtStartUp = Flags.shuffle;
   Options.PreferSmall = Flags.prefer_small;
   Options.ReloadIntervalSec = Flags.reload;
   Options.OnlyASCII = Flags.only_ascii;
   Options.DetectLeaks = Flags.detect_leaks;
+  Options.PurgeAllocatorIntervalSec = Flags.purge_allocator_interval;
   Options.TraceMalloc = Flags.trace_malloc;
   Options.RssLimitMb = Flags.rss_limit_mb;
   if (Flags.runs >= 0)
     Options.MaxNumberOfRuns = Flags.runs;
   if (!Inputs->empty() && !Flags.minimize_crash_internal_step)
     Options.OutputCorpus = (*Inputs)[0];
   Options.ReportSlowUnits = Flags.report_slow_units;
   if (Flags.artifact_prefix)

compiler-rt/trunk/lib/fuzzer/FuzzerExtFunctions.def

 // Sanitizer functions
 EXT_FUNC(__lsan_enable, void, (), false);
 EXT_FUNC(__lsan_disable, void, (), false);
 EXT_FUNC(__lsan_do_recoverable_leak_check, int, (), false);
 EXT_FUNC(__sanitizer_install_malloc_and_free_hooks, int,
          (void (*malloc_hook)(const volatile void *, size_t),
           void (*free_hook)(const volatile void *)),
          false);
+EXT_FUNC(__sanitizer_purge_allocator, void, (), false);
 EXT_FUNC(__sanitizer_print_memory_profile, int, (size_t, size_t), false);
 EXT_FUNC(__sanitizer_print_stack_trace, void, (), true);
 EXT_FUNC(__sanitizer_symbolize_pc, void,
          (void *, const char *fmt, char *out_buf, size_t out_buf_size), false);
 EXT_FUNC(__sanitizer_get_module_and_offset_for_pc, int,
          (void *pc, char *module_path,
          size_t module_path_len,void **pc_offset), false);
 EXT_FUNC(__sanitizer_set_death_callback, void, (void (*)(void)), true);
 EXT_FUNC(__sanitizer_set_report_fd, void, (void*), false);
 EXT_FUNC(__sanitizer_dump_coverage, void, (const uintptr_t *, uintptr_t),
          false);

compiler-rt/trunk/lib/fuzzer/FuzzerFlags.def

 FUZZER_FLAG_INT(handle_int, 1, "If 1, try to intercept SIGINT.")
 FUZZER_FLAG_INT(handle_term, 1, "If 1, try to intercept SIGTERM.")
 FUZZER_FLAG_INT(handle_xfsz, 1, "If 1, try to intercept SIGXFSZ.")
 FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; "
     "if 2, close stderr; if 3, close both. "
     "Be careful, this will also close e.g. asan's stderr/stdout.")
 FUZZER_FLAG_INT(detect_leaks, 1, "If 1, and if LeakSanitizer is enabled "
     "try to detect memory leaks during fuzzing (i.e. not only at shut down).")
+FUZZER_FLAG_INT(purge_allocator_interval, 1, "Purge allocator caches and "
+    "quarantines every <N> seconds. When rss_limit_mb is specified (>0), "
+    "purging starts when RSS exceeds 50% of rss_limit_mb. Pass "
+    "purge_allocator_interval=-1 to disable this functionality.")
 FUZZER_FLAG_INT(trace_malloc, 0, "If >= 1 will print all mallocs/frees. "
     "If >= 2 will also print stack traces.")
 FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon"
     "reaching this limit of RSS memory usage.")
 FUZZER_FLAG_STRING(exit_on_src_pos, "Exit if a newly found PC originates"
     " from the given source location. Example: -exit_on_src_pos=foo.cc:123. "
     "Used primarily for testing libFuzzer itself.")
 FUZZER_FLAG_STRING(exit_on_item, "Exit if an item with a given sha1 sum"

compiler-rt/trunk/lib/fuzzer/FuzzerInternal.h

 
 private:
   void AlarmCallback();
   void CrashCallback();
   void ExitCallback();
   void CrashOnOverwrittenData();
   void InterruptCallback();
   void MutateAndTestOne();
+  void PurgeAllocator();
   void ReportNewCoverage(InputInfo *II, const Unit &U);
   void PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size);
   void WriteToOutputCorpus(const Unit &U);
   void WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix);
   void PrintStats(const char *Where, const char *End = "\n", size_t Units = 0);
   void PrintStatusForNewUnit(const Unit &U, const char *Text);
   void CheckExitOnSrcPosOrItem();
 
private:
 
   size_t LastCorpusUpdateRun = 0;
   system_clock::time_point LastCorpusUpdateTime = system_clock::now();
 
 
   bool HasMoreMallocsThanFrees = false;
   size_t NumberOfLeakDetectionAttempts = 0;
 
+  system_clock::time_point LastAllocatorPurgeAttemptTime = system_clock::now();
+
   UserCallback CB;
   InputCorpus &Corpus;
   MutationDispatcher &MD;
   FuzzingOptions Options;
 
   system_clock::time_point ProcessStartTime = system_clock::now();
   system_clock::time_point UnitStartTime, UnitStopTime;
   long TimeOfLongestUnitInSeconds = 0;

compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp

void Fuzzer::MutateAndTestOne() {
   assert(CurrentMaxMutationLen > 0);
 
   for (int i = 0; i < Options.MutateDepth; i++) {
     if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
       break;
     size_t NewSize = 0;
     NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen);
     assert(NewSize > 0 && "Mutator returned empty unit");
-    assert(NewSize <= CurrentMaxMutationLen && "Mutator return overisized unit");
+    assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit");
     Size = NewSize;
     II.NumExecutedMutations++;
     if (RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II))
       ReportNewCoverage(&II, {CurrentUnitData, CurrentUnitData + Size});
 
     TryDetectingAMemoryLeak(CurrentUnitData, Size,
                             /*DuringInitialCorpusExecution*/ false);
   }
 }
 
+void Fuzzer::PurgeAllocator() {
+  if (Options.PurgeAllocatorIntervalSec < 0 ||
+      !EF->__sanitizer_purge_allocator) {
+    return;
+  }
+  if (duration_cast<seconds>(system_clock::now() -
+                             LastAllocatorPurgeAttemptTime).count() <
+      Options.PurgeAllocatorIntervalSec) {
+    return;
+  }
+
+  if (Options.RssLimitMb <= 0 ||
+      GetPeakRSSMb() > static_cast<size_t>(Options.RssLimitMb) / 2) {
+    EF->__sanitizer_purge_allocator();
+  }
+
+  LastAllocatorPurgeAttemptTime = system_clock::now();
+}
+
 void Fuzzer::ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs) {
   const size_t kMaxSaneLen = 1 << 20;
   const size_t kMinDefaultLen = 4096;
   Vector<SizedFile> SizedFiles;
   size_t MaxSize = 0;
   size_t MinSize = -1;
   size_t TotalSize = 0;
   size_t LastNumFiles = 0;
if (Options.ExperimentalLenControl) {
                  TmpMaxMutationLen);
       }
     } else {
       TmpMaxMutationLen = MaxMutationLen;
     }
 
     // Perform several mutations and runs.
     MutateAndTestOne();
+
+    PurgeAllocator();
   }
 
   PrintStats("DONE  ", "\n");
   MD.PrintRecommendedDictionary();
 }
 
 void Fuzzer::MinimizeCrashLoop(const Unit &U) {
   if (U.size() <= 1) return;

compiler-rt/trunk/lib/fuzzer/FuzzerOptions.h

struct FuzzingOptions {
   bool PrintNewCovPcs = false;
   int PrintNewCovFuncs = 0;
   bool PrintFinalStats = false;
   bool PrintCorpusStats = false;
   bool PrintCoverage = false;
   bool DumpCoverage = false;
   bool UseClangCoverage = false;
   bool DetectLeaks = true;
+  int PurgeAllocatorIntervalSec = 1;
   int UseFeatureFrequency = false;
   int  TraceMalloc = 0;
   bool HandleAbrt = false;
   bool HandleBus = false;
   bool HandleFpe = false;
   bool HandleIll = false;
   bool HandleInt = false;
   bool HandleSegv = false;
   bool HandleTerm = false;
   bool HandleXfsz = false;
 };
 
 }  // namespace fuzzer
 
 #endif  // LLVM_FUZZER_OPTIONS_H