[LibFuzzer] Add support for an optional user defined callback (`LLVMFuzzerOnExitHandler()`)

[delcypher]

Summary

[LibFuzzer] Add support for an optional user defined callback (LLVMFuzzerOnExitHandler())
which is called when LibFuzzer exits.

This callback will be called by LibFuzzer when it exits in most
scenarios. It currently doesn't call the callback if:

• LibFuzzer is launched incorrectly.
• LibFuzzer observes that the client corrupts the fuzzing input.

This upstreams a change from the JFS solver's fork of LibFuzzer. JFS's use
case is to ensure that an event log gets flushed to disk whenever
LibFuzzer decides that it wants to exit. There are probably other use
cases too.

[kcc]

Why is it not enough to cal atexit() in LLVMFuzzerInitialize?

[delcypher]

In D45762#1071308, @kcc wrote:

Why is it not enough to cal atexit() in LLVMFuzzerInitialize?

LibFuzzer currently uses _Exit() in several cases which means that it is not guaranteed that registered atexit() callbacks will be called by LibFuzer. In particular in JFS we currently use abort() in the program under test to detect we've reached the target and we implement a timeout externally (there are technical reasons for doing this) and send SIGINT to LibFuzzer when the timeout occurs. In both of these cases atexit handlers would not be called. Therefore we needed to implement a callback for the fuzzer exiting. This patch seemed like the most general way of doing this.

[delcypher]

@kcc ping

lib/fuzzer/FuzzerDefs.h

 #    define ATTRIBUTE_TARGET_POPCNT __attribute__((target("popcnt")))
 #  else
 #    define ATTRIBUTE_TARGET_POPCNT
 #  endif
 #else
 #  define ATTRIBUTE_TARGET_POPCNT
 #endif
 
+#if __has_attribute(noreturn)
+#  define ATTRIBUTE_NO_RETURN __attribute__((noreturn))
+#else
+#  define ATTRIBUTE_NO_RETURN
+#endif
 
 #ifdef __clang__  // avoid gcc warning.
 #  if __has_attribute(no_sanitize)
 #    define ATTRIBUTE_NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory")))
 #  else
 #    define ATTRIBUTE_NO_SANITIZE_MEMORY
 #  endif
 #  define ALWAYS_INLINE __attribute__((always_inline))

lib/fuzzer/FuzzerDriver.cpp

for (auto &Path : *Inputs) {
       auto MS = duration_cast<milliseconds>(StopTime - StartTime).count();
       Printf("Executed %s in %zd ms\n", Path.c_str(), (long)MS);
     }
     Printf("***\n"
            "*** NOTE: fuzzing was not performed, you have only\n"
            "***       executed the target code on a fixed set of inputs.\n"
            "***\n");
     F->PrintFinalStats();
-    exit(0);
+    F->ExitWithUserCallBack(0);
   }
 
   if (Flags.merge) {
     F->CrashResistantMerge(Args, *Inputs,
                            Flags.load_coverage_summary,
                            Flags.save_coverage_summary,
                            Flags.merge_control_file);
     exit(0);
int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   }
 
   F->Loop(*Inputs);
 
   if (Flags.verbosity)
     Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(),
            F->secondsSinceProcessStartUp());
   F->PrintFinalStats();
-
-  exit(0);  // Don't let F destroy itself.
+  F->ExitWithUserCallBack(0); // Don't let F destory itself.
 }
 
 // Storage for global ExternalFunctions object.
 ExternalFunctions *EF = nullptr;
 
 }  // namespace fuzzer

lib/fuzzer/FuzzerExtFunctions.def

 EXT_FUNC(LLVMFuzzerCustomMutator, size_t,
          (uint8_t * Data, size_t Size, size_t MaxSize, unsigned int Seed),
          false);
 EXT_FUNC(LLVMFuzzerCustomCrossOver, size_t,
          (const uint8_t * Data1, size_t Size1,
           const uint8_t * Data2, size_t Size2,
           uint8_t * Out, size_t MaxOutSize, unsigned int Seed),
          false);
+EXT_FUNC(LLVMFuzzerOnExitHandler, void, (), false);
 
 // Sanitizer functions
 EXT_FUNC(__lsan_enable, void, (), false);
 EXT_FUNC(__lsan_disable, void, (), false);
 EXT_FUNC(__lsan_do_recoverable_leak_check, int, (), false);
 EXT_FUNC(__sanitizer_install_malloc_and_free_hooks, int,
          (void (*malloc_hook)(const volatile void *, size_t),
           void (*free_hook)(const volatile void *)),

lib/fuzzer/FuzzerInterface.h

 
 // Experimental, may go away in future.
 // libFuzzer-provided function to be used inside LLVMFuzzerCustomMutator.
 // Mutates raw data in [Data, Data+Size) inplace.
 // Returns the new size, which is not greater than MaxSize.
 __attribute__((visibility("default"))) size_t
 LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize);
 
+// Optional user-provided exit function.  If provided, libFuzzer will try to
+// call this function when libFuzzer exits, in most scenarios
+__attribute__((visibility("default"))) void LLVMFuzzerOnExitHandler();
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif  // __cplusplus
 
 #endif  // LLVM_FUZZER_INTERFACE_H

lib/fuzzer/FuzzerInternal.h

public:
 
   bool InFuzzingThread() const { return IsMyThread; }
   size_t GetCurrentUnitInFuzzingThead(const uint8_t **Data) const;
   void TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size,
                                bool DuringInitialCorpusExecution);
 
   void HandleMalloc(size_t Size);
   void AnnounceOutput(const uint8_t *Data, size_t Size);
+  ATTRIBUTE_NO_RETURN static void ExitWithUserCallBack(int Status,
+                                                       bool CleanUp = true);
 
 private:
   void AlarmCallback();
   void CrashCallback();
   void ExitCallback();
   void MaybeExitGracefully();
   void CrashOnOverwrittenData();
   void InterruptCallback();

lib/fuzzer/FuzzerLoop.cpp

void Fuzzer::HandleMalloc(size_t Size) {
   Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(),
          Size);
   Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
   if (EF->__sanitizer_print_stack_trace)
     EF->__sanitizer_print_stack_trace();
   DumpCurrentUnit("oom-");
   Printf("SUMMARY: libFuzzer: out-of-memory\n");
   PrintFinalStats();
-  _Exit(Options.ErrorExitCode); // Stop right now.
+  ExitWithUserCallBack(Options.ErrorExitCode,
+                       /*CleanUp=*/false); // Stop right now.
 }
 
 Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
                FuzzingOptions Options)
     : CB(CB), Corpus(Corpus), MD(MD), Options(Options) {
   if (EF->__sanitizer_set_death_callback)
     EF->__sanitizer_set_death_callback(StaticDeathCallback);
   assert(!F);

 void Fuzzer::StaticGracefulExitCallback() {
   assert(F);
   F->GracefulExitRequested = true;
   Printf("INFO: signal received, trying to exit gracefully\n");
 }
 
 void Fuzzer::StaticFileSizeExceedCallback() {
   Printf("==%lu== ERROR: libFuzzer: file size exceeded\n", GetPid());
-  exit(1);
+  // FIXME: Why is `exit()` being used rather than `_Exit()`
+  // here?
+  // FIXME: Exit code should really be `Options.ErrorExitCode`.
+  ExitWithUserCallBack(1, /*CleanUp=*/true);
 }
 
 void Fuzzer::CrashCallback() {
   Printf("==%lu== ERROR: libFuzzer: deadly signal\n", GetPid());
   if (EF->__sanitizer_print_stack_trace)
     EF->__sanitizer_print_stack_trace();
   Printf("NOTE: libFuzzer has rudimentary signal handlers.\n"
          "      Combine libFuzzer with AddressSanitizer or similar for better "
          "crash reports.\n");
   Printf("SUMMARY: libFuzzer: deadly signal\n");
   DumpCurrentUnit("crash-");
   PrintFinalStats();
-  _Exit(Options.ErrorExitCode); // Stop right now.
+  ExitWithUserCallBack(Options.ErrorExitCode,
+                       /*CleanUp=*/false); // Stop right now
 }
 
 void Fuzzer::ExitCallback() {
   if (!RunningCB)
     return; // This exit did not come from the user callback
   Printf("==%lu== ERROR: libFuzzer: fuzz target exited\n", GetPid());
   if (EF->__sanitizer_print_stack_trace)
     EF->__sanitizer_print_stack_trace();
   Printf("SUMMARY: libFuzzer: fuzz target exited\n");
   DumpCurrentUnit("crash-");
   PrintFinalStats();
-  _Exit(Options.ErrorExitCode);
+  ExitWithUserCallBack(Options.ErrorExitCode, /*CleanUp=*/false);
 }
 
 void Fuzzer::MaybeExitGracefully() {
   if (!GracefulExitRequested) return;
   Printf("==%lu== INFO: libFuzzer: exiting as requested\n", GetPid());
   PrintFinalStats();
-  _Exit(0);
+  ExitWithUserCallBack(0, /*CleanUp=*/false);
 }
 
 void Fuzzer::InterruptCallback() {
   Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid());
   PrintFinalStats();
-  _Exit(0); // Stop right now, don't perform any at-exit actions.
+  // Stop right now, don't perform any at-exit actions except the user defined
+  // callback.
+  ExitWithUserCallBack(0, /*CleanUp=*/false);
 }
 
 NO_SANITIZE_MEMORY
 void Fuzzer::AlarmCallback() {
   assert(Options.UnitTimeoutSec > 0);
   // In Windows Alarm callback is executed by a different thread.
 #if !LIBFUZZER_WINDOWS
   if (!InFuzzingThread())
Printf("       and the timeout value is %d (use -timeout=N to change)\n",
            Options.UnitTimeoutSec);
     DumpCurrentUnit("timeout-");
     Printf("==%lu== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),
            Seconds);
     if (EF->__sanitizer_print_stack_trace)
       EF->__sanitizer_print_stack_trace();
     Printf("SUMMARY: libFuzzer: timeout\n");
     PrintFinalStats();
-    _Exit(Options.TimeoutExitCode); // Stop right now.
+    ExitWithUserCallBack(Options.TimeoutExitCode,
+                         /*CleanUp=*/false); // Stop right now.
   }
 }
 
 void Fuzzer::RssLimitCallback() {
   Printf(
       "==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
       GetPid(), GetPeakRSSMb(), Options.RssLimitMb);
   Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
   if (EF->__sanitizer_print_memory_profile)
     EF->__sanitizer_print_memory_profile(95, 8);
   DumpCurrentUnit("oom-");
   Printf("SUMMARY: libFuzzer: out-of-memory\n");
   PrintFinalStats();
-  _Exit(Options.ErrorExitCode); // Stop right now.
+  ExitWithUserCallBack(Options.ErrorExitCode,
+                       /*CleanUp=*/false); // Stop right now.
 }
 
 void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) {
   size_t ExecPerSec = execPerSec();
   if (!Options.Verbosity)
     return;
   Printf("#%zd\t%s", TotalNumberOfRuns, Where);
   if (size_t N = TPC.GetTotalPCCoverage())
if (!Options.ExitOnSrcPos.empty()) {
     static auto *PCsSet = new Set<uintptr_t>;
     auto HandlePC = [&](uintptr_t PC) {
       if (!PCsSet->insert(PC).second)
         return;
       std::string Descr = DescribePC("%F %L", PC + 1);
       if (Descr.find(Options.ExitOnSrcPos) != std::string::npos) {
         Printf("INFO: found line matching '%s', exiting.\n",
                Options.ExitOnSrcPos.c_str());
-        _Exit(0);
+        ExitWithUserCallBack(0, /*CleanUp=*/false);
       }
     };
     TPC.ForEachObservedPC(HandlePC);
   }
   if (!Options.ExitOnItem.empty()) {
     if (Corpus.HasUnit(Options.ExitOnItem)) {
       Printf("INFO: found item with checksum '%s', exiting.\n",
              Options.ExitOnItem.c_str());
-      _Exit(0);
+      ExitWithUserCallBack(0, /*CleanUp=*/false);
     }
   }
 }
 
 void Fuzzer::RereadOutputCorpus(size_t MaxSize) {
   if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec)
     return;
   Vector<Unit> AdditionalCorpus;
size_t Fuzzer::GetCurrentUnitInFuzzingThead(const uint8_t **Data) const {
   return CurrentUnitSize;
 }
 
 void Fuzzer::CrashOnOverwrittenData() {
   Printf("==%d== ERROR: libFuzzer: fuzz target overwrites it's const input\n",
          GetPid());
   DumpCurrentUnit("crash-");
   Printf("SUMMARY: libFuzzer: out-of-memory\n");
+  // Don't use ExitWithUserCallBack(). The client code has abused our API
+  // so it's probably best to avoid calling `LLVMFuzzerOnExitHandler`.
   _Exit(Options.ErrorExitCode); // Stop right now.
 }
 
 // Compare two arrays, but not all bytes if the arrays are large.
 static bool LooseMemeq(const uint8_t *A, const uint8_t *B, size_t Size) {
   const size_t Limit = 64;
   if (Size <= 64)
     return !memcmp(A, B, Size);
void Fuzzer::TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size,
   // we don't call it too often.
   if (EF->__lsan_do_recoverable_leak_check()) { // Leak is found, report it.
     if (DuringInitialCorpusExecution)
       Printf("\nINFO: a leak has been found in the initial corpus.\n\n");
     Printf("INFO: to ignore leaks on libFuzzer side use -detect_leaks=0.\n\n");
     CurrentUnitSize = Size;
     DumpCurrentUnit("leak-");
     PrintFinalStats();
-    _Exit(Options.ErrorExitCode); // not exit() to disable lsan further on.
+    // CleanUp=false to disable lsan further on.
+    ExitWithUserCallBack(Options.ErrorExitCode, /*CleanUp=*/false);
   }
 }
 
 void Fuzzer::MutateAndTestOne() {
   MD.StartMutationSequence();
 
   auto &II = Corpus.ChooseUnitToMutate(MD.GetRand());
   if (Options.UseFeatureFrequency)
if (Size != OtherSize || memcmp(Data, OtherData, Size) != 0) {
       DumpCurrentUnit("mismatch-");
       Printf("SUMMARY: libFuzzer: equivalence-mismatch\n");
       PrintFinalStats();
       _Exit(Options.ErrorExitCode);
     }
   }
 }
 
+void Fuzzer::ExitWithUserCallBack(int Status, bool CleanUp) {
+  if (EF->LLVMFuzzerOnExitHandler) {
+    EF->LLVMFuzzerOnExitHandler();
+  }
+  if (CleanUp)
+    exit(Status);
+  _Exit(Status);
+}
+
 } // namespace fuzzer
 
 extern "C" {
 
 __attribute__((visibility("default"))) size_t
 LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) {
   assert(fuzzer::F);
   return fuzzer::F->GetMD().DefaultMutate(Data, Size, MaxSize);

test/fuzzer/FuzzerOnExitHandlerTest.cpp

+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// Test for a fuzzer: must find the case where a particular basic block is
+// executed many times.
+#include <algorithm>
+#include <cassert>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+__attribute__((noinline)) uint8_t *getNull() { return 0; }
+uint8_t *v = NULL;
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+#if defined(TEST_MODE_LOOP)
+  return 0;
+#elif defined(TEST_MODE_ABORT)
+  abort();
+#elif defined(TEST_MODE_CRASH)
+  uint8_t *v = getNull();
+  v[0] = 'x'; // BOOM
+  return 0;
+#elif defined(TEST_MODE_HANG)
+  uint8_t *v = 0;
+  while (true) {
+    v = getNull();
+  }
+  return 0;
+#elif defined(TEST_MODE_LARGE_ALLOC)
+  if (Size == 0)
+    return 0;                      // Don't alloc during INIT
+  v = (uint8_t *)malloc(30 << 20); // 30MiB
+  assert(v != NULL);
+  // Do something with the allocated memory to avoid Clang optimizing
+  // malloc/free out.
+  memcpy(v, Data, Size);
+  free(v);
+  return 0;
+#elif defined(TEST_MODE_LEAK)
+  v = (uint8_t *)malloc(128); // Leak
+  return 0;
+#elif defined(TEST_MODE_EXIT_ON_SRC_LOC)
+  if (Size > 0 && Data[0] == 'H') {
+    v = (uint8_t *)0x0;
+    if (Size > 1 && Data[1] == 'i') {
+      v = (uint8_t *)0x1; // This is the source location
+      if (Size > 2 && Data[2] == '!') {
+        v = (uint8_t *)0x2;
+      }
+    }
+  }
+  return 0;
+#else
+#error TEST_MODE must be set
+#endif
+}
+
+extern "C" void LLVMFuzzerOnExitHandler() {
+  printf("Custom LLVMFuzzerOnExitHandler called\n");
+  fflush(stdout);
+}

test/fuzzer/fuzzer-on-exit-handler-leaks.test

+// FIXME: Update as LSan support is implemented in more targets and LibFuzzer.
+REQUIRES: linux
+RUN: %cpp_compiler -fsanitize=leak -DTEST_MODE_LARGE_ALLOC %S/FuzzerOnExitHandlerTest.cpp -o %t-FuzzerAtExitHandlerTestLargeAllocLSan
+
+// Malloc limit
+RUN: not %t-FuzzerAtExitHandlerTestLargeAllocLSan -malloc_limit_mb=1 2>&1 | FileCheck %s
+// Clean exit
+
+// Leak check
+RUN: %cpp_compiler -fsanitize=leak -DTEST_MODE_LEAK %S/FuzzerOnExitHandlerTest.cpp -o %t-FuzzerAtExitHandlerTestLeak
+RUN: not %t-FuzzerAtExitHandlerTestLeak -detect_leaks=1 2>&1 | FileCheck %s
+
+CHECK: Custom LLVMFuzzerOnExitHandler called

test/fuzzer/fuzzer-on-exit-handler.test

+// Clean exit
+RUN: %cpp_compiler -DTEST_MODE_LOOP %S/FuzzerOnExitHandlerTest.cpp -o %t-FuzzerAtExitHandlerTestLoop
+RUN: %t-FuzzerAtExitHandlerTestLoop -runs=20 2>&1 | FileCheck %s
+
+
+// Abort called
+RUN: %cpp_compiler -DTEST_MODE_ABORT %S/FuzzerOnExitHandlerTest.cpp -o %t-FuzzerAtExitHandlerTestAbort
+RUN: not %t-FuzzerAtExitHandlerTestAbort 2>&1 | FileCheck %s
+
+// Crash
+RUN: %cpp_compiler -DTEST_MODE_CRASH %S/FuzzerOnExitHandlerTest.cpp -o %t-FuzzerAtExitHandlerTestCrash
+RUN: not %t-FuzzerAtExitHandlerTestCrash 2>&1 | FileCheck %s
+
+// Total timeout
+RUN: %t-FuzzerAtExitHandlerTestLoop -max_total_time=2 2>&1 | FileCheck %s
+
+// Per unit timeout
+RUN: %cpp_compiler -DTEST_MODE_HANG %S/FuzzerOnExitHandlerTest.cpp -o %t-FuzzerAtExitHandlerTestHang
+RUN: not %t-FuzzerAtExitHandlerTestHang -timeout=2 2>&1 | FileCheck %s
+
+
+// Run on files only
+RUN: echo "input one" >> %t-input-one.txt
+RUN: echo "input two" >> %t-input-two.txt
+RUN: %t-FuzzerAtExitHandlerTestLoop %t-input-one.txt %t-input-two.txt 2>&1 | FileCheck %s
+
+// RSS limit
+RUN: %cpp_compiler -DTEST_MODE_LARGE_ALLOC %S/FuzzerOnExitHandlerTest.cpp -o %t-FuzzerAtExitHandlerTestLargeAlloc
+RUN: not %t-FuzzerAtExitHandlerTestLargeAlloc -rss_limit_mb=30 2>&1 | FileCheck %s
+
+// Exit on source location
+// NOTE: Compile flags taken from exit_on_src_pos.test
+RUN: %cpp_compiler -O0 -DTEST_MODE_EXIT_ON_SRC_LOC %S/FuzzerOnExitHandlerTest.cpp -mllvm -use-unknown-locations=Disable -o %t-FuzzerAtExitHandlerTestSimpleTest
+RUN: %t-FuzzerAtExitHandlerTestSimpleTest -max_len=2 -exit_on_src_pos=FuzzerOnExitHandlerTest.cpp:49 2>&1 | FileCheck %s
+
+// Exit on unit
+RUN: %t-FuzzerAtExitHandlerTestSimpleTest -seed=1 -max_len=2 -exit_on_item=adc83b19e793491b1c6ea0fd8b46cd9f32e592fc 2>&1 | FileCheck %s
+
+// FIXME: We should really test SIGINT but there's no portable way to do that.
+
+CHECK: Custom LLVMFuzzerOnExitHandler called