[libFuzzer] Guard symbolization with try-lock.

[morehouse]

Summary

When out-of-memory or timeout occurs, threads can be stopped during
symbolization, thereby causing a deadlock when the OOM/TO handlers
attempt symbolization. We avoid this deadlock by skipping symbolization
if another thread is symbolizing.

[kcc]

code LGTM.
Any chance to get a test (something that hangs not, and doesn't with this change)?
I understand it's not trivial...

[morehouse]

• Add (probabilistic) deadlock test.

[morehouse]

At least on my machine this test deadlocks ~75% of the time. But of course it is tough to say what will happen on other machines.

[kcc]

LGTM, thanks!

compiler-rt/lib/fuzzer/FuzzerLoop.cpp

void MallocHook(const volatile void *ptr, size_t size) {
   size_t N = AllocTracer.Mallocs++;
   F->HandleMalloc(size);
   if (int TraceLevel = AllocTracer.TraceLevel) {
     TraceLock Lock;
     if (Lock.IsDisabled())
       return;
     Printf("MALLOC[%zd] %p %zd\n", N, ptr, size);
     if (TraceLevel >= 2 && EF)
-      EF->__sanitizer_print_stack_trace();
+      PrintStackTrace();
   }
 }
 
 ATTRIBUTE_NO_SANITIZE_MEMORY
 void FreeHook(const volatile void *ptr) {
   size_t N = AllocTracer.Frees++;
   if (int TraceLevel = AllocTracer.TraceLevel) {
     TraceLock Lock;
     if (Lock.IsDisabled())
       return;
     Printf("FREE[%zd]   %p\n", N, ptr);
     if (TraceLevel >= 2 && EF)
-      EF->__sanitizer_print_stack_trace();
+      PrintStackTrace();
   }
 }
 
 // Crash on a single malloc that exceeds the rss limit.
 void Fuzzer::HandleMalloc(size_t Size) {
   if (!Options.MallocLimitMb || (Size >> 20) < (size_t)Options.MallocLimitMb)
     return;
   Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(),
          Size);
   Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
-  if (EF->__sanitizer_print_stack_trace)
-    EF->__sanitizer_print_stack_trace();
+  PrintStackTrace();
   DumpCurrentUnit("oom-");
   Printf("SUMMARY: libFuzzer: out-of-memory\n");
   PrintFinalStats();
   _Exit(Options.ErrorExitCode); // Stop right now.
 }
 
 Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
                FuzzingOptions Options)
void Fuzzer::StaticFileSizeExceedCallback() {
   Printf("==%lu== ERROR: libFuzzer: file size exceeded\n", GetPid());
   exit(1);
 }
 
 void Fuzzer::CrashCallback() {
   if (EF->__sanitizer_acquire_crash_state)
     EF->__sanitizer_acquire_crash_state();
   Printf("==%lu== ERROR: libFuzzer: deadly signal\n", GetPid());
-  if (EF->__sanitizer_print_stack_trace)
-    EF->__sanitizer_print_stack_trace();
+  PrintStackTrace();
   Printf("NOTE: libFuzzer has rudimentary signal handlers.\n"
          "      Combine libFuzzer with AddressSanitizer or similar for better "
          "crash reports.\n");
   Printf("SUMMARY: libFuzzer: deadly signal\n");
   DumpCurrentUnit("crash-");
   PrintFinalStats();
   _Exit(Options.ErrorExitCode); // Stop right now.
 }
 
 void Fuzzer::ExitCallback() {
   if (!RunningCB)
     return; // This exit did not come from the user callback
   if (EF->__sanitizer_acquire_crash_state &&
       !EF->__sanitizer_acquire_crash_state())
     return;
   Printf("==%lu== ERROR: libFuzzer: fuzz target exited\n", GetPid());
-  if (EF->__sanitizer_print_stack_trace)
-    EF->__sanitizer_print_stack_trace();
+  PrintStackTrace();
   Printf("SUMMARY: libFuzzer: fuzz target exited\n");
   DumpCurrentUnit("crash-");
   PrintFinalStats();
   _Exit(Options.ErrorExitCode);
 }
 
 void Fuzzer::MaybeExitGracefully() {
   if (!GracefulExitRequested) return;
if (EF->__sanitizer_acquire_crash_state &&
         !EF->__sanitizer_acquire_crash_state())
       return;
     Printf("ALARM: working on the last Unit for %zd seconds\n", Seconds);
     Printf("       and the timeout value is %d (use -timeout=N to change)\n",
            Options.UnitTimeoutSec);
     DumpCurrentUnit("timeout-");
     Printf("==%lu== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),
            Seconds);
-    if (EF->__sanitizer_print_stack_trace)
-      EF->__sanitizer_print_stack_trace();
+    PrintStackTrace();
     Printf("SUMMARY: libFuzzer: timeout\n");
     PrintFinalStats();
     _Exit(Options.TimeoutExitCode); // Stop right now.
   }
 }
 
 void Fuzzer::RssLimitCallback() {
   if (EF->__sanitizer_acquire_crash_state &&
       !EF->__sanitizer_acquire_crash_state())
     return;
   Printf(
       "==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
       GetPid(), GetPeakRSSMb(), Options.RssLimitMb);
   Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
-  if (EF->__sanitizer_print_memory_profile)
-    EF->__sanitizer_print_memory_profile(95, 8);
+  PrintMemoryProfile();
   DumpCurrentUnit("oom-");
   Printf("SUMMARY: libFuzzer: out-of-memory\n");
   PrintFinalStats();
   _Exit(Options.ErrorExitCode); // Stop right now.
 }
 
 void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) {
   size_t ExecPerSec = execPerSec();

compiler-rt/lib/fuzzer/FuzzerUtil.h

 bool IsASCII(const uint8_t *Data, size_t Size);
 
 std::string Base64(const Unit &U);
 
 void PrintPC(const char *SymbolizedFMT, const char *FallbackFMT, uintptr_t PC);
 
 std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC);
 
+void PrintStackTrace();
+
+void PrintMemoryProfile();
+
 unsigned NumberOfCpuCores();
 
 // Platform specific functions.
 void SetSignalHandler(const FuzzingOptions& Options);
 
 void SleepSeconds(int Seconds);
 
 unsigned long GetPid();

compiler-rt/lib/fuzzer/FuzzerUtil.cpp

 
 #include "FuzzerUtil.h"
 #include "FuzzerIO.h"
 #include "FuzzerInternal.h"
 #include <cassert>
 #include <chrono>
 #include <cstring>
 #include <errno.h>
+#include <mutex>
 #include <signal.h>
 #include <sstream>
 #include <stdio.h>
 #include <sys/types.h>
 #include <thread>
 
 namespace fuzzer {
 
if (i + 1 == U.size()) {
     Res += Table[(x >> 18) & 63];
     Res += Table[(x >> 12) & 63];
     Res += Table[(x >> 6) & 63];
     Res += "=";
   }
   return Res;
 }
 
+static std::mutex SymbolizeMutex;
+
 std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC) {
-  if (!EF->__sanitizer_symbolize_pc) return "<can not symbolize>";
+  std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock);
+  if (!EF->__sanitizer_symbolize_pc || !l.owns_lock())
+    return "<can not symbolize>";
   char PcDescr[1024] = {};
   EF->__sanitizer_symbolize_pc(reinterpret_cast<void*>(PC),
                                SymbolizedFMT, PcDescr, sizeof(PcDescr));
   PcDescr[sizeof(PcDescr) - 1] = 0;  // Just in case.
   return PcDescr;
 }
 
 void PrintPC(const char *SymbolizedFMT, const char *FallbackFMT, uintptr_t PC) {
   if (EF->__sanitizer_symbolize_pc)
     Printf("%s", DescribePC(SymbolizedFMT, PC).c_str());
   else
     Printf(FallbackFMT, PC);
 }
 
+void PrintStackTrace() {
+  std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock);
+  if (EF->__sanitizer_print_stack_trace && l.owns_lock())
+    EF->__sanitizer_print_stack_trace();
+}
+
+void PrintMemoryProfile() {
+  std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock);
+  if (EF->__sanitizer_print_memory_profile && l.owns_lock())
+    EF->__sanitizer_print_memory_profile(95, 8);
+}
+
 unsigned NumberOfCpuCores() {
   unsigned N = std::thread::hardware_concurrency();
   if (!N) {
     Printf("WARNING: std::thread::hardware_concurrency not well defined for "
            "your platform. Assuming CPU count of 1.\n");
     N = 1;
   }
   return N;