[fuzzer] Fix threaded stack printing and nested mallocs

[vitalybuka]

Summary

Nested mallocs are possible with internal symbolizer.

[vitalybuka]

ping

[kcc]

LGTM

compiler-rt/trunk/lib/fuzzer/FuzzerLoop.cpp

 #include "FuzzerInternal.h"
 #include "FuzzerMutate.h"
 #include "FuzzerRandom.h"
 #include "FuzzerShmem.h"
 #include "FuzzerTracePC.h"
 #include <algorithm>
 #include <cstring>
 #include <memory>
+#include <mutex>
 #include <set>
 
 #if defined(__has_include)
 #if __has_include(<sanitizer / lsan_interface.h>)
 #include <sanitizer/lsan_interface.h>
 #endif
 #endif
 
struct MallocFreeTracer {
   }
   std::atomic<size_t> Mallocs;
   std::atomic<size_t> Frees;
   int TraceLevel = 0;
 };
 
 static MallocFreeTracer AllocTracer;
 
+static thread_local bool IsMallocFreeHookDisabled;
+static std::mutex MallocFreeStackMutex;
+
+struct MallocFreeHookDisabler {
+  MallocFreeHookDisabler() { IsMallocFreeHookDisabled = true; }
+  ~MallocFreeHookDisabler() { IsMallocFreeHookDisabled = false; }
+};
+
 ATTRIBUTE_NO_SANITIZE_MEMORY
 void MallocHook(const volatile void *ptr, size_t size) {
+  // Avoid nested hooks for mallocs/frees in sanitizer.
+  if (IsMallocFreeHookDisabled)
+    return;
+  MallocFreeHookDisabler Disable;
   size_t N = AllocTracer.Mallocs++;
   F->HandleMalloc(size);
   if (int TraceLevel = AllocTracer.TraceLevel) {
+    std::lock_guard<std::mutex> Lock(MallocFreeStackMutex);
     Printf("MALLOC[%zd] %p %zd\n", N, ptr, size);
     if (TraceLevel >= 2 && EF)
       EF->__sanitizer_print_stack_trace();
   }
 }
 
 ATTRIBUTE_NO_SANITIZE_MEMORY
 void FreeHook(const volatile void *ptr) {
+  // Avoid nested hooks for mallocs/frees in sanitizer.
+  if (IsMallocFreeHookDisabled)
+    return;
+  MallocFreeHookDisabler Disable;
   size_t N = AllocTracer.Frees++;
   if (int TraceLevel = AllocTracer.TraceLevel) {
+    std::lock_guard<std::mutex> Lock(MallocFreeStackMutex);
     Printf("FREE[%zd]   %p\n", N, ptr);
     if (TraceLevel >= 2 && EF)
       EF->__sanitizer_print_stack_trace();
   }
 }
 
 // Crash on a single malloc that exceeds the rss limit.
 void Fuzzer::HandleMalloc(size_t Size) {

compiler-rt/trunk/test/fuzzer/TraceMallocThreadedTest.cpp

+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// Check that allocation tracing from different threads does not cause
+// interleaving of stack traces.
+#include <assert.h>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <thread>
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  auto C = [&] {
+    volatile void *a = malloc(5639);
+    free((void *)a);
+  };
+  std::thread T[] = {std::thread(C), std::thread(C), std::thread(C),
+                     std::thread(C), std::thread(C), std::thread(C)};
+  for (auto &X : T)
+    X.join();
+  return 0;
+}

compiler-rt/trunk/test/fuzzer/trace-malloc-threaded.test

+// FIXME: This test infinite loops on darwin because it crashes
+// printing a stack trace repeatedly
+UNSUPPORTED: darwin
+
+RUN: %cpp_compiler %S/TraceMallocThreadedTest.cpp -o %t-TraceMallocThreadedTest
+
+RUN: %t-TraceMallocThreadedTest -trace_malloc=2 -runs=1 2>&1 | FileCheck %s
+CHECK: {{MALLOC\[[0-9]+] +0x[0-9]+ 5639}}
+CHECK-NEXT: {{ +\#0 +}}
+CHECK-NEXT: {{ +\#1 +}}
+CHECK-NEXT: {{ +\#2 +}}
+
+CHECK: {{MALLOC\[[0-9]+] +0x[0-9]+ 5639}}
+CHECK-NEXT: {{ +\#0 +}}
+CHECK-NEXT: {{ +\#1 +}}
+CHECK-NEXT: {{ +\#2 +}}
+
+CHECK: {{MALLOC\[[0-9]+] +0x[0-9]+ 5639}}
+CHECK-NEXT: {{ +\#0 +}}
+CHECK-NEXT: {{ +\#1 +}}
+CHECK-NEXT: {{ +\#2 +}}
+
+CHECK: {{MALLOC\[[0-9]+] +0x[0-9]+ 5639}}
+CHECK-NEXT: {{ +\#0 +}}
+CHECK-NEXT: {{ +\#1 +}}
+CHECK-NEXT: {{ +\#2 +}}
+
+CHECK: {{MALLOC\[[0-9]+] +0x[0-9]+ 5639}}
+CHECK-NEXT: {{ +\#0 +}}
+CHECK-NEXT: {{ +\#1 +}}
+CHECK-NEXT: {{ +\#2 +}}
+
+CHECK: {{MALLOC\[[0-9]+] +0x[0-9]+ 5639}}
+CHECK-NEXT: {{ +\#0 +}}
+CHECK-NEXT: {{ +\#1 +}}
+CHECK-NEXT: {{ +\#2 +}}