Add FuzzerProxy

[aarongreen]

Summary

This change implements the FuzzerRemoteInterface for the local libFuzzer process. When combined with an IPC library, this can allow remote processes to forward their coverage.

This is change 16 of (at least) 20 for cross-process fuzzing support.

[aarongreen]

Rebase

[aarongreen]

Multiprocess fuzzing will not be supported by the libFuzzer maintainers. Fuchsia has implemented a new approach with their Component Fuzzing Framework (RFC-117).

compiler-rt/lib/fuzzer/CMakeLists.txt

 set(LIBFUZZER_BASE_HEADERS
   FuzzerDefs.h
   FuzzerExtFunctions.def
   FuzzerExtFunctions.h
   FuzzerLock.h
   FuzzerMonitor.h
   FuzzerIO.h
   FuzzerPlatform.h
+  FuzzerProxy.h
   FuzzerRemoteInterface.h
   FuzzerUtil.h)
 
 set(LIBFUZZER_SOURCES
   FuzzerCrossOver.cpp
   FuzzerDataFlowTrace.cpp
   FuzzerDriver.cpp
   FuzzerDSORelative.cpp
   FuzzerExtraCounters.cpp
   FuzzerFork.cpp
   FuzzerLoop.cpp
   FuzzerMerge.cpp
   FuzzerMutate.cpp
+  FuzzerProxy.cpp
   FuzzerSHA1.cpp
   FuzzerTracePC.cpp)
 
 set(LIBFUZZER_HEADERS
   FuzzerBuiltins.h
   FuzzerBuiltinsMsvc.h
   FuzzerCommand.h
   FuzzerCorpus.h
   FuzzerDataFlowTrace.h
   FuzzerDictionary.h
   FuzzerDSORelative.h
   FuzzerFlags.def
   FuzzerFork.h
   FuzzerInterface.h
   FuzzerInternal.h
   FuzzerMerge.h
   FuzzerMutate.h
   FuzzerOptions.h
   FuzzerRandom.h
   FuzzerSHA1.h
   FuzzerTracePC.h
   FuzzerValueBitMap.h)
 
-
 include_directories(../../include)
 
 CHECK_CXX_SOURCE_COMPILES("
   static thread_local int blah;
   int main() {
   return 0;
   }
   " HAS_THREAD_LOCAL)

compiler-rt/lib/fuzzer/FuzzerInternal.h

 #ifndef LLVM_FUZZER_INTERNAL_H
 #define LLVM_FUZZER_INTERNAL_H
 
 #include "FuzzerDataFlowTrace.h"
 #include "FuzzerDefs.h"
 #include "FuzzerExtFunctions.h"
 #include "FuzzerInterface.h"
 #include "FuzzerOptions.h"
+#include "FuzzerProxy.h"
 #include "FuzzerSHA1.h"
 #include "FuzzerValueBitMap.h"
 #include <algorithm>
 #include <atomic>
 #include <chrono>
 #include <climits>
 #include <cstdlib>
 #include <string.h>
public:
   void PrintFinalStats();
   void SetMaxInputLen(size_t MaxInputLen);
   void SetMaxMutationLen(size_t MaxMutationLen);
 
   bool InFuzzingThread() const { return IsMyThread; }
   size_t GetCurrentUnitInFuzzingThead(const uint8_t **Data) const;
   void TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size,
                                bool DuringInitialCorpusExecution);
+  void DetectRemoteLeaksAtExit();
 
   void HandleMalloc(size_t Size);
   static void MaybeExitGracefully();
   std::string WriteToOutputCorpus(const Unit &U);
 
   void AlarmCallback();
   void CrashCallback(unsigned long PID);
   void DeathCallback();
private:
 
   system_clock::time_point LastAllocatorPurgeAttemptTime = system_clock::now();
 
   UserCallback CB;
   InputCorpus &Corpus;
   MutationDispatcher &MD;
   FuzzingOptions Options;
   DataFlowTrace DFT;
+  Proxy *Remote;
 
   system_clock::time_point ProcessStartTime = system_clock::now();
   system_clock::time_point UnitStartTime, UnitStopTime;
   long TimeOfLongestUnitInSeconds = 0;
   long EpochOfLastReadOfOutputCorpus = 0;
 
   size_t MaxInputLen = 0;
   size_t MaxMutationLen = 0;

compiler-rt/lib/fuzzer/FuzzerLoop.cpp

if (Options.Verbosity)
     TPC.PrintModuleInfo();
   if (!Options.OutputCorpus.empty() && Options.ReloadIntervalSec)
     EpochOfLastReadOfOutputCorpus = GetEpoch(Options.OutputCorpus);
   MaxInputLen = MaxMutationLen = Options.MaxLen;
   TmpMaxMutationLen = 0;  // Will be set once we load the corpus.
   AllocateCurrentUnitData();
   CurrentUnitSize = 0;
   memset(BaseSha1, 0, sizeof(BaseSha1));
+  Remote = Proxy::GetInstance();
+  Remote->SetOptions(Options);
 }
 
 Fuzzer::~Fuzzer() {}
 
 void Fuzzer::AllocateCurrentUnitData() {
   if (CurrentUnitData || MaxInputLen == 0)
     return;
   CurrentUnitData = new uint8_t[MaxInputLen];
if (Seconds >= (size_t)Options.UnitTimeoutSec) {
     Printf("       and the timeout value is %d (use -timeout=N to change)\n",
            Options.UnitTimeoutSec);
     DumpCurrentUnit("timeout-");
     Printf("==%lu== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),
            Seconds);
     PrintStackTrace();
     Printf("SUMMARY: libFuzzer: timeout\n");
     PrintFinalStats();
+    Remote->Shutdown();
     _Exit(Options.TimeoutExitCode); // Stop right now.
   }
 }
 
 void Fuzzer::OutOfMemoryCallback(unsigned long PID, size_t LastMalloc) {
   if (EF->__sanitizer_acquire_crash_state &&
       !EF->__sanitizer_acquire_crash_state())
     return;
void Fuzzer::LeakCallback(unsigned long PID) {
   if (ExecutingSeedCorpora)
     Printf("\nINFO: a leak has been found in the initial corpus.\n\n");
   Printf("INFO: to ignore leaks on libFuzzer side use -detect_leaks=0.\n\n");
   DumpCurrentUnit("leak-");
   PrintFinalStats();
   _Exit(Options.ErrorExitCode); // not exit() to disable lsan further on.
 }
 
-unsigned long Fuzzer::GetPid(unsigned long PID) { return fuzzer::GetPid(); }
+unsigned long Fuzzer::GetPid(unsigned long PID) {
+  return PID ? PID : fuzzer::GetPid();
+}
 
-void Fuzzer::PrintStackTrace(unsigned long PID) { fuzzer::PrintStackTrace(); }
+void Fuzzer::PrintStackTrace(unsigned long PID) {
+  if (PID)
+    Remote->PrintStackTrace(PID);
+  else
+    fuzzer::PrintStackTrace();
+}
 
 void Fuzzer::PrintMemoryProfile(unsigned long PID) {
+  if (PID)
+    Remote->PrintMemoryProfile(PID);
+  else
     fuzzer::PrintMemoryProfile();
 }
 
 void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units,
                         size_t Features) {
   size_t ExecPerSec = execPerSec();
   if (!Options.Verbosity)
     return;
   Printf("#%zd\t%s", TotalNumberOfRuns, Where);
static void WriteEdgeToMutationGraphFile(const std::string &MutationGraphFile,
   AppendToFile(OutputString, MutationGraphFile);
 }
 
 bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
                     InputInfo *II, bool ForceAddToCorpus,
                     bool *FoundUniqFeatures) {
   if (!Size)
     return false;
-
   ExecuteCallback(Data, Size);
   auto TimeOfUnit = duration_cast<microseconds>(UnitStopTime - UnitStartTime);
 
   UniqFeatureSetTmp.clear();
   size_t FoundUniqFeaturesOfII = 0;
   size_t NumUpdatesBefore = Corpus.NumFeatureUpdates();
   TPC.CollectFeatures([&](size_t Feature) {
     if (Corpus.AddFeature(Feature, Size, Options.Shrink))
if (EF->__msan_unpoison)
     EF->__msan_unpoison(DataCopy, Size);
   if (EF->__msan_unpoison_param)
     EF->__msan_unpoison_param(2);
   if (CurrentUnitData && CurrentUnitData != Data)
     memcpy(CurrentUnitData, Data, Size);
   CurrentUnitSize = Size;
   {
     ScopedEnableMsanInterceptorChecks S;
+    Remote->StartExecution();
     AllocMonitorStartTracing(Options.TraceMalloc);
     UnitStartTime = system_clock::now();
     TPC.ResetMaps();
     RunningUserCallback = true;
     int Res = CB(DataCopy, Size);
     RunningUserCallback = false;
     UnitStopTime = system_clock::now();
     (void)Res;
     assert(Res == 0);
     HasMoreMallocsThanFrees = AllocMonitorStopTracing();
+    HasMoreMallocsThanFrees |= Remote->FinishExecution();
   }
   if (!LooseMemeq(DataCopy, Data, Size))
     CrashOnOverwrittenData();
   CurrentUnitSize = 0;
   delete[] DataCopy;
 }
 
 std::string Fuzzer::WriteToOutputCorpus(const Unit &U) {
if (!DuringInitialCorpusExecution &&
     return;
   if (!&(EF->__lsan_enable) || !&(EF->__lsan_disable) ||
       !(EF->__lsan_do_recoverable_leak_check))
     return; // No lsan.
   // Run the target once again, but with lsan disabled so that if there is
   // a real leak we do not report it twice.
   ExecutingSeedCorpora = DuringInitialCorpusExecution;
   EF->__lsan_disable();
+  Remote->TryDetectingAMemoryLeak();
   ExecuteCallback(Data, Size);
   EF->__lsan_enable();
   if (!HasMoreMallocsThanFrees)
     return; // a leak is unlikely.
   if (NumberOfLeakDetectionAttempts++ > 1000) {
     Options.DetectLeaks = false;
     Printf("INFO: libFuzzer disabled leak detection after every mutation.\n"
            "      Most likely the target function accumulates allocated\n"
            "      memory in a global state w/o actually leaking it.\n"
            "      You may try running this binary with -trace_malloc=[12]"
            "      to get a trace of mallocs and frees.\n"
            "      If LeakSanitizer is enabled in this process it will still\n"
            "      run on the process shutdown.\n");
     return;
   }
   // Now perform the actual lsan pass. This is expensive and we must ensure
   // we don't call it too often.
   if (EF->__lsan_do_recoverable_leak_check()) { // Leak is found, report it.
     CurrentUnitSize = Size;
     LeakCallback(0);
   }
 }
 
+void Fuzzer::DetectRemoteLeaksAtExit() { Remote->DetectLeaksAtExit(); }
+
 void Fuzzer::MutateAndTestOne() {
   MD.StartMutationSequence();
 
   auto &II = Corpus.ChooseUnitToMutate(MD.GetRand());
   if (Options.DoCrossOver) {
     auto &CrossOverII = Corpus.ChooseUnitToCrossOverWith(
         MD.GetRand(), Options.CrossOverUniformDist);
     MD.SetCrossOverWith(&CrossOverII.U);

compiler-rt/lib/fuzzer/FuzzerProxy.h

+//===- FuzzerProxy.h --------------------------------------------*- C++ -* ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Proxy interface used by libFuzzer.
+//
+// When fuzzing multipe processes communicating via IPC, the process linked
+// against libFuzzer is considered the "proxy", and the others are "remote"
+// processes.
+//
+// The FuzzerProxy can be called directly by libFuzzer, and by the remote
+// processes via its implementations of the FuzzerProxy* interface methods. It
+// can also invoke remote methods using the FuzzerRemote* interface methods.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FUZZER_PROXY_H
+#define LLVM_FUZZER_PROXY_H
+
+#include "FuzzerDefs.h"
+#include "FuzzerLock.h"
+#include "FuzzerOptions.h"
+#include "FuzzerRemoteInterface.h"
+#include "FuzzerTracePC.h"
+#include <atomic>
+#include <condition_variable>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <unordered_map>
+
+namespace fuzzer {
+
+class Proxy final {
+public:
+  // In order to uniquely map PCs back to individual remote processes, the PCs
+  // they report are mapped into separate ranges. This is currently achieved by
+  // using the 16 unused high bits of x86_64 and ARMv8 virtual addresses to
+  // index which process the PCs belong to.
+  static constexpr int kPCShift = 48;
+  static constexpr uint64_t kPCMask = (1ULL << kPCShift) - 1;
+
+  static Proxy *GetInstance();
+  ~Proxy();
+
+  // Setup methods
+  void SetOptions(const FuzzingOptions &Options) EXCLUDES(Mutex);
+
+  // Connection-related methods
+  void Connect(unsigned long PID, void *Options, size_t OptionsLen)
+      EXCLUDES(Mutex);
+  void Disconnect(unsigned long PID) EXCLUDES(Mutex);
+
+  // Coverage-related methods
+  uintptr_t AddCoverage(unsigned long PID, uint8_t *CountersBegin,
+                        uint8_t *CountersEnd, const uintptr_t *PCsBegin,
+                        const uintptr_t *PCsEnd) EXCLUDES(Mutex);
+  Vector<uintptr_t> GetIndicesForPID(unsigned long PID) EXCLUDES(Mutex);
+
+  // These return true if the PC was recognized as belonging to a remote process
+  // and a request sent; otherwise false.
+  static bool PrintPC(const char *SymbolizedFMT, const char *FallbackFMT,
+                      uintptr_t PC);
+  static bool DescribePC(const char *SymbolizedFMT, uintptr_t PC, char *Desc,
+                         size_t DescLen);
+
+  // Execution callback related methods
+  void TryDetectingAMemoryLeak();
+  void StartExecution() EXCLUDES(Mutex);
+  void ExecutionStarted(unsigned long PID) EXCLUDES(Mutex);
+  bool FinishExecution() EXCLUDES(Mutex);
+  void ExecutionFinished(unsigned long PID, bool HasMoreMallocsThanFrees)
+      EXCLUDES(Mutex);
+  void PrintStackTrace(unsigned long PID);
+  void PrintMemoryProfile(unsigned long PID);
+
+  // Teardown methods
+  void DetectLeaksAtExit();
+  void Shutdown() EXCLUDES(Mutex);
+
+private:
+  Proxy();
+  void StopLocked() REQUIRES(Mutex);
+
+  // Coverage-related methods
+  unsigned long GetPidForPCLocked(uintptr_t PC) REQUIRES(Mutex);
+  bool DescribePCImpl(const char *SymbolizedFMT, uintptr_t PC, char *Desc,
+                      size_t DescLen) EXCLUDES(Mutex);
+  bool PrintPCImpl(const char *SymbolizedFMT, const char *FallbackFMT,
+                   uintptr_t PC) EXCLUDES(Mutex);
+
+  // Execution callback related methods
+  void WaitForResponses() EXCLUDES(Mutex);
+  void ExecutionStartedLocked(unsigned long PID) REQUIRES(Mutex);
+  void ExecutionFinishedLocked(unsigned long PID, bool HasMoreMallocsThanFrees)
+      REQUIRES(Mutex);
+
+  // If false, all calls to the proxy return immediately. Set by SetOptions.
+  std::atomic<bool> Enabled;
+
+  // Calls from libFuzzer and the FuzzerProxy* interface methods may be
+  // interleaved.
+  std::mutex Mutex;
+  bool Verbose GUARDED_BY(Mutex);
+  ProxiedOptions Proxied GUARDED_BY(Mutex);
+
+  // Maps PIDs of connected remote processes to coverage and execution details.
+  struct Remote {
+    bool Executing;
+    uint64_t PCOffset;
+    Vector<uintptr_t> Indices;
+  };
+  std::unordered_map<unsigned long, Remote> PidToRemote GUARDED_BY(Mutex);
+  std::condition_variable ConnectionCV;
+
+  // Maps ranges of PCs to PIDs of connected remote processes.
+  std::map<uint64_t, unsigned long> PCOffsetToPid GUARDED_BY(Mutex);
+
+  // Offset to add to PCs from the next process to connect.
+  uint64_t NextPCOffset = 1ULL << kPCShift;
+
+  // Tables for remote process PCs that lives as long as the fuzzer can can be
+  // "recycled" for remote processes that disconnect and subsequently respawn.
+  Vector<std::unique_ptr<TracePC::PCTableEntry[]>> AllRebasedPCs;
+
+  // Maps hashes of DSOs from disconnected remote processes to PCTable indices.
+  std::unordered_multimap<uint64_t, uintptr_t>
+      HashToRecyclableIdx GUARDED_BY(Mutex);
+
+  // Varibles used to control execution of a single fuzzing iteration.
+  bool FirstIteration GUARDED_BY(Mutex) = true;
+  bool Executing GUARDED_BY(Mutex) = false;
+  bool DetectLeaks GUARDED_BY(Mutex) = false;
+  bool LeakLikely GUARDED_BY(Mutex) = false;
+  std::condition_variable CompletionCV;
+
+  FUZZER_NO_COPY_OR_MOVE(Proxy);
+};
+
+} // namespace fuzzer
+
+#endif // LLVM_FUZZER_PROXY_H

compiler-rt/lib/fuzzer/FuzzerProxy.cpp

+//===- FuzzerProxy.cpp ------------------------------------------*- C++ -* ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// FuzzerRemoteInterface implementation for the fuzzing engine.
+//
+// When Options.Remote=1, libFuzzer acts as a fuzzing "proxy" that can be used
+// to fuzz multiple processes ("remotes") communicating via IPC.
+//
+// The FuzzerProxy can be called directly by libFuzzer, and by the remote
+// processes via its implementations of the FuzzerProxy* interface methods. It
+// can also invoke remote methods using the FuzzerRemote* interface methods.
+//===----------------------------------------------------------------------===//
+
+#include "FuzzerProxy.h"
+#include "FuzzerDSORelative.h"
+#include "FuzzerExtFunctions.h"
+#include "FuzzerIO.h"
+#include "FuzzerInternal.h"
+#include "FuzzerMonitor.h"
+#include <algorithm>
+
+extern "C" {
+
+void __sanitizer_cov_8bit_counters_init(uint8_t *, uint8_t *);
+void __sanitizer_cov_pcs_init(const uintptr_t *, const uintptr_t *);
+
+} // extern "C"
+
+namespace fuzzer {
+
+constexpr int Proxy::kPCShift;
+constexpr uint64_t Proxy::kPCMask;
+
+Proxy *Proxy::GetInstance() {
+  static Proxy Singleton;
+  return &Singleton;
+}
+
+Proxy::Proxy() : Enabled(false) {}
+
+Proxy::~Proxy() { Shutdown(); }
+
+void Proxy::SetOptions(const FuzzingOptions &Options) {
+  {
+    // Require no connections when setting options.
+    std::lock_guard<std::mutex> Lock(Mutex);
+    assert(PidToRemote.empty());
+    Verbose = Options.Verbosity >= 2;
+    Proxied.Pack(Options);
+  }
+  if (!Options.UseRemote)
+    return;
+  if (sizeof(uintptr_t) < sizeof(uint64_t)) {
+    Printf("INFO: FuzzerProxy requires 64-bit support.\n"
+           "      Ignoring remote processes.\n");
+    return;
+  }
+  if (!EF->FuzzerAcceptRemotes || !EF->FuzzerShutdownRemotes ||
+      !EF->FuzzerRemoteStartExecution || !EF->FuzzerRemoteFinishExecution ||
+      !EF->FuzzerRemotePrintPC || !EF->FuzzerRemoteDescribePC ||
+      !EF->FuzzerRemotePrintStackTrace || !EF->FuzzerRemotePrintMemoryProfile ||
+      !EF->FuzzerRemoteDetectLeaksAtExit) {
+    Printf("ERROR: FuzzerRemote method(s) missing.\n");
+    exit(1);
+  }
+  EF->FuzzerAcceptRemotes();
+  Enabled = true;
+}
+
+// Connection-related methods
+
+void Proxy::Connect(unsigned long PID, void *Options, size_t OptionsLen) {
+  if (!Enabled)
+    return;
+  {
+    std::lock_guard<std::mutex> Lock(Mutex);
+    PidToRemote.emplace(PID, Remote{Executing, 0, {}});
+    if (OptionsLen != sizeof(Proxied)) {
+      VPrintf(Verbose, "==%lu== ERROR: option length mismatch: %zu vs. %zu\n",
+              PID, OptionsLen, sizeof(Proxied));
+      exit(1);
+    }
+    memcpy(Options, &Proxied, OptionsLen);
+  }
+  // See ExecutionFinished, below.
+  ConnectionCV.notify_one();
+}
+
+void Proxy::Disconnect(unsigned long PID) {
+  std::lock_guard<std::mutex> Lock(Mutex);
+  auto I = PidToRemote.find(PID);
+  if (I == PidToRemote.end())
+    return;
+  auto &Remote = I->second;
+  if (Remote.Executing) {
+    ExecutionFinishedLocked(I->first, /* HasMoreMallocsThanFrees= */ false);
+    ConnectionCV.notify_one();
+  }
+  if (Remote.PCOffset) {
+    for (auto Idx : Remote.Indices) {
+      struct DSOInfo Info;
+      TPC.DSOInfoByIdx(Idx, &Info);
+      HashToRecyclableIdx.insert({{Info.Hash, Idx}});
+    }
+    PCOffsetToPid.erase(Remote.PCOffset);
+  }
+  PidToRemote.erase(I);
+}
+
+// Coverage-related methods
+
+static bool CoverageMatches(const struct DSOInfo &Info,
+                            const TracePC::PCTableEntry *TE, size_t NumTEs,
+                            uint64_t *PCOffset) {
+  if (Info.LastIdx - Info.FirstIdx != NumTEs)
+    return false;
+  auto *Other = TPC.PCTableEntryByIdx(Info.FirstIdx);
+  uint64_t Delta = Other->PC - TE->PC;
+  for (size_t i = 0; i < NumTEs; ++i)
+    if (TE[i].PC + Delta != Other[i].PC || TE[i].PCFlags != Other[i].PCFlags)
+      return false;
+  *PCOffset = Delta;
+  return true;
+}
+
+uintptr_t Proxy::AddCoverage(unsigned long PID, uint8_t *CountersBegin,
+                             uint8_t *CountersEnd, const uintptr_t *PCsBegin,
+                             const uintptr_t *PCsEnd) {
+  if (!Enabled)
+    return kInvalidIdx;
+  std::lock_guard<std::mutex> Lock(Mutex);
+  auto I = PidToRemote.find(PID);
+  if (I == PidToRemote.end()) {
+    VPrintf(Verbose, "WARNING: Ignoring unconnected process: %lu\n", PID);
+    return kInvalidIdx;
+  }
+  if (!CountersBegin || !CountersEnd || CountersEnd <= CountersBegin) {
+    VPrintf(Verbose,
+            "WARNING: Ignoring invalid inline 8-bit counters: [%p, %p)\n",
+            CountersBegin, CountersEnd);
+    return kInvalidIdx;
+  }
+  if (!PCsBegin || !PCsEnd || PCsEnd <= PCsBegin) {
+    VPrintf(Verbose, "WARNING: Ignoring invalid PC table: [%p, %p)\n", PCsBegin,
+            PCsEnd);
+    return kInvalidIdx;
+  }
+  auto *B = reinterpret_cast<const TracePC::PCTableEntry *>(PCsBegin);
+  auto *E = reinterpret_cast<const TracePC::PCTableEntry *>(PCsEnd);
+  size_t NumPCs = E - B;
+  uint64_t PCOffset = 0;
+  uintptr_t Idx = kInvalidIdx;
+
+  // First check if this processes already has this DSO mapped.
+  auto Hash = TracePC::HashPCs(PCsBegin, PCsEnd);
+  for (auto ActiveIdx : I->second.Indices) {
+    struct DSOInfo Info;
+    TPC.DSOInfoByIdx(ActiveIdx, &Info);
+    if (Info.Hash == Hash && CoverageMatches(Info, B, NumPCs, &PCOffset))
+      return ActiveIdx;
+  }
+
+  // This process may be one that ran before, exited, and is now restarting.
+  // So try next to "recycle" the counters from a previously exited process.
+  auto Range = HashToRecyclableIdx.equal_range(Hash);
+  for (auto J = Range.first; J != Range.second; ++J) {
+    struct DSOInfo Info;
+    auto RecyclableIdx = J->second;
+    TPC.DSOInfoByIdx(RecyclableIdx, &Info);
+    if (CoverageMatches(Info, B, NumPCs, &PCOffset)) {
+      // Coverage matches that from a process that previously exited. Replace
+      //  the counters and reuse the PC tables.
+      TPC.HandleInline8bitCountersInit(CountersBegin, CountersEnd,
+                                       Info.PCTableOffset);
+      Idx = RecyclableIdx;
+      HashToRecyclableIdx.erase(J);
+      break;
+    }
+  }
+
+  // Fallback to adding the coverage as normal after rebasing the PC values to
+  // ensure they are unique. DescribePC and PrintPC reverse this shift when
+  // sending values back to the proxied process.
+  if (Idx == kInvalidIdx) {
+    if (!TPC.HandleInline8bitCountersInit(CountersBegin, CountersEnd))
+      return kInvalidIdx;
+    PCOffset = NextPCOffset;
+    NextPCOffset += 1ULL << kPCShift;
+    std::unique_ptr<TracePC::PCTableEntry[]> RebasedPCs(
+        new TracePC::PCTableEntry[NumPCs]);
+    for (size_t i = 0; i < NumPCs; ++i) {
+      auto &Src = B[i];
+      auto &Dst = RebasedPCs[i];
+      assert((Src.PC & ~kPCMask) == 0); // Ensure we can rebase.
+      Dst.PC = Src.PC + PCOffset;
+      Dst.PCFlags = Src.PCFlags;
+    }
+    auto *RebasedBegin = reinterpret_cast<uintptr_t *>(RebasedPCs.get());
+    auto *RebasedEnd = RebasedBegin + (PCsEnd - PCsBegin);
+    TPC.HandlePCsInit(RebasedBegin, RebasedEnd, Hash);
+    Idx = TPC.PCTableEntryIdx(RebasedPCs.get());
+    AllRebasedPCs.push_back(std::move(RebasedPCs));
+  }
+  PCOffsetToPid[PCOffset] = PID;
+
+  auto &Remote = I->second;
+  Remote.PCOffset = PCOffset;
+  Remote.Indices.push_back(Idx);
+  return Idx;
+}
+
+Vector<uintptr_t> Proxy::GetIndicesForPID(unsigned long PID) {
+  std::lock_guard<std::mutex> Lock(Mutex);
+  auto I = PidToRemote.find(PID);
+  if (I == PidToRemote.end())
+    return Vector<uintptr_t>();
+  return I->second.Indices;
+}
+
+unsigned long Proxy::GetPidForPCLocked(uintptr_t PC) {
+  auto P = PCOffsetToPid.find(PC & ~kPCMask);
+  return P == PCOffsetToPid.end() ? 0 : P->second;
+}
+
+bool Proxy::PrintPC(const char *SymbolizedFMT, const char *FallbackFMT,
+                    uintptr_t PC) {
+  auto *Proxy = fuzzer::Proxy::GetInstance();
+  return Proxy->PrintPCImpl(SymbolizedFMT, FallbackFMT, PC);
+}
+
+bool Proxy::PrintPCImpl(const char *SymbolizedFMT, const char *FallbackFMT,
+                        uintptr_t PC) {
+  if (!Enabled)
+    return false;
+  std::lock_guard<std::mutex> Lock(Mutex);
+  unsigned long PID = GetPidForPCLocked(PC);
+  if (PID == 0)
+    return false;
+  EF->FuzzerRemotePrintPC(PID, SymbolizedFMT, FallbackFMT, PC & kPCMask);
+  return true;
+}
+
+bool Proxy::DescribePC(const char *SymbolizedFMT, uintptr_t PC, char *Desc,
+                       size_t DescLen) {
+  auto *Proxy = fuzzer::Proxy::GetInstance();
+  return Proxy->DescribePCImpl(SymbolizedFMT, PC, Desc, DescLen);
+}
+
+bool Proxy::DescribePCImpl(const char *SymbolizedFMT, uintptr_t PC, char *Desc,
+                           size_t DescLen) {
+  if (!Enabled)
+    return false;
+  std::lock_guard<std::mutex> Lock(Mutex);
+  unsigned long PID = GetPidForPCLocked(PC);
+  if (PID == 0)
+    return false;
+  EF->FuzzerRemoteDescribePC(PID, SymbolizedFMT, PC & kPCMask, Desc, DescLen);
+  return true;
+}
+
+// Execution callback related methods
+
+void Proxy::TryDetectingAMemoryLeak() {
+  std::lock_guard<std::mutex> Lock(Mutex);
+  DetectLeaks = true;
+}
+
+void Proxy::StartExecution() {
+  if (!Enabled)
+    return;
+  {
+    std::lock_guard<std::mutex> Lock(Mutex);
+    Executing = true;
+    uint32_t Flags = 0;
+    Flags |= DetectLeaks ? kLeakDetection : 0;
+    // Need a proper iterator to be able to mutate the map.
+    for (auto I = PidToRemote.begin(); I != PidToRemote.end(); ++I)
+      EF->FuzzerRemoteStartExecution(I->first, Flags);
+  }
+  WaitForResponses();
+}
+
+bool Proxy::FinishExecution() {
+  if (!Enabled)
+    return false;
+  {
+    UniqueLock Lock(Mutex);
+    Executing = false;
+    LeakLikely = false;
+    for (auto I : PidToRemote)
+      if (I.second.Executing)
+        EF->FuzzerRemoteFinishExecution(I.first);
+  }
+  WaitForResponses();
+  {
+    std::lock_guard<std::mutex> Lock(Mutex);
+    DetectLeaks = false;
+    return LeakLikely;
+  }
+}
+
+void Proxy::WaitForResponses() {
+  UniqueLock Lock(Mutex);
+  // Remote processes typcially won't be started before the first iteration. If
+  // there aren't any, it could either be because one or more processes are
+  // still starting up and connecting, or because the remote process isn't
+  // instrumented. The former case is a simple matter of waiting, but the latter
+  // is an error condition caught by Fuzzer::ReadAndExecuteSeedCorpora.
+  // Unfortunately, at this point there's no way of knowing which case it is.
+  // The best we can do is play it safe and wait a bit, hoping for a connection.
+  if (FirstIteration)
+    FirstIteration = false;
+  else if (!ConnectionCV.wait_for(
+               Lock.get(), std::chrono::seconds(3),
+               [&]() REQUIRES(Mutex) { return !PidToRemote.empty(); }))
+    return;
+
+  while (true) {
+    // Get a new iterator each time; other calls while waiting may have
+    // invalidated the previous one.
+    auto I = PidToRemote.begin();
+    while (I != PidToRemote.end() && I->second.Executing == Executing)
+      I++;
+    if (I == PidToRemote.end())
+      break;
+    CompletionCV.wait(Lock.get());
+  }
+}
+
+void Proxy::ExecutionStarted(unsigned long PID) {
+  if (!Enabled)
+    return;
+  std::lock_guard<std::mutex> Lock(Mutex);
+  auto I = PidToRemote.find(PID);
+  if (I == PidToRemote.end())
+    return;
+  I->second.Executing = true;
+  CompletionCV.notify_one();
+}
+
+void Proxy::ExecutionFinished(unsigned long PID, bool HasMoreMallocsThanFrees) {
+  if (!Enabled)
+    return;
+  std::lock_guard<std::mutex> Lock(Mutex);
+  ExecutionFinishedLocked(PID, HasMoreMallocsThanFrees);
+}
+
+void Proxy::ExecutionFinishedLocked(unsigned long PID,
+                                    bool HasMoreMallocsThanFrees) {
+  auto I = PidToRemote.find(PID);
+  if (I == PidToRemote.end())
+    return;
+  I->second.Executing = false;
+  LeakLikely |= HasMoreMallocsThanFrees;
+  CompletionCV.notify_one();
+}
+
+void Proxy::PrintStackTrace(unsigned long PID) {
+  if (!Enabled)
+    return;
+  EF->FuzzerRemotePrintStackTrace(PID);
+}
+
+void Proxy::PrintMemoryProfile(unsigned long PID) {
+  if (!Enabled)
+    return;
+  EF->FuzzerRemotePrintMemoryProfile(PID);
+}
+
+void Proxy::DetectLeaksAtExit() {
+  if (!Enabled)
+    return;
+  if (EF && EF->FuzzerRemoteDetectLeaksAtExit) {
+    std::lock_guard<std::mutex> Lock(Mutex);
+    for (auto &C : PidToRemote)
+      EF->FuzzerRemoteDetectLeaksAtExit(C.first);
+  }
+}
+
+void Proxy::Shutdown() {
+  if (!Enabled)
+    return;
+  if (EF && EF->FuzzerShutdownRemotes)
+    EF->FuzzerShutdownRemotes();
+  Enabled = false;
+}
+
+} // namespace fuzzer
+
+extern "C" {
+
+// libFuzzer's implementation of the FuzzerProxy* interface.
+
+ATTRIBUTE_INTERFACE void FuzzerProxyConnect(unsigned long PID, void *Options,
+                                            size_t OptionsLen) {
+  auto *Proxy = fuzzer::Proxy::GetInstance();
+  Proxy->Connect(PID, Options, OptionsLen);
+}
+
+ATTRIBUTE_INTERFACE uintptr_t FuzzerProxyAddCoverage(unsigned long PID,
+                                                     uint8_t *CountersBegin,
+                                                     uint8_t *CountersEnd,
+                                                     const uintptr_t *PCsBegin,
+                                                     const uintptr_t *PCsEnd) {
+  auto *Proxy = fuzzer::Proxy::GetInstance();
+  return Proxy->AddCoverage(PID, CountersBegin, CountersEnd, PCsBegin, PCsEnd);
+}
+
+ATTRIBUTE_INTERFACE void FuzzerProxyExecutionStarted(unsigned long PID) {
+  auto *Proxy = fuzzer::Proxy::GetInstance();
+  Proxy->ExecutionStarted(PID);
+}
+
+ATTRIBUTE_INTERFACE void
+FuzzerProxyExecutionFinished(unsigned long PID, int HasMoreMallocsThanFrees) {
+  auto *Proxy = fuzzer::Proxy::GetInstance();
+  Proxy->ExecutionFinished(PID, HasMoreMallocsThanFrees);
+}
+
+ATTRIBUTE_INTERFACE void FuzzerProxyDisconnect(unsigned long PID) {
+  auto *Proxy = fuzzer::Proxy::GetInstance();
+  Proxy->Disconnect(PID);
+}
+
+} // extern "C"

compiler-rt/lib/fuzzer/FuzzerRemote.cpp

 // The FuzzerProxy can be called directly by libFuzzer, and by the remote
 // processes via its implementations of the FuzzerProxy* interface methods. It
 // can also invoke remote methods using the FuzzerRemote* interface methods.
 
 #include "FuzzerExtFunctions.h"
 #include "FuzzerLock.h"
 #include "FuzzerMonitor.h"
 #include "FuzzerOptions.h"
+#include "FuzzerProxy.h"
 #include "FuzzerRemoteInterface.h"
 #include "FuzzerTracePC.h"
 #include "FuzzerUtil.h"
 #include <deque>
 
 namespace fuzzer {
 
 // Storage for global ExternalFunctions object.
 ExternalFunctions *EF = nullptr;
 
+// These are referenced by FuzzerUtil.cpp, but always return false in a remote
+// process.
+bool Proxy::PrintPC(const char *SymbolizedFMT, const char *FallbackFMT,
+                    uintptr_t PC) {
+  return false;
+}
+bool Proxy::DescribePC(const char *SymbolizedFMT, uintptr_t PC, char *Desc,
+                       size_t DescLen) {
+  return false;
+}
+
 namespace {
 
 // Singleton that keeps track of coverage initialization and fuzzer loop
 // execution.
 class Remote final {
 public:
   static Remote &GetInstance() {
     static Remote Singleton;

compiler-rt/lib/fuzzer/FuzzerUtil.cpp

if (i + 1 == U.size()) {
     Buffer[j + 3] = '=';
   }
   return Buffer;
 }
 
 static std::mutex SymbolizeMutex;
 
 std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC) {
+  char PcDescr[1024] = {};
+  if (!Proxy::DescribePC(SymbolizedFMT, PC, PcDescr, sizeof(PcDescr))) {
     std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock);
     if (!EF->__sanitizer_symbolize_pc || !l.owns_lock())
       return "<can not symbolize>";
-  char PcDescr[1024] = {};
-  EF->__sanitizer_symbolize_pc(reinterpret_cast<void*>(PC),
-                               SymbolizedFMT, PcDescr, sizeof(PcDescr));
+    EF->__sanitizer_symbolize_pc(reinterpret_cast<void *>(PC), SymbolizedFMT,
+                                 PcDescr, sizeof(PcDescr));
+  }
   PcDescr[sizeof(PcDescr) - 1] = 0;  // Just in case.
   return PcDescr;
 }
 
 void PrintPC(const char *SymbolizedFMT, const char *FallbackFMT, uintptr_t PC) {
+  if (!Proxy::PrintPC(SymbolizedFMT, FallbackFMT, PC))
+    return;
   if (EF->__sanitizer_symbolize_pc)
     Printf("%s", DescribePC(SymbolizedFMT, PC).c_str());
   else
     Printf(FallbackFMT, PC);
 }
 
 void PrintStackTrace() {
   std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock);

compiler-rt/lib/fuzzer/tests/FuzzerUnittest.cpp

 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 // Avoid ODR violations (LibFuzzer is built without ASan and this test is built
 // with ASan) involving C++ standard library types when using libcxx.
 #define _LIBCPP_HAS_NO_ASAN
 
 // Do not attempt to use LLVM ostream etc from gtest.
 #define GTEST_NO_LLVM_SUPPORT 1
 
 #include "FuzzerCorpus.h"
 #include "FuzzerDSORelative.h"
 #include "FuzzerDictionary.h"
 #include "FuzzerFork.h"
 #include "FuzzerInternal.h"
+#include "FuzzerLock.h"
 #include "FuzzerMerge.h"
 #include "FuzzerMutate.h"
+#include "FuzzerProxy.h"
 #include "FuzzerRandom.h"
 #include "FuzzerTracePC.h"
 #include "tests/FuzzerTestUtil.h"
 #include "gtest/gtest.h"
 #include <memory>
 #include <set>
 #include <sstream>
+#include <thread>
 
 using namespace fuzzer;
 
 // For now, have LLVMFuzzerTestOneInput just to make it link.
 // Later we may want to make unittests that actually call LLVMFuzzerTestOneInput.
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   abort();
 }
TEST(Entropic, ComputeEnergy) {
 
   II->FeatureFreqs[0].second++;
   II->FeatureFreqs.push_back(std::pair<uint32_t, uint16_t>(42, 6));
   II->NumExecutedMutations = 20;
   II->UpdateEnergy(10, false, std::chrono::microseconds(0));
   EXPECT_LT(SubAndSquare(II->Energy, 1.792831), Precision);
 }
 
+// Helper struct to disconnect fake remotes on test completion.
+struct ScopedConnection final {
+  explicit ScopedConnection(unsigned long NewPID = 0) : Proxied() {
+    Reset(NewPID);
+  }
+
+  ~ScopedConnection() { Reset(); }
+
+  unsigned long Reset(unsigned long NewPID = 0) {
+    if (PID)
+      FuzzerProxyDisconnect(PID);
+    auto OldPID = PID;
+    PID = NewPID;
+    if (PID)
+      FuzzerProxyConnect(PID, &Proxied, sizeof(Proxied));
+    return OldPID;
+  }
+
+  unsigned long PID = 0;
+  ProxiedOptions Proxied;
+};
+
+TEST(Proxy, SetOptionsAndConnect) {
+  auto *P = Proxy::GetInstance();
+  pid_t PID = GetPid() + 1;
+  std::vector<uint8_t> Zeros(sizeof(ProxiedOptions), 0);
+
+  // All of these differ from their defaults.
+  FuzzingOptions Expected;
+  Expected.Verbosity = 2;
+  Expected.OOMExitCode = 17;
+  Expected.InterruptExitCode = 27;
+  Expected.IgnoreOOMs = false;
+  Expected.IgnoreCrashes = true;
+  Expected.RssLimitMb = 2048;
+  Expected.MallocLimitMb = 256;
+  Expected.UseCounters = true;
+  Expected.UseMemmem = false;
+  Expected.UseCmp = true;
+  Expected.UseValueProfile = 2;
+  Expected.PurgeAllocatorIntervalSec = 10;
+  Expected.TraceMalloc = 2;
+  Expected.HandleAbrt = true;
+  Expected.HandleBus = true;
+  Expected.HandleFpe = true;
+  Expected.HandleIll = true;
+  Expected.HandleInt = true;
+  Expected.HandleSegv = true;
+  Expected.HandleTerm = true;
+
+  // No effect if not enabled.
+  Expected.UseRemote = false;
+  P->SetOptions(Expected);
+
+  ProxiedOptions Proxied;
+  memset(&Proxied, 0, sizeof(Proxied));
+
+  ScopedConnection SC(PID);
+  EXPECT_EQ(memcmp(&SC.Proxied, &Zeros[0], Zeros.size()), 0);
+
+  // Enable the proxy.
+  Expected.UseRemote = true;
+  P->SetOptions(Expected);
+  SC.Reset(PID);
+
+  FuzzingOptions Actual;
+  SC.Proxied.Unpack(&Actual);
+  EXPECT_EQ(Expected.Verbosity, Actual.Verbosity);
+  EXPECT_EQ(Expected.OOMExitCode, Actual.OOMExitCode);
+  EXPECT_EQ(Expected.InterruptExitCode, Actual.InterruptExitCode);
+  EXPECT_EQ(Expected.RssLimitMb, Actual.RssLimitMb);
+  EXPECT_EQ(Expected.MallocLimitMb, Actual.MallocLimitMb);
+  EXPECT_EQ(Expected.PurgeAllocatorIntervalSec,
+            Actual.PurgeAllocatorIntervalSec);
+  EXPECT_EQ(Expected.TraceMalloc, Actual.TraceMalloc);
+  EXPECT_EQ(Expected.UseCounters, Actual.UseCounters);
+  EXPECT_EQ(Expected.UseMemmem, Actual.UseMemmem);
+  EXPECT_EQ(Expected.UseCmp, Actual.UseCmp);
+  EXPECT_EQ(Expected.UseValueProfile, Actual.UseValueProfile);
+  EXPECT_EQ(Expected.IgnoreOOMs, Actual.IgnoreOOMs);
+  EXPECT_EQ(Expected.IgnoreCrashes, Actual.IgnoreCrashes);
+  EXPECT_EQ(Expected.HandleAbrt, Actual.HandleAbrt);
+  EXPECT_EQ(Expected.HandleBus, Actual.HandleBus);
+  EXPECT_EQ(Expected.HandleFpe, Actual.HandleFpe);
+  EXPECT_EQ(Expected.HandleIll, Actual.HandleIll);
+  EXPECT_EQ(Expected.HandleInt, Actual.HandleInt);
+  EXPECT_EQ(Expected.HandleSegv, Actual.HandleSegv);
+  EXPECT_EQ(Expected.HandleTerm, Actual.HandleTerm);
+}
+
+TEST(Proxy, AddCoverage) {
+  auto *P = Proxy::GetInstance();
+
+  // If this test is repeated, use a new PID.
+  static pid_t PID = GetPid();
+  ++PID;
+
+  // No effect if not enabled.
+  FuzzingOptions Options;
+  Options.UseRemote = false;
+  P->SetOptions(Options);
+
+  // Enable the proxy.
+  Options.UseRemote = true;
+  P->SetOptions(Options);
+
+  static FakeDSO<0x10> DSO1;
+  P->AddCoverage(PID, DSO1.Counters, DSO1.CountersEnd(), DSO1.PCs,
+                 DSO1.PCsEnd());
+  EXPECT_TRUE(P->GetIndicesForPID(PID).empty());
+
+  // Unconnected process is ignored.
+  P->AddCoverage(PID, DSO1.Counters, DSO1.CountersEnd(), DSO1.PCs,
+                 DSO1.PCsEnd());
+  EXPECT_TRUE(P->GetIndicesForPID(PID).empty());
+
+  // Invalid arguments are ignored.
+  ScopedConnection SC(PID);
+  P->AddCoverage(PID, nullptr, DSO1.CountersEnd(), DSO1.PCs, DSO1.PCsEnd());
+  P->AddCoverage(PID, DSO1.Counters, nullptr, DSO1.PCs, DSO1.PCsEnd());
+  P->AddCoverage(PID, DSO1.Counters, DSO1.CountersEnd(), nullptr,
+                 DSO1.PCsEnd());
+  P->AddCoverage(PID, DSO1.Counters, DSO1.CountersEnd(), DSO1.PCs, nullptr);
+  P->AddCoverage(PID, DSO1.Counters, DSO1.Counters, DSO1.PCs, DSO1.PCsEnd());
+  P->AddCoverage(PID, DSO1.Counters, DSO1.CountersEnd(), DSO1.PCs, DSO1.PCs);
+  P->AddCoverage(PID, DSO1.CountersEnd(), DSO1.Counters, DSO1.PCs,
+                 DSO1.PCsEnd());
+  P->AddCoverage(PID, DSO1.Counters, DSO1.CountersEnd(), DSO1.PCsEnd(),
+                 DSO1.PCs);
+  EXPECT_TRUE(P->GetIndicesForPID(PID).empty());
+
+  // Add valid coverage
+  P->AddCoverage(PID, DSO1.Counters, DSO1.CountersEnd(), DSO1.PCs,
+                 DSO1.PCsEnd());
+  auto Indices1 = P->GetIndicesForPID(PID);
+  EXPECT_FALSE(P->GetIndicesForPID(PID).empty());
+
+  // AddCoverage should be idempotent, i.e. calling it with the same arguments
+  // again should not change anything.
+  P->AddCoverage(PID, DSO1.Counters, DSO1.CountersEnd(), DSO1.PCs,
+                 DSO1.PCsEnd());
+  auto Indices2 = P->GetIndicesForPID(PID);
+  EXPECT_EQ(Indices1, Indices2);
+
+  // This breaks the abstraction of SimpleFastHash a bit, but should be the
+  // minimal needed steps to generate a collision.
+  static FakeDSO<0x10> DSO2;
+  ASSERT_EQ(sizeof(DSO1.Counters), sizeof(DSO2.Counters));
+  ASSERT_EQ(sizeof(DSO1.PCs), sizeof(DSO2.PCs));
+  memcpy(DSO2.Counters, DSO1.Counters, sizeof(DSO2.Counters));
+  memcpy(DSO2.PCs, DSO1.PCs, sizeof(DSO2.PCs));
+  auto Hash = TracePC::HashPCs(DSO2.PCs, DSO2.PCsEnd());
+  uint8_t *PCByte = reinterpret_cast<uint8_t *>(DSO2.PCs);
+  // Find the byte corresponding to the first PCFlags for either endianness.
+  PCByte += sizeof(uintptr_t);
+  for (size_t i = 0; *PCByte != 1; ++i)
+    ASSERT_NE(i, sizeof(uintptr_t));
+  PCByte[0] -= 1;
+  PCByte[1] += 11;
+  EXPECT_EQ(Hash, TPC.HashPCs(DSO2.PCs, DSO2.PCsEnd()));
+
+  // New coverage should be added, even though the hash collides, since the
+  // contents differ.
+  P->AddCoverage(PID, DSO2.Counters, DSO2.CountersEnd(), DSO2.PCs,
+                 DSO2.PCsEnd());
+  Indices2 = P->GetIndicesForPID(PID);
+  EXPECT_NE(Indices1, Indices2);
+
+  // Marks the process's coverage as "recyclable".
+  SC.Reset(++PID);
+
+  // Coverage should be recycled, since it matches a region previously added for
+  // a process that disconnected.
+  PCByte[0] += 1;
+  PCByte[1] -= 11;
+  P->AddCoverage(PID, DSO2.Counters, DSO2.CountersEnd(), DSO2.PCs,
+                 DSO2.PCsEnd());
+  Indices2 = P->GetIndicesForPID(PID);
+  EXPECT_EQ(Indices1, Indices2);
+}
+
+TEST(Proxy, ExecuteCallback) {
+  auto *P = Proxy::GetInstance();
+
+  // Set remote functions. Since this test involves competing threads, most
+  // setup and verification needs to hold the mutex.
+  static std::mutex Mutex;
+  static Set<unsigned long> ReceivedPIDs GUARDED_BY(Mutex);
+  static uint32_t ReceivedExecOptions GUARDED_BY(Mutex);
+  static bool ReturnedHasMoreMallocsThanFrees GUARDED_BY(Mutex);
+
+  EF->FuzzerRemoteStartExecution = [](unsigned long PID, uint32_t ExecOptions) {
+    std::lock_guard<std::mutex> Lock(Mutex);
+    ReceivedPIDs.insert(PID);
+    ReceivedExecOptions = ExecOptions;
+    std::thread([PID]() {
+      Proxy::GetInstance()->ExecutionStarted(PID);
+    }).detach();
+  };
+
+  EF->FuzzerRemoteFinishExecution = [](unsigned long PID) {
+    std::lock_guard<std::mutex> Lock(Mutex);
+    ReceivedPIDs.insert(PID);
+    bool HasMoreMallocsThanFrees = ReturnedHasMoreMallocsThanFrees;
+    std::thread([PID, HasMoreMallocsThanFrees]() {
+      Proxy::GetInstance()->ExecutionFinished(PID, HasMoreMallocsThanFrees);
+    }).detach();
+  };
+
+  // Doesn't block if not enabled.
+  FuzzingOptions Options;
+  Options.UseRemote = false;
+  P->SetOptions(Options);
+
+  P->StartExecution();
+  EXPECT_FALSE(P->FinishExecution());
+
+  // Enable the proxy
+  Options.UseRemote = true;
+  P->SetOptions(Options);
+
+  ScopedConnection SCs[3];
+  Set<unsigned long> PIDs;
+  for (size_t i = 0; i < 3; ++i) {
+    SCs[i].Reset(GetPid() + i + 1);
+    PIDs.insert(SCs[i].PID);
+  }
+  // Simulate normal execution.
+  P->StartExecution();
+  {
+    std::lock_guard<std::mutex> Lock(Mutex);
+    EXPECT_EQ(ReceivedPIDs, PIDs);
+    EXPECT_EQ(ReceivedExecOptions & kLeakDetection, 0U);
+    ReceivedPIDs.clear();
+    ReturnedHasMoreMallocsThanFrees = false;
+  }
+  EXPECT_FALSE(P->FinishExecution());
+  {
+    std::lock_guard<std::mutex> Lock(Mutex);
+    EXPECT_EQ(ReceivedPIDs, PIDs);
+
+    ReceivedPIDs.clear();
+  }
+  // Simulate leak detection.
+  P->TryDetectingAMemoryLeak();
+  P->StartExecution();
+  {
+    std::lock_guard<std::mutex> Lock(Mutex);
+    EXPECT_EQ(ReceivedPIDs, PIDs);
+    EXPECT_NE(ReceivedExecOptions & kLeakDetection, 0U);
+    ReceivedPIDs.clear();
+    ReturnedHasMoreMallocsThanFrees = false;
+  }
+  EXPECT_FALSE(P->FinishExecution());
+  {
+    std::lock_guard<std::mutex> Lock(Mutex);
+    EXPECT_EQ(ReceivedPIDs, PIDs);
+    ReceivedPIDs.clear();
+  }
+  // Simulate malloc/free mismatch.
+  P->StartExecution();
+  {
+    std::lock_guard<std::mutex> Lock(Mutex);
+    EXPECT_EQ(ReceivedPIDs, PIDs);
+    EXPECT_EQ(ReceivedExecOptions & kLeakDetection, 0U);
+    ReceivedPIDs.clear();
+    ReturnedHasMoreMallocsThanFrees = true;
+  }
+  EXPECT_TRUE(P->FinishExecution());
+  {
+    std::lock_guard<std::mutex> Lock(Mutex);
+    EXPECT_EQ(ReceivedPIDs, PIDs);
+  }
+}
+
+// Helper for adding fake DSOs as if they came from another process. Returns the
+// PC offset for the given process.
+template <size_t N>
+uint64_t AddCoverageForPID(unsigned long PID, FakeDSO<N> &DSO) {
+  auto *P = Proxy::GetInstance();
+  auto H = TracePC::HashPCs(DSO.PCs, DSO.PCsEnd());
+  P->AddCoverage(PID, DSO.Counters, DSO.CountersEnd(), DSO.PCs, DSO.PCsEnd());
+  auto *TE = TPC.PCTableEntryByDSO(H, 0);
+  return TE->PC & ~Proxy::kPCMask;
+}
+
+TEST(Proxy, DescribeAndPrintPC) {
+  auto *P = Proxy::GetInstance();
+  constexpr const char *kSymbolizedFMT = "SymbolizedFMT: PC=%lu";
+  constexpr const char *kFallbackFMT = "FallbackFMT";
+  char Buf[256];
+
+  // Set up remote functions.
+  static unsigned long ReceivedPID;
+  static uintptr_t ReceivedPC;
+  EF->FuzzerRemotePrintPC = [](unsigned long PID, const char *SymbolizedFMT,
+                               const char *FallbackFMT, uintptr_t PC) {
+    ReceivedPID = PID;
+    EXPECT_STREQ(SymbolizedFMT, kSymbolizedFMT);
+    EXPECT_STREQ(FallbackFMT, kFallbackFMT);
+    ReceivedPC = PC;
+  };
+
+  EF->FuzzerRemoteDescribePC = [](unsigned long PID, const char *SymbolizedFMT,
+                                  uintptr_t PC, char *Desc, size_t DescLen) {
+    ReceivedPID = PID;
+    EXPECT_STREQ(SymbolizedFMT, kSymbolizedFMT);
+    ReceivedPC = PC;
+    snprintf(Desc, DescLen, SymbolizedFMT, PC);
+  };
+
+  // Always returns false if proxy is disabled.
+  FuzzingOptions Options;
+  Options.UseRemote = false;
+  P->SetOptions(Options);
+  for (size_t Idx = 0;; ++Idx) {
+    SCOPED_TRACE(Idx);
+    auto *TE = TPC.PCTableEntryByIdx(Idx);
+    if (!TE)
+      break;
+    EXPECT_FALSE(Proxy::PrintPC(kSymbolizedFMT, kFallbackFMT, TE->PC));
+    EXPECT_FALSE(Proxy::DescribePC(kSymbolizedFMT, TE->PC, Buf, sizeof(Buf)));
+  }
+
+  // Enable the proxy.
+  Options.UseRemote = true;
+  P->SetOptions(Options);
+
+  // Track what PCs we expect to be able to print and describe.
+  auto PID = GetPid() + 1;
+  constexpr size_t N = 3;
+  ScopedConnection SCs[N];
+  for (size_t i = 0; i < N; ++i)
+    SCs[i].Reset(PID + i);
+  uint64_t PCOffsets[N];
+  static FakeDSO<0x100> DSO1;
+  PCOffsets[0] = AddCoverageForPID(SCs[0].PID, DSO1);
+  static FakeDSO<0x200> DSO2;
+  PCOffsets[1] = AddCoverageForPID(SCs[1].PID, DSO2);
+  static FakeDSO<0x300> DSO3;
+  PCOffsets[2] = AddCoverageForPID(SCs[2].PID, DSO3);
+
+  // Missing offset causes the PC to be unrecognized.
+  uint64_t PC = 0;
+  EXPECT_FALSE(Proxy::PrintPC(kSymbolizedFMT, kFallbackFMT, PC));
+  EXPECT_FALSE(Proxy::DescribePC(kSymbolizedFMT, PC, Buf, sizeof(Buf)));
+
+  PC = Proxy::kPCMask;
+  EXPECT_FALSE(Proxy::PrintPC(kSymbolizedFMT, kFallbackFMT, PC));
+  EXPECT_FALSE(Proxy::DescribePC(kSymbolizedFMT, PC, Buf, sizeof(Buf)));
+
+  for (size_t i = 0; i < N; ++i) {
+    // Valid PC range for a remote process.
+    PC = PCOffsets[i];
+    snprintf(Buf, sizeof(Buf), kSymbolizedFMT, 0UL);
+
+    ReceivedPC = 0;
+    ReceivedPID = 0;
+    PrintPC(kSymbolizedFMT, kFallbackFMT, PC);
+    EXPECT_EQ(ReceivedPC, 0U);
+    EXPECT_EQ(ReceivedPID, SCs[i].PID);
+
+    ReceivedPC = 0;
+    ReceivedPID = 0;
+    auto Actual = DescribePC(kSymbolizedFMT, PC);
+    EXPECT_EQ(ReceivedPC, 0U);
+    EXPECT_EQ(ReceivedPID, SCs[i].PID);
+    EXPECT_STREQ(Buf, Actual.c_str());
+
+    PC |= Proxy::kPCMask;
+    snprintf(Buf, sizeof(Buf), kSymbolizedFMT, Proxy::kPCMask);
+
+    ReceivedPC = 0;
+    ReceivedPID = 0;
+    PrintPC(kSymbolizedFMT, kFallbackFMT, PC);
+    EXPECT_EQ(ReceivedPC, Proxy::kPCMask);
+    EXPECT_EQ(ReceivedPID, SCs[i].PID);
+
+    ReceivedPC = 0;
+    ReceivedPID = 0;
+    Actual = DescribePC(kSymbolizedFMT, PC);
+    EXPECT_EQ(ReceivedPC, Proxy::kPCMask);
+    EXPECT_EQ(ReceivedPID, SCs[i].PID);
+    EXPECT_STREQ(Buf, Actual.c_str());
+  }
+
+  // An invalid process is also unrecognized.
+  PC = PCOffsets[2] + (1ULL << Proxy::kPCShift);
+  EXPECT_FALSE(Proxy::PrintPC(kSymbolizedFMT, kFallbackFMT, PC));
+  EXPECT_FALSE(Proxy::DescribePC(kSymbolizedFMT, PC, Buf, sizeof(Buf)));
+
+  // Disconnect; should subsequently return false.
+  for (size_t i = 0; i < N; ++i)
+    SCs[i].Reset();
+  for (size_t i = 0; i < N; ++i) {
+    PC = PCOffsets[i];
+    EXPECT_FALSE(Proxy::PrintPC(kSymbolizedFMT, kFallbackFMT, PC));
+    EXPECT_FALSE(Proxy::DescribePC(kSymbolizedFMT, PC, Buf, sizeof(Buf)));
+    PC |= Proxy::kPCMask;
+    EXPECT_FALSE(Proxy::PrintPC(kSymbolizedFMT, kFallbackFMT, PC));
+    EXPECT_FALSE(Proxy::DescribePC(kSymbolizedFMT, PC, Buf, sizeof(Buf)));
+  }
+}
+
 int main(int argc, char **argv) {
   testing::InitGoogleTest(&argc, argv);
   testing::AddGlobalTestEnvironment(new fuzzer::TestEnvironment());
   return RUN_ALL_TESTS();
 }