-output_csv libfuzzer option

[aizatsky]

Summary

The option outputs statistics in CSV format preceded by 1 header line.
This is intended for machine processing of the output.
-verbosity=0 should likely be set.

[kcc]

LGTM with one nit

lib/Fuzzer/FuzzerLoop.cpp

440 ↗

(On Diff #39995)

Why "" as second param? It will not terminate the line

[aizatsky]

review

[aizatsky]

submitting.

llvm/trunk/lib/Fuzzer/FuzzerDriver.cpp

int FuzzerDriver(const std::vector<std::string> &Args,
   Options.UseTraces = Flags.use_traces;
   Options.ShuffleAtStartUp = Flags.shuffle;
   Options.PreferSmallDuringInitialShuffle =
       Flags.prefer_small_during_initial_shuffle;
   Options.Reload = Flags.reload;
   Options.OnlyASCII = Flags.only_ascii;
   Options.TBMDepth = Flags.tbm_depth;
   Options.TBMWidth = Flags.tbm_width;
+  Options.OutputCSV = Flags.output_csv;
   if (Flags.runs >= 0)
     Options.MaxNumberOfRuns = Flags.runs;
   if (!Inputs->empty())
     Options.OutputCorpus = (*Inputs)[0];
   if (Flags.sync_command)
     Options.SyncCommand = Flags.sync_command;
   Options.SyncTimeout = Flags.sync_timeout;
   Options.ReportSlowUnits = Flags.report_slow_units;

llvm/trunk/lib/Fuzzer/FuzzerFlags.def

 FUZZER_FLAG_INT(tbm_width, 5, "Apply at most this number of independent"
                                "trace-based-mutations (tbm)")
 FUZZER_FLAG_STRING(test_single_input, "Use specified file as test input.")
 FUZZER_FLAG_STRING(artifact_prefix, "Write fuzzing artifacts (crash, "
                                     "timeout, or slow inputs) as "
                                     "$(artifact_prefix)file")
 FUZZER_FLAG_INT(drill, 0, "Experimental: fuzz using a single unit as the seed "
                           "corpus, then merge with the initial corpus")
+FUZZER_FLAG_INT(output_csv, 0, "Enable pulse output in CSV format.")

llvm/trunk/lib/Fuzzer/FuzzerInternal.h

struct FuzzingOptions {
     bool OnlyASCII = false;
     int TBMDepth = 10;
     int TBMWidth = 10;
     std::string OutputCorpus;
     std::string SyncCommand;
     std::string ArtifactPrefix = "./";
     bool SaveArtifacts = true;
     bool PrintNEW = true;  // Print a status line when new units are found;
+    bool OutputCSV = false;
   };
   Fuzzer(UserSuppliedFuzzer &USF, FuzzingOptions Options);
   void AddToCorpus(const Unit &U) { Corpus.push_back(U); }
   size_t ChooseUnitIdxToMutate();
   const Unit &ChooseUnitToMutate() { return Corpus[ChooseUnitIdxToMutate()]; };
   void Loop();
   void Drill();
   void ShuffleAndMinimize();

llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp

if (Seconds >= (size_t)Options.UnitTimeoutSec) {
     if (__sanitizer_print_stack_trace)
       __sanitizer_print_stack_trace();
     Printf("SUMMARY: libFuzzer: timeout\n");
     exit(1);
   }
 }
 
 void Fuzzer::PrintStats(const char *Where, const char *End) {
-  if (!Options.Verbosity) return;
   size_t Seconds = secondsSinceProcessStartUp();
   size_t ExecPerSec = (Seconds ? TotalNumberOfRuns / Seconds : 0);
+
+  if (Options.OutputCSV) {
+    static bool csvHeaderPrinted = false;
+    if (!csvHeaderPrinted) {
+      csvHeaderPrinted = true;
+      Printf("runs,block_cov,bits,cc_cov,corpus,execs_per_sec,tbms,reason\n");
+    }
+    Printf("%zd,%zd,%zd,%zd,%zd,%zd,%zd,%s\n", TotalNumberOfRuns,
+           LastRecordedBlockCoverage, TotalBits(),
+           LastRecordedCallerCalleeCoverage, Corpus.size(), ExecPerSec,
+           TotalNumberOfExecutedTraceBasedMutations, Where);
+  }
+
+  if (!Options.Verbosity)
+    return;
   Printf("#%zd\t%s", TotalNumberOfRuns, Where);
   if (LastRecordedBlockCoverage)
     Printf(" cov: %zd", LastRecordedBlockCoverage);
   if (auto TB = TotalBits())
     Printf(" bits: %zd", TB);
   if (LastRecordedCallerCalleeCoverage)
     Printf(" indir: %zd", LastRecordedCallerCalleeCoverage);
   Printf(" units: %zd exec/s: %zd", Corpus.size(), ExecPerSec);
void Fuzzer::RereadOutputCorpus() {
   for (auto &X : AdditionalCorpus) {
     if (X.size() > (size_t)Options.MaxLen)
       X.resize(Options.MaxLen);
     if (UnitHashesAddedToCorpus.insert(Hash(X)).second) {
       CurrentUnit.clear();
       CurrentUnit.insert(CurrentUnit.begin(), X.begin(), X.end());
       if (RunOne(CurrentUnit)) {
         Corpus.push_back(X);
-        if (Options.Verbosity >= 1)
-          PrintStats("RELOAD");
       }
     }
   }
 }
 
 void Fuzzer::ShuffleAndMinimize() {
   bool PreferSmall = (Options.PreferSmallDuringInitialShuffle == 1 ||
                       (Options.PreferSmallDuringInitialShuffle == -1 &&
bool Fuzzer::RunOne(const Unit &U) {
 
   PrepareCoverageBeforeRun();
   ExecuteCallback(U);
   bool Res = CheckCoverageAfterRun();
 
   auto UnitStopTime = system_clock::now();
   auto TimeOfUnit =
       duration_cast<seconds>(UnitStopTime - UnitStartTime).count();
-  if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1))
-      && secondsSinceProcessStartUp() >= 2
-      && Options.Verbosity)
+  if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) &&
+      secondsSinceProcessStartUp() >= 2)
     PrintStats("pulse ");
   if (TimeOfUnit > TimeOfLongestUnitInSeconds &&
       TimeOfUnit >= Options.ReportSlowUnits) {
     TimeOfLongestUnitInSeconds = TimeOfUnit;
     Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds);
     WriteUnitToFileWithPrefix(U, "slow-unit-");
   }
   return Res;

 }
 
 void Fuzzer::Loop() {
   while (true) {
     size_t J1 = ChooseUnitIdxToMutate();;
     SyncCorpus();
     RereadOutputCorpus();
     if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
-      return;
+      break;
     if (Options.MaxTotalTimeSec > 0 &&
         secondsSinceProcessStartUp() >
         static_cast<size_t>(Options.MaxTotalTimeSec))
-      return;
+      break;
     CurrentUnit = Corpus[J1];
     // Optionally, cross with another unit.
     if (Options.DoCrossOver && USF.GetRand().RandBool()) {
       size_t J2 = ChooseUnitIdxToMutate();
       if (!Corpus[J1].empty() && !Corpus[J2].empty()) {
         assert(!Corpus[J2].empty());
         CurrentUnit.resize(Options.MaxLen);
         size_t NewSize = USF.CrossOver(
             Corpus[J1].data(), Corpus[J1].size(), Corpus[J2].data(),
             Corpus[J2].size(), CurrentUnit.data(), CurrentUnit.size());
         assert(NewSize > 0 && "CrossOver returned empty unit");
         assert(NewSize <= (size_t)Options.MaxLen &&
                "CrossOver returned overisized unit");
         CurrentUnit.resize(NewSize);
       }
     }
     // Perform several mutations and runs.
     MutateAndTestOne(&CurrentUnit);
   }
+
+  PrintStats("DONE  ", "\n");
 }
 
 void Fuzzer::SyncCorpus() {
   if (Options.SyncCommand.empty() || Options.OutputCorpus.empty()) return;
   auto Now = system_clock::now();
   if (duration_cast<seconds>(Now - LastExternalSync).count() <
       Options.SyncTimeout)
     return;
   LastExternalSync = Now;
   ExecuteCommand(Options.SyncCommand + " " + Options.OutputCorpus);
 }
 
 }  // namespace fuzzer