[libFuzzer] Initial implementation of mutation usefulness percentages.

[kodewilliams]

Summary

Now, the mutation statistics are displayed in a concise manner, along with the usefulness percentages below.

[Dor1s]

Hey Kode, is this ready for review, or you've just uploaded it for yourself? Next time, just leave a comment like "please take a look" or "PTAL" or "ready for review", so Jonathan and I can look at it without asking that question. Otherwise, we usually assume that it's a temporary version that may not necessary need our attention.

Another comment would be, please format commit title and description to be more consistent for other libFuzzer commits. Take a look at other commit messages for an example: https://github.com/llvm-mirror/compiler-rt/commits/master/lib/fuzzer

[metzman]

I assumed I was supposed to review this. +1 to Max's comment about saying "PTAL"
I don't really think this should be its own option. I'd just print the percentages, but I think this is for Max to decide since he has in mind what will be done with the stats better than I.

lib/fuzzer/FuzzerInternal.h

83

I think it would better if PrintFinalStats handled printing of all stats including the mutation ones, instead of calling these three functions all over.

lib/fuzzer/FuzzerMutationStats.cpp

70

This looks like an unnecessary change, please undo

[Dor1s]

In D48336#1137043, @metzman wrote:

...
I don't really think this should be its own option. I'd just print the percentages, but I think this is for Max to decide since he has in mind what will be done with the stats better than I.

Fair point. Yeah, let's print both using the same flag (i.e. PrintMutationStats). At the end of the day, I think, we may remove the detailed information with mutation names at all, but it might be useful in the meantime while we're testing the things.

[Dor1s]

Please take a look at my comment to your previous CL: https://reviews.llvm.org/D48054#1137281

Let's incorporate this changes into that CL as well, that would be much simpler in terms of adding a test and getting it landed.

[kodewilliams]

Working on previous CL instead.

lib/fuzzer/FuzzerDriver.cpp

if (Flags.verbosity > 0 && !Dictionary.empty())
     Printf("Dictionary: %zd entries\n", Dictionary.size());
   bool DoPlainRun = AllInputsAreFiles();
   Options.SaveArtifacts =
       !DoPlainRun || Flags.minimize_crash_internal_step;
   Options.PrintNewCovPcs = Flags.print_pcs;
   Options.PrintNewCovFuncs = Flags.print_funcs;
   Options.PrintFinalStats = Flags.print_final_stats;
   Options.PrintMutationStats = Flags.print_mutation_stats;
+  Options.PrintMutationUsefulness = Flags.print_mutation_usefulness;
   Options.PrintCorpusStats = Flags.print_corpus_stats;
   Options.PrintCoverage = Flags.print_coverage;
   Options.DumpCoverage = Flags.dump_coverage;
   if (Flags.exit_on_src_pos)
     Options.ExitOnSrcPos = Flags.exit_on_src_pos;
   if (Flags.exit_on_item)
     Options.ExitOnItem = Flags.exit_on_item;
   if (Flags.focus_function)
for (auto &Path : *Inputs) {
       auto MS = duration_cast<milliseconds>(StopTime - StartTime).count();
       Printf("Executed %s in %zd ms\n", Path.c_str(), (long)MS);
     }
     Printf("***\n"
            "*** NOTE: fuzzing was not performed, you have only\n"
            "***       executed the target code on a fixed set of inputs.\n"
            "***\n");
     F->PrintFinalStats();
+    F->PrintMutationStats();
+    F->PrintMutationUsefulness();
     exit(0);
   }
 
   if (Flags.merge) {
     F->CrashResistantMerge(Args, *Inputs,
                            Flags.load_coverage_summary,
                            Flags.save_coverage_summary,
                            Flags.merge_control_file);
#endif
   }
 
   F->Loop(*Inputs);
 
   if (Flags.verbosity)
     Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(),
            F->secondsSinceProcessStartUp());
   F->PrintFinalStats();
+  F->PrintMutationStats();
+  F->PrintMutationUsefulness();
 
   exit(0);  // Don't let F destroy itself.
 }
 
 // Storage for global ExternalFunctions object.
 ExternalFunctions *EF = nullptr;
 
 }  // namespace fuzzer

lib/fuzzer/FuzzerFlags.def

FUZZER_FLAG_STRING(focus_function, "Experimental. "
      "Fuzzing will focus on inputs that trigger calls to this function")
 
 FUZZER_DEPRECATED_FLAG(run_equivalence_server)
 FUZZER_DEPRECATED_FLAG(use_equivalence_server)
 FUZZER_FLAG_INT(analyze_dict, 0, "Experimental")
 FUZZER_DEPRECATED_FLAG(use_clang_coverage)
 FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace")
 FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental")
+FUZZER_FLAG_INT(print_mutation_usefulness, 0, "Experimental")

lib/fuzzer/FuzzerInternal.h

public:
   void CrashResistantMerge(const Vector<std::string> &Args,
                            const Vector<std::string> &Corpora,
                            const char *CoverageSummaryInputPathOrNull,
                            const char *CoverageSummaryOutputPathOrNull,
                            const char *MergeControlFilePathOrNull);
   void CrashResistantMergeInternalStep(const std::string &ControlFilePath);
   MutationDispatcher &GetMD() { return MD; }
   void PrintFinalStats();
+  void PrintMutationStats();
+  void PrintMutationUsefulness();
   void SetMaxInputLen(size_t MaxInputLen);
   void SetMaxMutationLen(size_t MaxMutationLen);
   void RssLimitCallback();
 
   bool InFuzzingThread() const { return IsMyThread; }
   size_t GetCurrentUnitInFuzzingThead(const uint8_t **Data) const;
   void TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size,
                                bool DuringInitialCorpusExecution);

lib/fuzzer/FuzzerLoop.cpp

if (!Options.MallocLimitMb || (Size >> 20) < (size_t)Options.MallocLimitMb)
     return;
   Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(),
          Size);
   Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
   PrintStackTrace();
   DumpCurrentUnit("oom-");
   Printf("SUMMARY: libFuzzer: out-of-memory\n");
   PrintFinalStats();
+  PrintMutationStats();
+  PrintMutationUsefulness();
   _Exit(Options.ErrorExitCode); // Stop right now.
 }
 
 Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
                FuzzingOptions Options)
     : CB(CB), Corpus(Corpus), MD(MD), Options(Options) {
   if (EF->__sanitizer_set_death_callback)
     EF->__sanitizer_set_death_callback(StaticDeathCallback);
void Fuzzer::DumpCurrentUnit(const char *Prefix) {
   WriteUnitToFileWithPrefix({CurrentUnitData, CurrentUnitData + UnitSize},
                             Prefix);
 }
 
 NO_SANITIZE_MEMORY
 void Fuzzer::DeathCallback() {
   DumpCurrentUnit("crash-");
   PrintFinalStats();
+  PrintMutationStats();
+  PrintMutationUsefulness();
 }
 
 void Fuzzer::StaticAlarmCallback() {
   assert(F);
   F->AlarmCallback();
 }
 
 void Fuzzer::StaticCrashSignalCallback() {
void Fuzzer::CrashCallback() {
   Printf("==%lu== ERROR: libFuzzer: deadly signal\n", GetPid());
   PrintStackTrace();
   Printf("NOTE: libFuzzer has rudimentary signal handlers.\n"
          "      Combine libFuzzer with AddressSanitizer or similar for better "
          "crash reports.\n");
   Printf("SUMMARY: libFuzzer: deadly signal\n");
   DumpCurrentUnit("crash-");
   PrintFinalStats();
+  PrintMutationStats();
+  PrintMutationUsefulness();
   _Exit(Options.ErrorExitCode); // Stop right now.
 }
 
 void Fuzzer::ExitCallback() {
   if (!RunningCB)
     return; // This exit did not come from the user callback
   if (EF->__sanitizer_acquire_crash_state &&
       !EF->__sanitizer_acquire_crash_state())
     return;
   Printf("==%lu== ERROR: libFuzzer: fuzz target exited\n", GetPid());
   PrintStackTrace();
   Printf("SUMMARY: libFuzzer: fuzz target exited\n");
   DumpCurrentUnit("crash-");
   PrintFinalStats();
+  PrintMutationStats();
+  PrintMutationUsefulness();
   _Exit(Options.ErrorExitCode);
 }
 
 void Fuzzer::MaybeExitGracefully() {
   if (!GracefulExitRequested) return;
   Printf("==%lu== INFO: libFuzzer: exiting as requested\n", GetPid());
   PrintFinalStats();
+  PrintMutationStats();
+  PrintMutationUsefulness();
   _Exit(0);
 }
 
 void Fuzzer::InterruptCallback() {
   Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid());
   PrintFinalStats();
+  PrintMutationStats();
+  PrintMutationUsefulness();
   _Exit(0); // Stop right now, don't perform any at-exit actions.
 }
 
 NO_SANITIZE_MEMORY
 void Fuzzer::AlarmCallback() {
   assert(Options.UnitTimeoutSec > 0);
   // In Windows Alarm callback is executed by a different thread.
 #if !LIBFUZZER_WINDOWS
if (Seconds >= (size_t)Options.UnitTimeoutSec) {
     Printf("       and the timeout value is %d (use -timeout=N to change)\n",
            Options.UnitTimeoutSec);
     DumpCurrentUnit("timeout-");
     Printf("==%lu== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),
            Seconds);
     PrintStackTrace();
     Printf("SUMMARY: libFuzzer: timeout\n");
     PrintFinalStats();
+    PrintMutationStats();
+    PrintMutationUsefulness();
     _Exit(Options.TimeoutExitCode); // Stop right now.
   }
 }
 
 void Fuzzer::RssLimitCallback() {
   if (EF->__sanitizer_acquire_crash_state &&
       !EF->__sanitizer_acquire_crash_state())
     return;
   Printf(
       "==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
       GetPid(), GetPeakRSSMb(), Options.RssLimitMb);
   Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
   PrintMemoryProfile();
   DumpCurrentUnit("oom-");
   Printf("SUMMARY: libFuzzer: out-of-memory\n");
   PrintFinalStats();
+  PrintMutationStats();
+  PrintMutationUsefulness();
   _Exit(Options.ErrorExitCode); // Stop right now.
 }
 
 void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) {
   size_t ExecPerSec = execPerSec();
   if (!Options.Verbosity)
     return;
   Printf("#%zd\t%s", TotalNumberOfRuns, Where);
void Fuzzer::PrintFinalStats() {
   if (!Options.PrintFinalStats)
     return;
   size_t ExecPerSec = execPerSec();
   Printf("stat::number_of_executed_units: %zd\n", TotalNumberOfRuns);
   Printf("stat::average_exec_per_sec:     %zd\n", ExecPerSec);
   Printf("stat::new_units_added:          %zd\n", NumberOfNewUnitsAdded);
   Printf("stat::slowest_unit_time_sec:    %zd\n", TimeOfLongestUnitInSeconds);
   Printf("stat::peak_rss_mb:              %zd\n", GetPeakRSSMb());
+}
 
+void Fuzzer::PrintMutationStats() {
   if (Options.PrintMutationStats)
     MStats->PrintMutationCounts();
 }
 
+void Fuzzer::PrintMutationUsefulness() {
+  if (Options.PrintMutationUsefulness)
+    MStats->PrintMutationUsefulness();
+}
+
 void Fuzzer::SetMaxInputLen(size_t MaxInputLen) {
   assert(this->MaxInputLen == 0); // Can only reset MaxInputLen from 0 to non-0.
   assert(MaxInputLen);
   this->MaxInputLen = MaxInputLen;
   this->MaxMutationLen = MaxInputLen;
   AllocateCurrentUnitData();
   Printf("INFO: -max_len is not provided; "
          "libFuzzer will not generate inputs larger than %zd bytes\n",
void Fuzzer::TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size,
   // we don't call it too often.
   if (EF->__lsan_do_recoverable_leak_check()) { // Leak is found, report it.
     if (DuringInitialCorpusExecution)
       Printf("\nINFO: a leak has been found in the initial corpus.\n\n");
     Printf("INFO: to ignore leaks on libFuzzer side use -detect_leaks=0.\n\n");
     CurrentUnitSize = Size;
     DumpCurrentUnit("leak-");
     PrintFinalStats();
+    PrintMutationStats();
+    PrintMutationUsefulness();
     _Exit(Options.ErrorExitCode); // not exit() to disable lsan further on.
   }
 }
 
 void Fuzzer::MutateAndTestOne() {
   MD.StartMutationSequence();
 
   auto &II = Corpus.ChooseUnitToMutate(MD.GetRand());
if (Size != OtherSize || memcmp(Data, OtherData, Size) != 0) {
         if (Data[i] != OtherData[i])
           break;
       Printf("==%lu== ERROR: libFuzzer: equivalence-mismatch. Sizes: %zd %zd; "
              "offset %zd\n",
              GetPid(), Size, OtherSize, i);
       DumpCurrentUnit("mismatch-");
       Printf("SUMMARY: libFuzzer: equivalence-mismatch\n");
       PrintFinalStats();
+      PrintMutationStats();
+      PrintMutationUsefulness();
       _Exit(Options.ErrorExitCode);
     }
   }
 }
 
 } // namespace fuzzer
 
 extern "C" {

lib/fuzzer/FuzzerMutationStats.h

 namespace fuzzer {
 
 class MutationStats {
 public:
   ~MutationStats() {}
   void PrintMutationCounts();
   void IncTotalMutationCount(MutationType MType);
   void IncUsefulMutationCount(MutationType MType);
+  void PrintMutationUsefulness();
 
 private:
   // A total count of each mutation used in the fuzzing process.
   std::array<size_t, MaxNumberOfMutationTypes> TotalMutations;
   // The number of each mutation that resulted in new coverage.
   std::array<size_t, MaxNumberOfMutationTypes> UsefulMutations;
 };
 
 extern MutationStats *MStats;
 
 }

lib/fuzzer/FuzzerMutationStats.cpp

 //===- FuzzerMutate.cpp - Mutate a test input -----------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 // Mutate a test input.
 //===----------------------------------------------------------------------===//
 
-#include "FuzzerMutate.h"
 #include "FuzzerMutationStats.h"
 #include "FuzzerIO.h"
+#include "FuzzerMutate.h"
 
 namespace fuzzer {
 
-MutationStats* MStats = new MutationStats();
+MutationStats *MStats = new MutationStats();
 
 void MutationStats::PrintMutationCounts() {
   // Outputs the number of each mutation used
-  Printf("\nTotal Mutations ----------\n");
+  Printf("\nstat::total_mutations:          ");
   for (int i = 0; i < MaxNumberOfMutationTypes; i++) {
-    auto current = kMutationNames.find((MutationType) i);
-    Printf("%s: %d\n", current->second.c_str(), TotalMutations.at(i));
+    if (i < MaxNumberOfMutationTypes-1)
+      Printf("%zd, ", TotalMutations.at(i));
+    else
+      Printf("%zd", TotalMutations.at(i));
   }
 
-  Printf("\nUseful Mutations ----------\n");
+  Printf("\nstat::useful_mutations:         ");
   for (int i = 0; i < MaxNumberOfMutationTypes; i++) {
-    auto current = kMutationNames.find((MutationType) i);
-    Printf("%s: %d\n", current->second.c_str(), UsefulMutations.at(i));
+    if (i < MaxNumberOfMutationTypes-1)
+      Printf("%zd, ", UsefulMutations.at(i));
+    else
+      Printf("%zd", UsefulMutations.at(i));
+  }
+}
+
+void MutationStats::PrintMutationUsefulness() {
+  Printf("\nstat::mutation_usefulness:      ");
+  for (int i = 0; i < MaxNumberOfMutationTypes; i++) {
+    auto Current = kMutationNames.find((MutationType)i);
+    double CurrentRatio =
+        (TotalMutations.at(i) != 0)
+            ? (UsefulMutations.at(i) /
+               static_cast<double>(TotalMutations.at(i)) * 100)
+            : 0;
+    if (i < MaxNumberOfMutationTypes-1)
+      Printf("%.3f, ", Current->second.c_str(), CurrentRatio);
+    else
+      Printf("%.3f", Current->second.c_str(), CurrentRatio);
   }
+  Printf("\n\n");
 }
 
-// Updates the count of mutations (total/useful) depending on type.
-// Types: (0) Total, (1) Useful
 void MutationStats::IncTotalMutationCount(MutationType MType) {
   if ((MType < 0) || MType >= MaxNumberOfMutationTypes)
     return;
   else
     TotalMutations[MType]++;
 }
 
 void MutationStats::IncUsefulMutationCount(MutationType MType) {
   if ((MType < 0) || MType >= MaxNumberOfMutationTypes)
     return;
   else
     UsefulMutations[MType]++;
 }
 
-}  // namespace fuzzer
+} // namespace fuzzer

lib/fuzzer/FuzzerOptions.h

struct FuzzingOptions {
   std::string FocusFunction;
   std::string DataFlowTrace;
   bool SaveArtifacts = true;
   bool PrintNEW = true; // Print a status line when new units are found;
   bool PrintNewCovPcs = false;
   int PrintNewCovFuncs = 0;
   bool PrintFinalStats = false;
   bool PrintMutationStats = false;
+  bool PrintMutationUsefulness = false;
   bool PrintCorpusStats = false;
   bool PrintCoverage = false;
   bool DumpCoverage = false;
   bool DetectLeaks = true;
   int PurgeAllocatorIntervalSec = 1;
   int  TraceMalloc = 0;
   bool HandleAbrt = false;
   bool HandleBus = false;