Index: lib/fuzzer/FuzzerDefs.h =================================================================== --- lib/fuzzer/FuzzerDefs.h +++ lib/fuzzer/FuzzerDefs.h @@ -16,10 +16,10 @@ #include #include #include +#include +#include #include #include -#include -#include // Platform detection. #ifdef __linux__ Index: lib/fuzzer/FuzzerDriver.cpp =================================================================== --- lib/fuzzer/FuzzerDriver.cpp +++ lib/fuzzer/FuzzerDriver.cpp @@ -613,6 +613,7 @@ Options.PrintNewCovPcs = Flags.print_pcs; Options.PrintNewCovFuncs = Flags.print_funcs; Options.PrintFinalStats = Flags.print_final_stats; + Options.PrintMutationStats = Flags.print_mutation_stats; Options.PrintCorpusStats = Flags.print_corpus_stats; Options.PrintCoverage = Flags.print_coverage; Options.DumpCoverage = Flags.dump_coverage; Index: lib/fuzzer/FuzzerFlags.def =================================================================== --- lib/fuzzer/FuzzerFlags.def +++ lib/fuzzer/FuzzerFlags.def @@ -153,3 +153,4 @@ FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") FUZZER_DEPRECATED_FLAG(use_clang_coverage) FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace") +FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental") Index: lib/fuzzer/FuzzerLoop.cpp =================================================================== --- lib/fuzzer/FuzzerLoop.cpp +++ lib/fuzzer/FuzzerLoop.cpp @@ -355,6 +355,8 @@ TPC.DumpCoverage(); if (Options.PrintCorpusStats) Corpus.PrintStats(); + if (Options.PrintMutationStats) + MD.PrintMutationStats(); if (!Options.PrintFinalStats) return; size_t ExecPerSec = execPerSec(); Index: lib/fuzzer/FuzzerMutate.h =================================================================== --- lib/fuzzer/FuzzerMutate.h +++ lib/fuzzer/FuzzerMutate.h @@ -19,6 +19,8 @@ namespace fuzzer { +const int kNumMutationTypes = 15; + class MutationDispatcher { public: MutationDispatcher(Random &Rand, const FuzzingOptions &Options); @@ -86,13 +88,18 @@ Random &GetRand() { return Rand; } -private: + void PrintMutationStats(); + + void CountCurrentMutatorSequence(); struct Mutator { size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max); + int Identifier; const char *Name; }; + +private: size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, size_t MaxSize); size_t MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize, @@ -143,6 +150,11 @@ Vector Mutators; Vector DefaultMutators; + + // A total count of each mutation used in the fuzzing process. + Vector TotalMutations; + // The number of each mutation that resulted in new coverage. + Vector UsefulMutations; }; } // namespace fuzzer Index: lib/fuzzer/FuzzerMutate.cpp =================================================================== --- lib/fuzzer/FuzzerMutate.cpp +++ lib/fuzzer/FuzzerMutate.cpp @@ -30,34 +30,41 @@ DefaultMutators.insert( DefaultMutators.begin(), { - {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"}, - {&MutationDispatcher::Mutate_InsertByte, "InsertByte"}, + {&MutationDispatcher::Mutate_EraseBytes, 11, "EraseBytes"}, + {&MutationDispatcher::Mutate_InsertByte, 12, "InsertByte"}, {&MutationDispatcher::Mutate_InsertRepeatedBytes, - "InsertRepeatedBytes"}, - {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"}, - {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"}, - {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"}, - {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"}, - {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"}, - {&MutationDispatcher::Mutate_CopyPart, "CopyPart"}, - {&MutationDispatcher::Mutate_CrossOver, "CrossOver"}, + 13, "InsertRepeatedBytes"}, + {&MutationDispatcher::Mutate_ChangeByte, 6, "ChangeByte"}, + {&MutationDispatcher::Mutate_ChangeBit, 5, "ChangeBit"}, + {&MutationDispatcher::Mutate_ShuffleBytes, 14, + "ShuffleBytes"}, + {&MutationDispatcher::Mutate_ChangeASCIIInteger, 3, + "ChangeASCIIInt"}, + {&MutationDispatcher::Mutate_ChangeBinaryInteger, + 4, "ChangeBinInt"}, + {&MutationDispatcher::Mutate_CopyPart, 7, "CopyPart"}, + {&MutationDispatcher::Mutate_CrossOver, 8, "CrossOver"}, {&MutationDispatcher::Mutate_AddWordFromManualDictionary, - "ManualDict"}, + 0, "ManualDict"}, {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary, - "PersAutoDict"}, + 1, "PersAutoDict"}, }); if(Options.UseCmp) - DefaultMutators.push_back( - {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"}); + DefaultMutators.push_back({&MutationDispatcher::Mutate_AddWordFromTORC, + 2, "CMP"}); if (EF->LLVMFuzzerCustomMutator) - Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"}); + Mutators.push_back({&MutationDispatcher::Mutate_Custom, 10, "Custom"}); else Mutators = DefaultMutators; if (EF->LLVMFuzzerCustomCrossOver) - Mutators.push_back( - {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"}); + Mutators.push_back({&MutationDispatcher::Mutate_CustomCrossOver, + 9, "CustomCrossOver"}); + + // Initialize mutation statistic counters. + TotalMutations.resize(kNumMutationTypes, 0); + UsefulMutations.resize(kNumMutationTypes, 0); } static char RandCh(Random &Rand) { @@ -465,6 +472,7 @@ if (!PersistentAutoDictionary.ContainsWord(DE->GetW())) PersistentAutoDictionary.push_back({DE->GetW(), 1}); } + CountCurrentMutatorSequence(); } void MutationDispatcher::PrintRecommendedDictionary() { @@ -520,6 +528,7 @@ if (Options.OnlyASCII) ToASCII(Data, NewSize); CurrentMutatorSequence.push_back(M); + TotalMutations.at(M.Identifier)++; return NewSize; } } @@ -532,4 +541,23 @@ {W, std::numeric_limits::max()}); } +void MutationDispatcher::CountCurrentMutatorSequence() { + for (const auto &M : CurrentMutatorSequence) + UsefulMutations.at(M.Identifier)++; +} + +void MutationDispatcher::PrintMutationStats() { + Printf("\nstat::mutation_usefulness: "); + for (int i = 0; i < kNumMutationTypes; i++) { + double UsefulPercentage = + TotalMutations.at(i) + ? (100.0 * UsefulMutations.at(i)) / TotalMutations.at(i) + : 0; + Printf("%.3f", UsefulPercentage); + if (i < kNumMutationTypes - 1) + Printf(","); + } + Printf("\n"); +} + } // namespace fuzzer Index: lib/fuzzer/FuzzerOptions.h =================================================================== --- lib/fuzzer/FuzzerOptions.h +++ lib/fuzzer/FuzzerOptions.h @@ -52,6 +52,8 @@ bool PrintNewCovPcs = false; int PrintNewCovFuncs = 0; bool PrintFinalStats = false; + bool PrintMutationStats = false; + bool PrintMutationUsefulness = false; bool PrintCorpusStats = false; bool PrintCoverage = false; bool DumpCoverage = false; Index: test/fuzzer/fuzzer-mutationstats.test =================================================================== --- /dev/null +++ test/fuzzer/fuzzer-mutationstats.test @@ -0,0 +1,3 @@ +RUN: %cpp_compiler %S/SimpleTest.cpp -o %t-MutationStatsTest +RUN: not %run %t-MutationStatsTest -max_total_time=360 -print_final_stats=1 -print_mutation_stats=1 -print_mutation_usefulness=1 2>&1 | FileCheck %s +CHECK: stat::mutation_usefulness: {{.*[0-9]+\.[0-9]+[1-9]+.*}}