Index: lib/fuzzer/CMakeLists.txt =================================================================== --- lib/fuzzer/CMakeLists.txt +++ lib/fuzzer/CMakeLists.txt @@ -12,6 +12,7 @@ FuzzerLoop.cpp FuzzerMerge.cpp FuzzerMutate.cpp + FuzzerMutationStats.cpp FuzzerSHA1.cpp FuzzerShmemFuchsia.cpp FuzzerShmemPosix.cpp Index: lib/fuzzer/FuzzerDefs.h =================================================================== --- lib/fuzzer/FuzzerDefs.h +++ lib/fuzzer/FuzzerDefs.h @@ -12,14 +12,15 @@ #ifndef LLVM_FUZZER_DEFS_H #define LLVM_FUZZER_DEFS_H +#include #include #include #include #include +#include +#include #include #include -#include -#include // Platform detection. #ifdef __linux__ Index: lib/fuzzer/FuzzerDriver.cpp =================================================================== --- lib/fuzzer/FuzzerDriver.cpp +++ lib/fuzzer/FuzzerDriver.cpp @@ -613,6 +613,8 @@ Options.PrintNewCovPcs = Flags.print_pcs; Options.PrintNewCovFuncs = Flags.print_funcs; Options.PrintFinalStats = Flags.print_final_stats; + Options.PrintMutationStats = Flags.print_mutation_stats; + Options.PrintMutationUsefulness = Flags.print_mutation_usefulness; Options.PrintCorpusStats = Flags.print_corpus_stats; Options.PrintCoverage = Flags.print_coverage; Options.DumpCoverage = Flags.dump_coverage; Index: lib/fuzzer/FuzzerFlags.def =================================================================== --- lib/fuzzer/FuzzerFlags.def +++ lib/fuzzer/FuzzerFlags.def @@ -153,3 +153,5 @@ FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") FUZZER_DEPRECATED_FLAG(use_clang_coverage) FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace") +FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental") +FUZZER_FLAG_INT(print_mutation_usefulness, 0, "Experimental") Index: lib/fuzzer/FuzzerLoop.cpp =================================================================== --- lib/fuzzer/FuzzerLoop.cpp +++ lib/fuzzer/FuzzerLoop.cpp @@ -13,6 +13,7 @@ #include "FuzzerIO.h" #include "FuzzerInternal.h" #include "FuzzerMutate.h" +#include "FuzzerMutationStats.h" #include "FuzzerRandom.h" #include "FuzzerShmem.h" #include "FuzzerTracePC.h" @@ -363,6 +364,12 @@ Printf("stat::new_units_added: %zd\n", NumberOfNewUnitsAdded); Printf("stat::slowest_unit_time_sec: %zd\n", TimeOfLongestUnitInSeconds); Printf("stat::peak_rss_mb: %zd\n", GetPeakRSSMb()); + + if (Options.PrintMutationStats) + MStats->PrintMutationCounts(); + + if (Options.PrintMutationUsefulness) + MStats->PrintMutationUsefulness(); } void Fuzzer::SetMaxInputLen(size_t MaxInputLen) { @@ -660,6 +667,7 @@ /*DuringInitialCorpusExecution*/ false); if (NewCov) { ReportNewCoverage(&II, {CurrentUnitData, CurrentUnitData + Size}); + MD.CountCurrentMutatorSequence(); break; // We will mutate this input more in the next rounds. } if (Options.ReduceDepth && !FoundUniqFeatures) Index: lib/fuzzer/FuzzerMutate.h =================================================================== --- lib/fuzzer/FuzzerMutate.h +++ lib/fuzzer/FuzzerMutate.h @@ -19,6 +19,25 @@ namespace fuzzer { +typedef enum { + ADD_WORD_FROM_MANUAL_DICTIONARY, + ADD_WORD_FROM_PERSISTENT_AUTO_DICTIONARY, + ADD_WORD_FROM_TORC, + CHANGE_ASCII_INTEGER, + CHANGE_BINARY_INTEGER, + CHANGE_BIT, + CHANGE_BYTE, + COPY_PART, + CROSS_OVER, + CUSTOM_CROSS_OVER, + CUSTOM, + ERASE_BYTES, + INSERT_BYTE, + INSERT_REPEATED_BYTES, + SHUFFLE_BYTES, + NUMBER_OF_MUTATION_TYPES +} MutationType; + class MutationDispatcher { public: MutationDispatcher(Random &Rand, const FuzzingOptions &Options); @@ -86,13 +105,15 @@ Random &GetRand() { return Rand; } -private: + void CountCurrentMutatorSequence(); struct Mutator { size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max); + MutationType Identifier; const char *Name; }; +private: size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, size_t MaxSize); size_t MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize, Index: lib/fuzzer/FuzzerMutate.cpp =================================================================== --- lib/fuzzer/FuzzerMutate.cpp +++ lib/fuzzer/FuzzerMutate.cpp @@ -10,6 +10,7 @@ //===----------------------------------------------------------------------===// #include "FuzzerMutate.h" +#include "FuzzerMutationStats.h" #include "FuzzerCorpus.h" #include "FuzzerDefs.h" #include "FuzzerExtFunctions.h" @@ -30,34 +31,39 @@ DefaultMutators.insert( DefaultMutators.begin(), { - {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"}, - {&MutationDispatcher::Mutate_InsertByte, "InsertByte"}, + {&MutationDispatcher::Mutate_EraseBytes, ERASE_BYTES, "EraseBytes"}, + {&MutationDispatcher::Mutate_InsertByte, INSERT_BYTE, "InsertByte"}, {&MutationDispatcher::Mutate_InsertRepeatedBytes, - "InsertRepeatedBytes"}, - {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"}, - {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"}, - {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"}, - {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"}, - {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"}, - {&MutationDispatcher::Mutate_CopyPart, "CopyPart"}, - {&MutationDispatcher::Mutate_CrossOver, "CrossOver"}, + INSERT_REPEATED_BYTES, "InsertRepeatedBytes"}, + {&MutationDispatcher::Mutate_ChangeByte, CHANGE_BYTE, "ChangeByte"}, + {&MutationDispatcher::Mutate_ChangeBit, CHANGE_BIT,"ChangeBit"}, + {&MutationDispatcher::Mutate_ShuffleBytes, SHUFFLE_BYTES, + "ShuffleBytes"}, + {&MutationDispatcher::Mutate_ChangeASCIIInteger, CHANGE_ASCII_INTEGER, + "ChangeASCIIInt"}, + {&MutationDispatcher::Mutate_ChangeBinaryInteger, + CHANGE_BINARY_INTEGER, "ChangeBinInt"}, + {&MutationDispatcher::Mutate_CopyPart, COPY_PART, "CopyPart"}, + {&MutationDispatcher::Mutate_CrossOver, CROSS_OVER, "CrossOver"}, {&MutationDispatcher::Mutate_AddWordFromManualDictionary, - "ManualDict"}, + ADD_WORD_FROM_MANUAL_DICTIONARY, "ManualDict"}, {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary, - "PersAutoDict"}, + ADD_WORD_FROM_PERSISTENT_AUTO_DICTIONARY, "PersAutoDict"}, }); if(Options.UseCmp) DefaultMutators.push_back( - {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"}); + {&MutationDispatcher::Mutate_AddWordFromTORC, ADD_WORD_FROM_TORC, + "CMP"}); if (EF->LLVMFuzzerCustomMutator) - Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"}); + Mutators.push_back({&MutationDispatcher::Mutate_Custom, CUSTOM, "Custom"}); else Mutators = DefaultMutators; if (EF->LLVMFuzzerCustomCrossOver) Mutators.push_back( - {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"}); + {&MutationDispatcher::Mutate_CustomCrossOver, CUSTOM_CROSS_OVER, + "CustomCrossOver"}); } static char RandCh(Random &Rand) { @@ -520,6 +526,7 @@ if (Options.OnlyASCII) ToASCII(Data, NewSize); CurrentMutatorSequence.push_back(M); + MStats->IncTotalMutationCount(M.Identifier); return NewSize; } } @@ -532,4 +539,10 @@ {W, std::numeric_limits::max()}); } +void MutationDispatcher::CountCurrentMutatorSequence() { + for (auto M : CurrentMutatorSequence) + // Increment using map look up and enum for index + MStats->IncUsefulMutationCount(M.Identifier); +} + } // namespace fuzzer Index: lib/fuzzer/FuzzerMutationStats.h =================================================================== --- /dev/null +++ lib/fuzzer/FuzzerMutationStats.h @@ -0,0 +1,33 @@ +//===- FuzzerMutationStats.h - Header for mutation tracking ----------*- C++ -* ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// fuzzer::MutationStats +//===----------------------------------------------------------------------===// + +#include "FuzzerMutate.h" + +namespace fuzzer { + +class MutationStats { +public: + ~MutationStats() {} + void PrintMutationCounts(); + void IncTotalMutationCount(MutationType MType); + void IncUsefulMutationCount(MutationType MType); + void PrintMutationUsefulness(); + +private: + // A total count of each mutation used in the fuzzing process. + std::array TotalMutations; + // The number of each mutation that resulted in new coverage. + std::array UsefulMutations; +}; + +extern MutationStats *MStats; + +} Index: lib/fuzzer/FuzzerMutationStats.cpp =================================================================== --- /dev/null +++ lib/fuzzer/FuzzerMutationStats.cpp @@ -0,0 +1,59 @@ +//===- FuzzerMutate.cpp - Mutate a test input -----------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Mutate a test input. +//===----------------------------------------------------------------------===// + +#include "FuzzerIO.h" +#include "FuzzerMutate.h" +#include "FuzzerMutationStats.h" + +namespace fuzzer { + +MutationStats *MStats = new MutationStats(); + +void MutationStats::PrintMutationCounts() { + // Outputs the number of each mutation used + Printf("\nstat::total_mutations: "); + for (auto& M : TotalMutations) { + Printf("%zd", M); + if (&M != &TotalMutations.back()) Printf(","); + } + + Printf("\nstat::useful_mutations: "); + for (auto& M : UsefulMutations) { + Printf("%zd", M); + if (&M != &UsefulMutations.back()) Printf(","); + } +} + +void MutationStats::PrintMutationUsefulness() { + Printf("\nstat::mutation_usefulness: "); + for (int i = 0; i < NUMBER_OF_MUTATION_TYPES; i++) { + double CurrentRatio = + (TotalMutations.at(i) != 0) + ? (UsefulMutations.at(i) / + static_cast(TotalMutations.at(i)) * 100) + : 0; + Printf("%.3f", CurrentRatio); + if (i < NUMBER_OF_MUTATION_TYPES-1) Printf(","); + else if (i == NUMBER_OF_MUTATION_TYPES-1) Printf("\n\n"); + } +} + +void MutationStats::IncTotalMutationCount(MutationType MType) { + assert(MType >= 0 && MType < NUMBER_OF_MUTATION_TYPES); + TotalMutations[MType]++; +} + +void MutationStats::IncUsefulMutationCount(MutationType MType) { + assert(MType >= 0 && MType < NUMBER_OF_MUTATION_TYPES); + UsefulMutations[MType]++; +} + +} // namespace fuzzer Index: lib/fuzzer/FuzzerOptions.h =================================================================== --- lib/fuzzer/FuzzerOptions.h +++ lib/fuzzer/FuzzerOptions.h @@ -52,6 +52,8 @@ bool PrintNewCovPcs = false; int PrintNewCovFuncs = 0; bool PrintFinalStats = false; + bool PrintMutationStats = false; + bool PrintMutationUsefulness = false; bool PrintCorpusStats = false; bool PrintCoverage = false; bool DumpCoverage = false; Index: test/fuzzer/fuzzer-mutationstats.test =================================================================== --- /dev/null +++ test/fuzzer/fuzzer-mutationstats.test @@ -0,0 +1,3 @@ +RUN: %cpp_compiler %S/SimpleTest.cpp -o %t-MutationStatsTest +RUN: not %run %t-MutationStatsTest -max_total_time=360 -print_final_stats=1 -print_mutation_stats=1 -print_mutation_usefulness=1 2>&1 | FileCheck %s +CHECK: stat::mutation_usefulness: {{.*[0-9]+\.[0-9]+[1-9]+.*}}