Index: lib/fuzzer/CMakeLists.txt =================================================================== --- lib/fuzzer/CMakeLists.txt +++ lib/fuzzer/CMakeLists.txt @@ -12,6 +12,7 @@ FuzzerLoop.cpp FuzzerMerge.cpp FuzzerMutate.cpp + FuzzerMutationStats.cpp FuzzerSHA1.cpp FuzzerShmemFuchsia.cpp FuzzerShmemPosix.cpp Index: lib/fuzzer/FuzzerDefs.h =================================================================== --- lib/fuzzer/FuzzerDefs.h +++ lib/fuzzer/FuzzerDefs.h @@ -12,14 +12,16 @@ #ifndef LLVM_FUZZER_DEFS_H #define LLVM_FUZZER_DEFS_H +#include #include #include #include #include +#include +#include #include +#include #include -#include -#include // Platform detection. #ifdef __linux__ Index: lib/fuzzer/FuzzerDriver.cpp =================================================================== --- lib/fuzzer/FuzzerDriver.cpp +++ lib/fuzzer/FuzzerDriver.cpp @@ -613,6 +613,7 @@ Options.PrintNewCovPcs = Flags.print_pcs; Options.PrintNewCovFuncs = Flags.print_funcs; Options.PrintFinalStats = Flags.print_final_stats; + Options.PrintMutationStats = Flags.print_mutation_stats; Options.PrintCorpusStats = Flags.print_corpus_stats; Options.PrintCoverage = Flags.print_coverage; Options.DumpCoverage = Flags.dump_coverage; Index: lib/fuzzer/FuzzerFlags.def =================================================================== --- lib/fuzzer/FuzzerFlags.def +++ lib/fuzzer/FuzzerFlags.def @@ -153,3 +153,4 @@ FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") FUZZER_DEPRECATED_FLAG(use_clang_coverage) FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace") +FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental") Index: lib/fuzzer/FuzzerLoop.cpp =================================================================== --- lib/fuzzer/FuzzerLoop.cpp +++ lib/fuzzer/FuzzerLoop.cpp @@ -13,6 +13,7 @@ #include "FuzzerIO.h" #include "FuzzerInternal.h" #include "FuzzerMutate.h" +#include "FuzzerMutationStats.h" #include "FuzzerRandom.h" #include "FuzzerShmem.h" #include "FuzzerTracePC.h" @@ -363,6 +364,9 @@ Printf("stat::new_units_added: %zd\n", NumberOfNewUnitsAdded); Printf("stat::slowest_unit_time_sec: %zd\n", TimeOfLongestUnitInSeconds); Printf("stat::peak_rss_mb: %zd\n", GetPeakRSSMb()); + + if (Options.PrintMutationStats) + MStats->PrintMutationCounts(); } void Fuzzer::SetMaxInputLen(size_t MaxInputLen) { @@ -660,6 +664,7 @@ /*DuringInitialCorpusExecution*/ false); if (NewCov) { ReportNewCoverage(&II, {CurrentUnitData, CurrentUnitData + Size}); + MD.CountCurrentMutatorSequence(); break; // We will mutate this input more in the next rounds. } if (Options.ReduceDepth && !FoundUniqFeatures) Index: lib/fuzzer/FuzzerMutate.h =================================================================== --- lib/fuzzer/FuzzerMutate.h +++ lib/fuzzer/FuzzerMutate.h @@ -86,13 +86,16 @@ Random &GetRand() { return Rand; } -private: + void CountCurrentMutatorSequence(); struct Mutator { size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max); - const char *Name; + MutationType Identifier; }; + +private: + size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, size_t MaxSize); size_t MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize, @@ -145,6 +148,24 @@ Vector DefaultMutators; }; +const std::unordered_map kMutationNames = { + {ManualDict, "AddWordFromManualDictionaryCount"}, + {PersAutoDict, "AddWordFromPersistentAutoDictionaryCount"}, + {CMP, "AddWordFromTORCCount"}, + {ChangeAsciiInt, "ChangeASCIIIntegerCount"}, + {ChangeBinInt, "ChangeBinaryIntegerCount"}, + {ChangeBit, "ChangeBitCount"}, + {CopyPart, "CopyPartCount"}, + {ChangeByte, "ChangeByteCount"}, + {CrossOver, "CrossOverCount"}, + {CustomCrossOver, "CustomCrossOverCount"}, + {CustomMutation, "CustomMutationCount"}, + {EraseBytes, "EraseBytesCount"}, + {InsertByte, "InsertByteCount"}, + {InsertRepeatedBytes, "InsertRepeatedBytesCount"}, + {ShuffleBytes, "ShuffleBytesCount"} +}; + } // namespace fuzzer #endif // LLVM_FUZZER_MUTATE_H Index: lib/fuzzer/FuzzerMutate.cpp =================================================================== --- lib/fuzzer/FuzzerMutate.cpp +++ lib/fuzzer/FuzzerMutate.cpp @@ -10,6 +10,7 @@ //===----------------------------------------------------------------------===// #include "FuzzerMutate.h" +#include "FuzzerMutationStats.h" #include "FuzzerCorpus.h" #include "FuzzerDefs.h" #include "FuzzerExtFunctions.h" @@ -30,34 +31,34 @@ DefaultMutators.insert( DefaultMutators.begin(), { - {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"}, - {&MutationDispatcher::Mutate_InsertByte, "InsertByte"}, + {&MutationDispatcher::Mutate_EraseBytes, EraseBytes}, + {&MutationDispatcher::Mutate_InsertByte, InsertByte}, {&MutationDispatcher::Mutate_InsertRepeatedBytes, - "InsertRepeatedBytes"}, - {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"}, - {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"}, - {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"}, - {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"}, - {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"}, - {&MutationDispatcher::Mutate_CopyPart, "CopyPart"}, - {&MutationDispatcher::Mutate_CrossOver, "CrossOver"}, + InsertRepeatedBytes}, + {&MutationDispatcher::Mutate_ChangeByte, ChangeByte}, + {&MutationDispatcher::Mutate_ChangeBit, ChangeBit}, + {&MutationDispatcher::Mutate_ShuffleBytes, ShuffleBytes}, + {&MutationDispatcher::Mutate_ChangeASCIIInteger, ChangeAsciiInt}, + {&MutationDispatcher::Mutate_ChangeBinaryInteger, ChangeBinInt}, + {&MutationDispatcher::Mutate_CopyPart, CopyPart}, + {&MutationDispatcher::Mutate_CrossOver, CrossOver}, {&MutationDispatcher::Mutate_AddWordFromManualDictionary, - "ManualDict"}, + ManualDict}, {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary, - "PersAutoDict"}, + PersAutoDict}, }); if(Options.UseCmp) DefaultMutators.push_back( - {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"}); + {&MutationDispatcher::Mutate_AddWordFromTORC, CMP}); if (EF->LLVMFuzzerCustomMutator) - Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"}); + Mutators.push_back({&MutationDispatcher::Mutate_Custom, CustomMutation}); else Mutators = DefaultMutators; if (EF->LLVMFuzzerCustomCrossOver) Mutators.push_back( - {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"}); + {&MutationDispatcher::Mutate_CustomCrossOver, CustomCrossOver}); } static char RandCh(Random &Rand) { @@ -68,6 +69,7 @@ size_t MutationDispatcher::Mutate_Custom(uint8_t *Data, size_t Size, size_t MaxSize) { + MStats->IncrementCount(CustomMutation, 0); return EF->LLVMFuzzerCustomMutator(Data, Size, MaxSize, Rand.Rand()); } @@ -87,6 +89,7 @@ return 0; assert(NewSize <= MaxSize && "CustomCrossOver returned overisized unit"); memcpy(Data, U.data(), NewSize); + MStats->IncrementCount(CustomCrossOver, 0); return NewSize; } @@ -98,6 +101,7 @@ size_t ShuffleStart = Rand(Size - ShuffleAmount); assert(ShuffleStart + ShuffleAmount <= Size); std::shuffle(Data + ShuffleStart, Data + ShuffleStart + ShuffleAmount, Rand); + MStats->IncrementCount(ShuffleBytes, 0); return Size; } @@ -110,6 +114,7 @@ // Erase Data[Idx:Idx+N]. memmove(Data + Idx, Data + Idx + N, Size - Idx - N); // Printf("Erase: %zd %zd => %zd; Idx %zd\n", N, Size, Size - N, Idx); + MStats->IncrementCount(EraseBytes, 0); return Size - N; } @@ -120,6 +125,7 @@ // Insert new value at Data[Idx]. memmove(Data + Idx + 1, Data + Idx, Size - Idx); Data[Idx] = RandCh(Rand); + MStats->IncrementCount(InsertByte, 0); return Size + 1; } @@ -138,6 +144,7 @@ uint8_t Byte = Rand.RandBool() ? Rand(256) : (Rand.RandBool() ? 0 : 255); for (size_t i = 0; i < N; i++) Data[Idx + i] = Byte; + MStats->IncrementCount(InsertRepeatedBytes, 0); return Size + N; } @@ -146,6 +153,7 @@ if (Size > MaxSize) return 0; size_t Idx = Rand(Size); Data[Idx] = RandCh(Rand); + MStats->IncrementCount(ChangeByte, 0); return Size; } @@ -154,12 +162,14 @@ if (Size > MaxSize) return 0; size_t Idx = Rand(Size); Data[Idx] ^= 1 << Rand(8); + MStats->IncrementCount(ChangeBit, 0); return Size; } size_t MutationDispatcher::Mutate_AddWordFromManualDictionary(uint8_t *Data, size_t Size, size_t MaxSize) { + MStats->IncrementCount(ManualDict, 0); return AddWordFromDictionary(ManualDictionary, Data, Size, MaxSize); } @@ -275,11 +285,13 @@ kCmpDictionaryEntriesDequeSize]; DERef = DE; CurrentDictionaryEntrySequence.push_back(&DERef); + MStats->IncrementCount(CMP, 0); return Size; } size_t MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary( uint8_t *Data, size_t Size, size_t MaxSize) { + MStats->IncrementCount(PersAutoDict, 0); return AddWordFromDictionary(PersistentAutoDictionary, Data, Size, MaxSize); } @@ -338,6 +350,7 @@ size_t MutationDispatcher::Mutate_CopyPart(uint8_t *Data, size_t Size, size_t MaxSize) { + MStats->IncrementCount(CopyPart, 0); if (Size > MaxSize || Size == 0) return 0; // If Size == MaxSize, `InsertPartOf(...)` will // fail so there's no point using it in this case. @@ -378,6 +391,7 @@ Data[Idx] = (Val % 10) + '0'; Val /= 10; } + MStats->IncrementCount(ChangeAsciiInt, 0); return Size; } @@ -409,6 +423,7 @@ size_t MutationDispatcher::Mutate_ChangeBinaryInteger(uint8_t *Data, size_t Size, size_t MaxSize) { + MStats->IncrementCount(ChangeBinInt, 0); if (Size > MaxSize) return 0; switch (Rand(4)) { case 3: return ChangeBinaryInteger(Data, Size, Rand); @@ -447,6 +462,7 @@ assert(NewSize > 0 && "CrossOver returned empty unit"); assert(NewSize <= MaxSize && "CrossOver returned overisized unit"); memcpy(Data, U.data(), NewSize); + MStats->IncrementCount(CrossOver, 0); return NewSize; } @@ -485,8 +501,10 @@ void MutationDispatcher::PrintMutationSequence() { Printf("MS: %zd ", CurrentMutatorSequence.size()); - for (auto M : CurrentMutatorSequence) - Printf("%s-", M.Name); + for (auto M : CurrentMutatorSequence) { + auto currentMutation = kMutationNames.find(M.Identifier); + Printf("%s-", currentMutation->second.c_str()); + } if (!CurrentDictionaryEntrySequence.empty()) { Printf(" DE: "); for (auto DE : CurrentDictionaryEntrySequence) { @@ -532,4 +550,10 @@ {W, std::numeric_limits::max()}); } +void MutationDispatcher::CountCurrentMutatorSequence() { + for (auto M : CurrentMutatorSequence) + // Increment using map look up and enum for index + MStats->IncrementCount(M.Identifier, 1); +} + } // namespace fuzzer Index: lib/fuzzer/FuzzerMutationStats.h =================================================================== --- /dev/null +++ lib/fuzzer/FuzzerMutationStats.h @@ -0,0 +1,50 @@ +//===- FuzzerMutationStats.h - Header for mutation tracking ----------*- C++ -* ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// fuzzer::MutationStats +//===----------------------------------------------------------------------===// + + +namespace fuzzer { + +enum MutationType { + ManualDict, + PersAutoDict, + CMP, + ChangeAsciiInt, + ChangeBinInt, + ChangeBit, + ChangeByte, + CopyPart, + CrossOver, + CustomCrossOver, + CustomMutation, + EraseBytes, + InsertByte, + InsertRepeatedBytes, + ShuffleBytes, + MaxNumberOfMutationTypes +}; + +class MutationStats { +public: + ~MutationStats() {} + void PrintMutationCounts(); + void IncTotalMutationCount(MutationType MType); + void IncUsefulMutationCount(MutationType MType); + +private: + // A total count of each mutation used in the fuzzing process. + std::array TotalMutations; + // The number of each mutation that resulted in new coverage. + std::array UsefulMutations; +}; + +extern MutationStats *MStats; + +} Index: lib/fuzzer/FuzzerMutationStats.cpp =================================================================== --- /dev/null +++ lib/fuzzer/FuzzerMutationStats.cpp @@ -0,0 +1,55 @@ +//===- FuzzerMutate.cpp - Mutate a test input -----------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Mutate a test input. +//===----------------------------------------------------------------------===// + +#include "FuzzerMutate.h" +#include "FuzzerMutationStats.h" +#include "FuzzerIO.h" + +#define NAMEOFSTAT(var) #var + +namespace fuzzer { + +MutationStats* MStats = new MutationStats(); + +void MutationStats::PrintMutationCounts() { + // Outputs the number of each mutation used + + Printf("\nTotal Mutations ----------\n"); + for (int i = 0; i < MaxNumberOfMutationTypes; i++) { + auto current = kMutationNames.find(i); + Printf("%s: %d\n", current->second.c_str(), TotalMutations.at(i)); + } + + Printf("\nUseful Mutations ----------\n"); + for (int i = 0; i < MaxNumberOfMutationTypes; i++) { + auto current = kMutationNames.find(i); + Printf("%s: %d\n", current->second.c_str(), UsefulMutations.at(i)); + } + +} + +// Updates the count of mutations (total/useful) depending on type. +// Types: (0) Total, (1) Useful +void MutationStats::IncTotalMutationCount(MutationType MType) { + if ((MType < 0) || MType >= MaxNumberOfMutationTypes) + return; + else + TotalMutations[MType]++; +} + +void MutationStats::IncTotalMutationCount(MutationType MType) { + if ((MType < 0) || MType >= MaxNumberOfMutationTypes) + return; + else + UsefulMutations[MType]++; +} + +} // namespace fuzzer Index: lib/fuzzer/FuzzerOptions.h =================================================================== --- lib/fuzzer/FuzzerOptions.h +++ lib/fuzzer/FuzzerOptions.h @@ -52,6 +52,7 @@ bool PrintNewCovPcs = false; int PrintNewCovFuncs = 0; bool PrintFinalStats = false; + bool PrintMutationStats = false; bool PrintCorpusStats = false; bool PrintCoverage = false; bool DumpCoverage = false;