Index: FuzzerDefs.h =================================================================== --- FuzzerDefs.h +++ FuzzerDefs.h @@ -16,10 +16,10 @@ #include #include #include +#include +#include #include #include -#include -#include // Platform detection. #ifdef __linux__ Index: FuzzerDriver.cpp =================================================================== --- FuzzerDriver.cpp +++ FuzzerDriver.cpp @@ -613,6 +613,7 @@ Options.PrintNewCovPcs = Flags.print_pcs; Options.PrintNewCovFuncs = Flags.print_funcs; Options.PrintFinalStats = Flags.print_final_stats; + Options.PrintMutationStats = Flags.print_mutation_stats; Options.PrintCorpusStats = Flags.print_corpus_stats; Options.PrintCoverage = Flags.print_coverage; Options.DumpCoverage = Flags.dump_coverage; Index: FuzzerFlags.def =================================================================== --- FuzzerFlags.def +++ FuzzerFlags.def @@ -153,3 +153,4 @@ FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") FUZZER_DEPRECATED_FLAG(use_clang_coverage) FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace") +FUZZER_FLAG_INT(print_mutation_stats, 0, "Experimental") Index: FuzzerLoop.cpp =================================================================== --- FuzzerLoop.cpp +++ FuzzerLoop.cpp @@ -355,6 +355,8 @@ TPC.DumpCoverage(); if (Options.PrintCorpusStats) Corpus.PrintStats(); + if (Options.PrintMutationStats) + MD.PrintMutationStats(); if (!Options.PrintFinalStats) return; size_t ExecPerSec = execPerSec(); Index: FuzzerMutate.h =================================================================== --- FuzzerMutate.h +++ FuzzerMutate.h @@ -86,13 +86,16 @@ Random &GetRand() { return Rand; } -private: + void PrintMutationStats(); + + void CountCurrentMutatorSequence(); struct Mutator { size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max); const char *Name; }; +private: size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, size_t MaxSize); size_t MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize, @@ -130,6 +133,7 @@ Vector CurrentMutatorSequence; Vector CurrentDictionaryEntrySequence; + Vector CurrentMutatorIdxSequence; static const size_t kCmpDictionaryEntriesDequeSize = 16; DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; @@ -143,6 +147,11 @@ Vector Mutators; Vector DefaultMutators; + + // A total count of each mutation used in the fuzzing process. + Vector TotalMutations; + // The number of each mutation that resulted in new coverage. + Vector UsefulMutations; }; } // namespace fuzzer Index: FuzzerMutate.cpp =================================================================== --- FuzzerMutate.cpp +++ FuzzerMutate.cpp @@ -46,7 +46,7 @@ {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary, "PersAutoDict"}, }); - if(Options.UseCmp) + if (Options.UseCmp) DefaultMutators.push_back( {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"}); @@ -58,10 +58,15 @@ if (EF->LLVMFuzzerCustomCrossOver) Mutators.push_back( {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"}); + + // Initialize mutation statistic counters. + TotalMutations.resize(Mutators.size(), 0); + UsefulMutations.resize(Mutators.size(), 0); } static char RandCh(Random &Rand) { - if (Rand.RandBool()) return Rand(256); + if (Rand.RandBool()) + return Rand(256); const char Special[] = "!*'();:@&=+$,/?%#[]012Az-`~.\xff\x00"; return Special[Rand(sizeof(Special) - 1)]; } @@ -92,7 +97,8 @@ size_t MutationDispatcher::Mutate_ShuffleBytes(uint8_t *Data, size_t Size, size_t MaxSize) { - if (Size > MaxSize || Size == 0) return 0; + if (Size > MaxSize || Size == 0) + return 0; size_t ShuffleAmount = Rand(std::min(Size, (size_t)8)) + 1; // [1,8] and <= Size. size_t ShuffleStart = Rand(Size - ShuffleAmount); @@ -103,7 +109,8 @@ size_t MutationDispatcher::Mutate_EraseBytes(uint8_t *Data, size_t Size, size_t MaxSize) { - if (Size <= 1) return 0; + if (Size <= 1) + return 0; size_t N = Rand(Size / 2) + 1; assert(N < Size); size_t Idx = Rand(Size - N + 1); @@ -115,7 +122,8 @@ size_t MutationDispatcher::Mutate_InsertByte(uint8_t *Data, size_t Size, size_t MaxSize) { - if (Size >= MaxSize) return 0; + if (Size >= MaxSize) + return 0; size_t Idx = Rand(Size + 1); // Insert new value at Data[Idx]. memmove(Data + Idx + 1, Data + Idx, Size - Idx); @@ -127,7 +135,8 @@ size_t Size, size_t MaxSize) { const size_t kMinBytesToInsert = 3; - if (Size + kMinBytesToInsert >= MaxSize) return 0; + if (Size + kMinBytesToInsert >= MaxSize) + return 0; size_t MaxBytesToInsert = std::min(MaxSize - Size, (size_t)128); size_t N = Rand(MaxBytesToInsert - kMinBytesToInsert + 1) + kMinBytesToInsert; assert(Size + N <= MaxSize && N); @@ -143,7 +152,8 @@ size_t MutationDispatcher::Mutate_ChangeByte(uint8_t *Data, size_t Size, size_t MaxSize) { - if (Size > MaxSize) return 0; + if (Size > MaxSize) + return 0; size_t Idx = Rand(Size); Data[Idx] = RandCh(Rand); return Size; @@ -151,7 +161,8 @@ size_t MutationDispatcher::Mutate_ChangeBit(uint8_t *Data, size_t Size, size_t MaxSize) { - if (Size > MaxSize) return 0; + if (Size > MaxSize) + return 0; size_t Idx = Rand(Size); Data[Idx] ^= 1 << Rand(8); return Size; @@ -170,14 +181,16 @@ bool UsePositionHint = DE.HasPositionHint() && DE.GetPositionHint() + W.size() < Size && Rand.RandBool(); - if (Rand.RandBool()) { // Insert W. - if (Size + W.size() > MaxSize) return 0; + if (Rand.RandBool()) { // Insert W. + if (Size + W.size() > MaxSize) + return 0; size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size + 1); memmove(Data + Idx + W.size(), Data + Idx, Size - Idx); memcpy(Data + Idx, W.data(), W.size()); Size += W.size(); - } else { // Overwrite some bytes with W. - if (W.size() > Size) return 0; + } else { // Overwrite some bytes with W. + if (W.size() > Size) + return 0; size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size - W.size()); memcpy(Data + Idx, W.data(), W.size()); } @@ -191,9 +204,8 @@ // input and if it succeeds it creates a DE with a position hint. // Otherwise it creates a DE with one of the arguments w/o a position hint. DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP( - const void *Arg1, const void *Arg2, - const void *Arg1Mutation, const void *Arg2Mutation, - size_t ArgSize, const uint8_t *Data, + const void *Arg1, const void *Arg2, const void *Arg1Mutation, + const void *Arg2Mutation, size_t ArgSize, const uint8_t *Data, size_t Size) { ScopedDoingMyOwnMemOrStr scoped_doing_my_own_mem_os_str; bool HandleFirst = Rand.RandBool(); @@ -204,7 +216,7 @@ ExistingBytes = HandleFirst ? Arg1 : Arg2; DesiredBytes = HandleFirst ? Arg2Mutation : Arg1Mutation; HandleFirst = !HandleFirst; - W.Set(reinterpret_cast(DesiredBytes), ArgSize); + W.Set(reinterpret_cast(DesiredBytes), ArgSize); const size_t kMaxNumPositions = 8; size_t Positions[kMaxNumPositions]; size_t NumPositions = 0; @@ -212,22 +224,25 @@ Cur < End && NumPositions < kMaxNumPositions; Cur++) { Cur = (const uint8_t *)SearchMemory(Cur, End - Cur, ExistingBytes, ArgSize); - if (!Cur) break; + if (!Cur) + break; Positions[NumPositions++] = Cur - Data; } - if (!NumPositions) continue; + if (!NumPositions) + continue; return DictionaryEntry(W, Positions[Rand(NumPositions)]); } DictionaryEntry DE(W); return DE; } - template DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP( T Arg1, T Arg2, const uint8_t *Data, size_t Size) { - if (Rand.RandBool()) Arg1 = Bswap(Arg1); - if (Rand.RandBool()) Arg2 = Bswap(Arg2); + if (Rand.RandBool()) + Arg1 = Bswap(Arg1); + if (Rand.RandBool()) + Arg2 = Bswap(Arg2); T Arg1Mutation = Arg1 + Rand(-1, 1); T Arg2Mutation = Arg2 + Rand(-1, 1); return MakeDictionaryEntryFromCMP(&Arg1, &Arg2, &Arg1Mutation, &Arg2Mutation, @@ -240,8 +255,8 @@ Arg2.data(), Arg1.size(), Data, Size); } -size_t MutationDispatcher::Mutate_AddWordFromTORC( - uint8_t *Data, size_t Size, size_t MaxSize) { +size_t MutationDispatcher::Mutate_AddWordFromTORC(uint8_t *Data, size_t Size, + size_t MaxSize) { Word W; DictionaryEntry DE; switch (Rand(4)) { @@ -260,16 +275,20 @@ auto X = TPC.TORCW.Get(Rand.Rand()); DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); } break; - case 3: if (Options.UseMemmem) { - auto X = TPC.MMT.Get(Rand.Rand()); - DE = DictionaryEntry(X); - } break; + case 3: + if (Options.UseMemmem) { + auto X = TPC.MMT.Get(Rand.Rand()); + DE = DictionaryEntry(X); + } + break; default: assert(0); } - if (!DE.GetW().size()) return 0; + if (!DE.GetW().size()) + return 0; Size = ApplyDictionaryEntry(Data, Size, MaxSize, DE); - if (!Size) return 0; + if (!Size) + return 0; DictionaryEntry &DERef = CmpDictionaryEntriesDeque[CmpDictionaryEntriesDequeIdx++ % kCmpDictionaryEntriesDequeSize]; @@ -285,11 +304,14 @@ size_t MutationDispatcher::AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, size_t MaxSize) { - if (Size > MaxSize) return 0; - if (D.empty()) return 0; + if (Size > MaxSize) + return 0; + if (D.empty()) + return 0; DictionaryEntry &DE = D[Rand(D.size())]; Size = ApplyDictionaryEntry(Data, Size, MaxSize, DE); - if (!Size) return 0; + if (!Size) + return 0; DE.IncUseCount(); CurrentDictionaryEntrySequence.push_back(&DE); return Size; @@ -315,7 +337,8 @@ size_t MutationDispatcher::InsertPartOf(const uint8_t *From, size_t FromSize, uint8_t *To, size_t ToSize, size_t MaxToSize) { - if (ToSize >= MaxToSize) return 0; + if (ToSize >= MaxToSize) + return 0; size_t AvailableSpace = MaxToSize - ToSize; size_t MaxCopySize = std::min(AvailableSpace, FromSize); size_t CopySize = Rand(MaxCopySize) + 1; @@ -338,7 +361,8 @@ size_t MutationDispatcher::Mutate_CopyPart(uint8_t *Data, size_t Size, size_t MaxSize) { - if (Size > MaxSize || Size == 0) return 0; + if (Size > MaxSize || Size == 0) + return 0; // If Size == MaxSize, `InsertPartOf(...)` will // fail so there's no point using it in this case. if (Size == MaxSize || Rand.RandBool()) @@ -349,12 +373,16 @@ size_t MutationDispatcher::Mutate_ChangeASCIIInteger(uint8_t *Data, size_t Size, size_t MaxSize) { - if (Size > MaxSize) return 0; + if (Size > MaxSize) + return 0; size_t B = Rand(Size); - while (B < Size && !isdigit(Data[B])) B++; - if (B == Size) return 0; + while (B < Size && !isdigit(Data[B])) + B++; + if (B == Size) + return 0; size_t E = B; - while (E < Size && isdigit(Data[E])) E++; + while (E < Size && isdigit(Data[E])) + E++; assert(B < E); // now we have digits in [B, E). // strtol and friends don't accept non-zero-teminated data, parse it manually. @@ -363,13 +391,24 @@ Val = Val * 10 + Data[i] - '0'; // Mutate the integer value. - switch(Rand(5)) { - case 0: Val++; break; - case 1: Val--; break; - case 2: Val /= 2; break; - case 3: Val *= 2; break; - case 4: Val = Rand(Val * Val); break; - default: assert(0); + switch (Rand(5)) { + case 0: + Val++; + break; + case 1: + Val--; + break; + case 2: + Val /= 2; + break; + case 3: + Val *= 2; + break; + case 4: + Val = Rand(Val * Val); + break; + default: + assert(0); } // Just replace the bytes with the new ones, don't bother moving bytes. for (size_t i = B; i < E; i++) { @@ -381,9 +420,10 @@ return Size; } -template +template size_t ChangeBinaryInteger(uint8_t *Data, size_t Size, Random &Rand) { - if (Size < sizeof(T)) return 0; + if (Size < sizeof(T)) + return 0; size_t Off = Rand(Size - sizeof(T) + 1); assert(Off + sizeof(T) <= Size); T Val; @@ -409,40 +449,50 @@ size_t MutationDispatcher::Mutate_ChangeBinaryInteger(uint8_t *Data, size_t Size, size_t MaxSize) { - if (Size > MaxSize) return 0; + if (Size > MaxSize) + return 0; switch (Rand(4)) { - case 3: return ChangeBinaryInteger(Data, Size, Rand); - case 2: return ChangeBinaryInteger(Data, Size, Rand); - case 1: return ChangeBinaryInteger(Data, Size, Rand); - case 0: return ChangeBinaryInteger(Data, Size, Rand); - default: assert(0); + case 3: + return ChangeBinaryInteger(Data, Size, Rand); + case 2: + return ChangeBinaryInteger(Data, Size, Rand); + case 1: + return ChangeBinaryInteger(Data, Size, Rand); + case 0: + return ChangeBinaryInteger(Data, Size, Rand); + default: + assert(0); } return 0; } size_t MutationDispatcher::Mutate_CrossOver(uint8_t *Data, size_t Size, size_t MaxSize) { - if (Size > MaxSize) return 0; - if (!Corpus || Corpus->size() < 2 || Size == 0) return 0; + if (Size > MaxSize) + return 0; + if (!Corpus || Corpus->size() < 2 || Size == 0) + return 0; size_t Idx = Rand(Corpus->size()); const Unit &O = (*Corpus)[Idx]; - if (O.empty()) return 0; + if (O.empty()) + return 0; MutateInPlaceHere.resize(MaxSize); auto &U = MutateInPlaceHere; size_t NewSize = 0; - switch(Rand(3)) { - case 0: - NewSize = CrossOver(Data, Size, O.data(), O.size(), U.data(), U.size()); - break; - case 1: - NewSize = InsertPartOf(O.data(), O.size(), U.data(), U.size(), MaxSize); - if (!NewSize) - NewSize = CopyPartOf(O.data(), O.size(), U.data(), U.size()); - break; - case 2: + switch (Rand(3)) { + case 0: + NewSize = CrossOver(Data, Size, O.data(), O.size(), U.data(), U.size()); + break; + case 1: + NewSize = InsertPartOf(O.data(), O.size(), U.data(), U.size(), MaxSize); + if (!NewSize) NewSize = CopyPartOf(O.data(), O.size(), U.data(), U.size()); - break; - default: assert(0); + break; + case 2: + NewSize = CopyPartOf(O.data(), O.size(), U.data(), U.size()); + break; + default: + assert(0); } assert(NewSize > 0 && "CrossOver returned empty unit"); assert(NewSize <= MaxSize && "CrossOver returned overisized unit"); @@ -452,6 +502,7 @@ void MutationDispatcher::StartMutationSequence() { CurrentMutatorSequence.clear(); + CurrentMutatorIdxSequence.clear(); CurrentDictionaryEntrySequence.clear(); } @@ -465,6 +516,7 @@ if (!PersistentAutoDictionary.ContainsWord(DE->GetW())) PersistentAutoDictionary.push_back({DE->GetW(), 1}); } + CountCurrentMutatorSequence(); } void MutationDispatcher::PrintRecommendedDictionary() { @@ -472,9 +524,10 @@ for (auto &DE : PersistentAutoDictionary) if (!ManualDictionary.ContainsWord(DE.GetW())) V.push_back(DE); - if (V.empty()) return; + if (V.empty()) + return; Printf("###### Recommended dictionary. ######\n"); - for (auto &DE: V) { + for (auto &DE : V) { assert(DE.GetW().size()); Printf("\""); PrintASCII(DE.GetW(), "\""); @@ -514,22 +567,43 @@ // in which case they will return 0. // Try several times before returning un-mutated data. for (int Iter = 0; Iter < 100; Iter++) { - auto M = Mutators[Rand(Mutators.size())]; + size_t MutatorIdx = Rand(Mutators.size()); + auto M = Mutators[MutatorIdx]; size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize); if (NewSize && NewSize <= MaxSize) { if (Options.OnlyASCII) ToASCII(Data, NewSize); CurrentMutatorSequence.push_back(M); + CurrentMutatorIdxSequence.push_back(MutatorIdx); + TotalMutations[MutatorIdx]++; return NewSize; } } *Data = ' '; - return 1; // Fallback, should not happen frequently. + return 1; // Fallback, should not happen frequently. } void MutationDispatcher::AddWordToManualDictionary(const Word &W) { - ManualDictionary.push_back( - {W, std::numeric_limits::max()}); + ManualDictionary.push_back({W, std::numeric_limits::max()}); +} + +void MutationDispatcher::CountCurrentMutatorSequence() { + for (const size_t M : CurrentMutatorIdxSequence) + UsefulMutations[M]++; +} + +void MutationDispatcher::PrintMutationStats() { + Printf("\nstat::mutation_usefulness: "); + for (unsigned int i = 0; i < Mutators.size(); i++) { + double UsefulPercentage = + TotalMutations[i] + ? (100.0 * UsefulMutations[i]) / TotalMutations[i] + : 0; + Printf("%.3f", UsefulPercentage); + if (i < Mutators.size() - 1) + Printf(","); + } + Printf("\n"); } -} // namespace fuzzer +} // namespace fuzzer Index: FuzzerOptions.h =================================================================== --- FuzzerOptions.h +++ FuzzerOptions.h @@ -52,6 +52,7 @@ bool PrintNewCovPcs = false; int PrintNewCovFuncs = 0; bool PrintFinalStats = false; + bool PrintMutationStats = false; bool PrintCorpusStats = false; bool PrintCoverage = false; bool DumpCoverage = false;