[libfuzzer] hiding custom mutator handling in MutationDispatcher.

[aizatsky]

Summary

Refactoring, no functional changes.

[kcc]

lib/Fuzzer/FuzzerInternal.h

474 ↗

(On Diff #59586)

This works now, but Dan was going to extend this with more weak functions (__sanitizer_*) and EF will be needed in other places of libFuzzer.
Please leave it here.

lib/Fuzzer/FuzzerMutate.cpp

43 ↗

(On Diff #59586)

Why not Mutators = DefaultMUtators?
also remove {}

253 ↗

(On Diff #59586)

Probably also change LLVMFuzzerMutate to call this one.

[aizatsky]

addressing comments

[aizatsky]

all done.

lib/Fuzzer/FuzzerMutate.cpp

253 ↗

(On Diff #59586)

It already calls it.

[kcc]

LGTM with a nit

lib/Fuzzer/FuzzerMutate.cpp

42 ↗

(On Diff #59614)

remove these {} too

llvm/trunk/lib/Fuzzer/FuzzerInternal.h

 
 private:
   DictionaryEntry DE[kMaxDictSize];
   size_t Size = 0;
 };
 
 class MutationDispatcher {
 public:
-  MutationDispatcher(Random &Rand) : Rand(Rand) {}
+  MutationDispatcher(Random &Rand);
   ~MutationDispatcher() {}
   /// Indicate that we are about to start a new sequence of mutations.
   void StartMutationSequence();
   /// Print the current sequence of mutations.
   void PrintMutationSequence();
   /// Indicate that the current sequence of mutations was successfull.
   void RecordSuccessfulMutationSequence();
+  /// Mutates data by invoking user-provided mutator.
+  size_t Mutate_Custom(uint8_t *Data, size_t Size, size_t MaxSize);
   /// Mutates data by shuffling bytes.
   size_t Mutate_ShuffleBytes(uint8_t *Data, size_t Size, size_t MaxSize);
   /// Mutates data by erasing a byte.
   size_t Mutate_EraseByte(uint8_t *Data, size_t Size, size_t MaxSize);
   /// Mutates data by inserting a byte.
   size_t Mutate_InsertByte(uint8_t *Data, size_t Size, size_t MaxSize);
   /// Mutates data by chanding one byte.
   size_t Mutate_ChangeByte(uint8_t *Data, size_t Size, size_t MaxSize);
size_t Mutate_AddWordFromPersistentAutoDictionary(uint8_t *Data, size_t Size,
                                                     size_t MaxSize);
 
   /// Tries to find an ASCII integer in Data, changes it to another ASCII int.
   size_t Mutate_ChangeASCIIInteger(uint8_t *Data, size_t Size, size_t MaxSize);
 
   /// CrossOver Data with some other element of the corpus.
   size_t Mutate_CrossOver(uint8_t *Data, size_t Size, size_t MaxSize);
 
-  /// Applies one of the above mutations.
+  /// Applies one of the configured mutations.
   /// Returns the new size of data which could be up to MaxSize.
   size_t Mutate(uint8_t *Data, size_t Size, size_t MaxSize);
+  /// Applies one of the default mutations. Provided as a service
+  /// to mutation authors.
+  size_t DefaultMutate(uint8_t *Data, size_t Size, size_t MaxSize);
 
   /// Creates a cross-over of two pieces of Data, returns its size.
   size_t CrossOver(const uint8_t *Data1, size_t Size1, const uint8_t *Data2,
                    size_t Size2, uint8_t *Out, size_t MaxOutSize);
 
   void AddWordToManualDictionary(const Word &W);
 
   void AddWordToAutoDictionary(const Word &W, size_t PositionHint);
   void ClearAutoDictionary();
   void PrintRecommendedDictionary();
 
   void SetCorpus(const std::vector<Unit> *Corpus) { this->Corpus = Corpus; }
 
   Random &GetRand() { return Rand; }
 
 private:
 
   struct Mutator {
     size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max);
     const char *Name;
   };
 
   size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size,
                                size_t MaxSize);
+  size_t MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize,
+                    const std::vector<Mutator> &Mutators);
+
+  // Interface to functions that may or may not be available.
+  const ExternalFunctions EF;
 
   Random &Rand;
   // Dictionary provided by the user via -dict=DICT_FILE.
   Dictionary ManualDictionary;
   // Temporary dictionary modified by the fuzzer itself,
   // recreated periodically.
   Dictionary TempAutoDictionary;
   // Persistent dictionary modified by the fuzzer, consists of
   // entries that led to successfull discoveries in the past mutations.
   Dictionary PersistentAutoDictionary;
   std::vector<Mutator> CurrentMutatorSequence;
   std::vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
   const std::vector<Unit> *Corpus = nullptr;
   std::vector<uint8_t> MutateInPlaceHere;
 
-  static Mutator Mutators[];
+  std::vector<Mutator> Mutators;
+  std::vector<Mutator> DefaultMutators;
 };
 
 class Fuzzer {
 public:
   struct FuzzingOptions {
     int Verbosity = 1;
     size_t MaxLen = 0;
     int UnitTimeoutSec = 300;
private:
 
   // Maximum recorded coverage.
   Coverage MaxCoverage;
 
   // Need to know our own thread.
   static thread_local bool IsMyThread;
 
   // Interface to functions that may or may not be available.
-  ExternalFunctions EF;
+  // For future use, currently not used.
+  const ExternalFunctions EF;
 };
 
 }; // namespace fuzzer
 
 #endif // LLVM_FUZZER_INTERNAL_H

llvm/trunk/lib/Fuzzer/FuzzerLoop.cpp

void Fuzzer::MutateAndTestOne() {
   auto &U = ChooseUnitToMutate();
   assert(CurrentUnitData);
   size_t Size = U.size();
   assert(Size <= Options.MaxLen && "Oversized Unit");
   memcpy(CurrentUnitData, U.data(), Size);
 
   for (int i = 0; i < Options.MutateDepth; i++) {
     size_t NewSize = 0;
-    if (EF.LLVMFuzzerCustomMutator)
-      NewSize = EF.LLVMFuzzerCustomMutator(CurrentUnitData, Size,
-                                           Options.MaxLen, MD.GetRand().Rand());
-    else
-      NewSize = MD.Mutate(CurrentUnitData, Size, Options.MaxLen);
     assert(NewSize > 0 && "Mutator returned empty unit");
     assert(NewSize <= Options.MaxLen &&
            "Mutator return overisized unit");
     Size = NewSize;
     if (Options.OnlyASCII)
       ToASCII(CurrentUnitData, Size);
     if (i == 0)
       StartTraceRecording();

 }
 
 } // namespace fuzzer
 
 extern "C" {
 
 size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) {
   assert(fuzzer::F);
-  return fuzzer::F->GetMD().Mutate(Data, Size, MaxSize);
+  return fuzzer::F->GetMD().DefaultMutate(Data, Size, MaxSize);
 }
 }  // extern "C"

llvm/trunk/lib/Fuzzer/FuzzerMutate.cpp

 
 #include "FuzzerInternal.h"
 
 
 namespace fuzzer {
 
 const size_t Dictionary::kMaxDictSize;
 
-MutationDispatcher::Mutator MutationDispatcher::Mutators[] = {
+MutationDispatcher::MutationDispatcher(Random &Rand) : Rand(Rand) {
+  DefaultMutators.insert(
+      DefaultMutators.begin(),
+      {
           {&MutationDispatcher::Mutate_EraseByte, "EraseByte"},
           {&MutationDispatcher::Mutate_InsertByte, "InsertByte"},
           {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"},
           {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"},
           {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"},
           {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"},
           {&MutationDispatcher::Mutate_CrossOver, "CrossOver"},
           {&MutationDispatcher::Mutate_AddWordFromManualDictionary,
            "AddFromManualDict"},
           {&MutationDispatcher::Mutate_AddWordFromTemporaryAutoDictionary,
            "AddFromTempAutoDict"},
           {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary,
            "AddFromPersAutoDict"},
-};
+      });
+
+  if (EF.LLVMFuzzerCustomMutator)
+    Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"});
+  else
+    Mutators = DefaultMutators;
+}
 
 static char FlipRandomBit(char X, Random &Rand) {
   int Bit = Rand(8);
   char Mask = 1 << Bit;
   char R;
   if (X & (1 << Bit))
     R = X & ~Mask;
   else
     R = X | Mask;
   assert(R != X);
   return R;
 }
 
 static char RandCh(Random &Rand) {
   if (Rand.RandBool()) return Rand(256);
   const char *Special = "!*'();:@&=+$,/?%#[]123ABCxyz-`~.";
   return Special[Rand(sizeof(Special) - 1)];
 }
 
+size_t MutationDispatcher::Mutate_Custom(uint8_t *Data, size_t Size,
+                                         size_t MaxSize) {
+  return EF.LLVMFuzzerCustomMutator(Data, Size, MaxSize, Rand.Rand());
+}
+
 size_t MutationDispatcher::Mutate_ShuffleBytes(uint8_t *Data, size_t Size,
                                                size_t MaxSize) {
   assert(Size);
   size_t ShuffleAmount =
       Rand(std::min(Size, (size_t)8)) + 1; // [1,8] and <= Size.
   size_t ShuffleStart = Rand(Size - ShuffleAmount);
   assert(ShuffleStart + ShuffleAmount <= Size);
   std::random_shuffle(Data + ShuffleStart, Data + ShuffleStart + ShuffleAmount,
if (!CurrentDictionaryEntrySequence.empty()) {
     Printf(" DE: ");
     for (auto DE : CurrentDictionaryEntrySequence) {
       Printf("\"");
       PrintASCII(DE->GetW(), "\"-");
     }
   }
 }
 
-// Mutates Data in place, returns new size.
 size_t MutationDispatcher::Mutate(uint8_t *Data, size_t Size, size_t MaxSize) {
+  return MutateImpl(Data, Size, MaxSize, Mutators);
+}
+
+size_t MutationDispatcher::DefaultMutate(uint8_t *Data, size_t Size,
+                                         size_t MaxSize) {
+  return MutateImpl(Data, Size, MaxSize, DefaultMutators);
+}
+
+// Mutates Data in place, returns new size.
+size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size,
+                                      size_t MaxSize,
+                                      const std::vector<Mutator> &Mutators) {
   assert(MaxSize > 0);
   assert(Size <= MaxSize);
   if (Size == 0) {
     for (size_t i = 0; i < MaxSize; i++)
       Data[i] = RandCh(Rand);
     return MaxSize;
   }
   assert(Size > 0);
   // Some mutations may fail (e.g. can't insert more bytes if Size == MaxSize),
   // in which case they will return 0.
   // Try several times before returning un-mutated data.
   for (int Iter = 0; Iter < 10; Iter++) {
-    size_t NumMutators = sizeof(Mutators) / sizeof(Mutators[0]);
-    size_t MutatorIdx = Rand(NumMutators);
-    auto M = Mutators[MutatorIdx];
+    auto M = Mutators[Rand(Mutators.size())];
     size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize);
     if (NewSize) {
       CurrentMutatorSequence.push_back(M);
       return NewSize;
     }
   }
   return Size;
 }