Index: compiler-rt/lib/fuzzer/FuzzerDriver.cpp =================================================================== --- compiler-rt/lib/fuzzer/FuzzerDriver.cpp +++ compiler-rt/lib/fuzzer/FuzzerDriver.cpp @@ -8,6 +8,16 @@ // FuzzerDriver and flag parsing. //===----------------------------------------------------------------------===// +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include "FuzzerCommand.h" #include "FuzzerCorpus.h" #include "FuzzerFork.h" @@ -19,15 +29,6 @@ #include "FuzzerPlatform.h" #include "FuzzerRandom.h" #include "FuzzerTracePC.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include // This function should be present in the libFuzzer so that the client // binary can test for its existence. @@ -48,8 +49,8 @@ struct FlagDescription { const char *Name; const char *Description; - int Default; - int *IntFlag; + int Default; + int *IntFlag; const char **StrFlag; unsigned int *UIntFlag; }; @@ -66,15 +67,15 @@ #undef FUZZER_FLAG_STRING } Flags; -static const FlagDescription FlagDescriptions [] { -#define FUZZER_DEPRECATED_FLAG(Name) \ +static const FlagDescription FlagDescriptions[]{ +#define FUZZER_DEPRECATED_FLAG(Name) \ {#Name, "Deprecated; don't use", 0, nullptr, nullptr, nullptr}, -#define FUZZER_FLAG_INT(Name, Default, Description) \ +#define FUZZER_FLAG_INT(Name, Default, Description) \ {#Name, Description, Default, &Flags.Name, nullptr, nullptr}, -#define FUZZER_FLAG_UNSIGNED(Name, Default, Description) \ - {#Name, Description, static_cast(Default), \ - nullptr, nullptr, &Flags.Name}, -#define FUZZER_FLAG_STRING(Name, Description) \ +#define FUZZER_FLAG_UNSIGNED(Name, Default, Description) \ + {#Name, Description, static_cast(Default), \ + nullptr, nullptr, &Flags.Name}, +#define FUZZER_FLAG_STRING(Name, Description) \ {#Name, Description, 0, nullptr, &Flags.Name, nullptr}, #include "FuzzerFlags.def" #undef FUZZER_DEPRECATED_FLAG @@ -107,20 +108,20 @@ const auto &D = FlagDescriptions[F]; if (strstr(D.Description, "internal flag") == D.Description) continue; Printf(" %s", D.Name); - for (size_t i = 0, n = MaxFlagLen - strlen(D.Name); i < n; i++) - Printf(" "); + for (size_t i = 0, n = MaxFlagLen - strlen(D.Name); i < n; i++) Printf(" "); Printf("\t"); Printf("%d\t%s\n", D.Default, D.Description); } - Printf("\nFlags starting with '--' will be ignored and " - "will be passed verbatim to subprocesses.\n"); + Printf( + "\nFlags starting with '--' will be ignored and " + "will be passed verbatim to subprocesses.\n"); } static const char *FlagValue(const char *Param, const char *Name) { size_t Len = strlen(Name); if (Param[0] == '-' && strstr(Param + 1, Name) == Param + 1 && Param[Len + 1] == '=') - return &Param[Len + 2]; + return &Param[Len + 2]; return nullptr; } @@ -134,8 +135,7 @@ } for (size_t i = 0; Str[i]; i++) { char Ch = Str[i]; - if (Ch < '0' || Ch > '9') - return Res; + if (Ch < '0' || Ch > '9') return Res; Res = Res * 10 + (Ch - '0'); } return Res * Sign; @@ -157,23 +157,20 @@ for (size_t F = 0; F < kNumFlags; F++) { const char *Name = FlagDescriptions[F].Name; const char *Str = FlagValue(Param, Name); - if (Str) { + if (Str) { if (FlagDescriptions[F].IntFlag) { auto Val = MyStol(Str); *FlagDescriptions[F].IntFlag = static_cast(Val); - if (Flags.verbosity >= 2) - Printf("Flag: %s %d\n", Name, Val); + if (Flags.verbosity >= 2) Printf("Flag: %s %d\n", Name, Val); return true; } else if (FlagDescriptions[F].UIntFlag) { auto Val = std::stoul(Str); *FlagDescriptions[F].UIntFlag = static_cast(Val); - if (Flags.verbosity >= 2) - Printf("Flag: %s %u\n", Name, Val); + if (Flags.verbosity >= 2) Printf("Flag: %s %u\n", Name, Val); return true; } else if (FlagDescriptions[F].StrFlag) { *FlagDescriptions[F].StrFlag = Str; - if (Flags.verbosity >= 2) - Printf("Flag: %s %s\n", Name, Str); + if (Flags.verbosity >= 2) Printf("Flag: %s %s\n", Name, Str); return true; } else { // Deprecated flag. Printf("Flag: %s: deprecated, don't use\n", Name); @@ -181,8 +178,10 @@ } } } - Printf("\n\nWARNING: unrecognized flag '%s'; " - "use -help=1 to list all flags\n\n", Param); + Printf( + "\n\nWARNING: unrecognized flag '%s'; " + "use -help=1 to list all flags\n\n", + Param); return true; } @@ -195,22 +194,22 @@ if (FlagDescriptions[F].UIntFlag) *FlagDescriptions[F].UIntFlag = static_cast(FlagDescriptions[F].Default); - if (FlagDescriptions[F].StrFlag) - *FlagDescriptions[F].StrFlag = nullptr; + if (FlagDescriptions[F].StrFlag) *FlagDescriptions[F].StrFlag = nullptr; } // Disable len_control by default, if LLVMFuzzerCustomMutator is used. if (EF->LLVMFuzzerCustomMutator) { Flags.len_control = 0; - Printf("INFO: found LLVMFuzzerCustomMutator (%p). " - "Disabling -len_control by default.\n", EF->LLVMFuzzerCustomMutator); + Printf( + "INFO: found LLVMFuzzerCustomMutator (%p). " + "Disabling -len_control by default.\n", + EF->LLVMFuzzerCustomMutator); } Inputs = new Vector; for (size_t A = 1; A < Args.size(); A++) { if (ParseOneFlag(Args[A].c_str())) { - if (Flags.ignore_remaining_args) - break; + if (Flags.ignore_remaining_args) break; continue; } Inputs->push_back(Args[A]); @@ -241,8 +240,7 @@ Printf("%s\n", CommandLine.c_str()); } int ExitCode = ExecuteCommand(Cmd); - if (ExitCode != 0) - *HasErrors = true; + if (ExitCode != 0) *HasErrors = true; std::lock_guard Lock(Mu); Printf("================== Job %u exited with exit code %d ============\n", C, ExitCode); @@ -257,8 +255,7 @@ exit(1); } - if (IsDirectory(Path)) - return; + if (IsDirectory(Path)) return; if (CreateDirectory) { if (!MkDirRecursive(Path)) { @@ -272,12 +269,11 @@ exit(1); } -std::string CloneArgsWithoutX(const Vector &Args, - const char *X1, const char *X2) { +std::string CloneArgsWithoutX(const Vector &Args, const char *X1, + const char *X2) { std::string Cmd; for (auto &S : Args) { - if (FlagValue(S.c_str(), X1) || FlagValue(S.c_str(), X2)) - continue; + if (FlagValue(S.c_str(), X1) || FlagValue(S.c_str(), X2)) continue; Cmd += S + " "; } return Cmd; @@ -294,9 +290,9 @@ std::thread Pulse(PulseThread); Pulse.detach(); for (unsigned i = 0; i < NumWorkers; i++) - V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, &HasErrors)); - for (auto &T : V) - T.join(); + V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, + &HasErrors)); + for (auto &T : V) T.join(); return HasErrors ? 1 : 0; } @@ -304,22 +300,19 @@ while (true) { SleepSeconds(1); size_t Peak = GetPeakRSSMb(); - if (Peak > RssLimitMb) - F->RssLimitCallback(); + if (Peak > RssLimitMb) F->RssLimitCallback(); } } static void StartRssThread(Fuzzer *F, size_t RssLimitMb) { - if (!RssLimitMb) - return; + if (!RssLimitMb) return; std::thread T(RssThread, F, RssLimitMb); T.detach(); } int RunOneTest(Fuzzer *F, const char *InputFilePath, size_t MaxLen) { Unit U = FileToVector(InputFilePath); - if (MaxLen && MaxLen < U.size()) - U.resize(MaxLen); + if (MaxLen && MaxLen < U.size()) U.resize(MaxLen); F->ExecuteCallback(U.data(), U.size()); if (Flags.print_full_coverage) { // Leak detection is not needed when collecting full coverage data. @@ -333,26 +326,24 @@ static bool AllInputsAreFiles() { if (Inputs->empty()) return false; for (auto &Path : *Inputs) - if (!IsFile(Path)) - return false; + if (!IsFile(Path)) return false; return true; } static std::string GetDedupTokenFromCmdOutput(const std::string &S) { auto Beg = S.find("DEDUP_TOKEN:"); - if (Beg == std::string::npos) - return ""; + if (Beg == std::string::npos) return ""; auto End = S.find('\n', Beg); - if (End == std::string::npos) - return ""; + if (End == std::string::npos) return ""; return S.substr(Beg, End - Beg); } int CleanseCrashInput(const Vector &Args, - const FuzzingOptions &Options) { + const FuzzingOptions &Options) { if (Inputs->size() != 1 || !Flags.exact_artifact_path) { - Printf("ERROR: -cleanse_crash should be given one input file and" - " -exact_artifact_path\n"); + Printf( + "ERROR: -cleanse_crash should be given one input file and" + " -exact_artifact_path\n"); exit(1); } std::string InputFilePath = Inputs->at(0); @@ -416,9 +407,10 @@ assert(BaseCmd.hasArgument(InputFilePath)); BaseCmd.removeArgument(InputFilePath); if (Flags.runs <= 0 && Flags.max_total_time == 0) { - Printf("INFO: you need to specify -runs=N or " - "-max_total_time=N with -minimize_crash=1\n" - "INFO: defaulting to -max_total_time=600\n"); + Printf( + "INFO: you need to specify -runs=N or " + "-max_total_time=N with -minimize_crash=1\n" + "INFO: defaulting to -max_total_time=600\n"); BaseCmd.addFlag("max_total_time", "600"); } @@ -440,9 +432,10 @@ Printf("ERROR: the input %s did not crash\n", CurrentFilePath.c_str()); exit(1); } - Printf("CRASH_MIN: '%s' (%zd bytes) caused a crash. Will try to minimize " - "it further\n", - CurrentFilePath.c_str(), U.size()); + Printf( + "CRASH_MIN: '%s' (%zd bytes) caused a crash. Will try to minimize " + "it further\n", + CurrentFilePath.c_str(), U.size()); auto DedupToken1 = GetDedupTokenFromCmdOutput(CmdOutput); if (!DedupToken1.empty()) Printf("CRASH_MIN: DedupToken1: %s\n", DedupToken1.c_str()); @@ -475,8 +468,9 @@ CurrentFilePath = Flags.exact_artifact_path; WriteToFile(U, CurrentFilePath); } - Printf("CRASH_MIN: mismatch in dedup tokens" - " (looks like a different bug). Won't minimize further\n"); + Printf( + "CRASH_MIN: mismatch in dedup tokens" + " (looks like a different bug). Won't minimize further\n"); break; } @@ -525,14 +519,12 @@ for (auto &Path : NewFiles) F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen)); // We are done, delete the control file if it was a temporary one. - if (!Flags.merge_control_file) - RemoveFile(CFPath); + if (!Flags.merge_control_file) RemoveFile(CFPath); exit(0); } -int AnalyzeDictionary(Fuzzer *F, const Vector& Dict, - UnitVector& Corpus) { +int AnalyzeDictionary(Fuzzer *F, const Vector &Dict, UnitVector &Corpus) { Printf("Started dictionary minimization (up to %d tests)\n", Dict.size() * Corpus.size() * 2); @@ -546,35 +538,31 @@ // Get coverage for the testcase without modifications. F->ExecuteCallback(C.data(), C.size()); InitialFeatures.clear(); - TPC.CollectFeatures([&](size_t Feature) { - InitialFeatures.push_back(Feature); - }); + TPC.CollectFeatures( + [&](size_t Feature) { InitialFeatures.push_back(Feature); }); for (size_t i = 0; i < Dict.size(); ++i) { Vector Data = C; - auto StartPos = std::search(Data.begin(), Data.end(), - Dict[i].begin(), Dict[i].end()); + auto StartPos = + std::search(Data.begin(), Data.end(), Dict[i].begin(), Dict[i].end()); // Skip dictionary unit, if the testcase does not contain it. - if (StartPos == Data.end()) - continue; + if (StartPos == Data.end()) continue; ++Usages[i]; while (StartPos != Data.end()) { // Replace all occurrences of dictionary unit in the testcase. auto EndPos = StartPos + Dict[i].size(); - for (auto It = StartPos; It != EndPos; ++It) - *It ^= 0xFF; + for (auto It = StartPos; It != EndPos; ++It) *It ^= 0xFF; - StartPos = std::search(EndPos, Data.end(), - Dict[i].begin(), Dict[i].end()); + StartPos = + std::search(EndPos, Data.end(), Dict[i].begin(), Dict[i].end()); } // Get coverage for testcase with masked occurrences of dictionary unit. F->ExecuteCallback(Data.data(), Data.size()); ModifiedFeatures.clear(); - TPC.CollectFeatures([&](size_t Feature) { - ModifiedFeatures.push_back(Feature); - }); + TPC.CollectFeatures( + [&](size_t Feature) { ModifiedFeatures.push_back(Feature); }); if (InitialFeatures == ModifiedFeatures) --Scores[i]; @@ -586,8 +574,7 @@ Printf("###### Useless dictionary elements. ######\n"); for (size_t i = 0; i < Dict.size(); ++i) { // Dictionary units with positive score are treated as useful ones. - if (Scores[i] > 0) - continue; + if (Scores[i] > 0) continue; Printf("\""); PrintASCII(Dict[i].data(), Dict[i].size(), "\""); @@ -603,9 +590,9 @@ if (!seed_inputs) return Files; std::string SeedInputs; if (Flags.seed_inputs[0] == '@') - SeedInputs = FileToString(Flags.seed_inputs + 1); // File contains list. + SeedInputs = FileToString(Flags.seed_inputs + 1); // File contains list. else - SeedInputs = Flags.seed_inputs; // seed_inputs contains the list. + SeedInputs = Flags.seed_inputs; // seed_inputs contains the list. if (SeedInputs.empty()) { Printf("seed_inputs is empty or @file does not exist.\n"); exit(1); @@ -620,7 +607,8 @@ return Files; } -static Vector ReadCorpora(const Vector &CorpusDirs, +static Vector ReadCorpora( + const Vector &CorpusDirs, const Vector &ExtraSeedFiles) { Vector SizedFiles; size_t LastNumFiles = 0; @@ -631,8 +619,7 @@ LastNumFiles = SizedFiles.size(); } for (auto &File : ExtraSeedFiles) - if (auto Size = FileSize(File)) - SizedFiles.push_back({File, Size}); + if (auto Size = FileSize(File)) SizedFiles.push_back({File, Size}); return SizedFiles; } @@ -641,8 +628,7 @@ assert(argc && argv && "Argument pointers cannot be nullptr"); std::string Argv0((*argv)[0]); EF = new ExternalFunctions(); - if (EF->LLVMFuzzerInitialize) - EF->LLVMFuzzerInitialize(argc, argv); + if (EF->LLVMFuzzerInitialize) EF->LLVMFuzzerInitialize(argc, argv); if (EF->__msan_scoped_disable_interceptor_checks) EF->__msan_scoped_disable_interceptor_checks(); const Vector Args(*argv, *argv + *argc); @@ -658,15 +644,12 @@ return 0; } - if (Flags.close_fd_mask & 2) - DupAndCloseStderr(); - if (Flags.close_fd_mask & 1) - CloseStdout(); + if (Flags.close_fd_mask & 2) DupAndCloseStderr(); + if (Flags.close_fd_mask & 1) CloseStdout(); if (Flags.jobs > 0 && Flags.workers == 0) { Flags.workers = std::min(NumberOfCpuCores() / 2, Flags.jobs); - if (Flags.workers > 1) - Printf("Running %u workers\n", Flags.workers); + if (Flags.workers > 1) Printf("Running %u workers\n", Flags.workers); } if (Flags.workers > 0 && Flags.jobs > 0) @@ -703,10 +686,8 @@ Options.TraceMalloc = Flags.trace_malloc; Options.RssLimitMb = Flags.rss_limit_mb; Options.MallocLimitMb = Flags.malloc_limit_mb; - if (!Options.MallocLimitMb) - Options.MallocLimitMb = Options.RssLimitMb; - if (Flags.runs >= 0) - Options.MaxNumberOfRuns = Flags.runs; + if (!Options.MallocLimitMb) Options.MallocLimitMb = Options.RssLimitMb; + if (Flags.runs >= 0) Options.MaxNumberOfRuns = Flags.runs; if (!Inputs->empty() && !Flags.minimize_crash_internal_step) { // Ensure output corpus assumed to be the first arbitrary argument input // is not a path to an existing file. @@ -736,8 +717,7 @@ } Vector Dictionary; if (Flags.dict) - if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary)) - return 1; + if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary)) return 1; if (Flags.verbosity > 0 && !Dictionary.empty()) Printf("Dictionary: %zd entries\n", Dictionary.size()); bool RunIndividualFiles = AllInputsAreFiles(); @@ -749,14 +729,10 @@ Options.PrintCorpusStats = Flags.print_corpus_stats; Options.PrintCoverage = Flags.print_coverage; Options.PrintFullCoverage = Flags.print_full_coverage; - if (Flags.exit_on_src_pos) - Options.ExitOnSrcPos = Flags.exit_on_src_pos; - if (Flags.exit_on_item) - Options.ExitOnItem = Flags.exit_on_item; - if (Flags.focus_function) - Options.FocusFunction = Flags.focus_function; - if (Flags.data_flow_trace) - Options.DataFlowTrace = Flags.data_flow_trace; + if (Flags.exit_on_src_pos) Options.ExitOnSrcPos = Flags.exit_on_src_pos; + if (Flags.exit_on_item) Options.ExitOnItem = Flags.exit_on_item; + if (Flags.focus_function) Options.FocusFunction = Flags.focus_function; + if (Flags.data_flow_trace) Options.DataFlowTrace = Flags.data_flow_trace; if (Flags.features_dir) { Options.FeaturesDir = Flags.features_dir; ValidateDirectoryExists(Options.FeaturesDir, Flags.create_missing_dirs); @@ -765,8 +741,7 @@ Options.MutationGraphFile = Flags.mutation_graph_file; if (Flags.collect_data_flow) Options.CollectDataFlow = Flags.collect_data_flow; - if (Flags.stop_file) - Options.StopFile = Flags.stop_file; + if (Flags.stop_file) Options.StopFile = Flags.stop_file; Options.Entropic = Flags.entropic; Options.EntropicFeatureFrequencyThreshold = (size_t)Flags.entropic_feature_frequency_threshold; @@ -774,7 +749,7 @@ (size_t)Flags.entropic_number_of_rarest_features; Options.EntropicScalePerExecTime = Flags.entropic_scale_per_exec_time; if (!Options.FocusFunction.empty()) - Options.Entropic = false; // FocusFunction overrides entropic scheduling. + Options.Entropic = false; // FocusFunction overrides entropic scheduling. if (Options.Entropic) Printf("INFO: Running with entropic power schedule (0x%X, %d).\n", Options.EntropicFeatureFrequencyThreshold, @@ -791,16 +766,15 @@ if (Seed == 0) Seed = static_cast( std::chrono::system_clock::now().time_since_epoch().count() + GetPid()); - if (Flags.verbosity) - Printf("INFO: Seed: %u\n", Seed); + if (Flags.verbosity) Printf("INFO: Seed: %u\n", Seed); if (Flags.collect_data_flow && !Flags.fork && !Flags.merge) { if (RunIndividualFiles) return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, - ReadCorpora({}, *Inputs)); + ReadCorpora({}, *Inputs)); else return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, - ReadCorpora(*Inputs, {})); + ReadCorpora(*Inputs, {})); } Random Rand(Seed); @@ -808,7 +782,7 @@ auto *Corpus = new InputCorpus(Options.OutputCorpus, Entropic); auto *F = new Fuzzer(Callback, *Corpus, *MD, Options); - for (auto &U: Dictionary) + for (auto &U : Dictionary) if (U.size() <= Word::GetMaxSize()) MD->AddWordToManualDictionary(Word(U.data(), U.size())); @@ -816,7 +790,7 @@ // for now. #if !LIBFUZZER_EMSCRIPTEN StartRssThread(F, Flags.rss_limit_mb); -#endif // LIBFUZZER_EMSCRIPTEN +#endif // LIBFUZZER_EMSCRIPTEN Options.HandleAbrt = Flags.handle_abrt; Options.HandleAlrm = !Flags.minimize_crash; @@ -835,14 +809,12 @@ std::atexit(Fuzzer::StaticExitCallback); - if (Flags.minimize_crash) - return MinimizeCrashInput(Args, Options); + if (Flags.minimize_crash) return MinimizeCrashInput(Args, Options); if (Flags.minimize_crash_internal_step) return MinimizeCrashInputInternalStep(F, Corpus); - if (Flags.cleanse_crash) - return CleanseCrashInput(Args, Options); + if (Flags.cleanse_crash) return CleanseCrashInput(Args, Options); if (RunIndividualFiles) { Options.SaveArtifacts = false; @@ -858,24 +830,24 @@ auto MS = duration_cast(StopTime - StartTime).count(); Printf("Executed %s in %zd ms\n", Path.c_str(), (long)MS); } - Printf("***\n" - "*** NOTE: fuzzing was not performed, you have only\n" - "*** executed the target code on a fixed set of inputs.\n" - "***\n"); + Printf( + "***\n" + "*** NOTE: fuzzing was not performed, you have only\n" + "*** executed the target code on a fixed set of inputs.\n" + "***\n"); F->PrintFinalStats(); exit(0); } if (Flags.fork) - FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); + FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork, + Flags.NumCorpuses); - if (Flags.merge) - Merge(F, Options, Args, *Inputs, Flags.merge_control_file); + if (Flags.merge) Merge(F, Options, Args, *Inputs, Flags.merge_control_file); if (Flags.merge_inner) { const size_t kDefaultMaxMergeLen = 1 << 20; - if (Options.MaxLen == 0) - F->SetMaxInputLen(kDefaultMaxMergeLen); + if (Options.MaxLen == 0) F->SetMaxInputLen(kDefaultMaxMergeLen); assert(Flags.merge_control_file); F->CrashResistantMergeInternalStep(Flags.merge_control_file); exit(0); @@ -886,8 +858,8 @@ UnitVector InitialCorpus; for (auto &Inp : *Inputs) { Printf("Loading corpus dir: %s\n", Inp.c_str()); - ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr, - MaxLen, /*ExitOnError=*/false); + ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr, MaxLen, + /*ExitOnError=*/false); } if (Dictionary.empty() || Inputs->empty()) { @@ -913,9 +885,8 @@ exit(0); // Don't let F destroy itself. } -extern "C" ATTRIBUTE_INTERFACE int -LLVMFuzzerRunDriver(int *argc, char ***argv, - int (*UserCb)(const uint8_t *Data, size_t Size)) { +extern "C" ATTRIBUTE_INTERFACE int LLVMFuzzerRunDriver( + int *argc, char ***argv, int (*UserCb)(const uint8_t *Data, size_t Size)) { return FuzzerDriver(argc, argv, UserCb); } Index: compiler-rt/lib/fuzzer/FuzzerFlags.def =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFlags.def +++ compiler-rt/lib/fuzzer/FuzzerFlags.def @@ -12,58 +12,81 @@ FUZZER_FLAG_INT(verbosity, 1, "Verbosity level.") FUZZER_FLAG_UNSIGNED(seed, 0, "Random seed. If 0, seed is generated.") FUZZER_FLAG_INT(runs, -1, - "Number of individual test runs (-1 for infinite runs).") -FUZZER_FLAG_INT(max_len, 0, "Maximum length of the test input. " + "Number of individual test runs (-1 for infinite runs).") +FUZZER_FLAG_INT( + max_len, 0, + "Maximum length of the test input. " "If 0, libFuzzer tries to guess a good value based on the corpus " "and reports it. ") -FUZZER_FLAG_INT(len_control, 100, "Try generating small inputs first, " - "then try larger inputs over time. Specifies the rate at which the length " - "limit is increased (smaller == faster). If 0, immediately try inputs with " - "size up to max_len. Default value is 0, if LLVMFuzzerCustomMutator is used.") -FUZZER_FLAG_STRING(seed_inputs, "A comma-separated list of input files " - "to use as an additional seed corpus. Alternatively, an \"@\" followed by " - "the name of a file containing the comma-separated list.") -FUZZER_FLAG_INT(keep_seed, 0, "If 1, keep seed inputs in the corpus even if " - "they do not produce new coverage. When used with |reduce_inputs==1|, the " - "seed inputs will never be reduced. This option can be useful when seeds are" - "not properly formed for the fuzz target but still have useful snippets.") +FUZZER_FLAG_INT( + len_control, 100, + "Try generating small inputs first, " + "then try larger inputs over time. Specifies the rate at which the length " + "limit is increased (smaller == faster). If 0, immediately try inputs " + "with " + "size up to max_len. Default value is 0, if LLVMFuzzerCustomMutator is " + "used.") +FUZZER_FLAG_STRING( + seed_inputs, + "A comma-separated list of input files " + "to use as an additional seed corpus. Alternatively, an \"@\" followed by " + "the name of a file containing the comma-separated list.") +FUZZER_FLAG_INT( + keep_seed, 0, + "If 1, keep seed inputs in the corpus even if " + "they do not produce new coverage. When used with |reduce_inputs==1|, the " + "seed inputs will never be reduced. This option can be useful when seeds " + "are" + "not properly formed for the fuzz target but still have useful snippets.") FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.") -FUZZER_FLAG_INT(cross_over_uniform_dist, 0, "Experimental. If 1, use a " - "uniform probability distribution when choosing inputs to cross over with. " - "Some of the inputs in the corpus may never get chosen for mutation " - "depending on the input mutation scheduling policy. With this flag, all " - "inputs, regardless of the input mutation scheduling policy, can be chosen " - "as an input to cross over with. This can be particularly useful with " - "|keep_seed==1|; all the initial seed inputs, even though they do not " - "increase coverage because they are not properly formed, will still be " - "chosen as an input to cross over with.") +FUZZER_FLAG_INT( + cross_over_uniform_dist, 0, + "Experimental. If 1, use a " + "uniform probability distribution when choosing inputs to cross over with. " + "Some of the inputs in the corpus may never get chosen for mutation " + "depending on the input mutation scheduling policy. With this flag, all " + "inputs, regardless of the input mutation scheduling policy, can be chosen " + "as an input to cross over with. This can be particularly useful with " + "|keep_seed==1|; all the initial seed inputs, even though they do not " + "increase coverage because they are not properly formed, will still be " + "chosen as an input to cross over with.") FUZZER_FLAG_INT(mutate_depth, 5, - "Apply this number of consecutive mutations to each input.") -FUZZER_FLAG_INT(reduce_depth, 0, "Experimental/internal. " + "Apply this number of consecutive mutations to each input.") +FUZZER_FLAG_INT(reduce_depth, 0, + "Experimental/internal. " "Reduce depth if mutations lose unique features") FUZZER_FLAG_INT(shuffle, 1, "Shuffle inputs at startup") FUZZER_FLAG_INT(prefer_small, 1, - "If 1, always prefer smaller inputs during the corpus shuffle.") + "If 1, always prefer smaller inputs during the corpus shuffle.") FUZZER_FLAG_INT( timeout, 1200, "Timeout in seconds (if positive). " "If one unit runs more than this number of seconds the process will abort.") -FUZZER_FLAG_INT(error_exitcode, 77, "When libFuzzer itself reports a bug " - "this exit code will be used.") -FUZZER_FLAG_INT(timeout_exitcode, 70, "When libFuzzer reports a timeout " - "this exit code will be used.") -FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total " - "time in seconds to run the fuzzer.") +FUZZER_FLAG_INT(error_exitcode, 77, + "When libFuzzer itself reports a bug " + "this exit code will be used.") +FUZZER_FLAG_INT(timeout_exitcode, 70, + "When libFuzzer reports a timeout " + "this exit code will be used.") +FUZZER_FLAG_INT(max_total_time, 0, + "If positive, indicates the maximal total " + "time in seconds to run the fuzzer.") FUZZER_FLAG_INT(help, 0, "Print help.") -FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens " +FUZZER_FLAG_INT( + NumCorpuses, 1, + "FOR fork mode. Divide the main corpus into N parts according to size.") +FUZZER_FLAG_INT(fork, 0, + "Experimental mode where fuzzing happens " "in a subprocess") FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode") FUZZER_FLAG_INT(ignore_ooms, 1, "Ignore OOMs in fork mode") FUZZER_FLAG_INT(ignore_crashes, 0, "Ignore crashes in fork mode") -FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be " - "merged into the 1-st corpus. Only interesting units will be taken. " - "This flag can be used to minimize a corpus.") +FUZZER_FLAG_INT( + merge, 0, + "If 1, the 2-nd, 3-rd, etc corpora will be " + "merged into the 1-st corpus. Only interesting units will be taken. " + "This flag can be used to minimize a corpus.") FUZZER_FLAG_STRING(stop_file, "Stop fuzzing ASAP if this file exists") FUZZER_FLAG_STRING(merge_inner, "internal flag") FUZZER_FLAG_STRING(merge_control_file, @@ -72,27 +95,32 @@ "in a state suitable for resuming the merge. " "By default a temporary file will be used." "The same file can be used for multistep merge process.") -FUZZER_FLAG_INT(minimize_crash, 0, "If 1, minimizes the provided" - " crash input. Use with -runs=N or -max_total_time=N to limit " - "the number attempts." - " Use with -exact_artifact_path to specify the output." - " Combine with ASAN_OPTIONS=dedup_token_length=3 (or similar) to ensure that" - " the minimized input triggers the same crash." - ) -FUZZER_FLAG_INT(cleanse_crash, 0, "If 1, tries to cleanse the provided" - " crash input to make it contain fewer original bytes." - " Use with -exact_artifact_path to specify the output." - ) +FUZZER_FLAG_INT(minimize_crash, 0, + "If 1, minimizes the provided" + " crash input. Use with -runs=N or -max_total_time=N to limit " + "the number attempts." + " Use with -exact_artifact_path to specify the output." + " Combine with ASAN_OPTIONS=dedup_token_length=3 (or similar) " + "to ensure that" + " the minimized input triggers the same crash.") +FUZZER_FLAG_INT(cleanse_crash, 0, + "If 1, tries to cleanse the provided" + " crash input to make it contain fewer original bytes." + " Use with -exact_artifact_path to specify the output.") FUZZER_FLAG_INT(minimize_crash_internal_step, 0, "internal flag") -FUZZER_FLAG_STRING(features_dir, "internal flag. Used to dump feature sets on disk." - "Every time a new input is added to the corpus, a corresponding file in the features_dir" - " is created containing the unique features of that input." - " Features are stored in binary format.") -FUZZER_FLAG_STRING(mutation_graph_file, "Saves a graph (in DOT format) to" - " mutation_graph_file. The graph contains a vertex for each input that has" - " unique coverage; directed edges are provided between parents and children" - " where the child has unique coverage, and are recorded with the type of" - " mutation that caused the child.") +FUZZER_FLAG_STRING(features_dir, + "internal flag. Used to dump feature sets on disk." + "Every time a new input is added to the corpus, a " + "corresponding file in the features_dir" + " is created containing the unique features of that input." + " Features are stored in binary format.") +FUZZER_FLAG_STRING( + mutation_graph_file, + "Saves a graph (in DOT format) to" + " mutation_graph_file. The graph contains a vertex for each input that has" + " unique coverage; directed edges are provided between parents and children" + " where the child has unique coverage, and are recorded with the type of" + " mutation that caused the child.") FUZZER_FLAG_INT(use_counters, 1, "Use coverage counters") FUZZER_FLAG_INT(use_memmem, 1, "Use hints from intercepting memmem, strstr, etc") @@ -100,40 +128,47 @@ "Experimental. Use value profile to guide fuzzing.") FUZZER_FLAG_INT(use_cmp, 1, "Use CMP traces to guide mutations") FUZZER_FLAG_INT(shrink, 0, "Experimental. Try to shrink corpus inputs.") -FUZZER_FLAG_INT(reduce_inputs, 1, - "Try to reduce the size of inputs while preserving their full feature sets") -FUZZER_FLAG_UNSIGNED(jobs, 0, "Number of jobs to run. If jobs >= 1 we spawn" - " this number of jobs in separate worker processes" - " with stdout/stderr redirected to fuzz-JOB.log.") +FUZZER_FLAG_INT( + reduce_inputs, 1, + "Try to reduce the size of inputs while preserving their full feature sets") +FUZZER_FLAG_UNSIGNED(jobs, 0, + "Number of jobs to run. If jobs >= 1 we spawn" + " this number of jobs in separate worker processes" + " with stdout/stderr redirected to fuzz-JOB.log.") FUZZER_FLAG_UNSIGNED(workers, 0, - "Number of simultaneous worker processes to run the jobs." - " If zero, \"min(jobs,NumberOfCpuCores()/2)\" is used.") + "Number of simultaneous worker processes to run the jobs." + " If zero, \"min(jobs,NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload, 1, "Reload the main corpus every seconds to get new units" " discovered by other processes. If 0, disabled") -FUZZER_FLAG_INT(report_slow_units, 10, +FUZZER_FLAG_INT( + report_slow_units, 10, "Report slowest units if they run for more than this number of seconds.") FUZZER_FLAG_INT(only_ascii, 0, "If 1, generate only ASCII (isprint+isspace) inputs.") FUZZER_FLAG_STRING(dict, "Experimental. Use the dictionary file.") -FUZZER_FLAG_STRING(artifact_prefix, "Write fuzzing artifacts (crash, " - "timeout, or slow inputs) as " - "$(artifact_prefix)file") +FUZZER_FLAG_STRING(artifact_prefix, + "Write fuzzing artifacts (crash, " + "timeout, or slow inputs) as " + "$(artifact_prefix)file") FUZZER_FLAG_STRING(exact_artifact_path, "Write the single artifact on failure (crash, timeout) " "as $(exact_artifact_path). This overrides -artifact_prefix " "and will not use checksum in the file name. Do not " "use the same path for several parallel processes.") FUZZER_FLAG_INT(print_pcs, 0, "If 1, print out newly covered PCs.") -FUZZER_FLAG_INT(print_funcs, 2, "If >=1, print out at most this number of " - "newly covered functions.") +FUZZER_FLAG_INT(print_funcs, 2, + "If >=1, print out at most this number of " + "newly covered functions.") FUZZER_FLAG_INT(print_final_stats, 0, "If 1, print statistics at exit.") FUZZER_FLAG_INT(print_corpus_stats, 0, - "If 1, print statistics on corpus elements at exit.") -FUZZER_FLAG_INT(print_coverage, 0, "If 1, print coverage information as text" - " at exit.") -FUZZER_FLAG_INT(print_full_coverage, 0, "If 1, print full coverage information " - "(all branches) as text at exit.") + "If 1, print statistics on corpus elements at exit.") +FUZZER_FLAG_INT(print_coverage, 0, + "If 1, print coverage information as text" + " at exit.") +FUZZER_FLAG_INT(print_full_coverage, 0, + "If 1, print full coverage information " + "(all branches) as text at exit.") FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated.") FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.") FUZZER_FLAG_INT(handle_bus, 1, "If 1, try to intercept SIGBUS.") @@ -145,51 +180,73 @@ FUZZER_FLAG_INT(handle_xfsz, 1, "If 1, try to intercept SIGXFSZ.") FUZZER_FLAG_INT(handle_usr1, 1, "If 1, try to intercept SIGUSR1.") FUZZER_FLAG_INT(handle_usr2, 1, "If 1, try to intercept SIGUSR2.") -FUZZER_FLAG_INT(handle_winexcept, 1, "If 1, try to intercept uncaught Windows " - "Visual C++ Exceptions.") -FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; " - "if 2, close stderr; if 3, close both. " - "Be careful, this will also close e.g. stderr of asan.") -FUZZER_FLAG_INT(detect_leaks, 1, "If 1, and if LeakSanitizer is enabled " +FUZZER_FLAG_INT(handle_winexcept, 1, + "If 1, try to intercept uncaught Windows " + "Visual C++ Exceptions.") +FUZZER_FLAG_INT(close_fd_mask, 0, + "If 1, close stdout at startup; " + "if 2, close stderr; if 3, close both. " + "Be careful, this will also close e.g. stderr of asan.") +FUZZER_FLAG_INT( + detect_leaks, 1, + "If 1, and if LeakSanitizer is enabled " "try to detect memory leaks during fuzzing (i.e. not only at shut down).") -FUZZER_FLAG_INT(purge_allocator_interval, 1, "Purge allocator caches and " +FUZZER_FLAG_INT( + purge_allocator_interval, 1, + "Purge allocator caches and " "quarantines every seconds. When rss_limit_mb is specified (>0), " "purging starts when RSS exceeds 50% of rss_limit_mb. Pass " "purge_allocator_interval=-1 to disable this functionality.") -FUZZER_FLAG_INT(trace_malloc, 0, "If >= 1 will print all mallocs/frees. " - "If >= 2 will also print stack traces.") -FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon" - "reaching this limit of RSS memory usage.") -FUZZER_FLAG_INT(malloc_limit_mb, 0, "If non-zero, the fuzzer will exit " +FUZZER_FLAG_INT(trace_malloc, 0, + "If >= 1 will print all mallocs/frees. " + "If >= 2 will also print stack traces.") +FUZZER_FLAG_INT(rss_limit_mb, 2048, + "If non-zero, the fuzzer will exit upon" + "reaching this limit of RSS memory usage.") +FUZZER_FLAG_INT( + malloc_limit_mb, 0, + "If non-zero, the fuzzer will exit " "if the target tries to allocate this number of Mb with one malloc call. " "If zero (default) same limit as rss_limit_mb is applied.") -FUZZER_FLAG_STRING(exit_on_src_pos, "Exit if a newly found PC originates" +FUZZER_FLAG_STRING( + exit_on_src_pos, + "Exit if a newly found PC originates" " from the given source location. Example: -exit_on_src_pos=foo.cc:123. " "Used primarily for testing libFuzzer itself.") -FUZZER_FLAG_STRING(exit_on_item, "Exit if an item with a given sha1 sum" - " was added to the corpus. " - "Used primarily for testing libFuzzer itself.") -FUZZER_FLAG_INT(ignore_remaining_args, 0, "If 1, ignore all arguments passed " +FUZZER_FLAG_STRING(exit_on_item, + "Exit if an item with a given sha1 sum" + " was added to the corpus. " + "Used primarily for testing libFuzzer itself.") +FUZZER_FLAG_INT(ignore_remaining_args, 0, + "If 1, ignore all arguments passed " "after this one. Useful for fuzzers that need to do their own " "argument parsing.") -FUZZER_FLAG_STRING(focus_function, "Experimental. " - "Fuzzing will focus on inputs that trigger calls to this function. " - "If -focus_function=auto and -data_flow_trace is used, libFuzzer " - "will choose the focus functions automatically. Disables -entropic when " - "specified.") +FUZZER_FLAG_STRING( + focus_function, + "Experimental. " + "Fuzzing will focus on inputs that trigger calls to this function. " + "If -focus_function=auto and -data_flow_trace is used, libFuzzer " + "will choose the focus functions automatically. Disables -entropic when " + "specified.") FUZZER_FLAG_INT(entropic, 1, "Enables entropic power schedule.") -FUZZER_FLAG_INT(entropic_feature_frequency_threshold, 0xFF, "Experimental. If " - "entropic is enabled, all features which are observed less often than " - "the specified value are considered as rare.") -FUZZER_FLAG_INT(entropic_number_of_rarest_features, 100, "Experimental. If " - "entropic is enabled, we keep track of the frequencies only for the " - "Top-X least abundant features (union features that are considered as " - "rare).") -FUZZER_FLAG_INT(entropic_scale_per_exec_time, 0, "Experimental. If 1, " - "the Entropic power schedule gets scaled based on the input execution " - "time. Inputs with lower execution time get scheduled more (up to 30x). " - "Note that, if 1, fuzzer stops from being deterministic even if a " - "non-zero random seed is given.") +FUZZER_FLAG_INT( + entropic_feature_frequency_threshold, 0xFF, + "Experimental. If " + "entropic is enabled, all features which are observed less often than " + "the specified value are considered as rare.") +FUZZER_FLAG_INT( + entropic_number_of_rarest_features, 100, + "Experimental. If " + "entropic is enabled, we keep track of the frequencies only for the " + "Top-X least abundant features (union features that are considered as " + "rare).") +FUZZER_FLAG_INT( + entropic_scale_per_exec_time, 0, + "Experimental. If 1, " + "the Entropic power schedule gets scaled based on the input execution " + "time. Inputs with lower execution time get scheduled more (up to 30x). " + "Note that, if 1, fuzzer stops from being deterministic even if a " + "non-zero random seed is given.") FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") FUZZER_DEPRECATED_FLAG(use_clang_coverage) @@ -197,6 +254,8 @@ FUZZER_FLAG_STRING(collect_data_flow, "Experimental: collect the data flow trace") -FUZZER_FLAG_INT(create_missing_dirs, 0, "Automatically attempt to create " - "directories for arguments that would normally expect them to already " - "exist (i.e. artifact_prefix, exact_artifact_path, features_dir, corpus)") +FUZZER_FLAG_INT( + create_missing_dirs, 0, + "Automatically attempt to create " + "directories for arguments that would normally expect them to already " + "exist (i.e. artifact_prefix, exact_artifact_path, features_dir, corpus)") Index: compiler-rt/lib/fuzzer/FuzzerFork.h =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFork.h +++ compiler-rt/lib/fuzzer/FuzzerFork.h @@ -9,16 +9,17 @@ #ifndef LLVM_FUZZER_FORK_H #define LLVM_FUZZER_FORK_H +#include + #include "FuzzerDefs.h" #include "FuzzerOptions.h" #include "FuzzerRandom.h" -#include - namespace fuzzer { void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, const Vector &Args, - const Vector &CorpusDirs, int NumJobs); -} // namespace fuzzer + const Vector &CorpusDirs, int NumJobs, + int NumCorpuses); +} // namespace fuzzer -#endif // LLVM_FUZZER_FORK_H +#endif // LLVM_FUZZER_FORK_H Index: compiler-rt/lib/fuzzer/FuzzerFork.cpp =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFork.cpp +++ compiler-rt/lib/fuzzer/FuzzerFork.cpp @@ -8,14 +8,7 @@ // Spawn and orchestrate separate fuzzing processes. //===----------------------------------------------------------------------===// -#include "FuzzerCommand.h" #include "FuzzerFork.h" -#include "FuzzerIO.h" -#include "FuzzerInternal.h" -#include "FuzzerMerge.h" -#include "FuzzerSHA1.h" -#include "FuzzerTracePC.h" -#include "FuzzerUtil.h" #include #include @@ -27,6 +20,14 @@ #include #include +#include "FuzzerCommand.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include "FuzzerMerge.h" +#include "FuzzerSHA1.h" +#include "FuzzerTracePC.h" +#include "FuzzerUtil.h" + namespace fuzzer { struct Stats { @@ -55,8 +56,7 @@ size_t Val; ISS >> Name >> Val; for (size_t i = 0; NameVarPairs[i].Name; i++) - if (Name == NameVarPairs[i].Name) - *NameVarPairs[i].Var = Val; + if (Name == NameVarPairs[i].Name) *NameVarPairs[i].Var = Val; } return Res; } @@ -69,9 +69,9 @@ std::string LogPath; std::string SeedListPath; std::string CFPath; - size_t JobId; + size_t JobId; - int DftTimeInSeconds = 0; + int DftTimeInSeconds = 0; // Fuzzing Outputs. int ExitCode; @@ -95,6 +95,8 @@ Set Features, Cov; Set FilesWithDFT; Vector Files; + // This variable is used to store the size of the seed. + Vector FilesSizes; Random *Rand; std::chrono::system_clock::time_point ProcessStartTime; int Verbosity = 0; @@ -103,7 +105,6 @@ size_t NumOOMs = 0; size_t NumCrashes = 0; - size_t NumRuns = 0; std::string StopFile() { return DirPlusFile(TempDir, "STOP"); } @@ -114,12 +115,12 @@ .count(); } - FuzzJob *CreateNewJob(size_t JobId) { + FuzzJob *CreateNewJob(size_t JobId, int NumCorpuses) { Command Cmd(Args); Cmd.removeFlag("fork"); Cmd.removeFlag("runs"); Cmd.removeFlag("collect_data_flow"); - for (auto &C : CorpusDirs) // Remove all corpora from the args. + for (auto &C : CorpusDirs) // Remove all corpora from the args. Cmd.removeArgument(C); Cmd.addFlag("reload", "0"); // working in an isolated dir, no reload. Cmd.addFlag("print_final_stats", "1"); @@ -128,18 +129,33 @@ Cmd.addFlag("stop_file", StopFile()); if (!DataFlowBinary.empty()) { Cmd.addFlag("data_flow_trace", DFTDir); - if (!Cmd.hasFlag("focus_function")) - Cmd.addFlag("focus_function", "auto"); + if (!Cmd.hasFlag("focus_function")) Cmd.addFlag("focus_function", "auto"); } auto Job = new FuzzJob; std::string Seeds; if (size_t CorpusSubsetSize = std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) { + size_t AverageSize = Files.size() / NumCorpuses + 1; auto Time1 = std::chrono::system_clock::now(); + size_t StartIndex = ((JobId - 1) % NumCorpuses) * AverageSize; + // At this time, the seeds in the File variable are sorted according to + // the seed size, so by generating a uniformly distributed random + // number, the seeds are selected in a certain group. for (size_t i = 0; i < CorpusSubsetSize; i++) { - auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; - Seeds += (Seeds.empty() ? "" : ",") + SF; - CollectDFT(SF); + std::random_device rd; + std::mt19937 randomseed(rd()); + std::uniform_int_distribution<> rand(0, AverageSize); + size_t j = rand(randomseed); + size_t m = j + StartIndex; + if (m < Files.size()) { + auto &SF = Files[m]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } else { + auto &SF = Files[rand(randomseed)]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } } auto Time2 = std::chrono::system_clock::now(); auto DftTimeInSeconds = duration_cast(Time2 - Time1).count(); @@ -158,7 +174,6 @@ Job->CFPath = DirPlusFile(TempDir, std::to_string(JobId) + ".merge"); Job->JobId = JobId; - Cmd.addArgument(Job->CorpusDir); Cmd.addFlag("features_dir", Job->FeaturesDir); @@ -179,7 +194,7 @@ return Job; } - void RunOneMergeJob(FuzzJob *Job) { + void RunOneMergeJob(FuzzJob *Job, int NumCorpuses) { auto Stats = ParseFinalStatsFromLog(Job->LogPath); NumRuns += Stats.number_of_executed_units; @@ -203,11 +218,12 @@ } } // if (!FilesToAdd.empty() || Job->ExitCode != 0) - Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd " - "oom/timeout/crash: %zd/%zd/%zd time: %zds job: %zd dft_time: %d\n", - NumRuns, Cov.size(), Features.size(), Files.size(), - Stats.average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes, - secondsSinceProcessStartUp(), Job->JobId, Job->DftTimeInSeconds); + Printf( + "#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd " + "oom/timeout/crash: %zd/%zd/%zd time: %zds job: %zd dft_time: %d\n", + NumRuns, Cov.size(), Features.size(), Files.size(), + Stats.average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes, + secondsSinceProcessStartUp(), Job->JobId, Job->DftTimeInSeconds); if (MergeCandidates.empty()) return; @@ -219,7 +235,12 @@ auto U = FileToVector(Path); auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); WriteToFile(U, NewPath); - Files.push_back(NewPath); + // Seeds are inserted into Files according to size. + long usz = U.size(); + auto idx = std::upper_bound(FilesSizes.begin(), FilesSizes.end(), usz) - + FilesSizes.begin(); + FilesSizes.insert(FilesSizes.begin() + idx, usz); + Files.insert(Files.begin() + idx, NewPath); } Features.insert(NewFeatures.begin(), NewFeatures.end()); Cov.insert(NewCov.begin(), NewCov.end()); @@ -228,10 +249,8 @@ if (TPC.PcIsFuncEntry(TE)) PrintPC(" NEW_FUNC: %p %F %L\n", "", TPC.GetNextInstructionPc(TE->PC)); - } - void CollectDFT(const std::string &InputPath) { if (DataFlowBinary.empty()) return; if (!FilesWithDFT.insert(InputPath).second) return; @@ -240,14 +259,13 @@ Cmd.removeFlag("runs"); Cmd.addFlag("data_flow_trace", DFTDir); Cmd.addArgument(InputPath); - for (auto &C : CorpusDirs) // Remove all corpora from the args. + for (auto &C : CorpusDirs) // Remove all corpora from the args. Cmd.removeArgument(C); Cmd.setOutputFile(DirPlusFile(TempDir, "dft.log")); Cmd.combineOutAndErr(); // Printf("CollectDFT: %s\n", Cmd.toString().c_str()); ExecuteCommand(Cmd); } - }; struct JobQueue { @@ -265,7 +283,7 @@ FuzzJob *Pop() { std::unique_lock Lk(Mu); // std::lock_guard Lock(Mu); - Cv.wait(Lk, [&]{return !Qu.empty();}); + Cv.wait(Lk, [&] { return !Qu.empty(); }); assert(!Qu.empty()); auto Job = Qu.front(); Qu.pop(); @@ -284,7 +302,8 @@ // This is just a skeleton of an experimental -fork=1 feature. void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, const Vector &Args, - const Vector &CorpusDirs, int NumJobs) { + const Vector &CorpusDirs, int NumJobs, + int NumCorpuses) { Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs); GlobalEnv Env; @@ -296,8 +315,7 @@ Env.DataFlowBinary = Options.CollectDataFlow; Vector SeedFiles; - for (auto &Dir : CorpusDirs) - GetSizedFilesFromDir(Dir, &SeedFiles); + for (auto &Dir : CorpusDirs) GetSizedFilesFromDir(Dir, &SeedFiles); std::sort(SeedFiles.begin(), SeedFiles.end()); Env.TempDir = TempPath("FuzzWithFork", ".dir"); Env.DFTDir = DirPlusFile(Env.TempDir, "DFT"); @@ -305,15 +323,16 @@ MkDir(Env.TempDir); MkDir(Env.DFTDir); - if (CorpusDirs.empty()) MkDir(Env.MainCorpusDir = DirPlusFile(Env.TempDir, "C")); else Env.MainCorpusDir = CorpusDirs[0]; if (Options.KeepSeed) { - for (auto &File : SeedFiles) + for (auto &File : SeedFiles) { Env.Files.push_back(File.File); + Env.FilesSizes.push_back(File.Size); + } } else { auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); Set NewFeatures, NewCov; @@ -323,6 +342,11 @@ Env.Cov.insert(NewFeatures.begin(), NewFeatures.end()); RemoveFile(CFPath); } + + for (auto &path : Env.Files) { + Env.FilesSizes.push_back(FileSize(path)); + } + Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, Env.Files.size(), Env.TempDir.c_str()); @@ -331,8 +355,7 @@ JobQueue FuzzQ, MergeQ; auto StopJobs = [&]() { - for (int i = 0; i < NumJobs; i++) - FuzzQ.Push(nullptr); + for (int i = 0; i < NumJobs; i++) FuzzQ.Push(nullptr); MergeQ.Push(nullptr); WriteToFile(Unit({1}), Env.StopFile()); }; @@ -341,13 +364,12 @@ Vector Threads; for (int t = 0; t < NumJobs; t++) { Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ)); - FuzzQ.Push(Env.CreateNewJob(JobId++)); + FuzzQ.Push(Env.CreateNewJob(JobId++, NumCorpuses)); } while (true) { std::unique_ptr Job(MergeQ.Pop()); - if (!Job) - break; + if (!Job) break; ExitCode = Job->ExitCode; if (ExitCode == Options.InterruptExitCode) { Printf("==%lu== libFuzzer: a child was interrupted; exiting\n", GetPid()); @@ -356,7 +378,7 @@ } Fuzzer::MaybeExitGracefully(); - Env.RunOneMergeJob(Job.get()); + Env.RunOneMergeJob(Job.get(), NumCorpuses); // Continue if our crash is one of the ignorred ones. if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) @@ -398,12 +420,21 @@ StopJobs(); break; } - - FuzzQ.Push(Env.CreateNewJob(JobId++)); + // Since the number of corpus seeds will gradually increase, in order to + // control the number in each group to be about three times the number of + // seeds selected each time, the number of groups is dynamically adjusted. + if (Env.Files.size() >= 1 && Env.Files.size() < 1600) NumCorpuses = 16; + if (Env.Files.size() >= 1600 && Env.Files.size() < 3600) NumCorpuses = 20; + if (Env.Files.size() >= 3600 && Env.Files.size() < 6400) NumCorpuses = 24; + if (Env.Files.size() >= 6400 && Env.Files.size() < 8100) NumCorpuses = 32; + if (Env.Files.size() >= 8100 && Env.Files.size() < 12000) NumCorpuses = 36; + if (Env.Files.size() >= 12000 && Env.Files.size() < 16000) NumCorpuses = 40; + if (Env.Files.size() >= 16000 && Env.Files.size() < 30000) NumCorpuses = 60; + if (Env.Files.size() >= 30000) NumCorpuses = 80; + + FuzzQ.Push(Env.CreateNewJob(JobId++, NumCorpuses)); } - - for (auto &T : Threads) - T.join(); + for (auto &T : Threads) T.join(); // The workers have terminated. Don't try to remove the directory before they // terminate to avoid a race condition preventing cleanup on Windows. @@ -415,4 +446,4 @@ exit(ExitCode); } -} // namespace fuzzer +} // namespace fuzzer