Index: compiler-rt/lib/fuzzer/FuzzerDriver.cpp =================================================================== --- compiler-rt/lib/fuzzer/FuzzerDriver.cpp +++ compiler-rt/lib/fuzzer/FuzzerDriver.cpp @@ -10,0 +11,10 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + @@ -22,9 +31,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include @@ -43 +44 @@ -#endif // LIBFUZZER_MSVC +#endif // LIBFUZZER_MSVC @@ -51,2 +52,2 @@ - int Default; - int *IntFlag; + int Default; + int *IntFlag; @@ -69 +70 @@ -static const FlagDescription FlagDescriptions [] { +static const FlagDescription FlagDescriptions[]{ @@ -76 +77 @@ - nullptr, nullptr, &Flags.Name}, + nullptr, nullptr, &Flags.Name}, @@ -108 +109,2 @@ - if (strstr(D.Description, "internal flag") == D.Description) continue; + if (strstr(D.Description, "internal flag") == D.Description) + continue; @@ -116 +118 @@ - "will be passed verbatim to subprocesses.\n"); + "will be passed verbatim to subprocesses.\n"); @@ -123 +125 @@ - return &Param[Len + 2]; + return &Param[Len + 2]; @@ -145 +147,2 @@ - if (Param[0] != '-') return false; + if (Param[0] != '-') + return false; @@ -160 +163 @@ - if (Str) { + if (Str) { @@ -178 +181 @@ - } else { // Deprecated flag. + } else { // Deprecated flag. @@ -185 +188,2 @@ - "use -help=1 to list all flags\n\n", Param); + "use -help=1 to list all flags\n\n", + Param); @@ -206 +210,2 @@ - "Disabling -len_control by default.\n", EF->LLVMFuzzerCustomMutator); + "Disabling -len_control by default.\n", + EF->LLVMFuzzerCustomMutator); @@ -234 +239,2 @@ - if (C >= NumJobs) break; + if (C >= NumJobs) + break; @@ -275,2 +281,2 @@ -std::string CloneArgsWithoutX(const Vector &Args, - const char *X1, const char *X2) { +std::string CloneArgsWithoutX(const Vector &Args, const char *X1, + const char *X2) { @@ -297 +303,2 @@ - V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, &HasErrors)); + V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, + &HasErrors)); @@ -334 +341,2 @@ - if (Inputs->empty()) return false; + if (Inputs->empty()) + return false; @@ -352 +360 @@ - const FuzzingOptions &Options) { + const FuzzingOptions &Options) { @@ -355 +363 @@ - " -exact_artifact_path\n"); + " -exact_artifact_path\n"); @@ -401 +409,2 @@ - if (!Changed) break; + if (!Changed) + break; @@ -534,2 +543 @@ -int AnalyzeDictionary(Fuzzer *F, const Vector& Dict, - UnitVector& Corpus) { +int AnalyzeDictionary(Fuzzer *F, const Vector &Dict, UnitVector &Corpus) { @@ -549,3 +557,2 @@ - TPC.CollectFeatures([&](size_t Feature) { - InitialFeatures.push_back(Feature); - }); + TPC.CollectFeatures( + [&](size_t Feature) { InitialFeatures.push_back(Feature); }); @@ -555,2 +562,2 @@ - auto StartPos = std::search(Data.begin(), Data.end(), - Dict[i].begin(), Dict[i].end()); + auto StartPos = + std::search(Data.begin(), Data.end(), Dict[i].begin(), Dict[i].end()); @@ -568,2 +575,2 @@ - StartPos = std::search(EndPos, Data.end(), - Dict[i].begin(), Dict[i].end()); + StartPos = + std::search(EndPos, Data.end(), Dict[i].begin(), Dict[i].end()); @@ -575,3 +582,2 @@ - TPC.CollectFeatures([&](size_t Feature) { - ModifiedFeatures.push_back(Feature); - }); + TPC.CollectFeatures( + [&](size_t Feature) { ModifiedFeatures.push_back(Feature); }); @@ -590 +596 @@ - continue; + continue; @@ -603 +609,2 @@ - if (!seed_inputs) return Files; + if (!seed_inputs) + return Files; @@ -623,2 +630,3 @@ -static Vector ReadCorpora(const Vector &CorpusDirs, - const Vector &ExtraSeedFiles) { +static Vector +ReadCorpora(const Vector &CorpusDirs, + const Vector &ExtraSeedFiles) { @@ -800 +808 @@ - ReadCorpora({}, *Inputs)); + ReadCorpora({}, *Inputs)); @@ -803 +811 @@ - ReadCorpora(*Inputs, {})); + ReadCorpora(*Inputs, {})); @@ -811 +819 @@ - for (auto &U: Dictionary) + for (auto &U : Dictionary) @@ -870 +878,2 @@ - FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); + FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork, + Flags.group); @@ -885 +894 @@ - size_t MaxLen = INT_MAX; // Large max length. + size_t MaxLen = INT_MAX; // Large max length. @@ -889,2 +898,2 @@ - ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr, - MaxLen, /*ExitOnError=*/false); + ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr, MaxLen, + /*ExitOnError=*/false); @@ -913 +922 @@ - exit(0); // Don't let F destroy itself. + exit(0); // Don't let F destroy itself. @@ -925 +934 @@ -} // namespace fuzzer +} // namespace fuzzer Index: compiler-rt/lib/fuzzer/FuzzerFlags.def =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFlags.def +++ compiler-rt/lib/fuzzer/FuzzerFlags.def @@ -15,2 +15,4 @@ - "Number of individual test runs (-1 for infinite runs).") -FUZZER_FLAG_INT(max_len, 0, "Maximum length of the test input. " + "Number of individual test runs (-1 for infinite runs).") +FUZZER_FLAG_INT( + max_len, 0, + "Maximum length of the test input. " @@ -19,11 +21,20 @@ -FUZZER_FLAG_INT(len_control, 100, "Try generating small inputs first, " - "then try larger inputs over time. Specifies the rate at which the length " - "limit is increased (smaller == faster). If 0, immediately try inputs with " - "size up to max_len. Default value is 0, if LLVMFuzzerCustomMutator is used.") -FUZZER_FLAG_STRING(seed_inputs, "A comma-separated list of input files " - "to use as an additional seed corpus. Alternatively, an \"@\" followed by " - "the name of a file containing the comma-separated list.") -FUZZER_FLAG_INT(keep_seed, 0, "If 1, keep seed inputs in the corpus even if " - "they do not produce new coverage. When used with |reduce_inputs==1|, the " - "seed inputs will never be reduced. This option can be useful when seeds are" - "not properly formed for the fuzz target but still have useful snippets.") +FUZZER_FLAG_INT( + len_control, 100, + "Try generating small inputs first, " + "then try larger inputs over time. Specifies the rate at which the length " + "limit is increased (smaller == faster). If 0, immediately try inputs " + "with " + "size up to max_len. Default value is 0, if LLVMFuzzerCustomMutator is " + "used.") +FUZZER_FLAG_STRING( + seed_inputs, + "A comma-separated list of input files " + "to use as an additional seed corpus. Alternatively, an \"@\" followed by " + "the name of a file containing the comma-separated list.") +FUZZER_FLAG_INT( + keep_seed, 0, + "If 1, keep seed inputs in the corpus even if " + "they do not produce new coverage. When used with |reduce_inputs==1|, the " + "seed inputs will never be reduced. This option can be useful when seeds " + "are" + "not properly formed for the fuzz target but still have useful snippets.") @@ -31,9 +42,11 @@ -FUZZER_FLAG_INT(cross_over_uniform_dist, 0, "Experimental. If 1, use a " - "uniform probability distribution when choosing inputs to cross over with. " - "Some of the inputs in the corpus may never get chosen for mutation " - "depending on the input mutation scheduling policy. With this flag, all " - "inputs, regardless of the input mutation scheduling policy, can be chosen " - "as an input to cross over with. This can be particularly useful with " - "|keep_seed==1|; all the initial seed inputs, even though they do not " - "increase coverage because they are not properly formed, will still be " - "chosen as an input to cross over with.") +FUZZER_FLAG_INT( + cross_over_uniform_dist, 0, + "Experimental. If 1, use a " + "uniform probability distribution when choosing inputs to cross over with. " + "Some of the inputs in the corpus may never get chosen for mutation " + "depending on the input mutation scheduling policy. With this flag, all " + "inputs, regardless of the input mutation scheduling policy, can be chosen " + "as an input to cross over with. This can be particularly useful with " + "|keep_seed==1|; all the initial seed inputs, even though they do not " + "increase coverage because they are not properly formed, will still be " + "chosen as an input to cross over with.") @@ -42,2 +55,3 @@ - "Apply this number of consecutive mutations to each input.") -FUZZER_FLAG_INT(reduce_depth, 0, "Experimental/internal. " + "Apply this number of consecutive mutations to each input.") +FUZZER_FLAG_INT(reduce_depth, 0, + "Experimental/internal. " @@ -47 +61 @@ - "If 1, always prefer smaller inputs during the corpus shuffle.") + "If 1, always prefer smaller inputs during the corpus shuffle.") @@ -52,6 +66,9 @@ -FUZZER_FLAG_INT(error_exitcode, 77, "When libFuzzer itself reports a bug " - "this exit code will be used.") -FUZZER_FLAG_INT(timeout_exitcode, 70, "When libFuzzer reports a timeout " - "this exit code will be used.") -FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total " - "time in seconds to run the fuzzer.") +FUZZER_FLAG_INT(error_exitcode, 77, + "When libFuzzer itself reports a bug " + "this exit code will be used.") +FUZZER_FLAG_INT(timeout_exitcode, 70, + "When libFuzzer reports a timeout " + "this exit code will be used.") +FUZZER_FLAG_INT(max_total_time, 0, + "If positive, indicates the maximal total " + "time in seconds to run the fuzzer.") @@ -59 +76,5 @@ -FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens " +FUZZER_FLAG_INT(group, 1, + "FOR fork mode. Divide the main corpus into N groups " + "according to size.") +FUZZER_FLAG_INT(fork, 0, + "Experimental mode where fuzzing happens " @@ -64,3 +85,5 @@ -FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be " - "merged into the 1-st corpus. Only interesting units will be taken. " - "This flag can be used to minimize a corpus.") +FUZZER_FLAG_INT( + merge, 0, + "If 1, the 2-nd, 3-rd, etc corpora will be " + "merged into the 1-st corpus. Only interesting units will be taken. " + "This flag can be used to minimize a corpus.") @@ -75,11 +98,12 @@ -FUZZER_FLAG_INT(minimize_crash, 0, "If 1, minimizes the provided" - " crash input. Use with -runs=N or -max_total_time=N to limit " - "the number attempts." - " Use with -exact_artifact_path to specify the output." - " Combine with ASAN_OPTIONS=dedup_token_length=3 (or similar) to ensure that" - " the minimized input triggers the same crash." - ) -FUZZER_FLAG_INT(cleanse_crash, 0, "If 1, tries to cleanse the provided" - " crash input to make it contain fewer original bytes." - " Use with -exact_artifact_path to specify the output." - ) +FUZZER_FLAG_INT(minimize_crash, 0, + "If 1, minimizes the provided" + " crash input. Use with -runs=N or -max_total_time=N to limit " + "the number attempts." + " Use with -exact_artifact_path to specify the output." + " Combine with ASAN_OPTIONS=dedup_token_length=3 (or similar) " + "to ensure that" + " the minimized input triggers the same crash.") +FUZZER_FLAG_INT(cleanse_crash, 0, + "If 1, tries to cleanse the provided" + " crash input to make it contain fewer original bytes." + " Use with -exact_artifact_path to specify the output.") @@ -87,9 +111,13 @@ -FUZZER_FLAG_STRING(features_dir, "internal flag. Used to dump feature sets on disk." - "Every time a new input is added to the corpus, a corresponding file in the features_dir" - " is created containing the unique features of that input." - " Features are stored in binary format.") -FUZZER_FLAG_STRING(mutation_graph_file, "Saves a graph (in DOT format) to" - " mutation_graph_file. The graph contains a vertex for each input that has" - " unique coverage; directed edges are provided between parents and children" - " where the child has unique coverage, and are recorded with the type of" - " mutation that caused the child.") +FUZZER_FLAG_STRING(features_dir, + "internal flag. Used to dump feature sets on disk." + "Every time a new input is added to the corpus, a " + "corresponding file in the features_dir" + " is created containing the unique features of that input." + " Features are stored in binary format.") +FUZZER_FLAG_STRING( + mutation_graph_file, + "Saves a graph (in DOT format) to" + " mutation_graph_file. The graph contains a vertex for each input that has" + " unique coverage; directed edges are provided between parents and children" + " where the child has unique coverage, and are recorded with the type of" + " mutation that caused the child.") @@ -103,5 +131,7 @@ -FUZZER_FLAG_INT(reduce_inputs, 1, - "Try to reduce the size of inputs while preserving their full feature sets") -FUZZER_FLAG_UNSIGNED(jobs, 0, "Number of jobs to run. If jobs >= 1 we spawn" - " this number of jobs in separate worker processes" - " with stdout/stderr redirected to fuzz-JOB.log.") +FUZZER_FLAG_INT( + reduce_inputs, 1, + "Try to reduce the size of inputs while preserving their full feature sets") +FUZZER_FLAG_UNSIGNED(jobs, 0, + "Number of jobs to run. If jobs >= 1 we spawn" + " this number of jobs in separate worker processes" + " with stdout/stderr redirected to fuzz-JOB.log.") @@ -109,2 +139,2 @@ - "Number of simultaneous worker processes to run the jobs." - " If zero, \"min(jobs,NumberOfCpuCores()/2)\" is used.") + "Number of simultaneous worker processes to run the jobs." + " If zero, \"min(jobs,NumberOfCpuCores()/2)\" is used.") @@ -114 +144,2 @@ -FUZZER_FLAG_INT(report_slow_units, 10, +FUZZER_FLAG_INT( + report_slow_units, 10, @@ -128,2 +159,3 @@ -FUZZER_FLAG_INT(print_funcs, 2, "If >=1, print out at most this number of " - "newly covered functions.") +FUZZER_FLAG_INT(print_funcs, 2, + "If >=1, print out at most this number of " + "newly covered functions.") @@ -132,5 +164,7 @@ - "If 1, print statistics on corpus elements at exit.") -FUZZER_FLAG_INT(print_coverage, 0, "If 1, print coverage information as text" - " at exit.") -FUZZER_FLAG_INT(print_full_coverage, 0, "If 1, print full coverage information " - "(all branches) as text at exit.") + "If 1, print statistics on corpus elements at exit.") +FUZZER_FLAG_INT(print_coverage, 0, + "If 1, print coverage information as text" + " at exit.") +FUZZER_FLAG_INT(print_full_coverage, 0, + "If 1, print full coverage information " + "(all branches) as text at exit.") @@ -148,6 +182,10 @@ -FUZZER_FLAG_INT(handle_winexcept, 1, "If 1, try to intercept uncaught Windows " - "Visual C++ Exceptions.") -FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; " - "if 2, close stderr; if 3, close both. " - "Be careful, this will also close e.g. stderr of asan.") -FUZZER_FLAG_INT(detect_leaks, 1, "If 1, and if LeakSanitizer is enabled " +FUZZER_FLAG_INT(handle_winexcept, 1, + "If 1, try to intercept uncaught Windows " + "Visual C++ Exceptions.") +FUZZER_FLAG_INT(close_fd_mask, 0, + "If 1, close stdout at startup; " + "if 2, close stderr; if 3, close both. " + "Be careful, this will also close e.g. stderr of asan.") +FUZZER_FLAG_INT( + detect_leaks, 1, + "If 1, and if LeakSanitizer is enabled " @@ -155 +193,3 @@ -FUZZER_FLAG_INT(purge_allocator_interval, 1, "Purge allocator caches and " +FUZZER_FLAG_INT( + purge_allocator_interval, 1, + "Purge allocator caches and " @@ -159,5 +199,9 @@ -FUZZER_FLAG_INT(trace_malloc, 0, "If >= 1 will print all mallocs/frees. " - "If >= 2 will also print stack traces.") -FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon" - "reaching this limit of RSS memory usage.") -FUZZER_FLAG_INT(malloc_limit_mb, 0, "If non-zero, the fuzzer will exit " +FUZZER_FLAG_INT(trace_malloc, 0, + "If >= 1 will print all mallocs/frees. " + "If >= 2 will also print stack traces.") +FUZZER_FLAG_INT(rss_limit_mb, 2048, + "If non-zero, the fuzzer will exit upon" + "reaching this limit of RSS memory usage.") +FUZZER_FLAG_INT( + malloc_limit_mb, 0, + "If non-zero, the fuzzer will exit " @@ -166 +210,3 @@ -FUZZER_FLAG_STRING(exit_on_src_pos, "Exit if a newly found PC originates" +FUZZER_FLAG_STRING( + exit_on_src_pos, + "Exit if a newly found PC originates" @@ -170,3 +216,4 @@ - " was added to the corpus. " - "Used primarily for testing libFuzzer itself.") -FUZZER_FLAG_INT(ignore_remaining_args, 0, "If 1, ignore all arguments passed " + " was added to the corpus. " + "Used primarily for testing libFuzzer itself.") +FUZZER_FLAG_INT(ignore_remaining_args, 0, + "If 1, ignore all arguments passed " @@ -175,5 +222,7 @@ -FUZZER_FLAG_STRING(focus_function, "Experimental. " - "Fuzzing will focus on inputs that trigger calls to this function. " - "If -focus_function=auto and -data_flow_trace is used, libFuzzer " - "will choose the focus functions automatically. Disables -entropic when " - "specified.") +FUZZER_FLAG_STRING( + focus_function, + "Experimental. " + "Fuzzing will focus on inputs that trigger calls to this function. " + "If -focus_function=auto and -data_flow_trace is used, libFuzzer " + "will choose the focus functions automatically. Disables -entropic when " + "specified.") @@ -181,12 +230,18 @@ -FUZZER_FLAG_INT(entropic_feature_frequency_threshold, 0xFF, "Experimental. If " - "entropic is enabled, all features which are observed less often than " - "the specified value are considered as rare.") -FUZZER_FLAG_INT(entropic_number_of_rarest_features, 100, "Experimental. If " - "entropic is enabled, we keep track of the frequencies only for the " - "Top-X least abundant features (union features that are considered as " - "rare).") -FUZZER_FLAG_INT(entropic_scale_per_exec_time, 0, "Experimental. If 1, " - "the Entropic power schedule gets scaled based on the input execution " - "time. Inputs with lower execution time get scheduled more (up to 30x). " - "Note that, if 1, fuzzer stops from being deterministic even if a " - "non-zero random seed is given.") +FUZZER_FLAG_INT( + entropic_feature_frequency_threshold, 0xFF, + "Experimental. If " + "entropic is enabled, all features which are observed less often than " + "the specified value are considered as rare.") +FUZZER_FLAG_INT( + entropic_number_of_rarest_features, 100, + "Experimental. If " + "entropic is enabled, we keep track of the frequencies only for the " + "Top-X least abundant features (union features that are considered as " + "rare).") +FUZZER_FLAG_INT( + entropic_scale_per_exec_time, 0, + "Experimental. If 1, " + "the Entropic power schedule gets scaled based on the input execution " + "time. Inputs with lower execution time get scheduled more (up to 30x). " + "Note that, if 1, fuzzer stops from being deterministic even if a " + "non-zero random seed is given.") @@ -200,3 +255,8 @@ -FUZZER_FLAG_INT(create_missing_dirs, 0, "Automatically attempt to create " - "directories for arguments that would normally expect them to already " - "exist (i.e. artifact_prefix, exact_artifact_path, features_dir, corpus)") +FUZZER_FLAG_INT( + create_missing_dirs, 0, + "Automatically attempt to create " + "directories for arguments that would normally expect them to already " + "exist (i.e. artifact_prefix, exact_artifact_path, features_dir, corpus)") +FUZZER_FLAG_INT(group, 1, + "FOR fork mode. Divide the main corpus into N groups " + "according to size.") Index: compiler-rt/lib/fuzzer/FuzzerFork.h =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFork.h +++ compiler-rt/lib/fuzzer/FuzzerFork.h @@ -11,0 +12,2 @@ +#include + @@ -16,2 +17,0 @@ -#include - @@ -21 +21,2 @@ - const Vector &CorpusDirs, int NumJobs); + const Vector &CorpusDirs, int NumJobs, + int Group); Index: compiler-rt/lib/fuzzer/FuzzerFork.cpp =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFork.cpp +++ compiler-rt/lib/fuzzer/FuzzerFork.cpp @@ -11 +10,0 @@ -#include "FuzzerCommand.h" @@ -13,6 +11,0 @@ -#include "FuzzerIO.h" -#include "FuzzerInternal.h" -#include "FuzzerMerge.h" -#include "FuzzerSHA1.h" -#include "FuzzerTracePC.h" -#include "FuzzerUtil.h" @@ -29,0 +23,8 @@ +#include "FuzzerCommand.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include "FuzzerMerge.h" +#include "FuzzerSHA1.h" +#include "FuzzerTracePC.h" +#include "FuzzerUtil.h" + @@ -52 +53,2 @@ - if (Line.find("stat::") != 0) continue; + if (Line.find("stat::") != 0) + continue; @@ -72 +74 @@ - size_t JobId; + size_t JobId; @@ -74 +76 @@ - int DftTimeInSeconds = 0; + int DftTimeInSeconds = 0; @@ -97,0 +100,2 @@ + // Declare a variable to store the seed size. + Vector FilesSizes; @@ -106 +109,0 @@ - @@ -117 +120 @@ - FuzzJob *CreateNewJob(size_t JobId) { + FuzzJob *CreateNewJob(size_t JobId, int NumCorpuses, int Group) { @@ -124 +127 @@ - Cmd.addFlag("reload", "0"); // working in an isolated dir, no reload. + Cmd.addFlag("reload", "0"); // working in an isolated dir, no reload. @@ -126 +129 @@ - Cmd.addFlag("print_funcs", "0"); // no need to spend time symbolizing. + Cmd.addFlag("print_funcs", "0"); // no need to spend time symbolizing. @@ -135,0 +139 @@ + auto Time1 = std::chrono::system_clock::now(); @@ -138,5 +142,27 @@ - auto Time1 = std::chrono::system_clock::now(); - for (size_t i = 0; i < CorpusSubsetSize; i++) { - auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; - Seeds += (Seeds.empty() ? "" : ",") + SF; - CollectDFT(SF); + if (Group) { // flag: whether to group the corpus. + size_t AverageCorpusSize = Files.size() / NumCorpuses + 1; + if (Files.size() == 0) + AverageCorpusSize = 0; + size_t StartIndex = ((JobId - 1) % NumCorpuses) * AverageCorpusSize; + for (size_t i = 0; i < CorpusSubsetSize; i++) { + std::random_device rd; + std::mt19937 RandomSeed(rd()); + std::uniform_int_distribution<> rand(0, AverageCorpusSize); + size_t RandNum = rand(RandomSeed); + size_t Index = RandNum + StartIndex; + if (Index < Files.size()) { + auto &SF = Files[Index]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } else { + auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } + } + } else { + for (size_t i = 0; i < CorpusSubsetSize; i++) { + auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } @@ -144,4 +169,0 @@ - auto Time2 = std::chrono::system_clock::now(); - auto DftTimeInSeconds = duration_cast(Time2 - Time1).count(); - assert(DftTimeInSeconds < std::numeric_limits::max()); - Job->DftTimeInSeconds = static_cast(DftTimeInSeconds); @@ -148,0 +171,5 @@ + auto Time2 = std::chrono::system_clock::now(); + auto DftTimeInSeconds = duration_cast(Time2 - Time1).count(); + assert(DftTimeInSeconds < std::numeric_limits::max()); + Job->DftTimeInSeconds = static_cast(DftTimeInSeconds); + @@ -161 +187,0 @@ - @@ -182 +208 @@ - void RunOneMergeJob(FuzzJob *Job) { + void RunOneMergeJob(FuzzJob *Job, int Group) { @@ -212 +238,2 @@ - if (MergeCandidates.empty()) return; + if (MergeCandidates.empty()) + return; @@ -222 +249,9 @@ - Files.push_back(NewPath); + if (Group) { // Insert the queue according to the size of the seed. + long usz = U.size(); + auto idx = std::upper_bound(FilesSizes.begin(), FilesSizes.end(), usz) - + FilesSizes.begin(); + FilesSizes.insert(FilesSizes.begin() + idx, usz); + Files.insert(Files.begin() + idx, NewPath); + } else { + Files.push_back(NewPath); + } @@ -231 +265,0 @@ - @@ -234 +267,0 @@ - @@ -236,2 +269,4 @@ - if (DataFlowBinary.empty()) return; - if (!FilesWithDFT.insert(InputPath).second) return; + if (DataFlowBinary.empty()) + return; + if (!FilesWithDFT.insert(InputPath).second) + return; @@ -250 +284,0 @@ - @@ -268 +302 @@ - Cv.wait(Lk, [&]{return !Qu.empty();}); + Cv.wait(Lk, [&] { return !Qu.empty(); }); @@ -287 +321,2 @@ - const Vector &CorpusDirs, int NumJobs) { + const Vector &CorpusDirs, int NumJobs, + int Group) { @@ -304 +339 @@ - RmDirRecursive(Env.TempDir); // in case there is a leftover from old runs. + RmDirRecursive(Env.TempDir); // in case there is a leftover from old runs. @@ -308 +342,0 @@ - @@ -315 +349 @@ - for (auto &File : SeedFiles) + for (auto &File : SeedFiles) { @@ -316,0 +351,2 @@ + Env.FilesSizes.push_back(File.Size); + } @@ -325,0 +362,7 @@ + + if (Group) { + for (auto &path : Env.Files) { + Env.FilesSizes.push_back(FileSize(path)); + } + } + @@ -339,0 +383,3 @@ + size_t MergeCycle = 20; + size_t JobExecuted = 0; + int NumCorpuses = 8; @@ -344 +390 @@ - FuzzQ.Push(Env.CreateNewJob(JobId++)); + FuzzQ.Push(Env.CreateNewJob(JobId++, NumCorpuses, Group)); @@ -359 +405,30 @@ - Env.RunOneMergeJob(Job.get()); + Env.RunOneMergeJob(Job.get(), Group); + // merge the corpus . + JobExecuted++; + if (Group) { + if (JobExecuted >= MergeCycle) { + Vector CurrentSeedFiles; + for (auto &Dir : CorpusDirs) + GetSizedFilesFromDir(Dir, &CurrentSeedFiles); + std::sort(CurrentSeedFiles.begin(), CurrentSeedFiles.end()); + if (CorpusDirs.empty()) + MkDir(Env.MainCorpusDir = DirPlusFile(Env.TempDir, "C")); + else + Env.MainCorpusDir = CorpusDirs[0]; + + auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); + Set TmpNewFeatures, TmpNewCov; + Set TmpFeatures, TmpCov; + Env.Files.clear(); + Env.FilesSizes.clear(); + CrashResistantMerge(Env.Args, {}, CurrentSeedFiles, &Env.Files, + TmpFeatures, &TmpNewFeatures, TmpCov, &TmpNewCov, + CFPath, false); + for (auto &path : Env.Files) { + Env.FilesSizes.push_back(FileSize(path)); + } + RemoveFile(CFPath); + JobExecuted = 0; + MergeCycle += 5; + } + } @@ -402 +477,20 @@ - FuzzQ.Push(Env.CreateNewJob(JobId++)); + // Since the number of corpus seeds will gradually increase, in order to + // control the number in each group to be about three times the number of + // seeds selected each time, the number of groups is dynamically adjusted. + if (Env.Files.size() >= 1 && Env.Files.size() < 1600) + NumCorpuses = 16; + if (Env.Files.size() >= 1600 && Env.Files.size() < 3600) + NumCorpuses = 20; + if (Env.Files.size() >= 3600 && Env.Files.size() < 6400) + NumCorpuses = 24; + if (Env.Files.size() >= 6400 && Env.Files.size() < 8100) + NumCorpuses = 32; + if (Env.Files.size() >= 8100 && Env.Files.size() < 12000) + NumCorpuses = 36; + if (Env.Files.size() >= 12000 && Env.Files.size() < 16000) + NumCorpuses = 40; + if (Env.Files.size() >= 16000 && Env.Files.size() < 30000) + NumCorpuses = 60; + if (Env.Files.size() >= 30000) + NumCorpuses = 80; + FuzzQ.Push(Env.CreateNewJob(JobId++, NumCorpuses, Group)); @@ -404 +497,0 @@ - @@ -418,0 +512,16 @@ +// Declare a variable to store the seed size. +Vector FilesSizes; +auto Time1 = std::chrono::system_clock::now(); +auto Time2 = std::chrono::system_clock::now(); +auto DftTimeInSeconds = duration_cast(Time2 - Time1).count(); +assert(DftTimeInSeconds < std::numeric_limits::max()); +Job->DftTimeInSeconds = static_cast(DftTimeInSeconds); + +Env.FilesSizes.push_back(File.Size); +} + +if (Group) { + for (auto &path : Env.Files) { + Env.FilesSizes.push_back(FileSize(path)); + } +}