Index: compiler-rt/lib/fuzzer/FuzzerDriver.cpp =================================================================== --- compiler-rt/lib/fuzzer/FuzzerDriver.cpp +++ compiler-rt/lib/fuzzer/FuzzerDriver.cpp @@ -867,7 +867,8 @@ } if (Flags.fork) - FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); + FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork, + Flags.NumCorpuses); if (Flags.merge) Merge(F, Options, Args, *Inputs, Flags.merge_control_file); Index: compiler-rt/lib/fuzzer/FuzzerFlags.def =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFlags.def +++ compiler-rt/lib/fuzzer/FuzzerFlags.def @@ -56,6 +56,8 @@ FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total " "time in seconds to run the fuzzer.") FUZZER_FLAG_INT(help, 0, "Print help.") +FUZZER_FLAG_INT(NumCorpuses, 1, "FOR fork mode. Divide the main corpus into + N parts according to size.") FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens " "in a subprocess") FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode") Index: compiler-rt/lib/fuzzer/FuzzerFork.h =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFork.h +++ compiler-rt/lib/fuzzer/FuzzerFork.h @@ -18,7 +18,8 @@ namespace fuzzer { void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, const Vector &Args, - const Vector &CorpusDirs, int NumJobs); + const Vector &CorpusDirs, int NumJobs, + int NumCorpuses); } // namespace fuzzer #endif // LLVM_FUZZER_FORK_H Index: compiler-rt/lib/fuzzer/FuzzerFork.cpp =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFork.cpp +++ compiler-rt/lib/fuzzer/FuzzerFork.cpp @@ -95,6 +95,8 @@ Set Features, Cov; Set FilesWithDFT; Vector Files; + //This variable is used to store the size of the seed. + Vector FilesSizes; Random *Rand; std::chrono::system_clock::time_point ProcessStartTime; int Verbosity = 0; @@ -114,7 +116,7 @@ .count(); } - FuzzJob *CreateNewJob(size_t JobId) { + FuzzJob *CreateNewJob(size_t JobId, int NumCorpuses) { Command Cmd(Args); Cmd.removeFlag("fork"); Cmd.removeFlag("runs"); @@ -135,11 +137,28 @@ std::string Seeds; if (size_t CorpusSubsetSize = std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) { + size_t AverageSize = Files.size() / NumCorpuses + 1; auto Time1 = std::chrono::system_clock::now(); + size_t StartIndex = ((JobId - 1) % NumCorpuses) * AverageSize; + // At this time, the seeds in the File variable are sorted according to + // the seed size, so by generating a uniformly distributed random + // number, the seeds are selected in a certain group. for (size_t i = 0; i < CorpusSubsetSize; i++) { - auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; - Seeds += (Seeds.empty() ? "" : ",") + SF; - CollectDFT(SF); + std::random_device rd; + std::mt19937 randomseed(rd()); + std::uniform_int_distribution<> rand(0,AverageSize); + size_t j = rand(randomseed); + size_t m = j + StartIndex; + if(m < Files.size()) { + auto &SF = Files[m]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } + else{ + auto &SF = Files[rand(randomseed)]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } } auto Time2 = std::chrono::system_clock::now(); auto DftTimeInSeconds = duration_cast(Time2 - Time1).count(); @@ -179,7 +198,7 @@ return Job; } - void RunOneMergeJob(FuzzJob *Job) { + void RunOneMergeJob(FuzzJob *Job, int NumCorpuses) { auto Stats = ParseFinalStatsFromLog(Job->LogPath); NumRuns += Stats.number_of_executed_units; @@ -219,7 +238,12 @@ auto U = FileToVector(Path); auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); WriteToFile(U, NewPath); - Files.push_back(NewPath); + // Seeds are inserted into Files according to size. + long usz = U.size(); + auto idx = std::upper_bound(FilesSizes.begin(), FilesSizes.end(), usz) - + FilesSizes.begin(); + FilesSizes.insert(FilesSizes.begin() + idx, usz); + Files.insert(Files.begin() + idx, NewPath); } Features.insert(NewFeatures.begin(), NewFeatures.end()); Cov.insert(NewCov.begin(), NewCov.end()); @@ -284,7 +308,8 @@ // This is just a skeleton of an experimental -fork=1 feature. void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, const Vector &Args, - const Vector &CorpusDirs, int NumJobs) { + const Vector &CorpusDirs, int NumJobs, + int NumCorpuses) { Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs); GlobalEnv Env; @@ -311,10 +336,13 @@ else Env.MainCorpusDir = CorpusDirs[0]; - if (Options.KeepSeed) { - for (auto &File : SeedFiles) + if (Options.KeepSeed){ + for (auto &File : SeedFiles){ Env.Files.push_back(File.File); - } else { + Env.FilesSizes.push_back(File.Size); + } + } + else { auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); Set NewFeatures, NewCov; CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, Env.Features, @@ -323,9 +351,15 @@ Env.Cov.insert(NewFeatures.begin(), NewFeatures.end()); RemoveFile(CFPath); } + + for (auto &path : Env.Files) { + Env.FilesSizes.push_back(FileSize(path)); + } + Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, Env.Files.size(), Env.TempDir.c_str()); + int ExitCode = 0; JobQueue FuzzQ, MergeQ; @@ -341,7 +375,7 @@ Vector Threads; for (int t = 0; t < NumJobs; t++) { Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ)); - FuzzQ.Push(Env.CreateNewJob(JobId++)); + FuzzQ.Push(Env.CreateNewJob(JobId++, NumCorpuses)); } while (true) { @@ -356,7 +390,7 @@ } Fuzzer::MaybeExitGracefully(); - Env.RunOneMergeJob(Job.get()); + Env.RunOneMergeJob(Job.get(), NumCorpuses); // Continue if our crash is one of the ignorred ones. if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) @@ -398,10 +432,28 @@ StopJobs(); break; } - - FuzzQ.Push(Env.CreateNewJob(JobId++)); + // Since the number of corpus seeds will gradually increase, in order to + // control the number in each group to be about three times the number of + // seeds selected each time, the number of groups is dynamically adjusted. + if (Env.Files.size() >= 1 && Env.Files.size() < 1600) + NumCorpuses = 16; + if (Env.Files.size() >= 1600 && Env.Files.size() < 3600) + NumCorpuses = 20; + if (Env.Files.size() >= 3600 && Env.Files.size() < 6400) + NumCorpuses = 24; + if (Env.Files.size() >= 6400 && Env.Files.size() < 8100) + NumCorpuses = 32; + if (Env.Files.size() >= 8100 && Env.Files.size() < 12000) + NumCorpuses = 36; + if (Env.Files.size() >= 12000 && Env.Files.size() < 16000) + NumCorpuses = 40; + if (Env.Files.size() >= 16000 && Env.Files.size() < 30000) + NumCorpuses = 60; + if(Env.Files.size() >= 30000) + NumCorpuses = 80; + + FuzzQ.Push(Env.CreateNewJob(JobId++, NumCorpuses)); } - for (auto &T : Threads) T.join();