Index: compiler-rt/lib/fuzzer/FuzzerDriver.cpp =================================================================== --- compiler-rt/lib/fuzzer/FuzzerDriver.cpp +++ compiler-rt/lib/fuzzer/FuzzerDriver.cpp @@ -867,7 +867,7 @@ } if (Flags.fork) - FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); + FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork, Flags.NumCorpuses); if (Flags.merge) Merge(F, Options, Args, *Inputs, Flags.merge_control_file); Index: compiler-rt/lib/fuzzer/FuzzerFlags.def =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFlags.def +++ compiler-rt/lib/fuzzer/FuzzerFlags.def @@ -56,6 +56,7 @@ FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total " "time in seconds to run the fuzzer.") FUZZER_FLAG_INT(help, 0, "Print help.") +FUZZER_FLAG_INT(NumCorpuses, 1, "FOR fork mode. Divide the main corpus into N parts according to size.") FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens " "in a subprocess") FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode") Index: compiler-rt/lib/fuzzer/FuzzerFork.h =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFork.h +++ compiler-rt/lib/fuzzer/FuzzerFork.h @@ -18,7 +18,7 @@ namespace fuzzer { void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, const Vector &Args, - const Vector &CorpusDirs, int NumJobs); + const Vector &CorpusDirs, int NumJobs, int NumCorpuses); } // namespace fuzzer #endif // LLVM_FUZZER_FORK_H Index: compiler-rt/lib/fuzzer/FuzzerFork.cpp =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFork.cpp +++ compiler-rt/lib/fuzzer/FuzzerFork.cpp @@ -95,6 +95,7 @@ Set Features, Cov; Set FilesWithDFT; Vector Files; + Vector FilesSizes; Random *Rand; std::chrono::system_clock::time_point ProcessStartTime; int Verbosity = 0; @@ -114,7 +115,7 @@ .count(); } - FuzzJob *CreateNewJob(size_t JobId) { + FuzzJob *CreateNewJob(size_t JobId, int NumCorpuses) { Command Cmd(Args); Cmd.removeFlag("fork"); Cmd.removeFlag("runs"); @@ -135,11 +136,26 @@ std::string Seeds; if (size_t CorpusSubsetSize = std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) { + size_t AverageSize = Files.size() / NumCorpuses + 1; auto Time1 = std::chrono::system_clock::now(); + size_t StartIndex = ( (JobId-1) % NumCorpuses ) * AverageSize; for (size_t i = 0; i < CorpusSubsetSize; i++) { - auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; - Seeds += (Seeds.empty() ? "" : ",") + SF; - CollectDFT(SF); + //size_t j = Rand->SkewTowardsLast(AverageSize); + std::random_device rd; + std::mt19937 randomseed(rd()); + std::uniform_int_distribution<> rand(0,AverageSize); + size_t j = rand(randomseed); + size_t m = j + StartIndex; + if (m < Files.size()) { + auto &SF = Files[m]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } + else { + auto &SF = Files[rand(randomseed)]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } } auto Time2 = std::chrono::system_clock::now(); auto DftTimeInSeconds = duration_cast(Time2 - Time1).count(); @@ -179,7 +195,7 @@ return Job; } - void RunOneMergeJob(FuzzJob *Job) { + void RunOneMergeJob(FuzzJob *Job, int NumCorpuses) { auto Stats = ParseFinalStatsFromLog(Job->LogPath); NumRuns += Stats.number_of_executed_units; @@ -219,7 +235,10 @@ auto U = FileToVector(Path); auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); WriteToFile(U, NewPath); - Files.push_back(NewPath); + long usz = U.size(); + auto idx = std::upper_bound(FilesSizes.begin(), FilesSizes.end(), usz) - FilesSizes.begin(); + FilesSizes.insert(FilesSizes.begin() + idx, usz); + Files.insert(Files.begin() + idx, NewPath); } Features.insert(NewFeatures.begin(), NewFeatures.end()); Cov.insert(NewCov.begin(), NewCov.end()); @@ -284,7 +303,7 @@ // This is just a skeleton of an experimental -fork=1 feature. void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, const Vector &Args, - const Vector &CorpusDirs, int NumJobs) { + const Vector &CorpusDirs, int NumJobs, int NumCorpuses) { Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs); GlobalEnv Env; @@ -312,8 +331,10 @@ Env.MainCorpusDir = CorpusDirs[0]; if (Options.KeepSeed) { - for (auto &File : SeedFiles) + for (auto &File : SeedFiles){ Env.Files.push_back(File.File); + Env.FilesSizes.push_back(File.Size); + } } else { auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); Set NewFeatures, NewCov; @@ -323,9 +344,15 @@ Env.Cov.insert(NewFeatures.begin(), NewFeatures.end()); RemoveFile(CFPath); } + + for (auto &path : Env.Files) { + Env.FilesSizes.push_back(FileSize(path)); + } + Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, Env.Files.size(), Env.TempDir.c_str()); + int ExitCode = 0; JobQueue FuzzQ, MergeQ; @@ -341,7 +368,7 @@ Vector Threads; for (int t = 0; t < NumJobs; t++) { Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ)); - FuzzQ.Push(Env.CreateNewJob(JobId++)); + FuzzQ.Push(Env.CreateNewJob(JobId++, NumCorpuses)); } while (true) { @@ -356,7 +383,7 @@ } Fuzzer::MaybeExitGracefully(); - Env.RunOneMergeJob(Job.get()); + Env.RunOneMergeJob(Job.get(), NumCorpuses); // Continue if our crash is one of the ignorred ones. if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) @@ -398,10 +425,25 @@ StopJobs(); break; } - - FuzzQ.Push(Env.CreateNewJob(JobId++)); + if (Env.Files.size() >= 1 && Env.Files.size() < 1600 ) + NumCorpuses = 16; + if (Env.Files.size() >= 1600 && Env.Files.size() < 3600 ) + NumCorpuses = 20; + if (Env.Files.size() >= 3600 && Env.Files.size() < 6400 ) + NumCorpuses = 24; + if (Env.Files.size() >= 6400 && Env.Files.size() < 8100 ) + NumCorpuses = 32; + if (Env.Files.size() >= 8100 && Env.Files.size() < 12000 ) + NumCorpuses = 36; + if (Env.Files.size() >= 12000 && Env.Files.size() < 16000 ) + NumCorpuses = 40; + if (Env.Files.size() >= 16000 && Env.Files.size() < 30000 ) + NumCorpuses = 60; + if(Env.Files.size() >= 30000) + NumCorpuses = 80; + + FuzzQ.Push(Env.CreateNewJob(JobId++, NumCorpuses)); } - for (auto &T : Threads) T.join();