Index: compiler-rt/lib/fuzzer/FuzzerDriver.cpp =================================================================== --- compiler-rt/lib/fuzzer/FuzzerDriver.cpp +++ compiler-rt/lib/fuzzer/FuzzerDriver.cpp @@ -867,7 +867,8 @@ } if (Flags.fork) - FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); + FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork, + Flags.NumCorpuses); if (Flags.merge) Merge(F, Options, Args, *Inputs, Flags.merge_control_file); Index: compiler-rt/lib/fuzzer/FuzzerFlags.def =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFlags.def +++ compiler-rt/lib/fuzzer/FuzzerFlags.def @@ -56,6 +56,7 @@ FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total " "time in seconds to run the fuzzer.") FUZZER_FLAG_INT(help, 0, "Print help.") +FUZZER_FLAG_INT(NumCorpuses, 1, "FOR fork mode. Divide the main corpus into N parts according to size.") FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens " "in a subprocess") FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode") Index: compiler-rt/lib/fuzzer/FuzzerFork.h =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFork.h +++ compiler-rt/lib/fuzzer/FuzzerFork.h @@ -18,7 +18,8 @@ namespace fuzzer { void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, const Vector &Args, - const Vector &CorpusDirs, int NumJobs); + const Vector &CorpusDirs, int NumJobs, + int NumCorpuses); } // namespace fuzzer #endif // LLVM_FUZZER_FORK_H Index: compiler-rt/lib/fuzzer/FuzzerFork.cpp =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFork.cpp +++ compiler-rt/lib/fuzzer/FuzzerFork.cpp @@ -95,6 +95,8 @@ Set Features, Cov; Set FilesWithDFT; Vector Files; + // Declare a variable to store the seed size. + Vector FilesSizes; Random *Rand; std::chrono::system_clock::time_point ProcessStartTime; int Verbosity = 0; @@ -114,7 +116,7 @@ .count(); } - FuzzJob *CreateNewJob(size_t JobId) { + FuzzJob *CreateNewJob(size_t JobId, int NumCorpuses) { Command Cmd(Args); Cmd.removeFlag("fork"); Cmd.removeFlag("runs"); @@ -135,11 +137,28 @@ std::string Seeds; if (size_t CorpusSubsetSize = std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) { + size_t AverageSize = Files.size() / NumCorpuses + 1; auto Time1 = std::chrono::system_clock::now(); + size_t StartIndex = ((JobId - 1) % NumCorpuses) * AverageSize; + // At this time, the seeds in the File variable are sorted according to + // the seed size, so by generating a uniformly distributed random + // number, the seeds are selected in a certain group for (size_t i = 0; i < CorpusSubsetSize; i++) { - auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; - Seeds += (Seeds.empty() ? "" : ",") + SF; - CollectDFT(SF); + std::random_device rd; + std::mt19937 randomseed(rd()); + std::uniform_int_distribution<> rand(0,AverageSize); + size_t j = rand(randomseed); + size_t m = j + StartIndex; + if (m < Files.size()) { + auto &SF = Files[m]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } + else { + auto &SF = Files[rand(randomseed)]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } } auto Time2 = std::chrono::system_clock::now(); auto DftTimeInSeconds = duration_cast(Time2 - Time1).count(); @@ -179,7 +198,7 @@ return Job; } - void RunOneMergeJob(FuzzJob *Job) { + void RunOneMergeJob(FuzzJob *Job, int NumCorpuses) { auto Stats = ParseFinalStatsFromLog(Job->LogPath); NumRuns += Stats.number_of_executed_units; @@ -219,8 +238,18 @@ auto U = FileToVector(Path); auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); WriteToFile(U, NewPath); - Files.push_back(NewPath); + // Insert the queue according to the size of the seed. + long usz = U.size(); + auto idx = std::upper_bound(FilesSizes.begin(), FilesSizes.end(), usz) - + FilesSizes.begin(); + FilesSizes.insert(FilesSizes.begin() + idx, usz); + Files.insert(Files.begin() + idx, NewPath); } + //printf("\n"); + //for (int i = 0; i < FilesSizes.size(); i++){ + // printf("size is %lu --",FilesSizes[i]); + //} + //printf("\n"); Features.insert(NewFeatures.begin(), NewFeatures.end()); Cov.insert(NewCov.begin(), NewCov.end()); for (auto Idx : NewCov) @@ -284,7 +313,7 @@ // This is just a skeleton of an experimental -fork=1 feature. void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, const Vector &Args, - const Vector &CorpusDirs, int NumJobs) { + const Vector &CorpusDirs, int NumJobs, int NumCorpuses) { Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs); GlobalEnv Env; @@ -294,6 +323,8 @@ Env.Verbosity = Options.Verbosity; Env.ProcessStartTime = std::chrono::system_clock::now(); Env.DataFlowBinary = Options.CollectDataFlow; + size_t JobExected = 0; + size_t MergeCycle = 80; Vector SeedFiles; for (auto &Dir : CorpusDirs) @@ -312,8 +343,10 @@ Env.MainCorpusDir = CorpusDirs[0]; if (Options.KeepSeed) { - for (auto &File : SeedFiles) + for (auto &File : SeedFiles){ Env.Files.push_back(File.File); + Env.FilesSizes.push_back(File.Size); + } } else { auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); Set NewFeatures, NewCov; @@ -323,9 +356,15 @@ Env.Cov.insert(NewFeatures.begin(), NewFeatures.end()); RemoveFile(CFPath); } + + for (auto &path : Env.Files) { + Env.FilesSizes.push_back(FileSize(path)); + } + Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, Env.Files.size(), Env.TempDir.c_str()); + int ExitCode = 0; JobQueue FuzzQ, MergeQ; @@ -341,7 +380,7 @@ Vector Threads; for (int t = 0; t < NumJobs; t++) { Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ)); - FuzzQ.Push(Env.CreateNewJob(JobId++)); + FuzzQ.Push(Env.CreateNewJob(JobId++, NumCorpuses)); } while (true) { @@ -356,7 +395,44 @@ } Fuzzer::MaybeExitGracefully(); - Env.RunOneMergeJob(Job.get()); + Env.RunOneMergeJob(Job.get(), NumCorpuses); + // merge the corpus . + JobExected ++; + if(JobExected >= MergeCycle){ + auto t1 = std::chrono::system_clock::now(); + Vector CurrentSeedFiles; + for (auto &Dir : CorpusDirs) + GetSizedFilesFromDir(Dir, &CurrentSeedFiles); + std::sort(CurrentSeedFiles.begin(), CurrentSeedFiles.end()); + if (CorpusDirs.empty()) + MkDir(Env.MainCorpusDir = DirPlusFile(Env.TempDir, "C")); + else + Env.MainCorpusDir = CorpusDirs[0]; + + auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); + Set TmpNewFeatures, TmpNewCov; + Set TmpFeatures, TmpCov; + Env.Files.clear(); + Env.FilesSizes.clear(); + CrashResistantMerge(Env.Args, {}, CurrentSeedFiles, &Env.Files, TmpFeatures, + &TmpNewFeatures, TmpCov, &TmpNewCov, CFPath, false); + for (auto &path : Env.Files) { + Env.FilesSizes.push_back(FileSize(path)); + } + //printf("\n"); + //for (int i = 0; i < Env.FilesSizes.size(); i++){ + // printf("Env.size is %lu --", Env.FilesSizes[i]); + //} + //printf("\n"); + RemoveFile(CFPath); + JobExected = 0; + MergeCycle += 20; + auto t2 = std::chrono::system_clock::now(); + auto TimeCost = duration_cast> (t2 - t1); + printf("Merge Successfully. Cost Times %f , SeedFiles is %zu Files is %zu \n ",TimeCost,CurrentSeedFiles.size(),Env.Files.size()); + } + + // Continue if our crash is one of the ignorred ones. if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) @@ -399,9 +475,27 @@ break; } - FuzzQ.Push(Env.CreateNewJob(JobId++)); + // Since the number of corpus seeds will gradually increase, in order to + // control the number in each group to be about three times the number of + // seeds selected each time, the number of groups is dynamically adjusted. + if (Env.Files.size() >= 1 && Env.Files.size() < 1600) + NumCorpuses = 16; + if (Env.Files.size() >= 1600 && Env.Files.size() < 3600) + NumCorpuses = 20; + if (Env.Files.size() >= 3600 && Env.Files.size() < 6400) + NumCorpuses = 24; + if (Env.Files.size() >= 6400 && Env.Files.size() < 8100) + NumCorpuses = 32; + if (Env.Files.size() >= 8100 && Env.Files.size() < 12000) + NumCorpuses = 36; + if (Env.Files.size() >= 12000 && Env.Files.size() < 16000) + NumCorpuses = 40; + if (Env.Files.size() >= 16000 && Env.Files.size() < 30000) + NumCorpuses = 60; + if(Env.Files.size() >= 30000) + NumCorpuses = 80; + FuzzQ.Push(Env.CreateNewJob(JobId++, NumCorpuses)); } - for (auto &T : Threads) T.join();