Index: compiler-rt/lib/fuzzer/FuzzerDriver.cpp =================================================================== --- compiler-rt/lib/fuzzer/FuzzerDriver.cpp +++ compiler-rt/lib/fuzzer/FuzzerDriver.cpp @@ -870 +870,2 @@ - FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); + FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork, + Flags.group); Index: compiler-rt/lib/fuzzer/FuzzerFlags.def =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFlags.def +++ compiler-rt/lib/fuzzer/FuzzerFlags.def @@ -58,0 +59,2 @@ +FUZZER_FLAG_INT(group, 1, "FOR fork mode. Divide the main corpus into N groups " + "according to size.") Index: compiler-rt/lib/fuzzer/FuzzerFork.h =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFork.h +++ compiler-rt/lib/fuzzer/FuzzerFork.h @@ -21 +21,2 @@ - const Vector &CorpusDirs, int NumJobs); + const Vector &CorpusDirs, int NumJobs, + int Group); Index: compiler-rt/lib/fuzzer/FuzzerFork.cpp =================================================================== --- compiler-rt/lib/fuzzer/FuzzerFork.cpp +++ compiler-rt/lib/fuzzer/FuzzerFork.cpp @@ -72 +72 @@ - size_t JobId; + size_t JobId; @@ -74 +74 @@ - int DftTimeInSeconds = 0; + int DftTimeInSeconds = 0; @@ -97,0 +98,2 @@ + // Declare a variable to store the seed size. + Vector FilesSizes; @@ -117 +119 @@ - FuzzJob *CreateNewJob(size_t JobId) { + FuzzJob *CreateNewJob(size_t JobId, int NumCorpuses, int Group) { @@ -135,0 +138 @@ + auto Time1 = std::chrono::system_clock::now(); @@ -137,11 +140,31 @@ - std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) { - auto Time1 = std::chrono::system_clock::now(); - for (size_t i = 0; i < CorpusSubsetSize; i++) { - auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; - Seeds += (Seeds.empty() ? "" : ",") + SF; - CollectDFT(SF); - } - auto Time2 = std::chrono::system_clock::now(); - auto DftTimeInSeconds = duration_cast(Time2 - Time1).count(); - assert(DftTimeInSeconds < std::numeric_limits::max()); - Job->DftTimeInSeconds = static_cast(DftTimeInSeconds); + std::min(Files.size(), (size_t)sqrt(Files.size() + 2))){ + if (Group) { // flag: whether to group the corpus. + size_t AverageCorpusSize = Files.size() / NumCorpuses + 1; + if (Files.size() == 0) + AverageCorpusSize = 0; + size_t StartIndex = ((JobId - 1) % NumCorpuses) * AverageCorpusSize; + for (size_t i = 0; i < CorpusSubsetSize; i++) { + std::random_device rd; + std::mt19937 RandomSeed(rd()); + std::uniform_int_distribution<> rand(0,AverageCorpusSize); + size_t RandNum = rand(RandomSeed); + size_t Index = RandNum + StartIndex; + if (Index < Files.size()) { + auto &SF = Files[Index]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } + else { + auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } + } + } + else{ + for (size_t i = 0; i < CorpusSubsetSize; i++) { + auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } + } @@ -148,0 +172,5 @@ + auto Time2 = std::chrono::system_clock::now(); + auto DftTimeInSeconds = duration_cast(Time2 - Time1).count(); + assert(DftTimeInSeconds < std::numeric_limits::max()); + Job->DftTimeInSeconds = static_cast(DftTimeInSeconds); + @@ -182 +210 @@ - void RunOneMergeJob(FuzzJob *Job) { + void RunOneMergeJob(FuzzJob *Job, int Group) { @@ -222 +250,10 @@ - Files.push_back(NewPath); + if (Group) {// Insert the queue according to the size of the seed. + long usz = U.size(); + auto idx = std::upper_bound(FilesSizes.begin(), FilesSizes.end(), usz) - + FilesSizes.begin(); + FilesSizes.insert(FilesSizes.begin() + idx, usz); + Files.insert(Files.begin() + idx, NewPath); + } + else { + Files.push_back(NewPath); + } @@ -287 +324,2 @@ - const Vector &CorpusDirs, int NumJobs) { + const Vector &CorpusDirs, int NumJobs, + int Group) { @@ -315 +353 @@ - for (auto &File : SeedFiles) + for (auto &File : SeedFiles){ @@ -316,0 +355,2 @@ + Env.FilesSizes.push_back(File.Size); + } @@ -325,0 +366,7 @@ + + if (Group) { + for (auto &path : Env.Files) { + Env.FilesSizes.push_back(FileSize(path)); + } + } + @@ -328,0 +376 @@ + @@ -339 +387,4 @@ - + + size_t MergeCycle = 20; + size_t JobExecuted = 0; + int NumCorpuses = 8; @@ -344 +395 @@ - FuzzQ.Push(Env.CreateNewJob(JobId++)); + FuzzQ.Push(Env.CreateNewJob(JobId++, NumCorpuses, Group)); @@ -359 +410,29 @@ - Env.RunOneMergeJob(Job.get()); + Env.RunOneMergeJob(Job.get(), Group); + // merge the corpus . + JobExecuted ++; + if (Group) { + if(JobExecuted >= MergeCycle){ + Vector CurrentSeedFiles; + for (auto &Dir : CorpusDirs) + GetSizedFilesFromDir(Dir, &CurrentSeedFiles); + std::sort(CurrentSeedFiles.begin(), CurrentSeedFiles.end()); + if (CorpusDirs.empty()) + MkDir(Env.MainCorpusDir = DirPlusFile(Env.TempDir, "C")); + else + Env.MainCorpusDir = CorpusDirs[0]; + + auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); + Set TmpNewFeatures, TmpNewCov; + Set TmpFeatures, TmpCov; + Env.Files.clear(); + Env.FilesSizes.clear(); + CrashResistantMerge(Env.Args, {}, CurrentSeedFiles, &Env.Files, TmpFeatures, + &TmpNewFeatures, TmpCov, &TmpNewCov, CFPath, false); + for (auto &path : Env.Files) { + Env.FilesSizes.push_back(FileSize(path)); + } + RemoveFile(CFPath); + JobExecuted = 0; + MergeCycle += 5; + } + } @@ -402 +481,20 @@ - FuzzQ.Push(Env.CreateNewJob(JobId++)); + // Since the number of corpus seeds will gradually increase, in order to + // control the number in each group to be about three times the number of + // seeds selected each time, the number of groups is dynamically adjusted. + if (Env.Files.size() >= 1 && Env.Files.size() < 1600) + NumCorpuses = 16; + if (Env.Files.size() >= 1600 && Env.Files.size() < 3600) + NumCorpuses = 20; + if (Env.Files.size() >= 3600 && Env.Files.size() < 6400) + NumCorpuses = 24; + if (Env.Files.size() >= 6400 && Env.Files.size() < 8100) + NumCorpuses = 32; + if (Env.Files.size() >= 8100 && Env.Files.size() < 12000) + NumCorpuses = 36; + if (Env.Files.size() >= 12000 && Env.Files.size() < 16000) + NumCorpuses = 40; + if (Env.Files.size() >= 16000 && Env.Files.size() < 30000) + NumCorpuses = 60; + if(Env.Files.size() >= 30000) + NumCorpuses = 80; + FuzzQ.Push(Env.CreateNewJob(JobId++, NumCorpuses, Group)); @@ -404 +501,0 @@ -