Index: bolt/tools/merge-fdata/merge-fdata.cpp =================================================================== --- bolt/tools/merge-fdata/merge-fdata.cpp +++ bolt/tools/merge-fdata/merge-fdata.cpp @@ -20,6 +20,7 @@ #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/Signals.h" +#include "llvm/Support/ThreadPool.h" #include using namespace llvm; @@ -258,15 +259,17 @@ void mergeLegacyProfiles(const SmallVectorImpl &Filenames) { errs() << "Using legacy profile format.\n"; std::optional BoltedCollection; - StringMap Entries; - for (const std::string &Filename : Filenames) { + typedef StringMap ProfileTy; + + auto ParseProfile = [&](const std::string &Filename, auto &Profiles) { + const llvm::thread::id tid = llvm::this_thread::get_id(); + if (isYAML(Filename)) report_error(Filename, "cannot mix YAML and legacy formats"); ErrorOr> MB = MemoryBuffer::getFileOrSTDIN(Filename); if (std::error_code EC = MB.getError()) report_error(Filename, EC); - errs() << "Merging data from " << Filename << "...\n"; StringRef Buf = MB.get()->getBuffer(); // Check if the string "boltedcollection" is in the first line @@ -295,15 +298,31 @@ uint64_t Count; if (Line.substr(Pos + 1, Line.size() - Pos).getAsInteger(10, Count)) report_error(Filename, "Malformed / corrupted profile counter"); - Count += Entries.lookup(Signature); - Entries.insert_or_assign(Signature, Count); + uint64_t NewCount = Profiles[tid].lookup(Signature) + Count; + Profiles[tid].insert_or_assign(Signature, NewCount); + } + }; + + // The final reduction has non-trivial cost, make sure each thread has at + // least 4 tasks. + ThreadPoolStrategy S = optimal_concurrency(Filenames.size() / 4); + ThreadPool Pool(S); + DenseMap ParsedProfiles(Pool.getThreadCount()); + for (const auto &Filename : Filenames) + Pool.async(ParseProfile, std::cref(Filename), std::ref(ParsedProfiles)); + Pool.wait(); + + ProfileTy MergedProfile; + for (const auto &[Thread, Profile] : ParsedProfiles) + for (const auto &[Key, Value] : Profile) { + uint64_t Count = MergedProfile.lookup(Key) + Value; + MergedProfile.insert_or_assign(Key, Count); } - } if (BoltedCollection) output() << "boltedcollection\n"; - for (const auto &Entry : Entries) - output() << Entry.getKey() << " " << Entry.getValue() << "\n"; + for (const auto &[Key, Value] : MergedProfile) + output() << Key << " " << Value << "\n"; errs() << "Profile from " << Filenames.size() << " files merged.\n"; }