Index: include/llvm/LTO/LTO.h =================================================================== --- include/llvm/LTO/LTO.h +++ include/llvm/LTO/LTO.h @@ -246,19 +246,20 @@ /// This ThinBackend runs the individual backend jobs in-process. ThinBackend createInProcessThinBackend(unsigned ParallelismLevel); +/// Defines a callback called for each module for the distributed ThinLTO +/// backend. The client implements this callback to transmit the information to +/// the distributed build system. +typedef std::function &ModuleToSummariesForIndex)> + WriteIndexFn; + /// This ThinBackend writes individual module indexes to files, instead of /// running the individual backend jobs. This backend is for distributed builds /// where separate processes will invoke the real backends. -/// -/// To find the path to write the index to, the backend checks if the path has a -/// prefix of OldPrefix; if so, it replaces that prefix with NewPrefix. It then -/// appends ".thinlto.bc" and writes the index to that path. If -/// ShouldEmitImportsFiles is true it also writes a list of imported files to a -/// similar path with ".imports" appended instead. -ThinBackend createWriteIndexesThinBackend(std::string OldPrefix, - std::string NewPrefix, - bool ShouldEmitImportsFiles, - std::string LinkedObjectsFile); +ThinBackend createWriteIndexesThinBackend(WriteIndexFn PerModuleCallback); /// This class implements a resolution-based interface to LLVM's LTO /// functionality. It supports regular LTO, parallel LTO code generation and Index: lib/LTO/LTO.cpp =================================================================== --- lib/LTO/LTO.cpp +++ lib/LTO/LTO.cpp @@ -523,91 +523,44 @@ } class WriteIndexesThinBackend : public ThinBackendProc { - std::string OldPrefix, NewPrefix; - bool ShouldEmitImportsFiles; - - std::string LinkedObjectsFileName; - std::unique_ptr LinkedObjectsFile; + WriteIndexFn PerModuleCallback; public: - WriteIndexesThinBackend(Config &Conf, ModuleSummaryIndex &CombinedIndex, - StringMap &ModuleToDefinedGVSummaries, - std::string OldPrefix, std::string NewPrefix, - bool ShouldEmitImportsFiles, - std::string LinkedObjectsFileName) + WriteIndexesThinBackend( + Config &Conf, ModuleSummaryIndex &CombinedIndex, + StringMap &ModuleToDefinedGVSummaries, // FIXME const + WriteIndexFn PerModuleCallback) : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries), - OldPrefix(OldPrefix), NewPrefix(NewPrefix), - ShouldEmitImportsFiles(ShouldEmitImportsFiles), - LinkedObjectsFileName(LinkedObjectsFileName) {} - - /// Given the original \p Path to an output file, replace any path - /// prefix matching \p OldPrefix with \p NewPrefix. Also, create the - /// resulting directory if it does not yet exist. - std::string getThinLTOOutputFile(const std::string &Path, - const std::string &OldPrefix, - const std::string &NewPrefix) { - if (OldPrefix.empty() && NewPrefix.empty()) - return Path; - SmallString<128> NewPath(Path); - llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix); - StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str()); - if (!ParentPath.empty()) { - // Make sure the new directory exists, creating it if necessary. - if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath)) - llvm::errs() << "warning: could not create directory '" << ParentPath - << "': " << EC.message() << '\n'; - } - return NewPath.str(); - } + PerModuleCallback(std::move(PerModuleCallback)) {} Error start(unsigned Task, MemoryBufferRef MBRef, const FunctionImporter::ImportMapTy &ImportList, MapVector &ModuleMap) override { StringRef ModulePath = MBRef.getBufferIdentifier(); - std::string NewModulePath = - getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix); - - std::error_code EC; - if (!LinkedObjectsFileName.empty()) { - if (!LinkedObjectsFile) { - LinkedObjectsFile = llvm::make_unique( - LinkedObjectsFileName, EC, sys::fs::OpenFlags::F_None); - if (EC) - return errorCodeToError(EC); - } - *LinkedObjectsFile << NewModulePath << '\n'; - } - std::map ModuleToSummariesForIndex; gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries, ImportList, ModuleToSummariesForIndex); - raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC, - sys::fs::OpenFlags::F_None); - if (EC) - return errorCodeToError(EC); - WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex); - - if (ShouldEmitImportsFiles) - return errorCodeToError( - EmitImportsFiles(ModulePath, NewModulePath + ".imports", ImportList)); + PerModuleCallback(Task, ModulePath, CombinedIndex, ImportList, + ModuleToSummariesForIndex); return Error(); } Error wait() override { return Error(); } }; -ThinBackend lto::createWriteIndexesThinBackend(std::string OldPrefix, - std::string NewPrefix, - bool ShouldEmitImportsFiles, - std::string LinkedObjectsFile) { - return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex, - StringMap &ModuleToDefinedGVSummaries, - AddOutputFn AddOutput) { - return llvm::make_unique( - Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix, NewPrefix, - ShouldEmitImportsFiles, LinkedObjectsFile); - }; +ThinBackend lto::createWriteIndexesThinBackend(WriteIndexFn PerModuleCallback) { + return std::bind( + [](WriteIndexFn PerModuleCallback, Config &Conf, + ModuleSummaryIndex &CombinedIndex, + StringMap &ModuleToDefinedGVSummaries, + AddOutputFn AddOutput) -> std::unique_ptr { + return llvm::make_unique( + Conf, CombinedIndex, ModuleToDefinedGVSummaries, + std::move(PerModuleCallback)); + }, + std::move(PerModuleCallback), std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3, std::placeholders::_4); } Error LTO::runThinLTO(AddOutputFn AddOutput) { Index: tools/gold/gold-plugin.cpp =================================================================== --- tools/gold/gold-plugin.cpp +++ tools/gold/gold-plugin.cpp @@ -686,6 +686,26 @@ }; } +/// Given the original \p Path to an output file, replace any path +/// prefix matching \p OldPrefix with \p NewPrefix. Also, create the +/// resulting directory if it does not yet exist. +static std::string getThinLTOOutputFile(const std::string &Path, + const std::string &OldPrefix, + const std::string &NewPrefix) { + if (OldPrefix.empty() && NewPrefix.empty()) + return Path; + SmallString<128> NewPath(Path); + llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix); + StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str()); + if (!ParentPath.empty()) { + // Make sure the new directory exists, creating it if necessary. + if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath)) + llvm::errs() << "warning: could not create directory '" << ParentPath + << "': " << EC.message() << '\n'; + } + return NewPath.str(); +} + static std::unique_ptr createLTO() { Config Conf; ThinBackend Backend; @@ -712,9 +732,38 @@ if (options::thinlto_index_only) { std::string OldPrefix, NewPrefix; getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix); - Backend = createWriteIndexesThinBackend( - OldPrefix, NewPrefix, options::thinlto_emit_imports_files, - options::thinlto_linked_objects_file); + + std::unique_ptr LinkedObjectsFile; + if (!options::thinlto_linked_objects_file.empty()) { + std::error_code EC; + LinkedObjectsFile = llvm::make_unique( + options::thinlto_linked_objects_file, EC, sys::fs::OpenFlags::F_None); + check(EC, "Can't open ", options::thinlto_linked_objects_file); + } + + Backend = createWriteIndexesThinBackend([&]( + unsigned Task, StringRef ModulePath, + const ModuleSummaryIndex &CombinedIndex, + const FunctionImporter::ImportMapTy &ImportList, + const std::map + &ModuleToSummariesForIndex) { + std::string NewModulePath = + getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix); + + // Add to the list of emitted objects + if (LinkedObjectsFile) + *LinkedObjectsFile << NewModulePath << '\n'; + raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC, + sys::fs::OpenFlags::F_None); + if (EC) + return errorCodeToError(EC); + WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex); + + if (options::thinlto_emit_imports_files) + check(EmitImportsFiles(ModulePath, NewModulePath + ".imports", + ImportList), + "Can't emit imports to " + options::thinlto_emit_imports_files); + }); } Conf.OverrideTriple = options::triple; Index: tools/llvm-lto2/llvm-lto2.cpp =================================================================== --- tools/llvm-lto2/llvm-lto2.cpp +++ tools/llvm-lto2/llvm-lto2.cpp @@ -16,6 +16,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Bitcode/ReaderWriter.h" #include "llvm/LTO/LTO.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/TargetSelect.h" @@ -146,9 +147,25 @@ ThinBackend Backend; if (ThinLTODistributedIndexes) - Backend = createWriteIndexesThinBackend("", "", true, ""); - else + Backend = createWriteIndexesThinBackend( + [&](unsigned Task, StringRef ModuleIdentifier, + const ModuleSummaryIndex &CombinedIndex, + const FunctionImporter::ImportMapTy &ImportList, + const std::map + &ModuleToSummariesForIndex) { + std::error_code EC; + auto IndexPath = (ModuleIdentifier + ".thinlto.bc").str(); + raw_fd_ostream OS(IndexPath, EC, sys::fs::OpenFlags::F_None); + check(EC, "Creating " + IndexPath); + WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex); + + auto ImportsPath = (ModuleIdentifier + ".imports").str(); + EmitImportsFiles(ModuleIdentifier, ImportsPath, ImportList); + check(EC, "Emitting Imports list to " + ImportsPath); + }); + else { Backend = createInProcessThinBackend(Threads); + } LTO Lto(std::move(Conf), std::move(Backend)); bool HasErrors = false;