Index: lld/ELF/Config.h =================================================================== --- lld/ELF/Config.h +++ lld/ELF/Config.h @@ -94,6 +94,8 @@ llvm::StringRef SoName; llvm::StringRef Sysroot; llvm::StringRef ThinLTOCacheDir; + llvm::StringRef ThinLTOIndexOnlyObjectsFile; + llvm::StringRef ThinLTOPrefixReplace; std::string Rpath; std::vector VersionDefinitions; std::vector AuxiliaryList; @@ -156,6 +158,7 @@ bool SysvHash = false; bool Target1Rel; bool Trace; + bool ThinLTOIndexOnly; bool UndefinedVersion; bool WarnBackrefs; bool WarnCommon; Index: lld/ELF/Driver.cpp =================================================================== --- lld/ELF/Driver.cpp +++ lld/ELF/Driver.cpp @@ -777,27 +777,37 @@ // Parse LTO plugin-related options for compatibility with gold. for (auto *Arg : Args.filtered(OPT_plugin_opt)) { StringRef S = Arg->getValue(); - if (S == "disable-verify") + if (S == "disable-verify") { Config->DisableVerify = true; - else if (S == "save-temps") + } else if (S == "save-temps") { Config->SaveTemps = true; - else if (S.startswith("O")) + } else if (S.startswith("O")) { Config->LTOO = parseInt(S.substr(1), Arg); - else if (S.startswith("lto-partitions=")) + } else if (S.startswith("lto-partitions=")) { Config->LTOPartitions = parseInt(S.substr(15), Arg); - else if (S.startswith("jobs=")) + } else if (S.startswith("jobs=")) { Config->ThinLTOJobs = parseInt(S.substr(5), Arg); - else if (S.startswith("mcpu=")) + } else if (S.startswith("mcpu=")) { parseClangOption(Saver.save("-" + S), Arg->getSpelling()); - else if (S == "new-pass-manager") + } else if (S == "new-pass-manager") { Config->LTONewPassManager = true; - else if (S == "debug-pass-manager") + } else if (S == "debug-pass-manager") { Config->LTODebugPassManager = true; - else if (S.startswith("sample-profile=")) - Config->LTOSampleProfile = S.substr(strlen("sample-profile=")); - else if (!S.startswith("/") && !S.startswith("-fresolution=") && - !S.startswith("-pass-through=") && !S.startswith("thinlto")) + } else if (S.startswith("sample-profile=")) { + Config->LTOSampleProfile = S.substr(15); + } else if (S == "thinlto-index-only") { + Config->ThinLTOIndexOnly = true; + } else if (S.startswith("thinlto-index-only=")) { + Config->ThinLTOIndexOnly = true; + Config->ThinLTOIndexOnlyObjectsFile = S.substr(19); + } else if (S.startswith("thinlto-prefix-replace=")) { + Config->ThinLTOPrefixReplace = S.substr(23); + if (!Config->ThinLTOPrefixReplace.contains(';')) + error("thinlto-prefix-replace expects 'old;new' format"); + } else if (!S.startswith("/") && !S.startswith("-fresolution=") && + !S.startswith("-pass-through=") && !S.startswith("thinlto")) { parseClangOption(S, Arg->getSpelling()); + } } // Parse -mllvm options. @@ -1001,6 +1011,13 @@ ++InputFile::NextGroupId; break; case OPT_start_lib: + // With ThinLTOIndexOnly option, Thinlto needs to generate object files + // for all bitcode files(even with no symbols). After index files are + // created, we exit from linker and LTO Distributed Backend runs. This + // distributed build system will want to confirm that all expected outputs + // are created based on all of the modules provided to the linker. + if (Config->ThinLTOIndexOnly) + break; if (InLib) error("nested --start-lib"); if (InputFile::IsInGroup) @@ -1009,6 +1026,8 @@ InputFile::IsInGroup = true; break; case OPT_end_lib: + if (Config->ThinLTOIndexOnly) + break; if (!InLib) error("stray --end-lib"); InLib = false; Index: lld/ELF/InputFiles.cpp =================================================================== --- lld/ELF/InputFiles.cpp +++ lld/ELF/InputFiles.cpp @@ -1024,9 +1024,10 @@ // this causes a collision which result in only one of the objects being // taken into consideration at LTO time (which very likely causes undefined // symbols later in the link stage). - MemoryBufferRef MBRef(MB.getBuffer(), - Saver.save(ArchiveName + MB.getBufferIdentifier() + - utostr(OffsetInArchive))); + MemoryBufferRef MBRef( + MB.getBuffer(), + Saver.save(ArchiveName + MB.getBufferIdentifier() + + (ArchiveName.empty() ? "" : utostr(OffsetInArchive)))); Obj = CHECK(lto::InputFile::create(MBRef), this); Triple T(Obj->getTargetTriple()); Index: lld/ELF/LTO.h =================================================================== --- lld/ELF/LTO.h +++ lld/ELF/LTO.h @@ -24,6 +24,7 @@ #include "lld/Common/LLVM.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallString.h" +#include "llvm/Support/raw_ostream.h" #include #include @@ -52,6 +53,7 @@ std::vector> Buff; std::vector> Files; llvm::DenseSet UsedStartStop; + std::unique_ptr LinkedObjects; }; } // namespace elf } // namespace lld Index: lld/ELF/LTO.cpp =================================================================== --- lld/ELF/LTO.cpp +++ lld/ELF/LTO.cpp @@ -20,6 +20,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/LTO/Caching.h" #include "llvm/LTO/Config.h" @@ -29,7 +31,6 @@ #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/raw_ostream.h" #include #include #include @@ -66,7 +67,24 @@ [&](ErrorInfoBase &EIB) { error(EIB.message()); }); } -static std::unique_ptr createLTO() { +// Creates and returns output stream with a list of object files for final +// linking of distributed ThinLTO. +static std::unique_ptr createLinkedObjectsFile() { + if (Config->ThinLTOIndexOnlyObjectsFile.empty()) + return nullptr; + std::error_code EC; + auto LinkedObjectsFile = llvm::make_unique( + Config->ThinLTOIndexOnlyObjectsFile, EC, sys::fs::OpenFlags::F_None); + if (EC) + error("cannot create " + Config->ThinLTOIndexOnlyObjectsFile + ": " + + EC.message()); + return LinkedObjectsFile; +} + +// Creates instance of LTO. +// LinkedObjectsFile is an output stream to write the list of object files for +// the final ThinLTO linking. Can be nullptr. +static std::unique_ptr createLTO(raw_fd_ostream *LinkedObjectsFile) { lto::Config Conf; // LLD supports the new relocations. @@ -105,6 +123,13 @@ if (Config->ThinLTOJobs != -1u) Backend = lto::createInProcessThinBackend(Config->ThinLTOJobs); + if (Config->ThinLTOIndexOnly) { + std::string OldPrefix, NewPrefix; + std::tie(OldPrefix, NewPrefix) = Config->ThinLTOPrefixReplace.split(';'); + Backend = llvm::lto::createWriteIndexesThinBackend( + OldPrefix, NewPrefix, true, LinkedObjectsFile, nullptr); + } + Conf.SampleProfile = Config->LTOSampleProfile; Conf.UseNewPM = Config->LTONewPassManager; Conf.DebugPassManager = Config->LTODebugPassManager; @@ -113,7 +138,10 @@ Config->LTOPartitions); } -BitcodeCompiler::BitcodeCompiler() : LTOObj(createLTO()) { +BitcodeCompiler::BitcodeCompiler() { + if (Config->ThinLTOIndexOnly) + LinkedObjects = createLinkedObjectsFile(); + LTOObj = createLTO(LinkedObjects.get()); for (Symbol *Sym : Symtab->getSymbols()) { StringRef Name = Sym->getName(); for (StringRef Prefix : {"__start_", "__stop_"}) @@ -222,6 +250,12 @@ Ret.push_back(Obj); } + // ThinLTO with index only option is required to generate only the index + // files. After that, we exit from linker and ThinLTO backend runs in a + // distributed environment. + if (Config->ThinLTOIndexOnly) + exit(0); + for (std::unique_ptr &File : Files) if (File) Ret.push_back(createObjectFile(*File)); Index: lld/ELF/Options.td =================================================================== --- lld/ELF/Options.td +++ lld/ELF/Options.td @@ -425,6 +425,10 @@ defm thinlto_cache_policy: Eq<"thinlto-cache-policy">, HelpText<"Pruning policy for the ThinLTO cache">; def thinlto_jobs: J<"thinlto-jobs=">, HelpText<"Number of ThinLTO jobs">; +def thinlto_index_only: J<"thinlto-index-only=">, + HelpText<"Write individual backend index files for LTO ">; +def thinlto_prefix_replace : F<"thinlto-prefix-replace">, + HelpText<"Controls location of distributed backend files">; // Ignore LTO plugin-related options. // clang -flto passes -plugin and -plugin-opt to the linker. This is required Index: lld/test/ELF/lto/thinlto.ll =================================================================== --- lld/test/ELF/lto/thinlto.ll +++ lld/test/ELF/lto/thinlto.ll @@ -1,8 +1,24 @@ ; REQUIRES: x86 + +; First ensure that the ThinLTO handling in lld handles +; bitcode without summary sections gracefully. +; RUN: llvm-as %s -o %t.o +; RUN: llvm-as %p/Inputs/thinlto.ll -o %t2.o +; RUN: ld.lld -m elf_x86_64 --plugin-opt=thinlto-index-only -shared %t.o %t2.o -o %t3 +; RUN: not test -e %t3 +; RUN: ld.lld -m elf_x86_64 -shared %t.o %t2.o -o %t4 +; RUN: llvm-nm %t4 | FileCheck %s --check-prefix=NM + ; Basic ThinLTO tests. ; RUN: opt -module-summary %s -o %t.o ; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.o +; Ensure lld generates an index and not a binary if requested. +; RUN: ld.lld -m elf_x86_64 --plugin-opt=thinlto-index-only -shared %t.o %t2.o -o %t3 +; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND1 +; RUN: llvm-bcanalyzer -dump %t2.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND2 +; RUN: not test -e %t3 + ; First force single-threaded mode ; RUN: rm -f %t.lto.o %t1.lto.o ; RUN: ld.lld -save-temps --thinlto-jobs=1 -shared %t.o %t2.o -o %t @@ -15,16 +31,43 @@ ; RUN: llvm-nm %t21.lto.o | FileCheck %s --check-prefix=NM1 ; RUN: llvm-nm %t22.lto.o | FileCheck %s --check-prefix=NM2 -; NM1: T f -; NM1-NOT: U g - -; NM2: T g - ; Then check without --thinlto-jobs (which currently default to hardware_concurrency) ; We just check that we don't crash or fail (as it's not sure which tests are ; stable on the final output file itself. ; RUN: ld.lld -shared %t.o %t2.o -o %t2 +; NM: T f +; NM1: T f +; NM1-NOT: U g +; NM2: T g + +; The backend index for this module contains summaries from itself and +; Inputs/thinlto.ll, as it imports from the latter. +; BACKEND1: