Index: lld/ELF/Config.h =================================================================== --- lld/ELF/Config.h +++ lld/ELF/Config.h @@ -94,6 +94,8 @@ llvm::StringRef SoName; llvm::StringRef Sysroot; llvm::StringRef ThinLTOCacheDir; + llvm::StringRef ThinLTOLinkedObjectsFile; + llvm::StringRef ThinLTOPrefixReplace; std::string Rpath; std::vector VersionDefinitions; std::vector AuxiliaryList; @@ -156,6 +158,7 @@ bool SysvHash = false; bool Target1Rel; bool Trace; + bool ThinLTOIndexOnly; bool UndefinedVersion; bool WarnBackrefs; bool WarnCommon; Index: lld/ELF/Driver.cpp =================================================================== --- lld/ELF/Driver.cpp +++ lld/ELF/Driver.cpp @@ -795,7 +795,18 @@ Config->LTODebugPassManager = true; else if (S.startswith("sample-profile=")) Config->LTOSampleProfile = S.substr(strlen("sample-profile=")); - else if (!S.startswith("/") && !S.startswith("-fresolution=") && + else if (S == "thinlto-index-only") + Config->ThinLTOIndexOnly = true; + else if (S.startswith("thinlto-index-only=")) { + Config->ThinLTOIndexOnly = true; + Config->ThinLTOLinkedObjectsFile = + S.substr(strlen("thinlto-index-only=")); + } else if (S.startswith("thinlto-prefix-replace=")) { + Config->ThinLTOPrefixReplace = + S.substr(strlen("thinlto-prefix-replace=")); + if (Config->ThinLTOPrefixReplace.find(';') == std::string::npos) + fatal("thinlto-prefix-replace expects 'old;new' format"); + } else if (!S.startswith("/") && !S.startswith("-fresolution=") && !S.startswith("-pass-through=") && !S.startswith("thinlto")) parseClangOption(S, Arg->getSpelling()); } @@ -1001,19 +1012,24 @@ ++InputFile::NextGroupId; break; case OPT_start_lib: - if (InLib) - error("nested --start-lib"); - if (InputFile::IsInGroup) - error("may not nest --start-lib in --start-group"); - InLib = true; - InputFile::IsInGroup = true; + // ThinLTO needs to generate object files for all bitcode files + if (!Config->ThinLTOIndexOnly) { + if (InLib) + error("nested --start-lib"); + if (InputFile::IsInGroup) + error("may not nest --start-lib in --start-group"); + InLib = true; + InputFile::IsInGroup = true; + } break; case OPT_end_lib: - if (!InLib) - error("stray --end-lib"); - InLib = false; - InputFile::IsInGroup = false; - ++InputFile::NextGroupId; + if (!Config->ThinLTOIndexOnly) { + if (!InLib) + error("stray --end-lib"); + InLib = false; + InputFile::IsInGroup = false; + ++InputFile::NextGroupId; + } break; } } Index: lld/ELF/InputFiles.cpp =================================================================== --- lld/ELF/InputFiles.cpp +++ lld/ELF/InputFiles.cpp @@ -1024,9 +1024,10 @@ // this causes a collision which result in only one of the objects being // taken into consideration at LTO time (which very likely causes undefined // symbols later in the link stage). - MemoryBufferRef MBRef(MB.getBuffer(), - Saver.save(ArchiveName + MB.getBufferIdentifier() + - utostr(OffsetInArchive))); + MemoryBufferRef MBRef( + MB.getBuffer(), + Saver.save(ArchiveName + MB.getBufferIdentifier() + + (ArchiveName.empty() ? "" : utostr(OffsetInArchive)))); Obj = CHECK(lto::InputFile::create(MBRef), this); Triple T(Obj->getTargetTriple()); Index: lld/ELF/LTO.h =================================================================== --- lld/ELF/LTO.h +++ lld/ELF/LTO.h @@ -24,6 +24,7 @@ #include "lld/Common/LLVM.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallString.h" +#include "llvm/Support/raw_ostream.h" #include #include @@ -52,6 +53,8 @@ std::vector> Buff; std::vector> Files; llvm::DenseSet UsedStartStop; + llvm::StringMap ObjectToIndexFileState; + std::unique_ptr LinkedObjects; }; } // namespace elf } // namespace lld Index: lld/ELF/LTO.cpp =================================================================== --- lld/ELF/LTO.cpp +++ lld/ELF/LTO.cpp @@ -20,6 +20,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/LTO/Caching.h" #include "llvm/LTO/Config.h" @@ -29,7 +31,6 @@ #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/raw_ostream.h" #include #include #include @@ -66,7 +67,38 @@ [&](ErrorInfoBase &EIB) { error(EIB.message()); }); } -static std::unique_ptr createLTO() { +// Creates and returns output stream with a list of object files for final +// linking of distributed ThinLTO. +static std::unique_ptr CreateLinkedObjectsFile() { + if (Config->ThinLTOLinkedObjectsFile.empty()) return nullptr; + assert(Config->ThinLTOIndexOnly); + std::error_code EC; + auto LinkedObjectsFile = llvm::make_unique( + Config->ThinLTOLinkedObjectsFile, EC, sys::fs::OpenFlags::F_None); + if (EC) + error("cannot create " + Config->ThinLTOLinkedObjectsFile + ": " + + EC.message()); + return LinkedObjectsFile; +} + +// Parse the thinlto_prefix_replace option into the \p OldPrefix and +// \p NewPrefix strings, if it was specified. +static void getThinLTOOldAndNewPrefix(std::string &OldPrefix, + std::string &NewPrefix) { + assert(Config->ThinLTOPrefixReplace.empty() || + Config->ThinLTOPrefixReplace.find(";") != StringRef::npos); + StringRef PrefixReplace = Config->ThinLTOPrefixReplace; + std::pair Split = PrefixReplace.split(";"); + OldPrefix = Split.first.str(); + NewPrefix = Split.second.str(); +} + +// Creates instance of LTO. +// OnIndexWrite is callback to let caller know when LTO writes index files. +// LinkedObjectsFile is an output stream to write the list of object files for +// the final ThinLTO linking. Can be nullptr. +static std::unique_ptr createLTO(llvm::lto::IndexWriteCallback OnIndexWrite, + raw_fd_ostream *LinkedObjectsFile) { lto::Config Conf; // LLD supports the new relocations. @@ -105,6 +137,13 @@ if (Config->ThinLTOJobs != -1u) Backend = lto::createInProcessThinBackend(Config->ThinLTOJobs); + if (Config->ThinLTOIndexOnly) { + std::string OldPrefix, NewPrefix; + getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix); + Backend = llvm::lto::createWriteIndexesThinBackend( + OldPrefix, NewPrefix, true, LinkedObjectsFile, OnIndexWrite); + } + Conf.SampleProfile = Config->LTOSampleProfile; Conf.UseNewPM = Config->LTONewPassManager; Conf.DebugPassManager = Config->LTODebugPassManager; @@ -113,7 +152,13 @@ Config->LTOPartitions); } -BitcodeCompiler::BitcodeCompiler() : LTOObj(createLTO()) { +BitcodeCompiler::BitcodeCompiler() { + if (Config->ThinLTOIndexOnly) LinkedObjects = CreateLinkedObjectsFile(); + LTOObj = createLTO( + [&](const std::string &Identifier) { + ObjectToIndexFileState[Identifier] = true; + }, + LinkedObjects.get()); for (Symbol *Sym : Symtab->getSymbols()) { StringRef Name = Sym->getName(); for (StringRef Prefix : {"__start_", "__stop_"}) @@ -222,6 +267,8 @@ Ret.push_back(Obj); } + if (Config->ThinLTOIndexOnly) exit(0); + for (std::unique_ptr &File : Files) if (File) Ret.push_back(createObjectFile(*File)); Index: lld/ELF/Options.td =================================================================== --- lld/ELF/Options.td +++ lld/ELF/Options.td @@ -425,6 +425,10 @@ defm thinlto_cache_policy: Eq<"thinlto-cache-policy">, HelpText<"Pruning policy for the ThinLTO cache">; def thinlto_jobs: J<"thinlto-jobs=">, HelpText<"Number of ThinLTO jobs">; +def thinlto_index_only: J<"thinlto-index-only=">, + HelpText<"Write individual backend index files for LTO ">; +def thinlto_prefix_replace : F<"thinlto-prefix-replace">, + HelpText<"Controls location of distributed backend files">; // Ignore LTO plugin-related options. // clang -flto passes -plugin and -plugin-opt to the linker. This is required Index: lld/test/ELF/lto/thinlto.ll =================================================================== --- lld/test/ELF/lto/thinlto.ll +++ lld/test/ELF/lto/thinlto.ll @@ -1,8 +1,24 @@ ; REQUIRES: x86 + +; First ensure that the ThinLTO handling in lld handles +; bitcode without summary sections gracefully. +; RUN: llvm-as %s -o %t.o +; RUN: llvm-as %p/Inputs/thinlto.ll -o %t2.o +; RUN: ld.lld -m elf_x86_64 --plugin-opt=thinlto-index-only -shared %t.o %t2.o -o %t3 +; RUN: not test -e %t3 +; RUN: ld.lld -m elf_x86_64 -shared %t.o %t2.o -o %t4 +; RUN: llvm-nm %t4 | FileCheck %s --check-prefix=NM + ; Basic ThinLTO tests. ; RUN: opt -module-summary %s -o %t.o ; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.o +; Ensure lld generates an index and not a binary if requested. +; RUN: ld.lld -m elf_x86_64 --plugin-opt=thinlto-index-only -shared %t.o %t2.o -o %t3 +; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND1 +; RUN: llvm-bcanalyzer -dump %t2.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND2 +; RUN: not test -e %t3 + ; First force single-threaded mode ; RUN: rm -f %t.lto.o %t1.lto.o ; RUN: ld.lld -save-temps --thinlto-jobs=1 -shared %t.o %t2.o -o %t @@ -15,16 +31,43 @@ ; RUN: llvm-nm %t21.lto.o | FileCheck %s --check-prefix=NM1 ; RUN: llvm-nm %t22.lto.o | FileCheck %s --check-prefix=NM2 -; NM1: T f -; NM1-NOT: U g - -; NM2: T g - ; Then check without --thinlto-jobs (which currently default to hardware_concurrency) ; We just check that we don't crash or fail (as it's not sure which tests are ; stable on the final output file itself. ; RUN: ld.lld -shared %t.o %t2.o -o %t2 +; NM: T f +; NM1: T f +; NM1-NOT: U g +; NM2: T g + +; The backend index for this module contains summaries from itself and +; Inputs/thinlto.ll, as it imports from the latter. +; BACKEND1: