diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h --- a/lld/COFF/Config.h +++ b/lld/COFF/Config.h @@ -179,6 +179,9 @@ // Used for /lldmap. std::string MapFile; + // Used for /thinlto-index-only: + llvm::StringRef ThinLTOIndexOnlyArg; + uint64_t ImageBase = -1; uint64_t FileAlign = 512; uint64_t StackReserve = 1024 * 1024; @@ -209,6 +212,8 @@ bool Repro = false; bool SwaprunCD = false; bool SwaprunNet = false; + bool ThinLTOEmitImportsFiles; + bool ThinLTOIndexOnly; }; extern Configuration *Config; diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp --- a/lld/COFF/Driver.cpp +++ b/lld/COFF/Driver.cpp @@ -1441,6 +1441,11 @@ fatal("/manifestinput: requires /manifest:embed"); } + Config->ThinLTOEmitImportsFiles = Args.hasArg(OPT_thinlto_emit_imports_files); + Config->ThinLTOIndexOnly = Args.hasArg(OPT_thinlto_index_only) || + Args.hasArg(OPT_thinlto_index_only_arg); + Config->ThinLTOIndexOnlyArg = + Args.getLastArgValue(OPT_thinlto_index_only_arg); // Handle miscellaneous boolean flags. Config->AllowBind = Args.hasFlag(OPT_allowbind, OPT_allowbind_no, true); Config->AllowIsolation = @@ -1727,8 +1732,18 @@ return; // Do LTO by compiling bitcode input files to a set of native COFF files then - // link those files. + // link those files (unless -thinlto-index-only was given, in which case we + // resolve symbols and write indices, but don't generate native code or link). Symtab->addCombinedLTOObjects(); + + // If -thinlto-index-only is given, we should create only "index + // files" and not object files. Index file creation is already done + // in addCombinedLTOObject, so we are done if that's the case. + if (Config->ThinLTOIndexOnly) + return; + + // If we generated native object files from bitcode files, this resolves + // references to the symbols we use from them. run(); if (Args.hasArg(OPT_include_optional)) { diff --git a/lld/COFF/LTO.h b/lld/COFF/LTO.h --- a/lld/COFF/LTO.h +++ b/lld/COFF/LTO.h @@ -21,7 +21,9 @@ #define LLD_COFF_LTO_H #include "lld/Common/LLVM.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallString.h" +#include "llvm/Support/raw_ostream.h" #include #include @@ -49,6 +51,8 @@ std::unique_ptr LTOObj; std::vector> Buf; std::vector> Files; + std::unique_ptr IndexFile; + llvm::DenseSet ThinIndices; }; } } diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -18,6 +18,7 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" +#include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/LTO/Caching.h" #include "llvm/LTO/Config.h" @@ -41,7 +42,19 @@ using namespace lld; using namespace lld::coff; -static std::unique_ptr createLTO() { +// Creates an empty file to and returns a raw_fd_ostream to write to it. +static std::unique_ptr openFile(StringRef File) { + std::error_code EC; + auto Ret = + llvm::make_unique(File, EC, sys::fs::OpenFlags::F_None); + if (EC) { + error("cannot open " + File + ": " + EC.message()); + return nullptr; + } + return Ret; +} + +static lto::Config createConfig() { lto::Config C; C.Options = initTargetOptionsFromCodeGenFlags(); @@ -67,14 +80,27 @@ if (Config->SaveTemps) checkError(C.addSaveTemps(std::string(Config->OutputFile) + ".", /*UseInputModulePath*/ true)); + return C; +} + +BitcodeCompiler::BitcodeCompiler() { + // Initialize IndexFile. + if (!Config->ThinLTOIndexOnlyArg.empty()) + IndexFile = openFile(Config->ThinLTOIndexOnlyArg); + + // Initialize LTOObj. lto::ThinBackend Backend; - if (Config->ThinLTOJobs != 0) + if (Config->ThinLTOIndexOnly) { + auto OnIndexWrite = [&](StringRef S) { ThinIndices.erase(S); }; + Backend = lto::createWriteIndexesThinBackend( + "", "", Config->ThinLTOEmitImportsFiles, IndexFile.get(), OnIndexWrite); + } else if (Config->ThinLTOJobs != 0) { Backend = lto::createInProcessThinBackend(Config->ThinLTOJobs); - return llvm::make_unique(std::move(C), Backend, - Config->LTOPartitions); -} + } -BitcodeCompiler::BitcodeCompiler() : LTOObj(createLTO()) {} + LTOObj = llvm::make_unique(createConfig(), Backend, + Config->LTOPartitions); +} BitcodeCompiler::~BitcodeCompiler() = default; @@ -86,6 +112,9 @@ std::vector SymBodies = F.getSymbols(); std::vector Resols(SymBodies.size()); + if (Config->ThinLTOIndexOnly) + ThinIndices.insert(Obj.getName()); + // Provide a resolution to the LTO API for each symbol. for (const lto::InputFile::Symbol &ObjSym : Obj.symbols()) { Symbol *Sym = SymBodies[SymNum]; @@ -129,6 +158,23 @@ }, Cache)); + // Emit empty index files for non-indexed files + for (StringRef S : ThinIndices) { + std::string Path(S); + openFile(Path + ".thinlto.bc"); + if (Config->ThinLTOEmitImportsFiles) + openFile(Path + ".imports"); + } + + // ThinLTO with index only option is required to generate only the index + // files. After that, we exit from linker and ThinLTO backend runs in a + // distributed environment. + if (Config->ThinLTOIndexOnly) { + if (IndexFile) + IndexFile->close(); + return {}; + } + if (!Config->LTOCache.empty()) pruneCache(Config->LTOCache, Config->LTOCachePolicy); diff --git a/lld/COFF/Options.td b/lld/COFF/Options.td --- a/lld/COFF/Options.td +++ b/lld/COFF/Options.td @@ -176,6 +176,15 @@ "Base path used to make relative source file path absolute in PDB">; def rsp_quoting : Joined<["--"], "rsp-quoting=">, HelpText<"Quoting style for response files, 'windows' (default) or 'posix'">; +def thinlto_emit_imports_files : + F<"thinlto-emit-imports-files">, + HelpText<"Emit .imports files with -thinlto-index-only">; +def thinlto_index_only : + F<"thinlto-index-only">, + HelpText<"Instead of linking, emit ThinLTO index files">; +def thinlto_index_only_arg : P< + "thinlto-index-only", + "-thinlto-index-only and also write native module names to file">; def dash_dash_version : Flag<["--"], "version">, HelpText<"Print version information">; defm threads: B<"threads", diff --git a/lld/test/COFF/Inputs/thinlto-empty.ll b/lld/test/COFF/Inputs/thinlto-empty.ll new file mode 100644 --- /dev/null +++ b/lld/test/COFF/Inputs/thinlto-empty.ll @@ -0,0 +1,2 @@ +target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc19.0.24215" diff --git a/lld/test/COFF/Inputs/thinlto.ll b/lld/test/COFF/Inputs/thinlto.ll new file mode 100644 --- /dev/null +++ b/lld/test/COFF/Inputs/thinlto.ll @@ -0,0 +1,6 @@ +target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc19.0.24215" + +define void @g() { + ret void +} diff --git a/lld/test/COFF/thinlto-emit-imports.ll b/lld/test/COFF/thinlto-emit-imports.ll new file mode 100644 --- /dev/null +++ b/lld/test/COFF/thinlto-emit-imports.ll @@ -0,0 +1,59 @@ +; REQUIRES: x86 + +; Generate summary sections and test lld handling. +; RUN: opt -module-summary %s -o %t1.obj +; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.obj + +; Include a file with an empty module summary index, to ensure that the expected +; output files are created regardless, for a distributed build system. +; RUN: opt -module-summary %p/Inputs/thinlto-empty.ll -o %t3.obj + +; Ensure lld generates imports files if requested for distributed backends. +; RUN: rm -f %t3.obj.imports %t3.obj.thinlto.bc +; RUN: lld-link -entry:main -thinlto-index-only \ +; RUN: -thinlto-emit-imports-files %t1.obj %t2.obj %t3.obj -out:%t4.exe + +; The imports file for this module contains the bitcode file for +; Inputs/thinlto.ll +; RUN: cat %t1.obj.imports | count 1 +; RUN: cat %t1.obj.imports | FileCheck %s --check-prefix=IMPORTS1 +; IMPORTS1: thinlto-emit-imports.ll.tmp2.obj + +; The imports file for Input/thinlto.ll is empty as it does not import anything. +; RUN: cat %t2.obj.imports | count 0 + +; The imports file for Input/thinlto_empty.ll is empty but should exist. +; RUN: cat %t3.obj.imports | count 0 + +; The index file should be created even for the input with an empty summary. +; RUN: ls %t3.obj.thinlto.bc + +; Ensure lld generates error if unable to write to imports file. +; RUN: rm -f %t3.obj.imports +; RUN: touch %t3.obj.imports +; RUN: chmod 400 %t3.obj.imports +; RUN: not lld-link -entry:main -thinlto-index-only \ +; RUN: -thinlto-emit-imports-files %t1.obj %t2.obj %t3.obj \ +; RUN: -out:%t4.exe 2>&1 | FileCheck %s --check-prefix=ERR +; ERR: cannot open {{.*}}3.obj.imports: {{P|p}}ermission denied + +; Ensure lld doesn't generate import files when thinlto-index-only is not enabled +; RUN: rm -f %t1.obj.imports +; RUN: rm -f %t2.obj.imports +; RUN: rm -f %t3.obj.imports +; RUN: lld-link -entry:main -thinlto-emit-imports-files \ +; RUN: %t1.obj %t2.obj %t3.obj -out:%t4.exe +; RUN: not ls %t1.obj.imports +; RUN: not ls %t2.obj.imports +; RUN: not ls %t3.obj.imports + +target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc19.0.24215" + +declare void @g(...) + +define void @main() { +entry: + call void (...) @g() + ret void +} diff --git a/lld/test/COFF/thinlto-index-only.ll b/lld/test/COFF/thinlto-index-only.ll new file mode 100644 --- /dev/null +++ b/lld/test/COFF/thinlto-index-only.ll @@ -0,0 +1,52 @@ +; REQUIRES: x86 + +; Basic ThinLTO tests. +; RUN: opt -thinlto-bc %s -o %t1.obj +; RUN: opt -thinlto-bc %p/Inputs/thinlto.ll -o %t2.obj +; RUN: opt -thinlto-bc %p/Inputs/thinlto-empty.ll -o %t3.obj + +; Ensure lld generates an index and not a binary if requested. +; RUN: rm -f %t4.exe +; RUN: lld-link -thinlto-index-only -entry:main %t1.obj %t2.obj -out:%t4.exe +; RUN: llvm-bcanalyzer -dump %t1.obj.thinlto.bc | FileCheck %s --check-prefix=BACKEND1 +; RUN: llvm-bcanalyzer -dump %t2.obj.thinlto.bc | FileCheck %s --check-prefix=BACKEND2 +; RUN: not test -e %t4.exe + +; The backend index for this module contains summaries from itself and +; Inputs/thinlto.ll, as it imports from the latter. +; BACKEND1: