Index: lld/trunk/COFF/Config.h =================================================================== --- lld/trunk/COFF/Config.h +++ lld/trunk/COFF/Config.h @@ -179,6 +179,9 @@ // Used for /lldmap. std::string mapFile; + // Used for /thinlto-index-only: + llvm::StringRef thinLTOIndexOnlyArg; + uint64_t imageBase = -1; uint64_t fileAlign = 512; uint64_t stackReserve = 1024 * 1024; @@ -209,6 +212,8 @@ bool repro = false; bool swaprunCD = false; bool swaprunNet = false; + bool thinLTOEmitImportsFiles; + bool thinLTOIndexOnly; }; extern Configuration *config; Index: lld/trunk/COFF/Driver.cpp =================================================================== --- lld/trunk/COFF/Driver.cpp +++ lld/trunk/COFF/Driver.cpp @@ -1441,6 +1441,11 @@ fatal("/manifestinput: requires /manifest:embed"); } + config->thinLTOEmitImportsFiles = args.hasArg(OPT_thinlto_emit_imports_files); + config->thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) || + args.hasArg(OPT_thinlto_index_only_arg); + config->thinLTOIndexOnlyArg = + args.getLastArgValue(OPT_thinlto_index_only_arg); // Handle miscellaneous boolean flags. config->allowBind = args.hasFlag(OPT_allowbind, OPT_allowbind_no, true); config->allowIsolation = @@ -1727,8 +1732,18 @@ return; // Do LTO by compiling bitcode input files to a set of native COFF files then - // link those files. + // link those files (unless -thinlto-index-only was given, in which case we + // resolve symbols and write indices, but don't generate native code or link). symtab->addCombinedLTOObjects(); + + // If -thinlto-index-only is given, we should create only "index + // files" and not object files. Index file creation is already done + // in addCombinedLTOObject, so we are done if that's the case. + if (config->thinLTOIndexOnly) + return; + + // If we generated native object files from bitcode files, this resolves + // references to the symbols we use from them. run(); if (args.hasArg(OPT_include_optional)) { Index: lld/trunk/COFF/LTO.h =================================================================== --- lld/trunk/COFF/LTO.h +++ lld/trunk/COFF/LTO.h @@ -21,7 +21,9 @@ #define LLD_COFF_LTO_H #include "lld/Common/LLVM.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallString.h" +#include "llvm/Support/raw_ostream.h" #include #include @@ -49,6 +51,8 @@ std::unique_ptr ltoObj; std::vector> buf; std::vector> files; + std::unique_ptr indexFile; + llvm::DenseSet thinIndices; }; } } Index: lld/trunk/COFF/LTO.cpp =================================================================== --- lld/trunk/COFF/LTO.cpp +++ lld/trunk/COFF/LTO.cpp @@ -18,6 +18,7 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" +#include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/LTO/Caching.h" #include "llvm/LTO/Config.h" @@ -41,7 +42,19 @@ using namespace lld; using namespace lld::coff; -static std::unique_ptr createLTO() { +// Creates an empty file to and returns a raw_fd_ostream to write to it. +static std::unique_ptr openFile(StringRef file) { + std::error_code ec; + auto ret = + llvm::make_unique(file, ec, sys::fs::OpenFlags::F_None); + if (ec) { + error("cannot open " + file + ": " + ec.message()); + return nullptr; + } + return ret; +} + +static lto::Config createConfig() { lto::Config c; c.Options = initTargetOptionsFromCodeGenFlags(); @@ -67,14 +80,27 @@ if (config->saveTemps) checkError(c.addSaveTemps(std::string(config->outputFile) + ".", /*UseInputModulePath*/ true)); + return c; +} + +BitcodeCompiler::BitcodeCompiler() { + // Initialize indexFile. + if (!config->thinLTOIndexOnlyArg.empty()) + indexFile = openFile(config->thinLTOIndexOnlyArg); + + // Initialize ltoObj. lto::ThinBackend backend; - if (config->thinLTOJobs != 0) + if (config->thinLTOIndexOnly) { + auto OnIndexWrite = [&](StringRef S) { thinIndices.erase(S); }; + backend = lto::createWriteIndexesThinBackend( + "", "", config->thinLTOEmitImportsFiles, indexFile.get(), OnIndexWrite); + } else if (config->thinLTOJobs != 0) { backend = lto::createInProcessThinBackend(config->thinLTOJobs); - return llvm::make_unique(std::move(c), backend, - config->ltoPartitions); -} + } -BitcodeCompiler::BitcodeCompiler() : ltoObj(createLTO()) {} + ltoObj = llvm::make_unique(createConfig(), backend, + config->ltoPartitions); +} BitcodeCompiler::~BitcodeCompiler() = default; @@ -86,6 +112,9 @@ std::vector symBodies = f.getSymbols(); std::vector resols(symBodies.size()); + if (config->thinLTOIndexOnly) + thinIndices.insert(obj.getName()); + // Provide a resolution to the LTO API for each symbol. for (const lto::InputFile::Symbol &objSym : obj.symbols()) { Symbol *sym = symBodies[symNum]; @@ -129,6 +158,23 @@ }, cache)); + // Emit empty index files for non-indexed files + for (StringRef S : thinIndices) { + std::string Path(S); + openFile(Path + ".thinlto.bc"); + if (config->thinLTOEmitImportsFiles) + openFile(Path + ".imports"); + } + + // ThinLTO with index only option is required to generate only the index + // files. After that, we exit from linker and ThinLTO backend runs in a + // distributed environment. + if (config->thinLTOIndexOnly) { + if (indexFile) + indexFile->close(); + return {}; + } + if (!config->ltoCache.empty()) pruneCache(config->ltoCache, config->ltoCachePolicy); Index: lld/trunk/COFF/Options.td =================================================================== --- lld/trunk/COFF/Options.td +++ lld/trunk/COFF/Options.td @@ -176,6 +176,15 @@ "Base path used to make relative source file path absolute in PDB">; def rsp_quoting : Joined<["--"], "rsp-quoting=">, HelpText<"Quoting style for response files, 'windows' (default) or 'posix'">; +def thinlto_emit_imports_files : + F<"thinlto-emit-imports-files">, + HelpText<"Emit .imports files with -thinlto-index-only">; +def thinlto_index_only : + F<"thinlto-index-only">, + HelpText<"Instead of linking, emit ThinLTO index files">; +def thinlto_index_only_arg : P< + "thinlto-index-only", + "-thinlto-index-only and also write native module names to file">; def dash_dash_version : Flag<["--"], "version">, HelpText<"Print version information">; defm threads: B<"threads", Index: lld/trunk/test/COFF/Inputs/thinlto-empty.ll =================================================================== --- lld/trunk/test/COFF/Inputs/thinlto-empty.ll +++ lld/trunk/test/COFF/Inputs/thinlto-empty.ll @@ -0,0 +1,2 @@ +target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc19.0.24215" Index: lld/trunk/test/COFF/Inputs/thinlto.ll =================================================================== --- lld/trunk/test/COFF/Inputs/thinlto.ll +++ lld/trunk/test/COFF/Inputs/thinlto.ll @@ -0,0 +1,6 @@ +target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc19.0.24215" + +define void @g() { + ret void +} Index: lld/trunk/test/COFF/thinlto-emit-imports.ll =================================================================== --- lld/trunk/test/COFF/thinlto-emit-imports.ll +++ lld/trunk/test/COFF/thinlto-emit-imports.ll @@ -0,0 +1,59 @@ +; REQUIRES: x86 + +; Generate summary sections and test lld handling. +; RUN: opt -module-summary %s -o %t1.obj +; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.obj + +; Include a file with an empty module summary index, to ensure that the expected +; output files are created regardless, for a distributed build system. +; RUN: opt -module-summary %p/Inputs/thinlto-empty.ll -o %t3.obj + +; Ensure lld generates imports files if requested for distributed backends. +; RUN: rm -f %t3.obj.imports %t3.obj.thinlto.bc +; RUN: lld-link -entry:main -thinlto-index-only \ +; RUN: -thinlto-emit-imports-files %t1.obj %t2.obj %t3.obj -out:%t4.exe + +; The imports file for this module contains the bitcode file for +; Inputs/thinlto.ll +; RUN: cat %t1.obj.imports | count 1 +; RUN: cat %t1.obj.imports | FileCheck %s --check-prefix=IMPORTS1 +; IMPORTS1: thinlto-emit-imports.ll.tmp2.obj + +; The imports file for Input/thinlto.ll is empty as it does not import anything. +; RUN: cat %t2.obj.imports | count 0 + +; The imports file for Input/thinlto_empty.ll is empty but should exist. +; RUN: cat %t3.obj.imports | count 0 + +; The index file should be created even for the input with an empty summary. +; RUN: ls %t3.obj.thinlto.bc + +; Ensure lld generates error if unable to write to imports file. +; RUN: rm -f %t3.obj.imports +; RUN: touch %t3.obj.imports +; RUN: chmod 400 %t3.obj.imports +; RUN: not lld-link -entry:main -thinlto-index-only \ +; RUN: -thinlto-emit-imports-files %t1.obj %t2.obj %t3.obj \ +; RUN: -out:%t4.exe 2>&1 | FileCheck %s --check-prefix=ERR +; ERR: cannot open {{.*}}3.obj.imports: {{P|p}}ermission denied + +; Ensure lld doesn't generate import files when thinlto-index-only is not enabled +; RUN: rm -f %t1.obj.imports +; RUN: rm -f %t2.obj.imports +; RUN: rm -f %t3.obj.imports +; RUN: lld-link -entry:main -thinlto-emit-imports-files \ +; RUN: %t1.obj %t2.obj %t3.obj -out:%t4.exe +; RUN: not ls %t1.obj.imports +; RUN: not ls %t2.obj.imports +; RUN: not ls %t3.obj.imports + +target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc19.0.24215" + +declare void @g(...) + +define void @main() { +entry: + call void (...) @g() + ret void +} Index: lld/trunk/test/COFF/thinlto-index-only.ll =================================================================== --- lld/trunk/test/COFF/thinlto-index-only.ll +++ lld/trunk/test/COFF/thinlto-index-only.ll @@ -0,0 +1,52 @@ +; REQUIRES: x86 + +; Basic ThinLTO tests. +; RUN: opt -thinlto-bc %s -o %t1.obj +; RUN: opt -thinlto-bc %p/Inputs/thinlto.ll -o %t2.obj +; RUN: opt -thinlto-bc %p/Inputs/thinlto-empty.ll -o %t3.obj + +; Ensure lld generates an index and not a binary if requested. +; RUN: rm -f %t4.exe +; RUN: lld-link -thinlto-index-only -entry:main %t1.obj %t2.obj -out:%t4.exe +; RUN: llvm-bcanalyzer -dump %t1.obj.thinlto.bc | FileCheck %s --check-prefix=BACKEND1 +; RUN: llvm-bcanalyzer -dump %t2.obj.thinlto.bc | FileCheck %s --check-prefix=BACKEND2 +; RUN: not test -e %t4.exe + +; The backend index for this module contains summaries from itself and +; Inputs/thinlto.ll, as it imports from the latter. +; BACKEND1: