diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h --- a/lld/ELF/Config.h +++ b/lld/ELF/Config.h @@ -171,6 +171,7 @@ llvm::StringRef soName; llvm::StringRef sysroot; llvm::StringRef thinLTOCacheDir; + llvm::StringRef thinLTOIndex; llvm::StringRef thinLTOIndexOnlyArg; llvm::StringRef whyExtract; StringRef zBtiReport = "none"; diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -1273,10 +1273,13 @@ parseCachePruningPolicy(args.getLastArgValue(OPT_thinlto_cache_policy)), "--thinlto-cache-policy: invalid cache policy"); config->thinLTOEmitImportsFiles = args.hasArg(OPT_thinlto_emit_imports_files); - config->thinLTOEmitIndexFiles = args.hasArg(OPT_thinlto_emit_index_files) || + config->thinLTOIndex = args.getLastArgValue(OPT_thinlto_index); + config->thinLTOEmitIndexFiles = config->thinLTOIndex.size() || + args.hasArg(OPT_thinlto_emit_index_files) || args.hasArg(OPT_thinlto_index_only) || args.hasArg(OPT_thinlto_index_only_eq); - config->thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) || + config->thinLTOIndexOnly = config->thinLTOIndex.size() || + args.hasArg(OPT_thinlto_index_only) || args.hasArg(OPT_thinlto_index_only_eq); config->thinLTOIndexOnlyArg = args.getLastArgValue(OPT_thinlto_index_only_eq); config->thinLTOObjectSuffixReplace = @@ -1293,7 +1296,7 @@ "--thinlto-emit-index-files"); } if (!config->thinLTOPrefixReplaceNativeObject.empty() && - config->thinLTOIndexOnlyArg.empty()) { + config->thinLTOIndex.empty() && config->thinLTOIndexOnlyArg.empty()) { error("--thinlto-prefix-replace=old_dir;new_dir;obj_dir must be used with " "--thinlto-index-only="); } @@ -2732,7 +2735,7 @@ // Skip the normal linked output if some LTO options are specified. // - // For --thinlto-index-only, index file creation is performed in + // For --thinlto-index{,-only}, index file creation is performed in // compileBitcodeFiles, so we are done afterwards. --plugin-opt=emit-llvm and // --plugin-opt=emit-asm create output files in bitcode or assembly code, // respectively. When only certain thinLTO modules are specified for diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp --- a/lld/ELF/LTO.cpp +++ b/lld/ELF/LTO.cpp @@ -209,6 +209,28 @@ ltoObj = std::make_unique(createConfig(), backend, config->ltoPartitions); + // Write a remapping file for the final native link. Non-lazy bitcode files + // are mapped to native object files. If lazy bitcode files is a minimized + // bitcode, it cannot participate the final link. Print /dev/null to ignore + // it. + if (!config->thinLTOIndex.empty()) { + if (auto os = openFile(config->thinLTOIndex)) { + for (BitcodeFile *file : ctx.bitcodeFiles) { + StringRef nativeDir = config->thinLTOPrefixReplaceNativeObject.empty() + ? config->thinLTOPrefixReplaceNew + : config->thinLTOPrefixReplaceNativeObject; + *os << file->getName() << '=' + << lto::getThinLTOOutputFile(replaceThinLTOSuffix(file->getName()), + config->thinLTOPrefixReplaceOld, + nativeDir) + << '\n'; + } + for (BitcodeFile *file : ctx.lazyBitcodeFiles) + if (file->lazy) + *os << file->getName() << "=/dev/null\n"; + } + } + // Initialize usedStartStop. if (ctx.bitcodeFiles.empty()) return; diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td --- a/lld/ELF/Options.td +++ b/lld/ELF/Options.td @@ -618,6 +618,7 @@ defm thinlto_cache_policy: EEq<"thinlto-cache-policy", "Pruning policy for the ThinLTO cache">; def thinlto_emit_imports_files: FF<"thinlto-emit-imports-files">; def thinlto_emit_index_files: FF<"thinlto-emit-index-files">; +def thinlto_index: JJ<"thinlto-index=">; def thinlto_index_only: FF<"thinlto-index-only">; def thinlto_index_only_eq: JJ<"thinlto-index-only=">; def thinlto_jobs_eq: JJ<"thinlto-jobs=">, diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst --- a/lld/docs/ReleaseNotes.rst +++ b/lld/docs/ReleaseNotes.rst @@ -28,6 +28,9 @@ * ``--remap-inputs=`` and ``--remap-inputs-file=`` are added to remap input files. (`D148859 `_) +* Experimental ``--thinlto-index=`` and ``--remapping-file=`` are added for distributed ThinLTO. + They address some symbol resolution issues with the old ``--thinlto-index-only=``. + (`D130229 `_) Breaking changes ---------------- diff --git a/lld/test/ELF/lto/thinlto-index-file.ll b/lld/test/ELF/lto/thinlto-index-file.ll --- a/lld/test/ELF/lto/thinlto-index-file.ll +++ b/lld/test/ELF/lto/thinlto-index-file.ll @@ -1,4 +1,7 @@ ; REQUIRES: x86 +;; Test --thinlto-index-only= for distributed ThinLTO. +;; This option is discouraged in favor of --thinlto-index=. + ; RUN: rm -rf %t && mkdir %t && cd %t ; RUN: opt -module-summary %s -o 1.o ; RUN: opt -module-summary %p/Inputs/thinlto.ll -o 2.o diff --git a/lld/test/ELF/lto/thinlto-index.ll b/lld/test/ELF/lto/thinlto-index.ll new file mode 100644 --- /dev/null +++ b/lld/test/ELF/lto/thinlto-index.ll @@ -0,0 +1,102 @@ +; REQUIRES: x86 +;; Test --thinlto-index= for distributed ThinLTO. + +; RUN: rm -rf %t && split-file %s %t && mkdir %t/thin && cd %t +; RUN: llvm-mc -filetype=obj -triple=x86_64 main.s -o main.o +; RUN: llvm-mc -filetype=obj -triple=x86_64 c.s -o c.o +; RUN: opt -thinlto-bc a.ll -o thin/a.o -thin-link-bitcode-file=thin/a.min.o +; RUN: opt -thinlto-bc bc.ll -o thin/bc.o -thin-link-bitcode-file=thin/bc.min.o +; RUN: opt -thinlto-bc %p/Inputs/thinlto_empty.ll -o thin/empty.o -thin-link-bitcode-file=thin/empty.min.o +; RUN: opt -thinlto-bc %p/Inputs/thinlto_empty.ll -o thin/empty1.o -thin-link-bitcode-file=thin/empty1.min.o + +; RUN: ld.lld --thinlto-index=1.map --thinlto-emit-imports-files --thinlto-prefix-replace='thin;obj' \ +; RUN: --thinlto-object-suffix-replace='.min.o;.o' main.o thin/a.min.o \ +; RUN: --start-lib thin/bc.min.o thin/empty.min.o --end-lib thin/empty1.min.o --start-lib c.o --end-lib -o 1 +; RUN: FileCheck --input-file=1.map %s --implicit-check-not={{.}} --match-full-lines --strict-whitespace + +;; No entry for empty.min.o which is not extracted. empty1.min.o is present, +;; otherwise the final link may try in-process ThinLTO backend compilation. +; CHECK:thin/a.min.o=obj/a.o +; CHECK-NEXT:thin/bc.min.o=obj/bc.o +; CHECK-NEXT:thin/empty1.min.o=obj/empty1.o +; CHECK-NEXT:thin/empty.min.o=/dev/null + +;; Nevertheless, empty.o.{imports,thinlto.bc} exist to meet the build system requirement. +; RUN: ls obj/a.o.imports obj/bc.o.imports obj/empty.o.imports obj/empty1.o.imports + +; RUN: llvm-bcanalyzer -dump obj/a.o.thinlto.bc | FileCheck %s --check-prefix=BACKENDA +; RUN: llvm-bcanalyzer -dump obj/bc.o.thinlto.bc | FileCheck %s --check-prefix=BACKENDB +; RUN: llvm-bcanalyzer -dump obj/empty.o.thinlto.bc | FileCheck %s --check-prefix=BACKENDE +; RUN: llvm-bcanalyzer -dump obj/empty1.o.thinlto.bc | FileCheck %s --check-prefix=BACKENDE1 + +; BACKENDA: + +; BACKENDB: + +; BACKENDE: + +; BACKENDE1: + +;; Thin archives can be used as well. +; RUN: llvm-ar rcTS thin.a thin/bc.min.o thin/empty.min.o +; RUN: ld.lld --thinlto-index=2.map --thinlto-prefix-replace='thin;obj' \ +; RUN: --thinlto-object-suffix-replace='.min.o;.o' main.o \ +; RUN: thin/a.min.o thin.a thin/empty1.min.o -o 2 +; RUN: FileCheck --input-file=2.map %s --implicit-check-not={{.}} --match-full-lines --strict-whitespace + +;; For regular archives, the filename may be insufficient to locate the archive and the particular member. +; RUN: llvm-ar rcS bc.a thin/bc.min.o thin/empty.min.o +; RUN: ld.lld --thinlto-index=3.map --thinlto-prefix-replace='thin;obj' \ +; RUN: --thinlto-object-suffix-replace='.min.o;.o' main.o \ +; RUN: thin/a.min.o bc.a thin/empty1.min.o -o 3 +; RUN: FileCheck --input-file=3.map %s --check-prefix=ARCHIVE --implicit-check-not={{.}} --match-full-lines --strict-whitespace + +; ARCHIVE:thin/a.min.o=obj/a.o +; ARCHIVE-NEXT:bc.min.o=bc.o +; ARCHIVE-NEXT:thin/empty1.min.o=obj/empty1.o +; ARCHIVE-NEXT:empty.min.o=/dev/null + +;--- a.ll +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +declare void @b() +declare void @c() + +define void @a() { + call void () @b() + call void () @c() + ret void +} + +;--- bc.ll +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @b() { + ret void +} +define void @c() { + ret void +} + +;--- main.s +.globl _start +_start: + call a + +;--- c.s +.globl c +c: diff --git a/lld/test/ELF/lto/thinlto-single-module.ll b/lld/test/ELF/lto/thinlto-single-module.ll --- a/lld/test/ELF/lto/thinlto-single-module.ll +++ b/lld/test/ELF/lto/thinlto-single-module.ll @@ -48,9 +48,20 @@ ; RUN: ls | FileCheck --implicit-check-not='thin.{{.*}}.thinlto.bc' /dev/null ; RUN: FileCheck %s --check-prefix=IDX < single5.idx ; RUN: count 1 < single5.idx +; RUN: rm main.o.thinlto.bc ; IDX: main.o +; RUN: ld.lld main.o thin.a --thinlto-single-module=main.o --thinlto-index=single5.map +; RUN: ls main.o.thinlto.bc +; RUN: ls | FileCheck --implicit-check-not='thin.{{.*}}.thinlto.bc' /dev/null +; RUN: FileCheck --input-file=single5.map %s --check-prefix=REMAP --implicit-check-not={{.}} + +;; Currently the --thinlto-index= file is not affected by --thinlto-single-module. +; REMAP: main.o main.o +; REMAP-NEXT: thin1.o thin1.o +; REMAP-NEXT: thin2.o thin2.o + ;; Check temporary output generated for main.o only. ; RUN: ld.lld main.o thin.a --thinlto-single-module=main.o --save-temps ; RUN: ls main.o.0.preopt.bc