diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h --- a/lld/ELF/Config.h +++ b/lld/ELF/Config.h @@ -134,6 +134,7 @@ llvm::StringRef soName; llvm::StringRef sysroot; llvm::StringRef thinLTOCacheDir; + llvm::StringRef thinLTOIndex; llvm::StringRef thinLTOIndexOnlyArg; llvm::StringRef whyExtract; StringRef zBtiReport = "none"; @@ -365,6 +366,8 @@ // this means to map the primary and thread stacks as PROT_MTE. Note: This is // not supported on Android 11 & 12. bool androidMemtagStack; + + llvm::DenseMap remappingFile; }; // The only instance of Configuration struct. diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -1017,6 +1017,26 @@ return arg == "none" || arg == "warning" || arg == "error"; } +// A remapping file contains a tab-separated pair: from, to. Distributed ThinLTO +// uses the file to redirect an input bitcode file to a backend compiled +// ELF object file . We make this option generic potentially for other use +// cases. +static void readRemappingFile(StringRef filename) { + Optional buffer = readFile(filename); + if (!buffer) + return; + SmallVector fields; + for (StringRef line : args::getLines(*buffer)) { + fields.clear(); + line.split(fields, '\t'); + if (fields.size() != 2) { + error(filename + ": parse error"); + return; + } + config->remappingFile[fields[0]] = fields[1]; + } +} + // Initializes Config members by the command line options. static void readConfigs(opt::InputArgList &args) { errorHandler().verbose = args.hasArg(OPT_verbose); @@ -1186,10 +1206,13 @@ parseCachePruningPolicy(args.getLastArgValue(OPT_thinlto_cache_policy)), "--thinlto-cache-policy: invalid cache policy"); config->thinLTOEmitImportsFiles = args.hasArg(OPT_thinlto_emit_imports_files); - config->thinLTOEmitIndexFiles = args.hasArg(OPT_thinlto_emit_index_files) || + config->thinLTOIndex = args.getLastArgValue(OPT_thinlto_index); + config->thinLTOEmitIndexFiles = config->thinLTOIndex.size() || + args.hasArg(OPT_thinlto_emit_index_files) || args.hasArg(OPT_thinlto_index_only) || args.hasArg(OPT_thinlto_index_only_eq); - config->thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) || + config->thinLTOIndexOnly = config->thinLTOIndex.size() || + args.hasArg(OPT_thinlto_index_only) || args.hasArg(OPT_thinlto_index_only_eq); config->thinLTOIndexOnlyArg = args.getLastArgValue(OPT_thinlto_index_only_eq); config->thinLTOObjectSuffixReplace = @@ -1261,6 +1284,9 @@ config->optEL = true; } + if (const auto *arg = args.getLastArg(OPT_remapping_file)) + readRemappingFile(arg->getValue()); + for (opt::Arg *arg : args.filtered(OPT_shuffle_sections)) { constexpr StringRef errPrefix = "--shuffle-sections=: "; std::pair kv = StringRef(arg->getValue()).split('='); @@ -2590,7 +2616,7 @@ // Skip the normal linked output if some LTO options are specified. // - // For --thinlto-index-only, index file creation is performed in + // For --thinlto-index{,-only}, index file creation is performed in // compileBitcodeFiles, so we are done afterwards. --plugin-opt=emit-llvm and // --plugin-opt=emit-asm create output files in bitcode or assembly code, // respectively. When only certain thinLTO modules are specified for diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -105,6 +105,10 @@ if (!config->chroot.empty() && path.startswith("/")) path = saver().save(config->chroot + path); + auto it = config->remappingFile.find(path); + if (it != config->remappingFile.end()) + path = it->second; + log(path); config->dependencyFiles.insert(llvm::CachedHashString(path)); diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp --- a/lld/ELF/LTO.cpp +++ b/lld/ELF/LTO.cpp @@ -207,6 +207,12 @@ ltoObj = std::make_unique(createConfig(), backend, config->ltoPartitions); + if (!config->thinLTOIndex.empty()) + if (auto os = openFile(config->thinLTOIndex)) + for (BitcodeFile *file : ctx->bitcodeFiles) + *os << file->getName() << '\t' + << getThinLTOOutputFile(replaceThinLTOSuffix(file->getName())) + << '\n'; // Initialize usedStartStop. if (ctx->bitcodeFiles.empty()) diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td --- a/lld/ELF/Options.td +++ b/lld/ELF/Options.td @@ -352,6 +352,10 @@ "Enable target-specific relaxations if supported (default)", "Disable target-specific relaxations">; +def remapping_file: JJ<"remapping-file=">, + HelpText<"Each line contains (from,to) separated by a tab. An input file matching is redirected to ">, + MetaVarName<"">; + defm reproduce: EEq<"reproduce", "Write tar file containing inputs and command to reproduce link">; @@ -601,6 +605,7 @@ defm thinlto_cache_policy: EEq<"thinlto-cache-policy", "Pruning policy for the ThinLTO cache">; def thinlto_emit_imports_files: FF<"thinlto-emit-imports-files">; def thinlto_emit_index_files: FF<"thinlto-emit-index-files">; +def thinlto_index: JJ<"thinlto-index=">; def thinlto_index_only: FF<"thinlto-index-only">; def thinlto_index_only_eq: JJ<"thinlto-index-only=">; def thinlto_jobs: JJ<"thinlto-jobs=">, diff --git a/lld/test/ELF/lto/thinlto-index-file.ll b/lld/test/ELF/lto/thinlto-index-file.ll --- a/lld/test/ELF/lto/thinlto-index-file.ll +++ b/lld/test/ELF/lto/thinlto-index-file.ll @@ -1,4 +1,7 @@ ; REQUIRES: x86 +;; Test --thinlto-index-only= for distributed ThinLTO. +;; This option is discouraged in favor of --thinlto-index=. + ; RUN: rm -rf %t && mkdir %t && cd %t ; RUN: opt -module-summary %s -o 1.o ; RUN: opt -module-summary %p/Inputs/thinlto.ll -o 2.o diff --git a/lld/test/ELF/lto/thinlto-index.ll b/lld/test/ELF/lto/thinlto-index.ll new file mode 100644 --- /dev/null +++ b/lld/test/ELF/lto/thinlto-index.ll @@ -0,0 +1,79 @@ +; REQUIRES: x86 +;; Test --thinlto-index= for distributed ThinLTO. + +; RUN: rm -rf %t && split-file %s %t && mkdir %t/out && cd %t +; RUN: llvm-mc -filetype=obj -triple=x86_64 main.s -o main.o +; RUN: opt -module-summary a.ll -o out/a.indexing.o +; RUN: opt -module-summary %p/Inputs/thinlto.ll -o out/b.indexing.o +; RUN: opt -module-summary %p/Inputs/thinlto_empty.ll -o out/c.indexing.o +; RUN: cp out/c.indexing.o out/d.indexing.o + +; RUN: ld.lld --thinlto-index=1.map --thinlto-prefix-replace='out;lto' \ +; RUN: --thinlto-object-suffix-replace='.indexing.o;.o' main.o \ +; RUN: out/a.indexing.o --start-lib out/b.indexing.o out/c.indexing.o --end-lib out/d.indexing.o -o 1 +; RUN: FileCheck --input-file=1.map %s --implicit-check-not={{.}} --match-full-lines --strict-whitespace + +;; No entry for c.indexing.o which is not extracted. d.indexing.o is present, +;; otherwise the final link may try in-process ThinLTO backend compilation. +; CHECK:out/a.indexing.o lto/a.o +; CHECK-NEXT:out/b.indexing.o lto/b.o +; CHECK-NEXT:out/d.indexing.o lto/d.o + +;; Nevertheless, c.o.thinlto.bc exists to meet the build system requirement. +; RUN: ls lto/a.o.thinlto.bc lto/b.o.thinlto.bc lto/c.o.thinlto.bc lto/d.o.thinlto.bc + +; RUN: llvm-bcanalyzer -dump lto/a.o.thinlto.bc | FileCheck %s --check-prefix=BACKENDA +; RUN: llvm-bcanalyzer -dump lto/b.o.thinlto.bc | FileCheck %s --check-prefix=BACKENDB +; RUN: llvm-bcanalyzer -dump lto/c.o.thinlto.bc | FileCheck %s --check-prefix=BACKENDC +; RUN: llvm-bcanalyzer -dump lto/d.o.thinlto.bc | FileCheck %s --check-prefix=BACKENDD + +; BACKENDA: + +; BACKENDB: + +; BACKENDC: + +; BACKENDD: + +;; Thin archives can be used as well. +; RUN: llvm-ar rcT thin.a out/b.indexing.o out/c.indexing.o +; RUN: ld.lld --thinlto-index=2.map --thinlto-prefix-replace='out;lto' \ +; RUN: --thinlto-object-suffix-replace='.indexing.o;.o' main.o \ +; RUN: out/a.indexing.o thin.a out/d.indexing.o -o 2 +; RUN: FileCheck --input-file=2.map %s --implicit-check-not={{.}} --match-full-lines --strict-whitespace + +;; For regular archives, the filename may insufficient to locate the archive member. +; RUN: llvm-ar rc bc.a out/b.indexing.o out/c.indexing.o +; RUN: ld.lld --thinlto-index=3.map --thinlto-prefix-replace='out;lto' \ +; RUN: --thinlto-object-suffix-replace='.indexing.o;.o' main.o \ +; RUN: out/a.indexing.o bc.a out/d.indexing.o -o 3 +; RUN: FileCheck --input-file=3.map %s --check-prefix=ARCHIVE --implicit-check-not={{.}} --match-full-lines --strict-whitespace + +; ARCHIVE:out/a.indexing.o lto/a.o +; ARCHIVE-NEXT:b.indexing.o b.o +; ARCHIVE-NEXT:out/d.indexing.o lto/d.o + +;--- a.ll +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +declare void @g() + +define void @f() { +entry: + call void () @g() + ret void +} + +;--- main.s +.globl _start +_start: + call f diff --git a/lld/test/ELF/lto/thinlto-single-module.ll b/lld/test/ELF/lto/thinlto-single-module.ll --- a/lld/test/ELF/lto/thinlto-single-module.ll +++ b/lld/test/ELF/lto/thinlto-single-module.ll @@ -48,9 +48,20 @@ ; RUN: ls | FileCheck --implicit-check-not='thin.{{.*}}.thinlto.bc' /dev/null ; RUN: FileCheck %s --check-prefix=IDX < single5.idx ; RUN: count 1 < single5.idx +; RUN: rm main.o.thinlto.bc ; IDX: main.o +; RUN: ld.lld main.o thin.a --thinlto-single-module=main.o --thinlto-index=single5.map +; RUN: ls main.o.thinlto.bc +; RUN: ls | FileCheck --implicit-check-not='thin.{{.*}}.thinlto.bc' /dev/null +; RUN: FileCheck --input-file=single5.map %s --check-prefix=REMAP --implicit-check-not={{.}} + +;; Currently the --thinlto-index= file is not affected by --thinlto-single-module. +; REMAP: main.o main.o +; REMAP-NEXT: thin1.o thin1.o +; REMAP-NEXT: thin2.o thin2.o + ;; Check temporary output generated for main.o only. ; RUN: ld.lld main.o thin.a --thinlto-single-module=main.o --save-temps ; RUN: ls main.o.0.preopt.bc diff --git a/lld/test/ELF/remapping-file.test b/lld/test/ELF/remapping-file.test new file mode 100644 --- /dev/null +++ b/lld/test/ELF/remapping-file.test @@ -0,0 +1,46 @@ +# REQUIRES: x86 +## --remapping-file redirects an input file to another file. + +# RUN: rm -rf %t && split-file %s %t && cd %t +# RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o a.o +# RUN: llvm-as b.ll -o b.o +# RUN: llvm-mc -filetype=obj -triple=x86_64 c.s -o c.o && llvm-ar rc c.a c.o +# RUN: llvm-mc -filetype=obj -triple=x86_64 d.s -o d.o && ld.lld -shared -soname=d d.o -o d.so +# RUN: ld.lld --remapping-file=1.map --reproduce=repro.tar aa.o bb.bc cc.a dd.so -o 1 + +# RUN: tar tf repro.tar | FileCheck %s --check-prefix=REPRO + +# REPRO: 1.map +# REPRO-NEXT: a.o +# REPRO-NEXT: b.o +# REPRO-NEXT: c.a +# REPRO-NEXT: d.so + +#--- a.s +.globl _start +_start: + call b + call c + call d + +#--- b.ll +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @b() { + ret void +} + +#--- c.s +.globl c +c: + +#--- d.s +.globl d +d: + +#--- 1.map +aa.o a.o +bb.bc b.o +cc.a c.a +dd.so d.so