[lld/mac] Add --reproduce option

[thakis]

Summary

This adds support for ld.lld's --reproduce / lld-link's /reproduce:
flag to the MachO port. This flag can be added to a link command
to make the link write a tar file containing all inputs to the link
and a response file containing the link command. This can be used
to reproduce the link on another machine, which is useful for sharing
bug report inputs or performance test loads.

Since the linker is usually called through the clang driver and
adding linker flags can be a bit cumbersome, setting the env var
LLD_REPRODUCE=foo.tar triggers the feature as well.

The file response.txt in the archive can be used with
ld64.lld.darwinnew $(cat response.txt) as long as the contents are
smaller than the command-line limit, or with `ld64.lld.darwinnew
@response.txt` once D92149 is in.

The support in this patch is sufficient to create a tar file for
Chromium's base_unittests that can link after unpacking on a different
machine.

[thakis]

lld/Common/Reproduce.cpp

61

(Turns out neither ELF nor COFF ports have MultiArg<> flags. Covered by the -platform_version flag in the test.)

[int3]

lgtm

lld/Common/Reproduce.cpp

61

lld/MachO/DriverUtils.cpp

122

lld/MachO/InputFiles.h

40

lld/test/MachO/reproduce.s

34

nit: align with previous line

[thakis]

Thanks!

lld/Common/Reproduce.cpp

 }
 
 // Converts an Arg to a string representation suitable for a response file.
 // To show an Arg in a diagnostic, use Arg::getAsString() instead.
 std::string lld::toString(const opt::Arg &arg) {
   std::string k = std::string(arg.getSpelling());
   if (arg.getNumValues() == 0)
     return k;
-  std::string v = quote(arg.getValue());
+  std::string v;
+  for (size_t i = 0; i < arg.getNumValues(); ++i) {
+    if (i > 0)
+      v.push_back(' ');
+    v += quote(arg.getValue(i));
+  }
   if (arg.getOption().getRenderStyle() == opt::Option::RenderJoinedStyle)
     return k + v;
   return k + " " + v;
 }

lld/ELF/DriverUtils.cpp

for (auto *arg : args) {
     case OPT_INPUT:
       os << quote(rewritePath(arg->getValue())) << "\n";
       break;
     case OPT_o:
       // If -o path contains directories, "lld @response.txt" will likely
       // fail because the archive we are creating doesn't contain empty
       // directories for the output path (-o doesn't create directories).
       // Strip directories to prevent the issue.
-      os << "-o " << quote(sys::path::filename(arg->getValue())) << "\n";
+      os << "-o " << quote(path::filename(arg->getValue())) << "\n";
       break;
     case OPT_dynamic_list:
     case OPT_library_path:
     case OPT_rpath:
     case OPT_script:
     case OPT_symbol_ordering_file:
     case OPT_sysroot:
     case OPT_version_script:

lld/MachO/Driver.h

 // Create enum with OPT_xxx values for each option in Options.td
 enum {
   OPT_INVALID = 0,
 #define OPTION(_1, _2, ID, _4, _5, _6, _7, _8, _9, _10, _11, _12) OPT_##ID,
 #include "Options.inc"
 #undef OPTION
 };
 
+std::string createResponseFile(const llvm::opt::InputArgList &args);
+
 // Check for both libfoo.dylib and libfoo.tbd (in that order).
 llvm::Optional<std::string> resolveDylibPath(llvm::StringRef path);
 
 llvm::Optional<DylibFile *> makeDylibFromTAPI(llvm::MemoryBufferRef mbref,
                                               DylibFile *umbrella = nullptr);
 
 } // namespace macho
 } // namespace lld
 
 #endif

lld/MachO/Driver.cpp

 #include "llvm/BinaryFormat/Magic.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/TarWriter.h"
 #include "llvm/Support/TargetSelect.h"
 
 #include <algorithm>
 
 using namespace llvm;
 using namespace llvm::MachO;
 using namespace llvm::object;
 using namespace llvm::opt;
case OPT_grp_ignored:
     break;
   default:
     warn("Option `" + opt.getPrefixedName() +
          "' is not yet implemented. Stay tuned...");
     break;
   }
 }
 
+static const char *getReproduceOption(opt::InputArgList &args) {
+  if (auto *arg = args.getLastArg(OPT_reproduce))
+    return arg->getValue();
+  return getenv("LLD_REPRODUCE");
+}
+
 bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
                  raw_ostream &stdoutOS, raw_ostream &stderrOS) {
   lld::stdoutOS = &stdoutOS;
   lld::stderrOS = &stderrOS;
 
   stderrOS.enable_colors(stderrOS.has_colors());
   // TODO: Set up error handler properly, e.g. the errorLimitExceededMsg
 
   errorHandler().cleanupCallback = []() { freeArena(); };
 
   MachOOptTable parser;
   opt::InputArgList args = parser.parse(argsArr.slice(1));
 
   if (args.hasArg(OPT_help_hidden)) {
     parser.printHelp(argsArr[0], /*showHidden=*/true);
     return true;
   } else if (args.hasArg(OPT_help)) {
     parser.printHelp(argsArr[0], /*showHidden=*/false);
     return true;
   }
 
+  if (const char *path = getReproduceOption(args)) {
+    // Note that --reproduce is a debug option so you can ignore it
+    // if you are trying to understand the whole picture of the code.
+    Expected<std::unique_ptr<TarWriter>> errOrWriter =
+        TarWriter::create(path, path::stem(path));
+    if (errOrWriter) {
+      tar = std::move(*errOrWriter);
+      tar->append("response.txt", createResponseFile(args));
+      tar->append("version.txt", getLLDVersion() + "\n");
+    } else {
+      error("--reproduce: " + toString(errOrWriter.takeError()));
+    }
+  }
+
   config = make<Configuration>();
   symtab = make<SymbolTable>();
   target = createTargetInfo(args);
 
   config->entry = symtab->addUndefined(args.getLastArgValue(OPT_e, "_main"));
   config->outputFile = args.getLastArgValue(OPT_o, "a.out");
   config->installName =
       args.getLastArgValue(OPT_install_name, config->outputFile);

lld/MachO/DriverUtils.cpp

 //===- DriverUtils.cpp ----------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "Driver.h"
 #include "InputFiles.h"
 
+#include "lld/Common/Args.h"
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
+#include "lld/Common/Reproduce.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Path.h"
 #include "llvm/TextAPI/MachO/TextAPIReader.h"
 
 using namespace llvm;

 }
 
 void MachOOptTable::printHelp(const char *argv0, bool showHidden) const {
   PrintHelp(lld::outs(), (std::string(argv0) + " [options] file...").c_str(),
             "LLVM Linker", showHidden);
   lld::outs() << "\n";
 }
 
+static std::string rewritePath(StringRef s) {
+  if (fs::exists(s))
+    return relativeToRoot(s);
+  return std::string(s);
+}
+
+// Reconstructs command line arguments so that so that you can re-run
+// the same command with the same inputs. This is for --reproduce.
+std::string macho::createResponseFile(const opt::InputArgList &args) {
+  SmallString<0> data;
+  raw_svector_ostream os(data);
+
+  // Copy the command line to the output while rewriting paths.
+  for (auto *arg : args) {
+    switch (arg->getOption().getID()) {
+    case OPT_reproduce:
+      break;
+    case OPT_INPUT:
+      os << quote(rewritePath(arg->getValue())) << "\n";
+      break;
+    case OPT_o:
+      os << "-o " << quote(path::filename(arg->getValue())) << "\n";
+      break;
+    case OPT_filelist:
+      if (Optional<MemoryBufferRef> buffer = readFile(arg->getValue()))
+        for (StringRef path : args::getLines(*buffer))
+          os << quote(rewritePath(path)) << "\n";
+      break;
+    case OPT_force_load:
+    case OPT_rpath:
+    case OPT_syslibroot:
+    case OPT_F:
+    case OPT_L:
+    case OPT_order_file:
+      os << arg->getSpelling() << " " << quote(rewritePath(arg->getValue()))
+         << "\n";
+      break;
+    case OPT_sectcreate:
+      os << arg->getSpelling() << " " << quote(arg->getValue(0)) << " "
+         << quote(arg->getValue(1)) << " "
+         << quote(rewritePath(arg->getValue(2))) << "\n";
+      break;
+    default:
+      os << toString(*arg) << "\n";
+    }
+  }
+  return std::string(data.str());
+}
+
 Optional<std::string> macho::resolveDylibPath(StringRef path) {
   // TODO: if a tbd and dylib are both present, we should check to make sure
   // they are consistent.
   if (fs::exists(path))
     return std::string(path);
 
   SmallString<261> location = path;
   path::replace_extension(location, ".tbd");

lld/MachO/InputFiles.h

 
 #include <map>
 #include <vector>
 
 namespace llvm {
 namespace lto {
 class InputFile;
 } // namespace lto
+class TarWriter;
 } // namespace llvm
 
 namespace lld {
 namespace macho {
 
 class InputSection;
 class Symbol;
 struct Reloc;
 
+// If --reproduce option is given, all input files are written
+// to this tar archive.
+extern std::unique_ptr<llvm::TarWriter> tar;
+
 // If .subsections_via_symbols is set, each InputSection will be split along
 // symbol boundaries. The keys of a SubsectionMap represent the offsets of
 // each subsection from the start of the original pre-split InputSection.
 using SubsectionMap = std::map<uint32_t, InputSection *>;
 
 class InputFile {
 public:
   enum Kind {

lld/MachO/InputFiles.cpp

 #include "OutputSection.h"
 #include "OutputSegment.h"
 #include "SymbolTable.h"
 #include "Symbols.h"
 #include "Target.h"
 
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
+#include "lld/Common/Reproduce.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/TarWriter.h"
 
 using namespace llvm;
 using namespace llvm::MachO;
 using namespace llvm::support::endian;
 using namespace llvm::sys;
 using namespace lld;
 using namespace lld::macho;
 
 std::vector<InputFile *> macho::inputFiles;
+std::unique_ptr<TarWriter> macho::tar;
 
 // Open a given file path and return it as a memory-mapped file.
 Optional<MemoryBufferRef> macho::readFile(StringRef path) {
   // Open a file.
   auto mbOrErr = MemoryBuffer::getFile(path);
   if (auto ec = mbOrErr.getError()) {
     error("cannot open " + path + ": " + ec.message());
     return None;
   }
 
   std::unique_ptr<MemoryBuffer> &mb = *mbOrErr;
   MemoryBufferRef mbref = mb->getMemBufferRef();
   make<std::unique_ptr<MemoryBuffer>>(std::move(mb)); // take mb ownership
 
   // If this is a regular non-fat file, return it.
   const char *buf = mbref.getBufferStart();
   auto *hdr = reinterpret_cast<const MachO::fat_header *>(buf);
-  if (read32be(&hdr->magic) != MachO::FAT_MAGIC)
+  if (read32be(&hdr->magic) != MachO::FAT_MAGIC) {
+    if (tar)
+      tar->append(relativeToRoot(path), mbref.getBuffer());
     return mbref;
+  }
 
   // Object files and archive files may be fat files, which contains
   // multiple real files for different CPU ISAs. Here, we search for a
   // file that matches with the current link target and returns it as
   // a MemoryBufferRef.
   auto *arch = reinterpret_cast<const MachO::fat_arch *>(buf + sizeof(*hdr));
 
   for (uint32_t i = 0, n = read32be(&hdr->nfat_arch); i < n; ++i) {
     if (reinterpret_cast<const char *>(arch + i + 1) >
         buf + mbref.getBufferSize()) {
       error(path + ": fat_arch struct extends beyond end of file");
       return None;
     }
 
     if (read32be(&arch[i].cputype) != target->cpuType ||
         read32be(&arch[i].cpusubtype) != target->cpuSubtype)
       continue;
 
     uint32_t offset = read32be(&arch[i].offset);
     uint32_t size = read32be(&arch[i].size);
     if (offset + size > mbref.getBufferSize())
       error(path + ": slice extends beyond end of file");
+    if (tar)
+      tar->append(relativeToRoot(path), mbref.getBuffer());
     return MemoryBufferRef(StringRef(buf + offset, size), path.copy(bAlloc));
   }
 
   error("unable to find matching architecture in " + path);
   return None;
 }
 
 const load_command *macho::findCommand(const mach_header_64 *hdr,

lld/MachO/Options.td

 include "llvm/Option/OptParser.td"
 
 // Flags that lld/MachO understands but ld64 doesn't. These take
 // '--' instead of '-' and use dashes instead of underscores, so
 // they don't collide with the ld64 compat options.
 
 def help : Flag<["-", "--"], "help">;
 def help_hidden : Flag<["--"], "help-hidden">,
   HelpText<"Display help for hidden options">;
 def color_diagnostics: Flag<["--"], "color-diagnostics">,
   HelpText<"Alias for --color-diagnostics=always">;
 def no_color_diagnostics: Flag<["--"], "no-color-diagnostics">,
   HelpText<"Alias for --color-diagnostics=never">;
 def color_diagnostics_eq: Joined<["--"], "color-diagnostics=">,
   HelpText<"Use colors in diagnostics (default: auto)">,
   MetaVarName<"[auto,always,never]">;
+def reproduce: Separate<["--"], "reproduce">;
+def reproduce_eq: Joined<["--"], "reproduce=">,
+    Alias<!cast<Separate>(reproduce)>,
+    HelpText<"Write tar file containing inputs and command to reproduce link">;
 
 
 // This is a complete Options.td compiled from Apple's ld(1) manpage
 // dated 2018-03-07 and cross checked with ld64 source code in repo
 // https://github.com/apple-opensource/ld64 at git tag "512.4" dated
 // 2018-03-18.
 
 // Flags<[HelpHidden]> marks options that are not yet ported to lld,

lld/test/MachO/reproduce.s

+# REQUIRES: x86
+
+# RUN: rm -rf %t.dir
+# RUN: mkdir -p %t.dir/build1
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos %s -o %t.dir/build1/foo.o
+# RUN: cd %t.dir
+# RUN: %lld -platform_version macos 10.10.0 11.0 build1/foo.o -o bar --reproduce repro1.tar
+# RUN: tar xOf repro1.tar repro1/%:t.dir/build1/foo.o > build1-foo.o
+# RUN: cmp build1/foo.o build1-foo.o
+
+# RUN: tar xf repro1.tar repro1/response.txt repro1/version.txt
+# RUN: FileCheck %s --check-prefix=RSP1 < repro1/response.txt
+# RSP1:      {{^}}-platform_version macos 10.10.0 11.0{{$}}
+# RSP1-NOT:  {{^}}repro1{{[/\\]}}
+# RSP1-NEXT: {{[/\\]}}foo.o
+# RSP1-NEXT: -o bar
+# RSP1-NOT:  --reproduce
+
+# RUN: FileCheck %s --check-prefix=VERSION < repro1/version.txt
+# VERSION: LLD
+
+# RUN: mkdir -p %t.dir/build2/a/b/c
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos %s -o %t.dir/build2/foo.o
+# RUN: cd %t.dir/build2/a/b/c
+# RUN: echo ./../../../foo.o > %t.dir/build2/filelist
+# RUN: env LLD_REPRODUCE=repro2.tar %lld -filelist %t.dir/build2/filelist -o /dev/null
+# RUN: tar xOf repro2.tar repro2/%:t.dir/build2/foo.o > build2-foo.o
+# RUN: cmp %t.dir/build2/foo.o build2-foo.o
+
+# RUN: tar xf repro2.tar repro2/response.txt repro2/version.txt
+# RUN: FileCheck %s --check-prefix=RSP2 < repro2/response.txt
+# RSP2-NOT:  {{^}}repro2{{[/\\]}}
+# RSP2:      {{[/\\]}}foo.o
+
+.globl _main
+_main:
+  ret