Add --reproduce to lld/wasm.

[ruiu]

Summary

--reproduce is a convenient option for debugging. If you invoke lld
with --reproduce=repro.tar, it creates repro.tar with all input
files and the command line options given to the linker, so that it is
very easy to run lld with the exact same inputs.

ELF and Windows lld have this option.

This patch add that option to lld/wasm.

[grimar]

Few cosmetic issues I noticed are below.

lld/wasm/Driver.cpp

31

Unsorted.

555

--reproduce

lld/wasm/InputFiles.cpp

35

Seems Loading should be lower case for conistency with log in dumpInfo() below.

lld/wasm/InputFiles.h

36

Use double dashes for consistency with the other comments?

[ruiu]

lld/wasm/Driver.cpp

31

It's actually sorted ASCIIbetically.

lld/wasm/InputFiles.cpp

35

Well, true, but we shouldn't fix that in this patch.

[ruiu]

• address review comments

[sbc100]

lld/test/wasm/reproduce.ll

10

If you are trying to verify the contents here why not something more explicit like tar t repro.tar | FileCheck %s -check-prefix=CONTENTS?

lld/wasm/InputFiles.cpp

58

Seems unrelated. Also, I kind of like the vertical separation here.

[sbc100]

The convention I've been using is to prefex the commit subject with [WebAssembly]

[ruiu]

lld/test/wasm/reproduce.ll

10

Done.

lld/wasm/InputFiles.cpp

58

Done.

lld/test/wasm/reproduce.ll

+; REQUIRES: shell
+; RUN: rm -rf %t.dir
+; RUN: mkdir -p %t.dir
+; RUN: llc -filetype=obj %s -o %t.dir/foo.o
+; RUN: wasm-ld --reproduce=%t.dir/repro.tar -o out.wasm %t.dir/foo.o
+
+; RUN: cd %t.dir
+; RUN: tar xf repro.tar
+; RUN: ls repro/%t.dir/foo.o
+
+; RUN: FileCheck %s < repro/response.txt
+
+; CHECK: -o out.wasm
+; CHECK: {{.*}}/foo.o
+
+; RUN: FileCheck %s --check-prefix=VERSION < repro/version.txt
+; VERSION: LLD
+
+target triple = "wasm32-unknown-unknown"
+
+define void @_start() {
+  ret void
+}

lld/wasm/Driver.cpp

 #include "InputChunks.h"
 #include "InputGlobal.h"
 #include "MarkLive.h"
 #include "SymbolTable.h"
 #include "Writer.h"
 #include "lld/Common/Args.h"
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
+#include "lld/Common/Reproduce.h"
 #include "lld/Common/Strings.h"
 #include "lld/Common/Threads.h"
 #include "lld/Common/Version.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Object/Wasm.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
+#include "llvm/Support/TarWriter.h"
 #include "llvm/Support/TargetSelect.h"
 
 #define DEBUG_TYPE "lld"
 
 using namespace llvm;
 using namespace llvm::object;
 using namespace llvm::sys;
 using namespace llvm::wasm;
 
if (Config->Pic) {
     WasmSym::MemoryBase->markLive();
     WasmSym::TableBase->markLive();
   }
 
   WasmSym::DsoHandle = Symtab->addSyntheticDataSymbol(
       "__dso_handle", WASM_SYMBOL_VISIBILITY_HIDDEN);
 }
 
+// Reconstructs command line arguments so that so that you can re-run
+// the same command with the same inputs. This is for --reproduce.
+static std::string createResponseFile(const opt::InputArgList &Args) {
+  SmallString<0> Data;
+  raw_svector_ostream OS(Data);
+
+  // Copy the command line to the output while rewriting paths.
+  for (auto *Arg : Args) {
+    switch (Arg->getOption().getUnaliasedOption().getID()) {
+    case OPT_reproduce:
+      break;
+    case OPT_INPUT:
+      OS << quote(relativeToRoot(Arg->getValue())) << "\n";
+      break;
+    case OPT_o:
+      // If -o path contains directories, "lld @response.txt" will likely
+      // fail because the archive we are creating doesn't contain empty
+      // directories for the output path (-o doesn't create directories).
+      // Strip directories to prevent the issue.
+      OS << "-o " << quote(sys::path::filename(Arg->getValue())) << "\n";
+      break;
+    default:
+      OS << toString(*Arg) << "\n";
+    }
+  }
+  return Data.str();
+}
+
 void LinkerDriver::link(ArrayRef<const char *> ArgsArr) {
   WasmOptTable Parser;
   opt::InputArgList Args = Parser.parse(ArgsArr.slice(1));
 
   // Handle --help
   if (Args.hasArg(OPT_help)) {
     Parser.PrintHelp(outs(),
                      (std::string(ArgsArr[0]) + " [options] file...").c_str(),
                      "LLVM Linker", false);
     return;
   }
 
   // Handle --version
   if (Args.hasArg(OPT_version) || Args.hasArg(OPT_v)) {
     outs() << getLLDVersion() << "\n";
     return;
   }
 
+  // Handle --reprodue
+  if (auto *Arg = Args.getLastArg(OPT_reproduce)) {
+    StringRef Path = Arg->getValue();
+    Expected<std::unique_ptr<TarWriter>> ErrOrWriter =
+        TarWriter::create(Path, path::stem(Path));
+    if (ErrOrWriter) {
+      Tar = std::move(*ErrOrWriter);
+      Tar->append("response.txt", createResponseFile(Args));
+      Tar->append("version.txt", getLLDVersion() + "\n");
+    } else {
+      error("--reproduce: " + toString(ErrOrWriter.takeError()));
+    }
+  }
+
   // Parse and evaluate -mllvm options.
   std::vector<const char *> V;
   V.push_back("wasm-ld (LLVM option parsing)");
   for (auto *Arg : Args.filtered(OPT_mllvm))
     V.push_back(Arg->getValue());
   cl::ParseCommandLineOptions(V.size(), V.data());
 
   errorHandler().ErrorLimit = args::getInteger(Args, OPT_error_limit, 20);

lld/wasm/InputFiles.h

 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/Wasm.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <vector>
 
+namespace llvm {
+class TarWriter;
+}
+
 namespace lld {
 namespace wasm {
 
 class InputChunk;
 class InputFunction;
 class InputSegment;
 class InputGlobal;
 class InputEvent;
 class InputSection;
 
+// If -reproduce option is given, all input files are written
+// to this tar archive.
+extern std::unique_ptr<llvm::TarWriter> Tar;
+
 class InputFile {
 public:
   enum Kind {
     ObjectKind,
     SharedKind,
     ArchiveKind,
     BitcodeKind,
   };

lld/wasm/InputFiles.cpp

 //===- InputFiles.cpp -----------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "InputFiles.h"
 #include "Config.h"
 #include "InputChunks.h"
 #include "InputEvent.h"
 #include "InputGlobal.h"
 #include "SymbolTable.h"
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
+#include "lld/Common/Reproduce.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/Wasm.h"
+#include "llvm/Support/TarWriter.h"
 #include "llvm/Support/raw_ostream.h"
 
 #define DEBUG_TYPE "lld"
 
 using namespace lld;
 using namespace lld::wasm;
 
 using namespace llvm;
 using namespace llvm::object;
 using namespace llvm::wasm;
 
+std::unique_ptr<llvm::TarWriter> lld::wasm::Tar;
+
 Optional<MemoryBufferRef> lld::wasm::readFile(StringRef Path) {
   log("Loading: " + Path);
 
   auto MBOrErr = MemoryBuffer::getFile(Path);
   if (auto EC = MBOrErr.getError()) {
     error("cannot open " + Path + ": " + EC.message());
     return None;
   }
   std::unique_ptr<MemoryBuffer> &MB = *MBOrErr;
   MemoryBufferRef MBRef = MB->getMemBufferRef();
   make<std::unique_ptr<MemoryBuffer>>(std::move(MB)); // take MB ownership
 
+  if (Tar)
+    Tar->append(relativeToRoot(Path), MBRef.getBuffer());
   return MBRef;
 }
 
 InputFile *lld::wasm::createObjectFile(MemoryBufferRef MB,
                                        StringRef ArchiveName) {
   file_magic Magic = identify_magic(MB.getBuffer());
   if (Magic == file_magic::wasm_object) {
     std::unique_ptr<Binary> Bin = check(createBinary(MB));
     auto *Obj = cast<WasmObjectFile>(Bin.get());
     if (Obj->isSharedObject())
       return make<SharedFile>(MB);
     return make<ObjFile>(MB, ArchiveName);
   }
 
   if (Magic == file_magic::bitcode)
     return make<BitcodeFile>(MB, ArchiveName);
-
   fatal("unknown file type: " + MB.getBufferIdentifier());
 }
 
 void ObjFile::dumpInfo() const {
   log("info for: " + toString(this) +
       "\n              Symbols : " + Twine(Symbols.size()) +
       "\n     Function Imports : " + Twine(WasmObj->getNumImportedFunctions()) +
       "\n       Global Imports : " + Twine(WasmObj->getNumImportedGlobals()) +

lld/wasm/Options.td

defm pie: B<"pie",
     "Do not create a position independent executable (default)">;
 
 defm print_gc_sections: B<"print-gc-sections",
     "List removed unused sections",
     "Do not list removed unused sections">;
 
 def relocatable: F<"relocatable">, HelpText<"Create relocatable object file">;
 
+defm reproduce: Eq<"reproduce", "Dump linker invocation and input files for debugging">;
+
 def shared: F<"shared">, HelpText<"Build a shared object">;
 
 def strip_all: F<"strip-all">, HelpText<"Strip all symbols">;
 
 def strip_debug: F<"strip-debug">, HelpText<"Strip debugging information">;
 
 def threads: F<"threads">, HelpText<"Run the linker multi-threaded">;