[lld-link] Add /reproduce: support for several flags

[thakis]

Summary

/reproduce: now works correctly with:

• /call-graph-ordering-file:
• /def:
• /natvis:
• /order:
• /pdbstream:

I went through all instances of MemoryBuffer::getFile() and made sure
everything that didn't already do so called takeBuffer().

For natvis, that wasn't possible since DebugInfo/PDB wants to take
owernship of the natvis buffer. For that case, I'm manually adding the
tar file entry.

/natvis: and /pdbstream: is slightly awkward, since createResponseFile()
always adds these flags to the response file but createPDB() (which
ultimately adds the files referenced by the flags) is only called if
/debug is also passed. So when using /natvis: without /debug with
/reproduce:, lld won't warn, but when linking using the response
file from the archive, it won't find the natvis file since it's not
in the tar. This isn't a new issue though, and after this patch things
at least work with using /natvis: _with_ debug with /reproduce:.
(Same for /pdbstream:)

[thakis]

add missing trailing backslash in one RUN line

[rnk]

lgtm

lld/COFF/Driver.cpp

633

(nit) Descriptive names can be more helpful than comments. The comment is fine, but how about renaming to createReproducerRspFile or createRspFileForReproduce?

[thakis]

Thanks!

lld/COFF/Driver.cpp

633

The ELF and MachO ports also have the same comment. I can rename all versions of these functions and remove the comments in a follow-up :)

[thakis]

Not sure why phab managed to add a link to https://reviews.llvm.org/rGe6d1f261a5a08d4c05b25509a6f4217a8a1c2188 but then didn't close this. Anyways, closed!

[thakis]

Aha, because I typod "Revision" as "Revison".

lld/COFF/Driver.h

void enqueueArchiveMember(const Archive::Child &c, const Archive::Symbol &sym,
                             StringRef parentName);
 
   void enqueuePDB(StringRef Path) { enqueuePath(Path, false, false); }
 
   MemoryBufferRef takeBuffer(std::unique_ptr<MemoryBuffer> mb);
 
   void enqueuePath(StringRef path, bool wholeArchive, bool lazy);
 
-private:
   std::unique_ptr<llvm::TarWriter> tar; // for /linkrepro
 
+private:
   // Searches a file from search paths.
   Optional<StringRef> findFile(StringRef filename);
   Optional<StringRef> findLib(StringRef filename);
   StringRef doFindFile(StringRef filename);
   StringRef doFindLib(StringRef filename);
   StringRef doFindLibMinGW(StringRef filename);
 
   // Parses LIB environment which contains a list of search paths.

lld/COFF/Driver.cpp

 #include "llvm/ToolDrivers/llvm-lib/LibDriver.h"
 #include <algorithm>
 #include <future>
 #include <memory>
 
 using namespace llvm;
 using namespace llvm::object;
 using namespace llvm::COFF;
-using llvm::sys::Process;
+using namespace llvm::sys;
 
 namespace lld {
 namespace coff {
 
 static Timer inputFileTimer("Input File Reading", Timer::root());
 
 Configuration *config;
 LinkerDriver *driver;

 }
 
 static uint64_t getDefaultImageBase() {
   if (config->is64())
     return config->dll ? 0x180000000 : 0x140000000;
   return config->dll ? 0x10000000 : 0x400000;
 }
 
+static std::string rewritePath(StringRef s) {
+  if (fs::exists(s))
+    return relativeToRoot(s);
+  return std::string(s);
+}
+
+// Reconstructs command line arguments so that so that you can re-run
+// the same command with the same inputs. This is for --reproduce.
 static std::string createResponseFile(const opt::InputArgList &args,
                                       ArrayRef<StringRef> filePaths,
                                       ArrayRef<StringRef> searchPaths) {
   SmallString<0> data;
   raw_svector_ostream os(data);
 
   for (auto *arg : args) {
     switch (arg->getOption().getID()) {
     case OPT_linkrepro:
     case OPT_reproduce:
     case OPT_INPUT:
     case OPT_defaultlib:
     case OPT_libpath:
     case OPT_manifest:
     case OPT_manifest_colon:
     case OPT_manifestdependency:
     case OPT_manifestfile:
     case OPT_manifestinput:
     case OPT_manifestuac:
       break;
+    case OPT_call_graph_ordering_file:
+    case OPT_deffile:
+    case OPT_natvis:
+      os << arg->getSpelling() << quote(rewritePath(arg->getValue())) << '\n';
+      break;
+    case OPT_order: {
+      StringRef orderFile = arg->getValue();
+      orderFile.consume_front("@");
+      os << arg->getSpelling() << '@' << quote(rewritePath(orderFile)) << '\n';
+      break;
+    }
+    case OPT_pdbstream: {
+      const std::pair<StringRef, StringRef> nameFile =
+          StringRef(arg->getValue()).split("=");
+      os << arg->getSpelling() << nameFile.first << '='
+         << quote(rewritePath(nameFile.second)) << '\n';
+      break;
+    }
     case OPT_implib:
     case OPT_pdb:
     case OPT_pdbstripped:
     case OPT_out:
       os << arg->getSpelling() << sys::path::filename(arg->getValue()) << "\n";
       break;
     default:
       os << toString(*arg) << "\n";

 }
 
 static void parseModuleDefs(StringRef path) {
   std::unique_ptr<MemoryBuffer> mb = CHECK(
       MemoryBuffer::getFile(path, -1, false, true), "could not open " + path);
   COFFModuleDefinition m = check(parseCOFFModuleDefinition(
       mb->getMemBufferRef(), config->machine, config->mingw));
 
+  // Include in /reproduce: output if applicable.
+  driver->takeBuffer(std::move(mb));
+
   if (config->outputFile.empty())
     config->outputFile = std::string(saver.save(m.OutputFile));
   config->importName = std::string(saver.save(m.ImportName));
   if (m.ImageBase)
     config->imageBase = m.ImageBase;
   if (m.StackReserve)
     config->stackReserve = m.StackReserve;
   if (m.StackCommit)
for (StringRef arg : args::getLines(mb->getMemBufferRef())) {
 
     if (set.count(s) == 0) {
       if (config->warnMissingOrderSymbol)
         warn("/order:" + arg + ": missing symbol: " + s + " [LNK4037]");
     }
     else
       config->order[s] = INT_MIN + config->order.size();
   }
+
+  // Include in /reproduce: output if applicable.
+  driver->takeBuffer(std::move(mb));
 }
 
 static void parseCallGraphFile(StringRef path) {
   std::unique_ptr<MemoryBuffer> mb = CHECK(
       MemoryBuffer::getFile(path, -1, false, true), "could not open " + path);
 
   // Build a map from symbol name to section.
   DenseMap<StringRef, Symbol *> map;
if (fields.size() != 3 || !to_integer(fields[2], count)) {
       error(path + ": parse error");
       return;
     }
 
     if (SectionChunk *from = findSection(fields[0]))
       if (SectionChunk *to = findSection(fields[1]))
         config->callGraphProfile[{from, to}] += count;
   }
+
+  // Include in /reproduce: output if applicable.
+  driver->takeBuffer(std::move(mb));
 }
 
 static void readCallGraphsFromObjectFiles() {
   for (ObjFile *obj : ObjFile::instances) {
     if (obj->callgraphSec) {
       ArrayRef<uint8_t> contents;
       cantFail(
           obj->getCOFFObj()->getSectionContents(obj->callgraphSec, contents));

lld/COFF/PDB.cpp

 void PDBLinker::addNatvisFiles() {
   for (StringRef file : config->natvisFiles) {
     ErrorOr<std::unique_ptr<MemoryBuffer>> dataOrErr =
         MemoryBuffer::getFile(file);
     if (!dataOrErr) {
       warn("Cannot open input file: " + file);
       continue;
     }
-    builder.addInjectedSource(file, std::move(*dataOrErr));
+    std::unique_ptr<MemoryBuffer> data = std::move(*dataOrErr);
+
+    // Can't use takeBuffer() here since addInjectedSource() takes ownership.
+    if (driver->tar)
+      driver->tar->append(relativeToRoot(data->getBufferIdentifier()),
+                          data->getBuffer());
+
+    builder.addInjectedSource(file, std::move(data));
   }
 }
 
 void PDBLinker::addNamedStreams() {
   for (const auto &streamFile : config->namedStreams) {
     const StringRef stream = streamFile.getKey(), file = streamFile.getValue();
     ErrorOr<std::unique_ptr<MemoryBuffer>> dataOrErr =
         MemoryBuffer::getFile(file);
     if (!dataOrErr) {
       warn("Cannot open input file: " + file);
       continue;
     }
-    exitOnErr(builder.addNamedStream(stream, (*dataOrErr)->getBuffer()));
+    std::unique_ptr<MemoryBuffer> data = std::move(*dataOrErr);
+    exitOnErr(builder.addNamedStream(stream, data->getBuffer()));
+    driver->takeBuffer(std::move(data));
   }
 }
 
 static codeview::CPUType toCodeViewMachine(COFF::MachineTypes machine) {
   switch (machine) {
   case COFF::IMAGE_FILE_MACHINE_AMD64:
     return codeview::CPUType::X64;
   case COFF::IMAGE_FILE_MACHINE_ARM:

lld/test/COFF/linkrepro-manifest.test

 REQUIRES: x86, gnutar, manifest_tool
 
+manifest-related files are compiled to a .res file and the .res file is
+added to the repro archive, instead of adding the inputs.
+
 RUN: rm -rf %t && mkdir %t && cd %t
-RUN: lld-link -entry:__ImageBase -nodefaultlib -linkrepro:%t -manifest:embed %p/Inputs/std32.lib -subsystem:console
+RUN: lld-link -entry:__ImageBase -nodefaultlib -linkrepro:%t \
+RUN:     -manifest:embed %p/Inputs/std32.lib -subsystem:console \
+RUN:     -manifestinput:%p/Inputs/manifestinput.test
+
 RUN: tar tf repro.tar | FileCheck --check-prefix=LIST %s
 RUN: tar xOf repro.tar repro/response.txt | FileCheck %s
 
 LIST: manifest.res
 
 CHECK-NOT: -manifest:
 CHECK: .manifest.res
 CHECK-NOT: -manifest:

lld/test/COFF/linkrepro-pdb.test

 REQUIRES: x86, gnutar
 
 RUN: rm -rf %t && mkdir -p %t && cd %t
 RUN: yaml2obj %S/Inputs/pdb-type-server-simple-a.yaml -o a.obj
 RUN: yaml2obj %S/Inputs/pdb-type-server-simple-b.yaml -o b.obj
 RUN: llvm-pdbutil yaml2pdb %S/Inputs/pdb-type-server-simple-ts.yaml -pdb ts.pdb
-RUN: lld-link a.obj b.obj -entry:main -debug -out:%t.exe -pdb:%t.pdb -nodefaultlib -linkrepro:.
-RUN: tar xOf repro.tar repro/%:t/ts.pdb > repro-ts.pdb
-RUN: diff ts.pdb repro-ts.pdb
-
+RUN: cp %p/Inputs/natvis-1.natvis %t.natvis
+RUN: cp %p/Inputs/stream.txt %t.txt
+RUN: lld-link a.obj b.obj -entry:main -debug -out:%t.exe -pdb:%t.pdb \
+RUN:     -nodefaultlib -linkrepro:. -natvis:%t.natvis \
+RUN:     -pdbstream:srcsrv=%t.txt
+RUN: tar tf repro.tar | FileCheck --check-prefix=LIST %s
 RUN: tar xf repro.tar
-RUN: cat repro/response.txt | FileCheck -check-prefix=PDB %s
+RUN: diff ts.pdb repro/%:t/ts.pdb
+RUN: diff %t.natvis repro/%:t.natvis
+RUN: diff %t.txt repro/%:t.txt
+RUN: cat repro/response.txt | FileCheck -check-prefix=RSP %s
+
+LIST: .obj
+LIST: response.txt
+LIST: .natvis
 
-PDB: -out:linkrepro-pdb.test.tmp.exe
-PDB-NEXT: -pdb:linkrepro-pdb.test.tmp.pdb
+RSP: -out:linkrepro-pdb.test.tmp.exe
+RSP-NEXT: -pdb:linkrepro-pdb.test.tmp.pdb
+RSP-NEXT: -nodefaultlib
+RSP-NOT: -natvis:/
+RSP-NOT: -pdbstream:srcsrv=/
 
 RUN: yaml2obj %p/Inputs/export.yaml -o %t1.obj
-RUN: lld-link /out:%t1.dll /dll %t1.obj /implib:%t1.lib /export:exportfn1 /export:exportfn2 /linkrepro:.
-RUN: tar xf repro.tar
-RUN: cat repro/response.txt | FileCheck -check-prefix=IMP %s
+RUN: lld-link /out:%t1.dll /dll %t1.obj /implib:%t1.lib /export:exportfn1 /export:exportfn2 /reproduce:repro2.tar
+RUN: tar xf repro2.tar
+RUN: cat repro2/response.txt | FileCheck -check-prefix=IMP %s
 
 IMP: /implib:linkrepro-pdb.test.tmp1.lib

lld/test/COFF/linkrepro.test

 # REQUIRES: x86, shell
 
 # RUN: rm -rf %t.dir
-# RUN: mkdir -p %t.dir/build1 %t.dir/build2 %t.dir/build3 %t.dir/build4
 # RUN: yaml2obj %p/Inputs/hello32.yaml -o %t.obj
+# RUN: echo '_main@0' > %t.order
+# RUN: touch %t.def
+# RUN: touch %t.cg
 
+Test link.exe-style /linkrepro: flag.
+# RUN: mkdir -p %t.dir/build1
 # RUN: cd %t.dir/build1
 # RUN: lld-link %t.obj %p/Inputs/std32.lib /subsystem:console \
 # RUN:   /entry:main@0 /linkrepro:. /out:%t.exe
 # RUN: tar xf repro.tar
 # RUN: diff %t.obj repro/%:t.obj
 # RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro/response.txt
 
-# RUN: cd %t.dir/build1
+Test lld-style /reproduce: flag.
 # RUN: lld-link %t.obj %p/Inputs/std32.lib /subsystem:console \
 # RUN:   /entry:main@0 /reproduce:repro2.tar /out:%t.exe
 # RUN: tar xf repro2.tar
 # RUN: diff %t.obj repro2/%:t.obj
 # RUN: diff %p/Inputs/std32.lib repro2/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro2/response.txt
 
+Test LLD_REPRODUCE env var.
+# RUN: mkdir -p %t.dir/build2
 # RUN: cd %t.dir/build2
 # RUN: env LLD_REPRODUCE=repro.tar lld-link %t.obj %p/Inputs/std32.lib \
 # RUN:    /subsystem:console /entry:main@0 /out:%t.exe
 # RUN: tar xf repro.tar
 # RUN: diff %t.obj repro/%:t.obj
 # RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro/response.txt
 
+Test adding .lib files with /libpath: to repro archive,
+and various other flags.
+# RUN: mkdir -p %t.dir/build3
 # RUN: cd %t.dir/build3
 # RUN: lld-link %t.obj /libpath:%p/Inputs /defaultlib:std32 /subsystem:console \
-# RUN:   /entry:main@0 /linkrepro:. /out:%t.exe
+# RUN:   /entry:main@0 /linkrepro:. /out:%t.exe /order:@%t.order /def:%t.def
+# RUN: tar tf repro.tar | FileCheck --check-prefix=LIST %s
 # RUN: tar xf repro.tar
 # RUN: diff %t.obj repro/%:t.obj
+# RUN: diff %t.order repro/%:t.order
+# RUN: diff %t.def repro/%:t.def
 # RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro/response.txt
+# RUN: cd repro; lld-link @response.txt
 
+Test adding .lib files with LIB env var to repro archive,
+and various other flags.
+# RUN: mkdir -p %t.dir/build4
 # RUN: cd %t.dir/build4
 # RUN: env LIB=%p/Inputs lld-link %t.obj /defaultlib:std32 /subsystem:console \
-# RUN:   /entry:main@0 /linkrepro:. /out:%t.exe
+# RUN:   /entry:main@0 /linkrepro:. /out:%t.exe /order:@%t.order /def:%t.def
+# RUN: tar tf repro.tar | FileCheck --check-prefix=LIST %s
 # RUN: tar xf repro.tar
 # RUN: diff %t.obj repro/%:t.obj
+# RUN: diff %t.order repro/%:t.order
+# RUN: diff %t.def repro/%:t.def
 # RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro/response.txt
+# RUN: cd repro; lld-link @response.txt
+
+# LIST: .obj
+# LIST: std32.lib
+# LIST: response.txt
+# LIST: .def
+# LIST: .order
 
 # RSP: /subsystem:console
 # RSP: /entry:main@0
-# RSP-NOT: /linkrepro:
 # RSP: /out:
+# RSP-NOT: /order:@/
+# RSP-NOT: /def:/
 # RSP: linkrepro.test.tmp.obj
 # RSP-NOT: defaultlib
 # RSP: std32.lib
+
+Test /call-graph-ordering-file (can't be used with /order:, needs separate test)
+# RUN: mkdir -p %t.dir/build5
+# RUN: cd %t.dir/build5
+# RUN: lld-link %t.obj %p/Inputs/std32.lib /subsystem:console \
+# RUN:     /entry:main@0 /linkrepro:. /out:%t.exe /call-graph-ordering-file:%t.cg
+# RUN: tar tf repro.tar | FileCheck --check-prefix=LISTCG %s
+# RUN: tar xf repro.tar
+# RUN: diff %t.obj repro/%:t.obj
+# RUN: diff %t.cg repro/%:t.cg
+# RUN: FileCheck %s --check-prefix=RSPCG < repro/response.txt
+# RUN: cd repro; lld-link @response.txt
+
+# LISTCG: .obj
+# LISTCG: response.txt
+# LISTCG: .cg
+
+# RSPCG-NOT: /call-graph-ordering-file:/