[ms] Add new /PDBSTREAM option to lld-link allowing injection of streams into PDB files.

[epastor]

Summary

/PDBSTREAM:<name>=<file> adds the contents of <file> to stream <name> in the resulting PDB.

This allows native uses with workflows that (for example) add srcsrv streams to PDB files to provide a location for the build's source files.

Results should be equivalent to linking with lld-link, then running Microsoft's pdbstr tool with the command line:
pdbstr.exe -w -p:<PDB LOCATION> -s:<name> -i:<file>
except in cases where the named stream overlaps with a default named stream, such as "/names". In those cases, the added stream will be overridden, making the /pdbstream option a no-op.

[thakis]

Seems reasonable. Can you add a test?

lld/COFF/Driver.cpp

1281

Should this do anything if you try to overwrite an existing stream? (Like, error?)

[epastor]

Add test

lld/COFF/Driver.cpp

1281

I've mimicked other command line flags; the last one with the same name will take precedence.

[thakis]

lld/COFF/Driver.cpp

1281

I mean, what if this conflicts with a "built-in" pdb named stream (natvis, or what have you)?

[epastor]

lld/COFF/Driver.cpp

1281

Currently, the "built-in" named stream will win. I think that's probably an acceptable outcome.

lld/COFF/Config.h

struct Configuration {
   bool debugSymtab = false;
   bool driver = false;
   bool driverUponly = false;
   bool driverWdm = false;
   bool showTiming = false;
   bool showSummary = false;
   unsigned debugTypes = static_cast<unsigned>(DebugType::None);
   std::vector<std::string> natvisFiles;
+  llvm::StringMap<std::string> namedStreams;
   llvm::SmallString<128> pdbAltPath;
   llvm::SmallString<128> pdbPath;
   llvm::SmallString<128> pdbSourcePath;
   std::vector<llvm::StringRef> argv;
 
   // Symbols in this set are considered as live by the garbage collector.
   std::vector<Symbol *> gcroot;
 

lld/COFF/Driver.cpp

bool shouldCreatePDB =
       (debug == DebugKind::Full || debug == DebugKind::GHash);
   if (shouldCreatePDB) {
     if (auto *arg = args.getLastArg(OPT_pdb))
       config->pdbPath = arg->getValue();
     if (auto *arg = args.getLastArg(OPT_pdbaltpath))
       config->pdbAltPath = arg->getValue();
     if (args.hasArg(OPT_natvis))
       config->natvisFiles = args.getAllArgValues(OPT_natvis);
+    if (args.hasArg(OPT_pdbstream)) {
+      for (const StringRef value : args.getAllArgValues(OPT_pdbstream)) {
+        const std::pair<StringRef, StringRef> nameFile = value.split("=");
+        const StringRef name = nameFile.first;
+        const std::string file = nameFile.second.str();
+        config->namedStreams[name] = file;
+      }
+    }
 
     if (auto *arg = args.getLastArg(OPT_pdb_source_path))
       config->pdbSourcePath = arg->getValue();
   }
 
   // Handle /pdbstripped
   if (args.hasArg(OPT_pdbstripped))
     warn("ignoring /pdbstripped flag, it is not yet supported");

lld/COFF/Options.td

 def order   : P<"order", "Put functions in order">;
 def out     : P<"out", "Path to file to write output">;
 def natvis : P<"natvis", "Path to natvis file to embed in the PDB">;
 def no_color_diagnostics: F<"no-color-diagnostics">,
     HelpText<"Do not use colors in diagnostics">;
 def pdb : P<"pdb", "PDB file path">;
 def pdbstripped : P<"pdbstripped", "Stripped PDB file path">;
 def pdbaltpath : P<"pdbaltpath", "PDB file path to embed in the image">;
+def pdbstream : Joined<["/", "-", "/?", "-?"], "pdbstream:">,
+    MetaVarName<"<name>=<file>">,
+    HelpText<"Embed the contents of <file> in the PDB as named stream <name>">;
 def section : P<"section", "Specify section attributes">;
 def stack   : P<"stack", "Size of the stack">;
 def stub    : P<"stub", "Specify DOS stub file">;
 def subsystem : P<"subsystem", "Specify subsystem">;
 def timestamp : P<"timestamp", "Specify the PE header timestamp">;
 def version : P<"version", "Specify a version number in the PE header">;
 def wholearchive_file : P<"wholearchive",
     "Include all object files from this library">;

lld/COFF/PDB.cpp

public:
   }
 
   /// Emit the basic PDB structure: initial streams, headers, etc.
   void initialize(llvm::codeview::DebugInfo *buildId);
 
   /// Add natvis files specified on the command line.
   void addNatvisFiles();
 
+  /// Add named streams specified on the command line.
+  void addNamedStreams();
+
   /// Link CodeView from each object file in the symbol table into the PDB.
   void addObjectsToPDB();
 
   /// Link info for each import file in the symbol table into the PDB.
   void addImportFilesToPDB(ArrayRef<OutputSection *> outputSections);
 
   /// Link CodeView from a single object file into the target (output) PDB.
   /// When a precompiled headers object is linked, its TPI map might be provided
for (StringRef file : config->natvisFiles) {
     if (!dataOrErr) {
       warn("Cannot open input file: " + file);
       continue;
     }
     builder.addInjectedSource(file, std::move(*dataOrErr));
   }
 }
 
+void PDBLinker::addNamedStreams() {
+  for (const auto &streamFile : config->namedStreams) {
+    const StringRef stream = streamFile.getKey(), file = streamFile.getValue();
+    ErrorOr<std::unique_ptr<MemoryBuffer>> dataOrErr =
+        MemoryBuffer::getFile(file);
+    if (!dataOrErr) {
+      warn("Cannot open input file: " + file);
+      continue;
+    }
+    exitOnErr(builder.addNamedStream(stream, (*dataOrErr)->getBuffer()));
+  }
+}
+
 static codeview::CPUType toCodeViewMachine(COFF::MachineTypes machine) {
   switch (machine) {
   case COFF::IMAGE_FILE_MACHINE_AMD64:
     return codeview::CPUType::X64;
   case COFF::IMAGE_FILE_MACHINE_ARM:
     return codeview::CPUType::ARM7;
   case COFF::IMAGE_FILE_MACHINE_ARM64:
     return codeview::CPUType::ARM64;
void lld::coff::createPDB(SymbolTable *symtab,
   ScopedTimer t1(totalPdbLinkTimer);
   PDBLinker pdb(symtab);
 
   pdb.initialize(buildId);
   pdb.addObjectsToPDB();
   pdb.addImportFilesToPDB(outputSections);
   pdb.addSections(outputSections, sectionTable);
   pdb.addNatvisFiles();
+  pdb.addNamedStreams();
 
   ScopedTimer t2(diskCommitTimer);
   codeview::GUID guid;
   pdb.commit(&guid);
   memcpy(&buildId->PDB70.Signature, &guid, 16);
 
   t2.stop();
   t1.stop();

lld/test/COFF/Inputs/stream.txt

+Stream contents
+Line 2
+Line 3

lld/test/COFF/pdbstream.test

+# RUN: yaml2obj %p/Inputs/empty.yaml > %t.obj
+
+# RUN: lld-link /entry:main %t.obj /out:%t.exe /debug /pdbstream:srcsrv=%p/Inputs/stream.txt
+# RUN: llvm-pdbutil export -stream=srcsrv -out=- %t.pdb | FileCheck %s
+
+CHECK: Stream contents
+CHECK-NEXT: Line 2
+CHECK-NEXT: Line 3