[DebugInfo][PDB] Don't write empty debug streams

[aganea]

Summary

Before, empty debug streams were written as 8 bytes (4 bytes signature + 4 bytes for the GlobalRefs count), see here.

With this patch, unused empty streams aren't emitted anymore. Modules now encode 65535 as an 'unused stream' value, by convention.
Also fix the * Linker * contrib section which wasn't correctly emitted previously.

As a side-effect, MinGW PDBs are now much smaller. Tested with Martin's dataset, using lld-link @response.txt /debug
libqt_plugin.pdb:

trunk	21 MB
this patch	13.3 MB

This was previously caused by the large amount of MinGW sections which create a separate module, each with its own debug stream:

                          Summary                           
============================================================
  Block Size: 4096
  Number of blocks: 5491
  Number of streams: 2151 <----------
  Signature: 382173742
  Age: 1
  GUID: {2E82C716-1297-3F14-4C4C-44205044422E}
  Features: 0x0
  Has Debug Info: true
  Has Types: true
  Has IDs: false
  Has Globals: true
  Has Publics: true
  Is incrementally linked: false
  Has conflicting types: false
  Is stripped: false
...
  Stream   13 (      8 bytes): [Module "C:\Users\aganea\Downloads\vlc-qt-plugin-repro\home\martin\fate\vlc\src\build-x86_64\modules\gui\qt\.libs\libqt_plugin_la-qt.o"]
               Blocks: [1775]
...
Mod 0003 | `C:\Users\aganea\Downloads\vlc-qt-plugin-repro\home\martin\fate\vlc\src\build-x86_64\modules\gui\qt\.libs\libqt_plugin_la-qt.o`: 
  Stream 13, 8 bytes
  
    Symbols
                                       Total:       0 entries (       0 bytes)
    --------------------------------------------------------------------------
  
    Chunks
                                       Total:       0 entries (       0 bytes)
    --------------------------------------------------------------------------

With this patch:

                          Summary                           
============================================================
  Block Size: 4096
  Number of blocks: 3397
  Number of streams: 45 <----------
  Signature: 4031447796
  Age: 1
  GUID: {F4024BF0-D475-9E33-4C4C-44205044422E}
  Features: 0x0
  Has Debug Info: true
  Has Types: true
  Has IDs: false
  Has Globals: true
  Has Publics: true
  Is incrementally linked: false
  Has conflicting types: false
  Is stripped: false
...
Mod 0003 | `C:\Users\aganea\Downloads\vlc-qt-plugin-repro\home\martin\fate\vlc\src\build-x86_64\modules\gui\qt\.libs\libqt_plugin_la-qt.o`: 
  Mod    3 (debug info not present): [C:\Users\aganea\Downloads\vlc-qt-plugin-repro\home\martin\fate\vlc\src\build-x86_64\modules\gui\qt\.libs\libqt_plugin_la-qt.o]

lld/trunk/COFF/PDB.cpp

static ArrayRef<uint8_t> relocateDebugChunk(BumpPtrAllocator &Alloc,
   assert(DebugChunk.OutputSectionOff == 0 &&
          "debug sections should not be in output sections");
   DebugChunk.readRelocTargets();
   DebugChunk.writeTo(Buffer);
   return makeArrayRef(Buffer, DebugChunk.getSize());
 }
 
 static pdb::SectionContrib createSectionContrib(const Chunk *C, uint32_t Modi) {
-  OutputSection *OS = C->getOutputSection();
+  OutputSection *OS = C ? C->getOutputSection() : nullptr;
   pdb::SectionContrib SC;
   memset(&SC, 0, sizeof(SC));
-  SC.ISect = OS->SectionIndex;
-  SC.Off = C->getRVA() - OS->getRVA();
-  SC.Size = C->getSize();
-  if (auto *SecChunk = dyn_cast<SectionChunk>(C)) {
+  SC.ISect = OS ? OS->SectionIndex : llvm::pdb::kInvalidStreamIndex;
+  SC.Off = C && OS ? C->getRVA() - OS->getRVA() : 0;
+  SC.Size = C ? C->getSize() : -1;
+  if (auto *SecChunk = dyn_cast_or_null<SectionChunk>(C)) {
     SC.Characteristics = SecChunk->Header->Characteristics;
     SC.Imod = SecChunk->File->ModuleDBI->getModuleIndex();
     ArrayRef<uint8_t> Contents = SecChunk->getContents();
     JamCRC CRC(0);
     ArrayRef<char> CharContents = makeArrayRef(
         reinterpret_cast<const char *>(Contents.data()), Contents.size());
     CRC.update(CharContents);
     SC.DataCrc = CRC.getCRC();
   } else {
-    SC.Characteristics = OS->Header.Characteristics;
+    SC.Characteristics = OS ? OS->Header.Characteristics : 0;
     // FIXME: When we start creating DBI for import libraries, use those here.
     SC.Imod = Modi;
   }
   SC.RelocCrc = 0; // FIXME
 
   return SC;
 }
 
for (OutputSection *OS : OutputSections) {
     addLinkerModuleSectionSymbol(LinkerModule, *OS, Alloc);
     for (Chunk *C : OS->Chunks) {
       pdb::SectionContrib SC =
           createSectionContrib(C, LinkerModule.getModuleIndex());
       Builder.getDbiBuilder().addSectionContrib(SC);
     }
   }
 
+  // The * Linker * first section contrib is only used along with /INCREMENTAL,
+  // to provide trampolines thunks for incremental function patching. Set this
+  // as "unused" because LLD doesn't support /INCREMENTAL link.
+  pdb::SectionContrib SC =
+      createSectionContrib(nullptr, llvm::pdb::kInvalidStreamIndex);
+  LinkerModule.setFirstSectionContrib(SC);
+
   // Add Section Map stream.
   ArrayRef<object::coff_section> Sections = {
       (const object::coff_section *)SectionTable.data(),
       SectionTable.size() / sizeof(object::coff_section)};
   SectionMap = pdb::DbiStreamBuilder::createSectionMap(Sections);
   DbiBuilder.setSectionMap(SectionMap);
 
   // Add COFF section header stream.

lld/trunk/test/COFF/pdb-global-gc.yaml

 # CHECK-NEXT:   Records
 # CHECK-NEXT:       20 | S_GDATA32 [size = 28] `__wc_mb_cur`
 # CHECK-NEXT:            type = 0x0070 (char), addr = 0000:0000
 
 # CHECK:                                Symbols
 # CHECK:      ============================================================
 # CHECK-NEXT:   Mod 0000 | `{{.*}}pdb-global-gc.yaml.tmp.obj`:
 # CHECK-NEXT:   Mod 0001 | `{{.*}}pdb-global-gc.yaml.tmp2.obj`:
+# CHECK-NEXT:   Error loading module stream 1.  The specified stream could not be loaded. Module stream not present
 # CHECK-NEXT:   Mod 0002 | `* Linker *`:
 
 --- !COFF
 header:
   Machine:         IMAGE_FILE_MACHINE_AMD64
   Characteristics: [  ]
 sections:
   - Name:            '.debug$S'

lld/trunk/test/COFF/pdb-lib.s

 # REQUIRES: x86
 # RUN: rm -rf %t && mkdir -p %t && cd %t
 # RUN: llvm-mc -filetype=obj -triple=i686-windows-msvc %s -o foo.obj
 # RUN: llc %S/Inputs/bar.ll -filetype=obj -mtriple=i686-windows-msvc -o bar.obj
 # RUN: llvm-lib bar.obj -out:bar.lib
 # RUN: lld-link -debug -pdb:foo.pdb foo.obj bar.lib -out:foo.exe -entry:main
 # RUN: llvm-pdbutil dump -modules %t/foo.pdb | FileCheck %s
 
 # Make sure that the PDB has module descriptors. foo.obj and bar.lib should be
 # absolute paths, and bar.obj should be the relative path passed to llvm-lib.
 
 # CHECK:                               Modules
 # CHECK-NEXT: ============================================================
 # CHECK-NEXT:   Mod 0000 | `{{.*pdb-lib.s.tmp[/\\]foo.obj}}`:
 # CHECK-NEXT:              Obj: `{{.*pdb-lib.s.tmp[/\\]foo.obj}}`:
-# CHECK-NEXT:              debug stream: 10, # files: 0, has ec info: false
+# CHECK-NEXT:              debug stream: 65535, # files: 0, has ec info: false
 # CHECK-NEXT:              pdb file ni: 0 ``, src file ni: 0 ``
 # CHECK-NEXT:   Mod 0001 | `bar.obj`:
 # CHECK-NEXT:              Obj: `{{.*pdb-lib.s.tmp[/\\]bar.lib}}`:
-# CHECK-NEXT:              debug stream: 11, # files: 0, has ec info: false
+# CHECK-NEXT:              debug stream: 65535, # files: 0, has ec info: false
 # CHECK-NEXT:              pdb file ni: 0 ``, src file ni: 0 ``
 # CHECK-NEXT:   Mod 0002 | `* Linker *`:
 # CHECK-NEXT:              Obj: ``:
-# CHECK-NEXT:              debug stream: 12, # files: 0, has ec info: false
+# CHECK-NEXT:              debug stream: 10, # files: 0, has ec info: false
 # CHECK-NEXT:              pdb file ni: 1 `{{.*foo.pdb}}`, src file ni: 0 ``
 
         .def     _main;
         .scl    2;
         .type   32;
         .endef
         .globl  _main
 _main:
         calll _bar
         xor %eax, %eax
         retl
 

lld/trunk/test/COFF/pdb.test

 RAW-NEXT:   Mod 0001 | `{{.*}}pdb.test.tmp2.obj`:
 RAW-NEXT:              SC[.text]  | mod = 1, 0001:0016, size = 6, data crc = 2139436471, reloc crc = 0
 RAW-NEXT:                           IMAGE_SCN_CNT_CODE | IMAGE_SCN_ALIGN_16BYTES | IMAGE_SCN_MEM_EXECUTE |
 RAW-NEXT:                           IMAGE_SCN_MEM_READ
 RAW-NEXT:              Obj: `{{.*}}pdb.test.tmp2.obj`:
 RAW-NEXT:              debug stream: 12, # files: 1, has ec info: false
 RAW-NEXT:              pdb file ni: 0 ``, src file ni: 0 ``
 RAW-NEXT:   Mod 0002 | `* Linker *`:
-RAW-NEXT:              SC[???]  | mod = 2, 0000:0000, size = 0, data crc = 0, reloc crc = 0
+RAW-NEXT:              SC[???]  | mod = 65535, 65535:0000, size = -1, data crc = 0, reloc crc = 0
 RAW-NEXT:                         none
 RAW-NEXT:              Obj: ``:
 RAW-NEXT:              debug stream: 13, # files: 0, has ec info: false
 RAW-NEXT:              pdb file ni: 1 `{{.*pdb.test.tmp.pdb}}`, src file ni: 0 ``
 RAW:                          Types (TPI Stream)
 RAW-NEXT: ============================================================
 RAW-NEXT:   Showing 5 records
 RAW-NEXT:   0x1000 | LF_ARGLIST [size = 8, hash = 0x32484]

llvm/trunk/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h

public:
   bool hasDebugSubsections() const;
 
   Error commit();
 
   Expected<codeview::DebugChecksumsSubsectionRef>
   findChecksumsSubsection() const;
 
 private:
+  Error reloadSerialize(BinaryStreamReader &Reader);
+
   DbiModuleDescriptor Mod;
 
   uint32_t Signature;
 
   std::shared_ptr<msf::MappedBlockStream> Stream;
 
   codeview::CVSymbolArray SymbolArray;
 

llvm/trunk/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp

 uint32_t DbiModuleDescriptorBuilder::calculateSerializedLength() const {
   uint32_t L = sizeof(Layout);
   uint32_t M = ModuleName.size() + 1;
   uint32_t O = ObjFileName.size() + 1;
   return alignTo(L + M + O, sizeof(uint32_t));
 }
 
 void DbiModuleDescriptorBuilder::finalize() {
-  Layout.SC.Imod = Layout.Mod;
   Layout.FileNameOffs = 0; // TODO: Fix this
   Layout.Flags = 0;        // TODO: Fix this
   Layout.C11Bytes = 0;
   Layout.C13Bytes = calculateC13DebugInfoSize();
   (void)Layout.Mod;         // Set in constructor
   (void)Layout.ModDiStream; // Set in finalizeMsfLayout
   Layout.NumFiles = SourceFiles.size();
   Layout.PdbFilePathNI = PdbFilePathNI;
   Layout.SrcFileNameNI = 0;
 
   // This value includes both the signature field as well as the record bytes
   // from the symbol stream.
-  Layout.SymBytes = SymbolByteSize + sizeof(uint32_t);
+  Layout.SymBytes =
+      Layout.ModDiStream == kInvalidStreamIndex ? 0 : getNextSymbolOffset();
 }
 
 Error DbiModuleDescriptorBuilder::finalizeMsfLayout() {
   this->Layout.ModDiStream = kInvalidStreamIndex;
   uint32_t C13Size = calculateC13DebugInfoSize();
+  if (!C13Size && !SymbolByteSize)
+    return Error::success();
   auto ExpectedSN =
       MSF.addStream(calculateDiSymbolStreamSize(SymbolByteSize, C13Size));
   if (!ExpectedSN)
     return ExpectedSN.takeError();
   Layout.ModDiStream = *ExpectedSN;
   return Error::success();
 }
 

llvm/trunk/lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp

 //===- ModuleDebugStream.cpp - PDB Module Info Stream Access --------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/DebugInfo/PDB/Native/ModuleDebugStream.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h"
 #include "llvm/DebugInfo/CodeView/SymbolDeserializer.h"
 #include "llvm/DebugInfo/CodeView/SymbolRecord.h"
 #include "llvm/DebugInfo/CodeView/SymbolRecordHelpers.h"
 #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h"
+#include "llvm/DebugInfo/PDB/Native/RawConstants.h"
 #include "llvm/DebugInfo/PDB/Native/RawError.h"
 #include "llvm/Support/BinaryStreamReader.h"
 #include "llvm/Support/BinaryStreamRef.h"
 #include "llvm/Support/Error.h"
 #include <algorithm>
 #include <cstdint>
 
 using namespace llvm;
 using namespace llvm::codeview;
 using namespace llvm::msf;
 using namespace llvm::pdb;
 
 ModuleDebugStreamRef::ModuleDebugStreamRef(
     const DbiModuleDescriptor &Module,
     std::unique_ptr<MappedBlockStream> Stream)
     : Mod(Module), Stream(std::move(Stream)) {}
 
 ModuleDebugStreamRef::~ModuleDebugStreamRef() = default;
 
 Error ModuleDebugStreamRef::reload() {
   BinaryStreamReader Reader(*Stream);
 
+  if (Mod.getModuleStreamIndex() != llvm::pdb::kInvalidStreamIndex) {
+    if (Error E = reloadSerialize(Reader))
+      return E;
+  }
+  if (Reader.bytesRemaining() > 0)
+    return make_error<RawError>(raw_error_code::corrupt_file,
+                                "Unexpected bytes in module stream.");
+  return Error::success();
+}
+
+Error ModuleDebugStreamRef::reloadSerialize(BinaryStreamReader &Reader) {
   uint32_t SymbolSize = Mod.getSymbolDebugInfoByteSize();
   uint32_t C11Size = Mod.getC11LineInfoByteSize();
   uint32_t C13Size = Mod.getC13LineInfoByteSize();
 
   if (C11Size > 0 && C13Size > 0)
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "Module has both C11 and C13 line info");
 
if (auto EC = SubsectionsReader.readArray(Subsections,
                                             SubsectionsReader.bytesRemaining()))
     return EC;
 
   uint32_t GlobalRefsSize;
   if (auto EC = Reader.readInteger(GlobalRefsSize))
     return EC;
   if (auto EC = Reader.readSubstream(GlobalRefsSubstream, GlobalRefsSize))
     return EC;
-  if (Reader.bytesRemaining() > 0)
-    return make_error<RawError>(raw_error_code::corrupt_file,
-                                "Unexpected bytes in module stream.");
-
   return Error::success();
 }
 
 const codeview::CVSymbolArray
 ModuleDebugStreamRef::getSymbolArrayForScope(uint32_t ScopeBegin) const {
   return limitSymbolArrayToScope(SymbolArray, ScopeBegin);
 }