[pdb/lld] Add an empty globals stream

[zturner]

Summary

This is similar to the patch for adding an empty publics stream, but this patch adds an empty globals stream. With this we are outputting at least something for every stream that MSVC outputs, even if our stream is empty

[zturner]

Note that this still doesn't get our PDBs working with MSVC. next steps:

1. try to figure out what to write for the various fields of th Publics stream header, as well as drilling into symbol streams and comparing record for record.
2. Write actual FPO data to the fpo stream
3. Write something for section headers and section map

[rnk]

lgtm

lld/trunk/COFF/PDB.cpp

if (!Publics.empty()) {
     std::sort(Publics.begin(), Publics.end(),
               [](const PublicSym32 &L, const PublicSym32 &R) {
                 return L.Name < R.Name;
               });
     auto &PublicsBuilder = Builder.getPublicsBuilder();
     for (const PublicSym32 &Pub : Publics)
       PublicsBuilder.addPublicSymbol(Pub);
   }
+  // Add globals stream.  For now we don't actually write any thing useful to
+  // the globals stream, but the act of "getting" it also creates it lazily so
+  // that we write an empty stream.
+  (void)Builder.getGlobalsBuilder();
 }
 
 static void addLinkerModuleSymbols(StringRef Path,
                                    pdb::DbiModuleDescriptorBuilder &Mod,
                                    BumpPtrAllocator &Allocator) {
   codeview::SymbolSerializer Serializer(Allocator, CodeViewContainer::Pdb);
   codeview::ObjNameSym ONS(SymbolRecordKind::ObjNameSym);
   codeview::Compile3Sym CS(SymbolRecordKind::Compile3Sym);

lld/trunk/test/COFF/pdb-diff.test

 CHECK-NEXT:   |----------------+---|
 CHECK-NEXT:   | Directory Size |
 CHECK-NEXT:   |----------------+---|
 CHECK-NEXT:   ------------------------------------
 CHECK-NEXT:   |         Stream Directory         |
 CHECK-NEXT:   |------------------------------+---|
 CHECK-NEXT:   |                         File |   |
 CHECK-NEXT:   |------------------------------+---|
-CHECK-NEXT:   |                 Stream Count | D |
+CHECK-NEXT:   |                 Stream Count | I |
 CHECK-NEXT:   |------------------------------+---|
 CHECK-NEXT:   |            Old MSF Directory | I |
 CHECK-NEXT:   |------------------------------+---|
 CHECK-NEXT:   |                   PDB Stream | I |
 CHECK-NEXT:   |------------------------------+---|
 CHECK-NEXT:   |                   TPI Stream | I |
 CHECK-NEXT:   |------------------------------+---|
 CHECK-NEXT:   |                   DBI Stream | I |

 CHECK-NEXT:   |                     TPI Hash | {{[EI]}} |
 CHECK-NEXT:   |------------------------------+---|
 CHECK-NEXT:   |                     IPI Hash | {{[EI]}} |
 CHECK-NEXT:   |------------------------------+---|
 CHECK-NEXT:   |           Public Symbol Hash | {{[EI]}} |
 CHECK-NEXT:   |------------------------------+---|
 CHECK-NEXT:   |        Public Symbol Records | {{[EI]}} |
 CHECK-NEXT:   |------------------------------+---|
-CHECK-NEXT:   |           Global Symbol Hash | D |
+CHECK-NEXT:   |           Global Symbol Hash | {{[EI]}} |
 CHECK-NEXT:   |------------------------------+---|
 CHECK-NEXT:   ------------------------------------
 CHECK-NEXT:   |           String Table           |
 CHECK-NEXT:   |------------------------------+---|
 CHECK-NEXT:   |                         File |   |
 CHECK-NEXT:   |------------------------------+---|
 CHECK-NEXT:   |            Number of Strings | D |
 CHECK-NEXT:   |------------------------------+---|

 CHECK-NEXT:   |                            DBG (Xdata) | I |
 CHECK-NEXT:   |----------------------------------------+---|
 CHECK-NEXT:   |                            DBG (Pdata) | I |
 CHECK-NEXT:   |----------------------------------------+---|
 CHECK-NEXT:   |                           DBG (NewFPO) | {{[EI]}} |
 CHECK-NEXT:   |----------------------------------------+---|
 CHECK-NEXT:   |                   DBG (SectionHdrOrig) | I |
 CHECK-NEXT:   |----------------------------------------+---|
-CHECK-NEXT:   |                         Globals Stream | D |
+CHECK-NEXT:   |                         Globals Stream | {{[EI]}} |
 CHECK-NEXT:   |----------------------------------------+---|
 CHECK-NEXT:   |                         Publics Stream | {{[EI]}} |
 CHECK-NEXT:   |----------------------------------------+---|
 CHECK-NEXT:   |                         Symbol Records | {{[EI]}} |
 CHECK-NEXT:   |----------------------------------------+---|
 CHECK-NEXT:   |                             Has CTypes | I |
 CHECK-NEXT:   |----------------------------------------+---|
 CHECK-NEXT:   |                Is Incrementally Linked | D |

 CHECK-NEXT:   |                        - C13 Byte Size | I |
 CHECK-NEXT:   |----------------------------------------+---|
 CHECK-NEXT:   |                           - # of files | I |
 CHECK-NEXT:   |----------------------------------------+---|
 CHECK-NEXT:   |                  - Pdb File Path Index | I |
 CHECK-NEXT:   |----------------------------------------+---|
 CHECK-NEXT:   |               - Source File Name Index | I |
 CHECK-NEXT:   |----------------------------------------+---|
-CHECK-NEXT:   |                     - Symbol Byte Size | D |
+CHECK-NEXT:   |                     - Symbol Byte Size |
 CHECK-NEXT:   |----------------------------------------+---|
 CHECK-NEXT:   |            Module "* Linker *"             |
 CHECK-NEXT:   |----------------------------------------+---|
 CHECK-NEXT:   |                                 - Modi | I |
 CHECK-NEXT:   |----------------------------------------+---|
 CHECK-NEXT:   |                        - Obj File Name | I |
 CHECK-NEXT:   |----------------------------------------+---|
 CHECK-NEXT:   |                         - Debug Stream | {{[EI]}} |

llvm/trunk/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h

public:
 
   // Add given bytes as a new stream.
   Error addDbgStream(pdb::DbgHeaderType Type, ArrayRef<uint8_t> Data);
 
   uint32_t addECName(StringRef Name);
 
   uint32_t calculateSerializedLength() const;
 
+  void setGlobalsStreamIndex(uint32_t Index);
   void setPublicsStreamIndex(uint32_t Index);
   void setSymbolRecordStreamIndex(uint32_t Index);
 
   Expected<DbiModuleDescriptorBuilder &> addModuleInfo(StringRef ModuleName);
   Error addModuleSourceFile(StringRef Module, StringRef File);
   Error addModuleSourceFile(DbiModuleDescriptorBuilder &Module, StringRef File);
   Expected<uint32_t> getSourceFileNameIndex(StringRef FileName);
 
private:
 
   Optional<PdbRaw_DbiVer> VerHeader;
   uint32_t Age;
   uint16_t BuildNumber;
   uint16_t PdbDllVersion;
   uint16_t PdbDllRbld;
   uint16_t Flags;
   PDB_Machine MachineType;
+  uint32_t GlobalsStreamIndex = kInvalidStreamIndex;
   uint32_t PublicsStreamIndex = kInvalidStreamIndex;
   uint32_t SymRecordStreamIndex = kInvalidStreamIndex;
 
   const DbiStreamHeader *Header;
 
   StringMap<std::unique_ptr<DbiModuleDescriptorBuilder>> ModiMap;
   std::vector<DbiModuleDescriptorBuilder *> ModiList;
 

llvm/trunk/include/llvm/DebugInfo/PDB/Native/GlobalsStreamBuilder.h

+//===- GlobalsStreamBuilder.h - PDB Globals Stream Creation -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_PDB_RAW_PDBGLOBALSTREAMBUILDER_H
+#define LLVM_DEBUGINFO_PDB_RAW_PDBGLOBALSTREAMBUILDER_H
+
+#include "llvm/DebugInfo/PDB/Native/RawConstants.h"
+#include "llvm/DebugInfo/PDB/Native/RawTypes.h"
+#include "llvm/Support/BinaryByteStream.h"
+#include "llvm/Support/BinaryStreamRef.h"
+#include "llvm/Support/BinaryStreamWriter.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace msf {
+class MSFBuilder;
+}
+namespace pdb {
+class GlobalsStream;
+
+class GlobalsStreamBuilder {
+public:
+  explicit GlobalsStreamBuilder(msf::MSFBuilder &Msf);
+  ~GlobalsStreamBuilder();
+
+  GlobalsStreamBuilder(const GlobalsStreamBuilder &) = delete;
+  GlobalsStreamBuilder &operator=(const GlobalsStreamBuilder &) = delete;
+
+  Error finalizeMsfLayout();
+  uint32_t calculateSerializedLength() const;
+
+  Error commit(BinaryStreamWriter &PublicsWriter);
+
+  uint32_t getStreamIndex() const { return StreamIdx; }
+
+private:
+  uint32_t StreamIdx = kInvalidStreamIndex;
+  msf::MSFBuilder &Msf;
+};
+} // namespace pdb
+} // namespace llvm
+
+#endif

llvm/trunk/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h

 namespace llvm {
 namespace msf {
 class MSFBuilder;
 }
 namespace pdb {
 class DbiStreamBuilder;
 class InfoStreamBuilder;
 class PublicsStreamBuilder;
+class GlobalsStreamBuilder;
 class TpiStreamBuilder;
 
 class PDBFileBuilder {
 public:
   explicit PDBFileBuilder(BumpPtrAllocator &Allocator);
   ~PDBFileBuilder();
   PDBFileBuilder(const PDBFileBuilder &) = delete;
   PDBFileBuilder &operator=(const PDBFileBuilder &) = delete;
 
   Error initialize(uint32_t BlockSize);
 
   msf::MSFBuilder &getMsfBuilder();
   InfoStreamBuilder &getInfoBuilder();
   DbiStreamBuilder &getDbiBuilder();
   TpiStreamBuilder &getTpiBuilder();
   TpiStreamBuilder &getIpiBuilder();
   PDBStringTableBuilder &getStringTableBuilder();
   PublicsStreamBuilder &getPublicsBuilder();
+  GlobalsStreamBuilder &getGlobalsBuilder();
 
   Error commit(StringRef Filename);
 
   Expected<uint32_t> getNamedStreamIndex(StringRef Name) const;
   Error addNamedStream(StringRef Name, uint32_t Size);
 
 private:
   Expected<msf::MSFLayout> finalizeMsfLayout();
 
   BumpPtrAllocator &Allocator;
 
   std::unique_ptr<msf::MSFBuilder> Msf;
   std::unique_ptr<InfoStreamBuilder> Info;
   std::unique_ptr<DbiStreamBuilder> Dbi;
+  std::unique_ptr<GlobalsStreamBuilder> Globals;
   std::unique_ptr<PublicsStreamBuilder> Publics;
   std::unique_ptr<TpiStreamBuilder> Tpi;
   std::unique_ptr<TpiStreamBuilder> Ipi;
 
   PDBStringTableBuilder Strings;
   NamedStreamMap NamedStreams;
 };
 }
 }
 
 #endif

llvm/trunk/lib/DebugInfo/PDB/CMakeLists.txt

 add_pdb_impl_folder(Native
   Native/DbiModuleDescriptor.cpp
   Native/DbiModuleDescriptorBuilder.cpp
   Native/DbiModuleList.cpp
   Native/DbiStream.cpp
   Native/DbiStreamBuilder.cpp
   Native/EnumTables.cpp
   Native/GlobalsStream.cpp
+  Native/GlobalsStreamBuilder.cpp
   Native/Hash.cpp
   Native/HashTable.cpp
   Native/InfoStream.cpp
   Native/InfoStreamBuilder.cpp
   Native/ModuleDebugStream.cpp
   Native/NativeBuiltinSymbol.cpp
   Native/NativeCompilandSymbol.cpp
   Native/NativeEnumModules.cpp

llvm/trunk/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp

 void DbiStreamBuilder::setFlags(uint16_t F) { Flags = F; }
 
 void DbiStreamBuilder::setMachineType(PDB_Machine M) { MachineType = M; }
 
 void DbiStreamBuilder::setSectionMap(ArrayRef<SecMapEntry> SecMap) {
   SectionMap = SecMap;
 }
 
+void DbiStreamBuilder::setGlobalsStreamIndex(uint32_t Index) {
+  GlobalsStreamIndex = Index;
+}
+
 void DbiStreamBuilder::setSymbolRecordStreamIndex(uint32_t Index) {
   SymRecordStreamIndex = Index;
 }
 
 void DbiStreamBuilder::setPublicsStreamIndex(uint32_t Index) {
   PublicsStreamIndex = Index;
 }
 
Error DbiStreamBuilder::finalize() {
   H->ModiSubstreamSize = calculateModiSubstreamSize();
   H->OptionalDbgHdrSize = DbgStreams.size() * sizeof(uint16_t);
   H->SecContrSubstreamSize = calculateSectionContribsStreamSize();
   H->SectionMapSize = calculateSectionMapStreamSize();
   H->TypeServerSize = 0;
   H->SymRecordStreamIndex = SymRecordStreamIndex;
   H->PublicSymbolStreamIndex = PublicsStreamIndex;
   H->MFCTypeServerIndex = kInvalidStreamIndex;
-  H->GlobalSymbolStreamIndex = kInvalidStreamIndex;
+  H->GlobalSymbolStreamIndex = GlobalsStreamIndex;
 
   Header = H;
   return Error::success();
 }
 
 Error DbiStreamBuilder::finalizeMsfLayout() {
   for (auto &MI : ModiList) {
     if (auto EC = MI->finalizeMsfLayout())

llvm/trunk/lib/DebugInfo/PDB/Native/GlobalsStreamBuilder.cpp

+//===- GlobalsStreamBuilder.cpp - PDB Globals Stream Creation ---*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/PDB/Native/GlobalsStreamBuilder.h"
+
+#include "llvm/DebugInfo/MSF/MSFBuilder.h"
+#include "llvm/DebugInfo/MSF/MSFCommon.h"
+#include "llvm/DebugInfo/MSF/MappedBlockStream.h"
+#include "llvm/DebugInfo/PDB/Native/GlobalsStream.h"
+
+using namespace llvm;
+using namespace llvm::msf;
+using namespace llvm::pdb;
+
+GlobalsStreamBuilder::GlobalsStreamBuilder(msf::MSFBuilder &Msf) : Msf(Msf) {}
+
+GlobalsStreamBuilder::~GlobalsStreamBuilder() {}
+
+uint32_t GlobalsStreamBuilder::calculateSerializedLength() const {
+  uint32_t Size = 0;
+  // First is the header
+  Size += sizeof(GSIHashHeader);
+
+  // Next is the records.  For now we don't write any records, just an empty
+  // stream.
+  // FIXME: Write records and account for their size here.
+  Size += 0;
+
+  // Next is a bitmap indicating which hash buckets are valid.  The bitmap
+  // is alway present, but we only write buckets for bitmap entries which are
+  // non-zero, which now is noting.
+  size_t BitmapSizeInBits = alignTo(IPHR_HASH + 1, 32);
+  uint32_t NumBitmapEntries = BitmapSizeInBits / 8;
+  Size += NumBitmapEntries;
+
+  // FIXME: Account for hash buckets.  For now since we we write a zero-bitmap
+  // indicating that no hash buckets are valid, we also write zero byets of hash
+  // bucket data.
+  Size += 0;
+  return Size;
+}
+
+Error GlobalsStreamBuilder::finalizeMsfLayout() {
+  Expected<uint32_t> Idx = Msf.addStream(calculateSerializedLength());
+  if (!Idx)
+    return Idx.takeError();
+  StreamIdx = *Idx;
+  return Error::success();
+}
+
+Error GlobalsStreamBuilder::commit(BinaryStreamWriter &PublicsWriter) {
+  GSIHashHeader GSH;
+
+  GSH.VerSignature = GSIHashHeader::HdrSignature;
+  GSH.VerHdr = GSIHashHeader::HdrVersion;
+  GSH.HrSize = 0;
+  GSH.NumBuckets = 0;
+
+  if (auto EC = PublicsWriter.writeObject(GSH))
+    return EC;
+
+  // FIXME: Once we start writing a value other than 0 for GSH.HrSize, we need
+  // to write the hash records here.
+  size_t BitmapSizeInBits = alignTo(IPHR_HASH + 1, 32);
+  uint32_t NumBitmapEntries = BitmapSizeInBits / 8;
+  std::vector<uint8_t> BitmapData(NumBitmapEntries);
+  // FIXME: Build an actual bitmap
+  if (auto EC = PublicsWriter.writeBytes(makeArrayRef(BitmapData)))
+    return EC;
+
+  // FIXME: Write actual hash buckets.
+  return Error::success();
+}

llvm/trunk/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp

 #include "llvm/DebugInfo/PDB/Native/PDBFileBuilder.h"
 
 #include "llvm/ADT/BitVector.h"
 
 #include "llvm/DebugInfo/MSF/MSFBuilder.h"
 #include "llvm/DebugInfo/PDB/GenericError.h"
 #include "llvm/DebugInfo/PDB/Native/DbiStream.h"
 #include "llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h"
+#include "llvm/DebugInfo/PDB/Native/GlobalsStreamBuilder.h"
 #include "llvm/DebugInfo/PDB/Native/InfoStream.h"
 #include "llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h"
 #include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h"
 #include "llvm/DebugInfo/PDB/Native/PublicsStreamBuilder.h"
 #include "llvm/DebugInfo/PDB/Native/RawError.h"
 #include "llvm/DebugInfo/PDB/Native/TpiStream.h"
 #include "llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h"
 #include "llvm/Support/BinaryStream.h"

 }
 
 PublicsStreamBuilder &PDBFileBuilder::getPublicsBuilder() {
   if (!Publics)
     Publics = llvm::make_unique<PublicsStreamBuilder>(*Msf);
   return *Publics;
 }
 
+GlobalsStreamBuilder &PDBFileBuilder::getGlobalsBuilder() {
+  if (!Globals)
+    Globals = llvm::make_unique<GlobalsStreamBuilder>(*Msf);
+  return *Globals;
+}
+
 Error PDBFileBuilder::addNamedStream(StringRef Name, uint32_t Size) {
   auto ExpectedStream = Msf->addStream(Size);
   if (!ExpectedStream)
     return ExpectedStream.takeError();
   NamedStreams.set(Name, *ExpectedStream);
   return Error::success();
 }
 
if (Publics) {
     if (auto EC = Publics->finalizeMsfLayout())
       return std::move(EC);
     if (Dbi) {
       Dbi->setPublicsStreamIndex(Publics->getStreamIndex());
       Dbi->setSymbolRecordStreamIndex(Publics->getRecordStreamIdx());
     }
   }
 
+  if (Globals) {
+    if (auto EC = Globals->finalizeMsfLayout())
+      return std::move(EC);
+    if (Dbi)
+      Dbi->setGlobalsStreamIndex(Globals->getStreamIndex());
+  }
+
   return Msf->build();
 }
 
 Expected<uint32_t> PDBFileBuilder::getNamedStreamIndex(StringRef Name) const {
   uint32_t SN = 0;
   if (!NamedStreams.get(Name, SN))
     return llvm::make_error<pdb::RawError>(raw_error_code::no_stream);
   return SN;
if (Publics) {
     auto PRS = WritableMappedBlockStream::createIndexedStream(
         Layout, Buffer, Publics->getRecordStreamIdx(), Allocator);
     BinaryStreamWriter PSWriter(*PS);
     BinaryStreamWriter RecWriter(*PRS);
     if (auto EC = Publics->commit(PSWriter, RecWriter))
       return EC;
   }
 
+  if (Globals) {
+    auto GS = WritableMappedBlockStream::createIndexedStream(
+        Layout, Buffer, Globals->getStreamIndex(), Allocator);
+    BinaryStreamWriter GSWriter(*GS);
+    if (auto EC = Globals->commit(GSWriter))
+      return EC;
+  }
+
   return Buffer.commit();
 }