pdbdump: Print "Publics" stream.

[ruiu]

Summary

Publics stream seems to contain information as to public symbols.
It actually contains a serialized hash table along with fixed-sized
headers. This patch is not complete. It scans only till the end of
the stream and dump the header information. I'll write code to
de-serialize the hash table later.

[zturner]

In Microsoft's code, is the hash table an NMT or an NMTNI? We can parse both of those using the NameMap class and NameHashTable classes in DebugInfoPDB. I forget which is which, but NameHashTable starts with a very obvious signature 0xEFFEEFFE, so you should be able to tell immediately which one it is.

[ruiu]

I don't think it is NMT nor NMTNI. That hash table is embedded to the stream and has no header. Microsoft gsi.cpp has code to read it.

[zturner]

include/llvm/DebugInfo/PDB/Raw/DbiStream.h

36 ↗

(On Diff #57239)

Now that we know what this actually is, can you change it to a better name? Like getPublicSymbolStreamIndex()?

lib/DebugInfo/PDB/Raw/DbiStream.cpp

187 ↗

(On Diff #57239)

Same here, can you change the field name in Header to be PublicSymbolStreamIndex?

lib/DebugInfo/PDB/Raw/PublicsStream.cpp

47 ↗

(On Diff #57239)

The name gsi suggests that this structure is used for more than just public symbols. I bet it stands for Global Symbol Info. There is another stream representing global symbols, and my guess is that they have the exact same format. Based on that, maybe we can call this entire class something more generic, like just SymbolStream. What do you think?

64 ↗

(On Diff #57239)

Interesting, this is very similar to the other hash table, whose version is 0xEFFEEFFE. I wonder what the difference is.

72–75 ↗

(On Diff #57239)

Do you know what the purpose of CRef is here? In TpiStream.cpp we have a very similar structure called EmbeddedBuff. In that case, the Offset points to a location in the stream, and the second field indicates the length of the buffer. If this is the same thing, perhaps it's worth raising that structure to a higher level so both can reuse it.

93–96 ↗

(On Diff #57239)

I think this line isn't necessary, because the condition will be checked automatically when you try to read the HeaderInfo, and again when you try to read the GSIHashHeader.

[ruiu]

include/llvm/DebugInfo/PDB/Raw/DbiStream.h

36 ↗

(On Diff #57239)

Will do.

lib/DebugInfo/PDB/Raw/DbiStream.cpp

187 ↗

(On Diff #57239)

Will do.

lib/DebugInfo/PDB/Raw/PublicsStream.cpp

47 ↗

(On Diff #57239)

I'm not sure yet if the format is the same, so I want to keep as it is for now. It shouldn't be too late to move it to a common file when we find that this is a common thing.

64 ↗

(On Diff #57239)

I have no idea. In Microsoft code, this magic number is declared as GSIHashSCImpvV70.

72–75 ↗

(On Diff #57239)

This should be related to a hash table, but I don"t understand the exact meaning yet. I just took the name from the reference.

93–96 ↗

(On Diff #57239)

Sure, then I'll remove this piece of code.

[ruiu]

• Updated as per zturner's comments.

llvm/trunk/include/llvm/DebugInfo/PDB/Raw/DbiStream.h

 
 public:
   DbiStream(PDBFile &File);
   ~DbiStream();
   Error reload();
 
   PdbRaw_DbiVer getDbiVersion() const;
   uint32_t getAge() const;
+  uint16_t getPublicSymbolStreamIndex() const;
 
   bool isIncrementallyLinked() const;
   bool hasCTypes() const;
   bool isStripped() const;
 
   uint16_t getBuildMajorVersion() const;
   uint16_t getBuildMinorVersion() const;
 

llvm/trunk/include/llvm/DebugInfo/PDB/Raw/PDBFile.h

 
 namespace llvm {
 class MemoryBuffer;
 
 namespace pdb {
 struct PDBFileContext;
 class DbiStream;
 class InfoStream;
+class PublicsStream;
 class TpiStream;
 
 class PDBFile {
 public:
   explicit PDBFile(std::unique_ptr<MemoryBuffer> MemBuffer);
   ~PDBFile();
 
   uint32_t getBlockSize() const;
public:
 
   static uint64_t blockToOffset(uint64_t BlockNumber, uint64_t BlockSize) {
     return BlockNumber * BlockSize;
   }
 
   Expected<InfoStream &> getPDBInfoStream();
   Expected<DbiStream &> getPDBDbiStream();
   Expected<TpiStream &> getPDBTpiStream();
+  Expected<PublicsStream &> getPDBPublicsStream();
 
 private:
   std::unique_ptr<PDBFileContext> Context;
   std::unique_ptr<InfoStream> Info;
   std::unique_ptr<DbiStream> Dbi;
   std::unique_ptr<TpiStream> Tpi;
+  std::unique_ptr<PublicsStream> Publics;
 };
 }
 }
 
 #endif

llvm/trunk/include/llvm/DebugInfo/PDB/Raw/PublicsStream.h

+//===- PublicsStream.h - PDB Public Symbol Stream -------- ------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_PDB_RAW_PUBLICSSTREAM_H
+#define LLVM_DEBUGINFO_PDB_RAW_PUBLICSSTREAM_H
+
+#include "llvm/DebugInfo/CodeView/TypeStream.h"
+#include "llvm/DebugInfo/PDB/PDBTypes.h"
+#include "llvm/DebugInfo/PDB/Raw/ByteStream.h"
+#include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
+#include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
+
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace pdb {
+class PDBFile;
+
+class PublicsStream {
+  struct HeaderInfo;
+  struct GSIHashHeader;
+  struct HRFile;
+
+public:
+  PublicsStream(PDBFile &File, uint32_t StreamNum);
+  ~PublicsStream();
+  Error reload();
+
+  uint32_t getStreamNum() const { return StreamNum; }
+  uint32_t getSymHash() const;
+  uint32_t getAddrMap() const;
+  uint32_t getNumBuckets() const { return NumBuckets; }
+
+private:
+  uint32_t StreamNum;
+  MappedBlockStream Stream;
+  uint32_t NumBuckets = 0;
+
+  std::unique_ptr<HeaderInfo> Header;
+  std::unique_ptr<GSIHashHeader> HashHdr;
+};
+}
+}
+
+#endif

llvm/trunk/lib/DebugInfo/PDB/CMakeLists.txt

add_pdb_impl_folder(Raw
   Raw/DbiStream.cpp
   Raw/InfoStream.cpp
   Raw/MappedBlockStream.cpp
   Raw/ModInfo.cpp
   Raw/ModStream.cpp
   Raw/NameHashTable.cpp
   Raw/NameMap.cpp
   Raw/PDBFile.cpp
+  Raw/PublicsStream.cpp
   Raw/RawError.cpp
   Raw/RawSession.cpp
   Raw/StreamReader.cpp
   Raw/TpiStream.cpp)
 
 list(APPEND LIBPDB_ADDITIONAL_HEADER_DIRS "${LLVM_MAIN_INCLUDE_DIR}/llvm/DebugInfo/PDB")
 
 add_llvm_library(LLVMDebugInfoPDB

llvm/trunk/lib/DebugInfo/PDB/Raw/DbiStream.cpp

 
 const uint16_t BuildMajorMask = 0x7F00;
 const uint16_t BuildMajorShift = 8;
 }
 
 struct DbiStream::HeaderInfo {
   little32_t VersionSignature;
   ulittle32_t VersionHeader;
-  ulittle32_t Age;                  // Should match InfoStream.
-  ulittle16_t GSSyms;               // Number of global symbols
-  ulittle16_t BuildNumber;          // See DbiBuildNo structure.
-  ulittle16_t PSSyms;               // Number of public symbols
-  ulittle16_t PdbDllVersion;        // version of mspdbNNN.dll
-  ulittle16_t SymRecords;           // Number of symbols
-  ulittle16_t PdbDllRbld;           // rbld number of mspdbNNN.dll
-  little32_t ModiSubstreamSize;     // Size of module info stream
-  little32_t SecContrSubstreamSize; // Size of sec. contribution stream
-  little32_t SectionMapSize;        // Size of sec. map substream
-  little32_t FileInfoSize;          // Size of file info substream
+  ulittle32_t Age;                     // Should match InfoStream.
+  ulittle16_t GSSyms;                  // Number of global symbols
+  ulittle16_t BuildNumber;             // See DbiBuildNo structure.
+  ulittle16_t PublicSymbolStreamIndex; // Number of public symbols
+  ulittle16_t PdbDllVersion;           // version of mspdbNNN.dll
+  ulittle16_t SymRecords;              // Number of symbols
+  ulittle16_t PdbDllRbld;              // rbld number of mspdbNNN.dll
+  little32_t ModiSubstreamSize;        // Size of module info stream
+  little32_t SecContrSubstreamSize;    // Size of sec. contribution stream
+  little32_t SectionMapSize;           // Size of sec. map substream
+  little32_t FileInfoSize;             // Size of file info substream
   little32_t TypeServerSize;      // Size of type server map
   ulittle32_t MFCTypeServerIndex; // Index of MFC Type Server
   little32_t OptionalDbgHdrSize;  // Size of DbgHeader info
   little32_t ECSubstreamSize;     // Size of EC stream (what is EC?)
   ulittle16_t Flags;              // See DbiFlags enum.
   ulittle16_t MachineType;        // See PDB_MachineType enum.
 
   ulittle32_t Reserved; // Pad to 64 bytes

 
 PdbRaw_DbiVer DbiStream::getDbiVersion() const {
   uint32_t Value = Header->VersionHeader;
   return static_cast<PdbRaw_DbiVer>(Value);
 }
 
 uint32_t DbiStream::getAge() const { return Header->Age; }
 
+uint16_t DbiStream::getPublicSymbolStreamIndex() const {
+  return Header->PublicSymbolStreamIndex;
+}
+
 bool DbiStream::isIncrementallyLinked() const {
   return (Header->Flags & FlagIncrementalMask) != 0;
 }
 
 bool DbiStream::hasCTypes() const {
   return (Header->Flags & FlagHasCTypesMask) != 0;
 }
 

llvm/trunk/lib/DebugInfo/PDB/Raw/PDBFile.cpp

 //===- PDBFile.cpp - Low level interface to a PDB file ----------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/DebugInfo/PDB/Raw/DbiStream.h"
 #include "llvm/DebugInfo/PDB/Raw/InfoStream.h"
+#include "llvm/DebugInfo/PDB/Raw/PublicsStream.h"
 #include "llvm/DebugInfo/PDB/Raw/RawError.h"
 #include "llvm/DebugInfo/PDB/Raw/TpiStream.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/MemoryBuffer.h"
 
 using namespace llvm;
 using namespace llvm::pdb;
 

 Expected<TpiStream &> PDBFile::getPDBTpiStream() {
   if (!Tpi) {
     Tpi.reset(new TpiStream(*this));
     if (auto EC = Tpi->reload())
       return std::move(EC);
   }
   return *Tpi;
 }
+
+Expected<PublicsStream &> PDBFile::getPDBPublicsStream() {
+  if (!Publics) {
+    auto DbiS = getPDBDbiStream();
+    if (auto EC = DbiS.takeError())
+      return std::move(EC);
+    uint32_t PublicsStreamNum = DbiS->getPublicSymbolStreamIndex();
+
+    Publics.reset(new PublicsStream(*this, PublicsStreamNum));
+    if (auto EC = Publics->reload())
+      return std::move(EC);
+  }
+  return *Publics;
+}

llvm/trunk/lib/DebugInfo/PDB/Raw/PublicsStream.cpp

+//===- PublicsStream.cpp - PDB Public Symbol Stream -----------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// The data structures defined in this file are based on the reference
+// implementation which is available at
+// https://github.com/Microsoft/microsoft-pdb/blob/master/PDB/dbi/gsi.h
+//
+// When you are reading the reference source code, you'd find the
+// information below useful.
+//
+//  - ppdb1->m_fMinimalDbgInfo seems to be always true.
+//  - SMALLBUCKETS macro is defined.
+//
+// The reference doesn't compile, so I learned just by reading code.
+// It's not guaranteed to be correct.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/PDB/Raw/PublicsStream.h"
+
+#include "llvm/DebugInfo/CodeView/CodeView.h"
+#include "llvm/DebugInfo/CodeView/TypeRecord.h"
+#include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
+#include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
+#include "llvm/DebugInfo/PDB/Raw/RawError.h"
+#include "llvm/DebugInfo/PDB/Raw/StreamReader.h"
+
+#include "llvm/ADT/BitVector.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace llvm;
+using namespace llvm::support;
+using namespace llvm::pdb;
+
+
+static const unsigned IPHR_HASH = 4096;
+
+// This is PSGSIHDR struct defined in
+// https://github.com/Microsoft/microsoft-pdb/blob/master/PDB/dbi/gsi.h
+struct PublicsStream::HeaderInfo {
+  ulittle32_t SymHash;
+  ulittle32_t AddrMap;
+  ulittle32_t NumThunks;
+  ulittle32_t SizeOfThunk;
+  ulittle16_t ISectThunkTable;
+  char Padding[2];
+  ulittle32_t OffThunkTable;
+  ulittle32_t NumSects;
+};
+
+
+// This is GSIHashHdr struct defined in
+struct PublicsStream::GSIHashHeader {
+  enum {
+    HdrSignature = -1,
+    HdrVersion = 0xeffe0000 + 19990810,
+  };
+  ulittle32_t VerSignature;
+  ulittle32_t VerHdr;
+  ulittle32_t HrSize;
+  ulittle32_t NumBuckets;
+};
+
+struct PublicsStream::HRFile {
+  ulittle32_t Off;
+  ulittle32_t CRef;
+};
+
+PublicsStream::PublicsStream(PDBFile &File, uint32_t StreamNum)
+    : StreamNum(StreamNum), Stream(StreamNum, File) {}
+
+PublicsStream::~PublicsStream() {}
+
+uint32_t PublicsStream::getSymHash() const { return Header->SymHash; }
+uint32_t PublicsStream::getAddrMap() const { return Header->AddrMap; }
+
+// Publics stream contains fixed-size headers and a serialized hash table.
+// This implementation is not complete yet. It reads till the end of the
+// stream so that we verify the stream is at least not corrupted. However,
+// we skip over the hash table which we believe contains information about
+// public symbols.
+Error PublicsStream::reload() {
+  StreamReader Reader(Stream);
+
+  // Check stream size.
+  if (Reader.bytesRemaining() < sizeof(HeaderInfo) + sizeof(GSIHashHeader))
+    return make_error<RawError>(raw_error_code::corrupt_file,
+                                "Publics Stream does not contain a header.");
+
+  // Read PSGSIHDR and GSIHashHdr structs.
+  Header.reset(new HeaderInfo());
+  if (Reader.readObject(Header.get()))
+    return make_error<RawError>(raw_error_code::corrupt_file,
+                                "Publics Stream does not contain a header.");
+  HashHdr.reset(new GSIHashHeader());
+  if (Reader.readObject(HashHdr.get()))
+    return make_error<RawError>(raw_error_code::corrupt_file,
+                                "Publics Stream does not contain a header.");
+
+  // An array of HRFile follows. Read them.
+  if (HashHdr->HrSize % sizeof(HRFile))
+    return make_error<RawError>(raw_error_code::corrupt_file,
+                                "Invalid HR array size.");
+  std::vector<HRFile> HRs(HashHdr->HrSize / sizeof(HRFile));
+  if (auto EC = Reader.readArray<HRFile>(HRs))
+    return make_error<RawError>(raw_error_code::corrupt_file,
+                                "Could not read an HR array");
+
+  // A bitmap of a fixed length follows.
+  size_t BitmapSizeInBits = alignTo(IPHR_HASH + 1, 32);
+  std::vector<uint8_t> Bitmap(BitmapSizeInBits / 8);
+  if (auto EC = Reader.readArray<uint8_t>(Bitmap))
+    return make_error<RawError>(raw_error_code::corrupt_file,
+                                "Could not read a bitmap.");
+  for (uint8_t B : Bitmap)
+    NumBuckets += countPopulation(B);
+
+  // Buckets follow.
+  if (Reader.bytesRemaining() < NumBuckets * sizeof(uint32_t))
+    return make_error<RawError>(raw_error_code::corrupt_file,
+                                "Hash buckets corrupted.");
+
+  return Error::success();
+}

llvm/trunk/test/DebugInfo/PDB/pdbdump-headers.test

 ; RUN: llvm-pdbdump --dump-headers -dump-tpi-records -dump-tpi-record-bytes -dump-module-syms \
-; RUN:              %p/Inputs/empty.pdb | FileCheck -check-prefix=EMPTY %s
+; RUN:              --dump-publics %p/Inputs/empty.pdb | FileCheck -check-prefix=EMPTY %s
 ; RUN: llvm-pdbdump --dump-headers %p/Inputs/big-read.pdb | FileCheck -check-prefix=BIG %s
 ; RUN: llvm-pdbdump --dump-headers %p/Inputs/bad-block-size.pdb | FileCheck -check-prefix=BAD-BLOCK-SIZE %s
 
 ; EMPTY:      FileHeaders {
 ; EMPTY-NEXT:   BlockSize: 4096
 ; EMPTY-NEXT:   Unknown0: 2
 ; EMPTY-NEXT:   NumBlocks: 25
 ; EMPTY-NEXT:   NumDirectoryBytes: 136

 ; EMPTY-NEXT:         0000: 02150300 01006170 6172746D 656E7400  |......apartment.|
 ; EMPTY-NEXT:         0010: 02150300 02007369 6E676C65 00F3F2F1  |......single....|
 ; EMPTY-NEXT:         0020: 02150300 03006672 656500F1 02150300  |......free......|
 ; EMPTY-NEXT:         0030: 04006E65 75747261 6C00F2F1 02150300  |..neutral.......|
 ; EMPTY-NEXT:         0040: 0500626F 746800F1                    |..both..|
 ; EMPTY-NEXT:       )
 ; EMPTY-NEXT:     }
 
+; EMPTY:      Publics Stream {
+; EMPTY-NEXT:   Stream number: 7
+; EMPTY-NEXT:   SymHash: 556
+; EMPTY-NEXT:   AddrMap: 8
+; EMPTY-NEXT:   Number of buckets: 2
+; EMPTY-NEXT: }
+
 ; BIG:      FileHeaders {
 ; BIG-NEXT:   BlockSize: 4096
 ; BIG-NEXT:   Unknown0: 2
 ; BIG-NEXT:   NumBlocks: 99
 ; BIG-NEXT:   NumDirectoryBytes: 616
 ; BIG-NEXT:   Unknown1: 0
 ; BIG-NEXT:   BlockMapAddr: 97
 ; BIG-NEXT:   NumDirectoryBlocks: 1

llvm/trunk/tools/llvm-pdbdump/llvm-pdbdump.h

 extern llvm::cl::list<std::string> ExcludeTypes;
 extern llvm::cl::list<std::string> ExcludeSymbols;
 extern llvm::cl::list<std::string> ExcludeCompilands;
 extern llvm::cl::list<std::string> IncludeTypes;
 extern llvm::cl::list<std::string> IncludeSymbols;
 extern llvm::cl::list<std::string> IncludeCompilands;
 }
 
-#endif
+#endif
 

llvm/trunk/tools/llvm-pdbdump/llvm-pdbdump.cpp

 #include "llvm/DebugInfo/PDB/PDBSymbolThunk.h"
 #include "llvm/DebugInfo/PDB/Raw/DbiStream.h"
 #include "llvm/DebugInfo/PDB/Raw/InfoStream.h"
 #include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
 #include "llvm/DebugInfo/PDB/Raw/ModInfo.h"
 #include "llvm/DebugInfo/PDB/Raw/ModStream.h"
 #include "llvm/DebugInfo/PDB/Raw/NameHashTable.h"
 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
+#include "llvm/DebugInfo/PDB/Raw/PublicsStream.h"
 #include "llvm/DebugInfo/PDB/Raw/RawError.h"
 #include "llvm/DebugInfo/PDB/Raw/RawSession.h"
 #include "llvm/DebugInfo/PDB/Raw/StreamReader.h"
 #include "llvm/DebugInfo/PDB/Raw/TpiStream.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Format.h"
cl::opt<bool>
     DumpTpiRecordBytes("dump-tpi-record-bytes",
                        cl::desc("dump CodeView type record raw bytes"),
                        cl::cat(NativeOtions));
 cl::opt<std::string> DumpStreamData("dump-stream", cl::desc("dump stream data"),
                                     cl::cat(NativeOtions));
 cl::opt<bool> DumpModuleSyms("dump-module-syms",
                              cl::desc("dump module symbols"),
                              cl::cat(NativeOtions));
+cl::opt<bool> DumpPublics("dump-publics",
+                          cl::desc("dump Publics stream data"),
+                          cl::cat(NativeOtions));
 
 cl::list<std::string>
     ExcludeTypes("exclude-types",
                  cl::desc("Exclude types by regular expression"),
                  cl::ZeroOrMore, cl::cat(FilterCategory));
 cl::list<std::string>
     ExcludeSymbols("exclude-symbols",
                    cl::desc("Exclude symbols by regular expression"),
if (opts::DumpTpiRecordBytes || opts::DumpTpiRecords) {
     }
     if (HadError)
       return make_error<RawError>(raw_error_code::corrupt_file,
                                   "TPI stream contained corrupt record");
   }
   return Error::success();
 }
 
+static Error dumpPublicsStream(ScopedPrinter &P, PDBFile &File) {
+  if (!opts::DumpPublics)
+    return Error::success();
+
+  DictScope D(P, "Publics Stream");
+  auto PublicsS = File.getPDBPublicsStream();
+  if (auto EC = PublicsS.takeError())
+    return EC;
+  PublicsStream &Publics = PublicsS.get();
+  P.printNumber("Stream number", Publics.getStreamNum());
+  P.printNumber("SymHash", Publics.getSymHash());
+  P.printNumber("AddrMap", Publics.getAddrMap());
+  P.printNumber("Number of buckets", Publics.getNumBuckets());
+  return Error::success();
+}
+
 static Error dumpStructure(RawSession &RS) {
   PDBFile &File = RS.getPDBFile();
   ScopedPrinter P(outs());
 
   if (auto EC = dumpFileHeaders(P, File))
     return EC;
 
   if (auto EC = dumpStreamSizes(P, File))
static Error dumpStructure(RawSession &RS) {
   if (auto EC = dumpNamedStream(P, File, "/names"))
     return EC;
 
   if (auto EC = dumpDbiStream(P, File))
     return EC;
 
   if (auto EC = dumpTpiStream(P, File))
     return EC;
-  return Error::success();
+
+  if (auto EC = dumpPublicsStream(P, File))
+    return EC;
 }
 
 static void dumpInput(StringRef Path) {
   std::unique_ptr<IPDBSession> Session;
   if (opts::DumpHeaders || !opts::DumpStreamData.empty()) {
     auto E = loadDataForPDB(PDB_ReaderType::Raw, Path, Session);
     if (!E) {
       RawSession *RS = static_cast<RawSession *>(Session.get());