pdbdump: print out symbol names referred by publics stream.

[ruiu]

Summary

DBI stream contains a stream number of the symbol record stream.
Symbol record streams is an array of length-type-value members.
Each member represents one symbol.

Publics stream contains offsets to the symbol record stream.
This patch is to print out all symbols that are referenced by
the publics stream.

Note that even with this patch, llvm-pdbdump cannot dump all the
information in a publics stream since it contains more information
than symbol names. I'll improve it in followup patches.

[zturner]

I would hold off on this for now. We have code to do all of this in DebugInfo/CodeView, but it is not completely reusable outside of llvm-readobj. We have a lot of information about the different kinds of symbol records and how to parse them in DebugInfo/CodeView. (see SymbolRecord.h). This is what I'm working on fixing right now, making it reusable so that we can dump full symbol details from llvm-pdbdump. If you want to dump 1 or 2 symbols as a proof of concept for now (like you're doing in this patch), that's fine, but I wouldn't add any more because it would be duplicated effort and we will have to remove it once we standardize on using the ones in DebugInfo/CodeView.

[ruiu]

I'm not going to add more than these symbol types as they are the only two types that exist in "empty.pdb". My focus is to read the hash table in the publics stream, but since the hash table refers to the symbols, I needed to read it.

[majnemer]

lib/DebugInfo/PDB/Raw/SymbolStream.cpp

57

Why not make this const char * ? Then you could use the implicit constructor to make a StringRef.

Also, variable names should be capitalized.

[rnk]

llvm/trunk/lib/DebugInfo/PDB/Raw/SymbolStream.cpp

38 ↗

(On Diff #57973)

In the future, please don't use these kind of flexible array members, they are how you get buffer overruns when the strings are not null terminated. Look at how we are deserializing codeview types instead.

[zturner]

I was going to comment on that, but this code will go away once I finish
the work on the symbol visitor and start using libcodeview from here.

include/llvm/DebugInfo/PDB/Raw/PDBFile.h

 namespace llvm {
 class MemoryBuffer;
 
 namespace pdb {
 struct PDBFileContext;
 class DbiStream;
 class InfoStream;
 class PublicsStream;
+class SymbolStream;
 class TpiStream;
 
 class PDBFile {
 public:
   explicit PDBFile(std::unique_ptr<MemoryBuffer> MemBuffer);
   ~PDBFile();
 
   uint32_t getBlockSize() const;
public:
   static uint64_t blockToOffset(uint64_t BlockNumber, uint64_t BlockSize) {
     return BlockNumber * BlockSize;
   }
 
   Expected<InfoStream &> getPDBInfoStream();
   Expected<DbiStream &> getPDBDbiStream();
   Expected<TpiStream &> getPDBTpiStream();
   Expected<PublicsStream &> getPDBPublicsStream();
+  Expected<SymbolStream &> getPDBSymbolStream();
 
 private:
   std::unique_ptr<PDBFileContext> Context;
   std::unique_ptr<InfoStream> Info;
   std::unique_ptr<DbiStream> Dbi;
   std::unique_ptr<TpiStream> Tpi;
   std::unique_ptr<PublicsStream> Publics;
+  std::unique_ptr<SymbolStream> Symbols;
 };
 }
 }
 
 #endif

include/llvm/DebugInfo/PDB/Raw/PublicsStream.h

 #include "llvm/DebugInfo/PDB/Raw/ByteStream.h"
 #include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
 #include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
 
 #include "llvm/Support/Error.h"
 
 namespace llvm {
 namespace pdb {
+class DbiStream;
 class PDBFile;
 
 class PublicsStream {
-  struct HeaderInfo;
   struct GSIHashHeader;
-  struct HRFile;
+  struct HashRecord;
+  struct HeaderInfo;
 
 public:
   PublicsStream(PDBFile &File, uint32_t StreamNum);
   ~PublicsStream();
   Error reload();
 
   uint32_t getStreamNum() const { return StreamNum; }
   uint32_t getSymHash() const;
   uint32_t getAddrMap() const;
   uint32_t getNumBuckets() const { return NumBuckets; }
+  std::vector<std::string> getSymbols() const;
   ArrayRef<uint32_t> getHashBuckets() const { return HashBuckets; }
   ArrayRef<uint32_t> getAddressMap() const { return AddressMap; }
   ArrayRef<uint32_t> getThunkMap() const { return ThunkMap; }
   ArrayRef<uint32_t> getSectionOffsets() const { return SectionOffsets; }
 
 private:
+  Error readSymbols();
+
+  PDBFile &Pdb;
+
   uint32_t StreamNum;
   MappedBlockStream Stream;
   uint32_t NumBuckets = 0;
+  std::vector<HashRecord> HashRecords;
   std::vector<uint32_t> HashBuckets;
   std::vector<uint32_t> AddressMap;
   std::vector<uint32_t> ThunkMap;
   std::vector<uint32_t> SectionOffsets;
 
   std::unique_ptr<HeaderInfo> Header;
   std::unique_ptr<GSIHashHeader> HashHdr;
 };
 }
 }
 
 #endif

include/llvm/DebugInfo/PDB/Raw/SymbolStream.h

+//===- SymbolStream.cpp - PDB Symbol Stream Access --------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_PDB_RAW_PDBSYMBOLSTREAM_H
+#define LLVM_DEBUGINFO_PDB_RAW_PDBSYMBOLSTREAM_H
+
+#include "llvm/DebugInfo/CodeView/TypeStream.h"
+#include "llvm/DebugInfo/PDB/PDBTypes.h"
+#include "llvm/DebugInfo/PDB/Raw/ByteStream.h"
+#include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
+#include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
+
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace pdb {
+class PDBFile;
+
+class SymbolStream {
+public:
+  SymbolStream(PDBFile &File, uint32_t StreamNum);
+  ~SymbolStream();
+  Error reload();
+
+  Expected<std::string> getSymbolName(uint32_t Offset) const;
+
+private:
+  MappedBlockStream Stream;
+};
+}
+}
+
+#endif

lib/DebugInfo/PDB/CMakeLists.txt

add_pdb_impl_folder(Raw
   Raw/ModStream.cpp
   Raw/NameHashTable.cpp
   Raw/NameMap.cpp
   Raw/PDBFile.cpp
   Raw/PublicsStream.cpp
   Raw/RawError.cpp
   Raw/RawSession.cpp
   Raw/StreamReader.cpp
+  Raw/SymbolStream.cpp
   Raw/TpiStream.cpp)
 
 list(APPEND LIBPDB_ADDITIONAL_HEADER_DIRS "${LLVM_MAIN_INCLUDE_DIR}/llvm/DebugInfo/PDB")
 
 add_llvm_library(LLVMDebugInfoPDB
   GenericError.cpp
   IPDBSourceFile.cpp
   PDB.cpp

lib/DebugInfo/PDB/Raw/PDBFile.cpp

 //===- PDBFile.cpp - Low level interface to a PDB file ----------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/DebugInfo/PDB/Raw/DbiStream.h"
 #include "llvm/DebugInfo/PDB/Raw/InfoStream.h"
 #include "llvm/DebugInfo/PDB/Raw/PublicsStream.h"
 #include "llvm/DebugInfo/PDB/Raw/RawError.h"
+#include "llvm/DebugInfo/PDB/Raw/SymbolStream.h"
 #include "llvm/DebugInfo/PDB/Raw/TpiStream.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/MemoryBuffer.h"
 
 using namespace llvm;
 using namespace llvm::pdb;
 
 namespace {
if (!Publics) {
     uint32_t PublicsStreamNum = DbiS->getPublicSymbolStreamIndex();
 
     Publics.reset(new PublicsStream(*this, PublicsStreamNum));
     if (auto EC = Publics->reload())
       return std::move(EC);
   }
   return *Publics;
 }
+
+Expected<SymbolStream &> PDBFile::getPDBSymbolStream() {
+  if (!Symbols) {
+    auto DbiS = getPDBDbiStream();
+    if (auto EC = DbiS.takeError())
+      return std::move(EC);
+    uint32_t SymbolStreamNum = DbiS->getSymRecordStreamIndex();
+
+    Symbols.reset(new SymbolStream(*this, SymbolStreamNum));
+    if (auto EC = Symbols->reload())
+      return std::move(EC);
+  }
+  return *Symbols;
+}

lib/DebugInfo/PDB/Raw/PublicsStream.cpp

 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/DebugInfo/PDB/Raw/PublicsStream.h"
 
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/TypeRecord.h"
 #include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
+#include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
 #include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
 #include "llvm/DebugInfo/PDB/Raw/RawError.h"
 #include "llvm/DebugInfo/PDB/Raw/StreamReader.h"
+#include "llvm/DebugInfo/PDB/Raw/SymbolStream.h"
 
 #include "llvm/ADT/BitVector.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/MathExtras.h"
 
 using namespace llvm;
 using namespace llvm::support;
struct PublicsStream::HeaderInfo {
   ulittle32_t SizeOfThunk;
   ulittle16_t ISectThunkTable;
   char Padding[2];
   ulittle32_t OffThunkTable;
   ulittle32_t NumSections;
 };
 
 
-// This is GSIHashHdr struct defined in
+// This is GSIHashHdr.
 struct PublicsStream::GSIHashHeader {
   enum : unsigned {
     HdrSignature = ~0U,
     HdrVersion = 0xeffe0000 + 19990810,
   };
   ulittle32_t VerSignature;
   ulittle32_t VerHdr;
   ulittle32_t HrSize;
   ulittle32_t NumBuckets;
 };
 
-struct PublicsStream::HRFile {
-  ulittle32_t Off;
+// This is HRFile.
+struct PublicsStream::HashRecord {
+  ulittle32_t Off;  // Offset in the symbol record stream
   ulittle32_t CRef;
 };
 
 // This struct is defined as "SO" in langapi/include/pdb.h.
 namespace {
 struct SectionOffset {
   ulittle32_t Off;
   ulittle16_t Isect;
   char Padding[2];
 };
 }
 
 PublicsStream::PublicsStream(PDBFile &File, uint32_t StreamNum)
-    : StreamNum(StreamNum), Stream(StreamNum, File) {}
+    : Pdb(File), StreamNum(StreamNum), Stream(StreamNum, File) {}
 
 PublicsStream::~PublicsStream() {}
 
 uint32_t PublicsStream::getSymHash() const { return Header->SymHash; }
 uint32_t PublicsStream::getAddrMap() const { return Header->AddrMap; }
 
 // Publics stream contains fixed-size headers and a serialized hash table.
 // This implementation is not complete yet. It reads till the end of the
Error PublicsStream::reload() {
   if (Reader.readObject(Header.get()))
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "Publics Stream does not contain a header.");
   HashHdr.reset(new GSIHashHeader());
   if (Reader.readObject(HashHdr.get()))
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "Publics Stream does not contain a header.");
 
-  // An array of HRFile follows. Read them.
-  if (HashHdr->HrSize % sizeof(HRFile))
+  // An array of HashRecord follows. Read them.
+  if (HashHdr->HrSize % sizeof(HashRecord))
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "Invalid HR array size.");
-  std::vector<HRFile> HRs(HashHdr->HrSize / sizeof(HRFile));
-  if (auto EC = Reader.readArray<HRFile>(HRs))
+  HashRecords.resize(HashHdr->HrSize / sizeof(HashRecord));
+  if (auto EC = Reader.readArray<HashRecord>(HashRecords))
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "Could not read an HR array");
 
   // A bitmap of a fixed length follows.
   size_t BitmapSizeInBits = alignTo(IPHR_HASH + 1, 32);
   std::vector<uint8_t> Bitmap(BitmapSizeInBits / 8);
   if (auto EC = Reader.readArray<uint8_t>(Bitmap))
     return make_error<RawError>(raw_error_code::corrupt_file,
for (auto &SO : Offsets) {
     SectionOffsets.push_back(SO.Isect);
   }
 
   if (Reader.bytesRemaining() > 0)
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "Corrupted publics stream.");
   return Error::success();
 }
+
+std::vector<std::string> PublicsStream::getSymbols() const {
+  auto SymbolS = Pdb.getPDBSymbolStream();
+  if (SymbolS.takeError())
+    return {};
+  SymbolStream &SS = SymbolS.get();
+
+  std::vector<std::string> Ret;
+  for (const HashRecord &HR : HashRecords) {
+    // For some reason, symbol offset is biased by one.
+    Expected<std::string> Name = SS.getSymbolName(HR.Off - 1);
+    if (Name.takeError())
+      return Ret;
+    Ret.push_back(std::move(Name.get()));
+  }
+  return Ret;
+}

lib/DebugInfo/PDB/Raw/SymbolStream.cpp

+//===- SymbolStream.cpp - PDB Symbol Stream Access ------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/PDB/Raw/SymbolStream.h"
+
+#include "llvm/DebugInfo/CodeView/CodeView.h"
+#include "llvm/DebugInfo/CodeView/TypeRecord.h"
+#include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
+#include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
+#include "llvm/DebugInfo/PDB/Raw/RawError.h"
+#include "llvm/DebugInfo/PDB/Raw/StreamReader.h"
+
+#include "llvm/Support/Endian.h"
+
+using namespace llvm;
+using namespace llvm::support;
+using namespace llvm::pdb;
+
+// Symbol stream is an array of symbol records. Each record starts with
+// length and type fields followed by type-specfic fields.
+namespace {
+struct SymbolHeader {
+  ulittle16_t Len;       // Record length
+  ulittle16_t Type;
+};
+
+// For S_PUB32 symbol type.
+struct DataSym32 {
+  ulittle32_t TypIndex; // Type index, or Metadata token if a managed symbol
+  ulittle32_t off;
+  ulittle16_t seg;
+  char name[1];
+};
+
+// For S_PROCREF symbol type.
+struct RefSym {
+  ulittle32_t SumName;   // SUC of the name (?)
+  ulittle32_t SymOffset; // Offset of actual symbol in $$Symbols
+  ulittle16_t Mod;       // Module containing the actual symbol
+  char name[1];
+};
+}
+
+SymbolStream::SymbolStream(PDBFile &File, uint32_t StreamNum)
+    : Stream(StreamNum, File) {}
+
+SymbolStream::~SymbolStream() {}
+
+Error SymbolStream::reload() { return Error::success(); }
+
+static StringRef makeStringRef(char *p) { return {p, strlen(p)}; }
+
+Expected<std::string> SymbolStream::getSymbolName(uint32_t Off) const {
+  StreamReader Reader(Stream);
+  Reader.setOffset(Off);
+
+  // Read length field.
+  SymbolHeader Hdr;
+  if (Reader.readObject(&Hdr))
+    return make_error<RawError>(raw_error_code::corrupt_file,
+                                "Corrupted symbol stream.");
+
+  // Read the entire record.
+  std::vector<uint8_t> Buf(Hdr.Len - sizeof(Hdr.Type));
+  if (Reader.readBytes(Buf))
+    return make_error<RawError>(raw_error_code::corrupt_file,
+                                "Corrupted symbol stream.");
+
+  switch (Hdr.Type) {
+  case codeview::S_PUB32:
+    return makeStringRef(reinterpret_cast<DataSym32 *>(Buf.data())->name);
+  case codeview::S_PROCREF:
+    return makeStringRef(reinterpret_cast<RefSym *>(Buf.data())->name);
+  default:
+    return make_error<RawError>(raw_error_code::corrupt_file,
+                                "Unknown symbol type");
+  }
+  return Error::success();
+}

test/DebugInfo/PDB/pdbdump-headers.test

 ; EMPTY-NEXT:   Stream number: 7
 ; EMPTY-NEXT:   SymHash: 556
 ; EMPTY-NEXT:   AddrMap: 8
 ; EMPTY-NEXT:   Number of buckets: 2
 ; EMPTY-NEXT:   Hash Buckets: [0, 12]
 ; EMPTY-NEXT:   Address Map: [36, 0]
 ; EMPTY-NEXT:   Thunk Map: [4112]
 ; EMPTY-NEXT:   Section Offsets: [4096, 1]
+; EMPTY-NEXT:   Symbols: [?__purecall@@3PAXA, _main]
 ; EMPTY-NEXT: }
 
 ; BIG:      FileHeaders {
 ; BIG-NEXT:   BlockSize: 4096
 ; BIG-NEXT:   Unknown0: 2
 ; BIG-NEXT:   NumBlocks: 99
 ; BIG-NEXT:   NumDirectoryBytes: 616
 ; BIG-NEXT:   Unknown1: 0

tools/llvm-pdbdump/llvm-pdbdump.cpp

static Error dumpPublicsStream(ScopedPrinter &P, PDBFile &File) {
   P.printNumber("Stream number", Publics.getStreamNum());
   P.printNumber("SymHash", Publics.getSymHash());
   P.printNumber("AddrMap", Publics.getAddrMap());
   P.printNumber("Number of buckets", Publics.getNumBuckets());
   P.printList("Hash Buckets", Publics.getHashBuckets());
   P.printList("Address Map", Publics.getAddressMap());
   P.printList("Thunk Map", Publics.getThunkMap());
   P.printList("Section Offsets", Publics.getSectionOffsets());
+  P.printList("Symbols", Publics.getSymbols());
   return Error::success();
 }
 
 static Error dumpStructure(RawSession &RS) {
   PDBFile &File = RS.getPDBFile();
   ScopedPrinter P(outs());
 
   if (auto EC = dumpFileHeaders(P, File))