Parse and dump PDB TPI Stream

[zturner]

Summary

This parses the TPI stream (stream 2) from the PDB file. This stream contains some header information followed by a series of codeview records. There is some additional complexity here in that alongside this stream of codeview records is a serialized hash table in order to efficiently query the types. We parse the necessary bookkeeping information to allow us to reconstruct the hash table, but we do not actually construct it yet as there is some additional complexity that needs to be understood before that is possible.

[ruiu]

include/llvm/DebugInfo/PDB/Raw/TpiStream.h

23

Why do you need __fastcall here?

lib/DebugInfo/PDB/Raw/ByteStream.cpp

66

I think you can use ArrayRef::slice.

[zturner]

include/llvm/DebugInfo/PDB/Raw/TpiStream.h

23

This was the calling convention in the reference implementation, but you're right, as long as I control both ends it's not important.

[zturner]

Fix suggestions from ruiu@

[rnk]

lgtm

include/llvm/DebugInfo/PDB/Raw/TpiStream.h

2

Maybe this? "PDB Type Information (TPI) stream (stream 2) access"

I was wondering what the mystery "P" was.

23

Also, it won't compile on Linux. :)

25

Doc comments here, with the acronym expansion as well would be good.

lib/DebugInfo/PDB/Raw/TpiStream.cpp

26

This is llvm::codeview::TypeIndex::FirstNonSimpleIndex.

96

Check for failure here and on the getArrayRef, otherwise there's not much point to std::error_code

[zturner]

lib/DebugInfo/PDB/Raw/TpiStream.cpp

96

The std::error_code is because in the generic case of a StreamReader for which you don't know what type of underlying stream backs it (for example a MappedBlockStream, you might get an error. In this case we know it's backed by a ByteStream (because that's the type of RecordsBuffer) which is guaranteed be contiguous and so getArrayRef should never return an error.

[rnk]

lib/DebugInfo/PDB/Raw/TpiStream.cpp

96

The getArrayRef could fail on an invalid type stream or if we get off track accidentally. Len - sizeof(Prefix.Leaf) could take us past the end of the stream.

[zturner]

lib/DebugInfo/PDB/Raw/TpiStream.cpp

96

Ahh fair point. We could have a corrupt PDB.

[Eugene.Zelenko]

Committed in r268343.

include/llvm/DebugInfo/PDB/Raw/ByteStream.h

 
   std::error_code readBytes(uint32_t Offset,
                             MutableArrayRef<uint8_t> Buffer) const override;
+
+  std::error_code getArrayRef(uint32_t Offset, ArrayRef<uint8_t> &Buffer,
+                              uint32_t Length) const override;
+
   uint32_t getLength() const override;
 
   ArrayRef<uint8_t> data() const { return Data; }

include/llvm/DebugInfo/PDB/Raw/MappedBlockStream.h

 
   std::error_code readBytes(uint32_t Offset,
                             MutableArrayRef<uint8_t> Buffer) const override;
+  std::error_code getArrayRef(uint32_t Offset, ArrayRef<uint8_t> &Buffer,
+                              uint32_t Length) const override;
+
   uint32_t getLength() const override { return StreamLength; }
 
 private:

include/llvm/DebugInfo/PDB/Raw/PDBFile.h

 struct PDBFileContext;
 class DbiStream;
 class InfoStream;
+class TpiStream;
 
 class PDBFile {
 public:

 
   InfoStream &getPDBInfoStream();
   DbiStream &getPDBDbiStream();
+  TpiStream &getPDBTpiStream();
 
 private:
   std::unique_ptr<PDBFileContext> Context;
   std::unique_ptr<InfoStream> Info;
   std::unique_ptr<DbiStream> Dbi;
+  std::unique_ptr<TpiStream> Tpi;
 };
 }
 }

include/llvm/DebugInfo/PDB/Raw/RawConstants.h

   PdbDbiV110 = 20091201
 };
 
+enum PdbRaw_TpiVer : uint32_t {
+  PdbTpiV40 = 19950410,
+  PdbTpiV41 = 19951122,
+  PdbTpiV50 = 19961031,
+  PdbTpiV70 = 19990903,
+  PdbTpiV80 = 20040203,
+};
+
 enum SpecialStream : uint32_t {
   StreamPDB = 1,
   StreamTPI = 2,

include/llvm/DebugInfo/PDB/Raw/StreamInterface.h

 
   virtual std::error_code readBytes(uint32_t Offset,
                                     MutableArrayRef<uint8_t> Buffer) const = 0;
+  virtual std::error_code getArrayRef(uint32_t Offset,
+                                      ArrayRef<uint8_t> &Buffer,
+                                      uint32_t Length) const = 0;
+
   virtual uint32_t getLength() const = 0;
 };
 }

include/llvm/DebugInfo/PDB/Raw/StreamReader.h

     return readBytes(Casted);
   }
 
+  std::error_code getArrayRef(ArrayRef<uint8_t> &Array, uint32_t Length);
+
   void setOffset(uint32_t Off) { Offset = Off; }
   uint32_t getOffset() const { return Offset; }
   uint32_t getLength() const { return Stream.getLength(); }

include/llvm/DebugInfo/PDB/Raw/TpiStream.h

+//===- TpiStream.h - PDB Tpi Stream (Stream 2) Access -----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_PDB_RAW_PDBTPISTREAM_H
+#define LLVM_DEBUGINFO_PDB_RAW_PDBTPISTREAM_H
+
+#include "llvm/DebugInfo/PDB/PDBTypes.h"
+#include "llvm/DebugInfo/PDB/Raw/ByteStream.h"
+#include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
+#include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
+
+namespace llvm {
+namespace pdb {
+class PDBFile;
+
+typedef uint32_t (*HashFunctionType)(uint8_t *, uint32_t);
+
+class TpiStream {
+  struct HeaderInfo;
+
+public:
+  struct HashedTypeRecord {
+    uint32_t Hash;
+    codeview::TypeLeafKind Kind;
+    ArrayRef<uint8_t> Record;
+  };
+
+  TpiStream(PDBFile &File);
+  ~TpiStream();
+  std::error_code reload();
+
+  PdbRaw_TpiVer getTpiVersion() const;
+
+  uint32_t TypeIndexBegin() const;
+  uint32_t TypeIndexEnd() const;
+  uint32_t NumTypeRecords() const;
+
+  ArrayRef<HashedTypeRecord> records() const;
+
+private:
+  PDBFile &Pdb;
+  MappedBlockStream Stream;
+  HashFunctionType HashFunction;
+
+  ByteStream RecordsBuffer;
+  ByteStream TypeIndexOffsetBuffer;
+  ByteStream HashValuesBuffer;
+  ByteStream HashAdjBuffer;
+
+  std::vector<HashedTypeRecord> TypeRecords;
+  std::unique_ptr<HeaderInfo> Header;
+};
+}
+}
+
+#endif

lib/DebugInfo/PDB/CMakeLists.txt

   Raw/NameHashTable.cpp
   Raw/NameMap.cpp
   Raw/RawSession.cpp
-  Raw/StreamReader.cpp)
+  Raw/StreamReader.cpp
+  Raw/TpiStream.cpp)
 
 list(APPEND LIBPDB_ADDITIONAL_HEADER_DIRS "${LLVM_MAIN_INCLUDE_DIR}/llvm/DebugInfo/PDB")
 

lib/DebugInfo/PDB/Raw/ByteStream.cpp

   return std::error_code();
 }
 
+std::error_code ByteStream::getArrayRef(uint32_t Offset,
+                                        ArrayRef<uint8_t> &Buffer,
+                                        uint32_t Length) const {
+  if (Data.size() < Length + Offset)
+    return std::make_error_code(std::errc::bad_address);
+  Buffer = Data.slice(Offset, Length);
+  return std::error_code();
+}
+
 uint32_t ByteStream::getLength() const { return Data.size(); }
 
 StringRef ByteStream::str() const {

lib/DebugInfo/PDB/Raw/MappedBlockStream.cpp

 
   return std::error_code();
 }
+
+std::error_code MappedBlockStream::getArrayRef(uint32_t Offset,
+                                               ArrayRef<uint8_t> &Buffer,
+                                               uint32_t Length) const {
+  return std::make_error_code(std::errc::operation_not_supported);
+}

lib/DebugInfo/PDB/Raw/PDBFile.cpp

 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/DebugInfo/PDB/Raw/DbiStream.h"
 #include "llvm/DebugInfo/PDB/Raw/InfoStream.h"
+#include "llvm/DebugInfo/PDB/Raw/TpiStream.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/MemoryBuffer.h"
 

 std::error_code PDBFile::parseFileHeaders() {
   std::error_code EC;
   MemoryBufferRef BufferRef = *Context->Buffer;
+  // Make sure the file is sufficiently large to hold a super block.
+  // Do this before attempting to read the super block.
   if (BufferRef.getBufferSize() < sizeof(SuperBlock))
     return std::make_error_code(std::errc::illegal_byte_sequence);
 

   if (BufferRef.getBufferSize() % SB->BlockSize != 0)
     return std::make_error_code(std::errc::illegal_byte_sequence);
 
-  // Make sure the file is sufficiently large to hold a super block.
-  if (BufferRef.getBufferSize() < sizeof(SuperBlock))
-    return std::make_error_code(std::errc::illegal_byte_sequence);
-
   // Check the magic bytes.
   if (memcmp(SB->MagicBytes, Magic, sizeof(Magic)) != 0)
     return std::make_error_code(std::errc::illegal_byte_sequence);

   }
   return *Dbi;
 }
+
+TpiStream &PDBFile::getPDBTpiStream() {
+  if (!Tpi) {
+    Tpi.reset(new TpiStream(*this));
+    Tpi->reload();
+  }
+  return *Tpi;
+}

lib/DebugInfo/PDB/Raw/StreamReader.cpp

   } while (C != '\0');
   return std::error_code();
 }
+
+std::error_code StreamReader::getArrayRef(ArrayRef<uint8_t> &Array,
+                                          uint32_t Length) {
+  if (auto EC = Stream.getArrayRef(Offset, Array, Length))
+    return EC;
+  Offset += Length;
+  return std::error_code();
+}

lib/DebugInfo/PDB/Raw/TpiStream.cpp

+//===- TpiStream.cpp - PDB Tpi Stream (Stream 2) Access -------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/PDB/Raw/TpiStream.h"
+
+#include "llvm/DebugInfo/CodeView/CodeView.h"
+#include "llvm/DebugInfo/CodeView/TypeRecord.h"
+#include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
+#include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
+#include "llvm/DebugInfo/PDB/Raw/StreamReader.h"
+
+#include "llvm/Support/Endian.h"
+
+using namespace llvm;
+using namespace llvm::support;
+using namespace llvm::pdb;
+
+namespace {
+const uint32_t MinTypeIndex = 0x1000;
+
+const uint32_t MinHashBuckets = 0x1000;
+const uint32_t MaxHashBuckets = 0x40000;
+
+uint32_t HashBufferV8(uint8_t *buffer, uint32_t NumBuckets) {
+  // Not yet implemented, this is probably some variation of CRC32 but we need
+  // to be sure of the precise implementation otherwise we won't be able to work
+  // with persisted hash values.
+  return 0;
+}
+}
+
+struct TpiStream::HeaderInfo {
+  struct EmbeddedBuf {
+    little32_t Off;
+    ulittle32_t Length;
+  };
+
+  ulittle32_t Version;
+  ulittle32_t HeaderSize;
+  ulittle32_t TypeIndexBegin;
+  ulittle32_t TypeIndexEnd;
+  ulittle32_t TypeRecordBytes;
+
+  ulittle16_t HashStreamIndex;
+  ulittle16_t HashAuxStreamIndex;
+  ulittle32_t HashKeySize;
+  ulittle32_t NumHashBuckets;
+
+  EmbeddedBuf HashValueBuffer;
+  EmbeddedBuf IndexOffsetBuffer;
+  EmbeddedBuf HashAdjBuffer;
+};
+
+TpiStream::TpiStream(PDBFile &File)
+    : Pdb(File), Stream(StreamTPI, File), HashFunction(nullptr) {}
+
+TpiStream::~TpiStream() {}
+
+std::error_code TpiStream::reload() {
+  StreamReader Reader(Stream);
+
+  if (Reader.bytesRemaining() < sizeof(HeaderInfo))
+    return std::make_error_code(std::errc::illegal_byte_sequence);
+
+  Header.reset(new HeaderInfo());
+  Reader.readObject(Header.get());
+
+  if (Header->Version != PdbTpiV80)
+    return std::make_error_code(std::errc::not_supported);
+
+  if (Header->HeaderSize != sizeof(HeaderInfo))
+    return std::make_error_code(std::errc::illegal_byte_sequence);
+
+  if (Header->HashKeySize != sizeof(ulittle32_t))
+    return std::make_error_code(std::errc::illegal_byte_sequence);
+
+  if (Header->NumHashBuckets < MinHashBuckets ||
+      Header->NumHashBuckets > MaxHashBuckets)
+    return std::make_error_code(std::errc::illegal_byte_sequence);
+
+  HashFunction = HashBufferV8;
+
+  // The actual type records themselves come from this stream
+  RecordsBuffer.initialize(Reader, Header->TypeRecordBytes);
+  TypeRecords.resize(TypeIndexEnd() - ::MinTypeIndex);
+  StreamReader RecordsReader(RecordsBuffer);
+  for (uint32_t I = TypeIndexBegin(); I < TypeIndexEnd(); ++I) {
+    HashedTypeRecord &Record = TypeRecords[I - ::MinTypeIndex];
+    codeview::TypeRecordPrefix Prefix;
+    RecordsReader.readObject(&Prefix);
+    Record.Kind =
+        static_cast<codeview::TypeLeafKind>(static_cast<uint16_t>(Prefix.Leaf));
+
+    // Since we read this entire buffer into a ByteStream, we are guaranteed
+    // that the entire buffer is contiguous (i.e. there's no longer a chance
+    // that it splits across a page boundary.  So we can request a reference
+    // directly into the stream buffer to avoid unnecessary memory copies.
+    RecordsReader.getArrayRef(Record.Record, Prefix.Len - sizeof(Prefix.Leaf));
+  }
+
+  // Hash indices, hash values, etc come from the hash stream.
+  MappedBlockStream HS(Header->HashStreamIndex, Pdb);
+  StreamReader HSR(HS);
+  HSR.setOffset(Header->HashValueBuffer.Off);
+  HashValuesBuffer.initialize(HSR, Header->HashValueBuffer.Length);
+
+  HSR.setOffset(Header->HashAdjBuffer.Off);
+  HashAdjBuffer.initialize(HSR, Header->HashAdjBuffer.Length);
+
+  HSR.setOffset(Header->IndexOffsetBuffer.Off);
+  TypeIndexOffsetBuffer.initialize(HSR, Header->IndexOffsetBuffer.Length);
+
+  return std::error_code();
+}
+
+PdbRaw_TpiVer TpiStream::getTpiVersion() const {
+  uint32_t Value = Header->Version;
+  return static_cast<PdbRaw_TpiVer>(Value);
+}
+
+uint32_t TpiStream::TypeIndexBegin() const { return Header->TypeIndexBegin; }
+
+uint32_t TpiStream::TypeIndexEnd() const { return Header->TypeIndexEnd; }
+
+uint32_t TpiStream::NumTypeRecords() const {
+  return TypeIndexEnd() - TypeIndexBegin();
+}
+
+ArrayRef<TpiStream::HashedTypeRecord> TpiStream::records() const {
+  const HashedTypeRecord *Begin =
+      &TypeRecords[TypeIndexBegin() - ::MinTypeIndex];
+  return ArrayRef<HashedTypeRecord>(Begin, NumTypeRecords());
+}

test/DebugInfo/PDB/pdbdump-headers.test

 ; EMPTY-NEXT:     Type Server Index: 0
 ; EMPTY-NEXT:     Has EC Info: 0
 ; EMPTY-NEXT:     0 Contributing Source Files:
+; EMPTY-NEXT: TPI Version: 20040203
+; EMPTY-NEXT: Record count: 75
+; EMPTY-NEXT:   Kind: 0x4609  Bytes: [00 00 00 00]
+; EMPTY-NEXT:   Kind: 0x4104  Bytes: [74 00 00 00 00 00 00 00 00 10 00 00]
+; EMPTY-NEXT:   Kind: 0x4611  Bytes: [02 15 03 00 01 00 61 70 61 72 74 6D 65 6E 74 00
+; EMPTY-NEXT:                         02 15 03 00 02 00 73 69 6E 67 6C 65 00 F3 F2 F1
+; EMPTY-NEXT:                         02 15 03 00 03 00 66 72 65 65 00 F1 02 15 03 00
+; EMPTY-NEXT:                         04 00 6E 65 75 74 72 61 6C 00 F2 F1 02 15 03 00
+; EMPTY-NEXT:                         05 00 62 6F 74 68 00 F1]
 
 BIG:      BlockSize: 4096
 BIG-NEXT: Unknown0: 2

tools/llvm-pdbdump/llvm-pdbdump.cpp

 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
 #include "llvm/DebugInfo/PDB/Raw/RawSession.h"
 #include "llvm/DebugInfo/PDB/Raw/StreamReader.h"
+#include "llvm/DebugInfo/PDB/Raw/TpiStream.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/FileSystem.h"

                          cl::cat(FilterCategory));
 }
 
+static void dumpBytes(raw_ostream &S, ArrayRef<uint8_t> Bytes,
+                      uint32_t BytesPerRow, uint32_t Indent) {
+  S << "[";
+  uint32_t I = 0;
+
+  uint32_t BytesRemaining = Bytes.size();
+  while (BytesRemaining > 0) {
+    uint32_t BytesThisLine = std::min(BytesRemaining, BytesPerRow);
+    for (size_t L = 0; L < BytesThisLine; ++L, ++I) {
+      S << format_hex_no_prefix(Bytes[I], 2, true);
+      if (L + 1 < BytesThisLine)
+        S << ' ';
+    }
+    BytesRemaining -= BytesThisLine;
+    if (BytesRemaining > 0) {
+      S << '\n';
+      S.indent(Indent);
+    }
+  }
+  S << ']';
+  S.flush();
+}
+
 static void dumpStructure(RawSession &RS) {
   PDBFile &File = RS.getPDBFile();
 

       outs().indent(8) << File << '\n';
     }
   }
+
+  TpiStream &Tpi = File.getPDBTpiStream();
+  outs() << "TPI Version: " << Tpi.getTpiVersion() << '\n';
+  outs() << "Record count: " << Tpi.NumTypeRecords() << '\n';
+  for (auto &Record : Tpi.records()) {
+    outs().indent(2) << "Kind: 0x" << Record.Kind;
+    outs().indent(2) << "Bytes: ";
+    dumpBytes(outs(), Record.Record, 16, 24);
+    outs() << '\n';
+  }
 }
 
 static void reportError(StringRef Path, PDB_ErrorCode Error) {