Write the TPI stream from Yaml

[zturner]

Summary

This implements writing of the TPI stream from Yaml. This actually implements a generic TPI Stream builder / writer that is not tied to Yaml, but Yaml is just the vehicle used here to do the testing since we don't have linker support for writing PDBs yet.

Getting this to work was a little nasty due to the interactions between the normally stateless Yaml mappings and the stateful TypeTableBuilder. Because of that, this depends on D24162 which adds the ability to pass some state between yaml mappings, although that patch has yet to be looked at.

[zturner]

Ping, anyone able to take a look?

[rnk]

lgtm

include/llvm/DebugInfo/CodeView/FieldListSerializationVisitor.h

1 ↗

(On Diff #70600)

Is this header actually used? It seems like this part is rolled into TypeSerializationVisitor.

include/llvm/DebugInfo/MSF/SequencedItemStream.h

28 ↗

(On Diff #70600)

I guess I'm surprised that we're still having to build scaffolding like this. I kind of thought that we had all the stream-y sequence-y primitives in MSF that we needed at this point.

I don't have any real suggestions on how to improve this, just a general comment.

36 ↗

(On Diff #70600)

typo

[zturner]

include/llvm/DebugInfo/MSF/SequencedItemStream.h

28 ↗

(On Diff #70600)

I think we do have everything we need for reading PDBs, but I guess when you're constructing these things in memory, the requirements change somewhat because you have to adapt the in-memory format -- where it's convenient to store stuff in standard data structures -- to the on-disk format rather than the other way around. All the stuff like FixedStreamArray etc are useful for going from the on-disk format to the in-memory format without a bunch of copying, but how do you go from vector of std::strings (for example) to a single buffer of null terminated strings? We don't have anything like that, which is kind of what this is supposed to solve.

Another solution would have been to allocate some buffer up front, then manually serialize each thing into it, but since we're trying to be conservative with memory usage since PDBs can be so large that's not a great idea, and plus it doesn't take advantage of any of the existing machinery for writing to streams.

llvm/trunk/include/llvm/DebugInfo/CodeView/MemoryTypeTableBuilder.h

 #include "llvm/DebugInfo/CodeView/TypeTableBuilder.h"
 #include <vector>
 
 namespace llvm {
 namespace codeview {
 
 class MemoryTypeTableBuilder : public TypeTableBuilder {
 public:
-  MemoryTypeTableBuilder() {}
+  explicit MemoryTypeTableBuilder(BumpPtrAllocator &Allocator)
+      : RecordStorage(Allocator) {}
 
   bool empty() const { return Records.empty(); }
 
   template <typename TFunc> void ForEachRecord(TFunc Func) {
     uint32_t Index = TypeIndex::FirstNonSimpleIndex;
 
     for (StringRef R : Records) {
       Func(TypeIndex(Index), R);
       ++Index;
     }
   }
 
-protected:
   TypeIndex writeRecord(llvm::StringRef Data) override;
 
+  ArrayRef<StringRef> getRecords() const { return Records; }
+
 private:
   std::vector<StringRef> Records;
-  BumpPtrAllocator RecordStorage;
+  BumpPtrAllocator &RecordStorage;
   DenseMap<StringRef, TypeIndex> HashedRecords;
 };
 
 } // end namespace codeview
 } // end namespace llvm
 
 #endif // LLVM_DEBUGINFO_CODEVIEW_MEMORYTYPETABLEBUILDER_H

llvm/trunk/include/llvm/DebugInfo/CodeView/TypeRecordBuilder.h

public:
   void writeEncodedUnsignedInteger(uint64_t Value);
   void writeNullTerminatedString(StringRef Value);
   void writeGuid(StringRef Guid);
   void writeBytes(StringRef Value) { Stream << Value; }
 
   llvm::StringRef str();
 
   uint64_t size() const { return Stream.tell(); }
+  TypeRecordKind kind() const { return Kind; }
 
   void truncate(uint64_t Size) {
     // This works because raw_svector_ostream is not buffered.
     assert(Size < Buffer.size());
     Buffer.resize(Size);
   }
 
   void reset(TypeRecordKind K) {
     Buffer.clear();
+    Kind = K;
     writeTypeRecordKind(K);
   }
 
 private:
+  TypeRecordKind Kind;
   llvm::SmallVector<char, 256> Buffer;
   llvm::raw_svector_ostream Stream;
   llvm::support::endian::Writer<llvm::support::endianness::little> Writer;
 };
 }
 }
 
 #endif

llvm/trunk/include/llvm/DebugInfo/CodeView/TypeSerializationVisitor.h

+//===- TypeSerializationVisitor.h -------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_CODEVIEW_TYPESERIALIZATIONVISITOR_H
+#define LLVM_DEBUGINFO_CODEVIEW_TYPESERIALIZATIONVISITOR_H
+
+#include "llvm/DebugInfo/CodeView/FieldListRecordBuilder.h"
+#include "llvm/DebugInfo/CodeView/MemoryTypeTableBuilder.h"
+#include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h"
+
+#include "llvm/ADT/StringRef.h"
+
+namespace llvm {
+namespace codeview {
+
+class TypeSerializationVisitor : public TypeVisitorCallbacks {
+public:
+  TypeSerializationVisitor(FieldListRecordBuilder &FieldListBuilder,
+                           MemoryTypeTableBuilder &TypeTableBuilder)
+      : FieldListBuilder(FieldListBuilder), TypeTableBuilder(TypeTableBuilder) {
+  }
+
+  virtual Expected<TypeLeafKind> visitTypeBegin(const CVType &Record) override {
+    if (Record.Type == TypeLeafKind::LF_FIELDLIST)
+      FieldListBuilder.reset();
+    return Record.Type;
+  }
+
+  virtual Error visitTypeEnd(const CVRecord<TypeLeafKind> &Record) override {
+    if (Record.Type == TypeLeafKind::LF_FIELDLIST)
+      TypeTableBuilder.writeFieldList(FieldListBuilder);
+    return Error::success();
+  }
+
+#define TYPE_RECORD(EnumName, EnumVal, Name)                                   \
+  virtual Error visitKnownRecord(const CVRecord<TypeLeafKind> &CVR,            \
+                                 Name##Record &Record) override {              \
+    visitKnownRecordImpl(Record);                                              \
+    return Error::success();                                                   \
+  }
+#define TYPE_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
+#define MEMBER_RECORD(EnumName, EnumVal, Name)                                 \
+  virtual Error visitKnownRecord(const CVRecord<TypeLeafKind> &CVR,            \
+                                 Name##Record &Record) override {              \
+    visitMemberRecordImpl(Record);                                             \
+    return Error::success();                                                   \
+  }
+#define MEMBER_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
+#include "llvm/DebugInfo/CodeView/TypeRecords.def"
+
+private:
+  template <typename RecordKind> void visitKnownRecordImpl(RecordKind &Record) {
+    TypeTableBuilder.writeKnownType(Record);
+  }
+  template <typename RecordKind>
+  void visitMemberRecordImpl(RecordKind &Record) {
+    FieldListBuilder.writeMemberType(Record);
+  }
+
+  void visitKnownRecordImpl(FieldListRecord &FieldList) {}
+
+  FieldListRecordBuilder &FieldListBuilder;
+  MemoryTypeTableBuilder &TypeTableBuilder;
+};
+}
+}
+
+#endif

llvm/trunk/include/llvm/DebugInfo/CodeView/TypeTableBuilder.h

public:
   TypeIndex writeKnownType(const MethodOverloadListRecord &Record);
   TypeIndex writeKnownType(const TypeServer2Record &Record);
 
   TypeIndex writeFieldList(FieldListRecordBuilder &FieldList);
 
   TypeIndex writeRecord(TypeRecordBuilder &builder);
 
   virtual TypeIndex writeRecord(llvm::StringRef record) = 0;
+
+  ArrayRef<TypeRecordKind> getRecordKinds() const { return RecordKinds; }
+
+private:
+  std::vector<TypeRecordKind> RecordKinds;
 };
 }
 }
 
 #endif

llvm/trunk/include/llvm/DebugInfo/MSF/SequencedItemStream.h

+//===- SequencedItemStream.h ------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_MSF_SEQUENCEDITEMSTREAM_H
+#define LLVM_DEBUGINFO_MSF_SEQUENCEDITEMSTREAM_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/DebugInfo/MSF/MSFError.h"
+#include "llvm/DebugInfo/MSF/StreamInterface.h"
+#include "llvm/Support/Error.h"
+#include <cstdint>
+#include <memory>
+#include <type_traits>
+
+namespace llvm {
+namespace msf {
+template <typename T> struct SequencedItemTraits {
+  static size_t length(const T &Item) = delete;
+  static ArrayRef<uint8_t> bytes(const T &Item) = delete;
+};
+
+/// SequencedItemStream represents a sequence of objects stored in a
+/// standard container but for which it is useful to view as a stream of
+/// contiguous bytes.  An example of this might be if you have a std::vector
+/// of TPI records, where each record contains a byte sequence that
+/// represents that one record serialized, but where each consecutive item
+/// might not be allocated immediately after the previous item.  Using a
+/// SequencedItemStream, we can adapt the VarStreamArray class to trivially
+/// extract one item at a time, allowing the data to be used anywhere a
+/// VarStreamArray could be used.
+template <typename T, typename Traits = SequencedItemTraits<T>>
+class SequencedItemStream : public ReadableStream {
+public:
+  SequencedItemStream() {}
+
+  Error readBytes(uint32_t Offset, uint32_t Size,
+                  ArrayRef<uint8_t> &Buffer) const override {
+    auto ExpectedIndex = translateOffsetIndex(Offset);
+    if (!ExpectedIndex)
+      return ExpectedIndex.takeError();
+    const auto &Item = Items[*ExpectedIndex];
+    if (Size > Traits::length(Item))
+      return make_error<MSFError>(msf_error_code::insufficient_buffer);
+    Buffer = Traits::bytes(Item).take_front(Size);
+    return Error::success();
+  }
+
+  Error readLongestContiguousChunk(uint32_t Offset,
+                                   ArrayRef<uint8_t> &Buffer) const override {
+    auto ExpectedIndex = translateOffsetIndex(Offset);
+    if (!ExpectedIndex)
+      return ExpectedIndex.takeError();
+    Buffer = Traits::bytes(Items[*ExpectedIndex]);
+    return Error::success();
+  }
+
+  void setItems(ArrayRef<T> ItemArray) { Items = ItemArray; }
+
+  uint32_t getLength() const override {
+    uint32_t Size = 0;
+    for (const auto &Item : Items)
+      Size += Traits::length(Item);
+    return Size;
+  }
+
+private:
+  Expected<uint32_t> translateOffsetIndex(uint32_t Offset) const {
+    uint32_t CurrentOffset = 0;
+    uint32_t CurrentIndex = 0;
+    for (const auto &Item : Items) {
+      if (CurrentOffset >= Offset)
+        break;
+      CurrentOffset += Traits::length(Item);
+      ++CurrentIndex;
+    }
+    if (CurrentOffset != Offset)
+      return make_error<MSFError>(msf_error_code::insufficient_buffer);
+    return CurrentIndex;
+  }
+  ArrayRef<T> Items;
+};
+} // end namespace msf
+} // end namespace llvm
+
+#endif // LLVM_DEBUGINFO_MSF_SEQUENCEDITEMSTREAM_H

llvm/trunk/include/llvm/DebugInfo/MSF/StreamArray.h

 //===- StreamArray.h - Array backed by an arbitrary stream ----------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_DEBUGINFO_MSF_STREAMARRAY_H
 #define LLVM_DEBUGINFO_MSF_STREAMARRAY_H
 
+#include "llvm/DebugInfo/MSF/SequencedItemStream.h"
 #include "llvm/DebugInfo/MSF/StreamRef.h"
 #include "llvm/Support/Error.h"
 
 #include <functional>
 #include <type_traits>
 
 namespace llvm {
 namespace msf {

llvm/trunk/include/llvm/DebugInfo/PDB/Raw/PDBFile.h

public:
   Expected<InfoStream &> getPDBInfoStream();
   Expected<DbiStream &> getPDBDbiStream();
   Expected<TpiStream &> getPDBTpiStream();
   Expected<TpiStream &> getPDBIpiStream();
   Expected<PublicsStream &> getPDBPublicsStream();
   Expected<SymbolStream &> getPDBSymbolStream();
   Expected<NameHashTable &> getStringTable();
 
+  BumpPtrAllocator &getAllocator() { return Allocator; }
+
 private:
   BumpPtrAllocator &Allocator;
 
   std::unique_ptr<msf::ReadableStream> Buffer;
 
   std::vector<uint32_t> FpmPages;
   msf::MSFLayout ContainerLayout;
 

llvm/trunk/include/llvm/DebugInfo/PDB/Raw/PDBFileBuilder.h

 
 namespace llvm {
 namespace msf {
 class MSFBuilder;
 }
 namespace pdb {
 class DbiStreamBuilder;
 class InfoStreamBuilder;
+class TpiStreamBuilder;
 
 class PDBFileBuilder {
 public:
   explicit PDBFileBuilder(BumpPtrAllocator &Allocator);
   PDBFileBuilder(const PDBFileBuilder &) = delete;
   PDBFileBuilder &operator=(const PDBFileBuilder &) = delete;
 
   Error initialize(const msf::SuperBlock &Super);
 
   msf::MSFBuilder &getMsfBuilder();
   InfoStreamBuilder &getInfoBuilder();
   DbiStreamBuilder &getDbiBuilder();
+  TpiStreamBuilder &getTpiBuilder();
 
   Expected<std::unique_ptr<PDBFile>>
   build(std::unique_ptr<msf::WritableStream> PdbFileBuffer);
 
   Error commit(const msf::WritableStream &Buffer);
 
 private:
   Expected<msf::MSFLayout> finalizeMsfLayout() const;
 
   BumpPtrAllocator &Allocator;
 
   std::unique_ptr<msf::MSFBuilder> Msf;
   std::unique_ptr<InfoStreamBuilder> Info;
   std::unique_ptr<DbiStreamBuilder> Dbi;
+  std::unique_ptr<TpiStreamBuilder> Tpi;
 };
 }
 }
 
 #endif

llvm/trunk/include/llvm/DebugInfo/PDB/Raw/RawTypes.h

 
 /// Defines a 128-bit unique identifier.  This maps to a GUID on Windows, but
 /// is abstracted here for the purposes of non-Windows platforms that don't have
 /// the GUID structure defined.
 struct PDB_UniqueId {
   char Guid[16];
 };
 
+// The header preceeding the global TPI stream.
+// This corresponds to `HDR` in PDB/dbi/tpi.h.
+struct TpiStreamHeader {
+  struct EmbeddedBuf {
+    support::little32_t Off;
+    support::ulittle32_t Length;
+  };
+
+  support::ulittle32_t Version;
+  support::ulittle32_t HeaderSize;
+  support::ulittle32_t TypeIndexBegin;
+  support::ulittle32_t TypeIndexEnd;
+  support::ulittle32_t TypeRecordBytes;
+
+  // The following members correspond to `TpiHash` in PDB/dbi/tpi.h.
+  support::ulittle16_t HashStreamIndex;
+  support::ulittle16_t HashAuxStreamIndex;
+  support::ulittle32_t HashKeySize;
+  support::ulittle32_t NumHashBuckets;
+
+  EmbeddedBuf HashValueBuffer;
+  EmbeddedBuf IndexOffsetBuffer;
+  EmbeddedBuf HashAdjBuffer;
+};
+
+const uint32_t MinTpiHashBuckets = 0x1000;
+const uint32_t MaxTpiHashBuckets = 0x40000;
+
 /// The header preceeding the global PDB Stream (Stream 1)
 struct InfoStreamHeader {
   support::ulittle32_t Version;
   support::ulittle32_t Signature;
   support::ulittle32_t Age;
   PDB_UniqueId Guid;
 };
 
 } // namespace pdb
 } // namespace llvm
 
 #endif

llvm/trunk/include/llvm/DebugInfo/PDB/Raw/TpiStream.h

 namespace llvm {
 namespace msf {
 class MappedBlockStream;
 }
 namespace pdb {
 class PDBFile;
 
 class TpiStream {
-  struct HeaderInfo;
+  friend class TpiStreamBuilder;
 
 public:
   TpiStream(const PDBFile &File,
             std::unique_ptr<msf::MappedBlockStream> Stream);
   ~TpiStream();
   Error reload();
 
   PdbRaw_TpiVer getTpiVersion() const;
private:
 
   codeview::CVTypeArray TypeRecords;
 
   std::unique_ptr<msf::MappedBlockStream> HashStream;
   msf::FixedStreamArray<support::ulittle32_t> HashValues;
   msf::FixedStreamArray<TypeIndexOffset> TypeIndexOffsets;
   msf::FixedStreamArray<TypeIndexOffset> HashAdjustments;
 
-  const HeaderInfo *Header;
+  const TpiStreamHeader *Header;
 };
 }
 }
 
 #endif

llvm/trunk/include/llvm/DebugInfo/PDB/Raw/TpiStreamBuilder.h

+//===- TpiStreamBuilder.h - PDB Tpi Stream Creation -------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_PDB_RAW_PDBTPISTREAMBUILDER_H
+#define LLVM_DEBUGINFO_PDB_RAW_PDBTPISTREAMBUILDER_H
+
+#include "llvm/ADT/Optional.h"
+#include "llvm/DebugInfo/CodeView/TypeRecord.h"
+#include "llvm/DebugInfo/MSF/ByteStream.h"
+#include "llvm/DebugInfo/MSF/SequencedItemStream.h"
+#include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Error.h"
+
+#include <vector>
+
+namespace llvm {
+namespace codeview {
+class TypeRecord;
+}
+namespace msf {
+struct MSFLayout;
+class ReadableStreamRef;
+class WritableStream;
+
+template <> struct SequencedItemTraits<llvm::codeview::CVType> {
+  static size_t length(const codeview::CVType &Item) { return Item.Length; }
+  static ArrayRef<uint8_t> bytes(const codeview::CVType &Item) {
+    return Item.RawData;
+  }
+};
+}
+namespace pdb {
+class PDBFile;
+class TpiStream;
+struct TpiStreamHeader;
+
+class TpiStreamBuilder {
+public:
+  explicit TpiStreamBuilder(BumpPtrAllocator &Allocator);
+  ~TpiStreamBuilder();
+
+  TpiStreamBuilder(const TpiStreamBuilder &) = delete;
+  TpiStreamBuilder &operator=(const TpiStreamBuilder &) = delete;
+
+  void setVersionHeader(PdbRaw_TpiVer Version);
+  void addTypeRecord(const codeview::CVType &Record);
+
+  Expected<std::unique_ptr<TpiStream>> build(PDBFile &File,
+                                             const msf::WritableStream &Buffer);
+
+  Error commit(const msf::MSFLayout &Layout, const msf::WritableStream &Buffer);
+
+  uint32_t calculateSerializedLength() const;
+
+private:
+  Error finalize();
+
+  BumpPtrAllocator &Allocator;
+
+  Optional<PdbRaw_TpiVer> VerHeader;
+  std::vector<codeview::CVType> TypeRecords;
+  msf::SequencedItemStream<codeview::CVType> TypeRecordStream;
+
+  const TpiStreamHeader *Header;
+};
+}
+}
+
+#endif

llvm/trunk/lib/CodeGen/AsmPrinter/CodeViewDebug.h

 
 class StringRef;
 class LexicalScope;
 struct ClassInfo;
 
 /// \brief Collects and handles line tables information in a CodeView format.
 class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
   MCStreamer &OS;
+  llvm::BumpPtrAllocator Allocator;
   codeview::MemoryTypeTableBuilder TypeTable;
 
   /// Represents the most general definition range.
   struct LocalVarDefRange {
     /// Indicates that variable data is stored in memory relative to the
     /// specified register.
     int InMemory : 1;
 

llvm/trunk/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp

 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
 
 using namespace llvm;
 using namespace llvm::codeview;
 using namespace llvm::msf;
 
 CodeViewDebug::CodeViewDebug(AsmPrinter *AP)
-    : DebugHandlerBase(AP), OS(*Asm->OutStreamer), CurFn(nullptr) {
+    : DebugHandlerBase(AP), OS(*Asm->OutStreamer), Allocator(),
+      TypeTable(Allocator), CurFn(nullptr) {
   // If module doesn't have named metadata anchors or COFF debug section
   // is not available, skip any debug info related stuff.
   if (!MMI->getModule()->getNamedMetadata("llvm.dbg.cu") ||
       !AP->getObjFileLowering().getCOFFDebugSymbolsSection()) {
     Asm = nullptr;
     return;
   }
 

llvm/trunk/lib/DebugInfo/CodeView/CVTypeVisitor.cpp

 
 Error CVTypeVisitor::visitTypeRecord(const CVRecord<TypeLeafKind> &Record) {
   TypeLeafKind Kind;
   if (auto ExpectedKind = Callbacks.visitTypeBegin(Record))
     Kind = *ExpectedKind;
   else
     return ExpectedKind.takeError();
 
+  CVType RecordCopy = Record;
+  RecordCopy.Type = Kind;
+
   switch (Kind) {
   default:
-    if (auto EC = Callbacks.visitUnknownType(Record))
+    if (auto EC = Callbacks.visitUnknownType(RecordCopy))
       return EC;
     break;
 #define TYPE_RECORD(EnumName, EnumVal, Name)                                   \
   case EnumName: {                                                             \
-    if (auto EC = visitKnownRecord<Name##Record>(Record, Callbacks))           \
+    if (auto EC = visitKnownRecord<Name##Record>(RecordCopy, Callbacks))       \
       return EC;                                                               \
     break;                                                                     \
   }
 #define TYPE_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)                  \
   TYPE_RECORD(EnumVal, EnumVal, AliasName)
 #define MEMBER_RECORD(EnumName, EnumVal, Name)                                 \
   TYPE_RECORD(EnumName, EnumVal, Name)
 #define MEMBER_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)                \
   MEMBER_RECORD(EnumName, EnumVal, AliasName)
 #include "llvm/DebugInfo/CodeView/TypeRecords.def"
   }
 
-  if (auto EC = Callbacks.visitTypeEnd(Record))
+  if (auto EC = Callbacks.visitTypeEnd(RecordCopy))
     return EC;
 
   return Error::success();
 }
 
 /// Visits the type records in Data. Sets the error flag on parse failures.
 Error CVTypeVisitor::visitTypeStream(const CVTypeArray &Types) {
   for (const auto &I : Types) {

llvm/trunk/lib/DebugInfo/CodeView/TypeRecordBuilder.cpp

 //===-- TypeRecordBuilder.cpp ---------------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/DebugInfo/CodeView/TypeRecordBuilder.h"
 
 using namespace llvm;
 using namespace codeview;
 
 TypeRecordBuilder::TypeRecordBuilder(TypeRecordKind Kind)
-    : Stream(Buffer), Writer(Stream) {
+    : Kind(Kind), Stream(Buffer), Writer(Stream) {
   writeTypeRecordKind(Kind);
 }
 
 StringRef TypeRecordBuilder::str() {
   return StringRef(Buffer.data(), Buffer.size());
 }
 
 void TypeRecordBuilder::writeUInt8(uint8_t Value) {

llvm/trunk/lib/DebugInfo/CodeView/TypeTableBuilder.cpp

TypeIndex TypeTableBuilder::writeKnownType(const BuildInfoRecord &Record) {
   assert(Record.getArgs().size() <= UINT16_MAX);
   Builder.writeUInt16(Record.getArgs().size());
   for (TypeIndex Arg : Record.getArgs())
     Builder.writeTypeIndex(Arg);
   return writeRecord(Builder);
 }
 
 TypeIndex TypeTableBuilder::writeRecord(TypeRecordBuilder &Builder) {
-  return writeRecord(Builder.str());
+  TypeIndex I = writeRecord(Builder.str());
+  RecordKinds.push_back(Builder.kind());
+  return I;
 }
 
 TypeIndex TypeTableBuilder::writeFieldList(FieldListRecordBuilder &FieldList) {
   return FieldList.writeListRecord(*this);
 }
 
 TypeIndex
 TypeTableBuilder::writeKnownType(const MethodOverloadListRecord &Record) {

llvm/trunk/lib/DebugInfo/PDB/CMakeLists.txt

add_pdb_impl_folder(Raw
   Raw/NameMap.cpp
   Raw/NameMapBuilder.cpp
   Raw/PDBFile.cpp
   Raw/PDBFileBuilder.cpp
   Raw/PublicsStream.cpp
   Raw/RawError.cpp
   Raw/RawSession.cpp
   Raw/SymbolStream.cpp
-  Raw/TpiStream.cpp)
+  Raw/TpiStream.cpp
+  Raw/TpiStreamBuilder.cpp)
 
 list(APPEND LIBPDB_ADDITIONAL_HEADER_DIRS "${LLVM_MAIN_INCLUDE_DIR}/llvm/DebugInfo/PDB/Raw")
 list(APPEND LIBPDB_ADDITIONAL_HEADER_DIRS "${LLVM_MAIN_INCLUDE_DIR}/llvm/DebugInfo/PDB")
 
 add_llvm_library(LLVMDebugInfoPDB
   GenericError.cpp
   IPDBSourceFile.cpp
   PDB.cpp

llvm/trunk/lib/DebugInfo/PDB/Raw/PDBFileBuilder.cpp

 #include "llvm/DebugInfo/MSF/MSFBuilder.h"
 #include "llvm/DebugInfo/MSF/StreamInterface.h"
 #include "llvm/DebugInfo/MSF/StreamWriter.h"
 #include "llvm/DebugInfo/PDB/Raw/DbiStream.h"
 #include "llvm/DebugInfo/PDB/Raw/DbiStreamBuilder.h"
 #include "llvm/DebugInfo/PDB/Raw/InfoStream.h"
 #include "llvm/DebugInfo/PDB/Raw/InfoStreamBuilder.h"
 #include "llvm/DebugInfo/PDB/Raw/RawError.h"
+#include "llvm/DebugInfo/PDB/Raw/TpiStream.h"
+#include "llvm/DebugInfo/PDB/Raw/TpiStreamBuilder.h"
 
 using namespace llvm;
 using namespace llvm::codeview;
 using namespace llvm::msf;
 using namespace llvm::pdb;
 using namespace llvm::support;
 
 PDBFileBuilder::PDBFileBuilder(BumpPtrAllocator &Allocator)

 }
 
 DbiStreamBuilder &PDBFileBuilder::getDbiBuilder() {
   if (!Dbi)
     Dbi = llvm::make_unique<DbiStreamBuilder>(Allocator);
   return *Dbi;
 }
 
+TpiStreamBuilder &PDBFileBuilder::getTpiBuilder() {
+  if (!Tpi)
+    Tpi = llvm::make_unique<TpiStreamBuilder>(Allocator);
+  return *Tpi;
+}
+
 Expected<msf::MSFLayout> PDBFileBuilder::finalizeMsfLayout() const {
   if (Info) {
     uint32_t Length = Info->calculateSerializedLength();
     if (auto EC = Msf->setStreamSize(StreamPDB, Length))
       return std::move(EC);
   }
   if (Dbi) {
     uint32_t Length = Dbi->calculateSerializedLength();
     if (auto EC = Msf->setStreamSize(StreamDBI, Length))
       return std::move(EC);
   }
+  if (Tpi) {
+    uint32_t Length = Tpi->calculateSerializedLength();
+    if (auto EC = Msf->setStreamSize(StreamTPI, Length))
+      return std::move(EC);
+  }
 
   return Msf->build();
 }
 
 Expected<std::unique_ptr<PDBFile>>
 PDBFileBuilder::build(std::unique_ptr<msf::WritableStream> PdbFileBuffer) {
   auto ExpectedLayout = finalizeMsfLayout();
   if (!ExpectedLayout)
PDBFileBuilder::build(std::unique_ptr<msf::WritableStream> PdbFileBuffer) {
 
   if (Dbi) {
     auto ExpectedDbi = Dbi->build(*File, *PdbFileBuffer);
     if (!ExpectedDbi)
       return ExpectedDbi.takeError();
     File->Dbi = std::move(*ExpectedDbi);
   }
 
+  if (Tpi) {
+    auto ExpectedTpi = Tpi->build(*File, *PdbFileBuffer);
+    if (!ExpectedTpi)
+      return ExpectedTpi.takeError();
+    File->Tpi = std::move(*ExpectedTpi);
+  }
+
   if (File->Info && File->Dbi && File->Info->getAge() != File->Dbi->getAge())
     return llvm::make_error<RawError>(
         raw_error_code::corrupt_file,
         "PDB Stream Age doesn't match Dbi Stream Age!");
 
   return std::move(File);
 }
 
if (auto EC = Info->commit(Layout, Buffer))
       return EC;
   }
 
   if (Dbi) {
     if (auto EC = Dbi->commit(Layout, Buffer))
       return EC;
   }
 
+  if (Tpi) {
+    if (auto EC = Tpi->commit(Layout, Buffer))
+      return EC;
+  }
+
   return Buffer.commit();
 }
 No newline at end of file

llvm/trunk/lib/DebugInfo/PDB/Raw/TpiStream.cpp

 #include "llvm/Support/Endian.h"
 
 using namespace llvm;
 using namespace llvm::codeview;
 using namespace llvm::support;
 using namespace llvm::msf;
 using namespace llvm::pdb;
 
-namespace {
-const uint32_t MinHashBuckets = 0x1000;
-const uint32_t MaxHashBuckets = 0x40000;
-}
-
-// This corresponds to `HDR` in PDB/dbi/tpi.h.
-struct TpiStream::HeaderInfo {
-  struct EmbeddedBuf {
-    little32_t Off;
-    ulittle32_t Length;
-  };
-
-  ulittle32_t Version;
-  ulittle32_t HeaderSize;
-  ulittle32_t TypeIndexBegin;
-  ulittle32_t TypeIndexEnd;
-  ulittle32_t TypeRecordBytes;
-
-  // The following members correspond to `TpiHash` in PDB/dbi/tpi.h.
-  ulittle16_t HashStreamIndex;
-  ulittle16_t HashAuxStreamIndex;
-  ulittle32_t HashKeySize;
-  ulittle32_t NumHashBuckets;
-
-  EmbeddedBuf HashValueBuffer;
-  EmbeddedBuf IndexOffsetBuffer;
-  EmbeddedBuf HashAdjBuffer;
-};
-
 TpiStream::TpiStream(const PDBFile &File,
                      std::unique_ptr<MappedBlockStream> Stream)
     : Pdb(File), Stream(std::move(Stream)) {}
 
 TpiStream::~TpiStream() {}
 
 // Corresponds to `fUDTAnon`.
 template <typename T> static bool isAnonymous(T &Rec) {
Error TpiStream::verifyHashValues() {
 
   CVTypeVisitor Visitor(Pipeline);
   return Visitor.visitTypeStream(TypeRecords);
 }
 
 Error TpiStream::reload() {
   StreamReader Reader(*Stream);
 
-  if (Reader.bytesRemaining() < sizeof(HeaderInfo))
+  if (Reader.bytesRemaining() < sizeof(TpiStreamHeader))
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "TPI Stream does not contain a header.");
 
   if (Reader.readObject(Header))
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "TPI Stream does not contain a header.");
 
   if (Header->Version != PdbTpiV80)
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "Unsupported TPI Version.");
 
-  if (Header->HeaderSize != sizeof(HeaderInfo))
+  if (Header->HeaderSize != sizeof(TpiStreamHeader))
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "Corrupt TPI Header size.");
 
   if (Header->HashKeySize != sizeof(ulittle32_t))
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "TPI Stream expected 4 byte hash key size.");
 
-  if (Header->NumHashBuckets < MinHashBuckets ||
-      Header->NumHashBuckets > MaxHashBuckets)
+  if (Header->NumHashBuckets < MinTpiHashBuckets ||
+      Header->NumHashBuckets > MaxTpiHashBuckets)
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "TPI Stream Invalid number of hash buckets.");
 
   // The actual type records themselves come from this stream
   if (auto EC = Reader.readArray(TypeRecords, Header->TypeRecordBytes))
     return EC;
 
   // Hash indices, hash values, etc come from the hash stream.
+  if (Header->HashStreamIndex != kInvalidStreamIndex) {
     if (Header->HashStreamIndex >= Pdb.getNumStreams())
       return make_error<RawError>(raw_error_code::corrupt_file,
                                   "Invalid TPI hash stream index.");
+
     auto HS = MappedBlockStream::createIndexedStream(
         Pdb.getMsfLayout(), Pdb.getMsfBuffer(), Header->HashStreamIndex);
     StreamReader HSR(*HS);
 
-  uint32_t NumHashValues = Header->HashValueBuffer.Length / sizeof(ulittle32_t);
+    uint32_t NumHashValues =
+        Header->HashValueBuffer.Length / sizeof(ulittle32_t);
     if (NumHashValues != NumTypeRecords())
       return make_error<RawError>(
           raw_error_code::corrupt_file,
           "TPI hash count does not match with the number of type records.");
     HSR.setOffset(Header->HashValueBuffer.Off);
     if (auto EC = HSR.readArray(HashValues, NumHashValues))
       return EC;
 
     HSR.setOffset(Header->IndexOffsetBuffer.Off);
     uint32_t NumTypeIndexOffsets =
         Header->IndexOffsetBuffer.Length / sizeof(TypeIndexOffset);
     if (auto EC = HSR.readArray(TypeIndexOffsets, NumTypeIndexOffsets))
       return EC;
 
     HSR.setOffset(Header->HashAdjBuffer.Off);
     uint32_t NumHashAdjustments =
         Header->HashAdjBuffer.Length / sizeof(TypeIndexOffset);
     if (auto EC = HSR.readArray(HashAdjustments, NumHashAdjustments))
       return EC;
 
     HashStream = std::move(HS);
 
     // TPI hash table is a parallel array for the type records.
     // Verify that the hash values match with type records.
     if (auto EC = verifyHashValues())
       return EC;
+  }
 
   return Error::success();
 }
 
 PdbRaw_TpiVer TpiStream::getTpiVersion() const {
   uint32_t Value = Header->Version;
   return static_cast<PdbRaw_TpiVer>(Value);
 }

llvm/trunk/lib/DebugInfo/PDB/Raw/TpiStreamBuilder.cpp

+#include "llvm/DebugInfo/PDB/Raw/TpiStreamBuilder.h"
+
+#include "llvm/DebugInfo/CodeView/TypeIndex.h"
+#include "llvm/DebugInfo/CodeView/TypeRecord.h"
+#include "llvm/DebugInfo/MSF/MappedBlockStream.h"
+#include "llvm/DebugInfo/MSF/StreamWriter.h"
+#include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
+#include "llvm/DebugInfo/PDB/Raw/RawError.h"
+#include "llvm/DebugInfo/PDB/Raw/TpiStream.h"
+#include "llvm/Support/Allocator.h"
+
+using namespace llvm;
+using namespace llvm::msf;
+using namespace llvm::pdb;
+using namespace llvm::support;
+
+TpiStreamBuilder::TpiStreamBuilder(BumpPtrAllocator &Allocator)
+    : Allocator(Allocator), Header(nullptr) {}
+
+TpiStreamBuilder::~TpiStreamBuilder() {}
+
+void TpiStreamBuilder::setVersionHeader(PdbRaw_TpiVer Version) {
+  VerHeader = Version;
+}
+
+void TpiStreamBuilder::addTypeRecord(const codeview::CVType &Record) {
+  TypeRecords.push_back(Record);
+  TypeRecordStream.setItems(TypeRecords);
+}
+
+Error TpiStreamBuilder::finalize() {
+  if (Header)
+    return Error::success();
+
+  TpiStreamHeader *H = Allocator.Allocate<TpiStreamHeader>();
+
+  uint32_t Count = TypeRecords.size();
+
+  H->Version = *VerHeader;
+  H->HeaderSize = sizeof(TpiStreamHeader);
+  H->TypeIndexBegin = codeview::TypeIndex::FirstNonSimpleIndex;
+  H->TypeIndexEnd = H->TypeIndexBegin + Count;
+  H->TypeRecordBytes = TypeRecordStream.getLength();
+
+  H->HashStreamIndex = kInvalidStreamIndex;
+  H->HashAuxStreamIndex = kInvalidStreamIndex;
+  H->HashKeySize = sizeof(ulittle32_t);
+  H->NumHashBuckets = MinTpiHashBuckets;
+
+  H->HashValueBuffer.Length = 0;
+  H->HashAdjBuffer.Length = 0;
+  H->IndexOffsetBuffer.Length = 0;
+
+  Header = H;
+  return Error::success();
+}
+
+uint32_t TpiStreamBuilder::calculateSerializedLength() const {
+  return sizeof(TpiStreamHeader) + TypeRecordStream.getLength();
+}
+
+Expected<std::unique_ptr<TpiStream>>
+TpiStreamBuilder::build(PDBFile &File, const msf::WritableStream &Buffer) {
+  if (!VerHeader.hasValue())
+    return make_error<RawError>(raw_error_code::unspecified,
+                                "Missing TPI Stream Version");
+  if (auto EC = finalize())
+    return std::move(EC);
+
+  auto StreamData = MappedBlockStream::createIndexedStream(File.getMsfLayout(),
+                                                           Buffer, StreamTPI);
+  auto Tpi = llvm::make_unique<TpiStream>(File, std::move(StreamData));
+  Tpi->Header = Header;
+  Tpi->TypeRecords = VarStreamArray<codeview::CVType>(TypeRecordStream);
+  return std::move(Tpi);
+}
+
+Error TpiStreamBuilder::commit(const msf::MSFLayout &Layout,
+                               const msf::WritableStream &Buffer) {
+  if (auto EC = finalize())
+    return EC;
+
+  auto InfoS =
+      WritableMappedBlockStream::createIndexedStream(Layout, Buffer, StreamTPI);
+
+  StreamWriter Writer(*InfoS);
+  if (auto EC = Writer.writeObject(*Header))
+    return EC;
+
+  auto RecordArray = VarStreamArray<codeview::CVType>(TypeRecordStream);
+  if (auto EC = Writer.writeArray(RecordArray))
+    return EC;
+
+  return Error::success();
+}

llvm/trunk/test/DebugInfo/PDB/pdbdump-readwrite.test

 RUN: llvm-pdbdump pdb2yaml -dbi-module-info -dbi-module-source-info \
-RUN:   -dbi-stream -pdb-stream -stream-directory -stream-metadata \
-RUN:   %p/Inputs/empty.pdb > %t.1
+RUN:   -dbi-stream -pdb-stream -tpi-stream -stream-directory \
+RUN:   -stream-metadata %p/Inputs/empty.pdb > %t.1
 RUN: llvm-pdbdump yaml2pdb -pdb=%t.2 %t.1
 
-RUN: llvm-pdbdump raw -headers %p/Inputs/empty.pdb | FileCheck %s
-RUN: llvm-pdbdump raw -headers %t.2 | FileCheck %s
+RUN: llvm-pdbdump raw -headers -tpi-records %p/Inputs/empty.pdb | FileCheck %s
+RUN: llvm-pdbdump raw -headers -tpi-records %t.2 | FileCheck %s
 
 CHECK:      FileHeaders {
 CHECK-NEXT:   BlockSize: 4096
 CHECK-NEXT:   FreeBlockMap: 2
 CHECK-NEXT:   NumBlocks: 25
 CHECK-NEXT:   NumDirectoryBytes: 136
 CHECK-NEXT:   Unknown1: 0
 CHECK-NEXT:   BlockMapAddr: 24
 CHECK-NEXT:   NumDirectoryBlocks: 1
 CHECK-NEXT:   DirectoryBlocks: [23]
 CHECK-NEXT:   NumStreams: 17
 CHECK-NEXT: }
-CHECK-NEXT: PDB Stream {
+CHECK:      PDB Stream {
 CHECK-NEXT:   Version: 20000404
 CHECK-NEXT:   Signature: 0x54E507E2
 CHECK-NEXT:   Age: 1
 CHECK-NEXT:   Guid: {0B355641-86A0-A249-896F-9988FAE52FF0}
 CHECK-NEXT: }
-CHECK-NEXT: DBI Stream {
+CHECK:      Type Info Stream (TPI) {
+CHECK-NEXT:   TPI Version: 20040203
+CHECK-NEXT:   Record count: 75
+CHECK:      DBI Stream {
 CHECK-NEXT:   Dbi Version: 19990903
 CHECK-NEXT:   Age: 1
 CHECK-NEXT:   Incremental Linking: Yes
 CHECK-NEXT:   Has CTypes: No
 CHECK-NEXT:   Is Stripped: No
 CHECK-NEXT:   Machine Type: x86
 CHECK-NEXT:   Symbol Record Stream Index:
 CHECK-NEXT:   Public Symbol Stream Index:
 CHECK-NEXT:   Global Symbol Stream Index:
 CHECK-NEXT:   Toolchain Version: 12.0
 CHECK-NEXT:   mspdb120.dll version: 12.0.31101
 CHECK-NEXT: }

llvm/trunk/test/DebugInfo/PDB/pdbdump-write.test

 ; This testcase checks to make sure that we can write PDB files.  It
 ; works by first reading a known good PDB file and dumping the contents
 ; to YAML.  Then it tries to reconstruct as much of the original PDB as
 ; possible, although depending on what flags are specified when generating
 ; the YAML, the PDB might be missing data required for any standard tool
 ; to recognize it.  Finally, it dumps the same set of fields from the newly
 ; constructed PDB to YAML, and verifies that the YAML is the same as the
 ; original YAML generated from the good PDB.  Note that when doing the
 ; final comparison it must dump the original and the new pdb without any
 ; stream metadata, since the layout of the MSF file might be different
 ; (for example if we don't write the entire stream)
 ;
-; RUN: llvm-pdbdump pdb2yaml -stream-metadata -stream-directory -pdb-stream %p/Inputs/empty.pdb > %t.1
+; RUN: llvm-pdbdump pdb2yaml -stream-metadata -stream-directory -pdb-stream -tpi-stream %p/Inputs/empty.pdb > %t.1
 ; RUN: llvm-pdbdump yaml2pdb -pdb=%t.2 %t.1
-; RUN: llvm-pdbdump pdb2yaml -pdb-stream %p/Inputs/empty.pdb > %t.3
-; RUN: llvm-pdbdump pdb2yaml -pdb-stream %t.2 > %t.4
+; RUN: llvm-pdbdump pdb2yaml -pdb-stream -tpi-stream %p/Inputs/empty.pdb > %t.3
+; RUN: llvm-pdbdump pdb2yaml -pdb-stream -tpi-stream %t.2 > %t.4
 ; RUN: diff %t.3 %t.4

llvm/trunk/tools/llvm-pdbdump/CodeViewYaml.h

 //===- PdbYAML.h ---------------------------------------------- *- C++ --*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_TOOLS_LLVMPDBDUMP_CODEVIEWYAML_H
 #define LLVM_TOOLS_LLVMPDBDUMP_CODEVIEWYAML_H
 
 #include "llvm/DebugInfo/CodeView/CodeView.h"
+#include "llvm/DebugInfo/CodeView/MemoryTypeTableBuilder.h"
 #include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h"
 #include "llvm/Support/YAMLTraits.h"
 
 namespace llvm {
+namespace pdb {
+namespace yaml {
+struct SerializationContext;
+}
+}
 namespace codeview {
 namespace yaml {
 class YamlTypeDumperCallbacks : public TypeVisitorCallbacks {
 public:
-  YamlTypeDumperCallbacks(llvm::yaml::IO &IO) : YamlIO(IO) {}
+  YamlTypeDumperCallbacks(llvm::yaml::IO &IO,
+                          llvm::pdb::yaml::SerializationContext &Context)
+      : YamlIO(IO), Context(Context) {}
 
   virtual Expected<TypeLeafKind>
   visitTypeBegin(const CVRecord<TypeLeafKind> &Record) override;
 
 #define TYPE_RECORD(EnumName, EnumVal, Name)                                   \
   Error visitKnownRecord(const CVRecord<TypeLeafKind> &CVR,                    \
                          Name##Record &Record) override {                      \
     visitKnownRecordImpl(#Name, CVR, Record);                                  \
     return Error::success();                                                   \
   }
 #define MEMBER_RECORD(EnumName, EnumVal, Name)                                 \
   TYPE_RECORD(EnumName, EnumVal, Name)
 #define TYPE_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
 #define MEMBER_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
 #include "llvm/DebugInfo/CodeView/TypeRecords.def"
 
 private:
   template <typename T>
   void visitKnownRecordImpl(const char *Name, const CVType &Type, T &Record) {
     YamlIO.mapRequired(Name, Record);
   }
 
-  void visitKnownRecordImpl(const char *Name, const CVType &Type,
+  void visitKnownRecordImpl(const char *Name, const CVType &CVR,
                             FieldListRecord &FieldList);
 
   llvm::yaml::IO &YamlIO;
+  llvm::pdb::yaml::SerializationContext &Context;
 };
 }
 }
+namespace pdb {
+namespace yaml {
+struct SerializationContext;
+}
+}
 }
 
 namespace llvm {
 namespace yaml {
 template <> struct MappingTraits<codeview::MemberPointerInfo> {
   static void mapping(IO &IO, codeview::MemberPointerInfo &Obj);
 };
 
 template <>
-struct MappingContextTraits<codeview::CVType,
-                            codeview::yaml::YamlTypeDumperCallbacks> {
+struct MappingContextTraits<codeview::CVType, pdb::yaml::SerializationContext> {
   static void mapping(IO &IO, codeview::CVType &Obj,
-                      codeview::yaml::YamlTypeDumperCallbacks &Context);
+                      pdb::yaml::SerializationContext &Context);
 };
 
 template <> struct ScalarEnumerationTraits<codeview::TypeLeafKind> {
   static void enumeration(IO &io, codeview::TypeLeafKind &Value);
 };
 
 #define TYPE_RECORD(EnumName, EnumVal, Name)                                   \
   template <> struct MappingTraits<codeview::Name##Record> {                   \
     static void mapping(IO &IO, codeview::Name##Record &Obj);                  \
   };
 #define MEMBER_RECORD(EnumName, EnumVal, Name)                                 \
   TYPE_RECORD(EnumName, EnumVal, Name)
 #define TYPE_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
 #define MEMBER_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
 #include "llvm/DebugInfo/CodeView/TypeRecords.def"
-
-template <>
-struct MappingContextTraits<codeview::FieldListRecord,
-                            codeview::yaml::YamlTypeDumperCallbacks> {
-  static void mapping(IO &IO, codeview::FieldListRecord &Record,
-                      codeview::yaml::YamlTypeDumperCallbacks &Context);
-};
 }
 }
 
 #endif

llvm/trunk/tools/llvm-pdbdump/CodeViewYaml.cpp

 //===- PdbYAML.cpp -------------------------------------------- *- C++ --*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "CodeViewYaml.h"
 #include "PdbYaml.h"
+#include "YamlSerializationContext.h"
 
 #include "llvm/DebugInfo/CodeView/CVTypeVisitor.h"
 #include "llvm/DebugInfo/CodeView/EnumTables.h"
 #include "llvm/DebugInfo/CodeView/TypeDeserializer.h"
 #include "llvm/DebugInfo/CodeView/TypeRecord.h"
 #include "llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h"
 
 using namespace llvm;
template <> struct ScalarTraits<APSInt> {
   static StringRef input(StringRef Scalar, void *Ctx, APSInt &S) {
     S = APSInt(Scalar);
     return "";
   }
 
   static bool mustQuote(StringRef Scalar) { return false; }
 };
 
-void MappingContextTraits<CVType, YamlTypeDumperCallbacks>::mapping(
-    IO &IO, CVType &Record, YamlTypeDumperCallbacks &Dumper) {
+void MappingContextTraits<CVType, pdb::yaml::SerializationContext>::mapping(
+    IO &IO, CVType &Record, pdb::yaml::SerializationContext &Context) {
   if (IO.outputting()) {
     codeview::TypeDeserializer Deserializer;
 
     codeview::TypeVisitorCallbackPipeline Pipeline;
     Pipeline.addCallbackToPipeline(Deserializer);
-    Pipeline.addCallbackToPipeline(Dumper);
+    Pipeline.addCallbackToPipeline(Context.Dumper);
 
     codeview::CVTypeVisitor Visitor(Pipeline);
     consumeError(Visitor.visitTypeRecord(Record));
   }
 }
 
-void MappingContextTraits<FieldListRecord, YamlTypeDumperCallbacks>::mapping(
-    IO &IO, FieldListRecord &FieldList, YamlTypeDumperCallbacks &Dumper) {
-  if (IO.outputting()) {
-    codeview::TypeDeserializer Deserializer;
-
-    codeview::TypeVisitorCallbackPipeline Pipeline;
-    Pipeline.addCallbackToPipeline(Deserializer);
-    Pipeline.addCallbackToPipeline(Dumper);
-
-    codeview::CVTypeVisitor Visitor(Pipeline);
-    consumeError(Visitor.visitFieldListMemberStream(FieldList.Data));
-  }
-}
-
 void MappingTraits<StringIdRecord>::mapping(IO &IO, StringIdRecord &String) {
   IO.mapRequired("Id", String.Id);
   IO.mapRequired("String", String.String);
 }
 
 void MappingTraits<ArgListRecord>::mapping(IO &IO, ArgListRecord &Args) {
   IO.mapRequired("ArgIndices", Args.StringIndices);
 }
llvm::codeview::yaml::YamlTypeDumperCallbacks::visitTypeBegin(
   // when we're inputting, we need to read the value.  Since `CVR.Type` is const
   // we do it into a temp variable.
   TypeLeafKind K = CVR.Type;
   YamlIO.mapRequired("Kind", K);
   return K;
 }
 
 void llvm::codeview::yaml::YamlTypeDumperCallbacks::visitKnownRecordImpl(
-    const char *Name, const CVType &Type, FieldListRecord &FieldList) {
-
-  std::vector<llvm::pdb::yaml::PdbTpiRecord> Records;
+    const char *Name, const CVType &CVR, FieldListRecord &FieldList) {
+  std::vector<llvm::pdb::yaml::PdbTpiRecord> FieldListRecords;
   if (YamlIO.outputting()) {
-    FieldListRecordSplitter Splitter(Records);
+    // If we are outputting, then `FieldList.Data` contains a huge chunk of data
+    // representing the serialized list of members.  We need to split it up into
+    // individual CVType records where each record represents an individual
+    // member.  This way, we can simply map the entire thing as a Yaml sequence,
+    // which will recurse back to the standard handler for top-level fields
+    // (top-level and member fields all have the exact same Yaml syntax so use
+    // the same parser).
+    //
+    // If we are not outputting, then the array contains no data starting out,
+    // and is instead populated from the sequence represented by the yaml --
+    // again, using the same logic that we use for top-level records.
+    FieldListRecordSplitter Splitter(FieldListRecords);
     CVTypeVisitor V(Splitter);
     consumeError(V.visitFieldListMemberStream(FieldList.Data));
   }
-  YamlIO.mapRequired(Name, Records);
+  YamlIO.mapRequired("FieldList", FieldListRecords, Context);
 }

llvm/trunk/tools/llvm-pdbdump/PdbYaml.h

 #include "llvm/Support/YAMLTraits.h"
 
 #include <vector>
 
 namespace llvm {
 namespace pdb {
 
 namespace yaml {
+struct SerializationContext;
+
 struct MSFHeaders {
   msf::SuperBlock SuperBlock;
   uint32_t NumDirectoryBlocks;
   std::vector<uint32_t> DirectoryBlocks;
   uint32_t NumStreams;
   uint32_t FileSize;
 };
 

 };
 
 struct PdbTpiStream {
   PdbRaw_TpiVer Version;
   std::vector<PdbTpiRecord> Records;
 };
 
 struct PdbObject {
+  explicit PdbObject(BumpPtrAllocator &Allocator) : Allocator(Allocator) {}
+
   Optional<MSFHeaders> Headers;
   Optional<std::vector<uint32_t>> StreamSizes;
   Optional<std::vector<StreamBlockList>> StreamMap;
   Optional<PdbInfoStream> PdbStream;
   Optional<PdbDbiStream> DbiStream;
   Optional<PdbTpiStream> TpiStream;
+
+  BumpPtrAllocator &Allocator;
 };
 }
 }
 }
 
 namespace llvm {
 namespace yaml {
 

 template <> struct MappingTraits<pdb::yaml::PdbInfoStream> {
   static void mapping(IO &IO, pdb::yaml::PdbInfoStream &Obj);
 };
 
 template <> struct MappingTraits<pdb::yaml::PdbDbiStream> {
   static void mapping(IO &IO, pdb::yaml::PdbDbiStream &Obj);
 };
 
-template <> struct MappingTraits<pdb::yaml::PdbTpiStream> {
-  static void mapping(IO &IO, pdb::yaml::PdbTpiStream &Obj);
+template <>
+struct MappingContextTraits<pdb::yaml::PdbTpiStream, llvm::BumpPtrAllocator> {
+  static void mapping(IO &IO, pdb::yaml::PdbTpiStream &Obj,
+                      llvm::BumpPtrAllocator &Allocator);
 };
 
 template <> struct MappingTraits<pdb::yaml::NamedStreamMapping> {
   static void mapping(IO &IO, pdb::yaml::NamedStreamMapping &Obj);
 };
 
 template <> struct MappingTraits<pdb::yaml::PdbDbiModuleInfo> {
   static void mapping(IO &IO, pdb::yaml::PdbDbiModuleInfo &Obj);
 };
 
-template <> struct MappingTraits<pdb::yaml::PdbTpiRecord> {
-  static void mapping(IO &IO, pdb::yaml::PdbTpiRecord &Obj);
+template <>
+struct MappingContextTraits<pdb::yaml::PdbTpiRecord,
+                            pdb::yaml::SerializationContext> {
+  static void mapping(IO &IO, pdb::yaml::PdbTpiRecord &Obj,
+                      pdb::yaml::SerializationContext &Context);
 };
 }
 }
 
 #endif // LLVM_TOOLS_LLVMPDBDUMP_PDBYAML_H

llvm/trunk/tools/llvm-pdbdump/PdbYaml.cpp

 //===- PdbYAML.cpp -------------------------------------------- *- C++ --*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "PdbYaml.h"
 
 #include "CodeViewYaml.h"
+#include "YamlSerializationContext.h"
 
 #include "llvm/DebugInfo/CodeView/CVTypeVisitor.h"
 #include "llvm/DebugInfo/CodeView/TypeDeserializer.h"
+#include "llvm/DebugInfo/CodeView/TypeSerializationVisitor.h"
 #include "llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h"
 #include "llvm/DebugInfo/PDB/PDBExtras.h"
 #include "llvm/DebugInfo/PDB/PDBTypes.h"
 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
 
 using namespace llvm;
 using namespace llvm::pdb;
 using namespace llvm::pdb::yaml;

 }
 
 void MappingTraits<PdbObject>::mapping(IO &IO, PdbObject &Obj) {
   IO.mapOptional("MSF", Obj.Headers);
   IO.mapOptional("StreamSizes", Obj.StreamSizes);
   IO.mapOptional("StreamMap", Obj.StreamMap);
   IO.mapOptional("PdbStream", Obj.PdbStream);
   IO.mapOptional("DbiStream", Obj.DbiStream);
-  IO.mapOptional("TpiStream", Obj.TpiStream);
+  IO.mapOptionalWithContext("TpiStream", Obj.TpiStream, Obj.Allocator);
 }
 
 void MappingTraits<MSFHeaders>::mapping(IO &IO, MSFHeaders &Obj) {
   IO.mapRequired("SuperBlock", Obj.SuperBlock);
   IO.mapRequired("NumDirectoryBlocks", Obj.NumDirectoryBlocks);
   IO.mapRequired("DirectoryBlocks", Obj.DirectoryBlocks);
   IO.mapRequired("NumStreams", Obj.NumStreams);
   IO.mapRequired("FileSize", Obj.FileSize);
void MappingTraits<PdbDbiStream>::mapping(IO &IO, PdbDbiStream &Obj) {
   IO.mapRequired("BuildNumber", Obj.BuildNumber);
   IO.mapRequired("PdbDllVersion", Obj.PdbDllVersion);
   IO.mapRequired("PdbDllRbld", Obj.PdbDllRbld);
   IO.mapRequired("Flags", Obj.Flags);
   IO.mapRequired("MachineType", Obj.MachineType);
   IO.mapOptional("Modules", Obj.ModInfos);
 }
 
-void MappingTraits<PdbTpiStream>::mapping(IO &IO,
-                                          pdb::yaml::PdbTpiStream &Obj) {
+void MappingContextTraits<PdbTpiStream, BumpPtrAllocator>::mapping(
+    IO &IO, pdb::yaml::PdbTpiStream &Obj, BumpPtrAllocator &Allocator) {
+  // Create a single serialization context that will be passed through the
+  // entire process of serializing / deserializing a Tpi Stream.  This is
+  // especially important when we are going from Pdb -> Yaml because we need
+  // to maintain state in a TypeTableBuilder across mappings, and at the end of
+  // the entire process, we need to have one TypeTableBuilder that has every
+  // record.
+  pdb::yaml::SerializationContext Context(IO, Allocator);
+
   IO.mapRequired("Version", Obj.Version);
-  IO.mapRequired("Records", Obj.Records);
+  IO.mapRequired("Records", Obj.Records, Context);
 }
 
 void MappingTraits<NamedStreamMapping>::mapping(IO &IO,
                                                 NamedStreamMapping &Obj) {
   IO.mapRequired("Name", Obj.StreamName);
   IO.mapRequired("StreamNum", Obj.StreamNumber);
 }
 
 void MappingTraits<PdbDbiModuleInfo>::mapping(IO &IO, PdbDbiModuleInfo &Obj) {
   IO.mapRequired("Module", Obj.Mod);
   IO.mapRequired("ObjFile", Obj.Obj);
   IO.mapOptional("SourceFiles", Obj.SourceFiles);
 }
 
-void MappingTraits<PdbTpiRecord>::mapping(IO &IO,
-                                          pdb::yaml::PdbTpiRecord &Obj) {
-  if (IO.outputting()) {
-    codeview::TypeDeserializer Deserializer;
-    codeview::yaml::YamlTypeDumperCallbacks Callbacks(IO);
+void MappingContextTraits<PdbTpiRecord, pdb::yaml::SerializationContext>::
+    mapping(IO &IO, pdb::yaml::PdbTpiRecord &Obj,
+            pdb::yaml::SerializationContext &Context) {
+  codeview::TypeVisitorCallbackPipeline Pipeline;
+  codeview::TypeSerializationVisitor Serializer(Context.FieldListBuilder,
+                                                Context.TypeTableBuilder);
 
-    codeview::TypeVisitorCallbackPipeline Pipeline;
+  if (IO.outputting()) {
+    // For PDB to Yaml, deserialize into a high level record type, then dump it.
     Pipeline.addCallbackToPipeline(Deserializer);
-    Pipeline.addCallbackToPipeline(Callbacks);
+    Pipeline.addCallbackToPipeline(Context.Dumper);
+  } else {
+    // For Yaml to PDB, extract from the high level record type, then write it
+    // to bytes.
+    Pipeline.addCallbackToPipeline(Context.Dumper);
+    Pipeline.addCallbackToPipeline(Serializer);
+  }
 
-    codeview::CVTypeVisitor Visitor(Pipeline);
-    consumeError(Visitor.visitTypeRecord(Obj.Record));
-  } else {
-    codeview::yaml::YamlTypeDumperCallbacks Callbacks(IO);
-    codeview::CVTypeVisitor Visitor(Callbacks);
-    consumeError(Visitor.visitTypeRecord(Obj.Record));
+
+  if (!IO.outputting()) {
+    // For Yaml to PDB, we need to update the input Object with the bytes for
+    // this record.
+    ArrayRef<StringRef> Records = Context.TypeTableBuilder.getRecords();
+    ArrayRef<codeview::TypeRecordKind> Kinds =
+        Context.TypeTableBuilder.getRecordKinds();
+
+    StringRef ThisRecord = Records.back();
+    Obj.Record.Type = static_cast<codeview::TypeLeafKind>(Kinds.back());
+    Obj.Record.Data =
+        ArrayRef<uint8_t>(ThisRecord.bytes_begin(), ThisRecord.bytes_end());
+    Obj.Record.RawData = Obj.Record.Data;
+    Obj.Record.Length = ThisRecord.size();
   }
 }

llvm/trunk/tools/llvm-pdbdump/YAMLOutputStyle.cpp

 #include "llvm/DebugInfo/PDB/Raw/InfoStream.h"
 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
 #include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
 #include "llvm/DebugInfo/PDB/Raw/TpiStream.h"
 
 using namespace llvm;
 using namespace llvm::pdb;
 
-YAMLOutputStyle::YAMLOutputStyle(PDBFile &File) : File(File), Out(outs()) {}
+YAMLOutputStyle::YAMLOutputStyle(PDBFile &File)
+    : File(File), Out(outs()), Obj(File.getAllocator()) {}
 
 Error YAMLOutputStyle::dump() {
   if (opts::pdb2yaml::StreamDirectory)
     opts::pdb2yaml::StreamMetadata = true;
   if (opts::pdb2yaml::DbiModuleSourceFileInfo)
     opts::pdb2yaml::DbiModuleInfo = true;
   if (opts::pdb2yaml::DbiModuleInfo)
     opts::pdb2yaml::DbiStream = true;

llvm/trunk/tools/llvm-pdbdump/YamlSerializationContext.h

+//===- YamlSerializationContext.h ----------------------------- *- C++ --*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVMPDBDUMP_YAMLSERIALIZATIONCONTEXT_H
+#define LLVM_TOOLS_LLVMPDBDUMP_YAMLSERIALIZATIONCONTEXT_H
+
+#include "CodeViewYaml.h"
+#include "PdbYaml.h"
+#include "llvm/DebugInfo/CodeView/FieldListRecordBuilder.h"
+#include "llvm/DebugInfo/CodeView/MemoryTypeTableBuilder.h"
+
+namespace llvm {
+namespace yaml {
+class IO;
+}
+
+namespace pdb {
+namespace yaml {
+struct SerializationContext {
+  explicit SerializationContext(llvm::yaml::IO &IO, BumpPtrAllocator &Allocator)
+      : Dumper(IO, *this), TypeTableBuilder(Allocator) {}
+  codeview::yaml::YamlTypeDumperCallbacks Dumper;
+  codeview::MemoryTypeTableBuilder TypeTableBuilder;
+  codeview::FieldListRecordBuilder FieldListBuilder;
+};
+}
+}
+}
+
+#endif
 No newline at end of file

llvm/trunk/tools/llvm-pdbdump/llvm-pdbdump.cpp

 #include "llvm/DebugInfo/PDB/Raw/DbiStreamBuilder.h"
 #include "llvm/DebugInfo/PDB/Raw/InfoStream.h"
 #include "llvm/DebugInfo/PDB/Raw/InfoStreamBuilder.h"
 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
 #include "llvm/DebugInfo/PDB/Raw/PDBFileBuilder.h"
 #include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
 #include "llvm/DebugInfo/PDB/Raw/RawError.h"
 #include "llvm/DebugInfo/PDB/Raw/RawSession.h"
+#include "llvm/DebugInfo/PDB/Raw/TpiStream.h"
+#include "llvm/DebugInfo/PDB/Raw/TpiStreamBuilder.h"
 #include "llvm/Support/COM.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/FileOutputBuffer.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/MemoryBuffer.h"
static void yamlToPdb(StringRef Path) {
 
   if (ErrorOrBuffer.getError()) {
     ExitOnErr(make_error<GenericError>(generic_error_code::invalid_path, Path));
   }
 
   std::unique_ptr<MemoryBuffer> &Buffer = ErrorOrBuffer.get();
 
   llvm::yaml::Input In(Buffer->getBuffer());
-  pdb::yaml::PdbObject YamlObj;
+  pdb::yaml::PdbObject YamlObj(Allocator);
   In >> YamlObj;
   if (!YamlObj.Headers.hasValue())
     ExitOnErr(make_error<GenericError>(generic_error_code::unspecified,
                                        "Yaml does not contain MSF headers"));
 
   auto OutFileOrError = FileOutputBuffer::create(
       opts::yaml2pdb::YamlPdbOutputFile, YamlObj.Headers->FileSize);
   if (OutFileOrError.getError())
if (YamlObj.DbiStream.hasValue()) {
     DbiBuilder.setVersionHeader(YamlObj.DbiStream->VerHeader);
     for (const auto &MI : YamlObj.DbiStream->ModInfos) {
       ExitOnErr(DbiBuilder.addModuleInfo(MI.Obj, MI.Mod));
       for (auto S : MI.SourceFiles)
         ExitOnErr(DbiBuilder.addModuleSourceFile(MI.Mod, S));
     }
   }
 
+  if (YamlObj.TpiStream.hasValue()) {
+    auto &TpiBuilder = Builder.getTpiBuilder();
+    TpiBuilder.setVersionHeader(YamlObj.TpiStream->Version);
+    for (const auto &R : YamlObj.TpiStream->Records)
+      TpiBuilder.addTypeRecord(R.Record);
+  }
+
   ExitOnErr(Builder.commit(*FileByteStream));
 }
 
 static void pdb2Yaml(StringRef Path) {
   std::unique_ptr<IPDBSession> Session;
   ExitOnErr(loadDataForPDB(PDB_ReaderType::Raw, Path, Session));
 
   RawSession *RS = static_cast<RawSession *>(Session.get());

llvm/trunk/tools/llvm-readobj/llvm-readobj.cpp

static bool isMipsArch(unsigned Arch) {
   case llvm::Triple::mipsel:
   case llvm::Triple::mips64:
   case llvm::Triple::mips64el:
     return true;
   default:
     return false;
   }
 }
-
-static llvm::codeview::MemoryTypeTableBuilder CVTypes;
+namespace {
+struct TypeTableBuilder {
+  TypeTableBuilder() : Allocator(), Builder(Allocator) {}
+
+  llvm::BumpPtrAllocator Allocator;
+  llvm::codeview::MemoryTypeTableBuilder Builder;
+};
+}
+static TypeTableBuilder CVTypes;
 
 /// @brief Creates an format-specific object file dumper.
 static std::error_code createDumper(const ObjectFile *Obj,
                                     ScopedPrinter &Writer,
                                     std::unique_ptr<ObjDumper> &Result) {
   if (!Obj)
     return readobj_error::unsupported_file_format;
 
if (opts::COFFDirectives)
       Dumper->printCOFFDirectives();
     if (opts::COFFBaseRelocs)
       Dumper->printCOFFBaseReloc();
     if (opts::COFFDebugDirectory)
       Dumper->printCOFFDebugDirectory();
     if (opts::CodeView)
       Dumper->printCodeViewDebugInfo();
     if (opts::CodeViewMergedTypes)
-      Dumper->mergeCodeViewTypes(CVTypes);
+      Dumper->mergeCodeViewTypes(CVTypes.Builder);
   }
   if (Obj->isMachO()) {
     if (opts::MachODataInCode)
       Dumper->printMachODataInCode();
     if (opts::MachOIndirectSymbols)
       Dumper->printMachOIndirectSymbols();
     if (opts::MachOLinkerOptions)
       Dumper->printMachOLinkerOptions();
int main(int argc, const char *argv[]) {
   if (opts::InputFilenames.size() == 0)
     opts::InputFilenames.push_back("-");
 
   std::for_each(opts::InputFilenames.begin(), opts::InputFilenames.end(),
                 dumpInput);
 
   if (opts::CodeViewMergedTypes) {
     ScopedPrinter W(outs());
-    dumpCodeViewMergedTypes(W, CVTypes);
+    dumpCodeViewMergedTypes(W, CVTypes.Builder);
   }
 
   return 0;
 }