Add a compact dumping mode to `llvm-pdbdump`.

[zturner]

Summary

This adds a command line option -compact-records. Currently this will only affect the behavior of tpi-records and -ipi-records but can eventually be updated to affect the behavior of symbol records as well.

Instead of dumping full record details, it outputs a format like this:

TPI Version: 20040203
Record count: 75
Records [
  Index: 0x1000 (8 bytes, offset 0) LF_ARGLIST "()"
  Index: 0x1001 (16 bytes, offset 8) LF_PROCEDURE "int ()"
  Index: 0x1002 (76 bytes, offset 24) LF_FIELDLIST "<field list>"
  Index: 0x1003 (120 bytes, offset 100) LF_ENUM "__vc_attributes::threadingAttribute::threading_e"
  Index: 0x1004 (100 bytes, offset 220) LF_STRUCTURE "__vc_attributes::threadingAttribute"
  Index: 0x1005 (12 bytes, offset 320) LF_POINTER "const __vc_attributes::threadingAttribute*"
  Index: 0x1006 (12 bytes, offset 332) LF_ARGLIST "(__vc_attributes::threadingAttribute::threading_e)"
  Index: 0x1007 (28 bytes, offset 344) LF_MFUNCTION "void __vc_attributes::threadingAttribute::(__vc_attributes::threadingAttribute::threading_e)"
  Index: 0x1008 (28 bytes, offset 372) LF_MFUNCTION "void __vc_attributes::threadingAttribute::()"
  Index: 0x1009 (20 bytes, offset 400) LF_METHODLIST ""
  Index: 0x100a (68 bytes, offset 420) LF_FIELDLIST "<field list>"
  Index: 0x100b (100 bytes, offset 488) LF_STRUCTURE "__vc_attributes::threadingAttribute"
  Index: 0x100c (48 bytes, offset 588) LF_FIELDLIST "<field list>"
  Index: 0x100d (120 bytes, offset 636) LF_ENUM "__vc_attributes::event_receiverAttribute::type_e"
  Index: 0x100e (112 bytes, offset 756) LF_STRUCTURE "__vc_attributes::event_receiverAttribute"
  Index: 0x100f (12 bytes, offset 868) LF_POINTER "const __vc_attributes::event_receiverAttribute*"
  Index: 0x1010 (16 bytes, offset 880) LF_ARGLIST "(__vc_attributes::event_receiverAttribute::type_e, bool)"
  Index: 0x1011 (28 bytes, offset 896) LF_MFUNCTION "void __vc_attributes::event_receiverAttribute::(__vc_attributes::event_receiverAttribute::type_e, bool)"
  Index: 0x1012 (12 bytes, offset 924) LF_ARGLIST "(__vc_attributes::event_receiverAttribute::type_e)"
  Index: 0x1013 (28 bytes, offset 936) LF_MFUNCTION "void __vc_attributes::event_receiverAttribute::(__vc_attributes::event_receiverAttribute::type_e)"
  Index: 0x1014 (28 bytes, offset 964) LF_MFUNCTION "void __vc_attributes::event_receiverAttribute::()"
  Index: 0x1015 (28 bytes, offset 992) LF_METHODLIST ""
  Index: 0x1016 (96 bytes, offset 1,020) LF_FIELDLIST "<field list>"
  Index: 0x1017 (112 bytes, offset 1,116) LF_STRUCTURE "__vc_attributes::event_receiverAttribute"

In addition, this patch prints out the type index offset array in the TPI / IPI streams. After printing all type records, you will get some output that looks like this:

  Index: 0x1691 (16 bytes, offset 153,924) LF_ARRAY ""
  Index: 0x1692 (720 bytes, offset 153,940) LF_FIELDLIST "<field list>"
  Index: 0x1693 (92 bytes, offset 154,660) LF_STRUCTURE "_IMAGE_LOAD_CONFIG_DIRECTORY32"
  Index: 0x1694 (7,484 bytes, offset 154,752) LF_FIELDLIST "<field list>"
  Index: 0x1695 (52 bytes, offset 162,236) LF_ENUM "CXCursorKind"
  Index: 0x1696 (7,660 bytes, offset 162,288) LF_FIELDLIST "<field list>"
  Index: 0x1697 (52 bytes, offset 169,948) LF_ENUM "CXCursorKind"
  Index: 0x1698 (7,716 bytes, offset 170,000) LF_FIELDLIST "<field list>"
  Index: 0x1699 (52 bytes, offset 177,716) LF_ENUM "CXCursorKind"
  TypeIndexOffsets [
    Index: 0x1000, Offset: 0
    Index: 0x103e, Offset: 9,408
    Index: 0x1093, Offset: 16,392
    Index: 0x1104, Offset: 24,624
    Index: 0x11b8, Offset: 32,808
    Index: 0x126a, Offset: 40,964
    Index: 0x133f, Offset: 52,388
    Index: 0x1343, Offset: 115,832
    Index: 0x1355, Offset: 123,100
    Index: 0x1431, Offset: 131,080
    Index: 0x1523, Offset: 141,164
    Index: 0x15c0, Offset: 147,472
    Index: 0x1695, Offset: 162,236
    Index: 0x1697, Offset: 169,948
    Index: 0x1699, Offset: 177,716
  ]
]

With these two options combined, I was able to decipher the way the Index / Offsets array is laid out. It is an array of pairs, sorted by type index, where each pair represents the Type Index of the first item in a "chunk" of records, and the offset at which that record begins. The number of records in chunk N is IndexOffsets[N+1].First - IndexOffsets[N].First, and the spacing is chosen such that each chunk contains the minimum number of consecutive records such that a chunk is >= 8KB.

There are some exceptions to this which I'm investigating. See for example Index 0x1695 and 0x1697 above, the distance between them is < 8KB. But it seems to hold true almost everywhere else, so it shouldn't be too hard to figure out the rest of this edge case.

[ruiu]

llvm/tools/llvm-pdbdump/LLVMOutputStyle.cpp

577

You have using namespace llvm.

llvm/tools/llvm-pdbdump/llvm-pdbdump.cpp

589

I think errs() is unbuffered, so you don't need this. Even if it is buffered, exit() should flush it.

llvm/tools/llvm-pdbdump/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   DebugInfoCodeView
   DebugInfoMSF
   DebugInfoPDB
   Object
   Support
   )
 
 add_llvm_tool(llvm-pdbdump
+  CompactTypeDumpVisitor.cpp
   llvm-pdbdump.cpp
   YamlSymbolDumper.cpp
   YamlTypeDumper.cpp
   LinePrinter.cpp
   LLVMOutputStyle.cpp
   PdbYaml.cpp
   PrettyBuiltinDumper.cpp
   PrettyClassDefinitionDumper.cpp

llvm/tools/llvm-pdbdump/CompactTypeDumpVisitor.h

+//===-- CompactTypeDumpVisitor.h - CodeView type info dumper ----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_CODEVIEW_COMPACTTYPEDUMPVISITOR_H
+#define LLVM_DEBUGINFO_CODEVIEW_COMPACTTYPEDUMPVISITOR_H
+
+#include "llvm/DebugInfo/CodeView/TypeIndex.h"
+#include "llvm/DebugInfo/CodeView/TypeRecord.h"
+#include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h"
+
+namespace llvm {
+class ScopedPrinter;
+namespace codeview {
+class TypeDatabase;
+}
+
+namespace pdb {
+
+/// Dumper for CodeView type streams found in COFF object files and PDB files.
+/// Dumps records on a single line, and ignores member records.
+class CompactTypeDumpVisitor : public codeview::TypeVisitorCallbacks {
+public:
+  CompactTypeDumpVisitor(codeview::TypeDatabase &TypeDB, ScopedPrinter *W);
+
+  /// Paired begin/end actions for all types. Receives all record data,
+  /// including the fixed-length record prefix.
+  Error visitTypeBegin(codeview::CVType &Record) override;
+  Error visitTypeEnd(codeview::CVType &Record) override;
+
+private:
+  ScopedPrinter *W;
+
+  codeview::TypeIndex TI;
+  uint32_t Offset;
+  codeview::TypeDatabase &TypeDB;
+};
+
+} // end namespace pdb
+} // end namespace llvm
+
+#endif

llvm/tools/llvm-pdbdump/CompactTypeDumpVisitor.cpp

+//===-- CompactTypeDumpVisitor.cpp - CodeView type info dumper --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CompactTypeDumpVisitor.h"
+#include "llvm/DebugInfo/CodeView/TypeDatabase.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/ScopedPrinter.h"
+
+using namespace llvm;
+using namespace llvm::codeview;
+using namespace llvm::pdb;
+
+static const EnumEntry<TypeLeafKind> LeafTypeNames[] = {
+#define CV_TYPE(enum, val) {#enum, enum},
+#include "llvm/DebugInfo/CodeView/TypeRecords.def"
+};
+
+static StringRef getLeafName(TypeLeafKind K) {
+  for (const auto &E : LeafTypeNames) {
+    if (E.Value == K)
+      return E.Name;
+  }
+  return StringRef();
+}
+
+CompactTypeDumpVisitor::CompactTypeDumpVisitor(TypeDatabase &TypeDB,
+                                               ScopedPrinter *W)
+    : W(W), TI(TypeIndex::None()), Offset(0), TypeDB(TypeDB) {}
+
+Error CompactTypeDumpVisitor::visitTypeBegin(CVType &Record) {
+  if (TI == TypeIndex::None())
+    TI.setIndex(TypeIndex::FirstNonSimpleIndex);
+  else
+    TI.setIndex(TI.getIndex() + 1);
+
+  return Error::success();
+}
+
+Error CompactTypeDumpVisitor::visitTypeEnd(CVType &Record) {
+  uint32_t I = TI.getIndex();
+  StringRef Leaf = getLeafName(Record.Type);
+  StringRef Name = TypeDB.getTypeName(TI);
+  W->printString(
+      llvm::formatv("Index: {0:x} ({1:N} bytes, offset {2:N}) {3} \"{4}\"", I,
+                    Record.length(), Offset, Leaf, Name)
+          .str());
+
+  Offset += Record.length();
+
+  return Error::success();
+}

llvm/tools/llvm-pdbdump/LLVMOutputStyle.cpp

 //===- LLVMOutputStyle.cpp ------------------------------------ *- C++ --*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "LLVMOutputStyle.h"
 
+#include "CompactTypeDumpVisitor.h"
 #include "llvm-pdbdump.h"
+
 #include "llvm/DebugInfo/CodeView/CVTypeDumper.h"
 #include "llvm/DebugInfo/CodeView/CVTypeVisitor.h"
 #include "llvm/DebugInfo/CodeView/EnumTables.h"
 #include "llvm/DebugInfo/CodeView/ModuleSubstreamVisitor.h"
 #include "llvm/DebugInfo/CodeView/SymbolDumper.h"
 #include "llvm/DebugInfo/CodeView/TypeDatabaseVisitor.h"
 #include "llvm/DebugInfo/CodeView/TypeDeserializer.h"
 #include "llvm/DebugInfo/CodeView/TypeDumpVisitor.h"
 #include "llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h"
 #include "llvm/DebugInfo/MSF/MappedBlockStream.h"
 #include "llvm/DebugInfo/MSF/StreamReader.h"
 #include "llvm/DebugInfo/PDB/PDBExtras.h"
 #include "llvm/DebugInfo/PDB/Raw/DbiStream.h"
 #include "llvm/DebugInfo/PDB/Raw/EnumTables.h"
 #include "llvm/DebugInfo/PDB/Raw/GlobalsStream.h"
 #include "llvm/DebugInfo/PDB/Raw/ISectionContribVisitor.h"
 #include "llvm/DebugInfo/PDB/Raw/InfoStream.h"
 #include "llvm/DebugInfo/PDB/Raw/ModInfo.h"
 #include "llvm/DebugInfo/PDB/Raw/ModStream.h"
 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
 #include "llvm/DebugInfo/PDB/Raw/PublicsStream.h"
 #include "llvm/DebugInfo/PDB/Raw/RawError.h"
 #include "llvm/DebugInfo/PDB/Raw/TpiStream.h"
 #include "llvm/Object/COFF.h"
+#include "llvm/Support/FormatVariadic.h"
 
 #include <unordered_map>
 
 using namespace llvm;
 using namespace llvm::codeview;
 using namespace llvm::msf;
 using namespace llvm::pdb;
 
Error LLVMOutputStyle::dumpTpiStream(uint32_t StreamIdx) {
   if (!DumpRecordBytes && !DumpRecords && !opts::raw::DumpModuleSyms)
     return Error::success();
 
   auto Tpi = (StreamIdx == StreamTPI) ? File.getPDBTpiStream()
                                       : File.getPDBIpiStream();
   if (!Tpi)
     return Tpi.takeError();
 
-  CVTypeDumper Dumper(TypeDB);
+  // Even if the user doesn't want to dump type records, we still need to
+  // iterate them in order to build the type database. So when they want to
+  // dump symbols but not types, don't stick a dumper on the end, just build
+  // the type database.
+  TypeDatabaseVisitor DBV(TypeDB);
+  CompactTypeDumpVisitor CTDV(TypeDB, &P);
+  TypeDumpVisitor TDV(TypeDB, &P, false);
+  TypeDeserializer Deserializer;
+  TypeVisitorCallbackPipeline Pipeline;
+  Pipeline.addCallbackToPipeline(Deserializer);
+  Pipeline.addCallbackToPipeline(DBV);
+
+  CVTypeVisitor Visitor(Pipeline);
+
   if (DumpRecords || DumpRecordBytes) {
     DictScope D(P, Label);
 
     P.printNumber(VerLabel, Tpi->getTpiVersion());
     P.printNumber("Record count", Tpi->NumTypeRecords());
-
     ListScope L(P, "Records");
 
     bool HadError = false;
-    for (auto &Type : Tpi->types(&HadError)) {
-      DictScope DD(P, "");
+    if (opts::raw::CompactRecords)
+      Pipeline.addCallbackToPipeline(CTDV);
+    else
+      Pipeline.addCallbackToPipeline(TDV);
+
+    for (auto Type : Tpi->types(&HadError)) {
+      std::unique_ptr<DictScope> Scope;
+      if (!opts::raw::CompactRecords)
+        Scope.reset(new DictScope(P, ""));
 
       if (DumpRecords) {
-        TypeDumpVisitor TDV(TypeDB, &P, false);
-        if (auto EC = Dumper.dump(Type, TDV))
+        if (auto EC = Visitor.visitTypeRecord(Type))
           return EC;
       }
 
       if (DumpRecordBytes)
         P.printBinaryBlock("Bytes", Type.content());
     }
     dumpTpiHash(P, *Tpi);
     if (HadError)
       return make_error<RawError>(raw_error_code::corrupt_file,
                                   "TPI stream contained corrupt record");
-  } else if (opts::raw::DumpModuleSyms) {
-    // Even if the user doesn't want to dump type records, we still need to
-    // iterate them in order to build the type database. So when they want to
-    // dump symbols but not types, don't stick a dumper on the end, just build
-    // the type database.
-    TypeDatabaseVisitor DBV(TypeDB);
-    TypeDeserializer Deserializer;
-    TypeVisitorCallbackPipeline Pipeline;
-    Pipeline.addCallbackToPipeline(Deserializer);
-    Pipeline.addCallbackToPipeline(DBV);
+    {
+      ListScope L(P, "TypeIndexOffsets");
+      for (const auto &IO : Tpi->getTypeIndexOffsets()) {
+        P.printString(llvm::formatv("Index: {0:x}, Offset: {1:N}",
+                                    IO.Type.getIndex(), (uint32_t)IO.Offset)
+                          .str());
+      }
+    }
 
-    CVTypeVisitor Visitor(Pipeline);
+  } else if (opts::raw::DumpModuleSyms) {
 
     bool HadError = false;
     for (auto Type : Tpi->types(&HadError)) {
       if (auto EC = Visitor.visitTypeRecord(Type))
         return EC;
     }
 
     dumpTpiHash(P, *Tpi);

llvm/tools/llvm-pdbdump/llvm-pdbdump.h

 struct BlockRange {
   uint32_t Min;
   llvm::Optional<uint32_t> Max;
 };
 
 extern llvm::Optional<BlockRange> DumpBlockRange;
 extern llvm::cl::list<uint32_t> DumpStreamData;
 
+extern llvm::cl::opt<bool> CompactRecords;
 extern llvm::cl::opt<bool> DumpGlobals;
 extern llvm::cl::opt<bool> DumpHeaders;
 extern llvm::cl::opt<bool> DumpStreamBlocks;
 extern llvm::cl::opt<bool> DumpStreamSummary;
 extern llvm::cl::opt<bool> DumpPageStats;
 extern llvm::cl::opt<bool> DumpTpiHash;
 extern llvm::cl::opt<bool> DumpTpiRecordBytes;
 extern llvm::cl::opt<bool> DumpTpiRecords;

llvm/tools/llvm-pdbdump/llvm-pdbdump.cpp

 
 cl::list<uint32_t>
     DumpStreamData("stream-data", cl::CommaSeparated, cl::ZeroOrMore,
                    cl::desc("Dump binary data from specified streams."),
                    cl::cat(MsfOptions), cl::sub(RawSubcommand));
 
 // TYPE OPTIONS
 cl::opt<bool>
+    CompactRecords("compact-records",
+                   cl::desc("Dump type and symbol records with less detail"),
+                   cl::cat(TypeOptions), cl::sub(RawSubcommand));
+
+cl::opt<bool>
     DumpTpiRecords("tpi-records",
                    cl::desc("dump CodeView type records from TPI stream"),
                    cl::cat(TypeOptions), cl::sub(RawSubcommand));
 cl::opt<bool> DumpTpiRecordBytes(
     "tpi-record-bytes",
     cl::desc("dump CodeView type record raw bytes from TPI stream"),
     cl::cat(TypeOptions), cl::sub(RawSubcommand));
 cl::opt<bool> DumpTpiHash("tpi-hash", cl::desc("dump CodeView TPI hash stream"),
if (!opts::raw::DumpBlockRangeOpt.empty()) {
     opts::raw::DumpBlockRange.emplace();
     Matches[1].getAsInteger(10, opts::raw::DumpBlockRange->Min);
     if (!Matches[3].empty()) {
       opts::raw::DumpBlockRange->Max.emplace();
       Matches[3].getAsInteger(10, *opts::raw::DumpBlockRange->Max);
     }
   }
 
-  if (opts::RawSubcommand && opts::raw::RawAll) {
+  if (opts::RawSubcommand) {
+    if (opts::raw::RawAll) {
       opts::raw::DumpHeaders = true;
       opts::raw::DumpModules = true;
       opts::raw::DumpModuleFiles = true;
       opts::raw::DumpModuleSyms = true;
       opts::raw::DumpGlobals = true;
       opts::raw::DumpPublics = true;
       opts::raw::DumpSectionHeaders = true;
       opts::raw::DumpStreamSummary = true;
       opts::raw::DumpPageStats = true;
       opts::raw::DumpStreamBlocks = true;
       opts::raw::DumpTpiRecords = true;
       opts::raw::DumpTpiHash = true;
       opts::raw::DumpIpiRecords = true;
       opts::raw::DumpSectionMap = true;
       opts::raw::DumpSectionContribs = true;
       opts::raw::DumpLineInfo = true;
       opts::raw::DumpFpo = true;
     }
 
+    if (opts::raw::CompactRecords &&
+        (opts::raw::DumpTpiRecordBytes || opts::raw::DumpIpiRecordBytes)) {
+      errs() << "-compact-records is incompatible with -tpi-record-bytes and "
+                "-ipi-record-bytes.\n";
+      errs().flush();
+      exit(1);
+    }
+  }
+
   llvm::sys::InitializeCOMRAII COM(llvm::sys::COMThreadingMode::MultiThreaded);
 
   if (opts::PdbToYamlSubcommand) {
     pdb2Yaml(opts::pdb2yaml::InputFilename.front());
   } else if (opts::YamlToPdbSubcommand) {
     yamlToPdb(opts::yaml2pdb::InputFilename.front());
   } else if (opts::PrettySubcommand) {
     if (opts::pretty::Lines)