Parse each module's module info stream

[zturner]

Summary

This basically jsut pulls the major components of the module debug info stream out. It doesn't interpret any of this data semantically because that requires more investigation. But hopefully this at least presents a high level picture of what data is in each module's debug info stream. Subsequent patches will dig deeper and try to parse the symbol substream and the line info substreams.

[rnk]

All of this code is still dead. Are you sure you don't want to hold off on committing this until some kind of byte dumping is hooked up from llvm-pdbdump?

include/llvm/DebugInfo/PDB/Raw/ModStream.h

11

Wouldn't the header guard be named ...RAW_MODSTREAM_H?

lib/DebugInfo/PDB/CMakeLists.txt

36

consider sorting these at some point

lib/DebugInfo/PDB/Raw/ModStream.cpp

32

Same thought as before about adding errorCorruptFile or something.

[ruiu]

lib/DebugInfo/PDB/Raw/ByteStream.cpp

27–29

Why do you manage ownership by hand instead of unique_ptr?

[zturner]

lib/DebugInfo/PDB/Raw/ByteStream.cpp

27–29

Whether or not the data is owned should be optional. I could change the bool to a unique_ptr and only initialize it optionally, seems like that would work and be fine.

[zturner]

Updated based on suggestions. Also now dumps some information from each module's stream, so that it's not dead code anymore.

[rnk]

lgtm

[Eugene.Zelenko]

Committed in r268942.

include/llvm/DebugInfo/PDB/Raw/ByteStream.h

 
 private:
   MutableArrayRef<uint8_t> Data;
-  bool Owned;
+  std::unique_ptr<uint8_t[]> Ownership;
 };
 }
 }

include/llvm/DebugInfo/PDB/Raw/ModStream.h

+//===- ModStream.h - PDB Module Info Stream Access ------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_PDB_RAW_MODSTREAM_H
+#define LLVM_DEBUGINFO_PDB_RAW_MODSTREAM_H
+
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/DebugInfo/CodeView/SymbolRecord.h"
+#include "llvm/DebugInfo/PDB/Raw/ByteStream.h"
+#include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace pdb {
+class PDBFile;
+class ModInfo;
+
+class ModStream {
+public:
+  ModStream(PDBFile &File, const ModInfo &Module);
+  ~ModStream();
+
+  Error reload();
+
+  iterator_range<codeview::SymbolIterator> symbols() const;
+
+private:
+  const ModInfo &Mod;
+
+  MappedBlockStream Stream;
+
+  ByteStream SymbolsSubstream;
+  ByteStream LinesSubstream;
+  ByteStream C13LinesSubstream;
+  ByteStream GlobalRefsSubstream;
+};
+}
+}
+
+#endif

lib/DebugInfo/PDB/CMakeLists.txt

 
 add_pdb_impl_folder(Raw
   Raw/ByteStream.cpp
-  Raw/MappedBlockStream.cpp
-  Raw/ModInfo.cpp
-  Raw/PDBFile.cpp
   Raw/DbiStream.cpp
   Raw/InfoStream.cpp
+  Raw/MappedBlockStream.cpp
+  Raw/ModInfo.cpp
+  Raw/ModStream.cpp
   Raw/NameHashTable.cpp
   Raw/NameMap.cpp
+  Raw/PDBFile.cpp
   Raw/RawError.cpp
   Raw/RawSession.cpp
   Raw/StreamReader.cpp

lib/DebugInfo/PDB/Raw/ByteStream.cpp

 using namespace llvm;
 using namespace llvm::pdb;
 
-ByteStream::ByteStream() : Owned(false) {}
+ByteStream::ByteStream() {}
 
-ByteStream::ByteStream(MutableArrayRef<uint8_t> Bytes) : Owned(false) {
-  initialize(Bytes);
-}
+ByteStream::ByteStream(MutableArrayRef<uint8_t> Bytes) { initialize(Bytes); }
 
-ByteStream::ByteStream(uint32_t Length) : Owned(false) { initialize(Length); }
+ByteStream::ByteStream(uint32_t Length) { initialize(Length); }
 
 ByteStream::~ByteStream() { reset(); }
 
 void ByteStream::reset() {
-  if (Owned)
-    delete[] Data.data();
-  Owned = false;
+  Ownership.reset();
   Data = MutableArrayRef<uint8_t>();
 }
 
 void ByteStream::initialize(MutableArrayRef<uint8_t> Bytes) {
   reset();
   Data = Bytes;
-  Owned = false;
 }
 
 void ByteStream::initialize(uint32_t Length) {
   reset();
-  Data = MutableArrayRef<uint8_t>(new uint8_t[Length], Length);
-  Owned = true;
+  if (Length > 0)
+    Data = MutableArrayRef<uint8_t>(new uint8_t[Length], Length);
+  Ownership.reset(Data.data());
 }
 
 Error ByteStream::initialize(StreamReader &Reader, uint32_t Length) {

lib/DebugInfo/PDB/Raw/ModStream.cpp

+//===- ModStream.cpp - PDB Module Info Stream Access ----------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/PDB/Raw/ModStream.h"
+#include "llvm/DebugInfo/PDB/Raw/ModInfo.h"
+#include "llvm/DebugInfo/PDB/Raw/RawError.h"
+#include "llvm/DebugInfo/PDB/Raw/StreamReader.h"
+
+using namespace llvm;
+using namespace llvm::pdb;
+
+ModStream::ModStream(PDBFile &File, const ModInfo &Module)
+    : Mod(Module), Stream(Module.getModuleStreamIndex(), File) {}
+
+ModStream::~ModStream() {}
+
+Error ModStream::reload() {
+  StreamReader Reader(Stream);
+
+  uint32_t SymbolSize = Mod.getSymbolDebugInfoByteSize();
+  uint32_t C11Size = Mod.getLineInfoByteSize();
+  uint32_t C13Size = Mod.getC13LineInfoByteSize();
+
+  if (C11Size > 0 && C13Size > 0)
+    return llvm::make_error<RawError>(raw_error_code::corrupt_file,
+                                      "Module has both C11 and C13 line info");
+
+  if (auto EC = SymbolsSubstream.initialize(Reader, SymbolSize))
+    return EC;
+  if (auto EC = LinesSubstream.initialize(Reader, C11Size))
+    return EC;
+  if (auto EC = C13LinesSubstream.initialize(Reader, C13Size))
+    return EC;
+
+  uint32_t GlobalRefsSize;
+  if (auto EC = Reader.readInteger(GlobalRefsSize))
+    return EC;
+  if (auto EC = GlobalRefsSubstream.initialize(Reader, GlobalRefsSize))
+    return EC;
+  if (Reader.bytesRemaining() > 0)
+    return llvm::make_error<RawError>(raw_error_code::corrupt_file,
+                                      "Unexpected bytes in module stream.");
+
+  return Error::success();
+}
+
+iterator_range<codeview::SymbolIterator> ModStream::symbols() const {
+  return codeview::makeSymbolRange(SymbolsSubstream.data().slice(4));
+}

test/DebugInfo/PDB/pdbdump-headers.test

-; RUN: llvm-pdbdump --dump-headers -dump-tpi-records -dump-tpi-record-bytes \
+; RUN: llvm-pdbdump --dump-headers -dump-tpi-records -dump-tpi-record-bytes -dump-module-syms \
 ; RUN:              %p/Inputs/empty.pdb | FileCheck -check-prefix=EMPTY %s
 ; RUN: llvm-pdbdump --dump-headers %p/Inputs/big-read.pdb | FileCheck -check-prefix=BIG %s
 ; RUN: llvm-pdbdump --dump-headers %p/Inputs/bad-block-size.pdb | FileCheck -check-prefix=BAD-BLOCK-SIZE %s

 ; EMPTY-NEXT:       1 Contributing Source Files [
 ; EMPTY-NEXT:         d:\src\llvm\test\debuginfo\pdb\inputs\empty.cpp
 ; EMPTY-NEXT:       ]
+; EMPTY-NEXT:       Symbols [
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x1101
+; EMPTY-NEXT:           Length: 54
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 00000000 643A5C73 72635C6C 6C766D5C  |....d:\src\llvm\|
+; EMPTY-NEXT:             0010: 74657374 5C446562 7567496E 666F5C50  |test\DebugInfo\P|
+; EMPTY-NEXT:             0020: 44425C49 6E707574 735C656D 7074792E  |DB\Inputs\empty.|
+; EMPTY-NEXT:             0030: 6F626A00                             |obj.|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x113C
+; EMPTY-NEXT:           Length: 58
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 01200000 07001200 00007D79 00001200  |. ........}y....|
+; EMPTY-NEXT:             0010: 00007D79 00004D69 63726F73 6F667420  |..}y..Microsoft |
+; EMPTY-NEXT:             0020: 28522920 4F707469 6D697A69 6E672043  |(R) Optimizing C|
+; EMPTY-NEXT:             0030: 6F6D7069 6C657200                    |ompiler.|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x1110
+; EMPTY-NEXT:           Length: 42
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 00000000 C4000000 00000000 0A000000  |................|
+; EMPTY-NEXT:             0010: 03000000 08000000 01100000 10000000  |................|
+; EMPTY-NEXT:             0020: 0100016D 61696E00                    |...main.|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x1012
+; EMPTY-NEXT:           Length: 30
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 00000000 00000000 00000000 00000000  |................|
+; EMPTY-NEXT:             0010: 00000000 00000082 12000000           |............|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x6
+; EMPTY-NEXT:           Length: 2
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x114C
+; EMPTY-NEXT:           Length: 6
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 0E100000                             |....|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:       ]
 ; EMPTY-NEXT:     }
 ; EMPTY-NEXT:     {
 ; EMPTY-NEXT:       Name: * Linker *

 ; EMPTY-NEXT:       Has EC Info: No
 ; EMPTY-NEXT:       0 Contributing Source Files [
 ; EMPTY-NEXT:       ]
+; EMPTY-NEXT:       Symbols [
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x1101
+; EMPTY-NEXT:           Length: 18
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 00000000 2A204C69 6E6B6572 202A0000  |....* Linker *..|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x113C
+; EMPTY-NEXT:           Length: 46
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 07000000 03000000 00000000 00000C00  |................|
+; EMPTY-NEXT:             0010: 00007D79 00004D69 63726F73 6F667420  |..}y..Microsoft |
+; EMPTY-NEXT:             0020: 28522920 4C494E4B 00000000           |(R) LINK....|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x113D
+; EMPTY-NEXT:           Length: 170
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 00637764 00643A5C 7372635C 6C6C766D  |.cwd.d:\src\llvm|
+; EMPTY-NEXT:             0010: 5C746573 745C4465 62756749 6E666F5C  |\test\DebugInfo\|
+; EMPTY-NEXT:             0020: 5044425C 496E7075 74730065 78650043  |PDB\Inputs.exe.C|
+; EMPTY-NEXT:             0030: 3A5C5072 6F677261 6D204669 6C657320  |:\Program Files |
+; EMPTY-NEXT:             0040: 28783836 295C4D69 63726F73 6F667420  |(x86)\Microsoft |
+; EMPTY-NEXT:             0050: 56697375 616C2053 74756469 6F203132  |Visual Studio 12|
+; EMPTY-NEXT:             0060: 2E305C56 435C4249 4E5C6C69 6E6B2E65  |.0\VC\BIN\link.e|
+; EMPTY-NEXT:             0070: 78650070 64620064 3A5C7372 635C6C6C  |xe.pdb.d:\src\ll|
+; EMPTY-NEXT:             0080: 766D5C74 6573745C 44656275 67496E66  |vm\test\DebugInf|
+; EMPTY-NEXT:             0090: 6F5C5044 425C496E 70757473 5C656D70  |o\PDB\Inputs\emp|
+; EMPTY-NEXT:             00A0: 74792E70 64620000                    |ty.pdb..|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x112C
+; EMPTY-NEXT:           Length: 18
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 00000500 05000000 10000000 01000100  |................|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x1136
+; EMPTY-NEXT:           Length: 26
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 01000C00 00100000 1A100000 20000060  |............ ..`|
+; EMPTY-NEXT:             0010: 2E746578 74000000                    |.text...|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x1137
+; EMPTY-NEXT:           Length: 26
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 1A100000 20000060 00000000 01002E74  |.... ..`.......t|
+; EMPTY-NEXT:             0010: 65787424 6D6E0000                    |ext$mn..|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x1136
+; EMPTY-NEXT:           Length: 26
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 02000C00 00300000 B2020000 40000040  |.....0......@..@|
+; EMPTY-NEXT:             0010: 2E726461 74610000                    |.rdata..|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x1137
+; EMPTY-NEXT:           Length: 26
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 43010000 40000040 00000000 02002E72  |C...@..@.......r|
+; EMPTY-NEXT:             0010: 64617461 00000000                    |data....|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x1137
+; EMPTY-NEXT:           Length: 26
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 00000000 40000040 43010000 02002E65  |....@..@C......e|
+; EMPTY-NEXT:             0010: 64617461 00000000                    |data....|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x1137
+; EMPTY-NEXT:           Length: 30
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 6E010000 40000040 44010000 02002E72  |n...@..@D......r|
+; EMPTY-NEXT:             0010: 64617461 24646562 75670000           |data$debug..|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x1136
+; EMPTY-NEXT:           Length: 26
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 03000C00 00400000 04000000 400000C0  |.....@......@...|
+; EMPTY-NEXT:             0010: 2E646174 61000000                    |.data...|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x1137
+; EMPTY-NEXT:           Length: 22
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 04000000 800000C0 00000000 03002E62  |...............b|
+; EMPTY-NEXT:             0010: 73730000                             |ss..|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:         {
+; EMPTY-NEXT:           Kind: 0x1136
+; EMPTY-NEXT:           Length: 26
+; EMPTY-NEXT:           Bytes (
+; EMPTY-NEXT:             0000: 04000C00 00500000 08000000 40000042  |.....P......@..B|
+; EMPTY-NEXT:             0010: 2E72656C 6F630000                    |.reloc..|
+; EMPTY-NEXT:           )
+; EMPTY-NEXT:         }
+; EMPTY-NEXT:       ]
 ; EMPTY-NEXT:     }
 ; EMPTY-NEXT:   ]
 ; EMPTY-NEXT: }

tools/llvm-pdbdump/llvm-pdbdump.cpp

 #include "llvm/DebugInfo/PDB/Raw/InfoStream.h"
 #include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
 #include "llvm/DebugInfo/PDB/Raw/ModInfo.h"
+#include "llvm/DebugInfo/PDB/Raw/ModStream.h"
 #include "llvm/DebugInfo/PDB/Raw/NameHashTable.h"
 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
 #include "llvm/DebugInfo/PDB/Raw/RawSession.h"

 cl::OptionCategory TypeCategory("Symbol Type Options");
 cl::OptionCategory FilterCategory("Filtering Options");
 cl::OptionCategory OtherOptions("Other Options");
+cl::OptionCategory NativeOtions("Native Options");
 
 cl::opt<bool> Compilands("compilands", cl::desc("Display compilands"),
                          cl::cat(TypeCategory));

     cl::cat(OtherOptions));
 
 cl::opt<bool> DumpHeaders("dump-headers", cl::desc("dump PDB headers"),
-                          cl::cat(OtherOptions));
+                          cl::cat(NativeOtions));
 cl::opt<bool> DumpStreamSizes("dump-stream-sizes",
                               cl::desc("dump PDB stream sizes"),
-                              cl::cat(OtherOptions));
+                              cl::cat(NativeOtions));
 cl::opt<bool> DumpStreamBlocks("dump-stream-blocks",
                                cl::desc("dump PDB stream blocks"),
-                               cl::cat(OtherOptions));
+                               cl::cat(NativeOtions));
 cl::opt<bool> DumpTpiRecords("dump-tpi-records",
                              cl::desc("dump CodeView type records"),
-                             cl::cat(OtherOptions));
+                             cl::cat(NativeOtions));
 cl::opt<bool>
     DumpTpiRecordBytes("dump-tpi-record-bytes",
                        cl::desc("dump CodeView type record raw bytes"),
-                       cl::cat(OtherOptions));
+                       cl::cat(NativeOtions));
 cl::opt<std::string> DumpStreamData("dump-stream", cl::desc("dump stream data"),
-                                    cl::cat(OtherOptions));
+                                    cl::cat(NativeOtions));
+cl::opt<bool> DumpModuleSyms("dump-module-syms",
+                             cl::desc("dump module symbols"),
+                             cl::cat(NativeOtions));
 
 cl::list<std::string>
     ExcludeTypes("exclude-types",

     P.printNumber("Symbol Byte Size", Modi.Info.getSymbolDebugInfoByteSize());
     P.printNumber("Type Server Index", Modi.Info.getTypeServerIndex());
     P.printBoolean("Has EC Info", Modi.Info.hasECInfo());
-    std::string FileListName =
-        to_string(Modi.SourceFiles.size()) + " Contributing Source Files";
-    ListScope LL(P, FileListName);
-    for (auto File : Modi.SourceFiles)
-      P.printString(File);
+    {
+      std::string FileListName =
+          to_string(Modi.SourceFiles.size()) + " Contributing Source Files";
+      ListScope LL(P, FileListName);
+      for (auto File : Modi.SourceFiles)
+        P.printString(File);
+    }
+    if (opts::DumpModuleSyms) {
+      ListScope SS(P, "Symbols");
+      ModStream ModS(File, Modi.Info);
+      if (auto EC = ModS.reload())
+        return EC;
+
+      for (auto &S : ModS.symbols()) {
+        DictScope SD(P);
+        P.printHex("Kind", S.Type);
+        P.printNumber("Length", S.Length);
+        P.printBinaryBlock("Bytes", S.Data);
+      }
+    }
   }
   return Error::success();
 }