Read the rest of the substreams from DBI, and parse source file information

[zturner]

Summary

We now read out the rest of the substreams from the DBI streams. One of these substreams, the FileInfo substream, contains information about which source files contribute to each module (aka compiland). This patch additionally parses out the file information from that substream, and dumps it in llvm-pdbdump.

[ruiu]

include/llvm/DebugInfo/PDB/Raw/PDBDbiStream.h

47

Why don't you return ArrayRef<ModuleInfoEx>?

lib/DebugInfo/PDB/Raw/PDBDbiStream.cpp

183–269

This can return ArrayRef<ModuleInfoEx>?

[zturner]

Return ArrayRef instead of an iterator range.

[majnemer]

include/llvm/DebugInfo/PDB/Raw/ModInfo.h

47

Maybe make the parameter be a const &

lib/DebugInfo/PDB/Raw/PDBDbiStream.cpp

51

Would it be easier to decode this field if it were named something like NumGlobalSyms or NumGlobalSymbols?

191

I'd be nice if the readBytes interface took a MutableArrayRef<uint8_t>. Then you'd just pass in the std::vector.

216–218

Couldn't you also do:

auto *Buf = reinterpret_cast<const ulittle16_t *>(&FileInfoSubstream[1]);

[zturner]

lib/DebugInfo/PDB/Raw/PDBDbiStream.cpp

51

Actually I'm not 100% sure that that's even what it is. I could change it now, or wait till we actually start using it (which I assume by then I'll be more certain about its purpose)

191

I've actually got a patch in progress that completely refactors the stream reading interface to make it so that I can read from mapped block streams and contiguous byte arrays (e.g. substreams) with the same interface. It might make more sense to change it in that patch since it focuses on the stream reading interface anyway.

[zturner]

lib/DebugInfo/PDB/Raw/PDBDbiStream.cpp

216–218

Did you mean

auto *Buf = reinterpret_cast<const ulittle16_t*>(FI + 1);

?

include/llvm/DebugInfo/PDB/Raw/ModInfo.h

 #include "llvm/ADT/StringRef.h"
 
 #include <stdint.h>
+#include <vector>
 
 namespace llvm {
 class PDBFile;

   const FileLayout *Layout;
 };
 
+struct ModuleInfoEx {
+  ModuleInfoEx(ModInfo Module) : Info(Module) {}
+
+  ModInfo Info;
+  std::vector<StringRef> SourceFiles;
+};
+
 class ModInfoIterator {
 public:
   ModInfoIterator(const uint8_t *Stream);

include/llvm/DebugInfo/PDB/Raw/PDBDbiStream.h

 
   PDB_Machine getMachineType() const;
 
-  llvm::iterator_range<ModInfoIterator> modules() const;
+  ArrayRef<ModuleInfoEx> modules() const;
 
 private:
+  std::error_code readSubstream(std::vector<uint8_t> &Bytes, uint32_t Size);
+  std::error_code initializeFileInfo();
+
   PDBFile &Pdb;
   PDBStream Stream;
 
+  std::vector<ModuleInfoEx> ModuleInfos;
+
   std::vector<uint8_t> ModInfoSubstream;
+  std::vector<uint8_t> SecContrSubstream;
+  std::vector<uint8_t> SecMapSubstream;
+  std::vector<uint8_t> FileInfoSubstream;
+  std::vector<uint8_t> TypeServerMapSubstream;
+  std::vector<uint8_t> ECSubstream;
   std::unique_ptr<HeaderInfo> Header;
 };
 }

lib/DebugInfo/PDB/Raw/PDBDbiStream.cpp

   little32_t VersionSignature;
   ulittle32_t VersionHeader;
   ulittle32_t Age; // Should match PDBInfoStream.
-  ulittle16_t GSSyms;
-  ulittle16_t BuildNumber; // See DbiBuildNo structure.
-  ulittle16_t PSSyms;
+  ulittle16_t GSSyms;               // Number of global symbols
+  ulittle16_t BuildNumber;          // See DbiBuildNo structure.
+  ulittle16_t PSSyms;               // Number of public symbols
   ulittle16_t PdbDllVersion;        // version of mspdbNNN.dll
   ulittle16_t SymRecords;           // Number of symbols
   ulittle16_t PdbDllRbld;           // rbld number of mspdbNNN.dll
   little32_t ModiSubstreamSize;     // Size of module info stream
   little32_t SecContrSubstreamSize; // Size of sec. contribution stream
-  little32_t SectionMapSize;
-  little32_t FileInfoSize;
+  little32_t SectionMapSize;        // Size of sec. map substream
+  little32_t FileInfoSize;          // Size of file info substream
   little32_t TypeServerSize;      // Size of type server map
   ulittle32_t MFCTypeServerIndex; // Index of MFC Type Server
   little32_t OptionalDbgHdrSize;  // Size of DbgHeader info

           Header->OptionalDbgHdrSize + Header->ECSubstreamSize)
     return std::make_error_code(std::errc::illegal_byte_sequence);
 
+  // Only certain substreams are guaranteed to be aligned.  Validate
+  // them here.
   if (Header->ModiSubstreamSize % sizeof(uint32_t) != 0)
     return std::make_error_code(std::errc::illegal_byte_sequence);
+  if (Header->SecContrSubstreamSize % sizeof(uint32_t) != 0)
+    return std::make_error_code(std::errc::illegal_byte_sequence);
+  if (Header->SectionMapSize % sizeof(uint32_t) != 0)
+    return std::make_error_code(std::errc::illegal_byte_sequence);
+  if (Header->FileInfoSize % sizeof(uint32_t) != 0)
+    return std::make_error_code(std::errc::illegal_byte_sequence);
+  if (Header->TypeServerSize % sizeof(uint32_t) != 0)
+    return std::make_error_code(std::errc::illegal_byte_sequence);
+
+  std::error_code EC;
+  if (EC = readSubstream(ModInfoSubstream, Header->ModiSubstreamSize))
+    return EC;
+
+  // Since each ModInfo in the stream is a variable length, we have to iterate
+  // them to know how many there actually are.
+  auto Range = llvm::make_range(ModInfoIterator(&ModInfoSubstream.front()),
+                                ModInfoIterator(&ModInfoSubstream.back() + 1));
+  for (auto Info : Range)
+    ModuleInfos.push_back(ModuleInfoEx(Info));
 
-  ModInfoSubstream.resize(Header->ModiSubstreamSize);
-  if (auto EC =
-          Stream.readBytes(&ModInfoSubstream[0], Header->ModiSubstreamSize))
+  if (EC = readSubstream(SecContrSubstream, Header->SecContrSubstreamSize))
+    return EC;
+  if (EC = readSubstream(SecMapSubstream, Header->SectionMapSize))
+    return EC;
+  if (EC = readSubstream(FileInfoSubstream, Header->FileInfoSize))
+    return EC;
+  if (EC = readSubstream(TypeServerMapSubstream, Header->TypeServerSize))
+    return EC;
+  if (EC = readSubstream(ECSubstream, Header->ECSubstreamSize))
+    return EC;
+
+  if (EC = initializeFileInfo())
     return EC;
 
   return std::error_code();

   return static_cast<PDB_Machine>(Machine);
 }
 
-llvm::iterator_range<ModInfoIterator> PDBDbiStream::modules() const {
-  return llvm::make_range(ModInfoIterator(&ModInfoSubstream.front()),
-                          ModInfoIterator(&ModInfoSubstream.back() + 1));
+ArrayRef<ModuleInfoEx> PDBDbiStream::modules() const { return ModuleInfos; }
+
+std::error_code PDBDbiStream::readSubstream(std::vector<uint8_t> &Bytes, uint32_t Size) {
+  Bytes.clear();
+  if (Size == 0)
+    return std::error_code();
+
+  Bytes.resize(Size);
+  return Stream.readBytes(&Bytes[0], Size);
+}
+
+std::error_code PDBDbiStream::initializeFileInfo() {
+  struct FileInfoSubstreamHeader {
+    ulittle16_t NumModules;     // Total # of modules, should match number of
+                                // records in the ModuleInfo substream.
+    ulittle16_t NumSourceFiles; // Total # of source files.  This value is not
+                                // accurate because PDB actually supports more
+                                // than 64k source files, so we ignore it and
+                                // compute the value from other stream fields.
+  };
+
+  // The layout of the FileInfoSubstream is like this:
+  // struct {
+  //   ulittle16_t NumModules;
+  //   ulittle16_t NumSourceFiles;
+  //   ulittle16_t ModIndices[NumModules];
+  //   ulittle16_t ModFileCounts[NumModules];
+  //   ulittle32_t FileNameOffsets[NumSourceFiles];
+  //   char Names[][NumSourceFiles];
+  // };
+  // with the caveat that `NumSourceFiles` cannot be trusted, so
+  // it is computed by summing `ModFileCounts`.
+  //
+  const uint8_t *Buf = &FileInfoSubstream[0];
+  auto FI = reinterpret_cast<const FileInfoSubstreamHeader *>(Buf);
+  Buf += sizeof(FileInfoSubstreamHeader);
+  // The number of modules in the stream should be the same as reported by
+  // the FileInfoSubstreamHeader.
+  if (FI->NumModules != ModuleInfos.size())
+    return std::make_error_code(std::errc::illegal_byte_sequence);
+
+  // First is an array of `NumModules` module indices.  This is not used for the
+  // same reason that `NumSourceFiles` is not used.  It's an array of uint16's,
+  // but it's possible there are more than 64k source files, which would imply
+  // more than 64k modules (e.g. object files) as well.  So we ignore this
+  // field.
+  llvm::ArrayRef<ulittle16_t> ModIndexArray(
+      reinterpret_cast<const ulittle16_t *>(Buf), ModuleInfos.size());
+
+  llvm::ArrayRef<ulittle16_t> ModFileCountArray(ModIndexArray.end(),
+                                                ModuleInfos.size());
+
+  // Compute the real number of source files.
+  uint32_t NumSourceFiles = 0;
+  for (auto Count : ModFileCountArray)
+    NumSourceFiles += Count;
+
+  // This is the array that in the reference implementation corresponds to
+  // `ModInfo::FileLayout::FileNameOffs`, which is commented there as being a
+  // pointer. Due to the mentioned problems of pointers causing difficulty
+  // when reading from the file on 64-bit systems, we continue to ignore that
+  // field in `ModInfo`, and instead build a vector of StringRefs and stores
+  // them in `ModuleInfoEx`.  The value written to and read from the file is
+  // not used anyway, it is only there as a way to store the offsets for the
+  // purposes of later accessing the names at runtime.
+  llvm::ArrayRef<little32_t> FileNameOffsets(
+      reinterpret_cast<const little32_t *>(ModFileCountArray.end()),
+      NumSourceFiles);
+
+  const char *Names = reinterpret_cast<const char *>(FileNameOffsets.end());
+
+  // We go through each ModuleInfo, determine the number N of source files for
+  // that module, and then get the next N offsets from the Offsets array, using
+  // them to get the corresponding N names from the Names buffer and associating
+  // each one with the corresponding module.
+  uint32_t NextFileIndex = 0;
+  for (size_t I = 0; I < ModuleInfos.size(); ++I) {
+    uint32_t NumFiles = ModFileCountArray[I];
+    ModuleInfos[I].SourceFiles.resize(NumFiles);
+    for (size_t J = 0; J < NumFiles; ++J, ++NextFileIndex) {
+      uint32_t FileIndex = FileNameOffsets[NextFileIndex];
+      ModuleInfos[I].SourceFiles[J] = StringRef(Names + FileIndex);
+    }
+  }
+
+  return std::error_code();
 }

test/DebugInfo/PDB/pdbdump-headers.test

 ; CHECK-NEXT:     Symbol Byte Size: 208
 ; CHECK-NEXT:     Type Server Index: 0
 ; CHECK-NEXT:     Has EC Info: 0
+; CHECK-NEXT:     1 Contributing Source Files:
+; CHECK-NEXT:         d:\src\llvm\test\debuginfo\pdb\inputs\empty.cpp
 ; CHECK-NEXT: * Linker *
 ; CHECK-NEXT:     Debug Stream Index: 14
 ; CHECK-NEXT:     Object File:

 ; CHECK-NEXT:     Symbol Byte Size: 516
 ; CHECK-NEXT:     Type Server Index: 0
 ; CHECK-NEXT:     Has EC Info: 0
+; CHECK-NEXT:     0 Contributing Source Files:

tools/llvm-pdbdump/llvm-pdbdump.cpp

          << Minor << "." << DbiStream.getPdbDllVersion() << '\n';
 
   outs() << "Modules: \n";
-  for (auto Modi : DbiStream.modules()) {
-    outs() << Modi.getModuleName() << '\n';
-    outs().indent(4) << "Debug Stream Index: " << Modi.getModuleStreamIndex()
-                     << '\n';
-    outs().indent(4) << "Object File: " << Modi.getObjFileName() << '\n';
-    outs().indent(4) << "Num Files: " << Modi.getNumberOfFiles() << '\n';
+  for (auto &Modi : DbiStream.modules()) {
+    outs() << Modi.Info.getModuleName() << '\n';
+    outs().indent(4) << "Debug Stream Index: "
+                     << Modi.Info.getModuleStreamIndex() << '\n';
+    outs().indent(4) << "Object File: " << Modi.Info.getObjFileName() << '\n';
+    outs().indent(4) << "Num Files: " << Modi.Info.getNumberOfFiles() << '\n';
     outs().indent(4) << "Source File Name Idx: "
-                     << Modi.getSourceFileNameIndex() << '\n';
-    outs().indent(4) << "Pdb File Name Idx: " << Modi.getPdbFilePathNameIndex()
-                     << '\n';
-    outs().indent(4) << "Line Info Byte Size: " << Modi.getLineInfoByteSize()
-                     << '\n';
+                     << Modi.Info.getSourceFileNameIndex() << '\n';
+    outs().indent(4) << "Pdb File Name Idx: "
+                     << Modi.Info.getPdbFilePathNameIndex() << '\n';
+    outs().indent(4) << "Line Info Byte Size: "
+                     << Modi.Info.getLineInfoByteSize() << '\n';
     outs().indent(4) << "C13 Line Info Byte Size: "
-                     << Modi.getC13LineInfoByteSize() << '\n';
+                     << Modi.Info.getC13LineInfoByteSize() << '\n';
     outs().indent(4) << "Symbol Byte Size: "
-                     << Modi.getSymbolDebugInfoByteSize() << '\n';
-    outs().indent(4) << "Type Server Index: " << Modi.getTypeServerIndex()
+                     << Modi.Info.getSymbolDebugInfoByteSize() << '\n';
+    outs().indent(4) << "Type Server Index: " << Modi.Info.getTypeServerIndex()
                      << '\n';
-    outs().indent(4) << "Has EC Info: " << Modi.hasECInfo() << '\n';
+    outs().indent(4) << "Has EC Info: " << Modi.Info.hasECInfo() << '\n';
+    outs().indent(4) << Modi.SourceFiles.size()
+                     << " Contributing Source Files: \n";
+    for (auto File : Modi.SourceFiles) {
+      outs().indent(8) << File << '\n';
+    }
   }
 }