pdbdump: print out TPI hashes

[ruiu]

Summary

This patch also fixes a bug in FixedArrayStream. It used to trigger
an assert check if an array is empty.

[zturner]

include/llvm/DebugInfo/CodeView/StreamArray.h

188 ↗

(On Diff #59487)

I actually fixed this same bug in a patch that went in. So I guess you will have a merge conflict. I did it a little bit differently, but I guess both ways work. When you do merge, feel free to keep whichever you prefer. This way might be a little faster since it drops 2 conditionals from the ++ operator (although it has the downside that if you increment an iterator that is already at end, it will no longer compare equal to end. Not sure if we care, since incrementing end is already undefined behavior.)

include/llvm/DebugInfo/PDB/Raw/TpiStream.h

33 ↗

(On Diff #59487)

the first member of OffCb isn't necessarily a type index though, so I don't think this comment is entirely accurate. I'm guessing that in the Microsoft code they are using an OffCb structure parsing this hash table, but I guess that's just because the Offset field coincidentally has the same type as a type index. I think we should probably just delete the comment. Also, I've been putting structs like this in RawTypes.h so that they can be re-used from other code when we need them.

72 ↗

(On Diff #59487)

I was doing this at first too, but I think some people had objections to overloading the << operator. I guess you should make a printTypeIndexOffset() function in llvm-pdbdump.cpp

[ruiu]

include/llvm/DebugInfo/CodeView/StreamArray.h

188 ↗

(On Diff #59487)

I'm not an C++ expert, but I think in the first place we shouldn't increment an iterator that is already at end(). At least I believe if an iterator is a pointer pointing to a plain C array, the result of incrementing an iterator at end() is undefined.

include/llvm/DebugInfo/PDB/Raw/TpiStream.h

33 ↗

(On Diff #59487)

Will do.

72 ↗

(On Diff #59487)

Will do.

[ruiu]

• Address review comments.

[zturner]

Looks good after rebasing and moving the print method

include/llvm/DebugInfo/CodeView/StreamArray.h

248 ↗

(On Diff #59593)

Perhaps assert that Index < Array.size() instead.

tools/llvm-pdbdump/llvm-pdbdump.cpp

698 ↗

(On Diff #59593)

I think you will need to rebase and move this method to a pure virtual method on OutputStyle and implement it in LLVMOutputStyle. Sorry :-/ Good news is I should be done with all the intrusive changes now and can work in isolation entirely on the yaml outputter.

llvm/trunk/include/llvm/DebugInfo/CodeView/StreamArray.h

public:
   }
 
   uint32_t size() const { return Stream.getLength() / sizeof(T); }
 
   FixedStreamArrayIterator<T> begin() const {
     return FixedStreamArrayIterator<T>(*this, 0);
   }
   FixedStreamArrayIterator<T> end() const {
-    return FixedStreamArrayIterator<T>(*this);
+    return FixedStreamArrayIterator<T>(*this, size());
   }
 
 private:
   StreamRef Stream;
 };
 
 template <typename T> class FixedStreamArrayIterator {
 public:
-  FixedStreamArrayIterator(const FixedStreamArray<T> &Array)
-      : Array(Array), Index(uint32_t(-1)) {}
-  FixedStreamArrayIterator(const FixedStreamArray<T> &Array, uint32_t ArrayIndex)
-      : Array(Array), Index(ArrayIndex) {
-    if (Array.size() <= Index)
-      Index = uint32_t(-1);
-  }
+  FixedStreamArrayIterator(const FixedStreamArray<T> &Array, uint32_t Index)
+      : Array(Array), Index(Index) {}
 
   bool operator==(const FixedStreamArrayIterator<T> &R) {
     assert(&Array == &R.Array);
     return Index == R.Index;
   }
 
   bool operator!=(const FixedStreamArrayIterator<T> &R) {
     return !(*this == R);
   }
 
   const T &operator*() const { return Array[Index]; }
 
   FixedStreamArrayIterator<T> &operator++() {
-    if (Index == uint32_t(-1))
-      return *this;
-    if (++Index >= Array.size())
-      Index = uint32_t(-1);
+    assert(Index < Array.size());
+    ++Index;
     return *this;
   }
 
   FixedStreamArrayIterator<T> operator++(int) {
     FixedStreamArrayIterator<T> Original = *this;
     ++*this;
     return Original;
   }

llvm/trunk/include/llvm/DebugInfo/PDB/Raw/RawTypes.h

 //===- RawTypes.h -----------------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_DEBUGINFO_PDB_RAW_RAWTYPES_H
 #define LLVM_DEBUGINFO_PDB_RAW_RAWTYPES_H
 
+#include "llvm/DebugInfo/CodeView/TypeRecord.h"
 #include "llvm/Support/Endian.h"
 
 namespace llvm {
 namespace pdb {
 // This struct is defined as "SO" in langapi/include/pdb.h.
 struct SectionOffset {
   support::ulittle32_t Off;
   support::ulittle16_t Isect;
support::ulittle16_t ClassName;     // Byte index of the class name in the
                                       // sstSegName table, or 0xFFFF.
   support::ulittle32_t Offset;        // Byte offset of the logical segment
                                       // within the specified physical segment.
                                       // If group is set in flags, offset is the
                                       // offset of the group.
   support::ulittle32_t SecByteLength; // Byte count of the segment or group.
 };
 
+// Used for serialized hash table in TPI stream.
+// In the reference, it is an array of TI and cbOff pair.
+struct TypeIndexOffset {
+  codeview::TypeIndex Type;
+  support::ulittle32_t Offset;
+};
+
 } // namespace pdb
 } // namespace llvm
 
 #endif

llvm/trunk/include/llvm/DebugInfo/PDB/Raw/TpiStream.h

 #define LLVM_DEBUGINFO_PDB_RAW_PDBTPISTREAM_H
 
 #include "llvm/DebugInfo/CodeView/StreamArray.h"
 #include "llvm/DebugInfo/CodeView/StreamRef.h"
 #include "llvm/DebugInfo/CodeView/TypeRecord.h"
 #include "llvm/DebugInfo/PDB/PDBTypes.h"
 #include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
 #include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
+#include "llvm/DebugInfo/PDB/Raw/RawTypes.h"
+#include "llvm/Support/raw_ostream.h"
 
 #include "llvm/Support/Error.h"
 
 namespace llvm {
 namespace pdb {
 class PDBFile;
 
 typedef uint32_t (*HashFunctionType)(uint8_t *, uint32_t);
public:
   PdbRaw_TpiVer getTpiVersion() const;
 
   uint32_t TypeIndexBegin() const;
   uint32_t TypeIndexEnd() const;
   uint32_t NumTypeRecords() const;
   uint16_t getTypeHashStreamIndex() const;
   uint16_t getTypeHashStreamAuxIndex() const;
 
+  codeview::FixedStreamArray<support::ulittle32_t> getHashValues() const;
+  codeview::FixedStreamArray<TypeIndexOffset> getTypeIndexOffsets() const;
+  codeview::FixedStreamArray<TypeIndexOffset> getHashAdjustments() const;
+
   iterator_range<codeview::CVTypeArray::Iterator> types(bool *HadError) const;
 
 private:
   PDBFile &Pdb;
   MappedBlockStream Stream;
   HashFunctionType HashFunction;
 
   codeview::CVTypeArray TypeRecords;
-  codeview::StreamRef TypeIndexOffsetBuffer;
-  codeview::StreamRef HashValuesBuffer;
-  codeview::StreamRef HashAdjBuffer;
+
+  std::unique_ptr<MappedBlockStream> HashStream;
+  codeview::FixedStreamArray<support::ulittle32_t> HashValues;
+  codeview::FixedStreamArray<TypeIndexOffset> TypeIndexOffsets;
+  codeview::FixedStreamArray<TypeIndexOffset> HashAdjustments;
 
   const HeaderInfo *Header;
 };
 }
 }
 
 #endif

llvm/trunk/lib/DebugInfo/PDB/Raw/TpiStream.cpp

 //===- TpiStream.cpp - PDB Type Info (TPI) Stream 2 Access ----------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/DebugInfo/PDB/Raw/TpiStream.h"
 
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/StreamReader.h"
+#include "llvm/DebugInfo/CodeView/TypeIndex.h"
 #include "llvm/DebugInfo/CodeView/TypeRecord.h"
 #include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
 #include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
 #include "llvm/DebugInfo/PDB/Raw/RawError.h"
+#include "llvm/DebugInfo/PDB/Raw/RawTypes.h"
 
 #include "llvm/Support/Endian.h"
 
 using namespace llvm;
 using namespace llvm::support;
 using namespace llvm::pdb;
 
 namespace {
Error TpiStream::reload() {
 
   HashFunction = HashBufferV8;
 
   // The actual type records themselves come from this stream
   if (auto EC = Reader.readArray(TypeRecords, Header->TypeRecordBytes))
     return EC;
 
   // Hash indices, hash values, etc come from the hash stream.
-  MappedBlockStream HS(Header->HashStreamIndex, Pdb);
-  codeview::StreamReader HSR(HS);
+  HashStream.reset(new MappedBlockStream(Header->HashStreamIndex, Pdb));
+  codeview::StreamReader HSR(*HashStream);
+  uint32_t NumHashValues = Header->HashValueBuffer.Length / sizeof(ulittle32_t);
   HSR.setOffset(Header->HashValueBuffer.Off);
-  if (auto EC =
-          HSR.readStreamRef(HashValuesBuffer, Header->HashValueBuffer.Length))
+  if (auto EC = HSR.readArray(HashValues, NumHashValues))
     return EC;
 
-  HSR.setOffset(Header->HashAdjBuffer.Off);
-  if (auto EC = HSR.readStreamRef(HashAdjBuffer, Header->HashAdjBuffer.Length))
+  HSR.setOffset(Header->IndexOffsetBuffer.Off);
+  uint32_t NumTypeIndexOffsets =
+      Header->IndexOffsetBuffer.Length / sizeof(TypeIndexOffset);
+  if (auto EC = HSR.readArray(TypeIndexOffsets, NumTypeIndexOffsets))
     return EC;
 
-  HSR.setOffset(Header->IndexOffsetBuffer.Off);
-  if (auto EC = HSR.readStreamRef(TypeIndexOffsetBuffer,
-                                  Header->IndexOffsetBuffer.Length))
+  HSR.setOffset(Header->HashAdjBuffer.Off);
+  uint32_t NumHashAdjustments =
+      Header->HashAdjBuffer.Length / sizeof(TypeIndexOffset);
+  if (auto EC = HSR.readArray(HashAdjustments, NumHashAdjustments))
     return EC;
 
   return Error::success();
 }
 
 PdbRaw_TpiVer TpiStream::getTpiVersion() const {
   uint32_t Value = Header->Version;
   return static_cast<PdbRaw_TpiVer>(Value);

 uint16_t TpiStream::getTypeHashStreamIndex() const {
   return Header->HashStreamIndex;
 }
 
 uint16_t TpiStream::getTypeHashStreamAuxIndex() const {
   return Header->HashAuxStreamIndex;
 }
 
+codeview::FixedStreamArray<support::ulittle32_t>
+TpiStream::getHashValues() const {
+  return HashValues;
+}
+
+codeview::FixedStreamArray<TypeIndexOffset>
+TpiStream::getTypeIndexOffsets() const {
+  return TypeIndexOffsets;
+}
+
+codeview::FixedStreamArray<TypeIndexOffset>
+TpiStream::getHashAdjustments() const {
+  return HashAdjustments;
+}
+
 iterator_range<codeview::CVTypeArray::Iterator>
 TpiStream::types(bool *HadError) const {
   return llvm::make_range(TypeRecords.begin(HadError), TypeRecords.end());
 }

llvm/trunk/test/DebugInfo/PDB/pdbdump-headers.test

 ; RUN: llvm-pdbdump -raw-headers -raw-tpi-records -raw-tpi-record-bytes -raw-module-syms \
 ; RUN:              -raw-sym-record-bytes -raw-publics -raw-module-files -raw-stream-name=/names \
 ; RUN:              -raw-stream-summary -raw-stream-blocks -raw-ipi-records -raw-ipi-record-bytes \
 ; RUN:              -raw-section-contribs -raw-section-map -raw-section-headers -raw-line-info \
-; RUN:              %p/Inputs/empty.pdb | FileCheck -check-prefix=EMPTY %s
+; RUN:              -raw-tpi-hash %p/Inputs/empty.pdb | FileCheck -check-prefix=EMPTY %s
 ; RUN: llvm-pdbdump -raw-all %p/Inputs/empty.pdb | FileCheck -check-prefix=ALL %s
 ; RUN: llvm-pdbdump -raw-headers -raw-stream-name=/names -raw-modules -raw-module-files \
 ; RUN:              %p/Inputs/big-read.pdb | FileCheck -check-prefix=BIG %s
 ; RUN: not llvm-pdbdump -raw-headers %p/Inputs/bad-block-size.pdb 2>&1 | FileCheck -check-prefix=BAD-BLOCK-SIZE %s
 
 ; EMPTY:      FileHeaders {
 ; EMPTY-NEXT:   BlockSize: 4096
 ; EMPTY-NEXT:   Unknown0: 2

 ; EMPTY-NEXT:       Bytes (
 ; EMPTY-NEXT:         0000: 02150300 01006170 6172746D 656E7400  |......apartment.|
 ; EMPTY-NEXT:         0010: 02150300 02007369 6E676C65 00F3F2F1  |......single....|
 ; EMPTY-NEXT:         0020: 02150300 03006672 656500F1 02150300  |......free......|
 ; EMPTY-NEXT:         0030: 04006E65 75747261 6C00F2F1 02150300  |..neutral.......|
 ; EMPTY-NEXT:         0040: 0500626F 746800F1                    |..both..|
 ; EMPTY-NEXT:       )
 ; EMPTY-NEXT:     }
+; EMPTY:          Hash {
+; EMPTY-NEXT:       Values: [205956, 163561, 59811, 208239, 16377, 247078, 194342, 254156, 194536, 167492, 185421, 119540, 261871, 198119, 48056, 251486, 134580, 148190, 113636, 53336, 55779, 220695, 198114, 148734, 81128, 60158, 217249, 174209, 159978, 249504, 141941, 238785, 6214, 94935, 151449, 135589, 73373, 96512, 254299, 17744, 239514, 173189, 130544, 204437, 238560, 144673, 115151, 197306, 256035, 101096, 231280, 52156, 48854, 170035, 177041, 102745, 16947, 183703, 98548, 35693, 171328, 203640, 139292, 49018, 43821, 202555, 165040, 215835, 142625, 52534, 44186, 103930, 110942, 17991, 213215]
+; EMPTY-NEXT:       Type Index Offsets: [{4096, 0}]
+; EMPTY-NEXT:       Hash Adjustments: []
+; EMPTY-NEXT:     }
 ; EMPTY:        Type Info Stream (IPI) {
 ; EMPTY-NEXT:     IPI Version: 20040203
 ; EMPTY-NEXT:     Record count: 15
 ; EMPTY-NEXT:     Records [
 ; EMPTY-NEXT:       {
 ; EMPTY-NEXT:         UnknownLeaf (0x104B) {
 ; EMPTY-NEXT:           TypeLeafKind: LF_UDT_MOD_SRC_LINE (0x1607)
 ; EMPTY-NEXT:           UnknownType {

 ; EMPTY-NEXT:             Kind: LF_UDT_MOD_SRC_LINE (0x1607)
 ; EMPTY-NEXT:             Length: 16
 ; EMPTY-NEXT:           }
 ; EMPTY-NEXT:         }
 ; EMPTY-NEXT:         Bytes (
 ; EMPTY-NEXT:           0000: 42100000 01000000 6C000000 0100F2F1  |B.......l.......|
 ; EMPTY-NEXT:         )
 ; EMPTY-NEXT:       }
+; EMPTY:          Hash {
+; EMPTY-NEXT:       Values: [7186, 7198, 7180, 7191, 7201, 7241, 7249, 80727, 154177, 75189, 253662, 193467, 222705, 186099, 257108]
+; EMPTY-NEXT:       Type Index Offsets: [{4096, 0}]
+; EMPTY-NEXT:       Hash Adjustments: []
+; EMPTY-NEXT:     }
 ; EMPTY:      DBI Stream {
 ; EMPTY-NEXT:   Dbi Version: 19990903
 ; EMPTY-NEXT:   Age: 1
 ; EMPTY-NEXT:   Incremental Linking: Yes
 ; EMPTY-NEXT:   Has CTypes: No
 ; EMPTY-NEXT:   Is Stripped: No
 ; EMPTY-NEXT:   Machine Type: x86
 ; EMPTY-NEXT:   Symbol Record Stream Index: 8

llvm/trunk/tools/llvm-pdbdump/LLVMOutputStyle.cpp

for (uint32_t ID : NameTable.name_ids()) {
       StringRef Str = NameTable.getStringForID(ID);
       if (!Str.empty())
         P.printString(to_string(ID), Str);
     }
   }
   return Error::success();
 }
 
+static void printTypeIndexOffset(raw_ostream &OS,
+                                 const TypeIndexOffset &TIOff) {
+  OS << "{" << TIOff.Type.getIndex() << ", " << TIOff.Offset << "}";
+}
+
+static void dumpTpiHash(ScopedPrinter &P, TpiStream &Tpi) {
+  if (!opts::DumpTpiHash)
+    return;
+  DictScope DD(P, "Hash");
+  codeview::FixedStreamArray<support::ulittle32_t> S = Tpi.getHashValues();
+  P.printList("Values", Tpi.getHashValues());
+  P.printList("Type Index Offsets", Tpi.getTypeIndexOffsets(),
+              printTypeIndexOffset);
+  P.printList("Hash Adjustments", Tpi.getHashAdjustments(),
+              printTypeIndexOffset);
+}
+
 Error LLVMOutputStyle::dumpTpiStream(uint32_t StreamIdx) {
   assert(StreamIdx == StreamTPI || StreamIdx == StreamIPI);
 
   bool DumpRecordBytes = false;
   bool DumpRecords = false;
   StringRef Label;
   StringRef VerLabel;
   if (StreamIdx == StreamTPI) {
for (auto &Type : Tpi.types(&HadError)) {
       DictScope DD(P, "");
 
       if (DumpRecords)
         TD.dump(Type);
 
       if (DumpRecordBytes)
         P.printBinaryBlock("Bytes", Type.Data);
     }
+    dumpTpiHash(P, Tpi);
     if (HadError)
       return make_error<RawError>(raw_error_code::corrupt_file,
                                   "TPI stream contained corrupt record");
   } else if (opts::DumpModuleSyms) {
     // Even if the user doesn't want to dump type records, we still need to
     // iterate them in order to build the list of types so that we can print
     // them when dumping module symbols. So when they want to dump symbols
     // but not types, use a null output stream.
     ScopedPrinter *OldP = TD.getPrinter();
     TD.setPrinter(nullptr);
 
     bool HadError = false;
     for (auto &Type : Tpi.types(&HadError))
       TD.dump(Type);
 
     TD.setPrinter(OldP);
+    dumpTpiHash(P, Tpi);
     if (HadError)
       return make_error<RawError>(raw_error_code::corrupt_file,
                                   "TPI stream contained corrupt record");
   }
   P.flush();
   return Error::success();
 }
 

llvm/trunk/tools/llvm-pdbdump/llvm-pdbdump.h

 extern llvm::cl::opt<bool> Globals;
 extern llvm::cl::opt<bool> Types;
 extern llvm::cl::opt<bool> All;
 
 extern llvm::cl::opt<OutputStyleTy> RawOutputStyle;
 extern llvm::cl::opt<bool> DumpHeaders;
 extern llvm::cl::opt<bool> DumpStreamBlocks;
 extern llvm::cl::opt<bool> DumpStreamSummary;
-extern llvm::cl::opt<bool> DumpTpiRecords;
+extern llvm::cl::opt<bool> DumpTpiHash;
 extern llvm::cl::opt<bool> DumpTpiRecordBytes;
+extern llvm::cl::opt<bool> DumpTpiRecords;
 extern llvm::cl::opt<bool> DumpIpiRecords;
 extern llvm::cl::opt<bool> DumpIpiRecordBytes;
 extern llvm::cl::opt<std::string> DumpStreamDataIdx;
 extern llvm::cl::opt<std::string> DumpStreamDataName;
 extern llvm::cl::opt<bool> DumpModules;
 extern llvm::cl::opt<bool> DumpModuleFiles;
 extern llvm::cl::opt<bool> DumpModuleSyms;
 extern llvm::cl::opt<bool> DumpPublics;

llvm/trunk/tools/llvm-pdbdump/llvm-pdbdump.cpp

 cl::opt<bool>
     DumpTpiRecords("raw-tpi-records",
                    cl::desc("dump CodeView type records from TPI stream"),
                    cl::cat(NativeOptions));
 cl::opt<bool> DumpTpiRecordBytes(
     "raw-tpi-record-bytes",
     cl::desc("dump CodeView type record raw bytes from TPI stream"),
     cl::cat(NativeOptions));
+cl::opt<bool> DumpTpiHash("raw-tpi-hash",
+                          cl::desc("dump CodeView TPI hash stream"),
+                          cl::cat(NativeOptions));
 cl::opt<bool>
     DumpIpiRecords("raw-ipi-records",
                    cl::desc("dump CodeView type records from IPI stream"),
                    cl::cat(NativeOptions));
 cl::opt<bool> DumpIpiRecordBytes(
     "raw-ipi-record-bytes",
     cl::desc("dump CodeView type record raw bytes from IPI stream"),
     cl::cat(NativeOptions));
bool isRawDumpEnabled() {
   if (opts::DumpStreamBlocks)
     return true;
   if (opts::DumpSymRecordBytes)
     return true;
   if (opts::DumpTpiRecordBytes)
     return true;
   if (opts::DumpTpiRecords)
     return true;
+  if (opts::DumpTpiHash)
+    return true;
   if (opts::DumpIpiRecords)
     return true;
   if (opts::DumpIpiRecordBytes)
     return true;
   if (opts::DumpSectionContribs)
     return true;
   if (opts::DumpSectionMap)
     return true;
if (opts::RawAll) {
     opts::DumpModules = true;
     opts::DumpModuleFiles = true;
     opts::DumpModuleSyms = true;
     opts::DumpPublics = true;
     opts::DumpSectionHeaders = true;
     opts::DumpStreamSummary = true;
     opts::DumpStreamBlocks = true;
     opts::DumpTpiRecords = true;
+    opts::DumpTpiHash = true;
     opts::DumpIpiRecords = true;
     opts::DumpSectionMap = true;
     opts::DumpSectionContribs = true;
     opts::DumpLineInfo = true;
   }
 
   // When adding filters for excluded compilands and types, we need to remember
   // that these are regexes.  So special characters such as * and \ need to be