pdbdump: print out TPI hashes

[ruiu]

Summary

This patch also fixes a bug in FixedArrayStream. It used to trigger
an assert check if an array is empty.

[zturner]

include/llvm/DebugInfo/CodeView/StreamArray.h

188

I actually fixed this same bug in a patch that went in. So I guess you will have a merge conflict. I did it a little bit differently, but I guess both ways work. When you do merge, feel free to keep whichever you prefer. This way might be a little faster since it drops 2 conditionals from the ++ operator (although it has the downside that if you increment an iterator that is already at end, it will no longer compare equal to end. Not sure if we care, since incrementing end is already undefined behavior.)

include/llvm/DebugInfo/PDB/Raw/TpiStream.h

33

the first member of OffCb isn't necessarily a type index though, so I don't think this comment is entirely accurate. I'm guessing that in the Microsoft code they are using an OffCb structure parsing this hash table, but I guess that's just because the Offset field coincidentally has the same type as a type index. I think we should probably just delete the comment. Also, I've been putting structs like this in RawTypes.h so that they can be re-used from other code when we need them.

72

I was doing this at first too, but I think some people had objections to overloading the << operator. I guess you should make a printTypeIndexOffset() function in llvm-pdbdump.cpp

[ruiu]

include/llvm/DebugInfo/CodeView/StreamArray.h

188

I'm not an C++ expert, but I think in the first place we shouldn't increment an iterator that is already at end(). At least I believe if an iterator is a pointer pointing to a plain C array, the result of incrementing an iterator at end() is undefined.

include/llvm/DebugInfo/PDB/Raw/TpiStream.h

33

Will do.

72

Will do.

[ruiu]

• Address review comments.

[zturner]

Looks good after rebasing and moving the print method

include/llvm/DebugInfo/CodeView/StreamArray.h

212

Perhaps assert that Index < Array.size() instead.

tools/llvm-pdbdump/llvm-pdbdump.cpp

618

I think you will need to rebase and move this method to a pure virtual method on OutputStyle and implement it in LLVMOutputStyle. Sorry :-/ Good news is I should be done with all the intrusive changes now and can work in isolation entirely on the yaml outputter.

include/llvm/DebugInfo/CodeView/StreamArray.h

public:
   }
 
   uint32_t size() const { return Stream.getLength() / sizeof(T); }
 
   FixedStreamArrayIterator<T> begin() const {
     return FixedStreamArrayIterator<T>(*this, 0);
   }
   FixedStreamArrayIterator<T> end() const {
-    return FixedStreamArrayIterator<T>(*this);
+    return FixedStreamArrayIterator<T>(*this, size());
   }
 
 private:
   StreamRef Stream;
 };
 
 template <typename T> class FixedStreamArrayIterator {
 public:
-  FixedStreamArrayIterator(const FixedStreamArray<T> &Array)
-      : Array(Array), Index(uint32_t(-1)) {}
   FixedStreamArrayIterator(const FixedStreamArray<T> &Array, uint32_t Index)
       : Array(Array), Index(Index) {}
 
   bool operator==(const FixedStreamArrayIterator<T> &R) {
     assert(&Array == &R.Array);
     return Index == R.Index;
   }
 
   bool operator!=(const FixedStreamArrayIterator<T> &R) {
     return !(*this == R);
   }
 
   const T &operator*() const { return Array[Index]; }
 
   FixedStreamArrayIterator<T> &operator++() {
-    if (Index == uint32_t(-1))
-      return *this;
-    if (++Index >= Array.size())
-      Index = uint32_t(-1);
+    ++Index;
     return *this;
   }
 
   FixedStreamArrayIterator<T> operator++(int) {
     FixedStreamArrayIterator<T> Original = *this;
     ++*this;
     return Original;
   }

include/llvm/DebugInfo/PDB/Raw/TpiStream.h

 #define LLVM_DEBUGINFO_PDB_RAW_PDBTPISTREAM_H
 
 #include "llvm/DebugInfo/CodeView/StreamArray.h"
 #include "llvm/DebugInfo/CodeView/StreamRef.h"
 #include "llvm/DebugInfo/CodeView/TypeRecord.h"
 #include "llvm/DebugInfo/PDB/PDBTypes.h"
 #include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
 #include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
+#include "llvm/Support/raw_ostream.h"
 
 #include "llvm/Support/Error.h"
 
 namespace llvm {
 namespace pdb {
 class PDBFile;
 
 typedef uint32_t (*HashFunctionType)(uint8_t *, uint32_t);
 
 class TpiStream {
   struct HeaderInfo;
 
 public:
+  // Corresponds to `OffCb`.
+  struct TypeIndexOffset {
+    codeview::TypeIndex Type;
+    support::ulittle32_t Offset;
+  };
+
   TpiStream(PDBFile &File, uint32_t StreamIdx);
   ~TpiStream();
   Error reload();
 
   PdbRaw_TpiVer getTpiVersion() const;
 
   uint32_t TypeIndexBegin() const;
   uint32_t TypeIndexEnd() const;
   uint32_t NumTypeRecords() const;
   uint16_t getTypeHashStreamIndex() const;
   uint16_t getTypeHashStreamAuxIndex() const;
 
+  codeview::FixedStreamArray<support::ulittle32_t> getHashValues() const;
+  codeview::FixedStreamArray<TypeIndexOffset> getTypeIndexOffsets() const;
+  codeview::FixedStreamArray<TypeIndexOffset> getHashAdjustments() const;
+
   iterator_range<codeview::CVTypeArray::Iterator> types(bool *HadError) const;
 
 private:
   PDBFile &Pdb;
   MappedBlockStream Stream;
   HashFunctionType HashFunction;
 
   codeview::CVTypeArray TypeRecords;
-  codeview::StreamRef TypeIndexOffsetBuffer;
-  codeview::StreamRef HashValuesBuffer;
-  codeview::StreamRef HashAdjBuffer;
+
+  std::unique_ptr<MappedBlockStream> HashStream;
+  codeview::FixedStreamArray<support::ulittle32_t> HashValues;
+  codeview::FixedStreamArray<TypeIndexOffset> TypeIndexOffsets;
+  codeview::FixedStreamArray<TypeIndexOffset> HashAdjustments;
 
   const HeaderInfo *Header;
 };
+
+raw_ostream &operator<<(raw_ostream &OS, const TpiStream::TypeIndexOffset &);
 }
 }
 
 #endif

lib/DebugInfo/PDB/Raw/TpiStream.cpp

 //===- TpiStream.cpp - PDB Type Info (TPI) Stream 2 Access ----------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/DebugInfo/PDB/Raw/TpiStream.h"
 
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/StreamReader.h"
+#include "llvm/DebugInfo/CodeView/TypeIndex.h"
 #include "llvm/DebugInfo/CodeView/TypeRecord.h"
 #include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h"
 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
 #include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
 #include "llvm/DebugInfo/PDB/Raw/RawError.h"
 
 #include "llvm/Support/Endian.h"
 
Error TpiStream::reload() {
 
   HashFunction = HashBufferV8;
 
   // The actual type records themselves come from this stream
   if (auto EC = Reader.readArray(TypeRecords, Header->TypeRecordBytes))
     return EC;
 
   // Hash indices, hash values, etc come from the hash stream.
-  MappedBlockStream HS(Header->HashStreamIndex, Pdb);
-  codeview::StreamReader HSR(HS);
+  HashStream.reset(new MappedBlockStream(Header->HashStreamIndex, Pdb));
+  codeview::StreamReader HSR(*HashStream);
+  uint32_t NumHashValues = Header->HashValueBuffer.Length / sizeof(ulittle32_t);
   HSR.setOffset(Header->HashValueBuffer.Off);
-  if (auto EC =
-          HSR.readStreamRef(HashValuesBuffer, Header->HashValueBuffer.Length))
+  if (auto EC = HSR.readArray(HashValues, NumHashValues))
     return EC;
 
-  HSR.setOffset(Header->HashAdjBuffer.Off);
-  if (auto EC = HSR.readStreamRef(HashAdjBuffer, Header->HashAdjBuffer.Length))
+  HSR.setOffset(Header->IndexOffsetBuffer.Off);
+  uint32_t NumTypeIndexOffsets =
+      Header->IndexOffsetBuffer.Length / sizeof(TypeIndexOffset);
+  if (auto EC = HSR.readArray(TypeIndexOffsets, NumTypeIndexOffsets))
     return EC;
 
-  HSR.setOffset(Header->IndexOffsetBuffer.Off);
-  if (auto EC = HSR.readStreamRef(TypeIndexOffsetBuffer,
-                                  Header->IndexOffsetBuffer.Length))
+  HSR.setOffset(Header->HashAdjBuffer.Off);
+  uint32_t NumHashAdjustments =
+      Header->HashAdjBuffer.Length / sizeof(TypeIndexOffset);
+  if (auto EC = HSR.readArray(HashAdjustments, NumHashAdjustments))
     return EC;
 
   return Error::success();
 }
 
 PdbRaw_TpiVer TpiStream::getTpiVersion() const {
   uint32_t Value = Header->Version;
   return static_cast<PdbRaw_TpiVer>(Value);

 uint16_t TpiStream::getTypeHashStreamIndex() const {
   return Header->HashStreamIndex;
 }
 
 uint16_t TpiStream::getTypeHashStreamAuxIndex() const {
   return Header->HashAuxStreamIndex;
 }
 
+codeview::FixedStreamArray<support::ulittle32_t>
+TpiStream::getHashValues() const {
+  return HashValues;
+}
+
+codeview::FixedStreamArray<TpiStream::TypeIndexOffset>
+TpiStream::getTypeIndexOffsets() const {
+  return TypeIndexOffsets;
+}
+
+codeview::FixedStreamArray<TpiStream::TypeIndexOffset>
+TpiStream::getHashAdjustments() const {
+  return HashAdjustments;
+}
+
 iterator_range<codeview::CVTypeArray::Iterator>
 TpiStream::types(bool *HadError) const {
   return llvm::make_range(TypeRecords.begin(HadError), TypeRecords.end());
 }
+
+raw_ostream &pdb::operator<<(raw_ostream &OS,
+                             const TpiStream::TypeIndexOffset &TIOff) {
+  OS << "{" << TIOff.Type.getIndex() << ", " << TIOff.Offset << "}";
+  return OS;
+}

test/DebugInfo/PDB/pdbdump-headers.test

 ; RUN: llvm-pdbdump -raw-headers -raw-tpi-records -raw-tpi-record-bytes -raw-module-syms \
 ; RUN:              -raw-sym-record-bytes -raw-publics -raw-module-files -raw-stream-name=/names \
 ; RUN:              -raw-stream-summary -raw-stream-blocks -raw-ipi-records -raw-ipi-record-bytes \
 ; RUN:              -raw-section-contribs -raw-section-map -raw-section-headers -raw-line-info \
-; RUN:              %p/Inputs/empty.pdb | FileCheck -check-prefix=EMPTY %s
+; RUN:              -raw-tpi-hash %p/Inputs/empty.pdb | FileCheck -check-prefix=EMPTY %s
 ; RUN: llvm-pdbdump -raw-all %p/Inputs/empty.pdb | FileCheck -check-prefix=ALL %s
 ; RUN: llvm-pdbdump -raw-headers -raw-stream-name=/names -raw-modules -raw-module-files \
 ; RUN:              %p/Inputs/big-read.pdb | FileCheck -check-prefix=BIG %s
 ; RUN: not llvm-pdbdump -raw-headers %p/Inputs/bad-block-size.pdb 2>&1 | FileCheck -check-prefix=BAD-BLOCK-SIZE %s
 
 ; EMPTY:      FileHeaders {
 ; EMPTY-NEXT:   BlockSize: 4096
 ; EMPTY-NEXT:   Unknown0: 2

 ; EMPTY-NEXT:       Bytes (
 ; EMPTY-NEXT:         0000: 02150300 01006170 6172746D 656E7400  |......apartment.|
 ; EMPTY-NEXT:         0010: 02150300 02007369 6E676C65 00F3F2F1  |......single....|
 ; EMPTY-NEXT:         0020: 02150300 03006672 656500F1 02150300  |......free......|
 ; EMPTY-NEXT:         0030: 04006E65 75747261 6C00F2F1 02150300  |..neutral.......|
 ; EMPTY-NEXT:         0040: 0500626F 746800F1                    |..both..|
 ; EMPTY-NEXT:       )
 ; EMPTY-NEXT:     }
+; EMPTY:          Hash {
+; EMPTY-NEXT:       Values: [205956, 163561, 59811, 208239, 16377, 247078, 194342, 254156, 194536, 167492, 185421, 119540, 261871, 198119, 48056, 251486, 134580, 148190, 113636, 53336, 55779, 220695, 198114, 148734, 81128, 60158, 217249, 174209, 159978, 249504, 141941, 238785, 6214, 94935, 151449, 135589, 73373, 96512, 254299, 17744, 239514, 173189, 130544, 204437, 238560, 144673, 115151, 197306, 256035, 101096, 231280, 52156, 48854, 170035, 177041, 102745, 16947, 183703, 98548, 35693, 171328, 203640, 139292, 49018, 43821, 202555, 165040, 215835, 142625, 52534, 44186, 103930, 110942, 17991, 213215]
+; EMPTY-NEXT:       Type Index Offsets: [{4096, 0}]
+; EMPTY-NEXT:       Hash Adjustments: []
+; EMPTY-NEXT:     }
 ; EMPTY:        Type Info Stream (IPI) {
 ; EMPTY-NEXT:     IPI Version: 20040203
 ; EMPTY-NEXT:     Record count: 15
 ; EMPTY-NEXT:     Records [
 ; EMPTY-NEXT:       {
 ; EMPTY-NEXT:         UnknownLeaf (0x104B) {
 ; EMPTY-NEXT:           TypeLeafKind: LF_UDT_MOD_SRC_LINE (0x1607)
 ; EMPTY-NEXT:           UnknownType {

 ; EMPTY-NEXT:             Kind: LF_UDT_MOD_SRC_LINE (0x1607)
 ; EMPTY-NEXT:             Length: 16
 ; EMPTY-NEXT:           }
 ; EMPTY-NEXT:         }
 ; EMPTY-NEXT:         Bytes (
 ; EMPTY-NEXT:           0000: 42100000 01000000 6C000000 0100F2F1  |B.......l.......|
 ; EMPTY-NEXT:         )
 ; EMPTY-NEXT:       }
+; EMPTY:          Hash {
+; EMPTY-NEXT:       Values: [7186, 7198, 7180, 7191, 7201, 7241, 7249, 80727, 154177, 75189, 253662, 193467, 222705, 186099, 257108]
+; EMPTY-NEXT:       Type Index Offsets: [{4096, 0}]
+; EMPTY-NEXT:       Hash Adjustments: []
+; EMPTY-NEXT:     }
 ; EMPTY:      DBI Stream {
 ; EMPTY-NEXT:   Dbi Version: 19990903
 ; EMPTY-NEXT:   Age: 1
 ; EMPTY-NEXT:   Incremental Linking: Yes
 ; EMPTY-NEXT:   Has CTypes: No
 ; EMPTY-NEXT:   Is Stripped: No
 ; EMPTY-NEXT:   Machine Type: x86
 ; EMPTY-NEXT:   Symbol Record Stream Index: 8

tools/llvm-pdbdump/llvm-pdbdump.cpp

 cl::opt<bool>
     DumpTpiRecords("raw-tpi-records",
                    cl::desc("dump CodeView type records from TPI stream"),
                    cl::cat(NativeOptions));
 cl::opt<bool> DumpTpiRecordBytes(
     "raw-tpi-record-bytes",
     cl::desc("dump CodeView type record raw bytes from TPI stream"),
     cl::cat(NativeOptions));
+cl::opt<bool> DumpTpiHash("raw-tpi-hash",
+                          cl::desc("dump CodeView TPI hash stream"),
+                          cl::cat(NativeOptions));
 cl::opt<bool>
     DumpIpiRecords("raw-ipi-records",
                    cl::desc("dump CodeView type records from IPI stream"),
                    cl::cat(NativeOptions));
 cl::opt<bool> DumpIpiRecordBytes(
     "raw-ipi-record-bytes",
     cl::desc("dump CodeView type record raw bytes from IPI stream"),
     cl::cat(NativeOptions));
for (auto &M : DS.getSectionMap()) {
     P.printNumber("ClassName", M.ClassName);
     P.printNumber("Offset", M.Offset);
     P.printNumber("SecByteLength", M.SecByteLength);
     P.flush();
   }
   return Error::success();
 }
 
+static void dumpTpiHash(ScopedPrinter &P, TpiStream &Tpi) {
+  if (!opts::DumpTpiHash)
+    return;
+  DictScope DD(P, "Hash");
+  codeview::FixedStreamArray<support::ulittle32_t> S = Tpi.getHashValues();
+  P.printList("Values", Tpi.getHashValues());
+  P.printList("Type Index Offsets", Tpi.getTypeIndexOffsets());
+  P.printList("Hash Adjustments", Tpi.getHashAdjustments());
+}
+
 static Error dumpTpiStream(ScopedPrinter &P, PDBFile &File,
                            codeview::CVTypeDumper &TD, uint32_t StreamIdx) {
   assert(StreamIdx == StreamTPI || StreamIdx == StreamIPI);
 
   bool DumpRecordBytes = false;
   bool DumpRecords = false;
   StringRef Label;
   StringRef VerLabel;
for (auto &Type : Tpi.types(&HadError)) {
       DictScope DD(P, "");
 
       if (DumpRecords)
         TD.dump(Type);
 
       if (DumpRecordBytes)
         P.printBinaryBlock("Bytes", Type.Data);
     }
+    dumpTpiHash(P, Tpi);
     if (HadError)
       return make_error<RawError>(raw_error_code::corrupt_file,
                                   "TPI stream contained corrupt record");
   } else if (opts::DumpModuleSyms) {
     // Even if the user doesn't want to dump type records, we still need to
     // iterate them in order to build the list of types so that we can print
     // them when dumping module symbols. So when they want to dump symbols
     // but not types, use a null output stream.
     ScopedPrinter *OldP = TD.getPrinter();
     TD.setPrinter(nullptr);
 
     bool HadError = false;
     for (auto &Type : Tpi.types(&HadError))
       TD.dump(Type);
 
     TD.setPrinter(OldP);
+    dumpTpiHash(P, Tpi);
     if (HadError)
       return make_error<RawError>(raw_error_code::corrupt_file,
                                   "TPI stream contained corrupt record");
   }
+
   P.flush();
   return Error::success();
 }
 
 static void printSectionOffset(llvm::raw_ostream &OS,
                                const SectionOffset &Off) {
   OS << Off.Off << ", " << Off.Isect;
 }
bool isRawDumpEnabled() {
   if (opts::DumpStreamBlocks)
     return true;
   if (opts::DumpSymRecordBytes)
     return true;
   if (opts::DumpTpiRecordBytes)
     return true;
   if (opts::DumpTpiRecords)
     return true;
+  if (opts::DumpTpiHash)
+    return true;
   if (opts::DumpIpiRecords)
     return true;
   if (opts::DumpIpiRecordBytes)
     return true;
   if (opts::DumpSectionContribs)
     return true;
   if (opts::DumpSectionMap)
     return true;
if (opts::RawAll) {
     opts::DumpModules = true;
     opts::DumpModuleFiles = true;
     opts::DumpModuleSyms = true;
     opts::DumpPublics = true;
     opts::DumpSectionHeaders = true;
     opts::DumpStreamSummary = true;
     opts::DumpStreamBlocks = true;
     opts::DumpTpiRecords = true;
+    opts::DumpTpiHash = true;
     opts::DumpIpiRecords = true;
     opts::DumpSectionMap = true;
     opts::DumpSectionContribs = true;
     opts::DumpLineInfo = true;
   }
 
   // When adding filters for excluded compilands and types, we need to remember
   // that these are regexes.  So special characters such as * and \ need to be