[LLD][COFF] Add more type record information to /summary

[aganea]

Summary

This adds the following two new lines to /summary:

      21351 Input OBJ files (expanded from all cmd-line inputs)
         61 PDB type server dependencies
         38 Precomp OBJ dependencies
 1420669231 Input type records         <<<<
78665073382 Input type records bytes   <<<<
    8801393 Merged TPI records
    3177158 Merged IPI records
      59194 Output PDB strings
   71576766 Global symbol records
   25416935 Module symbol records
    2103431 Public symbol records

Works with both /DEBUG and /DEBUG:GHASH.

[rnk]

lgtm, this is useful info

lld/COFF/PDB.cpp

977

A little voice in my head says, "addition is associative, parallelize it!" But I know the performance of this operation is not important. :)

lld/COFF/DebugTypes.h

public:
   struct MergedInfo {
     std::vector<uint8_t> recs;
     std::vector<uint16_t> recSizes;
     std::vector<uint32_t> recHashes;
   };
 
   MergedInfo mergedTpi;
   MergedInfo mergedIpi;
+
+  uint64_t nbTypeRecords = 0;
+  uint64_t nbTypeRecordsBytes = 0;
 };
 
 TpiSource *makeTpiSource(ObjFile *file);
 TpiSource *makeTypeServerSource(PDBInputFile *pdbInputFile);
 TpiSource *makeUseTypeServerSource(ObjFile *file,
                                    llvm::codeview::TypeServer2Record ts);
 TpiSource *makePrecompSource(ObjFile *file);
 TpiSource *makeUsePrecompSource(ObjFile *file,
                                 llvm::codeview::PrecompRecord ts);
 
 } // namespace coff
 } // namespace lld
 
 #endif

lld/COFF/DebugTypes.cpp

if (auto err = mergeTypeAndIdRecords(
     fatal("codeview::mergeTypeAndIdRecords failed: " +
           toString(std::move(err)));
 
   // In an object, there is only one mapping for both types and items.
   tpiMap = indexMapStorage;
   ipiMap = indexMapStorage;
 
   if (config->showSummary) {
+    nbTypeRecords = indexMapStorage.size() - nbHeadIndices;
+    nbTypeRecordsBytes = reader.getLength();
     // Count how many times we saw each type record in our input. This
     // calculation requires a second pass over the type records to classify each
     // record as a type or index. This is slow, but this code executes when
     // collecting statistics.
     m->tpiCounts.resize(m->getTypeTable().size());
     m->ipiCounts.resize(m->getIDTable().size());
     uint32_t srcIdx = nbHeadIndices;
     for (CVType &ty : types) {
Error TypeServerSource::mergeDebugT(TypeMerger *m) {
   if (maybeIpi) {
     if (auto err = mergeIdRecords(m->idTable, tpiMap, ipiSrc->indexMapStorage,
                                   maybeIpi->typeArray()))
       fatal("codeview::mergeIdRecords failed: " + toString(std::move(err)));
     ipiMap = ipiSrc->indexMapStorage;
   }
 
   if (config->showSummary) {
+    nbTypeRecords = tpiMap.size() + ipiMap.size();
+    nbTypeRecordsBytes =
+        expectedTpi->typeArray().getUnderlyingStream().getLength() +
+        (maybeIpi ? maybeIpi->typeArray().getUnderlyingStream().getLength()
+                  : 0);
+
     // Count how many times we saw each type record in our input. If a
     // destination type index is present in the source to destination type index
     // map, that means we saw it once in the input. Add it to our histogram.
     m->tpiCounts.resize(m->getTypeTable().size());
     m->ipiCounts.resize(m->getIDTable().size());
     for (TypeIndex ti : tpiMap)
       if (!ti.isSimple())
         ++m->tpiCounts[ti.toArrayIndex()];

 
 void TpiSource::remapTpiWithGHashes(GHashState *g) {
   assert(config->debugGHashes && "ghashes must be enabled");
   fillMapFromGHashes(g, indexMapStorage);
   tpiMap = indexMapStorage;
   ipiMap = indexMapStorage;
   mergeUniqueTypeRecords(file->debugTypes);
   // TODO: Free all unneeded ghash resources now that we have a full index map.
+
+  if (config->showSummary) {
+    nbTypeRecords = ghashes.size();
+    nbTypeRecordsBytes = file->debugTypes.size();
+  }
 }
 
 // PDBs do not actually store global hashes, so when merging a type server
 // PDB we have to synthesize global hashes.  To do this, we first synthesize
 // global hashes for the TPI stream, since it is independent, then we
 // synthesize hashes for the IPI stream, using the hashes for the TPI stream
 // as inputs.
 void TypeServerSource::loadGHashes() {
void TypeServerSource::remapTpiWithGHashes(GHashState *g) {
   if (pdbFile.hasPDBIpiStream()) {
     pdb::TpiStream &ipi = check(pdbFile.getPDBIpiStream());
     ipiSrc->indexMapStorage.resize(ipiSrc->ghashes.size());
     ipiSrc->fillMapFromGHashes(g, ipiSrc->indexMapStorage);
     ipiMap = ipiSrc->indexMapStorage;
     ipiSrc->tpiMap = tpiMap;
     ipiSrc->ipiMap = ipiMap;
     ipiSrc->mergeUniqueTypeRecords(typeArrayToBytes(ipi.typeArray()));
+
+    if (config->showSummary) {
+      nbTypeRecords = ipiSrc->ghashes.size();
+      nbTypeRecordsBytes = ipi.typeArray().getUnderlyingStream().getLength();
+    }
+  }
+
+  if (config->showSummary) {
+    nbTypeRecords += ghashes.size();
+    nbTypeRecordsBytes += tpi.typeArray().getUnderlyingStream().getLength();
   }
 }
 
 void UseTypeServerSource::remapTpiWithGHashes(GHashState *g) {
   // No remapping to do with /Zi objects. Simply use the index map from the type
   // server. Errors should have been reported earlier. Symbols from this object
   // will be ignored.
   Expected<TypeServerSource *> maybeTsSrc = getTypeServerSource();
void UsePrecompSource::remapTpiWithGHashes(GHashState *g) {
   }
 
   fillMapFromGHashes(g, indexMapStorage);
   tpiMap = indexMapStorage;
   ipiMap = indexMapStorage;
   mergeUniqueTypeRecords(file->debugTypes,
                          TypeIndex(precompDependency.getStartTypeIndex() +
                                    precompDependency.getTypesCount()));
+  if (config->showSummary) {
+    nbTypeRecords = ghashes.size();
+    nbTypeRecordsBytes = file->debugTypes.size();
+  }
 }
 
 namespace {
 /// A concurrent hash table for global type hashing. It is based on this paper:
 /// Concurrent Hash Tables: Fast and General(?)!
 /// https://dl.acm.org/doi/10.1145/3309206
 ///
 /// This hash table is meant to be used in two phases:

lld/COFF/PDB.cpp

private:
   DebugStringTableSubsection pdbStrTab;
 
   llvm::SmallString<128> nativePath;
 
   // For statistics
   uint64_t globalSymbols = 0;
   uint64_t moduleSymbols = 0;
   uint64_t publicSymbols = 0;
+  uint64_t nbTypeRecords = 0;
+  uint64_t nbTypeRecordsBytes = 0;
 };
 
 class DebugSHandler {
   PDBLinker &linker;
 
   /// The object file whose .debug$S sections we're processing.
   ObjFile &file;
 
void PDBLinker::addObjectsToPDB() {
   // Collect all the merged types.
   if (config->debugGHashes) {
     addGHashTypeInfo(builder);
   } else {
     addTypeInfo(builder.getTpiBuilder(), tMerger.getTypeTable());
     addTypeInfo(builder.getIpiBuilder(), tMerger.getIDTable());
   }
   t2.stop();
+
+  if (config->showSummary) {
+    for_each(TpiSource::instances, [&](TpiSource *source) {
+      nbTypeRecords += source->nbTypeRecords;
+      nbTypeRecordsBytes += source->nbTypeRecordsBytes;
+    });
+  }
 }
 
 void PDBLinker::addPublicsToPDB() {
   ScopedTimer t3(publicsLayoutTimer);
   // Compute the public symbols.
   auto &gsiBuilder = builder.getGsiBuilder();
   std::vector<pdb::BulkPublic> publics;
   symtab->forEachSymbol([&publics](Symbol *s) {
void PDBLinker::printStats() {
   auto print = [&](uint64_t v, StringRef s) {
     stream << format_decimal(v, 15) << " " << s << '\n';
   };
 
   print(ObjFile::instances.size(),
         "Input OBJ files (expanded from all cmd-line inputs)");
   print(TpiSource::countTypeServerPDBs(), "PDB type server dependencies");
   print(TpiSource::countPrecompObjs(), "Precomp OBJ dependencies");
+  print(nbTypeRecords, "Input type records");
+  print(nbTypeRecordsBytes, "Input type records bytes");
   print(builder.getTpiBuilder().getRecordCount(), "Merged TPI records");
   print(builder.getIpiBuilder().getRecordCount(), "Merged IPI records");
   print(pdbStrTab.size(), "Output PDB strings");
   print(globalSymbols, "Global symbol records");
   print(moduleSymbols, "Module symbol records");
   print(publicSymbols, "Public symbol records");
 
   auto printLargeInputTypeRecs = [&](StringRef name,

lld/test/COFF/pdb-type-server-simple.test

 CHECK:      200 | S_BUILDINFO [size = 8] BuildId = `[[B_BUILD]]`
 CHECK-LABEL:   Mod 0002 | `* Linker *`:
 
 SUMMARY:                                     Summary
 SUMMARY-NEXT: --------------------------------------------------------------------------------
 SUMMARY-NEXT:               2 Input OBJ files (expanded from all cmd-line inputs)
 SUMMARY-NEXT:               1 PDB type server dependencies
 SUMMARY-NEXT:               0 Precomp OBJ dependencies
+SUMMARY-NEXT:              25 Input type records
+SUMMARY-NEXT:             868 Input type records bytes
 SUMMARY-NEXT:               9 Merged TPI records
 SUMMARY-NEXT:              16 Merged IPI records
 SUMMARY-NEXT:               3 Output PDB strings
 SUMMARY-NEXT:               4 Global symbol records
 SUMMARY-NEXT:              14 Module symbol records
 SUMMARY-NEXT:               2 Public symbol records
 
 SUMMARY:      Top 10 types responsible for the most TPI input:

lld/test/COFF/precomp-link.test

 RUN: llvm-pdbutil dump -types %t.pdb | FileCheck %s
 
 
 SUMMARY:                                     Summary
 SUMMARY-NEXT: --------------------------------------------------------------------------------
 SUMMARY-NEXT:               3 Input OBJ files (expanded from all cmd-line inputs)
 SUMMARY-NEXT:               0 PDB type server dependencies
 SUMMARY-NEXT:               1 Precomp OBJ dependencies
-SUMMARY-NEXT:            874 Merged TPI records
-SUMMARY-NEXT:            170 Merged IPI records
+SUMMARY-NEXT:            1066 Input type records
+SUMMARY-NEXT:           55968 Input type records bytes
+SUMMARY-NEXT:             874 Merged TPI records
+SUMMARY-NEXT:             170 Merged IPI records
 SUMMARY-NEXT:               5 Output PDB strings
 SUMMARY-NEXT:             167 Global symbol records
 SUMMARY-NEXT:              20 Module symbol records
 SUMMARY-NEXT:               3 Public symbol records
 
 // precomp.h
 #pragma once
 int Function(char A);

lld/test/COFF/precomp-summary-fail.test

 
 The input files were tailored so that we end up with a resulting IPI stream
 smaller than the TPI stream, which would previously trigger a crash with
 /summary.
 
 RUN: rm -rf %t && mkdir %t
 RUN: yaml2obj < %S/Inputs/precomp2.yaml -o %t\precomp2.obj
 RUN: yaml2obj < %S/Inputs/precomp2-a.yaml -o %t\precomp2-a.obj
 RUN: lld-link %t\precomp2-a.obj %t\precomp2.obj /nodefaultlib /noentry \
 RUN:    /dll /out:%t.dll /debug /summary | FileCheck %s -check-prefix SUMMARY
 
 SUMMARY:                                     Summary
 SUMMARY-NEXT: --------------------------------------------------------------------------------
 SUMMARY-NEXT:               2 Input OBJ files (expanded from all cmd-line inputs)
 SUMMARY-NEXT:               0 PDB type server dependencies
 SUMMARY-NEXT:               1 Precomp OBJ dependencies
+SUMMARY-NEXT:               8 Input type records
+SUMMARY-NEXT:             232 Input type records bytes
 SUMMARY-NEXT:               3 Merged TPI records
 SUMMARY-NEXT:               2 Merged IPI records
 SUMMARY-NEXT:               1 Output PDB strings
 SUMMARY-NEXT:               0 Global symbol records
 SUMMARY-NEXT:               4 Module symbol records
 SUMMARY-NEXT:               0 Public symbol records