[msf] Figure out the rest of the FPM and dump more detailed page allocation stats

[zturner]

Summary

The FPM is split at regular intervals across the MSF file, as the MS code suggests. It turns out that the value of the interval is precisely the block size. If the block size is 4096, then there are two Fpm pages every 4096 blocks.

So here we teach the PDBFile class to parse a split FPM, and also add more options when dumping the FPM to display some additional information such as orphaned pages (pages which the FPM says are allocated, but which nothing appears to use), use after free pages (pages which the FPM says are not allocated, but which are referenced by a stream), and multiple use pages (pages which the FPM says are allocated but are used more than once).

[ruiu]

Do you want to add a test?

[zturner]

Fix the test to exercise the new option.

[ruiu]

LGTM

lib/DebugInfo/PDB/Raw/PDBFile.cpp

130

You are mixing kilo=1000 and kilo=1024 (32K=32*1024, 4K=4*1024, but 134MB=134*1000*1000). It's more accurate to say that the total is 128MB.

[zturner]

lib/DebugInfo/PDB/Raw/PDBFile.cpp

130

Ahh, thanks for catching that. Gotta be careful using Google. I wrote "4096 * 4096 * 8 bytes to megabytes" into Google and it told me the answer is 134 Megabytes. I guess I have to write KiB and MiB to be accurate, ugh.

include/llvm/DebugInfo/PDB/Raw/PDBFile.h

public:
   getStreamBlockList(uint32_t StreamIndex) const override;
   uint32_t getFileSize() const;
 
   Expected<ArrayRef<uint8_t>> getBlockData(uint32_t BlockIndex,
                                            uint32_t NumBytes) const override;
   Error setBlockData(uint32_t BlockIndex, uint32_t Offset,
                      ArrayRef<uint8_t> Data) const override;
 
+  ArrayRef<uint32_t> getFpmPages() const { return FpmPages; }
+
   ArrayRef<support::ulittle32_t> getStreamSizes() const {
     return ContainerLayout.StreamSizes;
   }
   ArrayRef<ArrayRef<support::ulittle32_t>> getStreamMap() const {
     return ContainerLayout.StreamMap;
   }
 
   const msf::MSFLayout &getMsfLayout() const { return ContainerLayout; }
public:
   Expected<SymbolStream &> getPDBSymbolStream();
   Expected<NameHashTable &> getStringTable();
 
 private:
   BumpPtrAllocator &Allocator;
 
   std::unique_ptr<msf::ReadableStream> Buffer;
 
+  std::vector<uint32_t> FpmPages;
   msf::MSFLayout ContainerLayout;
 
   std::unique_ptr<InfoStream> Info;
   std::unique_ptr<DbiStream> Dbi;
   std::unique_ptr<TpiStream> Tpi;
   std::unique_ptr<TpiStream> Ipi;
   std::unique_ptr<PublicsStream> Publics;
   std::unique_ptr<SymbolStream> Symbols;
   std::unique_ptr<msf::MappedBlockStream> DirectoryStream;
   std::unique_ptr<msf::MappedBlockStream> StringTableStream;
   std::unique_ptr<NameHashTable> StringTable;
 };
 }
 }
 
 #endif

lib/DebugInfo/PDB/Raw/PDBFile.cpp

if (auto EC = msf::validateSuperBlock(*SB))
     return EC;
 
   if (Buffer->getLength() % SB->BlockSize != 0)
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "File size is not a multiple of block size");
   ContainerLayout.SB = SB;
 
   // Initialize Free Page Map.
-  ContainerLayout.FreePageMap.resize(getBlockSize() * 8);
-  uint64_t FPMOffset = SB->FreeBlockMapBlock * getBlockSize();
-  ArrayRef<uint8_t> FPMBlock;
-  if (auto EC = Buffer->readBytes(FPMOffset, getBlockSize(), FPMBlock))
+  ContainerLayout.FreePageMap.resize(SB->NumBlocks);
+  ArrayRef<uint8_t> FpmBytes;
+  // The Fpm exists either at block 1 or block 2 of the Msf.  However, this
+  // allows for a maximum of getBlockSize() * 8 blocks bits in the Fpm, and
+  // thusly an equal number of total blocks in the file.  For a block size
+  // of 4KB (very common), this would yield 32K total blocks in file, for a
+  // maximum file size of 32K * 4K = 134MB.  Obviously this won't do, so
+  // the Fpm is split across the file at `getBlockSize()` intervals.  As a
+  // result, every block whose index is of the form |{1,2} + getBlockSize() * k|
+  // for any non-negative integer k is an Fpm block.  In theory, we only really
+  // need to reserve blocks of the form |{1,2} + getBlockSize() * 8 * k|, but
+  // current versions of the MSF format already expect the Fpm to be arranged
+  // at getBlockSize() intervals, so we have to be compatible.
+  // See the function fpmPn() for more information:
+  // https://github.com/Microsoft/microsoft-pdb/blob/master/PDB/msf/msf.cpp#L489
+
+  uint32_t BlocksPerSection = getBlockSize();
+  uint64_t FpmBlockOffset = SB->FreeBlockMapBlock;
+  uint32_t BlocksRemaining = getBlockCount();
+  for (uint32_t SI = 0; BlocksRemaining > 0; ++SI) {
+    uint32_t FpmFileOffset = FpmBlockOffset * getBlockSize();
+
+    if (auto EC = Buffer->readBytes(FpmFileOffset, getBlockSize(), FpmBytes))
       return EC;
-  for (uint32_t I = 0, E = getBlockSize() * 8; I != E; ++I)
-    if (FPMBlock[I / 8] & (1 << (I % 8)))
-      ContainerLayout.FreePageMap[I] = true;
+
+    uint32_t BlocksThisSection = std::min(BlocksRemaining, BlocksPerSection);
+    for (uint32_t I = 0; I < BlocksThisSection; ++I) {
+      uint32_t BI = I + BlocksPerSection * SI;
+
+      if (FpmBytes[I / 8] & (1 << (I % 8)))
+        ContainerLayout.FreePageMap[BI] = true;
+    }
+    BlocksRemaining -= BlocksThisSection;
+    FpmBlockOffset += BlocksPerSection;
+  }
 
   Reader.setOffset(getBlockMapOffset());
   if (auto EC = Reader.readArray(ContainerLayout.DirectoryBlocks,
                                  getNumDirectoryBlocks()))
     return EC;
 
   return Error::success();
 }

tools/llvm-pdbdump/LLVMOutputStyle.h

 #define LLVM_TOOLS_LLVMPDBDUMP_LLVMOUTPUTSTYLE_H
 
 #include "OutputStyle.h"
 
 #include "llvm/DebugInfo/CodeView/TypeDumper.h"
 #include "llvm/Support/ScopedPrinter.h"
 
 namespace llvm {
+class BitVector;
 namespace pdb {
 class LLVMOutputStyle : public OutputStyle {
 public:
   LLVMOutputStyle(PDBFile &File);
 
   Error dump() override;
 
 private:
   Error dumpFileHeaders();
   Error dumpStreamSummary();
   Error dumpFreePageMap();
   Error dumpStreamBlocks();
   Error dumpStreamData();
   Error dumpInfoStream();
   Error dumpNamedStream();
   Error dumpTpiStream(uint32_t StreamIdx);
   Error dumpDbiStream();
   Error dumpSectionContribs();
   Error dumpSectionMap();
   Error dumpPublicsStream();
   Error dumpSectionHeaders();
   Error dumpFpoStream();
 
+  void dumpBitVector(StringRef Name, const BitVector &V);
+
   void flush();
 
   PDBFile &File;
   ScopedPrinter P;
   codeview::CVTypeDumper TD;
 };
 }
 }
 
 #endif

tools/llvm-pdbdump/LLVMOutputStyle.cpp

 
 #include <unordered_map>
 
 using namespace llvm;
 using namespace llvm::codeview;
 using namespace llvm::msf;
 using namespace llvm::pdb;
 
+namespace {
+struct PageStats {
+  explicit PageStats(const BitVector &FreePages)
+      : Upm(FreePages), ActualUsedPages(FreePages.size()),
+        MultiUsePages(FreePages.size()), UseAfterFreePages(FreePages.size()) {
+    const_cast<BitVector &>(Upm).flip();
+    // To calculate orphaned pages, we start with the set of pages that the
+    // MSF thinks are used.  Each time we find one that actually *is* used,
+    // we unset it.  Whichever bits remain set at the end are orphaned.
+    OrphanedPages = Upm;
+  }
+
+  // The inverse of the MSF File's copy of the Fpm.  The basis for which we
+  // determine the allocation status of each page.
+  const BitVector Upm;
+
+  // Pages which are marked as used in the FPM and are used at least once.
+  BitVector ActualUsedPages;
+
+  // Pages which are marked as used in the FPM but are used more than once.
+  BitVector MultiUsePages;
+
+  // Pages which are marked as used in the FPM but are not used at all.
+  BitVector OrphanedPages;
+
+  // Pages which are marked free in the FPM but are used.
+  BitVector UseAfterFreePages;
+};
+}
+
+static void recordKnownUsedPage(PageStats &Stats, uint32_t UsedIndex) {
+  if (Stats.Upm.test(UsedIndex)) {
+    if (Stats.ActualUsedPages.test(UsedIndex))
+      Stats.MultiUsePages.set(UsedIndex);
+    Stats.ActualUsedPages.set(UsedIndex);
+    Stats.OrphanedPages.reset(UsedIndex);
+  } else {
+    // The MSF doesn't think this page is used, but it is.
+    Stats.UseAfterFreePages.set(UsedIndex);
+  }
+}
+
 static void printSectionOffset(llvm::raw_ostream &OS,
                                const SectionOffset &Off) {
   OS << Off.Off << ", " << Off.Isect;
 }
 
 LLVMOutputStyle::LLVMOutputStyle(PDBFile &File)
     : File(File), P(outs()), TD(&P, false) {}
 
Error LLVMOutputStyle::dumpStreamSummary() {
   if (!Info)
     consumeError(Info.takeError());
 
   P.flush();
   return Error::success();
 }
 
 Error LLVMOutputStyle::dumpFreePageMap() {
-  if (!opts::raw::DumpFreePageMap)
+  if (!opts::raw::DumpPageStats)
     return Error::success();
-  const BitVector &FPM = File.getMsfLayout().FreePageMap;
 
-  std::vector<uint32_t> Vec;
-  for (uint32_t I = 0, E = FPM.size(); I != E; ++I)
-    if (!FPM[I])
-      Vec.push_back(I);
-
-  // Prints out used pages instead of free pages because
+  // Start with used pages instead of free pages because
   // the number of free pages is far larger than used pages.
-  P.printList("Used Page Map", Vec);
+  BitVector FPM = File.getMsfLayout().FreePageMap;
+
+  PageStats PS(FPM);
+
+  recordKnownUsedPage(PS, 0); // MSF Super Block
+
+  uint32_t BlocksPerSection = File.getBlockSize();
+  uint32_t NumSections =
+      llvm::alignTo(File.getBlockCount(), BlocksPerSection) / BlocksPerSection;
+  for (uint32_t I = 0; I < NumSections; ++I) {
+    uint32_t Fpm0 = 1 + BlocksPerSection * I;
+    recordKnownUsedPage(PS, Fpm0);
+    recordKnownUsedPage(PS, Fpm0 + 1);
+  }
+
+  recordKnownUsedPage(PS, File.getBlockMapIndex()); // Stream Table
+
+  for (auto DB : File.getDirectoryBlockArray()) {
+    recordKnownUsedPage(PS, DB);
+  }
+  for (auto &SE : File.getStreamMap()) {
+    for (auto &S : SE) {
+      recordKnownUsedPage(PS, S);
+    }
+  }
+
+  dumpBitVector("Msf Free Pages", FPM);
+  dumpBitVector("Orphaned Pages", PS.OrphanedPages);
+  dumpBitVector("Multiply Used Pages", PS.MultiUsePages);
+  dumpBitVector("Use After Free Pages", PS.UseAfterFreePages);
   return Error::success();
 }
 
+void LLVMOutputStyle::dumpBitVector(StringRef Name, const BitVector &V) {
+  std::vector<uint32_t> Vec;
+  for (uint32_t I = 0, E = V.size(); I != E; ++I)
+    if (V[I])
+      Vec.push_back(I);
+  P.printList(Name, Vec);
+}
+
 Error LLVMOutputStyle::dumpStreamBlocks() {
   if (!opts::raw::DumpStreamBlocks)
     return Error::success();
 
   ListScope L(P, "StreamBlocks");
   uint32_t StreamCount = File.getNumStreams();
   for (uint32_t StreamIdx = 0; StreamIdx < StreamCount; ++StreamIdx) {
     std::string Name("Stream ");

tools/llvm-pdbdump/llvm-pdbdump.h

 extern llvm::cl::list<std::string> IncludeSymbols;
 extern llvm::cl::list<std::string> IncludeCompilands;
 }
 
 namespace raw {
 extern llvm::cl::opt<bool> DumpHeaders;
 extern llvm::cl::opt<bool> DumpStreamBlocks;
 extern llvm::cl::opt<bool> DumpStreamSummary;
-extern llvm::cl::opt<bool> DumpFreePageMap;
+extern llvm::cl::opt<bool> DumpPageStats;
 extern llvm::cl::opt<bool> DumpTpiHash;
 extern llvm::cl::opt<bool> DumpTpiRecordBytes;
 extern llvm::cl::opt<bool> DumpTpiRecords;
 extern llvm::cl::opt<bool> DumpIpiRecords;
 extern llvm::cl::opt<bool> DumpIpiRecordBytes;
 extern llvm::cl::opt<std::string> DumpStreamDataIdx;
 extern llvm::cl::opt<std::string> DumpStreamDataName;
 extern llvm::cl::opt<bool> DumpModules;

tools/llvm-pdbdump/llvm-pdbdump.cpp

 cl::opt<bool> DumpHeaders("headers", cl::desc("dump PDB headers"),
                           cl::cat(MsfOptions), cl::sub(RawSubcommand));
 cl::opt<bool> DumpStreamBlocks("stream-blocks",
                                cl::desc("dump PDB stream blocks"),
                                cl::cat(MsfOptions), cl::sub(RawSubcommand));
 cl::opt<bool> DumpStreamSummary("stream-summary",
                                 cl::desc("dump summary of the PDB streams"),
                                 cl::cat(MsfOptions), cl::sub(RawSubcommand));
-cl::opt<bool> DumpFreePageMap("fpm", cl::desc("dump free page bitmap"),
-                              cl::cat(MsfOptions), cl::sub(RawSubcommand));
+cl::opt<bool> DumpPageStats(
+    "page-stats",
+    cl::desc("dump allocation stats of the pages in the MSF file"),
 
 // TYPE OPTIONS
 cl::opt<bool>
     DumpTpiRecords("tpi-records",
                    cl::desc("dump CodeView type records from TPI stream"),
                    cl::cat(TypeOptions), cl::sub(RawSubcommand));
 cl::opt<bool> DumpTpiRecordBytes(
     "tpi-record-bytes",
int main(int argc_, const char *argv_[]) {
   if ((opts::PdbToYamlSubcommand || opts::RawSubcommand) && opts::raw::RawAll) {
     opts::raw::DumpHeaders = true;
     opts::raw::DumpModules = true;
     opts::raw::DumpModuleFiles = true;
     opts::raw::DumpModuleSyms = true;
     opts::raw::DumpPublics = true;
     opts::raw::DumpSectionHeaders = true;
     opts::raw::DumpStreamSummary = true;
-    opts::raw::DumpFreePageMap = true;
+    opts::raw::DumpPageStats = true;
     opts::raw::DumpStreamBlocks = true;
     opts::raw::DumpTpiRecords = true;
     opts::raw::DumpTpiHash = true;
     opts::raw::DumpIpiRecords = true;
     opts::raw::DumpSectionMap = true;
     opts::raw::DumpSectionContribs = true;
     opts::raw::DumpLineInfo = true;
     opts::raw::DumpFpo = true;