[msf] Make Fpm reader and writer use MappedBlockStream

[zturner]

Summary

MappedBlockStream can work with any sequence of block data where the ordering is specified by a list of block numbers. This is true even in the case of split FPMs. So here we add support for creating a MappedBlockStream representing the Fpm so we can just read it in one single operation. This is also nice because we have to read/write the Fpm from multiple places, so this allows us to centralize all the logic to make sure we aren't reproducing the split interval logic in many locations.

For some reason, this also seems to fix a bug. I no longer show any orphaned pages on larger PDBs.

[ruiu]

LGTM

include/llvm/DebugInfo/MSF/MSFCommon.h

   return BlockNumber * BlockSize;
 }
 
+inline uint32_t getNumFpmIntervals(const MSFLayout &L) {
+  return llvm::alignTo(L.SB->NumBlocks, L.SB->BlockSize) / L.SB->BlockSize;
+}
+
+inline uint32_t getFullFpmByteSize(const MSFLayout &L) {
+  return llvm::alignTo(L.SB->NumBlocks, 8) / 8;
+}
+
 Error validateSuperBlock(const SuperBlock &SB);
 } // namespace msf
 } // namespace llvm

include/llvm/DebugInfo/MSF/MSFStreamLayout.h

 #ifndef LLVM_DEBUGINFO_MSF_MSFSTREAMLAYOUT_H
 #define LLVM_DEBUGINFO_MSF_MSFSTREAMLAYOUT_H
 
-#include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/Endian.h"
 
 #include <cstdint>
+#include <vector>
 
 namespace llvm {
 namespace msf {

 class MSFStreamLayout {
 public:
   uint32_t Length;
-  ArrayRef<support::ulittle32_t> Blocks;
+  std::vector<support::ulittle32_t> Blocks;
 };
 } // namespace msf
 } // namespace llvm

include/llvm/DebugInfo/MSF/MappedBlockStream.h

 /// of bytes.
 class MappedBlockStream : public ReadableStream {
   friend class WritableMappedBlockStream;
-
 public:
   static std::unique_ptr<MappedBlockStream>
   createStream(uint32_t BlockSize, uint32_t NumBlocks,

                       uint32_t StreamIndex);
 
   static std::unique_ptr<MappedBlockStream>
+  createFpmStream(const MSFLayout &Layout, const ReadableStream &MsfData);
+
+  static std::unique_ptr<MappedBlockStream>
   createDirectoryStream(const MSFLayout &Layout, const ReadableStream &MsfData);
 
   Error readBytes(uint32_t Offset, uint32_t Size,

   static std::unique_ptr<WritableMappedBlockStream>
   createDirectoryStream(const MSFLayout &Layout, const WritableStream &MsfData);
 
+  static std::unique_ptr<WritableMappedBlockStream>
+  createFpmStream(const MSFLayout &Layout, const WritableStream &MsfData);
+
   Error readBytes(uint32_t Offset, uint32_t Size,
                   ArrayRef<uint8_t> &Buffer) const override;
   Error readLongestContiguousChunk(uint32_t Offset,

lib/DebugInfo/MSF/MappedBlockStream.cpp

 };
 }
 
+static void initializeFpmStreamLayout(const MSFLayout &Layout,
+                                      MSFStreamLayout &FpmLayout) {
+  uint32_t NumFpmIntervals = msf::getNumFpmIntervals(Layout);
+  support::ulittle32_t FpmBlock = Layout.SB->FreeBlockMapBlock;
+  assert(FpmBlock == 1 || FpmBlock == 2);
+  while (NumFpmIntervals > 0) {
+    FpmLayout.Blocks.push_back(FpmBlock);
+    FpmBlock += Layout.SB->BlockSize;
+    --NumFpmIntervals;
+  }
+  FpmLayout.Length = msf::getFullFpmByteSize(Layout);
+}
+
 typedef std::pair<uint32_t, uint32_t> Interval;
 static Interval intersect(const Interval &I1, const Interval &I2) {
   return std::make_pair(std::max(I1.first, I2.first),

   return createStream(Layout.SB->BlockSize, Layout.SB->NumBlocks, SL, MsfData);
 }
 
+std::unique_ptr<MappedBlockStream>
+MappedBlockStream::createFpmStream(const MSFLayout &Layout,
+                                   const ReadableStream &MsfData) {
+  MSFStreamLayout SL;
+  initializeFpmStreamLayout(Layout, SL);
+  return createStream(Layout.SB->BlockSize, Layout.SB->NumBlocks, SL, MsfData);
+}
+
 Error MappedBlockStream::readBytes(uint32_t Offset, uint32_t Size,
                                    ArrayRef<uint8_t> &Buffer) const {
   // Make sure we aren't trying to read beyond the end of the stream.

   return createStream(Layout.SB->BlockSize, Layout.SB->NumBlocks, SL, MsfData);
 }
 
+std::unique_ptr<WritableMappedBlockStream>
+WritableMappedBlockStream::createFpmStream(const MSFLayout &Layout,
+                                           const WritableStream &MsfData) {
+  MSFStreamLayout SL;
+  initializeFpmStreamLayout(Layout, SL);
+  return createStream(Layout.SB->BlockSize, Layout.SB->NumBlocks, SL, MsfData);
+}
+
 Error WritableMappedBlockStream::readBytes(uint32_t Offset, uint32_t Size,
                                            ArrayRef<uint8_t> &Buffer) const {
   return ReadInterface.readBytes(Offset, Size, Buffer);

lib/DebugInfo/PDB/Raw/PDBFile.cpp

 
   // Initialize Free Page Map.
   ContainerLayout.FreePageMap.resize(SB->NumBlocks);
-  ArrayRef<uint8_t> FpmBytes;
   // The Fpm exists either at block 1 or block 2 of the MSF.  However, this
   // allows for a maximum of getBlockSize() * 8 blocks bits in the Fpm, and
   // thusly an equal number of total blocks in the file.  For a block size

   // at getBlockSize() intervals, so we have to be compatible.
   // See the function fpmPn() for more information:
   // https://github.com/Microsoft/microsoft-pdb/blob/master/PDB/msf/msf.cpp#L489
-
-  uint32_t BlocksPerSection = getBlockSize();
-  uint64_t FpmBlockOffset = SB->FreeBlockMapBlock;
+  auto FpmStream = MappedBlockStream::createFpmStream(ContainerLayout, *Buffer);
+  StreamReader FpmReader(*FpmStream);
+  ArrayRef<uint8_t> FpmBytes;
+  if (auto EC = FpmReader.readBytes(FpmBytes,
+                                    msf::getFullFpmByteSize(ContainerLayout)))
+    return EC;
   uint32_t BlocksRemaining = getBlockCount();
-  for (uint32_t SI = 0; BlocksRemaining > 0; ++SI) {
-    uint32_t FpmFileOffset = FpmBlockOffset * getBlockSize();
-
-    if (auto EC = Buffer->readBytes(FpmFileOffset, getBlockSize(), FpmBytes))
-      return EC;
-
-    uint32_t BlocksThisSection = std::min(BlocksRemaining, BlocksPerSection);
-    for (uint32_t I = 0; I < BlocksThisSection; ++I) {
-      uint32_t BI = I + BlocksPerSection * SI;
-
-      if (FpmBytes[I / 8] & (1 << (I % 8)))
+  uint32_t BI = 0;
+  for (auto Byte : FpmBytes) {
+    uint32_t BlocksThisByte = std::min(BlocksRemaining, 8U);
+    for (uint32_t I = 0; I < BlocksThisByte; ++I) {
+      if (Byte & (1 << I))
         ContainerLayout.FreePageMap[BI] = true;
+      --BlocksRemaining;
+      ++BI;
     }
-    BlocksRemaining -= BlocksThisSection;
-    FpmBlockOffset += BlocksPerSection;
   }
 
   Reader.setOffset(getBlockMapOffset());

tools/llvm-pdbdump/LLVMOutputStyle.cpp

   recordKnownUsedPage(PS, 0); // MSF Super Block
 
   uint32_t BlocksPerSection = File.getBlockSize();
-  uint32_t NumSections =
-      llvm::alignTo(File.getBlockCount(), BlocksPerSection) / BlocksPerSection;
+  uint32_t NumSections = msf::getNumFpmIntervals(File.getMsfLayout());
   for (uint32_t I = 0; I < NumSections; ++I) {
     uint32_t Fpm0 = 1 + BlocksPerSection * I;
     // 2 Fpm blocks spaced at `getBlockSize()` block intervals