[scudo] Add PageReleaseContext to convey page usage status.

[Chia-hungDuan]

Summary

PageReleaseContext contains all the information needed for determing if
a page can be released. Splitting out the context increases the flexibility
of heterogenous free lists in the future. Also rename PackedCounterArray to
PageMap.

[cryptoad]

compiler-rt/lib/scudo/standalone/release.h

351

not const

387–390

nit const

[Chia-hungDuan]

Rebase only

[Chia-hungDuan]

Address review comment

[Chia-hungDuan]

compiler-rt/lib/scudo/standalone/release.h

351

Same as below

[cferris]

Mostly small issues.

compiler-rt/lib/scudo/standalone/release.h

147

the

227

Tabs snuck in, the same for a few of following lines.

269

More tabs.

[Chia-hungDuan]

address review comment

[Chia-hungDuan]

Address review comment

[Chia-hungDuan]

Try to handle tab problem...

[Chia-hungDuan]

Try to address tab problem...

[Chia-hungDuan]

Try to address tab problem...

[Chia-hungDuan]

Sorry for spamming patches. I was trying to see why phab keeps saying I added tab.
Tried several ways and couldn't remove it. I think I'm using space instead of tab...

[pcc]

In phabricator those markers just mean that you added indentation, not that you added a tab.

[Chia-hungDuan]

In D133895#3838929, @pcc wrote:

In phabricator those markers just mean that you added indentation, not that you added a tab.

Got it! Thanks!

[cferris]

LGTM

Any other reviewers, please take a quick look to make everything looks okay. If you feel that you might not understand all of the deeper changes, we'd still appreciate a higher level review to make sure there are no obvious errors.

compiler-rt/lib/scudo/standalone/primary32.h

NOINLINE uptr releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId,
     ReleaseRecorder Recorder(Base);
     auto SkipRegion = [this, First, ClassId](uptr RegionIndex) {
       return (PossibleRegions[First + RegionIndex] - 1U) != ClassId;
     };
     auto DecompactPtr = [](CompactPtrT CompactPtr) {
       return reinterpret_cast<uptr>(CompactPtr);
     };
     releaseFreeMemoryToOS(Sci->FreeList, RegionSize, NumberOfRegions, BlockSize,
-                          &Recorder, DecompactPtr, SkipRegion);
+                          Recorder, DecompactPtr, SkipRegion);
     if (Recorder.getReleasedRangesCount() > 0) {
       Sci->ReleaseInfo.PushedBlocksAtLastRelease = Sci->Stats.PushedBlocks;
       Sci->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount();
       Sci->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes();
       TotalReleasedBytes += Sci->ReleaseInfo.LastReleasedBytes;
     }
     Sci->ReleaseInfo.LastReleaseAtNs = getMonotonicTime();
 

compiler-rt/lib/scudo/standalone/primary64.h

NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
 
     ReleaseRecorder Recorder(Region->RegionBeg, &Region->Data);
     const uptr CompactPtrBase = getCompactPtrBaseByClassId(ClassId);
     auto DecompactPtr = [CompactPtrBase](CompactPtrT CompactPtr) {
       return decompactPtrInternal(CompactPtrBase, CompactPtr);
     };
     auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; };
     releaseFreeMemoryToOS(Region->FreeList, Region->AllocatedUser, 1U,
-                          BlockSize, &Recorder, DecompactPtr, SkipRegion);
+                          BlockSize, Recorder, DecompactPtr, SkipRegion);
 
     if (Recorder.getReleasedRangesCount() > 0) {
       Region->ReleaseInfo.PushedBlocksAtLastRelease =
           Region->Stats.PushedBlocks;
       Region->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount();
       Region->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes();
     }
     Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTime();
     return Recorder.getReleasedBytes();
   }
 };
 
 } // namespace scudo
 
 #endif // SCUDO_PRIMARY64_H_

compiler-rt/lib/scudo/standalone/release.h

 
 private:
   uptr ReleasedRangesCount = 0;
   uptr ReleasedBytes = 0;
   uptr Base = 0;
   MapPlatformData *Data = nullptr;
 };
 
-// A packed array of Counters. Each counter occupies 2^N bits, enough to store
-// counter's MaxValue. Ctor will try to use a static buffer first, and if that
-// fails (the buffer is too small or already locked), will allocate the
+// A Region page map is used to record the usage of pages in the regions. It
+// implements a packed array of Counters. Each counter occupies 2^N bits, enough
+// to store counter's MaxValue. Ctor will try to use a static buffer first, and
+// if that fails (the buffer is too small or already locked), will allocate the
 // required Buffer via map(). The caller is expected to check whether the
 // initialization was successful by checking isAllocated() result. For
 // performance sake, none of the accessors check the validity of the arguments,
 // It is assumed that Index is always in [0, N) range and the value is not
 // incremented past MaxValue.
-class PackedCounterArray {
+class RegionPageMap {
 public:
-  PackedCounterArray(uptr NumberOfRegions, uptr CountersPerRegion,
-                     uptr MaxValue)
-      : Regions(NumberOfRegions), NumCounters(CountersPerRegion) {
-    DCHECK_GT(Regions, 0);
-    DCHECK_GT(NumCounters, 0);
+  RegionPageMap()
+      : Regions(0),
+        NumCounters(0),
+        CounterSizeBitsLog(0),
+        CounterMask(0),
+        PackingRatioLog(0),
+        BitOffsetMask(0),
+        SizePerRegion(0),
+        BufferSize(0),
+        Buffer(nullptr) {}
+  RegionPageMap(uptr NumberOfRegions, uptr CountersPerRegion, uptr MaxValue) {
+    reset(NumberOfRegions, CountersPerRegion, MaxValue);
+  }
+  ~RegionPageMap() {
+    if (!isAllocated())
+      return;
+    if (Buffer == &StaticBuffer[0])
+      Mutex.unlock();
+    else
+      unmap(reinterpret_cast<void *>(Buffer),
+            roundUpTo(BufferSize, getPageSizeCached()));
+    Buffer = nullptr;
+  }
+
+  void reset(uptr NumberOfRegion, uptr CountersPerRegion, uptr MaxValue) {
+    DCHECK_GT(NumberOfRegion, 0);
+    DCHECK_GT(CountersPerRegion, 0);
     DCHECK_GT(MaxValue, 0);
+
+    Regions = NumberOfRegion;
+    NumCounters = CountersPerRegion;
+
     constexpr uptr MaxCounterBits = sizeof(*Buffer) * 8UL;
     // Rounding counter storage size up to the power of two allows for using
     // bit shifts calculating particular counter's Index and offset.
     const uptr CounterSizeBits =
         roundUpToPowerOfTwo(getMostSignificantSetBitIndex(MaxValue) + 1);
     DCHECK_LE(CounterSizeBits, MaxCounterBits);
     CounterSizeBitsLog = getLog2(CounterSizeBits);
     CounterMask = ~(static_cast<uptr>(0)) >> (MaxCounterBits - CounterSizeBits);
if (BufferSize <= (StaticBufferCount * sizeof(Buffer[0])) &&
       Buffer = &StaticBuffer[0];
       memset(Buffer, 0, BufferSize);
     } else {
       Buffer = reinterpret_cast<uptr *>(
           map(nullptr, roundUpTo(BufferSize, getPageSizeCached()),
               "scudo:counters", MAP_ALLOWNOMEM, &MapData));
     }
   }
-  ~PackedCounterArray() {
-    if (!isAllocated())
-      return;
-    if (Buffer == &StaticBuffer[0])
-      Mutex.unlock();
-    else
-      unmap(reinterpret_cast<void *>(Buffer),
-            roundUpTo(BufferSize, getPageSizeCached()), 0, &MapData);
-  }
 
   bool isAllocated() const { return !!Buffer; }
 
   uptr getCount() const { return NumCounters; }
 
   uptr get(uptr Region, uptr I) const {
     DCHECK_LT(Region, Regions);
     DCHECK_LT(I, NumCounters);
     const uptr Index = I >> PackingRatioLog;
     const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog;
     return (Buffer[Region * SizePerRegion + Index] >> BitOffset) & CounterMask;
   }
 
   void inc(uptr Region, uptr I) const {
     DCHECK_LT(get(Region, I), CounterMask);
     const uptr Index = I >> PackingRatioLog;
     const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog;
     DCHECK_LT(BitOffset, SCUDO_WORDSIZE);
+    DCHECK_EQ(isAllCounted(Region, I), false);
     Buffer[Region * SizePerRegion + Index] += static_cast<uptr>(1U)
                                               << BitOffset;
   }
 
   void incRange(uptr Region, uptr From, uptr To) const {
     DCHECK_LE(From, To);
     const uptr Top = Min(To + 1, NumCounters);
     for (uptr I = From; I < Top; I++)
       inc(Region, I);
   }
 
+  // Set the counter to the max value. Note that the max number of blocks in a
+  // page may vary. To provide an easier way to tell if all the blocks are
+  // counted for different pages, set to the same max value to denote the
+  // all-counted status.
+  void setAsAllCounted(uptr Region, uptr I) const {
+    DCHECK_LE(get(Region, I), CounterMask);
+    const uptr Index = I >> PackingRatioLog;
+    const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog;
+    DCHECK_LT(BitOffset, SCUDO_WORDSIZE);
+    Buffer[Region * SizePerRegion + Index] |= CounterMask << BitOffset;
+  }
+  bool isAllCounted(uptr Region, uptr I) const {
+    return get(Region, I) == CounterMask;
+  }
+
   uptr getBufferSize() const { return BufferSize; }
 
   static const uptr StaticBufferCount = 2048U;
 
 private:
-  const uptr Regions;
-  const uptr NumCounters;
+  uptr Regions;
+  uptr NumCounters;
   uptr CounterSizeBitsLog;
   uptr CounterMask;
   uptr PackingRatioLog;
   uptr BitOffsetMask;
 
   uptr SizePerRegion;
   uptr BufferSize;
   uptr *Buffer;
   [[no_unique_address]] MapPlatformData MapData = {};
 
   static HybridMutex Mutex;
   static uptr StaticBuffer[StaticBufferCount];
 };
 
 template <class ReleaseRecorderT> class FreePagesRangeTracker {
 public:
-  explicit FreePagesRangeTracker(ReleaseRecorderT *Recorder)
+  explicit FreePagesRangeTracker(ReleaseRecorderT &Recorder)
       : Recorder(Recorder), PageSizeLog(getLog2(getPageSizeCached())) {}
 
-  void processNextPage(bool Freed) {
-    if (Freed) {
+  void processNextPage(bool Released) {
+    if (Released) {
       if (!InRange) {
         CurrentRangeStatePage = CurrentPage;
         InRange = true;
       }
     } else {
       closeOpenedRange();
     }
     CurrentPage++;
   }
 
   void skipPages(uptr N) {
     closeOpenedRange();
     CurrentPage += N;
   }
 
   void finish() { closeOpenedRange(); }
 
 private:
   void closeOpenedRange() {
     if (InRange) {
-      Recorder->releasePageRangeToOS((CurrentRangeStatePage << PageSizeLog),
-                                     (CurrentPage << PageSizeLog));
+      Recorder.releasePageRangeToOS((CurrentRangeStatePage << PageSizeLog),
       InRange = false;
     }
   }
 
-  ReleaseRecorderT *const Recorder;
+  ReleaseRecorderT &Recorder;
   const uptr PageSizeLog;
   bool InRange = false;
   uptr CurrentPage = 0;
   uptr CurrentRangeStatePage = 0;
 };
 
-template <class TransferBatchT, class ReleaseRecorderT, typename DecompactPtrT,
-          typename SkipRegionT>
-NOINLINE void
-releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList,
-                      uptr RegionSize, uptr NumberOfRegions, uptr BlockSize,
-                      ReleaseRecorderT *Recorder, DecompactPtrT DecompactPtr,
-                      SkipRegionT SkipRegion) {
-  const uptr PageSize = getPageSizeCached();
-
-  // Figure out the number of chunks per page and whether we can take a fast
-  // path (the number of chunks per page is the same for all pages).
-  uptr FullPagesBlockCountMax;
-  bool SameBlockCountPerPage;
+struct PageReleaseContext {
+  PageReleaseContext(uptr BlockSize, uptr RegionSize, uptr NumberOfRegions) :
+      BlockSize(BlockSize),
+      RegionSize(RegionSize),
+      NumberOfRegions(NumberOfRegions) {
+    PageSize = getPageSizeCached();
     if (BlockSize <= PageSize) {
       if (PageSize % BlockSize == 0) {
         // Same number of chunks per page, no cross overs.
         FullPagesBlockCountMax = PageSize / BlockSize;
         SameBlockCountPerPage = true;
       } else if (BlockSize % (PageSize % BlockSize) == 0) {
         // Some chunks are crossing page boundaries, which means that the page
         // contains one or two partial chunks, but all pages contain the same
         // number of chunks.
         FullPagesBlockCountMax = PageSize / BlockSize + 1;
         SameBlockCountPerPage = true;
       } else {
         // Some chunks are crossing page boundaries, which means that the page
         // contains one or two partial chunks.
         FullPagesBlockCountMax = PageSize / BlockSize + 2;
         SameBlockCountPerPage = false;
       }
     } else {
       if (BlockSize % PageSize == 0) {
         // One chunk covers multiple pages, no cross overs.
         FullPagesBlockCountMax = 1;
         SameBlockCountPerPage = true;
       } else {
         // One chunk covers multiple pages, Some chunks are crossing page
         // boundaries. Some pages contain one chunk, some contain two.
         FullPagesBlockCountMax = 2;
         SameBlockCountPerPage = false;
       }
     }
 
-  const uptr PagesCount = roundUpTo(RegionSize, PageSize) / PageSize;
-  PackedCounterArray Counters(NumberOfRegions, PagesCount,
-                              FullPagesBlockCountMax);
-  if (!Counters.isAllocated())
-    return;
-
-  const uptr PageSizeLog = getLog2(PageSize);
-  const uptr RoundedRegionSize = PagesCount << PageSizeLog;
-  const uptr RoundedSize = NumberOfRegions * RoundedRegionSize;
+    PagesCount = roundUpTo(RegionSize, PageSize) / PageSize;
+    PageSizeLog = getLog2(PageSize);
+    RoundedRegionSize = PagesCount << PageSizeLog;
+    RoundedSize = NumberOfRegions * RoundedRegionSize;
+
+    PageMap.reset(NumberOfRegions, PagesCount, FullPagesBlockCountMax);
+    DCHECK(PageMap.isAllocated());
+  }
 
+  template<class TransferBatchT, typename DecompactPtrT>
+  void markFreeBlocks(const IntrusiveList<TransferBatchT> &FreeList,
+                      DecompactPtrT DecompactPtr, uptr Base) {
     // Iterate over free chunks and count how many free chunks affect each
     // allocated page.
     if (BlockSize <= PageSize && PageSize % BlockSize == 0) {
       // Each chunk affects one page only.
       for (const auto &It : FreeList) {
         for (u16 I = 0; I < It.getCount(); I++) {
-        const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase();
+          const uptr P = DecompactPtr(It.get(I)) - Base;
           if (P >= RoundedSize)
             continue;
           const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize;
           const uptr PInRegion = P - RegionIndex * RegionSize;
-        Counters.inc(RegionIndex, PInRegion >> PageSizeLog);
+          PageMap.inc(RegionIndex, PInRegion >> PageSizeLog);
         }
       }
     } else {
       // In all other cases chunks might affect more than one page.
       DCHECK_GE(RegionSize, BlockSize);
-    const uptr LastBlockInRegion = ((RegionSize / BlockSize) - 1U) * BlockSize;
+      const uptr LastBlockInRegion =
+          ((RegionSize / BlockSize) - 1U) * BlockSize;
       for (const auto &It : FreeList) {
         for (u16 I = 0; I < It.getCount(); I++) {
-        const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase();
+          const uptr P = DecompactPtr(It.get(I)) - Base;
           if (P >= RoundedSize)
             continue;
           const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize;
           uptr PInRegion = P - RegionIndex * RegionSize;
-        Counters.incRange(RegionIndex, PInRegion >> PageSizeLog,
+          PageMap.incRange(RegionIndex, PInRegion >> PageSizeLog,
                             (PInRegion + BlockSize - 1) >> PageSizeLog);
           // The last block in a region might straddle a page, so if it's
           // free, we mark the following "pretend" memory block(s) as free.
           if (PInRegion == LastBlockInRegion) {
             PInRegion += BlockSize;
             while (PInRegion < RoundedRegionSize) {
-            Counters.incRange(RegionIndex, PInRegion >> PageSizeLog,
+              PageMap.incRange(RegionIndex, PInRegion >> PageSizeLog,
                                 (PInRegion + BlockSize - 1) >> PageSizeLog);
               PInRegion += BlockSize;
             }
           }
         }
       }
     }
+  }
+
+  uptr BlockSize;
+  uptr RegionSize;
+  uptr NumberOfRegions;
+  uptr PageSize;
+  uptr PagesCount;
+  uptr PageSizeLog;
+  uptr RoundedRegionSize;
+  uptr RoundedSize;
+  uptr FullPagesBlockCountMax;
+  bool SameBlockCountPerPage;
+  RegionPageMap PageMap;
+};
+
+// Try to release the page which doesn't have any in-used block, i.e., they are
+// all free blocks. The `PageMap` will record the number of free blocks in each
+// page.
+template <class ReleaseRecorderT, typename SkipRegionT>
+NOINLINE void
+releaseFreeMemoryToOS(PageReleaseContext &Context,
+                      ReleaseRecorderT &Recorder, SkipRegionT SkipRegion) {
+  const uptr PageSize = Context.PageSize;
+  const uptr BlockSize = Context.BlockSize;
+  const uptr PagesCount = Context.PagesCount;
+  const uptr NumberOfRegions = Context.NumberOfRegions;
+  const uptr FullPagesBlockCountMax = Context.FullPagesBlockCountMax;
+  const bool SameBlockCountPerPage = Context.SameBlockCountPerPage;
+  RegionPageMap &PageMap = Context.PageMap;
 
   // Iterate over pages detecting ranges of pages with chunk Counters equal
   // to the expected number of chunks for the particular page.
   FreePagesRangeTracker<ReleaseRecorderT> RangeTracker(Recorder);
   if (SameBlockCountPerPage) {
     // Fast path, every page has the same number of chunks affecting it.
     for (uptr I = 0; I < NumberOfRegions; I++) {
       if (SkipRegion(I)) {
         RangeTracker.skipPages(PagesCount);
         continue;
       }
-      for (uptr J = 0; J < PagesCount; J++)
-        RangeTracker.processNextPage(Counters.get(I, J) ==
-                                     FullPagesBlockCountMax);
+      for (uptr J = 0; J < PagesCount; J++) {
+        const bool CanRelease = PageMap.get(I, J) == FullPagesBlockCountMax;
+        if (CanRelease)
+          PageMap.setAsAllCounted(I, J);
+        RangeTracker.processNextPage(CanRelease);
+      }
     }
   } else {
     // Slow path, go through the pages keeping count how many chunks affect
     // each page.
     const uptr Pn = BlockSize < PageSize ? PageSize / BlockSize : 1;
     const uptr Pnc = Pn * BlockSize;
     // The idea is to increment the current page pointer by the first chunk
     // size, middle portion size (the portion of the page covered by chunks
for (uptr I = 0; I < NumberOfRegions; I++) {
             BlocksPerPage++;
           CurrentBoundary += Pnc;
           if (CurrentBoundary < PageBoundary) {
             BlocksPerPage++;
             CurrentBoundary += BlockSize;
           }
         }
         PrevPageBoundary = PageBoundary;
-        RangeTracker.processNextPage(Counters.get(I, J) == BlocksPerPage);
+        const bool CanRelease = PageMap.get(I, J) == BlocksPerPage;
+        if (CanRelease)
+          PageMap.setAsAllCounted(I, J);
+        RangeTracker.processNextPage(CanRelease);
       }
     }
   }
   RangeTracker.finish();
 }
 
+// An overload releaseFreeMemoryToOS which doesn't require the page usage
+// information after releasing.
+template <class TransferBatchT, class ReleaseRecorderT, typename DecompactPtrT,
+          typename SkipRegionT>
+NOINLINE void
+releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList,
+                      uptr RegionSize, uptr NumberOfRegions, uptr BlockSize,
+                      ReleaseRecorderT &Recorder, DecompactPtrT DecompactPtr,
+                      SkipRegionT SkipRegion) {
+  PageReleaseContext Context(BlockSize, RegionSize, NumberOfRegions);
+  Context.markFreeBlocks(FreeList, DecompactPtr, Recorder.getBase());
+  releaseFreeMemoryToOS(Context, Recorder, SkipRegion);
+}
+
 } // namespace scudo
 
 #endif // SCUDO_RELEASE_H_

compiler-rt/lib/scudo/standalone/release.cpp

 //===-- release.cpp ---------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "release.h"
 
 namespace scudo {
 
-HybridMutex PackedCounterArray::Mutex = {};
-uptr PackedCounterArray::StaticBuffer[PackedCounterArray::StaticBufferCount];
+HybridMutex RegionPageMap::Mutex = {};
+uptr RegionPageMap::StaticBuffer[RegionPageMap::StaticBufferCount];
 
 } // namespace scudo

compiler-rt/lib/scudo/standalone/tests/release_test.cpp

 #include "size_class_map.h"
 
 #include <string.h>
 
 #include <algorithm>
 #include <random>
 #include <set>
 
-TEST(ScudoReleaseTest, PackedCounterArray) {
+TEST(ScudoReleaseTest, RegionPageMap) {
   for (scudo::uptr I = 0; I < SCUDO_WORDSIZE; I++) {
     // Various valid counter's max values packed into one word.
-    scudo::PackedCounterArray Counters2N(1U, 1U, 1UL << I);
-    EXPECT_EQ(sizeof(scudo::uptr), Counters2N.getBufferSize());
+    scudo::RegionPageMap PageMap2N(1U, 1U, 1UL << I);
+    EXPECT_EQ(sizeof(scudo::uptr), PageMap2N.getBufferSize());
     // Check the "all bit set" values too.
-    scudo::PackedCounterArray Counters2N1_1(1U, 1U, ~0UL >> I);
-    EXPECT_EQ(sizeof(scudo::uptr), Counters2N1_1.getBufferSize());
+    scudo::RegionPageMap PageMap2N1_1(1U, 1U, ~0UL >> I);
+    EXPECT_EQ(sizeof(scudo::uptr), PageMap2N1_1.getBufferSize());
     // Verify the packing ratio, the counter is Expected to be packed into the
     // closest power of 2 bits.
-    scudo::PackedCounterArray Counters(1U, SCUDO_WORDSIZE, 1UL << I);
+    scudo::RegionPageMap PageMap(1U, SCUDO_WORDSIZE, 1UL << I);
     EXPECT_EQ(sizeof(scudo::uptr) * scudo::roundUpToPowerOfTwo(I + 1),
-              Counters.getBufferSize());
+              PageMap.getBufferSize());
   }
 
   // Go through 1, 2, 4, 8, .. {32,64} bits per counter.
   for (scudo::uptr I = 0; (SCUDO_WORDSIZE >> I) != 0; I++) {
     // Make sure counters request one memory page for the buffer.
     const scudo::uptr NumCounters =
         (scudo::getPageSizeCached() / 8) * (SCUDO_WORDSIZE >> I);
-    scudo::PackedCounterArray Counters(1U, NumCounters,
+    scudo::RegionPageMap PageMap(1U, NumCounters,
                                        1UL << ((1UL << I) - 1));
-    Counters.inc(0U, 0U);
+    PageMap.inc(0U, 0U);
     for (scudo::uptr C = 1; C < NumCounters - 1; C++) {
-      EXPECT_EQ(0UL, Counters.get(0U, C));
-      Counters.inc(0U, C);
-      EXPECT_EQ(1UL, Counters.get(0U, C - 1));
+      EXPECT_EQ(0UL, PageMap.get(0U, C));
+      PageMap.inc(0U, C);
+      EXPECT_EQ(1UL, PageMap.get(0U, C - 1));
     }
-    EXPECT_EQ(0UL, Counters.get(0U, NumCounters - 1));
-    Counters.inc(0U, NumCounters - 1);
+    EXPECT_EQ(0UL, PageMap.get(0U, NumCounters - 1));
+    PageMap.inc(0U, NumCounters - 1);
     if (I > 0) {
-      Counters.incRange(0u, 0U, NumCounters - 1);
+      PageMap.incRange(0u, 0U, NumCounters - 1);
       for (scudo::uptr C = 0; C < NumCounters; C++)
-        EXPECT_EQ(2UL, Counters.get(0U, C));
+        EXPECT_EQ(2UL, PageMap.get(0U, C));
     }
   }
 }
 
 class StringRangeRecorder {
 public:
   std::string ReportedPages;
 
const char *TestCases[] = {
       ".x.x.x.x.x.x.x.x.x.x.x.x",
       ".x.x.x.x.x.x.x.x.x.x.x.x.",
       "x.x.x.x.x.x.x.x.x.x.x.x.x",
   };
   typedef scudo::FreePagesRangeTracker<StringRangeRecorder> RangeTracker;
 
   for (auto TestCase : TestCases) {
     StringRangeRecorder Recorder;
-    RangeTracker Tracker(&Recorder);
+    RangeTracker Tracker(Recorder);
     for (scudo::uptr I = 0; TestCase[I] != 0; I++)
       Tracker.processNextPage(TestCase[I] == 'x');
     Tracker.finish();
     // Strip trailing '.'-pages before comparing the results as they are not
     // going to be reported to range_recorder anyway.
     const char *LastX = strrchr(TestCase, 'x');
     std::string Expected(TestCase,
                          LastX == nullptr ? 0 : (LastX - TestCase + 1));
private:
   scudo::uptr Batch[MaxCount];
   scudo::u16 Count;
 };
 
 template <class SizeClassMap> void testReleaseFreeMemoryToOS() {
   typedef FreeBatch<SizeClassMap> Batch;
   const scudo::uptr PagesCount = 1024;
   const scudo::uptr PageSize = scudo::getPageSizeCached();
+  const scudo::uptr PageSizeLog = scudo::getLog2(PageSize);
   std::mt19937 R;
   scudo::u32 RandState = 42;
 
   for (scudo::uptr I = 1; I <= SizeClassMap::LargestClassId; I++) {
     const scudo::uptr BlockSize = SizeClassMap::getSizeByClassId(I);
     const scudo::uptr MaxBlocks = PagesCount * PageSize / BlockSize;
 
     // Generate the random free list.
for (auto const &Block : FreeArray) {
       if (CurrentBatch->getCount() == Batch::MaxCount)
         CurrentBatch = nullptr;
     }
 
     // Release the memory.
     auto SkipRegion = [](UNUSED scudo::uptr RegionIndex) { return false; };
     auto DecompactPtr = [](scudo::uptr P) { return P; };
     ReleasedPagesRecorder Recorder;
-    releaseFreeMemoryToOS(FreeList, MaxBlocks * BlockSize, 1U, BlockSize,
-                          &Recorder, DecompactPtr, SkipRegion);
+    scudo::PageReleaseContext Context(BlockSize,
+                                      /*RegionSize=*/MaxBlocks * BlockSize,
+                                      /*NumberOfRegions=*/1U);
+    Context.markFreeBlocks(FreeList, DecompactPtr, Recorder.getBase());
+    releaseFreeMemoryToOS(Context, Recorder, SkipRegion);
+    scudo::RegionPageMap &PageMap = Context.PageMap;
 
     // Verify that there are no released pages touched by used chunks and all
     // ranges of free chunks big enough to contain the entire memory pages had
     // these pages released.
     scudo::uptr VerifiedReleasedPages = 0;
     std::set<scudo::uptr> FreeBlocks(FreeArray.begin(), FreeArray.end());
 
     scudo::uptr CurrentBlock = 0;
for (scudo::uptr I = 0; I < MaxBlocks; I++) {
       } else {
         // Verify that this used chunk does not touch any released page.
         const scudo::uptr StartPage = CurrentBlock / PageSize;
         const scudo::uptr EndPage = (CurrentBlock + BlockSize - 1) / PageSize;
         for (scudo::uptr J = StartPage; J <= EndPage; J++) {
           const bool PageReleased = Recorder.ReportedPages.find(J * PageSize) !=
                                     Recorder.ReportedPages.end();
           EXPECT_EQ(false, PageReleased);
+          EXPECT_EQ(false,
+                    PageMap.isAllCounted(0, (J * PageSize) >> PageSizeLog));
         }
 
         if (InFreeRange) {
           InFreeRange = false;
           // Verify that all entire memory pages covered by this range of free
           // chunks were released.
           scudo::uptr P = scudo::roundUpTo(CurrentFreeRangeStart, PageSize);
           while (P + PageSize <= CurrentBlock) {
             const bool PageReleased =
                 Recorder.ReportedPages.find(P) != Recorder.ReportedPages.end();
             EXPECT_EQ(true, PageReleased);
+            EXPECT_EQ(true, PageMap.isAllCounted(0, P >> PageSizeLog));
             VerifiedReleasedPages++;
             P += PageSize;
           }
         }
       }
 
       CurrentBlock += BlockSize;
     }
 
     if (InFreeRange) {
       scudo::uptr P = scudo::roundUpTo(CurrentFreeRangeStart, PageSize);
       const scudo::uptr EndPage =
           scudo::roundUpTo(MaxBlocks * BlockSize, PageSize);
       while (P + PageSize <= EndPage) {
         const bool PageReleased =
             Recorder.ReportedPages.find(P) != Recorder.ReportedPages.end();
         EXPECT_EQ(true, PageReleased);
+        EXPECT_EQ(true, PageMap.isAllCounted(0, P >> PageSizeLog));
         VerifiedReleasedPages++;
         P += PageSize;
       }
     }
 
     EXPECT_EQ(Recorder.ReportedPages.size(), VerifiedReleasedPages);
 
     while (!FreeList.empty()) {