[scudo] Group poppedBlocks/pushedBlocks into BlocksInfo (NFC)

[Chia-hungDuan]

Summary

Create a new BlocksInfo to contain a list of blocks, poppedBlocks and
pushedBlocks. This is the preparation of adding new lock for operations
on freelist.

[Chia-hungDuan]

Rebase only

[Chia-hungDuan]

rebase only

[cferris]

LGTM.

compiler-rt/lib/scudo/standalone/primary32.h

TransferBatch *popBatch(CacheT *C, uptr ClassId) {
     TransferBatch *B = popBatchImpl(C, ClassId, Sci);
     if (UNLIKELY(!B)) {
       if (UNLIKELY(!populateFreeList(C, ClassId, Sci)))
         return nullptr;
       B = popBatchImpl(C, ClassId, Sci);
       // if `populateFreeList` succeeded, we are supposed to get free blocks.
       DCHECK_NE(B, nullptr);
     }
-    Sci->Stats.PoppedBlocks += B->getCount();
+    Sci->FreeListInfo.PoppedBlocks += B->getCount();
     return B;
   }
 
   // Push the array of free blocks to the designated batch group.
   void pushBlocks(CacheT *C, uptr ClassId, CompactPtrT *Array, u32 Size) {
     DCHECK_LT(ClassId, NumClasses);
     DCHECK_GT(Size, 0);
 
     SizeClassInfo *Sci = getSizeClassInfo(ClassId);
     if (ClassId == SizeClassMap::BatchClassId) {
       ScopedLock L(Sci->Mutex);
       // Constructing a batch group in the free list will use two blocks in
       // BatchClassId. If we are pushing BatchClassId blocks, we will use the
       // blocks in the array directly (can't delegate local cache which will
       // cause a recursive allocation). However, The number of free blocks may
       // be less than two. Therefore, populate the free list before inserting
       // the blocks.
       if (Size == 1 && !populateFreeList(C, ClassId, Sci))
         return;
       pushBlocksImpl(C, ClassId, Sci, Array, Size);
-      Sci->Stats.PushedBlocks += Size;
+      Sci->FreeListInfo.PushedBlocks += Size;
       return;
     }
 
     // TODO(chiahungduan): Consider not doing grouping if the group size is not
     // greater than the block size with a certain scale.
 
     // Sort the blocks so that blocks belonging to the same group can be pushed
     // together.
for (u32 I = 1; I < Size; ++I) {
         --J;
       }
       Array[J] = Cur;
     }
 
     ScopedLock L(Sci->Mutex);
     pushBlocksImpl(C, ClassId, Sci, Array, Size, SameGroup);
 
-    Sci->Stats.PushedBlocks += Size;
+    Sci->FreeListInfo.PushedBlocks += Size;
     if (ClassId != SizeClassMap::BatchClassId)
       releaseToOSMaybe(Sci, ClassId);
   }
 
   void disable() NO_THREAD_SAFETY_ANALYSIS {
     // The BatchClassId must be locked last since other classes can use it.
     for (sptr I = static_cast<sptr>(NumClasses) - 1; I >= 0; I--) {
       if (static_cast<uptr>(I) == SizeClassMap::BatchClassId)
void getStats(ScopedString *Str) {
     // TODO(kostyak): get the RSS per region.
     uptr TotalMapped = 0;
     uptr PoppedBlocks = 0;
     uptr PushedBlocks = 0;
     for (uptr I = 0; I < NumClasses; I++) {
       SizeClassInfo *Sci = getSizeClassInfo(I);
       ScopedLock L(Sci->Mutex);
       TotalMapped += Sci->AllocatedUser;
-      PoppedBlocks += Sci->Stats.PoppedBlocks;
-      PushedBlocks += Sci->Stats.PushedBlocks;
+      PoppedBlocks += Sci->FreeListInfo.PoppedBlocks;
+      PushedBlocks += Sci->FreeListInfo.PushedBlocks;
     }
     Str->append("Stats: SizeClassAllocator32: %zuM mapped in %zu allocations; "
                 "remains %zu\n",
                 TotalMapped >> 20, PoppedBlocks, PoppedBlocks - PushedBlocks);
     for (uptr I = 0; I < NumClasses; I++) {
       SizeClassInfo *Sci = getSizeClassInfo(I);
       ScopedLock L(Sci->Mutex);
       getStats(Str, I, Sci, 0);

 private:
   static const uptr NumClasses = SizeClassMap::NumClasses;
   static const uptr RegionSize = 1UL << Config::Primary::RegionSizeLog;
   static const uptr NumRegions =
       SCUDO_MMAP_RANGE_SIZE >> Config::Primary::RegionSizeLog;
   static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U;
   typedef FlatByteMap<NumRegions> ByteMap;
 
-  struct SizeClassStats {
-    uptr PoppedBlocks;
-    uptr PushedBlocks;
-  };
-
   struct ReleaseToOsInfo {
     uptr BytesInFreeListAtLastCheckpoint;
     uptr RangesReleased;
     uptr LastReleasedBytes;
     u64 LastReleaseAtNs;
   };
 
+  struct BlocksInfo {
+    SinglyLinkedList<BatchGroup> BlockList = {};
+    uptr PoppedBlocks = 0;
+    uptr PushedBlocks = 0;
+  };
+
   struct alignas(SCUDO_CACHE_LINE_SIZE) SizeClassInfo {
     HybridMutex Mutex;
-    SinglyLinkedList<BatchGroup> FreeList GUARDED_BY(Mutex);
+    BlocksInfo FreeListInfo GUARDED_BY(Mutex);
     uptr CurrentRegion GUARDED_BY(Mutex);
     uptr CurrentRegionAllocated GUARDED_BY(Mutex);
-    SizeClassStats Stats GUARDED_BY(Mutex);
     u32 RandState;
     uptr AllocatedUser GUARDED_BY(Mutex);
     // Lowest & highest region index allocated for this size class, to avoid
     // looping through the whole NumRegions.
     uptr MinRegionIndex GUARDED_BY(Mutex);
     uptr MaxRegionIndex GUARDED_BY(Mutex);
     ReleaseToOsInfo ReleaseInfo GUARDED_BY(Mutex);
   };
private:
 
   SizeClassInfo *getSizeClassInfo(uptr ClassId) {
     DCHECK_LT(ClassId, NumClasses);
     return &SizeClassInfoArray[ClassId];
   }
 
   // Push the blocks to their batch group. The layout will be like,
   //
-  // FreeList - > BG -> BG -> BG
-  //              |     |     |
-  //              v     v     v
-  //              TB    TB    TB
-  //              |
-  //              v
-  //              TB
+  // FreeListInfo.BlockList - > BG -> BG -> BG
+  //                            |     |     |
+  //                            v     v     v
+  //                            TB    TB    TB
+  //                            |
+  //                            v
+  //                            TB
   //
   // Each BlockGroup(BG) will associate with unique group id and the free blocks
   // are managed by a list of TransferBatch(TB). To reduce the time of inserting
   // blocks, BGs are sorted and the input `Array` are supposed to be sorted so
   // that we can get better performance of maintaining sorted property.
   // Use `SameGroup=true` to indicate that all blocks in the array are from the
   // same group then we will skip checking the group id of each block.
   //
auto InsertBlocks = [&](BatchGroup *BG, CompactPtrT *Array, u32 Size) {
         u16 AppendSize = static_cast<u16>(Min<u32>(UnusedSlots, Size - I));
         CurBatch->appendFromArray(&Array[I], AppendSize);
         I += AppendSize;
       }
 
       BG->PushedBlocks += Size;
     };
 
-    BatchGroup *Cur = Sci->FreeList.front();
+    BatchGroup *Cur = Sci->FreeListInfo.BlockList.front();
 
     if (ClassId == SizeClassMap::BatchClassId) {
       if (Cur == nullptr) {
         // Don't need to classify BatchClassId.
         Cur = CreateGroup(/*CompactPtrGroupBase=*/0);
-        Sci->FreeList.push_front(Cur);
+        Sci->FreeListInfo.BlockList.push_front(Cur);
       }
       InsertBlocks(Cur, Array, Size);
       return;
     }
 
     // In the following, `Cur` always points to the BatchGroup for blocks that
     // will be pushed next. `Prev` is the element right before `Cur`.
     BatchGroup *Prev = nullptr;
 
     while (Cur != nullptr &&
            compactPtrGroupBase(Array[0]) > Cur->CompactPtrGroupBase) {
       Prev = Cur;
       Cur = Cur->Next;
     }
 
     if (Cur == nullptr ||
         compactPtrGroupBase(Array[0]) != Cur->CompactPtrGroupBase) {
       Cur = CreateGroup(compactPtrGroupBase(Array[0]));
       if (Prev == nullptr)
-        Sci->FreeList.push_front(Cur);
+        Sci->FreeListInfo.BlockList.push_front(Cur);
       else
-        Sci->FreeList.insert(Prev, Cur);
+        Sci->FreeListInfo.BlockList.insert(Prev, Cur);
     }
 
     // All the blocks are from the same group, just push without checking group
     // id.
     if (SameGroup) {
       for (u32 I = 0; I < Size; ++I)
         DCHECK_EQ(compactPtrGroupBase(Array[I]), Cur->CompactPtrGroupBase);
 
for (u32 I = 1; I < Size; ++I) {
           Prev = Cur;
           Cur = Cur->Next;
         }
 
         if (Cur == nullptr ||
             compactPtrGroupBase(Array[I]) != Cur->CompactPtrGroupBase) {
           Cur = CreateGroup(compactPtrGroupBase(Array[I]));
           DCHECK_NE(Prev, nullptr);
-          Sci->FreeList.insert(Prev, Cur);
+          Sci->FreeListInfo.BlockList.insert(Prev, Cur);
         }
 
         Count = 1;
       } else {
         ++Count;
       }
     }
 
     InsertBlocks(Cur, Array + Size - Count, Count);
   }
 
   // Pop one TransferBatch from a BatchGroup. The BatchGroup with the smallest
   // group id will be considered first.
   //
   // The region mutex needs to be held while calling this method.
   TransferBatch *popBatchImpl(CacheT *C, uptr ClassId, SizeClassInfo *Sci)
       REQUIRES(Sci->Mutex) {
-    if (Sci->FreeList.empty())
+    if (Sci->FreeListInfo.BlockList.empty())
       return nullptr;
 
-    SinglyLinkedList<TransferBatch> &Batches = Sci->FreeList.front()->Batches;
+    SinglyLinkedList<TransferBatch> &Batches =
+        Sci->FreeListInfo.BlockList.front()->Batches;
     DCHECK(!Batches.empty());
 
     TransferBatch *B = Batches.front();
     Batches.pop_front();
     DCHECK_NE(B, nullptr);
     DCHECK_GT(B->getCount(), 0U);
 
     if (Batches.empty()) {
-      BatchGroup *BG = Sci->FreeList.front();
-      Sci->FreeList.pop_front();
+      BatchGroup *BG = Sci->FreeListInfo.BlockList.front();
+      Sci->FreeListInfo.BlockList.pop_front();
 
       // We don't keep BatchGroup with zero blocks to avoid empty-checking while
       // allocating. Note that block used by constructing BatchGroup is recorded
       // as free blocks in the last element of BatchGroup::Batches. Which means,
       // once we pop the last TransferBatch, the block is implicitly
       // deallocated.
       if (ClassId != SizeClassMap::BatchClassId)
         C->deallocate(SizeClassMap::BatchClassId, BG);
NOINLINE bool populateFreeList(CacheT *C, uptr ClassId, SizeClassInfo *Sci)
 
     return true;
   }
 
   void getStats(ScopedString *Str, uptr ClassId, SizeClassInfo *Sci, uptr Rss)
       REQUIRES(Sci->Mutex) {
     if (Sci->AllocatedUser == 0)
       return;
-    const uptr InUse = Sci->Stats.PoppedBlocks - Sci->Stats.PushedBlocks;
+    const uptr InUse =
+        Sci->FreeListInfo.PoppedBlocks - Sci->FreeListInfo.PushedBlocks;
     const uptr AvailableChunks = Sci->AllocatedUser / getSizeByClassId(ClassId);
     Str->append("  %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu "
                 "inuse: %6zu avail: %6zu rss: %6zuK releases: %6zu\n",
                 ClassId, getSizeByClassId(ClassId), Sci->AllocatedUser >> 10,
-                Sci->Stats.PoppedBlocks, Sci->Stats.PushedBlocks, InUse,
-                AvailableChunks, Rss >> 10, Sci->ReleaseInfo.RangesReleased);
+                Sci->FreeListInfo.PoppedBlocks, Sci->FreeListInfo.PushedBlocks,
+                InUse, AvailableChunks, Rss >> 10,
+                Sci->ReleaseInfo.RangesReleased);
   }
 
   NOINLINE uptr releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId,
                                  ReleaseToOS ReleaseType = ReleaseToOS::Normal)
       REQUIRES(Sci->Mutex) {
     const uptr BlockSize = getSizeByClassId(ClassId);
     const uptr PageSize = getPageSizeCached();
 
-    DCHECK_GE(Sci->Stats.PoppedBlocks, Sci->Stats.PushedBlocks);
+    DCHECK_GE(Sci->FreeListInfo.PoppedBlocks, Sci->FreeListInfo.PushedBlocks);
     const uptr BytesInFreeList =
         Sci->AllocatedUser -
-        (Sci->Stats.PoppedBlocks - Sci->Stats.PushedBlocks) * BlockSize;
+        (Sci->FreeListInfo.PoppedBlocks - Sci->FreeListInfo.PushedBlocks) *
+            BlockSize;
 
     if (UNLIKELY(BytesInFreeList == 0))
       return 0;
 
     if (BytesInFreeList <= Sci->ReleaseInfo.BytesInFreeListAtLastCheckpoint)
       Sci->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList;
 
     // Always update `BytesInFreeListAtLastCheckpoint` with the smallest value
NOINLINE uptr releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId,
 
     ReleaseRecorder Recorder(Base);
     PageReleaseContext Context(BlockSize, NumberOfRegions,
                                /*ReleaseSize=*/RegionSize);
 
     auto DecompactPtr = [](CompactPtrT CompactPtr) {
       return reinterpret_cast<uptr>(CompactPtr);
     };
-    for (BatchGroup &BG : Sci->FreeList) {
+    for (BatchGroup &BG : Sci->FreeListInfo.BlockList) {
       const uptr GroupBase = decompactGroupBase(BG.CompactPtrGroupBase);
       // The `GroupSize` may not be divided by `BlockSize`, which means there is
       // an unused space at the end of Region. Exclude that space to avoid
       // unused page map entry.
       uptr AllocatedGroupSize = GroupBase == CurGroupBase
                                     ? Sci->CurrentRegionAllocated
                                     : roundDownSlow(GroupSize, BlockSize);
       if (AllocatedGroupSize == 0)

compiler-rt/lib/scudo/standalone/primary64.h

public:
   TransferBatch *popBatch(CacheT *C, uptr ClassId) {
     DCHECK_LT(ClassId, NumClasses);
     RegionInfo *Region = getRegionInfo(ClassId);
     bool PrintStats = false;
     {
       ScopedLock L(Region->Mutex);
       TransferBatch *B = popBatchImpl(C, ClassId, Region);
       if (LIKELY(B)) {
-        Region->Stats.PoppedBlocks += B->getCount();
+        Region->FreeListInfo.PoppedBlocks += B->getCount();
         return B;
       }
 
       const bool RegionIsExhausted = Region->Exhausted;
       if (UNLIKELY(RegionIsExhausted ||
                    !populateFreeList(C, ClassId, Region))) {
         PrintStats = !RegionIsExhausted && Region->Exhausted;
       } else {
         B = popBatchImpl(C, ClassId, Region);
         // if `populateFreeList` succeeded, we are supposed to get free blocks.
         DCHECK_NE(B, nullptr);
-        Region->Stats.PoppedBlocks += B->getCount();
+        Region->FreeListInfo.PoppedBlocks += B->getCount();
         return B;
       }
     }
 
     // Note that `getStats()` requires locking each region so we can't call it
     // while locking the Region->Mutex in the above.
     if (UNLIKELY(PrintStats)) {
       ScopedString Str;
if (ClassId == SizeClassMap::BatchClassId) {
       {
         ScopedLock L(Region->Mutex);
         // Constructing a batch group in the free list will use two blocks in
         // BatchClassId. If we are pushing BatchClassId blocks, we will use the
         // blocks in the array directly (can't delegate local cache which will
         // cause a recursive allocation). However, The number of free blocks may
         // be less than two. Therefore, populate the free list before inserting
         // the blocks.
-        const bool NeedToRefill = Size == 1U && Region->FreeList.empty();
+        const bool NeedToRefill =
+            Size == 1U && Region->FreeListInfo.BlockList.empty();
         // If BatchClass has been exhausted, the program should have been
         // aborted.
         DCHECK(!Region->Exhausted);
 
         if (UNLIKELY(
                 NeedToRefill &&
                 !populateFreeList(C, SizeClassMap::BatchClassId, Region))) {
           PrintStats = true;
         } else {
           pushBlocksImpl(C, SizeClassMap::BatchClassId, Region, Array, Size);
-          Region->Stats.PushedBlocks += Size;
+          Region->FreeListInfo.PushedBlocks += Size;
         }
       }
 
       // Note that `getStats()` requires the lock of each region so we can't
       // call it while locking the Region->Mutex in the above.
       if (UNLIKELY(PrintStats)) {
         ScopedString Str;
         getStats(&Str);
for (u32 I = 1; I < Size; ++I) {
         --J;
       }
       Array[J] = Cur;
     }
 
     ScopedLock L(Region->Mutex);
     pushBlocksImpl(C, ClassId, Region, Array, Size, SameGroup);
 
-    Region->Stats.PushedBlocks += Size;
+    Region->FreeListInfo.PushedBlocks += Size;
     if (ClassId != SizeClassMap::BatchClassId)
       releaseToOSMaybe(Region, ClassId);
   }
 
   void disable() NO_THREAD_SAFETY_ANALYSIS {
     // The BatchClassId must be locked last since other classes can use it.
     for (sptr I = static_cast<sptr>(NumClasses) - 1; I >= 0; I--) {
       if (static_cast<uptr>(I) == SizeClassMap::BatchClassId)
void getStats(ScopedString *Str) {
     uptr TotalMapped = 0;
     uptr PoppedBlocks = 0;
     uptr PushedBlocks = 0;
     for (uptr I = 0; I < NumClasses; I++) {
       RegionInfo *Region = getRegionInfo(I);
       ScopedLock L(Region->Mutex);
       if (Region->MappedUser)
         TotalMapped += Region->MappedUser;
-      PoppedBlocks += Region->Stats.PoppedBlocks;
-      PushedBlocks += Region->Stats.PushedBlocks;
+      PoppedBlocks += Region->FreeListInfo.PoppedBlocks;
+      PushedBlocks += Region->FreeListInfo.PushedBlocks;
     }
     Str->append("Stats: SizeClassAllocator64: %zuM mapped (%uM rss) in %zu "
                 "allocations; remains %zu\n",
                 TotalMapped >> 20, 0U, PoppedBlocks,
                 PoppedBlocks - PushedBlocks);
 
     for (uptr I = 0; I < NumClasses; I++) {
       RegionInfo *Region = getRegionInfo(I);
private:
   static const uptr RegionSize = 1UL << Config::Primary::RegionSizeLog;
   static const uptr NumClasses = SizeClassMap::NumClasses;
   static const uptr PrimarySize = RegionSize * NumClasses;
 
   static const uptr MapSizeIncrement = Config::Primary::MapSizeIncrement;
   // Fill at most this number of batches from the newly map'd memory.
   static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U;
 
-  struct RegionStats {
-    uptr PoppedBlocks;
-    uptr PushedBlocks;
-  };
-
   struct ReleaseToOsInfo {
     uptr BytesInFreeListAtLastCheckpoint;
     uptr RangesReleased;
     uptr LastReleasedBytes;
     u64 LastReleaseAtNs;
   };
 
+  struct BlocksInfo {
+    SinglyLinkedList<BatchGroup> BlockList = {};
+    uptr PoppedBlocks = 0;
+    uptr PushedBlocks = 0;
+  };
+
   struct UnpaddedRegionInfo {
     HybridMutex Mutex;
-    SinglyLinkedList<BatchGroup> FreeList GUARDED_BY(Mutex);
+    BlocksInfo FreeListInfo GUARDED_BY(Mutex);
     // This is initialized before thread creation.
     uptr RegionBeg = 0;
-    RegionStats Stats GUARDED_BY(Mutex) = {};
     u32 RandState GUARDED_BY(Mutex) = 0;
     // Bytes mapped for user memory.
     uptr MappedUser GUARDED_BY(Mutex) = 0;
     // Bytes allocated for user memory.
     uptr AllocatedUser GUARDED_BY(Mutex) = 0;
     // The minimum size of pushed blocks to trigger page release.
     uptr TryReleaseThreshold GUARDED_BY(Mutex) = 0;
     MemMapT MemMap = {};
private:
 
   ALWAYS_INLINE static bool isLargeBlock(uptr BlockSize) {
     const uptr PageSize = getPageSizeCached();
     return BlockSize > PageSize;
   }
 
   // Push the blocks to their batch group. The layout will be like,
   //
-  // FreeList - > BG -> BG -> BG
-  //              |     |     |
-  //              v     v     v
-  //              TB    TB    TB
-  //              |
-  //              v
-  //              TB
+  // FreeListInfo.BlockList - > BG -> BG -> BG
+  //                            |     |     |
+  //                            v     v     v
+  //                            TB    TB    TB
+  //                            |
+  //                            v
+  //                            TB
   //
   // Each BlockGroup(BG) will associate with unique group id and the free blocks
   // are managed by a list of TransferBatch(TB). To reduce the time of inserting
   // blocks, BGs are sorted and the input `Array` are supposed to be sorted so
   // that we can get better performance of maintaining sorted property.
   // Use `SameGroup=true` to indicate that all blocks in the array are from the
   // same group then we will skip checking the group id of each block.
   //
auto InsertBlocks = [&](BatchGroup *BG, CompactPtrT *Array, u32 Size) {
         u16 AppendSize = static_cast<u16>(Min<u32>(UnusedSlots, Size - I));
         CurBatch->appendFromArray(&Array[I], AppendSize);
         I += AppendSize;
       }
 
       BG->PushedBlocks += Size;
     };
 
-    BatchGroup *Cur = Region->FreeList.front();
+    BatchGroup *Cur = Region->FreeListInfo.BlockList.front();
 
     if (ClassId == SizeClassMap::BatchClassId) {
       if (Cur == nullptr) {
         // Don't need to classify BatchClassId.
         Cur = CreateGroup(/*CompactPtrGroupBase=*/0);
-        Region->FreeList.push_front(Cur);
+        Region->FreeListInfo.BlockList.push_front(Cur);
       }
       InsertBlocks(Cur, Array, Size);
       return;
     }
 
     // In the following, `Cur` always points to the BatchGroup for blocks that
     // will be pushed next. `Prev` is the element right before `Cur`.
     BatchGroup *Prev = nullptr;
 
     while (Cur != nullptr &&
            compactPtrGroup(Array[0]) > Cur->CompactPtrGroupBase) {
       Prev = Cur;
       Cur = Cur->Next;
     }
 
     if (Cur == nullptr ||
         compactPtrGroup(Array[0]) != Cur->CompactPtrGroupBase) {
       Cur = CreateGroup(compactPtrGroup(Array[0]));
       if (Prev == nullptr)
-        Region->FreeList.push_front(Cur);
+        Region->FreeListInfo.BlockList.push_front(Cur);
       else
-        Region->FreeList.insert(Prev, Cur);
+        Region->FreeListInfo.BlockList.insert(Prev, Cur);
     }
 
     // All the blocks are from the same group, just push without checking group
     // id.
     if (SameGroup) {
       for (u32 I = 0; I < Size; ++I)
         DCHECK_EQ(compactPtrGroup(Array[I]), Cur->CompactPtrGroupBase);
 
for (u32 I = 1; I < Size; ++I) {
           Prev = Cur;
           Cur = Cur->Next;
         }
 
         if (Cur == nullptr ||
             compactPtrGroup(Array[I]) != Cur->CompactPtrGroupBase) {
           Cur = CreateGroup(compactPtrGroup(Array[I]));
           DCHECK_NE(Prev, nullptr);
-          Region->FreeList.insert(Prev, Cur);
+          Region->FreeListInfo.BlockList.insert(Prev, Cur);
         }
 
         Count = 1;
       } else {
         ++Count;
       }
     }
 
     InsertBlocks(Cur, Array + Size - Count, Count);
   }
 
   // Pop one TransferBatch from a BatchGroup. The BatchGroup with the smallest
   // group id will be considered first.
   //
   // The region mutex needs to be held while calling this method.
   TransferBatch *popBatchImpl(CacheT *C, uptr ClassId, RegionInfo *Region)
       REQUIRES(Region->Mutex) {
-    if (Region->FreeList.empty())
+    if (Region->FreeListInfo.BlockList.empty())
       return nullptr;
 
     SinglyLinkedList<TransferBatch> &Batches =
-        Region->FreeList.front()->Batches;
+        Region->FreeListInfo.BlockList.front()->Batches;
     DCHECK(!Batches.empty());
 
     TransferBatch *B = Batches.front();
     Batches.pop_front();
     DCHECK_NE(B, nullptr);
     DCHECK_GT(B->getCount(), 0U);
 
     if (Batches.empty()) {
-      BatchGroup *BG = Region->FreeList.front();
-      Region->FreeList.pop_front();
+      BatchGroup *BG = Region->FreeListInfo.BlockList.front();
+      Region->FreeListInfo.BlockList.pop_front();
 
       // We don't keep BatchGroup with zero blocks to avoid empty-checking while
       // allocating. Note that block used by constructing BatchGroup is recorded
       // as free blocks in the last element of BatchGroup::Batches. Which means,
       // once we pop the last TransferBatch, the block is implicitly
       // deallocated.
       if (ClassId != SizeClassMap::BatchClassId)
         C->deallocate(SizeClassMap::BatchClassId, BG);
NOINLINE bool populateFreeList(CacheT *C, uptr ClassId, RegionInfo *Region)
 
     return true;
   }
 
   void getStats(ScopedString *Str, uptr ClassId, RegionInfo *Region, uptr Rss)
       REQUIRES(Region->Mutex) {
     if (Region->MappedUser == 0)
       return;
-    const uptr InUse = Region->Stats.PoppedBlocks - Region->Stats.PushedBlocks;
+    const uptr InUse =
+        Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks;
     const uptr TotalChunks = Region->AllocatedUser / getSizeByClassId(ClassId);
     Str->append("%s %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu "
                 "inuse: %6zu total: %6zu rss: %6zuK releases: %6zu last "
                 "released: %6zuK region: 0x%zx (0x%zx)\n",
                 Region->Exhausted ? "F" : " ", ClassId,
                 getSizeByClassId(ClassId), Region->MappedUser >> 10,
-                Region->Stats.PoppedBlocks, Region->Stats.PushedBlocks, InUse,
-                TotalChunks, Rss >> 10, Region->ReleaseInfo.RangesReleased,
+                Region->FreeListInfo.PoppedBlocks,
+                Region->FreeListInfo.PushedBlocks, InUse, TotalChunks,
+                Rss >> 10, Region->ReleaseInfo.RangesReleased,
                 Region->ReleaseInfo.LastReleasedBytes >> 10, Region->RegionBeg,
                 getRegionBaseByClassId(ClassId));
   }
 
   NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
                                  ReleaseToOS ReleaseType = ReleaseToOS::Normal)
       REQUIRES(Region->Mutex) {
     const uptr BlockSize = getSizeByClassId(ClassId);
     const uptr PageSize = getPageSizeCached();
 
-    DCHECK_GE(Region->Stats.PoppedBlocks, Region->Stats.PushedBlocks);
+    DCHECK_GE(Region->FreeListInfo.PoppedBlocks,
+              Region->FreeListInfo.PushedBlocks);
     const uptr BytesInFreeList =
-        Region->AllocatedUser -
-        (Region->Stats.PoppedBlocks - Region->Stats.PushedBlocks) * BlockSize;
+        Region->AllocatedUser - (Region->FreeListInfo.PoppedBlocks -
+                                 Region->FreeListInfo.PushedBlocks) *
+                                    BlockSize;
 
     if (UNLIKELY(BytesInFreeList == 0))
       return 0;
 
     // Always update `BytesInFreeListAtLastCheckpoint` with the smallest value
     // so that we won't underestimate the releasable pages. For example, the
     // following is the region usage,
     //
NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
     // The first visit is to determine the range of BatchGroups we are going to
     // release. And we will extract those BatchGroups out and push into
     // `GroupToRelease`.
     SinglyLinkedList<BatchGroup> GroupToRelease;
     GroupToRelease.clear();
 
     // This is only used for debugging to ensure the consistency of the number
     // of groups.
-    uptr NumberOfBatchGroups = Region->FreeList.size();
+    uptr NumberOfBatchGroups = Region->FreeListInfo.BlockList.size();
 
     // We are examining each group and will take the minimum distance to the
     // release threshold as the next Region::TryReleaseThreshold(). Note that if
     // the size of free blocks has reached the release threshold, the distance
     // to the next release will be PageSize * SmallerBlockReleasePageDelta. See
     // the comment on `SmallerBlockReleasePageDelta` for more details.
     uptr MinDistToThreshold = GroupSize;
 
-    for (BatchGroup *BG = Region->FreeList.front(), *Prev = nullptr;
+    for (BatchGroup *BG = Region->FreeListInfo.BlockList.front(),
+                    *Prev = nullptr;
          BG != nullptr;) {
       // Group boundary is always GroupSize-aligned from CompactPtr base. The
       // layout of memory groups is like,
       //
       //     (CompactPtrBase)
       // #1 CompactPtrGroupBase   #2 CompactPtrGroupBase            ...
       //           |                       |                       |
       //           v                       v                       v
for (BatchGroup *BG = Region->FreeListInfo.BlockList.front(),
         if (!HighDensity || !ReachReleaseDelta) {
           Prev = BG;
           BG = BG->Next;
           continue;
         }
       }
 
       // If `BG` is the first BatchGroup in the list, we only need to advance
-      // `BG` and call FreeList::pop_front(). No update is needed for `Prev`.
+      // `BG` and call FreeListInfo.BlockList::pop_front(). No update is needed
+      // for `Prev`.
       //
       //         (BG)   (BG->Next)
       // Prev     Cur      BG
       //   |       |       |
       //   v       v       v
       //  nil     +--+    +--+
       //          |X | -> |  | -> ...
       //          +--+    +--+
       //
       // Otherwise, `Prev` will be used to extract the `Cur` from the
-      // `FreeList`.
+      // `FreeListInfo.BlockList`.
       //
       //         (BG)   (BG->Next)
       // Prev     Cur      BG
       //   |       |       |
       //   v       v       v
       //  +--+    +--+    +--+
       //  |  | -> |X | -> |  | -> ...
       //  +--+    +--+    +--+
       //
-      // After FreeList::extract(),
+      // After FreeListInfo.BlockList::extract(),
       //
       // Prev     Cur       BG
       //   |       |        |
       //   v       v        v
       //  +--+    +--+     +--+
       //  |  |-+  |X |  +->|  | -> ...
       //  +--+ |  +--+  |  +--+
       //       +--------+
       //
       // Note that we need to advance before pushing this BatchGroup to
       // GroupToRelease because it's a destructive operation.
 
       BatchGroup *Cur = BG;
       BG = BG->Next;
 
       // Ideally, we may want to update this only after successful release.
       // However, for smaller blocks, each block marking is a costly operation.
       // Therefore, we update it earlier.
       // TODO: Consider updating this after page release if `ReleaseRecorder`
       // can tell the releasd bytes in each group.
       Cur->BytesInBGAtLastCheckpoint = BytesInBG;
 
       if (Prev != nullptr)
-        Region->FreeList.extract(Prev, Cur);
+        Region->FreeListInfo.BlockList.extract(Prev, Cur);
       else
-        Region->FreeList.pop_front();
+        Region->FreeListInfo.BlockList.pop_front();
       GroupToRelease.push_back(Cur);
     }
 
     // Only small blocks have the adaptive `TryReleaseThreshold`.
     if (isSmallBlock(BlockSize)) {
       // If the MinDistToThreshold is not updated, that means each memory group
       // may have only pushed less than a page size. In that case, just set it
       // back to normal.
NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
 
     if (Recorder.getReleasedRangesCount() > 0) {
       Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint = BytesInFreeList;
       Region->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount();
       Region->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes();
     }
     Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTimeFast();
 
-    // Merge GroupToRelease back to the Region::FreeList. Note that both
-    // `Region->FreeList` and `GroupToRelease` are sorted.
-    for (BatchGroup *BG = Region->FreeList.front(), *Prev = nullptr;;) {
+    // Merge GroupToRelease back to the Region::FreeListInfo.BlockList. Note
+    // that both `Region->FreeListInfo.BlockList` and `GroupToRelease` are
+    // sorted.
+    for (BatchGroup *BG = Region->FreeListInfo.BlockList.front(),
+                    *Prev = nullptr;
+         ;) {
       if (BG == nullptr || GroupToRelease.empty()) {
         if (!GroupToRelease.empty())
-          Region->FreeList.append_back(&GroupToRelease);
+          Region->FreeListInfo.BlockList.append_back(&GroupToRelease);
         break;
       }
 
       DCHECK_NE(BG->CompactPtrGroupBase,
                 GroupToRelease.front()->CompactPtrGroupBase);
 
       if (BG->CompactPtrGroupBase <
           GroupToRelease.front()->CompactPtrGroupBase) {
         Prev = BG;
         BG = BG->Next;
         continue;
       }
 
       // At here, the `BG` is the first BatchGroup with CompactPtrGroupBase
       // larger than the first element in `GroupToRelease`. We need to insert
       // `GroupToRelease::front()` (which is `Cur` below)  before `BG`.
       //
       //   1. If `Prev` is nullptr, we simply push `Cur` to the front of
-      //      FreeList.
+      //      FreeListInfo.BlockList.
       //   2. Otherwise, use `insert()` which inserts an element next to `Prev`.
       //
       // Afterwards, we don't need to advance `BG` because the order between
       // `BG` and the new `GroupToRelease::front()` hasn't been checked.
       BatchGroup *Cur = GroupToRelease.front();
       GroupToRelease.pop_front();
       if (Prev == nullptr)
-        Region->FreeList.push_front(Cur);
+        Region->FreeListInfo.BlockList.push_front(Cur);
       else
-        Region->FreeList.insert(Prev, Cur);
+        Region->FreeListInfo.BlockList.insert(Prev, Cur);
       DCHECK_EQ(Cur->Next, BG);
       Prev = Cur;
     }
 
-    DCHECK_EQ(Region->FreeList.size(), NumberOfBatchGroups);
+    DCHECK_EQ(Region->FreeListInfo.BlockList.size(), NumberOfBatchGroups);
     (void)NumberOfBatchGroups;
 
     if (SCUDO_DEBUG) {
-      BatchGroup *Prev = Region->FreeList.front();
+      BatchGroup *Prev = Region->FreeListInfo.BlockList.front();
       for (BatchGroup *Cur = Prev->Next; Cur != nullptr;
            Prev = Cur, Cur = Cur->Next) {
         CHECK_LT(Prev->CompactPtrGroupBase, Cur->CompactPtrGroupBase);
       }
     }
 
     return Recorder.getReleasedBytes();
   }
 };
 
 } // namespace scudo
 
 #endif // SCUDO_PRIMARY64_H_