[scudo][standalone] Force release after burst of deallocs (64-b only)

[cryptoad]

Summary

Some situations are fairly adversarial to Scudo, notably when there is a lot of allocations in a small class size followed by deallocations. Since we do not release those except on forced-release, we end up with a higher RSS that's not going away.

Since releasing smaller classes is expensive, I tried to find the most balanced way to proceed.
The point of this change is to force-release classes when the % of freed chunks in a class size is very high (in the 9x% area), and if the number of blocks pushed in the freelist since last release is also high (I set that to 25%).
A very small size class (16 bytes) won't release significant memory unless it's mostly empty (98-99%) due to randomization.

There is a small drawback when an application does an M_PURGE shortly after a burst a deallocation, then we end up going through the release code twice as we still want to fulfill the release request.

compiler-rt/lib/scudo/standalone/local_cache.h

private:
   NOINLINE bool refill(PerClass *C, uptr ClassId) {
     initCacheMaybe(C);
     TransferBatch *B = Allocator->popBatch(this, ClassId);
     if (UNLIKELY(!B))
       return false;
     DCHECK_GT(B->getCount(), 0);
     C->Count = B->getCount();
     B->copyToArray(C->Chunks);
+    B->clear();
     destroyBatch(ClassId, B);
     return true;
   }
 
   NOINLINE void drain(PerClass *C, uptr ClassId) {
     const u32 Count = Min(C->MaxCount / 2, C->Count);
     TransferBatch *B = createBatch(ClassId, C->Chunks[0]);
     if (UNLIKELY(!B))

compiler-rt/lib/scudo/standalone/primary64.h

for (uptr I = 0; I < NumClasses; I++) {
       Region->RandState = getRandomU32(&Seed);
       // Releasing smaller size classes doesn't necessarily yield to a
       // meaningful RSS impact: there are more blocks per page, they are
       // randomized around, and thus pages are less likely to be entirely empty.
       // On top of this, attempting to release those require more iterations and
       // memory accesses which ends up being fairly costly. The current lower
       // limit is mostly arbitrary and based on empirical observations.
       // TODO(kostyak): make the lower limit a runtime option
-      Region->CanRelease = (I != SizeClassMap::BatchClassId) &&
-                           (getSizeByClassId(I) >= (PageSize / 32));
-      if (Region->CanRelease)
+      Region->UseInterval = (getSizeByClassId(I) >= 128U);
+      if (Region->UseInterval)
         Region->ReleaseInfo.LastReleaseAtNs = Time;
     }
     setReleaseToOsIntervalMs(ReleaseToOsInterval);
 
     if (SupportsMemoryTagging)
       UseMemoryTagging = systemSupportsMemoryTagging();
   }
   void init(s32 ReleaseToOsInterval) {
public:
   }
 
   void pushBatch(uptr ClassId, TransferBatch *B) {
     DCHECK_GT(B->getCount(), 0);
     RegionInfo *Region = getRegionInfo(ClassId);
     ScopedLock L(Region->Mutex);
     Region->FreeList.push_front(B);
     Region->Stats.PushedBlocks += B->getCount();
-    if (Region->CanRelease)
+    if (ClassId != SizeClassMap::BatchClassId)
       releaseToOSMaybe(Region, ClassId);
   }
 
   void disable() {
     // The BatchClassId must be locked last since other classes can use it.
     for (sptr I = static_cast<sptr>(NumClasses) - 1; I >= 0; I--) {
       if (static_cast<uptr>(I) == SizeClassMap::BatchClassId)
         continue;
struct ReleaseToOsInfo {
     uptr LastReleasedBytes;
     u64 LastReleaseAtNs;
   };
 
   struct UnpaddedRegionInfo {
     HybridMutex Mutex;
     SinglyLinkedList<TransferBatch> FreeList;
     RegionStats Stats;
-    bool CanRelease;
+    bool UseInterval;
     bool Exhausted;
     u32 RandState;
     uptr RegionBeg;
     uptr MappedUser;    // Bytes mapped for user memory.
     uptr AllocatedUser; // Bytes allocated for user memory.
     MapPlatformData Data;
     ReleaseToOsInfo ReleaseInfo;
   };
NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
     if (BytesInFreeList < PageSize)
       return 0; // No chance to release anything.
     const uptr BytesPushed = (Region->Stats.PushedBlocks -
                               Region->ReleaseInfo.PushedBlocksAtLastRelease) *
                              BlockSize;
     if (BytesPushed < PageSize)
       return 0; // Nothing new to release.
 
+    // Force a release if the number of bytes pushed since the last release and
+    // the number of bytes in the freelist are above given percentages of the
+    // allocated bytes for that class size, which usually indicates a burst in
+    // deallocation.
+    if (!Force && BytesPushed > (Region->AllocatedUser / 4U) &&
+        (BytesInFreeList / (Region->AllocatedUser / 100U)) >
+            (100U - getMostSignificantSetBitIndex(BlockSize)))
+      Force = true;
+
     if (!Force) {
+      if (!Region->UseInterval)
+        return 0;
       const s32 IntervalMs = getReleaseToOsIntervalMs();
       if (IntervalMs < 0)
         return 0;
       if (Region->ReleaseInfo.LastReleaseAtNs +
               static_cast<u64>(IntervalMs) * 1000000 >
           getMonotonicTime()) {
         return 0; // Memory was returned recently.
       }
     }
 
     ReleaseRecorder Recorder(Region->RegionBeg, &Region->Data);
     releaseFreeMemoryToOS(Region->FreeList, Region->RegionBeg,
                           Region->AllocatedUser, BlockSize, &Recorder);
 
     if (Recorder.getReleasedRangesCount() > 0) {
-      Region->ReleaseInfo.PushedBlocksAtLastRelease =
-          Region->Stats.PushedBlocks;
       Region->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount();
       Region->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes();
     }
+    Region->ReleaseInfo.PushedBlocksAtLastRelease = Region->Stats.PushedBlocks;
+    if (Region->UseInterval)
       Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTime();
     return Recorder.getReleasedBytes();
   }
 };
 
 } // namespace scudo
 
 #endif // SCUDO_PRIMARY64_H_