[scudo][standalone] Release secondary memory on purge

[cryptoad]

Summary

The Secondary's cache needs to be released when the Combined's
releaseToOS function is called (via M_PURGE for example),
which this CL adds.

Additionally, if doing a forced release, we'll release the
transfer batch class as well since now we can do that.

There is a couple of other house keeping changes as well:

• read the page size only once in the Secondary Cache retrieve
• remove the interval check for CanRelease: we are going to make that configurable via mallopt so this needs not be set in stone there.

[merge_guards_bot]

Unit tests: fail. 62352 tests passed, 1 failed and 839 were skipped.

failed: libc++.std/containers/sequences/array/array_creation/to_array.fail.cpp

clang-tidy: pass.

clang-format: pass.

Build artifacts: diff.json, clang-tidy.txt, clang-format.patch, CMakeCache.txt, console-log.txt, test-results.xml

Pre-merge checks is in beta. Report issue. Please join beta or enable it for your project.

[pcc]

LGTM

compiler-rt/lib/scudo/standalone/combined.h

void printStats() {
     getStats(&Str);
     enable();
     Str.output();
   }
 
   void releaseToOS() {
     initThreadMaybe();
     Primary.releaseToOS();
+    Secondary.releaseToOS();
   }
 
   // Iterate over all chunks and call a callback for all busy chunks located
   // within the provided memory range. Said callback must not use this allocator
   // or a deadlock can ensue. This fits Android's malloc_iterate() needs.
   void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback,
                          void *Arg) {
     initThreadMaybe();

compiler-rt/lib/scudo/standalone/primary32.h

if (UNLIKELY(!getRandom(reinterpret_cast<void *>(&Seed), sizeof(Seed))))
       Seed =
           static_cast<u32>(getMonotonicTime() ^
                            (reinterpret_cast<uptr>(SizeClassInfoArray) >> 6));
     const uptr PageSize = getPageSizeCached();
     for (uptr I = 0; I < NumClasses; I++) {
       SizeClassInfo *Sci = getSizeClassInfo(I);
       Sci->RandState = getRandomU32(&Seed);
       // See comment in the 64-bit primary about releasing smaller size classes.
-      Sci->CanRelease = (ReleaseToOsInterval >= 0) &&
-                        (I != SizeClassMap::BatchClassId) &&
+      Sci->CanRelease = (I != SizeClassMap::BatchClassId) &&
                         (getSizeByClassId(I) >= (PageSize / 32));
     }
     ReleaseToOsIntervalMs = ReleaseToOsInterval;
   }
   void init(s32 ReleaseToOsInterval) {
     memset(this, 0, sizeof(*this));
     initLinkerInitialized(ReleaseToOsInterval);
   }
Str->append("Stats: SizeClassAllocator32: %zuM mapped in %zu allocations; "
                 TotalMapped >> 20, PoppedBlocks, PoppedBlocks - PushedBlocks);
     for (uptr I = 0; I < NumClasses; I++)
       getStats(Str, I, 0);
   }
 
   uptr releaseToOS() {
     uptr TotalReleasedBytes = 0;
     for (uptr I = 0; I < NumClasses; I++) {
-      if (I == SizeClassMap::BatchClassId)
-        continue;
       SizeClassInfo *Sci = getSizeClassInfo(I);
       ScopedLock L(Sci->Mutex);
       TotalReleasedBytes += releaseToOSMaybe(Sci, I, /*Force=*/true);
     }
     return TotalReleasedBytes;
   }
 
   bool useMemoryTagging() { return false; }

compiler-rt/lib/scudo/standalone/primary64.h

for (uptr I = 0; I < NumClasses; I++) {
           getRegionBaseByClassId(I) + (getRandomModN(&Seed, 16) + 1) * PageSize;
       // Releasing smaller size classes doesn't necessarily yield to a
       // meaningful RSS impact: there are more blocks per page, they are
       // randomized around, and thus pages are less likely to be entirely empty.
       // On top of this, attempting to release those require more iterations and
       // memory accesses which ends up being fairly costly. The current lower
       // limit is mostly arbitrary and based on empirical observations.
       // TODO(kostyak): make the lower limit a runtime option
-      Region->CanRelease = (ReleaseToOsInterval >= 0) &&
-                           (I != SizeClassMap::BatchClassId) &&
+      Region->CanRelease = (I != SizeClassMap::BatchClassId) &&
                            (getSizeByClassId(I) >= (PageSize / 32));
       Region->RandState = getRandomU32(&Seed);
     }
     ReleaseToOsIntervalMs = ReleaseToOsInterval;
 
     if (SupportsMemoryTagging)
       UseMemoryTagging = systemSupportsMemoryTagging();
   }
void getStats(ScopedString *Str) const {
 
     for (uptr I = 0; I < NumClasses; I++)
       getStats(Str, I, 0);
   }
 
   uptr releaseToOS() {
     uptr TotalReleasedBytes = 0;
     for (uptr I = 0; I < NumClasses; I++) {
-      if (I == SizeClassMap::BatchClassId)
-        continue;
       RegionInfo *Region = getRegionInfo(I);
       ScopedLock L(Region->Mutex);
       TotalReleasedBytes += releaseToOSMaybe(Region, I, /*Force=*/true);
     }
     return TotalReleasedBytes;
   }
 
   bool useMemoryTagging() const {

compiler-rt/lib/scudo/standalone/secondary.h

public:
   void init(UNUSED s32 ReleaseToOsInterval) {}
   bool retrieve(UNUSED uptr Size, UNUSED LargeBlock::Header **H) {
     return false;
   }
   bool store(UNUSED LargeBlock::Header *H) { return false; }
   static bool canCache(UNUSED uptr Size) { return false; }
   void disable() {}
   void enable() {}
+  void releaseToOS() {}
 };
 
 template <uptr MaxEntriesCount = 32U, uptr MaxEntrySize = 1UL << 19>
 class MapAllocatorCache {
 public:
   // Fuchsia doesn't allow releasing Secondary blocks yet. Note that 0 length
   // arrays are an extension for some compilers.
   // FIXME(kostyak): support (partially) the cache on Fuchsia.
if (EmptyCache)
       empty();
     else if (ReleaseToOsIntervalMs >= 0)
       releaseOlderThan(Time -
                        static_cast<u64>(ReleaseToOsIntervalMs) * 1000000);
     return EntryCached;
   }
 
   bool retrieve(uptr Size, LargeBlock::Header **H) {
+    const uptr PageSize = getPageSizeCached();
     ScopedLock L(Mutex);
     if (EntriesCount == 0)
       return false;
     for (uptr I = 0; I < MaxEntriesCount; I++) {
       if (!Entries[I].Block)
         continue;
       const uptr BlockSize = Entries[I].BlockEnd - Entries[I].Block;
       if (Size > BlockSize)
         continue;
-      if (Size < BlockSize - getPageSizeCached() * 4U)
+      if (Size < BlockSize - PageSize * 4U)
         continue;
       *H = reinterpret_cast<LargeBlock::Header *>(Entries[I].Block);
       Entries[I].Block = 0;
       (*H)->BlockEnd = Entries[I].BlockEnd;
       (*H)->MapBase = Entries[I].MapBase;
       (*H)->MapSize = Entries[I].MapSize;
       EntriesCount--;
       return true;
     }
     return false;
   }
 
   static bool canCache(uptr Size) {
     return MaxEntriesCount != 0U && Size <= MaxEntrySize;
   }
 
+  void releaseToOS() { releaseOlderThan(UINT64_MAX); }
+
   void disable() { Mutex.lock(); }
 
   void enable() { Mutex.unlock(); }
 
 private:
   void empty() {
     struct {
       void *MapBase;
public:
 
   template <typename F> void iterateOverBlocks(F Callback) const {
     for (const auto &H : InUseBlocks)
       Callback(reinterpret_cast<uptr>(&H) + LargeBlock::getHeaderSize());
   }
 
   static uptr canCache(uptr Size) { return CacheT::canCache(Size); }
 
+  void releaseToOS() { Cache.releaseToOS(); }
+
 private:
   CacheT Cache;
 
   HybridMutex Mutex;
   DoublyLinkedList<LargeBlock::Header> InUseBlocks;
   uptr AllocatedBytes;
   uptr FreedBytes;
   uptr LargestSize;