[scudo] Add a method to force release everything.

[cferris]

Summary

The force flag to releaseToOSMaybe does not release everything
since it is an expensive operation. Modify the release flag to
have three states: normal, force, forceall. Force behaves the same
as setting Force to true from before this change. Forceall will
release everything regardless of how much time it takes, or
how much there is to release.

In addition, add a new mallopt that will call the release function
with the forceall flag set.

[Chia-hungDuan]

LGTM!

compiler-rt/lib/scudo/standalone/primary32.h

753–754

nit: I prefer having the brackets here because the predicate has multiple lines

BTW, it seems that I missed this before. I'm thinking if we should just delete the check here. The logic here is done while examining each memory group.

If we still want a high level rule, I would suggest to update the heuristic here. So far, I prefer to remove it but not a strong preference

[cferris]

compiler-rt/lib/scudo/standalone/primary32.h

753–754

I will remove this in a follow-on CL since I would prefer not to do this all in one.

[Chia-hungDuan]

compiler-rt/lib/scudo/standalone/primary32.h

753–754

Agree!

[cferris]

clang-format a few lines.

compiler-rt/lib/scudo/standalone/combined.h

#endif
   }
 
   void printStats() {
     ScopedString Str;
     getStats(&Str);
     Str.output();
   }
 
-  void releaseToOS() {
+  void releaseToOS(ReleaseToOS ReleaseType) {
     initThreadMaybe();
-    Primary.releaseToOS();
+    Primary.releaseToOS(ReleaseType);
     Secondary.releaseToOS();
   }
 
   // Iterate over all chunks and call a callback for all busy chunks located
   // within the provided memory range. Said callback must not use this allocator
   // or a deadlock can ensue. This fits Android's malloc_iterate() needs.
   void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback,
                          void *Arg) {

compiler-rt/lib/scudo/standalone/common.h

enum class Option : u8 {
   MemtagTuning,         // Whether to tune tagging for UAF or overflow.
   ThreadDisableMemInit, // Whether to disable automatic heap initialization and,
                         // where possible, memory tagging, on this thread.
   MaxCacheEntriesCount, // Maximum number of blocks that can be cached.
   MaxCacheEntrySize,    // Maximum size of a block that can be cached.
   MaxTSDsCount,         // Number of usable TSDs for the shared registry.
 };
 
+enum class ReleaseToOS : u8 {
+  Normal, // Follow the normal rules for releasing pages to the OS
+  Force,  // Force release pages to the OS, but avoid cases that take too long.
+  ForceAll, // Force release every page possible regardless of how long it will
+            // take.
+};
+
 constexpr unsigned char PatternFillByte = 0xAB;
 
 enum FillContentsMode {
   NoFill = 0,
   ZeroFill = 1,
   PatternOrZeroFill = 2 // Pattern fill unless the memory is known to be
                         // zero-initialized already.
 };
 
 } // namespace scudo
 
 #endif // SCUDO_COMMON_H_

compiler-rt/lib/scudo/standalone/include/scudo/interface.h

 #ifndef M_DECAY_TIME
 #define M_DECAY_TIME -100
 #endif
 
 #ifndef M_PURGE
 #define M_PURGE -101
 #endif
 
+#ifndef M_PURGE_ALL
+#define M_PURGE_ALL -104
+#endif
+
 // Tune the allocator's choice of memory tags to make it more likely that
 // a certain class of memory errors will be detected. The value argument should
 // be one of the M_MEMTAG_TUNING_* constants below.
 #ifndef M_MEMTAG_TUNING
 #define M_MEMTAG_TUNING -102
 #endif
 
 // Per-thread memory initialization tuning. The value argument should be one of:

compiler-rt/lib/scudo/standalone/primary32.h

if (O == Option::ReleaseInterval) {
           Config::PrimaryMinReleaseToOsIntervalMs);
       atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval);
       return true;
     }
     // Not supported by the Primary, but not an error either.
     return true;
   }
 
-  uptr releaseToOS() {
+  uptr releaseToOS(ReleaseToOS ReleaseType) {
     uptr TotalReleasedBytes = 0;
     for (uptr I = 0; I < NumClasses; I++) {
       if (I == SizeClassMap::BatchClassId)
         continue;
       SizeClassInfo *Sci = getSizeClassInfo(I);
       ScopedLock L(Sci->Mutex);
-      TotalReleasedBytes += releaseToOSMaybe(Sci, I, /*Force=*/true);
+      TotalReleasedBytes += releaseToOSMaybe(Sci, I, ReleaseType);
     }
     return TotalReleasedBytes;
   }
 
   const char *getRegionInfoArrayAddress() const { return nullptr; }
   static uptr getRegionInfoArraySize() { return 0; }
 
   static BlockInfo findNearestBlock(UNUSED const char *RegionInfoData,
void getStats(ScopedString *Str, uptr ClassId, SizeClassInfo *Sci, uptr Rss)
     Str->append("  %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu "
                 "inuse: %6zu avail: %6zu rss: %6zuK releases: %6zu\n",
                 ClassId, getSizeByClassId(ClassId), Sci->AllocatedUser >> 10,
                 Sci->Stats.PoppedBlocks, Sci->Stats.PushedBlocks, InUse,
                 AvailableChunks, Rss >> 10, Sci->ReleaseInfo.RangesReleased);
   }
 
   NOINLINE uptr releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId,
-                                 bool Force = false) REQUIRES(Sci->Mutex) {
+                                 ReleaseToOS ReleaseType = ReleaseToOS::Normal)
+      REQUIRES(Sci->Mutex) {
     const uptr BlockSize = getSizeByClassId(ClassId);
     const uptr PageSize = getPageSizeCached();
 
     DCHECK_GE(Sci->Stats.PoppedBlocks, Sci->Stats.PushedBlocks);
     const uptr BytesInFreeList =
         Sci->AllocatedUser -
         (Sci->Stats.PoppedBlocks - Sci->Stats.PushedBlocks) * BlockSize;
     if (BytesInFreeList < PageSize)
       return 0; // No chance to release anything.
     const uptr BytesPushed =
         (Sci->Stats.PushedBlocks - Sci->ReleaseInfo.PushedBlocksAtLastRelease) *
         BlockSize;
     if (BytesPushed < PageSize)
       return 0; // Nothing new to release.
 
-    const bool CheckDensity = BlockSize < PageSize / 16U;
+    const bool CheckDensity =
+        BlockSize < PageSize / 16U && ReleaseType != ReleaseToOS::ForceAll;
     // Releasing smaller blocks is expensive, so we want to make sure that a
     // significant amount of bytes are free, and that there has been a good
     // amount of batches pushed to the freelist before attempting to release.
     if (CheckDensity) {
-      if (!Force && BytesPushed < Sci->AllocatedUser / 16U)
+      if (ReleaseType == ReleaseToOS::Normal &&
+          BytesPushed < Sci->AllocatedUser / 16U)
         return 0;
     }
 
-    if (!Force) {
+    if (ReleaseType == ReleaseToOS::Normal) {
       const s32 IntervalMs = atomic_load_relaxed(&ReleaseToOsIntervalMs);
       if (IntervalMs < 0)
         return 0;
       if (Sci->ReleaseInfo.LastReleaseAtNs +
               static_cast<u64>(IntervalMs) * 1000000 >
           getMonotonicTimeFast()) {
         return 0; // Memory was returned recently.
       }

compiler-rt/lib/scudo/standalone/primary64.h

if (O == Option::ReleaseInterval) {
           Config::PrimaryMinReleaseToOsIntervalMs);
       atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval);
       return true;
     }
     // Not supported by the Primary, but not an error either.
     return true;
   }
 
-  uptr releaseToOS() {
+  uptr releaseToOS(ReleaseToOS ReleaseType) {
     uptr TotalReleasedBytes = 0;
     for (uptr I = 0; I < NumClasses; I++) {
       if (I == SizeClassMap::BatchClassId)
         continue;
       RegionInfo *Region = getRegionInfo(I);
       ScopedLock L(Region->Mutex);
-      TotalReleasedBytes += releaseToOSMaybe(Region, I, /*Force=*/true);
+      TotalReleasedBytes += releaseToOSMaybe(Region, I, ReleaseType);
     }
     return TotalReleasedBytes;
   }
 
   const char *getRegionInfoArrayAddress() const {
     return reinterpret_cast<const char *>(RegionInfoArray);
   }
 
Str->append("%s %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu "
                 getSizeByClassId(ClassId), Region->MappedUser >> 10,
                 Region->Stats.PoppedBlocks, Region->Stats.PushedBlocks, InUse,
                 TotalChunks, Rss >> 10, Region->ReleaseInfo.RangesReleased,
                 Region->ReleaseInfo.LastReleasedBytes >> 10, Region->RegionBeg,
                 getRegionBaseByClassId(ClassId));
   }
 
   NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
-                                 bool Force = false) REQUIRES(Region->Mutex) {
+                                 ReleaseToOS ReleaseType = ReleaseToOS::Normal)
+      REQUIRES(Region->Mutex) {
     const uptr BlockSize = getSizeByClassId(ClassId);
     const uptr PageSize = getPageSizeCached();
 
     DCHECK_GE(Region->Stats.PoppedBlocks, Region->Stats.PushedBlocks);
     const uptr BytesInFreeList =
         Region->AllocatedUser -
         (Region->Stats.PoppedBlocks - Region->Stats.PushedBlocks) * BlockSize;
     if (BytesInFreeList < PageSize)
       return 0; // No chance to release anything.
     const uptr BytesPushed = (Region->Stats.PushedBlocks -
                               Region->ReleaseInfo.PushedBlocksAtLastRelease) *
                              BlockSize;
     if (BytesPushed < PageSize)
       return 0; // Nothing new to release.
 
-    const bool CheckDensity = BlockSize < PageSize / 16U;
+    const bool CheckDensity =
+        BlockSize < PageSize / 16U && ReleaseType != ReleaseToOS::ForceAll;
     // Releasing smaller blocks is expensive, so we want to make sure that a
     // significant amount of bytes are free, and that there has been a good
     // amount of batches pushed to the freelist before attempting to release.
     if (CheckDensity) {
-      if (!Force && BytesPushed < Region->AllocatedUser / 16U)
+      if (ReleaseType == ReleaseToOS::Normal &&
+          BytesPushed < Region->AllocatedUser / 16U)
         return 0;
     }
 
-    if (!Force) {
+    if (ReleaseType == ReleaseToOS::Normal) {
       const s32 IntervalMs = atomic_load_relaxed(&ReleaseToOsIntervalMs);
       if (IntervalMs < 0)
         return 0;
       if (Region->ReleaseInfo.LastReleaseAtNs +
               static_cast<u64>(IntervalMs) * 1000000 >
           getMonotonicTimeFast()) {
         return 0; // Memory was returned recently.
       }

compiler-rt/lib/scudo/standalone/tests/combined_test.cpp

 };
 
 template <class TypeParam> struct ScudoCombinedTest : public Test {
   ScudoCombinedTest() {
     UseQuarantine = std::is_same<TypeParam, scudo::AndroidConfig>::value;
     Allocator = std::make_unique<AllocatorT>();
   }
   ~ScudoCombinedTest() {
-    Allocator->releaseToOS();
+    Allocator->releaseToOS(scudo::ReleaseToOS::Force);
     UseQuarantine = true;
   }
 
   void RunTest();
 
   void BasicTest(scudo::uptr SizeLog);
 
   using AllocatorT = TestAllocator<TypeParam>;
if (Allocator->useMemoryTaggingTestOnly()) {
     reinterpret_cast<char *>(P)[2048] = 0xaa;
     Allocator->deallocate(P, Origin);
 
     P = Allocator->allocate(2048, Origin);
     EXPECT_EQ(scudo::untagPointer(P), P);
     reinterpret_cast<char *>(P)[2048] = 0xaa;
     Allocator->deallocate(P, Origin);
 
-    Allocator->releaseToOS();
+    Allocator->releaseToOS(scudo::ReleaseToOS::Force);
   }
 }
 
 SCUDO_TYPED_TEST(ScudoCombinedTest, Stats) {
   auto *Allocator = this->Allocator.get();
 
   scudo::uptr BufferSize = 8192;
   std::vector<char> Buffer(BufferSize);
for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
     });
   {
     std::unique_lock<std::mutex> Lock(Mutex);
     Ready = true;
     Cv.notify_all();
   }
   for (auto &T : Threads)
     T.join();
-  Allocator->releaseToOS();
+  Allocator->releaseToOS(scudo::ReleaseToOS::Force);
 }
 
 // Test that multiple instantiations of the allocator have not messed up the
 // process's signal handlers (GWP-ASan used to do this).
 TEST(ScudoCombinedDeathTest, SKIP_ON_FUCHSIA(testSEGV)) {
   const scudo::uptr Size = 4 * scudo::getPageSizeCached();
   scudo::MapPlatformData Data = {};
   void *P = scudo::map(nullptr, Size, "testSEGV", MAP_NOACCESS, &Data);
TEST(ScudoCombinedTest, FullRegion) {
   }
   EXPECT_EQ(FailedAllocationsCount, 0U);
 }
 
 // Ensure that releaseToOS can be called prior to any other allocator
 // operation without issue.
 SCUDO_TYPED_TEST(ScudoCombinedTest, ReleaseToOS) {
   auto *Allocator = this->Allocator.get();
-  Allocator->releaseToOS();
+  Allocator->releaseToOS(scudo::ReleaseToOS::Force);
 }
 
 SCUDO_TYPED_TEST(ScudoCombinedTest, OddEven) {
   auto *Allocator = this->Allocator.get();
 
   if (!Allocator->useMemoryTaggingTestOnly())
     return;
 
for (scudo::uptr ClassId = 1U;
     ASSERT_NE(p, nullptr);
     free(p);
   }
 
   bool UnlockRequired;
   auto *TSD = Allocator->getTSDRegistry()->getTSDAndLock(&UnlockRequired);
   TSD->getCache().drain();
 
-  Allocator->releaseToOS();
+  Allocator->releaseToOS(scudo::ReleaseToOS::Force);
 }
 
 #endif
 #endif
 
 #if SCUDO_LINUX
 
 SCUDO_TYPED_TEST(ScudoCombinedTest, SoftRssLimit) {

compiler-rt/lib/scudo/standalone/tests/primary_test.cpp

for (scudo::uptr J = 0; J < NumberOfAllocations; J++) {
       void *P = Cache.allocate(ClassId);
       memset(P, 'B', Size);
       Pointers[J] = P;
     }
     for (scudo::uptr J = 0; J < NumberOfAllocations; J++)
       Cache.deallocate(ClassId, Pointers[J]);
   }
   Cache.destroy(nullptr);
-  Allocator->releaseToOS();
+  Allocator->releaseToOS(scudo::ReleaseToOS::Force);
   scudo::ScopedString Str;
   Allocator->getStats(&Str);
   Str.output();
 }
 
 struct SmallRegionsConfig {
   using SizeClassMap = scudo::DefaultSizeClassMap;
   static const scudo::uptr PrimaryRegionSizeLog = 21U;
TEST(ScudoPrimaryTest, Primary64OOM) {
   while (!Batches.empty()) {
     TransferBatch *B = Batches.back();
     Batches.pop_back();
     B->copyToArray(Blocks);
     Allocator.pushBlocks(&Cache, ClassId, Blocks, B->getCount());
     Cache.deallocate(Primary::SizeClassMap::BatchClassId, B);
   }
   Cache.destroy(nullptr);
-  Allocator.releaseToOS();
+  Allocator.releaseToOS(scudo::ReleaseToOS::Force);
   scudo::ScopedString Str;
   Allocator.getStats(&Str);
   Str.output();
   EXPECT_EQ(AllocationFailed, true);
   Allocator.unmapTestOnly();
 }
 
 SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryIterate) {
SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryIterate) {
   Allocator->enable();
   EXPECT_EQ(Found, V.size());
   while (!V.empty()) {
     auto Pair = V.back();
     Cache.deallocate(Pair.first, Pair.second);
     V.pop_back();
   }
   Cache.destroy(nullptr);
-  Allocator->releaseToOS();
+  Allocator->releaseToOS(scudo::ReleaseToOS::Force);
   scudo::ScopedString Str;
   Allocator->getStats(&Str);
   Str.output();
 }
 
 SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryThreaded) {
   using Primary = TestAllocator<TypeParam, scudo::SvelteSizeClassMap>;
   std::unique_ptr<Primary> Allocator(new Primary);
for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
     });
   {
     std::unique_lock<std::mutex> Lock(Mutex);
     Ready = true;
     Cv.notify_all();
   }
   for (auto &T : Threads)
     T.join();
-  Allocator->releaseToOS();
+  Allocator->releaseToOS(scudo::ReleaseToOS::Force);
   scudo::ScopedString Str;
   Allocator->getStats(&Str);
   Str.output();
 }
 
 // Through a simple allocation that spans two pages, verify that releaseToOS
 // actually releases some bytes (at least one page worth). This is a regression
 // test for an error in how the release criteria were computed.
 SCUDO_TYPED_TEST(ScudoPrimaryTest, ReleaseToOS) {
   using Primary = TestAllocator<TypeParam, scudo::DefaultSizeClassMap>;
   std::unique_ptr<Primary> Allocator(new Primary);
   Allocator->init(/*ReleaseToOsInterval=*/-1);
   typename Primary::CacheT Cache;
   Cache.init(nullptr, Allocator.get());
   const scudo::uptr Size = scudo::getPageSizeCached() * 2;
   EXPECT_TRUE(Primary::canAllocate(Size));
   const scudo::uptr ClassId = Primary::SizeClassMap::getClassIdBySize(Size);
   void *P = Cache.allocate(ClassId);
   EXPECT_NE(P, nullptr);
   Cache.deallocate(ClassId, P);
   Cache.destroy(nullptr);
-  EXPECT_GT(Allocator->releaseToOS(), 0U);
+  EXPECT_GT(Allocator->releaseToOS(scudo::ReleaseToOS::Force), 0U);
 }
 
 SCUDO_TYPED_TEST(ScudoPrimaryTest, MemoryGroup) {
   using Primary = TestAllocator<TypeParam, scudo::DefaultSizeClassMap>;
   std::unique_ptr<Primary> Allocator(new Primary);
   Allocator->init(/*ReleaseToOsInterval=*/-1);
   typename Primary::CacheT Cache;
   Cache.init(nullptr, Allocator.get());

compiler-rt/lib/scudo/standalone/wrappers_c.inc

if (SCUDO_ANDROID) {
         value = INT32_MAX;
       }
     }
 
     SCUDO_ALLOCATOR.setOption(scudo::Option::ReleaseInterval,
                               static_cast<scudo::sptr>(value));
     return 1;
   } else if (param == M_PURGE) {
-    SCUDO_ALLOCATOR.releaseToOS();
+    SCUDO_ALLOCATOR.releaseToOS(scudo::ReleaseToOS::Force);
+    return 1;
+  } else if (param == M_PURGE_ALL) {
+    SCUDO_ALLOCATOR.releaseToOS(scudo::ReleaseToOS::ForceAll);
     return 1;
   } else {
     scudo::Option option;
     switch (param) {
     case M_MEMTAG_TUNING:
       option = scudo::Option::MemtagTuning;
       break;
     case M_THREAD_DISABLE_MEM_INIT: