diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -745,9 +745,9 @@ Str.output(); } - void releaseToOS() { + void releaseToOS(ReleaseToOS ReleaseType) { initThreadMaybe(); - Primary.releaseToOS(); + Primary.releaseToOS(ReleaseType); Secondary.releaseToOS(); } diff --git a/compiler-rt/lib/scudo/standalone/common.h b/compiler-rt/lib/scudo/standalone/common.h --- a/compiler-rt/lib/scudo/standalone/common.h +++ b/compiler-rt/lib/scudo/standalone/common.h @@ -215,6 +215,13 @@ MaxTSDsCount, // Number of usable TSDs for the shared registry. }; +enum class ReleaseToOS : u8 { + Normal, // Follow the normal rules for releasing pages to the OS + Force, // Force release pages to the OS, but avoid cases that take too long. + ForceAll, // Force release every page possible regardless of how long it will + // take. +}; + constexpr unsigned char PatternFillByte = 0xAB; enum FillContentsMode { diff --git a/compiler-rt/lib/scudo/standalone/include/scudo/interface.h b/compiler-rt/lib/scudo/standalone/include/scudo/interface.h --- a/compiler-rt/lib/scudo/standalone/include/scudo/interface.h +++ b/compiler-rt/lib/scudo/standalone/include/scudo/interface.h @@ -118,6 +118,10 @@ #define M_PURGE -101 #endif +#ifndef M_PURGE_ALL +#define M_PURGE_ALL -104 +#endif + // Tune the allocator's choice of memory tags to make it more likely that // a certain class of memory errors will be detected. The value argument should // be one of the M_MEMTAG_TUNING_* constants below. diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -281,14 +281,14 @@ return true; } - uptr releaseToOS() { + uptr releaseToOS(ReleaseToOS ReleaseType) { uptr TotalReleasedBytes = 0; for (uptr I = 0; I < NumClasses; I++) { if (I == SizeClassMap::BatchClassId) continue; SizeClassInfo *Sci = getSizeClassInfo(I); ScopedLock L(Sci->Mutex); - TotalReleasedBytes += releaseToOSMaybe(Sci, I, /*Force=*/true); + TotalReleasedBytes += releaseToOSMaybe(Sci, I, ReleaseType); } return TotalReleasedBytes; } @@ -727,7 +727,8 @@ } NOINLINE uptr releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId, - bool Force = false) REQUIRES(Sci->Mutex) { + ReleaseToOS ReleaseType = ReleaseToOS::Normal) + REQUIRES(Sci->Mutex) { const uptr BlockSize = getSizeByClassId(ClassId); const uptr PageSize = getPageSizeCached(); @@ -743,16 +744,18 @@ if (BytesPushed < PageSize) return 0; // Nothing new to release. - const bool CheckDensity = BlockSize < PageSize / 16U; + const bool CheckDensity = + BlockSize < PageSize / 16U && ReleaseType != ReleaseToOS::ForceAll; // Releasing smaller blocks is expensive, so we want to make sure that a // significant amount of bytes are free, and that there has been a good // amount of batches pushed to the freelist before attempting to release. if (CheckDensity) { - if (!Force && BytesPushed < Sci->AllocatedUser / 16U) + if (ReleaseType == ReleaseToOS::Normal && + BytesPushed < Sci->AllocatedUser / 16U) return 0; } - if (!Force) { + if (ReleaseType == ReleaseToOS::Normal) { const s32 IntervalMs = atomic_load_relaxed(&ReleaseToOsIntervalMs); if (IntervalMs < 0) return 0; diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -321,14 +321,14 @@ return true; } - uptr releaseToOS() { + uptr releaseToOS(ReleaseToOS ReleaseType) { uptr TotalReleasedBytes = 0; for (uptr I = 0; I < NumClasses; I++) { if (I == SizeClassMap::BatchClassId) continue; RegionInfo *Region = getRegionInfo(I); ScopedLock L(Region->Mutex); - TotalReleasedBytes += releaseToOSMaybe(Region, I, /*Force=*/true); + TotalReleasedBytes += releaseToOSMaybe(Region, I, ReleaseType); } return TotalReleasedBytes; } @@ -805,7 +805,8 @@ } NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId, - bool Force = false) REQUIRES(Region->Mutex) { + ReleaseToOS ReleaseType = ReleaseToOS::Normal) + REQUIRES(Region->Mutex) { const uptr BlockSize = getSizeByClassId(ClassId); const uptr PageSize = getPageSizeCached(); @@ -821,16 +822,18 @@ if (BytesPushed < PageSize) return 0; // Nothing new to release. - const bool CheckDensity = BlockSize < PageSize / 16U; + const bool CheckDensity = + BlockSize < PageSize / 16U && ReleaseType != ReleaseToOS::ForceAll; // Releasing smaller blocks is expensive, so we want to make sure that a // significant amount of bytes are free, and that there has been a good // amount of batches pushed to the freelist before attempting to release. if (CheckDensity) { - if (!Force && BytesPushed < Region->AllocatedUser / 16U) + if (ReleaseType == ReleaseToOS::Normal && + BytesPushed < Region->AllocatedUser / 16U) return 0; } - if (!Force) { + if (ReleaseType == ReleaseToOS::Normal) { const s32 IntervalMs = atomic_load_relaxed(&ReleaseToOsIntervalMs); if (IntervalMs < 0) return 0; diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -92,7 +92,7 @@ Allocator = std::make_unique(); } ~ScudoCombinedTest() { - Allocator->releaseToOS(); + Allocator->releaseToOS(scudo::ReleaseToOS::Force); UseQuarantine = true; } @@ -412,7 +412,7 @@ reinterpret_cast(P)[2048] = 0xaa; Allocator->deallocate(P, Origin); - Allocator->releaseToOS(); + Allocator->releaseToOS(scudo::ReleaseToOS::Force); } } @@ -488,7 +488,7 @@ } for (auto &T : Threads) T.join(); - Allocator->releaseToOS(); + Allocator->releaseToOS(scudo::ReleaseToOS::Force); } // Test that multiple instantiations of the allocator have not messed up the @@ -601,7 +601,7 @@ // operation without issue. SCUDO_TYPED_TEST(ScudoCombinedTest, ReleaseToOS) { auto *Allocator = this->Allocator.get(); - Allocator->releaseToOS(); + Allocator->releaseToOS(scudo::ReleaseToOS::Force); } SCUDO_TYPED_TEST(ScudoCombinedTest, OddEven) { @@ -740,7 +740,7 @@ auto *TSD = Allocator->getTSDRegistry()->getTSDAndLock(&UnlockRequired); TSD->getCache().drain(); - Allocator->releaseToOS(); + Allocator->releaseToOS(scudo::ReleaseToOS::Force); } #endif diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp @@ -161,7 +161,7 @@ Cache.deallocate(ClassId, Pointers[J]); } Cache.destroy(nullptr); - Allocator->releaseToOS(); + Allocator->releaseToOS(scudo::ReleaseToOS::Force); scudo::ScopedString Str; Allocator->getStats(&Str); Str.output(); @@ -215,7 +215,7 @@ Cache.deallocate(Primary::SizeClassMap::BatchClassId, B); } Cache.destroy(nullptr); - Allocator.releaseToOS(); + Allocator.releaseToOS(scudo::ReleaseToOS::Force); scudo::ScopedString Str; Allocator.getStats(&Str); Str.output(); @@ -253,7 +253,7 @@ V.pop_back(); } Cache.destroy(nullptr); - Allocator->releaseToOS(); + Allocator->releaseToOS(scudo::ReleaseToOS::Force); scudo::ScopedString Str; Allocator->getStats(&Str); Str.output(); @@ -300,7 +300,7 @@ } for (auto &T : Threads) T.join(); - Allocator->releaseToOS(); + Allocator->releaseToOS(scudo::ReleaseToOS::Force); scudo::ScopedString Str; Allocator->getStats(&Str); Str.output(); @@ -322,7 +322,7 @@ EXPECT_NE(P, nullptr); Cache.deallocate(ClassId, P); Cache.destroy(nullptr); - EXPECT_GT(Allocator->releaseToOS(), 0U); + EXPECT_GT(Allocator->releaseToOS(scudo::ReleaseToOS::Force), 0U); } SCUDO_TYPED_TEST(ScudoPrimaryTest, MemoryGroup) { diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c.inc b/compiler-rt/lib/scudo/standalone/wrappers_c.inc --- a/compiler-rt/lib/scudo/standalone/wrappers_c.inc +++ b/compiler-rt/lib/scudo/standalone/wrappers_c.inc @@ -188,7 +188,10 @@ static_cast(value)); return 1; } else if (param == M_PURGE) { - SCUDO_ALLOCATOR.releaseToOS(); + SCUDO_ALLOCATOR.releaseToOS(scudo::ReleaseToOS::Force); + return 1; + } else if (param == M_PURGE_ALL) { + SCUDO_ALLOCATOR.releaseToOS(scudo::ReleaseToOS::ForceAll); return 1; } else { scudo::Option option;