diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -239,6 +239,7 @@ } TSDRegistryT *getTSDRegistry() { return &TSDRegistry; } + QuarantineT *getQuarantine() { return &Quarantine; } // The Cache must be provided zero-initialized. void initCache(CacheT *Cache) { Cache->init(&Stats, &Primary); } @@ -254,6 +255,13 @@ TSD->getCache().destroy(&Stats); } + void drainCache(TSD *TSD) { + Quarantine.drainAndRecycle(&TSD->getQuarantineCache(), + QuarantineCallback(*this, TSD->getCache())); + TSD->getCache().drain(); + } + void drainCaches() { TSDRegistry.drainCaches(this); } + ALWAYS_INLINE void *getHeaderTaggedPointer(void *Ptr) { if (!allocatorSupportsMemoryTagging()) return Ptr; @@ -747,6 +755,8 @@ void releaseToOS(ReleaseToOS ReleaseType) { initThreadMaybe(); + if (ReleaseType == ReleaseToOS::ForceAll) + drainCaches(); Primary.releaseToOS(ReleaseType); Secondary.releaseToOS(); } diff --git a/compiler-rt/lib/scudo/standalone/quarantine.h b/compiler-rt/lib/scudo/standalone/quarantine.h --- a/compiler-rt/lib/scudo/standalone/quarantine.h +++ b/compiler-rt/lib/scudo/standalone/quarantine.h @@ -192,6 +192,12 @@ uptr getMaxSize() const { return atomic_load_relaxed(&MaxSize); } uptr getCacheSize() const { return atomic_load_relaxed(&MaxCacheSize); } + // This is supposed to be used in test only. + bool isEmpty() { + ScopedLock L(CacheMutex); + return Cache.getSize() == 0U; + } + void put(CacheT *C, Callback Cb, Node *Ptr, uptr Size) { C->enqueue(Cb, Ptr, Size); if (C->getSize() > getCacheSize()) diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -457,6 +457,28 @@ TSD->unlock(); } +SCUDO_TYPED_TEST(ScudoCombinedTest, ForceCacheDrain) NO_THREAD_SAFETY_ANALYSIS { + auto *Allocator = this->Allocator.get(); + + std::vector V; + for (scudo::uptr I = 0; I < 64U; I++) + V.push_back(Allocator->allocate( + rand() % (TypeParam::Primary::SizeClassMap::MaxSize / 2U), Origin)); + for (auto P : V) + Allocator->deallocate(P, Origin); + + // `ForceAll` will also drain the caches. + Allocator->releaseToOS(scudo::ReleaseToOS::ForceAll); + + bool UnlockRequired; + auto *TSD = Allocator->getTSDRegistry()->getTSDAndLock(&UnlockRequired); + EXPECT_TRUE(TSD->getCache().isEmpty()); + EXPECT_EQ(TSD->getQuarantineCache().getSize(), 0U); + EXPECT_TRUE(Allocator->getQuarantine()->isEmpty()); + if (UnlockRequired) + TSD->unlock(); +} + SCUDO_TYPED_TEST(ScudoCombinedTest, ThreadedCombined) { std::mutex Mutex; std::condition_variable Cv; diff --git a/compiler-rt/lib/scudo/standalone/tsd_exclusive.h b/compiler-rt/lib/scudo/standalone/tsd_exclusive.h --- a/compiler-rt/lib/scudo/standalone/tsd_exclusive.h +++ b/compiler-rt/lib/scudo/standalone/tsd_exclusive.h @@ -59,6 +59,13 @@ Initialized = false; } + void drainCaches(Allocator *Instance) { + // We don't have a way to iterate all thread local `ThreadTSD`s. Simply + // drain the `ThreadTSD` of current thread and `FallbackTSD`. + Instance->drainCache(&ThreadTSD); + Instance->drainCache(&FallbackTSD); + } + ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, bool MinimalInit) { if (LIKELY(State.InitState != ThreadState::NotInitialized)) return; diff --git a/compiler-rt/lib/scudo/standalone/tsd_shared.h b/compiler-rt/lib/scudo/standalone/tsd_shared.h --- a/compiler-rt/lib/scudo/standalone/tsd_shared.h +++ b/compiler-rt/lib/scudo/standalone/tsd_shared.h @@ -54,6 +54,15 @@ Initialized = false; } + void drainCaches(Allocator *Instance) { + ScopedLock L(MutexTSDs); + for (uptr I = 0; I < NumberOfTSDs; ++I) { + TSDs[I].lock(); + Instance->drainCache(&TSDs[I]); + TSDs[I].unlock(); + } + } + ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, UNUSED bool MinimalInit) { if (LIKELY(getCurrentTSD()))