diff --git a/compiler-rt/lib/scudo/standalone/benchmarks/malloc_benchmark.cpp b/compiler-rt/lib/scudo/standalone/benchmarks/malloc_benchmark.cpp --- a/compiler-rt/lib/scudo/standalone/benchmarks/malloc_benchmark.cpp +++ b/compiler-rt/lib/scudo/standalone/benchmarks/malloc_benchmark.cpp @@ -29,7 +29,6 @@ std::unique_ptr Allocator(new AllocatorT, Deleter); CurrentAllocator = Allocator.get(); - Allocator->reset(); const size_t NBytes = State.range(0); size_t PageSize = scudo::getPageSizeCached(); @@ -70,7 +69,6 @@ std::unique_ptr Allocator(new AllocatorT, Deleter); CurrentAllocator = Allocator.get(); - Allocator->reset(); const size_t NumIters = State.range(0); size_t PageSize = scudo::getPageSizeCached(); diff --git a/compiler-rt/lib/scudo/standalone/bytemap.h b/compiler-rt/lib/scudo/standalone/bytemap.h --- a/compiler-rt/lib/scudo/standalone/bytemap.h +++ b/compiler-rt/lib/scudo/standalone/bytemap.h @@ -17,10 +17,9 @@ template class FlatByteMap { public: - void initLinkerInitialized() {} - void init() { memset(Map, 0, sizeof(Map)); } + void init() { DCHECK(Size == 0 || Map[0] == 0); } - void unmapTestOnly() {} + void unmapTestOnly() { memset(Map, 0, Size); } void set(uptr Index, u8 Value) { DCHECK_LT(Index, Size); @@ -36,7 +35,7 @@ void enable() {} private: - u8 Map[Size]; + u8 Map[Size] = {}; }; } // namespace scudo diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -132,7 +132,7 @@ typedef GlobalQuarantine QuarantineT; typedef typename QuarantineT::CacheT QuarantineCacheT; - void initLinkerInitialized() { + void init() { performSanityChecks(); // Check if hardware CRC32 is supported in the binary and by the platform, @@ -166,11 +166,10 @@ QuarantineMaxChunkSize = static_cast(getFlags()->quarantine_max_chunk_size); - Stats.initLinkerInitialized(); + Stats.init(); const s32 ReleaseToOsIntervalMs = getFlags()->release_to_os_interval_ms; - Primary.initLinkerInitialized(ReleaseToOsIntervalMs); - Secondary.initLinkerInitialized(&Stats, ReleaseToOsIntervalMs); - + Primary.init(ReleaseToOsIntervalMs); + Secondary.init(&Stats, ReleaseToOsIntervalMs); Quarantine.init( static_cast(getFlags()->quarantine_size_kb << 10), static_cast(getFlags()->thread_local_quarantine_size_kb << 10)); @@ -210,10 +209,8 @@ TSDRegistry.initThreadMaybe(this, MinimalInit); } - void reset() { memset(this, 0, sizeof(*this)); } - void unmapTestOnly() { - TSDRegistry.unmapTestOnly(); + TSDRegistry.unmapTestOnly(this); Primary.unmapTestOnly(); Secondary.unmapTestOnly(); #ifdef GWP_ASAN_HOOKS @@ -226,9 +223,7 @@ TSDRegistryT *getTSDRegistry() { return &TSDRegistry; } // The Cache must be provided zero-initialized. - void initCache(CacheT *Cache) { - Cache->initLinkerInitialized(&Stats, &Primary); - } + void initCache(CacheT *Cache) { Cache->init(&Stats, &Primary); } // Release the resources used by a TSD, which involves: // - draining the local quarantine cache to the global quarantine; diff --git a/compiler-rt/lib/scudo/standalone/local_cache.h b/compiler-rt/lib/scudo/standalone/local_cache.h --- a/compiler-rt/lib/scudo/standalone/local_cache.h +++ b/compiler-rt/lib/scudo/standalone/local_cache.h @@ -49,18 +49,14 @@ CompactPtrT Batch[MaxNumCached]; }; - void initLinkerInitialized(GlobalStats *S, SizeClassAllocator *A) { - Stats.initLinkerInitialized(); + void init(GlobalStats *S, SizeClassAllocator *A) { + DCHECK(isEmpty()); + Stats.init(); if (LIKELY(S)) S->link(&Stats); Allocator = A; } - void init(GlobalStats *S, SizeClassAllocator *A) { - memset(this, 0, sizeof(*this)); - initLinkerInitialized(S, A); - } - void destroy(GlobalStats *S) { drain(); if (LIKELY(S)) diff --git a/compiler-rt/lib/scudo/standalone/mutex.h b/compiler-rt/lib/scudo/standalone/mutex.h --- a/compiler-rt/lib/scudo/standalone/mutex.h +++ b/compiler-rt/lib/scudo/standalone/mutex.h @@ -22,7 +22,6 @@ class HybridMutex { public: - void init() { M = {}; } bool tryLock(); NOINLINE void lock() { if (LIKELY(tryLock())) diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -60,11 +60,11 @@ static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; } - void initLinkerInitialized(s32 ReleaseToOsInterval) { + void init(s32 ReleaseToOsInterval) { if (SCUDO_FUCHSIA) reportError("SizeClassAllocator32 is not supported on Fuchsia"); - PossibleRegions.initLinkerInitialized(); + PossibleRegions.init(); u32 Seed; const u64 Time = getMonotonicTime(); @@ -80,10 +80,6 @@ } setOption(Option::ReleaseInterval, static_cast(ReleaseToOsInterval)); } - void init(s32 ReleaseToOsInterval) { - memset(this, 0, sizeof(*this)); - initLinkerInitialized(ReleaseToOsInterval); - } void unmapTestOnly() { while (NumberOfStashedRegions > 0) @@ -96,6 +92,7 @@ MinRegionIndex = Sci->MinRegionIndex; if (Sci->MaxRegionIndex > MaxRegionIndex) MaxRegionIndex = Sci->MaxRegionIndex; + *Sci = {}; } for (uptr I = MinRegionIndex; I < MaxRegionIndex; I++) if (PossibleRegions[I]) diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -57,7 +57,8 @@ static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; } - void initLinkerInitialized(s32 ReleaseToOsInterval) { + void init(s32 ReleaseToOsInterval) { + DCHECK_EQ(PrimaryBase, 0U); // Reserve the space required for the Primary. PrimaryBase = reinterpret_cast( map(nullptr, PrimarySize, nullptr, MAP_NOACCESS, &Data)); @@ -77,13 +78,14 @@ } setOption(Option::ReleaseInterval, static_cast(ReleaseToOsInterval)); } - void init(s32 ReleaseToOsInterval) { - memset(this, 0, sizeof(*this)); - initLinkerInitialized(ReleaseToOsInterval); - } void unmapTestOnly() { + for (uptr I = 0; I < NumClasses; I++) { + RegionInfo *Region = getRegionInfo(I); + *Region = {}; + } unmap(reinterpret_cast(PrimaryBase), PrimarySize, UNMAP_ALL, &Data); + PrimaryBase = 0U; } TransferBatch *popBatch(CacheT *C, uptr ClassId) { diff --git a/compiler-rt/lib/scudo/standalone/quarantine.h b/compiler-rt/lib/scudo/standalone/quarantine.h --- a/compiler-rt/lib/scudo/standalone/quarantine.h +++ b/compiler-rt/lib/scudo/standalone/quarantine.h @@ -64,11 +64,7 @@ // Per-thread cache of memory blocks. template class QuarantineCache { public: - void initLinkerInitialized() {} - void init() { - memset(this, 0, sizeof(*this)); - initLinkerInitialized(); - } + void init() { DCHECK_EQ(atomic_load_relaxed(&Size), 0U); } // Total memory used, including internal accounting. uptr getSize() const { return atomic_load_relaxed(&Size); } @@ -175,7 +171,10 @@ public: typedef QuarantineCache CacheT; - void initLinkerInitialized(uptr Size, uptr CacheSize) { + void init(uptr Size, uptr CacheSize) { + DCHECK_EQ(atomic_load_relaxed(&MaxSize), 0U); + DCHECK_EQ(atomic_load_relaxed(&MinSize), 0U); + DCHECK_EQ(atomic_load_relaxed(&MaxCacheSize), 0U); // Thread local quarantine size can be zero only when global quarantine size // is zero (it allows us to perform just one atomic read per put() call). CHECK((Size == 0 && CacheSize == 0) || CacheSize != 0); @@ -184,16 +183,7 @@ atomic_store_relaxed(&MinSize, Size / 10 * 9); // 90% of max size. atomic_store_relaxed(&MaxCacheSize, CacheSize); - Cache.initLinkerInitialized(); - } - void init(uptr Size, uptr CacheSize) { - CacheMutex.init(); Cache.init(); - RecycleMutex.init(); - MinSize = {}; - MaxSize = {}; - MaxCacheSize = {}; - initLinkerInitialized(Size, CacheSize); } uptr getMaxSize() const { return atomic_load_relaxed(&MaxSize); } diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h --- a/compiler-rt/lib/scudo/standalone/secondary.h +++ b/compiler-rt/lib/scudo/standalone/secondary.h @@ -71,7 +71,6 @@ class MapAllocatorNoCache { public: - void initLinkerInitialized(UNUSED s32 ReleaseToOsInterval) {} void init(UNUSED s32 ReleaseToOsInterval) {} bool retrieve(UNUSED Options Options, UNUSED uptr Size, UNUSED uptr Alignment, UNUSED LargeBlock::Header **H, UNUSED bool *Zeroed) { @@ -121,17 +120,14 @@ Config::SecondaryCacheEntriesArraySize, ""); - void initLinkerInitialized(s32 ReleaseToOsInterval) { + void init(s32 ReleaseToOsInterval) { + DCHECK_EQ(EntriesCount, 0U); setOption(Option::MaxCacheEntriesCount, static_cast(Config::SecondaryCacheDefaultMaxEntriesCount)); setOption(Option::MaxCacheEntrySize, static_cast(Config::SecondaryCacheDefaultMaxEntrySize)); setOption(Option::ReleaseInterval, static_cast(ReleaseToOsInterval)); } - void init(s32 ReleaseToOsInterval) { - memset(this, 0, sizeof(*this)); - initLinkerInitialized(ReleaseToOsInterval); - } void store(Options Options, LargeBlock::Header *H) { if (!canCache(H->CommitSize)) @@ -404,16 +400,14 @@ template class MapAllocator { public: - void initLinkerInitialized(GlobalStats *S, s32 ReleaseToOsInterval = -1) { - Cache.initLinkerInitialized(ReleaseToOsInterval); - Stats.initLinkerInitialized(); + void init(GlobalStats *S, s32 ReleaseToOsInterval = -1) { + DCHECK_EQ(AllocatedBytes, 0U); + DCHECK_EQ(FreedBytes, 0U); + Cache.init(ReleaseToOsInterval); + Stats.init(); if (LIKELY(S)) S->link(&Stats); } - void init(GlobalStats *S, s32 ReleaseToOsInterval = -1) { - memset(this, 0, sizeof(*this)); - initLinkerInitialized(S, ReleaseToOsInterval); - } void *allocate(Options Options, uptr Size, uptr AlignmentHint = 0, uptr *BlockEnd = nullptr, diff --git a/compiler-rt/lib/scudo/standalone/stats.h b/compiler-rt/lib/scudo/standalone/stats.h --- a/compiler-rt/lib/scudo/standalone/stats.h +++ b/compiler-rt/lib/scudo/standalone/stats.h @@ -29,8 +29,10 @@ // LocalStats::add'ing, this is OK, we will still get a meaningful number. class LocalStats { public: - void initLinkerInitialized() {} - void init() { memset(this, 0, sizeof(*this)); } + void init() { + for (uptr I = 0; I < StatCount; I++) + DCHECK_EQ(get(static_cast(I)), 0U); + } void add(StatType I, uptr V) { V += atomic_load_relaxed(&StatsArray[I]); @@ -56,13 +58,7 @@ // Global stats, used for aggregation and querying. class GlobalStats : public LocalStats { public: - void initLinkerInitialized() {} - void init() { - LocalStats::init(); - Mutex.init(); - StatsList = {}; - initLinkerInitialized(); - } + void init() { LocalStats::init(); } void link(LocalStats *S) { ScopedLock L(Mutex); diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -68,7 +68,6 @@ template struct TestAllocator : scudo::Allocator { TestAllocator() { - this->reset(); this->initThreadMaybe(); if (scudo::archSupportsMemoryTagging() && !scudo::systemDetectsMemoryTagFaultsTestOnly()) diff --git a/compiler-rt/lib/scudo/standalone/tests/mutex_test.cpp b/compiler-rt/lib/scudo/standalone/tests/mutex_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/mutex_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/mutex_test.cpp @@ -82,7 +82,6 @@ TEST(ScudoMutexTest, Mutex) { scudo::HybridMutex M; - M.init(); TestData Data(M); pthread_t Threads[NumberOfThreads]; for (scudo::u32 I = 0; I < NumberOfThreads; I++) @@ -93,7 +92,6 @@ TEST(ScudoMutexTest, MutexTry) { scudo::HybridMutex M; - M.init(); TestData Data(M); pthread_t Threads[NumberOfThreads]; for (scudo::u32 I = 0; I < NumberOfThreads; I++) diff --git a/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp b/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/tsd_test.cpp @@ -26,15 +26,14 @@ using CacheT = struct MockCache { volatile scudo::uptr Canary; }; using QuarantineCacheT = struct MockQuarantine {}; - void initLinkerInitialized() { + void init() { // This should only be called once by the registry. EXPECT_FALSE(Initialized); Initialized = true; } - void reset() { memset(this, 0, sizeof(*this)); } - void unmapTestOnly() { TSDRegistry.unmapTestOnly(); } - void initCache(CacheT *Cache) { memset(Cache, 0, sizeof(*Cache)); } + void unmapTestOnly() { TSDRegistry.unmapTestOnly(this); } + void initCache(CacheT *Cache) { *Cache = {}; } void commitBack(scudo::TSD *TSD) {} TSDRegistryT *getTSDRegistry() { return &TSDRegistry; } void callPostInitCallback() {} @@ -42,7 +41,7 @@ bool isInitialized() { return Initialized; } private: - bool Initialized; + bool Initialized = false; TSDRegistryT TSDRegistry; }; @@ -69,11 +68,10 @@ }; std::unique_ptr Allocator(new AllocatorT, Deleter); - Allocator->reset(); EXPECT_FALSE(Allocator->isInitialized()); auto Registry = Allocator->getTSDRegistry(); - Registry->initLinkerInitialized(Allocator.get()); + Registry->init(Allocator.get()); EXPECT_TRUE(Allocator->isInitialized()); } @@ -84,7 +82,6 @@ }; std::unique_ptr Allocator(new AllocatorT, Deleter); - Allocator->reset(); EXPECT_FALSE(Allocator->isInitialized()); auto Registry = Allocator->getTSDRegistry(); @@ -153,7 +150,6 @@ }; std::unique_ptr Allocator(new AllocatorT, Deleter); - Allocator->reset(); std::thread Threads[32]; for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++) Threads[I] = std::thread(stressCache, Allocator.get()); @@ -209,7 +205,6 @@ }; std::unique_ptr Allocator(new AllocatorT, Deleter); - Allocator->reset(); // We attempt to use as many TSDs as the shared cache offers by creating a // decent amount of threads that will be run concurrently and attempt to get // and lock TSDs. We put them all in a set and count the number of entries diff --git a/compiler-rt/lib/scudo/standalone/tsd.h b/compiler-rt/lib/scudo/standalone/tsd.h --- a/compiler-rt/lib/scudo/standalone/tsd.h +++ b/compiler-rt/lib/scudo/standalone/tsd.h @@ -28,14 +28,11 @@ typename Allocator::QuarantineCacheT QuarantineCache; u8 DestructorIterations = 0; - void initLinkerInitialized(Allocator *Instance) { + void init(Allocator *Instance) { + DCHECK_EQ(DestructorIterations, 0U); Instance->initCache(&Cache); DestructorIterations = PTHREAD_DESTRUCTOR_ITERATIONS; } - void init(Allocator *Instance) { - memset(this, 0, sizeof(*this)); - initLinkerInitialized(Instance); - } void commitBack(Allocator *Instance) { Instance->commitBack(this); } diff --git a/compiler-rt/lib/scudo/standalone/tsd_exclusive.h b/compiler-rt/lib/scudo/standalone/tsd_exclusive.h --- a/compiler-rt/lib/scudo/standalone/tsd_exclusive.h +++ b/compiler-rt/lib/scudo/standalone/tsd_exclusive.h @@ -25,31 +25,35 @@ template void teardownThread(void *Ptr); template struct TSDRegistryExT { - void initLinkerInitialized(Allocator *Instance) { - Instance->initLinkerInitialized(); + void init(Allocator *Instance) { + DCHECK(!Initialized); + Instance->init(); CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread), 0); - FallbackTSD.initLinkerInitialized(Instance); + FallbackTSD.init(Instance); Initialized = true; } - void init(Allocator *Instance) { - memset(this, 0, sizeof(*this)); - initLinkerInitialized(Instance); - } void initOnceMaybe(Allocator *Instance) { ScopedLock L(Mutex); if (LIKELY(Initialized)) return; - initLinkerInitialized(Instance); // Sets Initialized. + init(Instance); // Sets Initialized. } - void unmapTestOnly() { - Allocator *Instance = - reinterpret_cast(pthread_getspecific(PThreadKey)); - if (!Instance) - return; - ThreadTSD.commitBack(Instance); + void unmapTestOnly(Allocator *Instance) { + DCHECK(Instance); + if (reinterpret_cast(pthread_getspecific(PThreadKey))) { + DCHECK_EQ(reinterpret_cast(pthread_getspecific(PThreadKey)), + Instance); + ThreadTSD.commitBack(Instance); + ThreadTSD = {}; + } + CHECK_EQ(pthread_key_delete(PThreadKey), 0); + PThreadKey = {}; + FallbackTSD.commitBack(Instance); + FallbackTSD = {}; State = {}; + Initialized = false; } ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, bool MinimalInit) { @@ -103,7 +107,7 @@ return; CHECK_EQ( pthread_setspecific(PThreadKey, reinterpret_cast(Instance)), 0); - ThreadTSD.initLinkerInitialized(Instance); + ThreadTSD.init(Instance); State.InitState = ThreadState::Initialized; Instance->callPostInitCallback(); } diff --git a/compiler-rt/lib/scudo/standalone/tsd_shared.h b/compiler-rt/lib/scudo/standalone/tsd_shared.h --- a/compiler-rt/lib/scudo/standalone/tsd_shared.h +++ b/compiler-rt/lib/scudo/standalone/tsd_shared.h @@ -24,28 +24,32 @@ template struct TSDRegistrySharedT { - void initLinkerInitialized(Allocator *Instance) { - Instance->initLinkerInitialized(); + void init(Allocator *Instance) { + DCHECK(!Initialized); + Instance->init(); for (u32 I = 0; I < TSDsArraySize; I++) - TSDs[I].initLinkerInitialized(Instance); + TSDs[I].init(Instance); const u32 NumberOfCPUs = getNumberOfCPUs(); setNumberOfTSDs((NumberOfCPUs == 0) ? DefaultTSDCount : Min(NumberOfCPUs, DefaultTSDCount)); Initialized = true; } - void init(Allocator *Instance) { - memset(this, 0, sizeof(*this)); - initLinkerInitialized(Instance); - } void initOnceMaybe(Allocator *Instance) { ScopedLock L(Mutex); if (LIKELY(Initialized)) return; - initLinkerInitialized(Instance); // Sets Initialized. + init(Instance); // Sets Initialized. } - void unmapTestOnly() { setCurrentTSD(nullptr); } + void unmapTestOnly(Allocator *Instance) { + for (u32 I = 0; I < TSDsArraySize; I++) { + TSDs[I].commitBack(Instance); + TSDs[I] = {}; + } + setCurrentTSD(nullptr); + Initialized = false; + } ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, UNUSED bool MinimalInit) {