diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.h b/compiler-rt/lib/scudo/standalone/allocator_config.h --- a/compiler-rt/lib/scudo/standalone/allocator_config.h +++ b/compiler-rt/lib/scudo/standalone/allocator_config.h @@ -25,13 +25,18 @@ struct DefaultConfig { using SizeClassMap = DefaultSizeClassMap; + static const bool MaySupportMemoryTagging = false; + #if SCUDO_CAN_USE_PRIMARY64 - // 1GB Regions - typedef SizeClassAllocator64 Primary; + typedef SizeClassAllocator64 Primary; + static const uptr PrimaryRegionSizeLog = 30U; #else - // 512KB regions - typedef SizeClassAllocator32 Primary; + typedef SizeClassAllocator32 Primary; + static const uptr PrimaryRegionSizeLog = 19U; #endif + static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + typedef MapAllocatorCache SecondaryCache; static const u32 SecondaryCacheEntriesArraySize = 32U; static const u32 SecondaryCacheDefaultMaxEntriesCount = 32U; @@ -44,15 +49,18 @@ struct AndroidConfig { using SizeClassMap = AndroidSizeClassMap; + static const bool MaySupportMemoryTagging = true; + #if SCUDO_CAN_USE_PRIMARY64 - // 256MB regions - typedef SizeClassAllocator64 - Primary; + typedef SizeClassAllocator64 Primary; + static const uptr PrimaryRegionSizeLog = 28U; #else - // 256KB regions - typedef SizeClassAllocator32 Primary; + typedef SizeClassAllocator32 Primary; + static const uptr PrimaryRegionSizeLog = 18U; #endif + static const s32 PrimaryMinReleaseToOsIntervalMs = 1000; + static const s32 PrimaryMaxReleaseToOsIntervalMs = 1000; + typedef MapAllocatorCache SecondaryCache; static const u32 SecondaryCacheEntriesArraySize = 256U; static const u32 SecondaryCacheDefaultMaxEntriesCount = 32U; @@ -66,13 +74,18 @@ struct AndroidSvelteConfig { using SizeClassMap = SvelteSizeClassMap; + static const bool MaySupportMemoryTagging = false; + #if SCUDO_CAN_USE_PRIMARY64 - // 128MB regions - typedef SizeClassAllocator64 Primary; + typedef SizeClassAllocator64 Primary; + static const uptr PrimaryRegionSizeLog = 27U; #else - // 64KB regions - typedef SizeClassAllocator32 Primary; + typedef SizeClassAllocator32 Primary; + static const uptr PrimaryRegionSizeLog = 16U; #endif + static const s32 PrimaryMinReleaseToOsIntervalMs = 1000; + static const s32 PrimaryMaxReleaseToOsIntervalMs = 1000; + typedef MapAllocatorCache SecondaryCache; static const u32 SecondaryCacheEntriesArraySize = 16U; static const u32 SecondaryCacheDefaultMaxEntriesCount = 4U; @@ -86,8 +99,14 @@ #if SCUDO_CAN_USE_PRIMARY64 struct FuchsiaConfig { - // 1GB Regions - typedef SizeClassAllocator64 Primary; + using SizeClassMap = DefaultSizeClassMap; + static const bool MaySupportMemoryTagging = false; + + typedef SizeClassAllocator64 Primary; + static const uptr PrimaryRegionSizeLog = 30U; + static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + typedef MapAllocatorNoCache SecondaryCache; template using TSDRegistryT = TSDRegistrySharedT; // Shared, max 8 TSDs. diff --git a/compiler-rt/lib/scudo/standalone/memtag.h b/compiler-rt/lib/scudo/standalone/memtag.h --- a/compiler-rt/lib/scudo/standalone/memtag.h +++ b/compiler-rt/lib/scudo/standalone/memtag.h @@ -268,6 +268,11 @@ *TaggedEnd = storeTags(*TaggedBegin, *TaggedBegin + Size); } +template +inline constexpr bool allocatorSupportsMemoryTagging() { + return archSupportsMemoryTagging() && Config::MaySupportMemoryTagging; +} + } // namespace scudo #endif diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -39,20 +39,15 @@ // Memory used by this allocator is never unmapped but can be partially // reclaimed if the platform allows for it. -template -class SizeClassAllocator32 { +template class SizeClassAllocator32 { public: - typedef SizeClassMapT SizeClassMap; + typedef typename Config::SizeClassMap SizeClassMap; // The bytemap can only track UINT8_MAX - 1 classes. static_assert(SizeClassMap::LargestClassId <= (UINT8_MAX - 1), ""); // Regions should be large enough to hold the largest Block. - static_assert((1UL << RegionSizeLog) >= SizeClassMap::MaxSize, ""); - typedef SizeClassAllocator32 - ThisT; + static_assert((1UL << Config::PrimaryRegionSizeLog) >= SizeClassMap::MaxSize, + ""); + typedef SizeClassAllocator32 ThisT; typedef SizeClassAllocatorLocalCache CacheT; typedef typename CacheT::TransferBatch TransferBatch; static const bool SupportsMemoryTagging = false; @@ -199,9 +194,9 @@ bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { - const s32 Interval = - Max(Min(static_cast(Value), MaxReleaseToOsIntervalMs), - MinReleaseToOsIntervalMs); + const s32 Interval = Max( + Min(static_cast(Value), Config::PrimaryMaxReleaseToOsIntervalMs), + Config::PrimaryMinReleaseToOsIntervalMs); atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval); return true; } @@ -236,8 +231,9 @@ private: static const uptr NumClasses = SizeClassMap::NumClasses; - static const uptr RegionSize = 1UL << RegionSizeLog; - static const uptr NumRegions = SCUDO_MMAP_RANGE_SIZE >> RegionSizeLog; + static const uptr RegionSize = 1UL << Config::PrimaryRegionSizeLog; + static const uptr NumRegions = + SCUDO_MMAP_RANGE_SIZE >> Config::PrimaryRegionSizeLog; static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U; typedef FlatByteMap ByteMap; @@ -270,7 +266,7 @@ static_assert(sizeof(SizeClassInfo) % SCUDO_CACHE_LINE_SIZE == 0, ""); uptr computeRegionId(uptr Mem) { - const uptr Id = Mem >> RegionSizeLog; + const uptr Id = Mem >> Config::PrimaryRegionSizeLog; CHECK_LT(Id, NumRegions); return Id; } diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -40,21 +40,14 @@ // The memory used by this allocator is never unmapped, but can be partially // released if the platform allows for it. -template -class SizeClassAllocator64 { +template class SizeClassAllocator64 { public: - typedef SizeClassMapT SizeClassMap; - typedef SizeClassAllocator64< - SizeClassMap, RegionSizeLog, MinReleaseToOsIntervalMs, - MaxReleaseToOsIntervalMs, MaySupportMemoryTagging> - ThisT; + typedef typename Config::SizeClassMap SizeClassMap; + typedef SizeClassAllocator64 ThisT; typedef SizeClassAllocatorLocalCache CacheT; typedef typename CacheT::TransferBatch TransferBatch; static const bool SupportsMemoryTagging = - MaySupportMemoryTagging && archSupportsMemoryTagging(); + allocatorSupportsMemoryTagging(); static uptr getSizeByClassId(uptr ClassId) { return (ClassId == SizeClassMap::BatchClassId) @@ -178,9 +171,9 @@ bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { - const s32 Interval = - Max(Min(static_cast(Value), MaxReleaseToOsIntervalMs), - MinReleaseToOsIntervalMs); + const s32 Interval = Max( + Min(static_cast(Value), Config::PrimaryMaxReleaseToOsIntervalMs), + Config::PrimaryMinReleaseToOsIntervalMs); atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval); return true; } @@ -258,7 +251,7 @@ AtomicOptions Options; private: - static const uptr RegionSize = 1UL << RegionSizeLog; + static const uptr RegionSize = 1UL << Config::PrimaryRegionSizeLog; static const uptr NumClasses = SizeClassMap::NumClasses; static const uptr PrimarySize = RegionSize * NumClasses; @@ -308,7 +301,7 @@ } uptr getRegionBaseByClassId(uptr ClassId) const { - return PrimaryBase + (ClassId << RegionSizeLog); + return PrimaryBase + (ClassId << Config::PrimaryRegionSizeLog); } NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId, diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -401,10 +401,15 @@ static const scudo::uptr DeathRegionSizeLog = 20U; struct DeathConfig { + static const bool MaySupportMemoryTagging = false; + // Tiny allocator, its Primary only serves chunks of four sizes. - using DeathSizeClassMap = scudo::FixedSizeClassMap; - typedef scudo::SizeClassAllocator64 - Primary; + using SizeClassMap = scudo::FixedSizeClassMap; + typedef scudo::SizeClassAllocator64 Primary; + static const scudo::uptr PrimaryRegionSizeLog = DeathRegionSizeLog; + static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + typedef scudo::MapAllocatorNoCache SecondaryCache; template using TSDRegistryT = scudo::TSDRegistrySharedT; }; @@ -460,13 +465,13 @@ std::vector V; scudo::uptr FailedAllocationsCount = 0; for (scudo::uptr ClassId = 1U; - ClassId <= DeathConfig::DeathSizeClassMap::LargestClassId; ClassId++) { + ClassId <= DeathConfig::SizeClassMap::LargestClassId; ClassId++) { const scudo::uptr Size = - DeathConfig::DeathSizeClassMap::getSizeByClassId(ClassId); + DeathConfig::SizeClassMap::getSizeByClassId(ClassId); // Allocate enough to fill all of the regions above this one. const scudo::uptr MaxNumberOfChunks = ((1U << DeathRegionSizeLog) / Size) * - (DeathConfig::DeathSizeClassMap::LargestClassId - ClassId + 1); + (DeathConfig::SizeClassMap::LargestClassId - ClassId + 1); void *P; for (scudo::uptr I = 0; I <= MaxNumberOfChunks; I++) { P = Allocator->allocate(Size - 64U, Origin); diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp @@ -52,20 +52,51 @@ Str.output(); } +template struct TestConfig1 { + using SizeClassMap = SizeClassMapT; + static const scudo::uptr PrimaryRegionSizeLog = 18U; + static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + static const bool MaySupportMemoryTagging = false; +}; + +template struct TestConfig2 { + using SizeClassMap = SizeClassMapT; + static const scudo::uptr PrimaryRegionSizeLog = 24U; + static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + static const bool MaySupportMemoryTagging = false; +}; + +template struct TestConfig3 { + using SizeClassMap = SizeClassMapT; + static const scudo::uptr PrimaryRegionSizeLog = 24U; + static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + static const bool MaySupportMemoryTagging = true; +}; + TEST(ScudoPrimaryTest, BasicPrimary) { using SizeClassMap = scudo::DefaultSizeClassMap; #if !SCUDO_FUCHSIA - testPrimary>(); + testPrimary>>(); #endif - testPrimary>(); - testPrimary>(); + testPrimary>>(); + testPrimary>>(); } +struct SmallRegionsConfig { + using SizeClassMap = scudo::DefaultSizeClassMap; + static const scudo::uptr PrimaryRegionSizeLog = 20U; + static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + static const bool MaySupportMemoryTagging = false; +}; + // The 64-bit SizeClassAllocator can be easily OOM'd with small region sizes. // For the 32-bit one, it requires actually exhausting memory, so we skip it. TEST(ScudoPrimaryTest, Primary64OOM) { - using Primary = scudo::SizeClassAllocator64; + using Primary = scudo::SizeClassAllocator64; using TransferBatch = Primary::CacheT::TransferBatch; Primary Allocator; Allocator.init(/*ReleaseToOsInterval=*/-1); @@ -142,11 +173,10 @@ TEST(ScudoPrimaryTest, PrimaryIterate) { using SizeClassMap = scudo::DefaultSizeClassMap; #if !SCUDO_FUCHSIA - testIteratePrimary>(); + testIteratePrimary>>(); #endif - testIteratePrimary>(); - testIteratePrimary>(); + testIteratePrimary>>(); + testIteratePrimary>>(); } static std::mutex Mutex; @@ -204,11 +234,10 @@ TEST(ScudoPrimaryTest, PrimaryThreaded) { using SizeClassMap = scudo::SvelteSizeClassMap; #if !SCUDO_FUCHSIA - testPrimaryThreaded>(); + testPrimaryThreaded>>(); #endif - testPrimaryThreaded>(); - testPrimaryThreaded>(); + testPrimaryThreaded>>(); + testPrimaryThreaded>>(); } // Through a simple allocation that spans two pages, verify that releaseToOS @@ -236,9 +265,8 @@ TEST(ScudoPrimaryTest, ReleaseToOS) { using SizeClassMap = scudo::DefaultSizeClassMap; #if !SCUDO_FUCHSIA - testReleaseToOS>(); + testReleaseToOS>>(); #endif - testReleaseToOS>(); - testReleaseToOS>(); + testReleaseToOS>>(); + testReleaseToOS>>(); }