diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.h b/compiler-rt/lib/scudo/standalone/allocator_config.h --- a/compiler-rt/lib/scudo/standalone/allocator_config.h +++ b/compiler-rt/lib/scudo/standalone/allocator_config.h @@ -19,6 +19,22 @@ #include "tsd_exclusive.h" #include "tsd_shared.h" +// To import a custom configuration, define `SCUDO_USE_CUSTOM_CONFIG` and +// aliasing the `Config` like: +// +// namespace scudo { +// // The instance of Scudo will be initiated with `Config`. +// typedef CustomConfig Config; +// // Aliasing as default configuration to run the tests with this config. +// typedef CustomConfig DefaultConfig; +// } // namespace scudo +// +// Put them in the header `custom_scudo_config.h` then you will be using the +// custom configuration and able to run all the tests as well. +#ifdef SCUDO_USE_CUSTOM_CONFIG +#include "custom_scudo_config.h" +#endif + namespace scudo { // The combined allocator uses a structure as a template argument that @@ -26,191 +42,265 @@ // allocator. // // struct ExampleConfig { -// // SizeClassMap to use with the Primary. -// using SizeClassMap = DefaultSizeClassMap; // // Indicates possible support for Memory Tagging. // static const bool MaySupportMemoryTagging = false; -// // Defines the Primary allocator to use. -// typedef SizeClassAllocator64 Primary; -// // Log2 of the size of a size class region, as used by the Primary. -// static const uptr PrimaryRegionSizeLog = 30U; -// // Log2 of the size of block group, as used by the Primary. Each group -// // contains a range of memory addresses, blocks in the range will belong to -// // the same group. In general, single region may have 1 or 2MB group size. -// // Multiple regions will have the group size equal to the region size -// // because the region size is usually smaller than 1 MB. -// // Smaller value gives fine-grained control of memory usage but the trade -// // off is that it may take longer time of deallocation. -// static const uptr PrimaryGroupSizeLog = 20U; -// // Defines the type and scale of a compact pointer. A compact pointer can -// // be understood as the offset of a pointer within the region it belongs -// // to, in increments of a power-of-2 scale. -// // eg: Ptr = Base + (CompactPtr << Scale). -// typedef u32 PrimaryCompactPtrT; -// static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; -// // Indicates support for offsetting the start of a region by -// // a random number of pages. Only used with primary64. -// static const bool PrimaryEnableRandomOffset = true; -// // Call map for user memory with at least this size. Only used with -// // primary64. -// static const uptr PrimaryMapSizeIncrement = 1UL << 18; -// // Defines the minimal & maximal release interval that can be set. -// static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; -// static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; -// // Defines the type of cache used by the Secondary. Some additional -// // configuration entries can be necessary depending on the Cache. -// typedef MapAllocatorNoCache SecondaryCache; +// // // Thread-Specific Data Registry used, shared or exclusive. // template using TSDRegistryT = TSDRegistrySharedT; +// +// struct Primary { +// // SizeClassMap to use with the Primary. +// using SizeClassMap = DefaultSizeClassMap; +// +// // Log2 of the size of a size class region, as used by the Primary. +// static const uptr RegionSizeLog = 30U; +// +// // Log2 of the size of block group, as used by the Primary. Each group +// // contains a range of memory addresses, blocks in the range will belong +// // to the same group. In general, single region may have 1 or 2MB group +// // size. Multiple regions will have the group size equal to the region +// // size because the region size is usually smaller than 1 MB. +// // Smaller value gives fine-grained control of memory usage but the +// // trade-off is that it may take longer time of deallocation. +// static const uptr GroupSizeLog = 20U; +// +// // Defines the type and scale of a compact pointer. A compact pointer can +// // be understood as the offset of a pointer within the region it belongs +// // to, in increments of a power-of-2 scale. +// // eg: Ptr = Base + (CompactPtr << Scale). +// typedef u32 CompactPtrT; +// static const uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; +// +// // Indicates support for offsetting the start of a region by +// // a random number of pages. Only used with primary64. +// static const bool EnableRandomOffset = true; +// +// // Call map for user memory with at least this size. Only used with +// // primary64. +// static const uptr MapSizeIncrement = 1UL << 18; +// +// // Defines the minimal & maximal release interval that can be set. +// static const s32 MinReleaseToOsIntervalMs = INT32_MIN; +// static const s32 MaxReleaseToOsIntervalMs = INT32_MAX; +// }; +// // Defines the type of Primary allocator to use. +// template using PrimaryT = SizeClassAllocator64; +// +// // Defines the type of cache used by the Secondary. Some additional +// // configuration entries can be necessary depending on the Cache. +// struct Secondary { +// struct Cache { +// static const u32 EntriesArraySize = 32U; +// static const u32 QuarantineSize = 0U; +// static const u32 DefaultMaxEntriesCount = 32U; +// static const uptr DefaultMaxEntrySize = 1UL << 19; +// static const s32 MinReleaseToOsIntervalMs = INT32_MIN; +// static const s32 MaxReleaseToOsIntervalMs = INT32_MAX; +// }; +// // Defines the type of Secondary Cache to use. +// template using CacheT = MapAllocatorCache; +// }; +// // Defines the type of Secondary allocator to use. +// template using SecondaryT = MapAllocator; // }; -// Default configurations for various platforms. +#ifndef SCUDO_USE_CUSTOM_CONFIG +// Default configurations for various platforms. Note this is only enabled when +// there's no custom configuration in the build system. struct DefaultConfig { - using SizeClassMap = DefaultSizeClassMap; static const bool MaySupportMemoryTagging = true; + template using TSDRegistryT = TSDRegistryExT; // Exclusive + struct Primary { + using SizeClassMap = DefaultSizeClassMap; +#if SCUDO_CAN_USE_PRIMARY64 + static const uptr RegionSizeLog = 32U; + static const uptr GroupSizeLog = 21U; + typedef uptr CompactPtrT; + static const uptr CompactPtrScale = 0; + static const bool EnableRandomOffset = true; + static const uptr MapSizeIncrement = 1UL << 18; +#else + static const uptr RegionSizeLog = 19U; + static const uptr GroupSizeLog = 19U; + typedef uptr CompactPtrT; +#endif + static const s32 MinReleaseToOsIntervalMs = INT32_MIN; + static const s32 MaxReleaseToOsIntervalMs = INT32_MAX; + }; #if SCUDO_CAN_USE_PRIMARY64 - typedef SizeClassAllocator64 Primary; - static const uptr PrimaryRegionSizeLog = 32U; - static const uptr PrimaryGroupSizeLog = 21U; - typedef uptr PrimaryCompactPtrT; - static const uptr PrimaryCompactPtrScale = 0; - static const bool PrimaryEnableRandomOffset = true; - static const uptr PrimaryMapSizeIncrement = 1UL << 18; + template using PrimaryT = SizeClassAllocator64; #else - typedef SizeClassAllocator32 Primary; - static const uptr PrimaryRegionSizeLog = 19U; - static const uptr PrimaryGroupSizeLog = 19U; - typedef uptr PrimaryCompactPtrT; + template using PrimaryT = SizeClassAllocator32; #endif - static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; - static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; - typedef MapAllocatorCache SecondaryCache; - static const u32 SecondaryCacheEntriesArraySize = 32U; - static const u32 SecondaryCacheQuarantineSize = 0U; - static const u32 SecondaryCacheDefaultMaxEntriesCount = 32U; - static const uptr SecondaryCacheDefaultMaxEntrySize = 1UL << 19; - static const s32 SecondaryCacheMinReleaseToOsIntervalMs = INT32_MIN; - static const s32 SecondaryCacheMaxReleaseToOsIntervalMs = INT32_MAX; + struct Secondary { + struct Cache { + static const u32 EntriesArraySize = 32U; + static const u32 QuarantineSize = 0U; + static const u32 DefaultMaxEntriesCount = 32U; + static const uptr DefaultMaxEntrySize = 1UL << 19; + static const s32 MinReleaseToOsIntervalMs = INT32_MIN; + static const s32 MaxReleaseToOsIntervalMs = INT32_MAX; + }; + template using CacheT = MapAllocatorCache; + }; - template using TSDRegistryT = TSDRegistryExT; // Exclusive + template using SecondaryT = MapAllocator; }; + +#endif // SCUDO_USE_CUSTOM_CONFIG + struct AndroidConfig { - using SizeClassMap = AndroidSizeClassMap; static const bool MaySupportMemoryTagging = true; + template + using TSDRegistryT = TSDRegistrySharedT; // Shared, max 8 TSDs. + struct Primary { + using SizeClassMap = AndroidSizeClassMap; +#if SCUDO_CAN_USE_PRIMARY64 + static const uptr RegionSizeLog = 28U; + typedef u32 CompactPtrT; + static const uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; + static const uptr GroupSizeLog = 20U; + static const bool EnableRandomOffset = true; + static const uptr MapSizeIncrement = 1UL << 18; +#else + static const uptr RegionSizeLog = 18U; + static const uptr GroupSizeLog = 18U; + typedef uptr CompactPtrT; +#endif + static const s32 MinReleaseToOsIntervalMs = 1000; + static const s32 MaxReleaseToOsIntervalMs = 1000; + }; #if SCUDO_CAN_USE_PRIMARY64 - typedef SizeClassAllocator64 Primary; - static const uptr PrimaryRegionSizeLog = 28U; - typedef u32 PrimaryCompactPtrT; - static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; - static const uptr PrimaryGroupSizeLog = 20U; - static const bool PrimaryEnableRandomOffset = true; - static const uptr PrimaryMapSizeIncrement = 1UL << 18; + template using PrimaryT = SizeClassAllocator64; #else - typedef SizeClassAllocator32 Primary; - static const uptr PrimaryRegionSizeLog = 18U; - static const uptr PrimaryGroupSizeLog = 18U; - typedef uptr PrimaryCompactPtrT; + template using PrimaryT = SizeClassAllocator32; #endif - static const s32 PrimaryMinReleaseToOsIntervalMs = 1000; - static const s32 PrimaryMaxReleaseToOsIntervalMs = 1000; - typedef MapAllocatorCache SecondaryCache; - static const u32 SecondaryCacheEntriesArraySize = 256U; - static const u32 SecondaryCacheQuarantineSize = 32U; - static const u32 SecondaryCacheDefaultMaxEntriesCount = 32U; - static const uptr SecondaryCacheDefaultMaxEntrySize = 2UL << 20; - static const s32 SecondaryCacheMinReleaseToOsIntervalMs = 0; - static const s32 SecondaryCacheMaxReleaseToOsIntervalMs = 1000; + struct Secondary { + struct Cache { + static const u32 EntriesArraySize = 256U; + static const u32 QuarantineSize = 32U; + static const u32 DefaultMaxEntriesCount = 32U; + static const uptr DefaultMaxEntrySize = 2UL << 20; + static const s32 MinReleaseToOsIntervalMs = 0; + static const s32 MaxReleaseToOsIntervalMs = 1000; + }; + template using CacheT = MapAllocatorCache; + }; - template - using TSDRegistryT = TSDRegistrySharedT; // Shared, max 8 TSDs. + template using SecondaryT = MapAllocator; }; struct AndroidSvelteConfig { - using SizeClassMap = SvelteSizeClassMap; static const bool MaySupportMemoryTagging = false; + template + using TSDRegistryT = TSDRegistrySharedT; // Shared, max 2 TSDs. + struct Primary { + using SizeClassMap = SvelteSizeClassMap; #if SCUDO_CAN_USE_PRIMARY64 - typedef SizeClassAllocator64 Primary; - static const uptr PrimaryRegionSizeLog = 27U; - typedef u32 PrimaryCompactPtrT; - static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; - static const uptr PrimaryGroupSizeLog = 18U; - static const bool PrimaryEnableRandomOffset = true; - static const uptr PrimaryMapSizeIncrement = 1UL << 18; + static const uptr RegionSizeLog = 27U; + typedef u32 CompactPtrT; + static const uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; + static const uptr GroupSizeLog = 18U; + static const bool EnableRandomOffset = true; + static const uptr MapSizeIncrement = 1UL << 18; #else - typedef SizeClassAllocator32 Primary; - static const uptr PrimaryRegionSizeLog = 16U; - static const uptr PrimaryGroupSizeLog = 16U; - typedef uptr PrimaryCompactPtrT; + static const uptr RegionSizeLog = 16U; + static const uptr GroupSizeLog = 16U; + typedef uptr CompactPtrT; #endif - static const s32 PrimaryMinReleaseToOsIntervalMs = 1000; - static const s32 PrimaryMaxReleaseToOsIntervalMs = 1000; + static const s32 MinReleaseToOsIntervalMs = 1000; + static const s32 MaxReleaseToOsIntervalMs = 1000; + }; - typedef MapAllocatorCache SecondaryCache; - static const u32 SecondaryCacheEntriesArraySize = 16U; - static const u32 SecondaryCacheQuarantineSize = 32U; - static const u32 SecondaryCacheDefaultMaxEntriesCount = 4U; - static const uptr SecondaryCacheDefaultMaxEntrySize = 1UL << 18; - static const s32 SecondaryCacheMinReleaseToOsIntervalMs = 0; - static const s32 SecondaryCacheMaxReleaseToOsIntervalMs = 0; +#if SCUDO_CAN_USE_PRIMARY64 + template using PrimaryT = SizeClassAllocator64; +#else + template using PrimaryT = SizeClassAllocator32; +#endif - template - using TSDRegistryT = TSDRegistrySharedT; // Shared, max 2 TSDs. + struct Secondary { + struct Cache { + static const u32 EntriesArraySize = 16U; + static const u32 QuarantineSize = 32U; + static const u32 DefaultMaxEntriesCount = 4U; + static const uptr DefaultMaxEntrySize = 1UL << 18; + static const s32 MinReleaseToOsIntervalMs = 0; + static const s32 MaxReleaseToOsIntervalMs = 0; + }; + template using CacheT = MapAllocatorCache; + }; + + template using SecondaryT = MapAllocator; }; #if SCUDO_CAN_USE_PRIMARY64 struct FuchsiaConfig { - using SizeClassMap = FuchsiaSizeClassMap; static const bool MaySupportMemoryTagging = false; + template + using TSDRegistryT = TSDRegistrySharedT; // Shared, max 8 TSDs. - typedef SizeClassAllocator64 Primary; -// Support 39-bit VMA for riscv-64 + struct Primary { + using SizeClassMap = FuchsiaSizeClassMap; #if SCUDO_RISCV64 - static const uptr PrimaryRegionSizeLog = 28U; - static const uptr PrimaryGroupSizeLog = 19U; + // Support 39-bit VMA for riscv-64 + static const uptr RegionSizeLog = 28U; + static const uptr GroupSizeLog = 19U; #else - static const uptr PrimaryRegionSizeLog = 30U; - static const uptr PrimaryGroupSizeLog = 21U; + static const uptr RegionSizeLog = 30U; + static const uptr GroupSizeLog = 21U; #endif - typedef u32 PrimaryCompactPtrT; - static const bool PrimaryEnableRandomOffset = true; - static const uptr PrimaryMapSizeIncrement = 1UL << 18; - static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; - static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; - static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; - - typedef MapAllocatorNoCache SecondaryCache; - template - using TSDRegistryT = TSDRegistrySharedT; // Shared, max 8 TSDs. + typedef u32 CompactPtrT; + static const bool EnableRandomOffset = true; + static const uptr MapSizeIncrement = 1UL << 18; + static const uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; + static const s32 MinReleaseToOsIntervalMs = INT32_MIN; + static const s32 MaxReleaseToOsIntervalMs = INT32_MAX; + }; + template using PrimaryT = SizeClassAllocator64; + + struct Secondary { + template using CacheT = MapAllocatorNoCache; + }; + template using SecondaryT = MapAllocator; }; struct TrustyConfig { - using SizeClassMap = TrustySizeClassMap; static const bool MaySupportMemoryTagging = false; - - typedef SizeClassAllocator64 Primary; - // Some apps have 1 page of heap total so small regions are necessary. - static const uptr PrimaryRegionSizeLog = 10U; - static const uptr PrimaryGroupSizeLog = 10U; - typedef u32 PrimaryCompactPtrT; - static const bool PrimaryEnableRandomOffset = false; - // Trusty is extremely memory-constrained so minimally round up map calls. - static const uptr PrimaryMapSizeIncrement = 1UL << 4; - static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; - static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; - static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; - - typedef MapAllocatorNoCache SecondaryCache; template using TSDRegistryT = TSDRegistrySharedT; // Shared, max 1 TSD. + + struct Primary { + static const bool MaySupportMemoryTagging = false; + using SizeClassMap = TrustySizeClassMap; + // Some apps have 1 page of heap total so small regions are necessary. + static const uptr RegionSizeLog = 10U; + static const uptr GroupSizeLog = 10U; + typedef u32 CompactPtrT; + static const bool EnableRandomOffset = false; + // Trusty is extremely memory-constrained so minimally round up map calls. + static const uptr MapSizeIncrement = 1UL << 4; + static const uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; + static const s32 MinReleaseToOsIntervalMs = INT32_MIN; + static const s32 MaxReleaseToOsIntervalMs = INT32_MAX; + }; + template using PrimaryT = SizeClassAllocator64; + + struct Secondary { + template using CacheT = MapAllocatorNoCache; + }; + + template using SecondaryT = MapAllocator; }; #endif +#ifndef SCUDO_USE_CUSTOM_CONFIG + #if SCUDO_ANDROID typedef AndroidConfig Config; #elif SCUDO_FUCHSIA @@ -221,6 +311,8 @@ typedef DefaultConfig Config; #endif +#endif // SCUDO_USE_CUSTOM_CONFIG + } // namespace scudo #endif // SCUDO_ALLOCATOR_CONFIG_H_ diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -43,13 +43,14 @@ namespace scudo { -template +template class Allocator { public: - using PrimaryT = typename Params::Primary; + using PrimaryT = typename Config::template PrimaryT; + using SecondaryT = typename Config::template SecondaryT; using CacheT = typename PrimaryT::CacheT; - typedef Allocator ThisT; - typedef typename Params::template TSDRegistryT TSDRegistryT; + typedef Allocator ThisT; + typedef typename Config::template TSDRegistryT TSDRegistryT; void callPostInitCallback() { pthread_once(&PostInitNonce, PostInitCallback); @@ -71,7 +72,7 @@ NewHeader.State = Chunk::State::Available; Chunk::compareExchangeHeader(Allocator.Cookie, Ptr, &NewHeader, &Header); - if (allocatorSupportsMemoryTagging()) + if (allocatorSupportsMemoryTagging()) Ptr = untagPointer(Ptr); void *BlockBegin = Allocator::getBlockBegin(Ptr, &NewHeader); Cache.deallocate(NewHeader.ClassId, BlockBegin); @@ -98,7 +99,7 @@ // Reset tag to 0 as this chunk may have been previously used for a tagged // user allocation. - if (UNLIKELY(useMemoryTagging(Allocator.Primary.Options.load()))) + if (UNLIKELY(useMemoryTagging(Allocator.Primary.Options.load()))) storeTags(reinterpret_cast(Ptr), reinterpret_cast(Ptr) + sizeof(QuarantineBatch)); @@ -162,7 +163,7 @@ Primary.Options.set(OptionBit::DeallocTypeMismatch); if (getFlags()->delete_size_mismatch) Primary.Options.set(OptionBit::DeleteSizeMismatch); - if (allocatorSupportsMemoryTagging() && + if (allocatorSupportsMemoryTagging() && systemSupportsMemoryTagging()) Primary.Options.set(OptionBit::UseMemoryTagging); Primary.Options.set(OptionBit::UseOddEvenTags); @@ -264,7 +265,7 @@ void drainCaches() { TSDRegistry.drainCaches(this); } ALWAYS_INLINE void *getHeaderTaggedPointer(void *Ptr) { - if (!allocatorSupportsMemoryTagging()) + if (!allocatorSupportsMemoryTagging()) return Ptr; auto UntaggedPtr = untagPointer(Ptr); if (UntaggedPtr != Ptr) @@ -276,7 +277,7 @@ } ALWAYS_INLINE uptr addHeaderTag(uptr Ptr) { - if (!allocatorSupportsMemoryTagging()) + if (!allocatorSupportsMemoryTagging()) return Ptr; return addFixedTag(Ptr, 2); } @@ -427,7 +428,7 @@ // // When memory tagging is enabled, zeroing the contents is done as part of // setting the tag. - if (UNLIKELY(useMemoryTagging(Options))) { + if (UNLIKELY(useMemoryTagging(Options))) { uptr PrevUserPtr; Chunk::UnpackedHeader Header; const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId); @@ -509,7 +510,7 @@ } else { Block = addHeaderTag(Block); Ptr = addHeaderTag(Ptr); - if (UNLIKELY(useMemoryTagging(Options))) { + if (UNLIKELY(useMemoryTagging(Options))) { storeTags(reinterpret_cast(Block), reinterpret_cast(Ptr)); storeSecondaryAllocationStackMaybe(Options, Ptr, Size); } @@ -676,7 +677,7 @@ (reinterpret_cast(OldTaggedPtr) + NewSize)) & Chunk::SizeOrUnusedBytesMask; Chunk::compareExchangeHeader(Cookie, OldPtr, &NewHeader, &OldHeader); - if (UNLIKELY(useMemoryTagging(Options))) { + if (UNLIKELY(useMemoryTagging(Options))) { if (ClassId) { resizeTaggedChunk(reinterpret_cast(OldTaggedPtr) + OldSize, reinterpret_cast(OldTaggedPtr) + NewSize, @@ -772,7 +773,7 @@ Base = untagPointer(Base); const uptr From = Base; const uptr To = Base + Size; - bool MayHaveTaggedPrimary = allocatorSupportsMemoryTagging() && + bool MayHaveTaggedPrimary = allocatorSupportsMemoryTagging() && systemSupportsMemoryTagging(); auto Lambda = [this, From, To, MayHaveTaggedPrimary, Callback, Arg](uptr Block) { @@ -794,9 +795,9 @@ } if (Header.State == Chunk::State::Allocated) { uptr TaggedChunk = Chunk; - if (allocatorSupportsMemoryTagging()) + if (allocatorSupportsMemoryTagging()) TaggedChunk = untagPointer(TaggedChunk); - if (useMemoryTagging(Primary.Options.load())) + if (useMemoryTagging(Primary.Options.load())) TaggedChunk = loadTag(Chunk); Callback(TaggedChunk, getSize(reinterpret_cast(Chunk), &Header), Arg); @@ -895,7 +896,7 @@ } bool useMemoryTaggingTestOnly() const { - return useMemoryTagging(Primary.Options.load()); + return useMemoryTagging(Primary.Options.load()); } void disableMemoryTagging() { // If we haven't been initialized yet, we need to initialize now in order to @@ -905,7 +906,7 @@ // callback), which may cause mappings to be created with memory tagging // enabled. TSDRegistry.initOnceMaybe(this); - if (allocatorSupportsMemoryTagging()) { + if (allocatorSupportsMemoryTagging()) { Secondary.disableMemoryTagging(); Primary.Options.clear(OptionBit::UseMemoryTagging); } @@ -989,7 +990,7 @@ const char *Memory, const char *MemoryTags, uintptr_t MemoryAddr, size_t MemorySize) { *ErrorInfo = {}; - if (!allocatorSupportsMemoryTagging() || + if (!allocatorSupportsMemoryTagging() || MemoryAddr + MemorySize < MemoryAddr) return; @@ -1017,7 +1018,6 @@ } private: - using SecondaryT = MapAllocator; typedef typename PrimaryT::SizeClassMap SizeClassMap; static const uptr MinAlignmentLog = SCUDO_MIN_ALIGNMENT_LOG; @@ -1029,7 +1029,7 @@ static_assert(MinAlignment >= sizeof(Chunk::PackedHeader), "Minimal alignment must at least cover a chunk header."); - static_assert(!allocatorSupportsMemoryTagging() || + static_assert(!allocatorSupportsMemoryTagging() || MinAlignment >= archMemoryTagGranuleSize(), ""); @@ -1129,7 +1129,7 @@ const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes; if (LIKELY(Header->ClassId)) return SizeOrUnusedBytes; - if (allocatorSupportsMemoryTagging()) + if (allocatorSupportsMemoryTagging()) Ptr = untagPointer(const_cast(Ptr)); return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) - reinterpret_cast(Ptr) - SizeOrUnusedBytes; @@ -1150,12 +1150,12 @@ NewHeader.State = Chunk::State::Available; else NewHeader.State = Chunk::State::Quarantined; - NewHeader.OriginOrWasZeroed = useMemoryTagging(Options) && + NewHeader.OriginOrWasZeroed = useMemoryTagging(Options) && NewHeader.ClassId && !TSDRegistry.getDisableMemInit(); Chunk::compareExchangeHeader(Cookie, Ptr, &NewHeader, Header); - if (UNLIKELY(useMemoryTagging(Options))) { + if (UNLIKELY(useMemoryTagging(Options))) { u8 PrevTag = extractTag(reinterpret_cast(TaggedPtr)); storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size); if (NewHeader.ClassId) { @@ -1172,7 +1172,7 @@ } } if (BypassQuarantine) { - if (allocatorSupportsMemoryTagging()) + if (allocatorSupportsMemoryTagging()) Ptr = untagPointer(Ptr); void *BlockBegin = getBlockBegin(Ptr, &NewHeader); const uptr ClassId = NewHeader.ClassId; @@ -1183,7 +1183,7 @@ if (UnlockRequired) TSD->unlock(); } else { - if (UNLIKELY(useMemoryTagging(Options))) + if (UNLIKELY(useMemoryTagging(Options))) storeTags(reinterpret_cast(BlockBegin), reinterpret_cast(Ptr)); Secondary.deallocate(Options, BlockBegin); diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -42,13 +42,14 @@ template class SizeClassAllocator32 { public: - typedef typename Config::PrimaryCompactPtrT CompactPtrT; - typedef typename Config::SizeClassMap SizeClassMap; - static const uptr GroupSizeLog = Config::PrimaryGroupSizeLog; + typedef typename Config::Primary::CompactPtrT CompactPtrT; + typedef typename Config::Primary::SizeClassMap SizeClassMap; + static const uptr GroupSizeLog = Config::Primary::GroupSizeLog; // The bytemap can only track UINT8_MAX - 1 classes. static_assert(SizeClassMap::LargestClassId <= (UINT8_MAX - 1), ""); // Regions should be large enough to hold the largest Block. - static_assert((1UL << Config::PrimaryRegionSizeLog) >= SizeClassMap::MaxSize, + static_assert((1UL << Config::Primary::RegionSizeLog) >= + SizeClassMap::MaxSize, ""); typedef SizeClassAllocator32 ThisT; typedef SizeClassAllocatorLocalCache CacheT; @@ -281,9 +282,9 @@ bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { - const s32 Interval = Max( - Min(static_cast(Value), Config::PrimaryMaxReleaseToOsIntervalMs), - Config::PrimaryMinReleaseToOsIntervalMs); + const s32 Interval = Max(Min(static_cast(Value), + Config::Primary::MaxReleaseToOsIntervalMs), + Config::Primary::MinReleaseToOsIntervalMs); atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval); return true; } @@ -315,9 +316,9 @@ private: static const uptr NumClasses = SizeClassMap::NumClasses; - static const uptr RegionSize = 1UL << Config::PrimaryRegionSizeLog; + static const uptr RegionSize = 1UL << Config::Primary::RegionSizeLog; static const uptr NumRegions = - SCUDO_MMAP_RANGE_SIZE >> Config::PrimaryRegionSizeLog; + SCUDO_MMAP_RANGE_SIZE >> Config::Primary::RegionSizeLog; static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U; typedef FlatByteMap ByteMap; @@ -350,7 +351,7 @@ static_assert(sizeof(SizeClassInfo) % SCUDO_CACHE_LINE_SIZE == 0, ""); uptr computeRegionId(uptr Mem) { - const uptr Id = Mem >> Config::PrimaryRegionSizeLog; + const uptr Id = Mem >> Config::Primary::RegionSizeLog; CHECK_LT(Id, NumRegions); return Id; } @@ -379,7 +380,7 @@ unmap(reinterpret_cast(End), MapEnd - End); DCHECK_EQ(Region % RegionSize, 0U); - static_assert(Config::PrimaryRegionSizeLog == GroupSizeLog, + static_assert(Config::Primary::RegionSizeLog == GroupSizeLog, "Memory group should be the same size as Region"); return Region; diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -45,11 +45,11 @@ template class SizeClassAllocator64 { public: - typedef typename Config::PrimaryCompactPtrT CompactPtrT; - static const uptr CompactPtrScale = Config::PrimaryCompactPtrScale; - static const uptr GroupSizeLog = Config::PrimaryGroupSizeLog; + typedef typename Config::Primary::CompactPtrT CompactPtrT; + typedef typename Config::Primary::SizeClassMap SizeClassMap; + static const uptr CompactPtrScale = Config::Primary::CompactPtrScale; + static const uptr GroupSizeLog = Config::Primary::GroupSizeLog; static const uptr GroupScale = GroupSizeLog - CompactPtrScale; - typedef typename Config::SizeClassMap SizeClassMap; typedef SizeClassAllocator64 ThisT; typedef SizeClassAllocatorLocalCache CacheT; typedef typename CacheT::TransferBatch TransferBatch; @@ -119,10 +119,11 @@ RegionInfo *Region = getRegionInfo(I); // The actual start of a region is offset by a random number of pages // when PrimaryEnableRandomOffset is set. - Region->RegionBeg = (PrimaryBase + (I << Config::PrimaryRegionSizeLog)) + - (Config::PrimaryEnableRandomOffset - ? ((getRandomModN(&Seed, 16) + 1) * PageSize) - : 0); + Region->RegionBeg = + (PrimaryBase + (I << Config::Primary::RegionSizeLog)) + + (Config::Primary::EnableRandomOffset + ? ((getRandomModN(&Seed, 16) + 1) * PageSize) + : 0); Region->RandState = getRandomU32(&Seed); // Releasing small blocks is expensive, set a higher threshold to avoid // frequent page releases. @@ -322,9 +323,9 @@ bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { - const s32 Interval = Max( - Min(static_cast(Value), Config::PrimaryMaxReleaseToOsIntervalMs), - Config::PrimaryMinReleaseToOsIntervalMs); + const s32 Interval = Max(Min(static_cast(Value), + Config::Primary::MaxReleaseToOsIntervalMs), + Config::Primary::MinReleaseToOsIntervalMs); atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval); return true; } @@ -420,11 +421,11 @@ AtomicOptions Options; private: - static const uptr RegionSize = 1UL << Config::PrimaryRegionSizeLog; + static const uptr RegionSize = 1UL << Config::Primary::RegionSizeLog; static const uptr NumClasses = SizeClassMap::NumClasses; static const uptr PrimarySize = RegionSize * NumClasses; - static const uptr MapSizeIncrement = Config::PrimaryMapSizeIncrement; + static const uptr MapSizeIncrement = Config::Primary::MapSizeIncrement; // Fill at most this number of batches from the newly map'd memory. static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U; diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h --- a/compiler-rt/lib/scudo/standalone/secondary.h +++ b/compiler-rt/lib/scudo/standalone/secondary.h @@ -72,7 +72,7 @@ MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); } -class MapAllocatorNoCache { +template class MapAllocatorNoCache { public: void init(UNUSED s32 ReleaseToOsInterval) {} bool retrieve(UNUSED Options Options, UNUSED uptr Size, UNUSED uptr Alignment, @@ -130,17 +130,18 @@ template class MapAllocatorCache { public: + using CacheConfig = typename Config::Secondary::Cache; // Ensure the default maximum specified fits the array. - static_assert(Config::SecondaryCacheDefaultMaxEntriesCount <= - Config::SecondaryCacheEntriesArraySize, + static_assert(CacheConfig::DefaultMaxEntriesCount <= + CacheConfig::EntriesArraySize, ""); void init(s32 ReleaseToOsInterval) NO_THREAD_SAFETY_ANALYSIS { DCHECK_EQ(EntriesCount, 0U); setOption(Option::MaxCacheEntriesCount, - static_cast(Config::SecondaryCacheDefaultMaxEntriesCount)); + static_cast(CacheConfig::DefaultMaxEntriesCount)); setOption(Option::MaxCacheEntrySize, - static_cast(Config::SecondaryCacheDefaultMaxEntrySize)); + static_cast(CacheConfig::DefaultMaxEntrySize)); setOption(Option::ReleaseInterval, static_cast(ReleaseToOsInterval)); } @@ -185,10 +186,9 @@ // just unmap it. break; } - if (Config::SecondaryCacheQuarantineSize && - useMemoryTagging(Options)) { + if (CacheConfig::QuarantineSize && useMemoryTagging(Options)) { QuarantinePos = - (QuarantinePos + 1) % Max(Config::SecondaryCacheQuarantineSize, 1u); + (QuarantinePos + 1) % Max(CacheConfig::QuarantineSize, 1u); if (!Quarantine[QuarantinePos].CommitBase) { Quarantine[QuarantinePos] = Entry; return; @@ -291,16 +291,15 @@ bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { - const s32 Interval = - Max(Min(static_cast(Value), - Config::SecondaryCacheMaxReleaseToOsIntervalMs), - Config::SecondaryCacheMinReleaseToOsIntervalMs); + const s32 Interval = Max( + Min(static_cast(Value), CacheConfig::MaxReleaseToOsIntervalMs), + CacheConfig::MinReleaseToOsIntervalMs); atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval); return true; } if (O == Option::MaxCacheEntriesCount) { const u32 MaxCount = static_cast(Value); - if (MaxCount > Config::SecondaryCacheEntriesArraySize) + if (MaxCount > CacheConfig::EntriesArraySize) return false; atomic_store_relaxed(&MaxEntriesCount, MaxCount); return true; @@ -317,7 +316,7 @@ void disableMemoryTagging() EXCLUDES(Mutex) { ScopedLock L(Mutex); - for (u32 I = 0; I != Config::SecondaryCacheQuarantineSize; ++I) { + for (u32 I = 0; I != CacheConfig::QuarantineSize; ++I) { if (Quarantine[I].CommitBase) { MemMapT &MemMap = Quarantine[I].MemMap; MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); @@ -342,11 +341,11 @@ private: void empty() { - MemMapT MapInfo[Config::SecondaryCacheEntriesArraySize]; + MemMapT MapInfo[CacheConfig::EntriesArraySize]; uptr N = 0; { ScopedLock L(Mutex); - for (uptr I = 0; I < Config::SecondaryCacheEntriesArraySize; I++) { + for (uptr I = 0; I < CacheConfig::EntriesArraySize; I++) { if (!Entries[I].CommitBase) continue; MapInfo[N] = Entries[I].MemMap; @@ -387,9 +386,9 @@ if (!EntriesCount || OldestTime == 0 || OldestTime > Time) return; OldestTime = 0; - for (uptr I = 0; I < Config::SecondaryCacheQuarantineSize; I++) + for (uptr I = 0; I < CacheConfig::QuarantineSize; I++) releaseIfOlderThan(Quarantine[I], Time); - for (uptr I = 0; I < Config::SecondaryCacheEntriesArraySize; I++) + for (uptr I = 0; I < CacheConfig::EntriesArraySize; I++) releaseIfOlderThan(Entries[I], Time); } @@ -402,9 +401,8 @@ u32 IsFullEvents GUARDED_BY(Mutex) = 0; atomic_s32 ReleaseToOsIntervalMs = {}; - CachedBlock - Entries[Config::SecondaryCacheEntriesArraySize] GUARDED_BY(Mutex) = {}; - NonZeroLengthArray + CachedBlock Entries[CacheConfig::EntriesArraySize] GUARDED_BY(Mutex) = {}; + NonZeroLengthArray Quarantine GUARDED_BY(Mutex) = {}; }; @@ -469,7 +467,7 @@ void unmapTestOnly() { Cache.unmapTestOnly(); } private: - typename Config::SecondaryCache Cache; + typename Config::Secondary::template CacheT Cache; mutable HybridMutex Mutex; DoublyLinkedList InUseBlocks GUARDED_BY(Mutex); diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -541,21 +541,29 @@ static const scudo::uptr DeathRegionSizeLog = 21U; struct DeathConfig { static const bool MaySupportMemoryTagging = false; - - // Tiny allocator, its Primary only serves chunks of four sizes. - using SizeClassMap = scudo::FixedSizeClassMap; - typedef scudo::SizeClassAllocator64 Primary; - static const scudo::uptr PrimaryRegionSizeLog = DeathRegionSizeLog; - static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; - static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; - typedef scudo::uptr PrimaryCompactPtrT; - static const scudo::uptr PrimaryCompactPtrScale = 0; - static const bool PrimaryEnableRandomOffset = true; - static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18; - static const scudo::uptr PrimaryGroupSizeLog = 18; - - typedef scudo::MapAllocatorNoCache SecondaryCache; template using TSDRegistryT = scudo::TSDRegistrySharedT; + + struct Primary { + // Tiny allocator, its Primary only serves chunks of four sizes. + using SizeClassMap = scudo::FixedSizeClassMap; + static const scudo::uptr RegionSizeLog = DeathRegionSizeLog; + static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX; + typedef scudo::uptr CompactPtrT; + static const scudo::uptr CompactPtrScale = 0; + static const bool EnableRandomOffset = true; + static const scudo::uptr MapSizeIncrement = 1UL << 18; + static const scudo::uptr GroupSizeLog = 18; + }; + template + using PrimaryT = scudo::SizeClassAllocator64; + + struct Secondary { + template + using CacheT = scudo::MapAllocatorNoCache; + }; + + template using SecondaryT = scudo::MapAllocator; }; TEST(ScudoCombinedDeathTest, DeathCombined) { @@ -600,13 +608,14 @@ std::vector V; scudo::uptr FailedAllocationsCount = 0; for (scudo::uptr ClassId = 1U; - ClassId <= DeathConfig::SizeClassMap::LargestClassId; ClassId++) { + ClassId <= DeathConfig::Primary::SizeClassMap::LargestClassId; + ClassId++) { const scudo::uptr Size = - DeathConfig::SizeClassMap::getSizeByClassId(ClassId); + DeathConfig::Primary::SizeClassMap::getSizeByClassId(ClassId); // Allocate enough to fill all of the regions above this one. const scudo::uptr MaxNumberOfChunks = ((1U << DeathRegionSizeLog) / Size) * - (DeathConfig::SizeClassMap::LargestClassId - ClassId + 1); + (DeathConfig::Primary::SizeClassMap::LargestClassId - ClassId + 1); void *P; for (scudo::uptr I = 0; I <= MaxNumberOfChunks; I++) { P = Allocator->allocate(Size - 64U, Origin); diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp @@ -25,82 +25,96 @@ // 32-bit architectures. It's not something we want to encourage, but we still // should ensure the tests pass. -struct TestConfig1 { - static const scudo::uptr PrimaryRegionSizeLog = 18U; - static const scudo::uptr PrimaryGroupSizeLog = 18U; - static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; - static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; +template struct TestConfig1 { static const bool MaySupportMemoryTagging = false; - typedef scudo::uptr PrimaryCompactPtrT; - static const scudo::uptr PrimaryCompactPtrScale = 0; - static const bool PrimaryEnableRandomOffset = true; - static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18; + + struct Primary { + using SizeClassMap = SizeClassMapT; + static const scudo::uptr RegionSizeLog = 18U; + static const scudo::uptr GroupSizeLog = 18U; + static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX; + typedef scudo::uptr CompactPtrT; + static const scudo::uptr CompactPtrScale = 0; + static const bool EnableRandomOffset = true; + static const scudo::uptr MapSizeIncrement = 1UL << 18; + }; }; -struct TestConfig2 { +template struct TestConfig2 { + static const bool MaySupportMemoryTagging = false; + + struct Primary { + using SizeClassMap = SizeClassMapT; #if defined(__mips__) - // Unable to allocate greater size on QEMU-user. - static const scudo::uptr PrimaryRegionSizeLog = 23U; + // Unable to allocate greater size on QEMU-user. + static const scudo::uptr RegionSizeLog = 23U; #else - static const scudo::uptr PrimaryRegionSizeLog = 24U; + static const scudo::uptr RegionSizeLog = 24U; #endif - static const scudo::uptr PrimaryGroupSizeLog = 20U; - static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; - static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; - static const bool MaySupportMemoryTagging = false; - typedef scudo::uptr PrimaryCompactPtrT; - static const scudo::uptr PrimaryCompactPtrScale = 0; - static const bool PrimaryEnableRandomOffset = true; - static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18; + static const scudo::uptr GroupSizeLog = 20U; + static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX; + typedef scudo::uptr CompactPtrT; + static const scudo::uptr CompactPtrScale = 0; + static const bool EnableRandomOffset = true; + static const scudo::uptr MapSizeIncrement = 1UL << 18; + }; }; -struct TestConfig3 { +template struct TestConfig3 { + static const bool MaySupportMemoryTagging = true; + + struct Primary { + using SizeClassMap = SizeClassMapT; #if defined(__mips__) - // Unable to allocate greater size on QEMU-user. - static const scudo::uptr PrimaryRegionSizeLog = 23U; + // Unable to allocate greater size on QEMU-user. + static const scudo::uptr RegionSizeLog = 23U; #else - static const scudo::uptr PrimaryRegionSizeLog = 24U; + static const scudo::uptr RegionSizeLog = 24U; #endif - static const scudo::uptr PrimaryGroupSizeLog = 20U; - static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; - static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; - static const bool MaySupportMemoryTagging = true; - typedef scudo::uptr PrimaryCompactPtrT; - static const scudo::uptr PrimaryCompactPtrScale = 0; - static const bool PrimaryEnableRandomOffset = true; - static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18; + static const scudo::uptr GroupSizeLog = 20U; + static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX; + typedef scudo::uptr CompactPtrT; + static const scudo::uptr CompactPtrScale = 0; + static const bool EnableRandomOffset = true; + static const scudo::uptr MapSizeIncrement = 1UL << 18; + }; }; -struct TestConfig4 { +template struct TestConfig4 { + static const bool MaySupportMemoryTagging = true; + + struct Primary { + using SizeClassMap = SizeClassMapT; #if defined(__mips__) - // Unable to allocate greater size on QEMU-user. - static const scudo::uptr PrimaryRegionSizeLog = 23U; + // Unable to allocate greater size on QEMU-user. + static const scudo::uptr RegionSizeLog = 23U; #else - static const scudo::uptr PrimaryRegionSizeLog = 24U; + static const scudo::uptr RegionSizeLog = 24U; #endif - static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; - static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; - static const bool MaySupportMemoryTagging = true; - static const scudo::uptr PrimaryCompactPtrScale = 3U; - static const scudo::uptr PrimaryGroupSizeLog = 20U; - typedef scudo::u32 PrimaryCompactPtrT; - static const bool PrimaryEnableRandomOffset = true; - static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18; + static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX; + static const scudo::uptr CompactPtrScale = 3U; + static const scudo::uptr GroupSizeLog = 20U; + typedef scudo::u32 CompactPtrT; + static const bool EnableRandomOffset = true; + static const scudo::uptr MapSizeIncrement = 1UL << 18; + }; }; -template -struct Config : public BaseConfig { - using SizeClassMap = SizeClassMapT; -}; +template