Index: compiler-rt/lib/asan/asan_allocator.h =================================================================== --- compiler-rt/lib/asan/asan_allocator.h +++ compiler-rt/lib/asan/asan_allocator.h @@ -118,39 +118,96 @@ void OnUnmap(uptr p, uptr size) const; }; -#if SANITIZER_CAN_USE_ALLOCATOR64 +#if defined(__aarch64__) +// AArch64 supports 39, 42 and 48-bit VMA. +const uptr kAllocatorSpace = ~(uptr)0; +#if SANITIZER_ANDROID +const uptr kAllocatorSize = 0x2000000000ULL; // 128G. +typedef VeryCompactSizeClassMap SizeClassMap64; +#else +const uptr kAllocatorSize = 0x40000000000ULL; // 4T. +typedef DefaultSizeClassMap SizeClassMap64; +#endif + +template +struct AP64 { // Allocator64 parameters. Deliberately using a short name. + static const uptr kSpaceBeg = kAllocatorSpace; + static const uptr kSpaceSize = kAllocatorSize; + static const uptr kMetadataSize = 0; + typedef __asan::SizeClassMap64 SizeClassMap; + typedef AsanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; + using AddressSpaceView = AddressSpaceViewTy; +}; +template +using Allocator64ASVT = SizeClassAllocator64>; +using Allocator64 = Allocator64ASVT; + +typedef CompactSizeClassMap SizeClassMap32; +template +struct AP32 { + static const uptr kSpaceBeg = 0; + static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; + static const uptr kMetadataSize = 16; + typedef __asan::SizeClassMap32 SizeClassMap; + static const uptr kRegionSizeLog = 20; + using AddressSpaceView = AddressSpaceViewTy; + typedef AsanMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; +template +using Allocator32ASVT = SizeClassAllocator32>; +using Allocator32 = Allocator32ASVT; +using Allocator32or64 = DoubleAllocator; + +static inline uptr getKNumClasses() { + if (Allocator32or64::UseAllocator1) + return SizeClassMap32::kNumClasses; + return SizeClassMap64::kNumClasses; +} + +static inline uptr getKMaxSize() { + if (Allocator32or64::UseAllocator1) + return SizeClassMap32::kMaxSize; + return SizeClassMap64::kMaxSize; +} + +static inline uptr getClassID(uptr size) { + if (Allocator32or64::UseAllocator1) + return SizeClassMap32::ClassID(size); + return SizeClassMap64::ClassID(size); +} + +static const uptr kMaxNumberOfSizeClasses = + SizeClassMap32::kNumClasses < SizeClassMap64::kNumClasses + ? SizeClassMap64::kNumClasses + : SizeClassMap32::kNumClasses; + +template +using PrimaryAllocatorASVT = DoubleAllocator, + Allocator64ASVT>; +#elif SANITIZER_CAN_USE_ALLOCATOR64 # if SANITIZER_FUCHSIA const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x40000000000ULL; // 4T. -typedef DefaultSizeClassMap SizeClassMap; # elif defined(__powerpc64__) const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x20000000000ULL; // 2T. -typedef DefaultSizeClassMap SizeClassMap; -# elif defined(__aarch64__) && SANITIZER_ANDROID -// Android needs to support 39, 42 and 48 bit VMA. -const uptr kAllocatorSpace = ~(uptr)0; -const uptr kAllocatorSize = 0x2000000000ULL; // 128G. -typedef VeryCompactSizeClassMap SizeClassMap; -# elif defined(__aarch64__) -// AArch64/SANITIZER_CAN_USE_ALLOCATOR64 is only for 42-bit VMA -// so no need to different values for different VMA. -const uptr kAllocatorSpace = 0x10000000000ULL; -const uptr kAllocatorSize = 0x10000000000ULL; // 3T. -typedef DefaultSizeClassMap SizeClassMap; -#elif defined(__sparc__) +# elif defined(__sparc__) const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x20000000000ULL; // 2T. -typedef DefaultSizeClassMap SizeClassMap; # elif SANITIZER_WINDOWS const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x8000000000ULL; // 500G -typedef DefaultSizeClassMap SizeClassMap; # else const uptr kAllocatorSpace = 0x600000000000ULL; const uptr kAllocatorSize = 0x40000000000ULL; // 4T. -typedef DefaultSizeClassMap SizeClassMap; # endif +typedef DefaultSizeClassMap SizeClassMap; +static inline uptr getKNumClasses() { return SizeClassMap::kNumClasses; } +static inline uptr getKMaxSize() { return SizeClassMap::kMaxSize; } +static inline uptr getClassID(uptr size) { return SizeClassMap::ClassID(size); } +static const uptr kMaxNumberOfSizeClasses = SizeClassMap::kNumClasses; template struct AP64 { // Allocator64 parameters. Deliberately using a short name. static const uptr kSpaceBeg = kAllocatorSpace; @@ -164,9 +221,12 @@ template using PrimaryAllocatorASVT = SizeClassAllocator64>; -using PrimaryAllocator = PrimaryAllocatorASVT; #else // Fallback to SizeClassAllocator32. typedef CompactSizeClassMap SizeClassMap; +static inline uptr getKNumClasses() { return SizeClassMap::kNumClasses; } +static inline uptr getKMaxSize() { return SizeClassMap::kMaxSize; } +static inline uptr getClassID(uptr size) { return SizeClassMap::ClassID(size); } +static const uptr kMaxNumberOfSizeClasses = SizeClassMap::kNumClasses; template struct AP32 { static const uptr kSpaceBeg = 0; @@ -180,16 +240,14 @@ }; template using PrimaryAllocatorASVT = SizeClassAllocator32 >; -using PrimaryAllocator = PrimaryAllocatorASVT; #endif // SANITIZER_CAN_USE_ALLOCATOR64 -static const uptr kNumberOfSizeClasses = SizeClassMap::kNumClasses; - template using AsanAllocatorASVT = CombinedAllocator>; using AsanAllocator = AsanAllocatorASVT; using AllocatorCache = AsanAllocator::AllocatorCache; +using PrimaryAllocator = PrimaryAllocatorASVT; struct AsanThreadLocalMallocStorage { uptr quarantine_cache[16]; Index: compiler-rt/lib/asan/asan_allocator.cc =================================================================== --- compiler-rt/lib/asan/asan_allocator.cc +++ compiler-rt/lib/asan/asan_allocator.cc @@ -30,6 +30,12 @@ #include "sanitizer_common/sanitizer_quarantine.h" #include "lsan/lsan_common.h" +// Initialize static member of DoubleAllocator: use allocator1 when the address +// space is too small for the 64-bit allocator. +template +bool __sanitizer::DoubleAllocator::UseAllocator1 = + GetMaxVirtualAddress() < (((uptr)1ULL << 48) - 1); + namespace __asan { // Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits. @@ -113,7 +119,7 @@ struct AsanChunk: ChunkBase { uptr Beg() { return reinterpret_cast(this) + kChunkHeaderSize; } uptr UsedSize(bool locked_version = false) { - if (user_requested_size != SizeClassMap::kMaxSize) + if (user_requested_size != getKMaxSize()) return user_requested_size; return *reinterpret_cast( get_allocator().GetMetaData(AllocBeg(locked_version))); @@ -499,7 +505,7 @@ CHECK(allocator.FromPrimary(allocated)); } else { CHECK(!allocator.FromPrimary(allocated)); - m->user_requested_size = SizeClassMap::kMaxSize; + m->user_requested_size = getKMaxSize(); uptr *meta = reinterpret_cast(allocator.GetMetaData(allocated)); meta[0] = size; meta[1] = chunk_beg; @@ -524,10 +530,10 @@ thread_stats.mallocs++; thread_stats.malloced += size; thread_stats.malloced_redzones += needed_size - size; - if (needed_size > SizeClassMap::kMaxSize) + if (needed_size > getKMaxSize()) thread_stats.malloc_large++; else - thread_stats.malloced_by_size[SizeClassMap::ClassID(needed_size)]++; + thread_stats.malloced_by_size[getClassID(needed_size)]++; void *res = reinterpret_cast(user_beg); if (can_fill && fl.max_malloc_fill_size) { Index: compiler-rt/lib/asan/asan_stats.h =================================================================== --- compiler-rt/lib/asan/asan_stats.h +++ compiler-rt/lib/asan/asan_stats.h @@ -38,7 +38,7 @@ uptr munmaps; uptr munmaped; uptr malloc_large; - uptr malloced_by_size[kNumberOfSizeClasses]; + uptr malloced_by_size[kMaxNumberOfSizeClasses]; // Ctor for global AsanStats (accumulated stats for dead threads). explicit AsanStats(LinkerInitialized) { } Index: compiler-rt/lib/asan/asan_stats.cc =================================================================== --- compiler-rt/lib/asan/asan_stats.cc +++ compiler-rt/lib/asan/asan_stats.cc @@ -30,9 +30,9 @@ } static void PrintMallocStatsArray(const char *prefix, - uptr (&array)[kNumberOfSizeClasses]) { + uptr *array) { Printf("%s", prefix); - for (uptr i = 0; i < kNumberOfSizeClasses; i++) { + for (uptr i = 0; i < getKNumClasses(); i++) { if (!array[i]) continue; Printf("%zu:%zu; ", i, array[i]); } Index: compiler-rt/lib/lsan/lsan_allocator.h =================================================================== --- compiler-rt/lib/lsan/lsan_allocator.h +++ compiler-rt/lib/lsan/lsan_allocator.h @@ -49,8 +49,45 @@ u32 stack_trace_id; }; -#if defined(__mips64) || defined(__aarch64__) || defined(__i386__) || \ - defined(__arm__) +#if defined(__aarch64__) +template +struct AP32 { + static const uptr kSpaceBeg = 0; + static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; + static const uptr kMetadataSize = sizeof(ChunkMetadata); + typedef __sanitizer::CompactSizeClassMap SizeClassMap; + static const uptr kRegionSizeLog = 20; + using AddressSpaceView = AddressSpaceViewTy; + typedef NoOpMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; +}; + +const uptr kAllocatorSpace = 0x600000000000ULL; +const uptr kAllocatorSize = 0x40000000000ULL; // 4T. + +template +struct AP64 { // Allocator64 parameters. Deliberately using a short name. + static const uptr kSpaceBeg = kAllocatorSpace; + static const uptr kSpaceSize = kAllocatorSize; + static const uptr kMetadataSize = sizeof(ChunkMetadata); + typedef DefaultSizeClassMap SizeClassMap; + typedef NoOpMapUnmapCallback MapUnmapCallback; + static const uptr kFlags = 0; + using AddressSpaceView = AddressSpaceViewTy; +}; + +template +using Allocator32ASVT = SizeClassAllocator32>; +template +using Allocator64ASVT = SizeClassAllocator64>; + +using Allocator32 = Allocator32ASVT; +using Allocator64 = Allocator64ASVT; + +template +using PrimaryAllocatorASVT = DoubleAllocator, + Allocator64ASVT>; +#elif defined(__mips64) || defined(__i386__) || defined(__arm__) template struct AP32 { static const uptr kSpaceBeg = 0; @@ -64,7 +101,6 @@ }; template using PrimaryAllocatorASVT = SizeClassAllocator32>; -using PrimaryAllocator = PrimaryAllocatorASVT; #elif defined(__x86_64__) || defined(__powerpc64__) # if defined(__powerpc64__) const uptr kAllocatorSpace = 0xa0000000000ULL; @@ -86,13 +122,13 @@ template using PrimaryAllocatorASVT = SizeClassAllocator64>; -using PrimaryAllocator = PrimaryAllocatorASVT; #endif template using AllocatorASVT = CombinedAllocator>; using Allocator = AllocatorASVT; using AllocatorCache = Allocator::AllocatorCache; +using PrimaryAllocator = PrimaryAllocatorASVT; Allocator::AllocatorCache *GetAllocatorCache(); Index: compiler-rt/lib/lsan/lsan_allocator.cc =================================================================== --- compiler-rt/lib/lsan/lsan_allocator.cc +++ compiler-rt/lib/lsan/lsan_allocator.cc @@ -25,6 +25,12 @@ extern "C" void *memset(void *ptr, int value, uptr num); +// Initialize static member of DoubleAllocator: use allocator1 when the address +// space is too small for the 64-bit allocator. +template +bool __sanitizer::DoubleAllocator::UseAllocator1 = + GetMaxVirtualAddress() < (((uptr)1ULL << 48) - 1); + namespace __lsan { #if defined(__i386__) || defined(__arm__) static const uptr kMaxAllowedMallocSize = 1UL << 30; Index: compiler-rt/lib/sanitizer_common/sanitizer_allocator.h =================================================================== --- compiler-rt/lib/sanitizer_common/sanitizer_allocator.h +++ compiler-rt/lib/sanitizer_common/sanitizer_allocator.h @@ -75,6 +75,7 @@ #include "sanitizer_allocator_local_cache.h" #include "sanitizer_allocator_secondary.h" #include "sanitizer_allocator_combined.h" +#include "sanitizer_doubleallocator.h" } // namespace __sanitizer Index: compiler-rt/lib/sanitizer_common/sanitizer_doubleallocator.h =================================================================== --- /dev/null +++ compiler-rt/lib/sanitizer_common/sanitizer_doubleallocator.h @@ -0,0 +1,164 @@ +//===-- sanitizer_allocator.h -----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Select one of the allocators at runtime. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_DOUBLEALLOCATOR_H +#define SANITIZER_DOUBLEALLOCATOR_H + +template +class DoubleAllocator { + Allocator1 a1; + Allocator2 a2; + + public: + static bool UseAllocator1; + + class DoubleAllocatorCache { + typename Allocator1::AllocatorCache a1; + typename Allocator2::AllocatorCache a2; + + public: + void Init(AllocatorGlobalStats *s) { + if (UseAllocator1) + a1.Init(s); + else + a2.Init(s); + } + void *Allocate(DoubleAllocator *allocator, uptr class_id) { + if (UseAllocator1) + return a1.Allocate(&allocator->a1, class_id); + return a2.Allocate(&allocator->a2, class_id); + } + + void Deallocate(DoubleAllocator *allocator, uptr class_id, void *p) { + if (UseAllocator1) + a1.Deallocate(&allocator->a1, class_id, p); + else + a2.Deallocate(&allocator->a2, class_id, p); + } + + void Drain(DoubleAllocator *allocator) { + if (UseAllocator1) + a1.Drain(&allocator->a1); + else + a2.Drain(&allocator->a2); + } + + void Destroy(DoubleAllocator *allocator, AllocatorGlobalStats *s) { + if (UseAllocator1) + a1.Destroy(&allocator->a1, s); + else + a2.Destroy(&allocator->a2, s); + } + }; + + using MapUnmapCallback = typename Allocator1::MapUnmapCallback; + using AddressSpaceView = typename Allocator1::AddressSpaceView; + using AllocatorCache = DoubleAllocatorCache; + + void Init(s32 release_to_os_interval_ms) { + if (UseAllocator1) + a1.Init(release_to_os_interval_ms); + else + a2.Init(release_to_os_interval_ms); + } + + static bool CanAllocate(uptr size, uptr alignment) { + if (UseAllocator1) + return Allocator1::CanAllocate(size, alignment); + return Allocator2::CanAllocate(size, alignment); + } + + static uptr ClassID(uptr size) { + if (UseAllocator1) + return Allocator1::ClassID(size); + return Allocator2::ClassID(size); + } + + bool PointerIsMine(const void *p) { + if (UseAllocator1) + return a1.PointerIsMine(p); + return a2.PointerIsMine(p); + } + + void *GetMetaData(const void *p) { + if (UseAllocator1) + return a1.GetMetaData(p); + return a2.GetMetaData(p); + } + + uptr GetSizeClass(const void *p) { + if (UseAllocator1) + return a1.GetSizeClass(p); + return a2.GetSizeClass(p); + } + + void ForEachChunk(ForEachChunkCallback callback, void *arg) { + if (UseAllocator1) + a1.ForEachChunk(callback, arg); + else + a2.ForEachChunk(callback, arg); + } + + void TestOnlyUnmap() { + if (UseAllocator1) + a1.TestOnlyUnmap(); + else + a2.TestOnlyUnmap(); + } + void ForceLock() { + if (UseAllocator1) + a1.ForceLock(); + else + a2.ForceLock(); + } + void ForceUnlock() { + if (UseAllocator1) + a1.ForceUnlock(); + else + a2.ForceUnlock(); + } + void *GetBlockBegin(const void *p) { + if (UseAllocator1) + return a1.GetBlockBegin(p); + return a2.GetBlockBegin(p); + } + uptr GetActuallyAllocatedSize(void *p) { + if (UseAllocator1) + return a1.GetActuallyAllocatedSize(p); + return a2.GetActuallyAllocatedSize(p); + } + void SetReleaseToOSIntervalMs(s32 release_to_os_interval_ms) { + if (UseAllocator1) + a1.SetReleaseToOSIntervalMs(release_to_os_interval_ms); + else + a2.SetReleaseToOSIntervalMs(release_to_os_interval_ms); + } + s32 ReleaseToOSIntervalMs() const { + if (UseAllocator1) + return a1.ReleaseToOSIntervalMs(); + return a2.ReleaseToOSIntervalMs(); + } + void ForceReleaseToOS() { + if (UseAllocator1) + a1.ForceReleaseToOS(); + else + a2.ForceReleaseToOS(); + } + void PrintStats() { + if (UseAllocator1) + a1.PrintStats(); + else + a2.PrintStats(); + } +}; + +#endif // SANITIZER_DOUBLEALLOCATOR_H Index: compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cc =================================================================== --- compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cc +++ compiler-rt/lib/sanitizer_common/tests/sanitizer_allocator_test.cc @@ -160,6 +160,9 @@ SizeClassAllocator32>; using Allocator32Compact = Allocator32CompactASVT; +using Allocator32or64Compact = + DoubleAllocator; + template void TestSizeClassMap() { typedef SizeClassMap SCMap; @@ -274,6 +277,13 @@ TestSizeClassAllocator(); } +TEST(SanitizerCommon, SizeClassAllocator32or64Compact) { + Allocator32or64Compact::UseAllocator1 = false; + TestSizeClassAllocator(); + Allocator32or64Compact::UseAllocator1 = true; + TestSizeClassAllocator(); +} + TEST(SanitizerCommon, SizeClassAllocator64Dense) { TestSizeClassAllocator(); } @@ -357,6 +367,12 @@ TEST(SanitizerCommon, SizeClassAllocator64CompactMetadataStress) { SizeClassAllocatorMetadataStress(); } +TEST(SanitizerCommon, SizeClassAllocator32or64CompactMetadataStress) { + Allocator32or64Compact::UseAllocator1 = false; + SizeClassAllocatorMetadataStress(); + Allocator32or64Compact::UseAllocator1 = true; + SizeClassAllocatorMetadataStress(); +} #endif #endif @@ -404,6 +420,12 @@ TEST(SanitizerCommon, SizeClassAllocator64CompactGetBlockBegin) { SizeClassAllocatorGetBlockBeginStress(1ULL << 33); } +TEST(SanitizerCommon, SizeClassAllocator32or64CompactGetBlockBegin) { + Allocator32or64Compact::UseAllocator1 = false; + SizeClassAllocatorGetBlockBeginStress(1ULL << 33); + Allocator32or64Compact::UseAllocator1 = true; + SizeClassAllocatorGetBlockBeginStress(1ULL << 33); +} #endif TEST(SanitizerCommon, SizeClassAllocator64VeryCompactGetBlockBegin) { // Does not have > 4Gb for each class. @@ -694,6 +716,12 @@ TEST(SanitizerCommon, CombinedAllocator64Compact) { TestCombinedAllocator(); } +TEST(SanitizerCommon, CombinedDoubleAllocator) { + Allocator32or64Compact::UseAllocator1 = false; + TestCombinedAllocator(); + Allocator32or64Compact::UseAllocator1 = true; + TestCombinedAllocator(); +} #endif TEST(SanitizerCommon, CombinedAllocator64VeryCompact) { @@ -755,6 +783,12 @@ TEST(SanitizerCommon, SizeClassAllocator64CompactLocalCache) { TestSizeClassAllocatorLocalCache(); } +TEST(SanitizerCommon, SizeClassAllocator32or64CompactLocalCache) { + Allocator32or64Compact::UseAllocator1 = false; + TestSizeClassAllocatorLocalCache(); + Allocator32or64Compact::UseAllocator1 = true; + TestSizeClassAllocatorLocalCache(); +} #endif TEST(SanitizerCommon, SizeClassAllocator64VeryCompactLocalCache) { TestSizeClassAllocatorLocalCache(); @@ -1333,6 +1367,12 @@ TEST(SanitizerCommon, SizeClassAllocator64CompactReleaseFreeMemoryToOS) { TestReleaseFreeMemoryToOS(); } +TEST(SanitizerCommon, SizeClassAllocator32or64CompactReleaseFreeMemoryToOS) { + Allocator32or64Compact::UseAllocator1 = false; + TestReleaseFreeMemoryToOS(); + Allocator32or64Compact::UseAllocator1 = true; + TestReleaseFreeMemoryToOS(); +} TEST(SanitizerCommon, SizeClassAllocator64VeryCompactReleaseFreeMemoryToOS) { TestReleaseFreeMemoryToOS();