diff --git a/compiler-rt/lib/scudo/standalone/size_class_map.h b/compiler-rt/lib/scudo/standalone/size_class_map.h --- a/compiler-rt/lib/scudo/standalone/size_class_map.h +++ b/compiler-rt/lib/scudo/standalone/size_class_map.h @@ -12,6 +12,10 @@ #include "common.h" #include "string_utils.h" +#if defined(__aarch64__) +#include +#endif + namespace scudo { // SizeClassMap maps allocation sizes into size classes and back, in an @@ -136,15 +140,170 @@ } }; +struct AndroidSizeClassMap { + static const uptr SizeTableSize = 30; + typedef const u32 SizeTableType[SizeTableSize]; + static const uptr MaxSize = 65552; + static const uptr MaxSizeLog = 16; + static const u32 MaxNumCachedHint = 14; + static const u32 MaxBytesCachedLog = 14; + + enum { + ST00 = 32, + ST01 = 48, + ST02 = 64, + ST03 = 80, + ST04 = 96, + ST05 = 144, + ST06 = 160, + ST07 = 176, + ST08 = 208, + ST09 = 240, + ST0a = 336, + ST0b = 416, + ST0c = 448, + ST0d = 592, + ST0e = 800, + ST0f = 1040, + ST10 = 1552, + ST11 = 2320, + ST12 = 2576, + ST13 = 3088, + ST14 = 4368, + ST15 = 7184, + ST16 = 8464, + ST17 = 12560, + ST18 = 13840, + ST19 = 16400, + ST1a = 18192, + ST1b = 23312, + ST1c = 28944, + ST1d = 65552, + }; + + __attribute__((visibility("hidden"))) static SizeTableType &getSizeTable() { + static constexpr u32 SizeTable[] = { + ST00, ST01, ST02, ST03, ST04, ST05, ST06, ST07, ST08, ST09, + ST0a, ST0b, ST0c, ST0d, ST0e, ST0f, ST10, ST11, ST12, ST13, + ST14, ST15, ST16, ST17, ST18, ST19, ST1a, ST1b, ST1c, ST1d, + }; + return SizeTable; + } + + static const uptr NumClasses = SizeTableSize + 1; + static const uptr LargestClassId = NumClasses - 1; + static const uptr BatchClassId = 0; + + static uptr getSizeByClassId(uptr ClassId) { + DCHECK_NE(ClassId, BatchClassId); + return getSizeTable()[ClassId - 1]; + } + + __attribute__((visibility("hidden"))) static uptr getClassIdBySize(uptr Size) { + uptr Min = 0; +#if defined(__aarch64__) + { + static constexpr uint8x16_t Vecs[2] = { + { + (ST0f + 15) / 16, + (ST0e + 15) / 16, + (ST0d + 15) / 16, + (ST0c + 15) / 16, + (ST0b + 15) / 16, + (ST0a + 15) / 16, + (ST09 + 15) / 16, + (ST08 + 15) / 16, + (ST07 + 15) / 16, + (ST06 + 15) / 16, + (ST05 + 15) / 16, + (ST04 + 15) / 16, + (ST03 + 15) / 16, + (ST02 + 15) / 16, + (ST01 + 15) / 16, + (ST00 + 15) / 16, + }, + { + 0xff, + 0xff, + (ST1d - ST0f + 255) / 256, + (ST1c - ST0f + 255) / 256, + (ST1b - ST0f + 255) / 256, + (ST1a - ST0f + 255) / 256, + (ST19 - ST0f + 255) / 256, + (ST18 - ST0f + 255) / 256, + (ST17 - ST0f + 255) / 256, + (ST16 - ST0f + 255) / 256, + (ST15 - ST0f + 255) / 256, + (ST14 - ST0f + 255) / 256, + (ST13 - ST0f + 255) / 256, + (ST12 - ST0f + 255) / 256, + (ST11 - ST0f + 255) / 256, + (ST10 - ST0f + 255) / 256, + }, + }; + + uint64_t Group; + uint8_t ScaledSize; + + if (Size <= ST0f) { + Group = 0; + ScaledSize = (Size + 15) / 16; + } else { + Group = 1; + ScaledSize = (Size - ST0f + 255) / 256; + } + + uint8x16_t SizeSplat = { + ScaledSize, ScaledSize, ScaledSize, ScaledSize, + ScaledSize, ScaledSize, ScaledSize, ScaledSize, + ScaledSize, ScaledSize, ScaledSize, ScaledSize, + ScaledSize, ScaledSize, ScaledSize, ScaledSize, + }; + + uint8x16_t mask = vcgeq_u8(Vecs[Group], SizeSplat); + + uint64_t mask1 = ((uint64_t *)&mask)[1]; + if (mask1) { + return Group * 16 + (__builtin_clzl(mask1) >> 3) + 1; + } else { + return Group * 16 + 8 + (__builtin_clzl(((uint64_t *)&mask)[0]) >> 3) + 1; + } + } +#endif + uptr Max = SizeTableSize; + while (Min < Max) { + uptr Cand = (Min + Max) / 2; + if (getSizeTable()[Cand] < Size) + Min = Cand + 1; + else + Max = Cand; + } + return Min + 1; + } + + static u32 getMaxCachedHint(uptr Size) { + DCHECK_LE(Size, MaxSize); + DCHECK_NE(Size, 0); + u32 N; + // Force a 32-bit division if the template parameters allow for it. + if (MaxBytesCachedLog > 31 || MaxSizeLog > 31) + N = static_cast((1UL << MaxBytesCachedLog) / Size); + else + N = (1U << MaxBytesCachedLog) / static_cast(Size); + return Max(1U, Min(MaxNumCachedHint, N)); + } + + static void print() {} + static void validate() {} +}; + typedef SizeClassMap<3, 5, 8, 17, 8, 10> DefaultSizeClassMap; // TODO(kostyak): further tune class maps for Android & Fuchsia. #if SCUDO_WORDSIZE == 64U typedef SizeClassMap<4, 4, 8, 14, 4, 10> SvelteSizeClassMap; -typedef SizeClassMap<2, 5, 9, 16, 14, 14> AndroidSizeClassMap; #else typedef SizeClassMap<4, 3, 7, 14, 5, 10> SvelteSizeClassMap; -typedef SizeClassMap<2, 5, 9, 16, 14, 14> AndroidSizeClassMap; #endif } // namespace scudo