diff --git a/compiler-rt/lib/scudo/standalone/local_cache.h b/compiler-rt/lib/scudo/standalone/local_cache.h --- a/compiler-rt/lib/scudo/standalone/local_cache.h +++ b/compiler-rt/lib/scudo/standalone/local_cache.h @@ -10,6 +10,7 @@ #define SCUDO_LOCAL_CACHE_H_ #include "internal_defs.h" +#include "platform.h" #include "report.h" #include "stats.h" @@ -20,8 +21,8 @@ typedef typename SizeClassAllocator::CompactPtrT CompactPtrT; struct TransferBatch { - static const u32 MaxNumCached = SizeClassMap::MaxNumCachedHint; - void setFromArray(CompactPtrT *Array, u32 N) { + static const u16 MaxNumCached = SizeClassMap::MaxNumCachedHint; + void setFromArray(CompactPtrT *Array, u16 N) { DCHECK_LE(N, MaxNumCached); Count = N; memcpy(Batch, Array, sizeof(Batch[0]) * Count); @@ -34,19 +35,19 @@ void copyToArray(CompactPtrT *Array) const { memcpy(Array, Batch, sizeof(Batch[0]) * Count); } - u32 getCount() const { return Count; } - CompactPtrT get(u32 I) const { + u16 getCount() const { return Count; } + CompactPtrT get(u16 I) const { DCHECK_LE(I, Count); return Batch[I]; } - static u32 getMaxCached(uptr Size) { + static u16 getMaxCached(uptr Size) { return Min(MaxNumCached, SizeClassMap::getMaxCachedHint(Size)); } TransferBatch *Next; private: - u32 Count; CompactPtrT Batch[MaxNumCached]; + u16 Count; }; void init(GlobalStats *S, SizeClassAllocator *A) { @@ -128,9 +129,9 @@ private: static const uptr NumClasses = SizeClassMap::NumClasses; static const uptr BatchClassId = SizeClassMap::BatchClassId; - struct PerClass { - u32 Count; - u32 MaxCount; + struct alignas(SCUDO_CACHE_LINE_SIZE) PerClass { + u16 Count; + u16 MaxCount; // Note: ClassSize is zero for the transfer batch. uptr ClassSize; CompactPtrT Chunks[2 * TransferBatch::MaxNumCached]; @@ -180,7 +181,7 @@ } NOINLINE void drain(PerClass *C, uptr ClassId) { - const u32 Count = Min(C->MaxCount / 2, C->Count); + const u16 Count = Min(static_cast(C->MaxCount / 2), C->Count); TransferBatch *B = createBatch(ClassId, Allocator->decompactPtr(ClassId, C->Chunks[0])); if (UNLIKELY(!B)) diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -351,7 +351,7 @@ } const uptr Size = getSizeByClassId(ClassId); - const u32 MaxCount = TransferBatch::getMaxCached(Size); + const u16 MaxCount = TransferBatch::getMaxCached(Size); DCHECK_GT(MaxCount, 0U); // The maximum number of blocks we should carve in the region is dictated // by the maximum number of batches we want to fill, and the amount of @@ -382,7 +382,8 @@ C->createBatch(ClassId, reinterpret_cast(ShuffleArray[I])); if (UNLIKELY(!B)) return nullptr; - const u32 N = Min(MaxCount, NumberOfBlocks - I); + // `MaxCount` is u16 so the result will also fit in u16. + const u16 N = static_cast(Min(MaxCount, NumberOfBlocks - I)); B->setFromArray(&ShuffleArray[I], N); Sci->FreeList.push_back(B); I += N; diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -333,7 +333,7 @@ NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId, RegionInfo *Region) { const uptr Size = getSizeByClassId(ClassId); - const u32 MaxCount = TransferBatch::getMaxCached(Size); + const u16 MaxCount = TransferBatch::getMaxCached(Size); const uptr RegionBeg = Region->RegionBeg; const uptr MappedUser = Region->MappedUser; @@ -392,7 +392,8 @@ CompactPtrBase, ShuffleArray[I]))); if (UNLIKELY(!B)) return nullptr; - const u32 N = Min(MaxCount, NumberOfBlocks - I); + // `MaxCount` is u16 so the result will also fit in u16. + const u16 N = static_cast(Min(MaxCount, NumberOfBlocks - I)); B->setFromArray(&ShuffleArray[I], N); Region->FreeList.push_back(B); I += N; diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h --- a/compiler-rt/lib/scudo/standalone/release.h +++ b/compiler-rt/lib/scudo/standalone/release.h @@ -242,7 +242,7 @@ if (BlockSize <= PageSize && PageSize % BlockSize == 0) { // Each chunk affects one page only. for (const auto &It : FreeList) { - for (u32 I = 0; I < It.getCount(); I++) { + for (u16 I = 0; I < It.getCount(); I++) { const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase(); if (P >= RoundedSize) continue; @@ -256,7 +256,7 @@ DCHECK_GE(RegionSize, BlockSize); const uptr LastBlockInRegion = ((RegionSize / BlockSize) - 1U) * BlockSize; for (const auto &It : FreeList) { - for (u32 I = 0; I < It.getCount(); I++) { + for (u16 I = 0; I < It.getCount(); I++) { const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase(); if (P >= RoundedSize) continue; diff --git a/compiler-rt/lib/scudo/standalone/size_class_map.h b/compiler-rt/lib/scudo/standalone/size_class_map.h --- a/compiler-rt/lib/scudo/standalone/size_class_map.h +++ b/compiler-rt/lib/scudo/standalone/size_class_map.h @@ -23,7 +23,7 @@ } template struct SizeClassMapBase { - static u32 getMaxCachedHint(uptr Size) { + static u16 getMaxCachedHint(uptr Size) { DCHECK_NE(Size, 0); u32 N; // Force a 32-bit division if the template parameters allow for it. @@ -31,7 +31,10 @@ N = static_cast((1UL << Config::MaxBytesCachedLog) / Size); else N = (1U << Config::MaxBytesCachedLog) / static_cast(Size); - return Max(1U, Min(Config::MaxNumCachedHint, N)); + + // Note that Config::MaxNumCachedHint is u16 so the result is guaranteed to + // fit in u16. + return static_cast(Max(1U, Min(Config::MaxNumCachedHint, N))); } }; @@ -65,7 +68,7 @@ static const uptr M = (1UL << S) - 1; public: - static const u32 MaxNumCachedHint = Config::MaxNumCachedHint; + static const u16 MaxNumCachedHint = Config::MaxNumCachedHint; static const uptr MaxSize = (1UL << Config::MaxSizeLog) + Config::SizeDelta; static const uptr NumClasses = @@ -99,7 +102,7 @@ return MidClass + 1 + scaledLog2(Size - 1, Config::MidSizeLog, S); } - static u32 getMaxCachedHint(uptr Size) { + static u16 getMaxCachedHint(uptr Size) { DCHECK_LE(Size, MaxSize); return Base::getMaxCachedHint(Size); } @@ -178,7 +181,7 @@ static constexpr LSBTable LTable = {}; public: - static const u32 MaxNumCachedHint = Config::MaxNumCachedHint; + static const u16 MaxNumCachedHint = Config::MaxNumCachedHint; static const uptr NumClasses = ClassesSize + 1; static_assert(NumClasses < 256, ""); @@ -212,7 +215,7 @@ return SzTable.Tab[scaledLog2(Size - 1, Config::MidSizeLog, S)]; } - static u32 getMaxCachedHint(uptr Size) { + static u16 getMaxCachedHint(uptr Size) { DCHECK_LE(Size, MaxSize); return Base::getMaxCachedHint(Size); } @@ -223,7 +226,7 @@ static const uptr MinSizeLog = 5; static const uptr MidSizeLog = 8; static const uptr MaxSizeLog = 17; - static const u32 MaxNumCachedHint = 14; + static const u16 MaxNumCachedHint = 14; static const uptr MaxBytesCachedLog = 10; static const uptr SizeDelta = 0; }; @@ -235,7 +238,7 @@ static const uptr MinSizeLog = 5; static const uptr MidSizeLog = 8; static const uptr MaxSizeLog = 17; - static const u32 MaxNumCachedHint = 12; + static const u16 MaxNumCachedHint = 12; static const uptr MaxBytesCachedLog = 10; static const uptr SizeDelta = Chunk::getHeaderSize(); }; @@ -248,7 +251,7 @@ static const uptr MinSizeLog = 4; static const uptr MidSizeLog = 6; static const uptr MaxSizeLog = 16; - static const u32 MaxNumCachedHint = 13; + static const u16 MaxNumCachedHint = 13; static const uptr MaxBytesCachedLog = 13; static constexpr u32 Classes[] = { @@ -263,7 +266,7 @@ static const uptr MinSizeLog = 4; static const uptr MidSizeLog = 7; static const uptr MaxSizeLog = 16; - static const u32 MaxNumCachedHint = 14; + static const u16 MaxNumCachedHint = 14; static const uptr MaxBytesCachedLog = 13; static constexpr u32 Classes[] = { @@ -292,7 +295,7 @@ static const uptr MinSizeLog = 4; static const uptr MidSizeLog = 8; static const uptr MaxSizeLog = 14; - static const u32 MaxNumCachedHint = 13; + static const u16 MaxNumCachedHint = 13; static const uptr MaxBytesCachedLog = 10; static const uptr SizeDelta = Chunk::getHeaderSize(); #else @@ -300,7 +303,7 @@ static const uptr MinSizeLog = 3; static const uptr MidSizeLog = 7; static const uptr MaxSizeLog = 14; - static const u32 MaxNumCachedHint = 14; + static const u16 MaxNumCachedHint = 14; static const uptr MaxBytesCachedLog = 10; static const uptr SizeDelta = Chunk::getHeaderSize(); #endif @@ -315,7 +318,7 @@ static const uptr MinSizeLog = 7; static const uptr MidSizeLog = 7; static const uptr MaxSizeLog = 7; - static const u32 MaxNumCachedHint = 8; + static const u16 MaxNumCachedHint = 8; static const uptr MaxBytesCachedLog = 10; static const uptr SizeDelta = 0; }; diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -506,7 +506,7 @@ static const scudo::uptr MinSizeLog = 10; static const scudo::uptr MidSizeLog = 10; static const scudo::uptr MaxSizeLog = 13; - static const scudo::u32 MaxNumCachedHint = 4; + static const scudo::u16 MaxNumCachedHint = 4; static const scudo::uptr MaxBytesCachedLog = 12; static const scudo::uptr SizeDelta = 0; }; diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp @@ -176,7 +176,7 @@ AllocationFailed = true; break; } - for (scudo::u32 J = 0; J < B->getCount(); J++) + for (scudo::u16 J = 0; J < B->getCount(); J++) memset(Allocator.decompactPtr(ClassId, B->get(J)), 'B', Size); Batches.push_back(B); } diff --git a/compiler-rt/lib/scudo/standalone/tests/release_test.cpp b/compiler-rt/lib/scudo/standalone/tests/release_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/release_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/release_test.cpp @@ -130,22 +130,22 @@ // Simplified version of a TransferBatch. template struct FreeBatch { - static const scudo::u32 MaxCount = SizeClassMap::MaxNumCachedHint; + static const scudo::u16 MaxCount = SizeClassMap::MaxNumCachedHint; void clear() { Count = 0; } void add(scudo::uptr P) { DCHECK_LT(Count, MaxCount); Batch[Count++] = P; } - scudo::u32 getCount() const { return Count; } - scudo::uptr get(scudo::u32 I) const { + scudo::u16 getCount() const { return Count; } + scudo::uptr get(scudo::u16 I) const { DCHECK_LE(I, Count); return Batch[I]; } FreeBatch *Next; private: - scudo::u32 Count; scudo::uptr Batch[MaxCount]; + scudo::u16 Count; }; template void testReleaseFreeMemoryToOS() { diff --git a/compiler-rt/lib/scudo/standalone/tests/size_class_map_test.cpp b/compiler-rt/lib/scudo/standalone/tests/size_class_map_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/size_class_map_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/size_class_map_test.cpp @@ -33,7 +33,7 @@ static const scudo::uptr MinSizeLog = 5; static const scudo::uptr MidSizeLog = 5; static const scudo::uptr MaxSizeLog = 5; - static const scudo::u32 MaxNumCachedHint = 0; + static const scudo::u16 MaxNumCachedHint = 0; static const scudo::uptr MaxBytesCachedLog = 0; static const scudo::uptr SizeDelta = 0; }; @@ -48,7 +48,7 @@ static const scudo::uptr MinSizeLog = 4; static const scudo::uptr MidSizeLog = 8; static const scudo::uptr MaxSizeLog = 63; - static const scudo::u32 MaxNumCachedHint = 128; + static const scudo::u16 MaxNumCachedHint = 128; static const scudo::uptr MaxBytesCachedLog = 16; static const scudo::uptr SizeDelta = 0; }; diff --git a/compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp b/compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp --- a/compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp +++ b/compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp @@ -140,7 +140,7 @@ static const uptr MinSizeLog = %zu; static const uptr MidSizeLog = %zu; static const uptr MaxSizeLog = %zu; - static const u32 MaxNumCachedHint = 14; + static const u16 MaxNumCachedHint = 14; static const uptr MaxBytesCachedLog = 14; static constexpr u32 Classes[] = {)",