diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -208,6 +208,7 @@ static const uptr NumClasses = SizeClassMap::NumClasses; static const uptr RegionSize = 1UL << RegionSizeLog; static const uptr NumRegions = SCUDO_MMAP_RANGE_SIZE >> RegionSizeLog; + static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U; typedef FlatByteMap ByteMap; struct SizeClassStats { @@ -225,6 +226,8 @@ struct ALIGNED(SCUDO_CACHE_LINE_SIZE) SizeClassInfo { HybridMutex Mutex; SinglyLinkedList FreeList; + uptr CurrentRegion; + uptr CurrentRegionAllocated; SizeClassStats Stats; bool CanRelease; u32 RandState; @@ -315,21 +318,36 @@ NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId, SizeClassInfo *Sci) { - const uptr Region = allocateRegion(ClassId); - if (UNLIKELY(!Region)) - return nullptr; + uptr Region; + uptr Offset; + if (Sci->CurrentRegion) { + Region = Sci->CurrentRegion; + DCHECK_GT(Sci->CurrentRegionAllocated, 0U); + Offset = Sci->CurrentRegionAllocated; + } else { + DCHECK_EQ(Sci->CurrentRegionAllocated, 0U); + Region = allocateRegion(ClassId); + if (UNLIKELY(!Region)) + return nullptr; + Sci->CurrentRegion = Region; + Offset = 0; + } C->getStats().add(StatMapped, RegionSize); const uptr Size = getSizeByClassId(ClassId); const u32 MaxCount = TransferBatch::getMaxCached(Size); - DCHECK_GT(MaxCount, 0); - const uptr NumberOfBlocks = RegionSize / Size; - DCHECK_GT(NumberOfBlocks, 0); + DCHECK_GT(MaxCount, 0U); + const u32 NumberOfBlocks = + Min(MaxNumBatches * MaxCount, + static_cast((RegionSize - Offset) / Size)); + DCHECK_GT(NumberOfBlocks, 0U); TransferBatch *B = nullptr; - constexpr u32 ShuffleArraySize = 8U * TransferBatch::MaxNumCached; + constexpr u32 ShuffleArraySize = + MaxNumBatches * TransferBatch::MaxNumCached; void *ShuffleArray[ShuffleArraySize]; u32 Count = 0; const uptr AllocatedUser = Size * NumberOfBlocks; - for (uptr I = Region; I < Region + AllocatedUser; I += Size) { + for (uptr I = Region + Offset; I < Region + Offset + AllocatedUser; + I += Size) { ShuffleArray[Count++] = reinterpret_cast(I); if (Count == ShuffleArraySize) { if (UNLIKELY(!populateBatches(C, Sci, ClassId, &B, MaxCount, @@ -352,6 +370,13 @@ DCHECK_GT(B->getCount(), 0); C->getStats().add(StatFree, AllocatedUser); + DCHECK_LE(Sci->CurrentRegionAllocated + AllocatedUser, RegionSize); + if (RegionSize - (Sci->CurrentRegionAllocated + AllocatedUser) < Size) { + Sci->CurrentRegion = 0; + Sci->CurrentRegionAllocated = 0; + } else { + Sci->CurrentRegionAllocated += AllocatedUser; + } Sci->AllocatedUser += AllocatedUser; if (Sci->CanRelease) Sci->ReleaseInfo.LastReleaseAtNs = getMonotonicTime(); diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -214,7 +214,7 @@ // Call map for user memory with at least this size. static const uptr MapSizeIncrement = 1UL << 18; // Fill at most this number of batches from the newly map'd memory. - static const u32 MaxNumBatches = 8U; + static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U; struct RegionStats { uptr PoppedBlocks;