diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -144,7 +144,10 @@ TSDRegistryT *getTSDRegistry() { return &TSDRegistry; } - void initCache(CacheT *Cache) { Cache->init(&Stats, &Primary); } + // The Cache must be provided zero-initialized. + void initCache(CacheT *Cache) { + Cache->initLinkerInitialized(&Stats, &Primary); + } // Release the resources used by a TSD, which involves: // - draining the local quarantine cache to the global quarantine; @@ -161,7 +164,7 @@ uptr Alignment = MinAlignment, bool ZeroContents = false) { initThreadMaybe(); - ZeroContents = ZeroContents || Options.ZeroContents; + ZeroContents |= static_cast(Options.ZeroContents); if (UNLIKELY(Alignment > MaxAlignment)) { if (Options.MayReturnNull) @@ -181,12 +184,13 @@ ((Alignment > MinAlignment) ? Alignment : Chunk::getHeaderSize()); // Takes care of extravagantly large sizes as well as integer overflows. - if (UNLIKELY(Size >= MaxAllowedMallocSize || - NeededSize >= MaxAllowedMallocSize)) { + COMPILER_CHECK(MaxAllowedMallocSize < UINTPTR_MAX - MaxAlignment); + if (UNLIKELY(Size >= MaxAllowedMallocSize)) { if (Options.MayReturnNull) return nullptr; reportAllocationSizeTooBig(Size, NeededSize, MaxAllowedMallocSize); } + DCHECK_LE(Size, NeededSize); void *Block; uptr ClassId; @@ -541,7 +545,9 @@ Chunk::UnpackedHeader NewHeader = *Header; // If the quarantine is disabled, the actual size of a chunk is 0 or larger // than the maximum allowed, we return a chunk directly to the backend. - const bool BypassQuarantine = !Quarantine.getCacheSize() || !Size || + // Logical Or can be short-circuited, which introduces unnecessary + // conditional jumps, so use bitwise Or and let the compiler be clever. + const bool BypassQuarantine = !Quarantine.getCacheSize() | !Size | (Size > Options.QuarantineMaxChunkSize); if (BypassQuarantine) { NewHeader.State = Chunk::State::Available; diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -300,10 +300,10 @@ const uptr NumberOfBlocks = RegionSize / Size; DCHECK_GT(NumberOfBlocks, 0); TransferBatch *B = nullptr; - constexpr uptr ShuffleArraySize = 48; + constexpr u32 ShuffleArraySize = 8U * TransferBatch::MaxNumCached; void *ShuffleArray[ShuffleArraySize]; u32 Count = 0; - const uptr AllocatedUser = NumberOfBlocks * Size; + const uptr AllocatedUser = Size * NumberOfBlocks; for (uptr I = Region; I < Region + AllocatedUser; I += Size) { ShuffleArray[Count++] = reinterpret_cast(I); if (Count == ShuffleArraySize) { @@ -319,6 +319,11 @@ return nullptr; } DCHECK(B); + if (!Sci->FreeList.empty()) { + Sci->FreeList.push_back(B); + B = Sci->FreeList.front(); + Sci->FreeList.pop_front(); + } DCHECK_GT(B->getCount(), 0); C->getStats().add(StatFree, AllocatedUser); diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -187,6 +187,8 @@ // Call map for user memory with at least this size. static const uptr MapSizeIncrement = 1UL << 17; + // Fill at most this number of batches from the newly map'd memory. + static const u32 MaxNumBatches = 8U; struct RegionStats { uptr PoppedBlocks; @@ -289,16 +291,18 @@ C->getStats().add(StatMapped, UserMapSize); } - const uptr NumberOfBlocks = Min( - 8UL * MaxCount, (Region->MappedUser - Region->AllocatedUser) / Size); + const u32 NumberOfBlocks = Min( + MaxNumBatches * MaxCount, + static_cast((Region->MappedUser - Region->AllocatedUser) / Size)); DCHECK_GT(NumberOfBlocks, 0); TransferBatch *B = nullptr; - constexpr uptr ShuffleArraySize = 48; + constexpr u32 ShuffleArraySize = + MaxNumBatches * TransferBatch::MaxNumCached; void *ShuffleArray[ShuffleArraySize]; u32 Count = 0; const uptr P = RegionBeg + Region->AllocatedUser; - const uptr AllocatedUser = NumberOfBlocks * Size; + const uptr AllocatedUser = Size * NumberOfBlocks; for (uptr I = P; I < P + AllocatedUser; I += Size) { ShuffleArray[Count++] = reinterpret_cast(I); if (Count == ShuffleArraySize) { @@ -314,6 +318,11 @@ return nullptr; } DCHECK(B); + if (!Region->FreeList.empty()) { + Region->FreeList.push_back(B); + B = Region->FreeList.front(); + Region->FreeList.pop_front(); + } DCHECK_GT(B->getCount(), 0); C->getStats().add(StatFree, AllocatedUser);