diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -311,6 +311,12 @@ } } + void getFragmentationInfo(ScopedString *Str) { + // TODO(chiahungduan): Organize the steps in releaseToOSMaybe() into + // functions which make the collection of fragmentation data easier. + Str->append("Fragmentation Stats: SizeClassAllocator32: Unsupported yet\n"); + } + bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { const s32 Interval = Max(Min(static_cast(Value), diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -367,6 +367,18 @@ } } + void getFragmentationInfo(ScopedString *Str) { + Str->append( + "Fragmentation Stats: SizeClassAllocator64: page size = %zu bytes\n", + getPageSizeCached()); + + for (uptr I = 1; I < NumClasses; I++) { + RegionInfo *Region = getRegionInfo(I); + ScopedLock L(Region->MMLock); + getRegionFragmentationInfo(Region, I, Str); + } + } + bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { const s32 Interval = Max(Min(static_cast(Value), @@ -951,10 +963,10 @@ if (Region->MemMapInfo.MappedUser == 0) return; const uptr BlockSize = getSizeByClassId(ClassId); - const uptr InUse = + const uptr InUseBlocks = Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks; const uptr BytesInFreeList = - Region->MemMapInfo.AllocatedUser - InUse * BlockSize; + Region->MemMapInfo.AllocatedUser - InUseBlocks * BlockSize; uptr RegionPushedBytesDelta = 0; if (BytesInFreeList >= Region->ReleaseInfo.BytesInFreeListAtLastCheckpoint) { @@ -968,13 +980,54 @@ "released: %6zuK latest pushed bytes: %6zuK region: 0x%zx (0x%zx)\n", Region->Exhausted ? "F" : " ", ClassId, getSizeByClassId(ClassId), Region->MemMapInfo.MappedUser >> 10, Region->FreeListInfo.PoppedBlocks, - Region->FreeListInfo.PushedBlocks, InUse, TotalChunks, + Region->FreeListInfo.PushedBlocks, InUseBlocks, TotalChunks, Region->ReleaseInfo.RangesReleased, Region->ReleaseInfo.LastReleasedBytes >> 10, RegionPushedBytesDelta >> 10, Region->RegionBeg, getRegionBaseByClassId(ClassId)); } + void getRegionFragmentationInfo(RegionInfo *Region, uptr ClassId, + ScopedString *Str) REQUIRES(Region->MMLock) { + const uptr BlockSize = getSizeByClassId(ClassId); + const uptr AllocatedUserEnd = + Region->MemMapInfo.AllocatedUser + Region->RegionBeg; + + SinglyLinkedList GroupsToRelease; + { + ScopedLock L(Region->FLLock); + GroupsToRelease = Region->FreeListInfo.BlockList; + Region->FreeListInfo.BlockList.clear(); + } + + FragmentationRecorder Recorder; + if (!GroupsToRelease.empty()) { + PageReleaseContext Context = + markFreeBlocks(Region, BlockSize, AllocatedUserEnd, + getCompactPtrBaseByClassId(ClassId), GroupsToRelease); + auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; }; + releaseFreeMemoryToOS(Context, Recorder, SkipRegion); + + mergeGroupsToReleaseBack(Region, GroupsToRelease); + } + + ScopedLock L(Region->FLLock); + const uptr PageSize = getPageSizeCached(); + const uptr TotalBlocks = Region->MemMapInfo.AllocatedUser / BlockSize; + const uptr InUseBlocks = + Region->FreeListInfo.PoppedBlocks - Region->FreeListInfo.PushedBlocks; + const uptr AllocatedPagesCount = + roundUp(Region->MemMapInfo.AllocatedUser, PageSize) / PageSize; + const uptr InUsePages = + AllocatedPagesCount - Recorder.getReleasedPagesCount(); + const uptr InUseBytes = InUsePages * PageSize; + + Str->append(" %02zu (%6zu): inuse/total blocks: %6zu/%6zu inuse/total " + "pages: %6zu/%6zu inuse bytes: %6zuK\n", + ClassId, BlockSize, InUseBlocks, TotalBlocks, InUsePages, + AllocatedPagesCount, InUseBytes >> 10); + } + NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId, ReleaseToOS ReleaseType = ReleaseToOS::Normal) REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) { diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h --- a/compiler-rt/lib/scudo/standalone/release.h +++ b/compiler-rt/lib/scudo/standalone/release.h @@ -80,6 +80,21 @@ MapPlatformData *Data = nullptr; }; +class FragmentationRecorder { +public: + FragmentationRecorder() = default; + + uptr getReleasedPagesCount() const { return ReleasedPagesCount; } + + void releasePageRangeToOS(uptr From, uptr To) { + DCHECK_EQ((To - From) % getPageSizeCached(), 0U); + ReleasedPagesCount += (To - From) / getPageSizeCached(); + } + +private: + uptr ReleasedPagesCount = 0; +}; + // A buffer pool which holds a fixed number of static buffers for fast buffer // allocation. If the request size is greater than `StaticBufferSize`, it'll // delegate the allocation to map(). diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp @@ -335,6 +335,7 @@ Allocator->releaseToOS(scudo::ReleaseToOS::Force); scudo::ScopedString Str; Allocator->getStats(&Str); + Allocator->getFragmentationInfo(&Str); Str.output(); }