diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -154,7 +154,6 @@ // if `populateFreeList` succeeded, we are supposed to get free blocks. DCHECK_NE(B, nullptr); } - Sci->FreeListInfo.PoppedBlocks += B->getCount(); return B; } @@ -175,7 +174,6 @@ if (Size == 1 && !populateFreeList(C, ClassId, Sci)) return; pushBlocksImpl(C, ClassId, Sci, Array, Size); - Sci->FreeListInfo.PushedBlocks += Size; return; } @@ -201,7 +199,6 @@ ScopedLock L(Sci->Mutex); pushBlocksImpl(C, ClassId, Sci, Array, Size, SameGroup); - Sci->FreeListInfo.PushedBlocks += Size; if (ClassId != SizeClassMap::BatchClassId) releaseToOSMaybe(Sci, ClassId); } @@ -533,6 +530,7 @@ BG->PushedBlocks += Size; }; + Sci->FreeListInfo.PushedBlocks += Size; BatchGroup *Cur = Sci->FreeListInfo.BlockList.front(); if (ClassId == SizeClassMap::BatchClassId) { @@ -635,6 +633,7 @@ C->deallocate(SizeClassMap::BatchClassId, BG); } + Sci->FreeListInfo.PoppedBlocks += B->getCount(); return B; } @@ -708,6 +707,12 @@ /*SameGroup=*/true); } + // Note that `PushedBlocks` and `PoppedBlocks` are supposed to only record + // the requests from `PushBlocks` and `PopBatch` which are external + // interfaces. `populateFreeList` is the internal interface so we should set + // the values back to avoid incorrectly setting the stats. + Sci->FreeListInfo.PushedBlocks -= NumberOfBlocks; + const uptr AllocatedUser = Size * NumberOfBlocks; C->getStats().add(StatFree, AllocatedUser); DCHECK_LE(Sci->CurrentRegionAllocated + AllocatedUser, RegionSize); diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -155,10 +155,8 @@ { ScopedLock L(Region->FLLock); TransferBatch *B = popBatchImpl(C, ClassId, Region); - if (LIKELY(B)) { - Region->FreeListInfo.PoppedBlocks += B->getCount(); + if (LIKELY(B)) return B; - } } bool PrintStats = false; @@ -174,10 +172,8 @@ { ScopedLock FL(Region->FLLock); TransferBatch *B = popBatchImpl(C, ClassId, Region); - if (LIKELY(B)) { - Region->FreeListInfo.PoppedBlocks += B->getCount(); + if (LIKELY(B)) return B; - } } const bool RegionIsExhausted = Region->Exhausted; @@ -229,7 +225,6 @@ Size == 1U && Region->FreeListInfo.BlockList.empty(); if (!NeedToRefill) { pushBlocksImpl(C, SizeClassMap::BatchClassId, Region, Array, Size); - Region->FreeListInfo.PushedBlocks += Size; return; } } @@ -274,7 +269,6 @@ { ScopedLock L(Region->FLLock); pushBlocksImpl(C, ClassId, Region, Array, Size, SameGroup); - Region->FreeListInfo.PushedBlocks += Size; } // Only non-BatchClass will be here, try to release the pages in the region. @@ -677,6 +671,7 @@ BG->PushedBlocks += Size; }; + Region->FreeListInfo.PushedBlocks += Size; BatchGroup *Cur = Region->FreeListInfo.BlockList.front(); if (ClassId == SizeClassMap::BatchClassId) { @@ -779,6 +774,8 @@ C->deallocate(SizeClassMap::BatchClassId, BG); } + Region->FreeListInfo.PoppedBlocks += B->getCount(); + return B; } @@ -861,6 +858,12 @@ /*SameGroup=*/true); } + // Note that `PushedBlocks` and `PoppedBlocks` are supposed to only record + // the requests from `PushBlocks` and `PopBatch` which are external + // interfaces. `populateFreeListAndPopBatch` is the internal interface so we + // should set the values back to avoid incorrectly setting the stats. + Region->FreeListInfo.PushedBlocks -= NumberOfBlocks; + const uptr AllocatedUser = Size * NumberOfBlocks; C->getStats().add(StatFree, AllocatedUser); Region->MemMapInfo.AllocatedUser += AllocatedUser;