diff --git a/compiler-rt/lib/scudo/standalone/common.h b/compiler-rt/lib/scudo/standalone/common.h --- a/compiler-rt/lib/scudo/standalone/common.h +++ b/compiler-rt/lib/scudo/standalone/common.h @@ -156,21 +156,20 @@ // by the function implementation. The Data parameter allows to pass opaque // platform specific data to the function. // Returns nullptr on error or dies if MAP_ALLOWNOMEM is not specified. -void *map(void *Addr, uptr Size, const char *Name, uptr Flags = 0, - MapPlatformData *Data = nullptr); +void *map(void *Addr, uptr Size, const char *Name, uptr Flags, + MapPlatformData *Data); // Indicates that we are getting rid of the whole mapping, which might have // further consequences on Data, depending on the platform. #define UNMAP_ALL (1U << 0) -void unmap(void *Addr, uptr Size, uptr Flags = 0, - MapPlatformData *Data = nullptr); +void unmap(void *Addr, uptr Size, uptr Flags, MapPlatformData *Data); void setMemoryPermission(uptr Addr, uptr Size, uptr Flags, - MapPlatformData *Data = nullptr); + MapPlatformData *Data); void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size, - MapPlatformData *Data = nullptr); + MapPlatformData *Data); // Internal map & unmap fatal error. This must not call map(). SizeIfOOM shall // hold the requested size on an out-of-memory error, 0 otherwise. diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -85,9 +85,11 @@ } void unmapTestOnly() { - while (NumberOfStashedRegions > 0) - unmap(reinterpret_cast(RegionsStash[--NumberOfStashedRegions]), - RegionSize); + while (NumberOfStashedRegions > 0) { + NumberOfStashedRegions -= 1; + unmap(reinterpret_cast(RegionsStash[NumberOfStashedRegions]), + RegionSize, 0, &StashMapData[NumberOfStashedRegions]); + } uptr MinRegionIndex = NumRegions, MaxRegionIndex = 0; for (uptr I = 0; I < NumClasses; I++) { SizeClassInfo *Sci = getSizeClassInfo(I); @@ -98,8 +100,10 @@ *Sci = {}; } for (uptr I = MinRegionIndex; I < MaxRegionIndex; I++) - if (PossibleRegions[I]) - unmap(reinterpret_cast(I * RegionSize), RegionSize); + if (PossibleRegions[I]) { + unmap(reinterpret_cast(I * RegionSize), RegionSize, 0, + &RegionsMapData[I]); + } PossibleRegions.unmapTestOnly(); } @@ -276,41 +280,49 @@ return Id; } - uptr allocateRegionSlow() { + uptr allocateRegionSlow(MapPlatformData *MapData) { uptr MapSize = 2 * RegionSize; const uptr MapBase = reinterpret_cast( - map(nullptr, MapSize, "scudo:primary", MAP_ALLOWNOMEM)); + map(nullptr, MapSize, "scudo:primary", MAP_ALLOWNOMEM, MapData)); if (!MapBase) return 0; const uptr MapEnd = MapBase + MapSize; uptr Region = MapBase; if (isAligned(Region, RegionSize)) { ScopedLock L(RegionsStashMutex); - if (NumberOfStashedRegions < MaxStashedRegions) - RegionsStash[NumberOfStashedRegions++] = MapBase + RegionSize; - else + if (NumberOfStashedRegions < MaxStashedRegions) { + RegionsStash[NumberOfStashedRegions] = MapBase + RegionSize; + StashMapData[NumberOfStashedRegions] = *MapData; + NumberOfStashedRegions += 1; + } else MapSize = RegionSize; } else { Region = roundUpTo(MapBase, RegionSize); - unmap(reinterpret_cast(MapBase), Region - MapBase); + unmap(reinterpret_cast(MapBase), Region - MapBase, 0, MapData); MapSize = RegionSize; } const uptr End = Region + MapSize; if (End != MapEnd) - unmap(reinterpret_cast(End), MapEnd - End); + unmap(reinterpret_cast(End), MapEnd - End, 0, MapData); return Region; } uptr allocateRegion(SizeClassInfo *Sci, uptr ClassId) { DCHECK_LT(ClassId, NumClasses); + MapPlatformData MapData; uptr Region = 0; { ScopedLock L(RegionsStashMutex); - if (NumberOfStashedRegions > 0) - Region = RegionsStash[--NumberOfStashedRegions]; + if (NumberOfStashedRegions > 0) { + NumberOfStashedRegions -= 1; + Region = RegionsStash[NumberOfStashedRegions]; + MapData = RegionsMapData[NumberOfStashedRegions]; + } + } + if (!Region) { + MapData = {}; + Region = allocateRegionSlow(&MapData); } - if (!Region) - Region = allocateRegionSlow(); if (LIKELY(Region)) { // Sci->Mutex is held by the caller, updating the Min/Max is safe. const uptr RegionIndex = computeRegionId(Region); @@ -319,6 +331,7 @@ if (RegionIndex > Sci->MaxRegionIndex) Sci->MaxRegionIndex = RegionIndex; PossibleRegions.set(RegionIndex, static_cast(ClassId + 1U)); + RegionsMapData[RegionIndex] = MapData; } return Region; } @@ -469,7 +482,8 @@ uptr TotalReleasedBytes = 0; const uptr Base = First * RegionSize; const uptr NumberOfRegions = Last - First + 1U; - ReleaseRecorder Recorder(Base); + ReleaseRecorder Recorder(Base, RegionSize, &RegionsMapData[First], + NumberOfRegions); auto SkipRegion = [this, First, ClassId](uptr RegionIndex) { return (PossibleRegions[First + RegionIndex] - 1U) != ClassId; }; @@ -500,6 +514,8 @@ HybridMutex RegionsStashMutex; uptr NumberOfStashedRegions = 0; uptr RegionsStash[MaxStashedRegions] = {}; + [[no_unique_address]] MapPlatformData StashMapData[MaxStashedRegions] = {}; + [[no_unique_address]] MapPlatformData RegionsMapData[NumRegions] = {}; }; } // namespace scudo diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -466,7 +466,7 @@ } } - ReleaseRecorder Recorder(Region->RegionBeg, &Region->Data); + ReleaseRecorder Recorder(Region->RegionBeg, RegionSize, &Region->Data, 1); const uptr CompactPtrBase = getCompactPtrBaseByClassId(ClassId); auto DecompactPtr = [CompactPtrBase](CompactPtrT CompactPtr) { return decompactPtrInternal(CompactPtrBase, CompactPtr); diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h --- a/compiler-rt/lib/scudo/standalone/release.h +++ b/compiler-rt/lib/scudo/standalone/release.h @@ -9,16 +9,18 @@ #ifndef SCUDO_RELEASE_H_ #define SCUDO_RELEASE_H_ -#include "common.h" -#include "list.h" -#include "mutex.h" +#include "internal/scudo/common.h" +#include "internal/scudo/list.h" +#include "internal/scudo/mutex.h" namespace scudo { class ReleaseRecorder { public: - ReleaseRecorder(uptr Base, MapPlatformData *Data = nullptr) - : Base(Base), Data(Data) {} + ReleaseRecorder(uptr _Base, uptr _RegionSize, MapPlatformData *_Data, + uptr _DataSize) + : Base(_Base), RegionSize(_RegionSize), Data(_Data), DataSize(_DataSize) { + } uptr getReleasedRangesCount() const { return ReleasedRangesCount; } @@ -29,7 +31,9 @@ // Releases [From, To) range of pages back to OS. void releasePageRangeToOS(uptr From, uptr To) { const uptr Size = To - From; - releasePagesToOS(Base, From, Size, Data); + const uptr Idx = From / RegionSize; + CHECK_LT(Idx, DataSize); + releasePagesToOS(Base, From, Size, &Data[Idx]); ReleasedRangesCount++; ReleasedBytes += Size; } @@ -38,7 +42,9 @@ uptr ReleasedRangesCount = 0; uptr ReleasedBytes = 0; uptr Base = 0; + uptr RegionSize = 0; MapPlatformData *Data = nullptr; + uptr DataSize = 0; }; // A packed array of Counters. Each counter occupies 2^N bits, enough to store