diff --git a/compiler-rt/lib/scudo/standalone/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/CMakeLists.txt --- a/compiler-rt/lib/scudo/standalone/CMakeLists.txt +++ b/compiler-rt/lib/scudo/standalone/CMakeLists.txt @@ -71,6 +71,7 @@ list.h local_cache.h memtag.h + mem_map.h mutex.h options.h platform.h @@ -103,6 +104,7 @@ flags.cpp fuchsia.cpp linux.cpp + mem_map.cpp release.cpp report.cpp rss_limit_checker.cpp diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.h b/compiler-rt/lib/scudo/standalone/allocator_config.h --- a/compiler-rt/lib/scudo/standalone/allocator_config.h +++ b/compiler-rt/lib/scudo/standalone/allocator_config.h @@ -12,6 +12,7 @@ #include "combined.h" #include "common.h" #include "flags.h" +#include "mem_map.h" #include "primary32.h" #include "primary64.h" #include "secondary.h" @@ -68,6 +69,7 @@ struct DefaultConfig { using SizeClassMap = DefaultSizeClassMap; + using MapReserver = DefaultMapReserver; static const bool MaySupportMemoryTagging = true; #if SCUDO_CAN_USE_PRIMARY64 @@ -99,6 +101,7 @@ }; struct AndroidConfig { using SizeClassMap = AndroidSizeClassMap; + using MapReserver = DefaultMapReserver; static const bool MaySupportMemoryTagging = true; #if SCUDO_CAN_USE_PRIMARY64 @@ -132,6 +135,7 @@ struct AndroidSvelteConfig { using SizeClassMap = SvelteSizeClassMap; + using MapReserver = DefaultMapReserver; static const bool MaySupportMemoryTagging = false; #if SCUDO_CAN_USE_PRIMARY64 @@ -166,6 +170,7 @@ #if SCUDO_CAN_USE_PRIMARY64 struct FuchsiaConfig { using SizeClassMap = FuchsiaSizeClassMap; + using MapReserver = DefaultMapReserver; static const bool MaySupportMemoryTagging = false; typedef SizeClassAllocator64 Primary; @@ -185,6 +190,7 @@ struct TrustyConfig { using SizeClassMap = TrustySizeClassMap; + using MapReserver = DefaultMapReserver; static const bool MaySupportMemoryTagging = false; typedef SizeClassAllocator64 Primary; diff --git a/compiler-rt/lib/scudo/standalone/mem_map.h b/compiler-rt/lib/scudo/standalone/mem_map.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/scudo/standalone/mem_map.h @@ -0,0 +1,129 @@ +//===-- mem_map.h -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_MEM_MAP_H_ +#define SCUDO_MEM_MAP_H_ + +#include "common.h" +#include "internal_defs.h" + +// TODO: This is only used for `MapPlatformData`. Remove these includes when we +// have all three platform specific `MemMap` and `MapReserver` implementations. +#include "fuchsia.h" +#include "linux.h" +#include "trusty.h" + +namespace scudo { + +// In Scudo, each page request from the system will be tracked by a +// platform-specific `MemMap` implementation. For example, the `DefaultMemMap` +// below records the base of the contiguous pages and its size. With these +// information, we are able to validate various page operations before it goes +// beyond the first place. + +// This class contains the required interfaces for a MemMap implementation. +template class MemMapBase { +public: + constexpr MemMapBase() {} + + // This is used to map a new contiguous of pages. Whether the holding pages + // (if nay) will be released is implementation defined. + bool map(uptr Addr, uptr Size, const char *Name, uptr Flags = 0) { + return static_cast(this)->mapImpl(Addr, Size, Name, Flags); + } + + // This is used to remap a mapped range. For example, we have reserved several + // pages and then we want to remap them with different flags. The validation + // of the remapping range is done by each implementation. + bool remap(uptr Addr, uptr Size, const char *Name, uptr Flags = 0) { + return static_cast(this)->remapImpl(Addr, Size, Name, Flags); + } + + // Release a contiguous physical pages back to the OS. Note that only physical + // pages can be released. Any releasing of virtual pages may lead to undefined + // behavior. + void releasePagesToOS(uptr From, uptr Size) { + return static_cast(this)->releasePagesToOSImpl(From, Size); + } + + uptr getMapBase() { return static_cast(this)->getMapBaseImpl(); } + + bool isAllocated() { return static_cast(this)->isAllocatedImpl(); } +}; + +// `MapReserver` reserves a contiguous pages for later dispatch. This ensures +// the memory map address is in a known range. +template +class MapReserver : public MemMapBase { +public: + using Base = MemMapBase; + using MemMapT = GeneralMapT; + + using Base::getMapBase; + using Base::isAllocated; + + // Reserve a chunk of memory. Note that if there's already a reserved range, + // the behavior is undefined. + void reserveMemory(uptr Addr, uptr Size, const char *Name) { + static_cast(this)->reserveMemoryImpl(Addr, Size, Name); + } + + // Prepare a MemMap from the reserved range. Note that any fragmentation of + // the reserved pages is managed by each implementation. + MemMapT extractMemory(uptr Addr, uptr Size) { + return static_cast(this)->extractMemoryImpl(Addr, Size); + } +}; + +// This will be deprecated when every allocator has been supported by each +// platform's `MemMap` implementation. +class DefaultMemMap : public MemMapBase { +public: + DefaultMemMap() = default; + DefaultMemMap(uptr Base, uptr Capacity) + : MapBase(Base), MapCapacity(Capacity) {} + + bool mapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags = 0); + bool remapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags = 0); + void releasePagesToOSImpl(uptr From, uptr Size); + + void setMapPlatformData(MapPlatformData &NewData) { Data = NewData; } + + uptr getMapBaseImpl() { return MapBase; } + uptr getCapacity() { return MapCapacity; } + + bool isAllocatedImpl() { return MapBase != 0; } + +private: + uptr MapBase = 0; + uptr MapCapacity = 0; + MapPlatformData Data = {}; +}; + +// This will be deprecated when every allocator has been supported by each +// platform's `MemMap` implementation. +class DefaultMapReserver + : public MapReserver { +public: + void reserveMemoryImpl(uptr Addr, uptr Size, const char *Name); + + MemMapT extractMemoryImpl(uptr Addr, uptr Size); + + uptr getMapBaseImpl() { return MapBase; } + + bool isAllocatedImpl() { return MapBase != 0; } + +private: + uptr MapBase = 0; + uptr MapCapacity = 0; + MapPlatformData Data = {}; +}; + +} // namespace scudo + +#endif // SCUDO_MEM_MAP_H_ diff --git a/compiler-rt/lib/scudo/standalone/mem_map.cpp b/compiler-rt/lib/scudo/standalone/mem_map.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/scudo/standalone/mem_map.cpp @@ -0,0 +1,54 @@ +//===-- mem_map.cpp ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mem_map.h" + +#include "common.h" + +namespace scudo { + +bool DefaultMemMap::mapImpl(uptr Addr, uptr Size, const char *Name, + uptr Flags) { + return ::scudo::map(reinterpret_cast(Addr), Size, Name, Flags, &Data); +} + +bool DefaultMemMap::remapImpl(uptr Addr, uptr Size, const char *Name, + uptr Flags) { + DCHECK_GE(Addr, MapBase); + DCHECK_LE(Addr + Size, MapBase + MapCapacity); + return ::scudo::map(reinterpret_cast(Addr), Size, Name, Flags, &Data); +} + +void DefaultMemMap::releasePagesToOSImpl(uptr From, uptr Size) { + DCHECK_GE(From, MapBase); + DCHECK_LE(From + Size, MapBase + MapCapacity); + return ::scudo::releasePagesToOS(MapBase, From - MapBase, Size, &Data); +} + +void DefaultMapReserver::reserveMemoryImpl(uptr Addr, uptr Size, + const char *Name) { + void *Reserved = ::scudo::map(reinterpret_cast(Addr), Size, Name, + MAP_NOACCESS, &Data); + if (Reserved == nullptr) + return; + + MapBase = reinterpret_cast(Reserved); + MapCapacity = Size; +} + +DefaultMapReserver::MemMapT DefaultMapReserver::extractMemoryImpl(uptr Addr, + uptr Size) { + DCHECK(isAllocated()); + DCHECK_GE(Addr, MapBase); + DCHECK_LE(Addr + Size, MapBase + MapCapacity); + DefaultMapReserver::MemMapT NewMap(Addr, Size); + NewMap.setMapPlatformData(Data); + return NewMap; +} + +} // namespace scudo diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -13,6 +13,7 @@ #include "common.h" #include "list.h" #include "local_cache.h" +#include "mem_map.h" #include "memtag.h" #include "options.h" #include "release.h" @@ -45,6 +46,8 @@ template class SizeClassAllocator64 { public: typedef typename Config::PrimaryCompactPtrT CompactPtrT; + typedef typename Config::MapReserver MapReserverT; + typedef typename MapReserverT::MemMapT MemMapT; static const uptr CompactPtrScale = Config::PrimaryCompactPtrScale; static const uptr GroupSizeLog = Config::PrimaryGroupSizeLog; static const uptr GroupScale = GroupSizeLog - CompactPtrScale; @@ -67,8 +70,10 @@ DCHECK_EQ(PrimaryBase, 0U); // Reserve the space required for the Primary. - PrimaryBase = reinterpret_cast(map( - nullptr, PrimarySize, "scudo:primary_reserve", MAP_NOACCESS, &Data)); + PrimaryMaps.reserveMemory(/*Addr=*/0U, PrimarySize, + "scudo:primary_reserve"); + PrimaryBase = PrimaryMaps.getMapBase(); + DCHECK_NE(PrimaryBase, 0U); u32 Seed; const u64 Time = getMonotonicTimeFast(); @@ -441,6 +446,7 @@ // Bytes allocated for user memory. uptr AllocatedUser GUARDED_BY(Mutex) = 0; MapPlatformData Data GUARDED_BY(Mutex) = {}; + MemMapT MemMap = {}; ReleaseToOsInfo ReleaseInfo GUARDED_BY(Mutex) = {}; bool Exhausted GUARDED_BY(Mutex) = false; }; @@ -450,7 +456,10 @@ }; static_assert(sizeof(RegionInfo) % SCUDO_CACHE_LINE_SIZE == 0, ""); + // TODO: `PrimaryBase` can be obtained from PrimaryMaps. This needs to be + // deprecated. uptr PrimaryBase = 0; + MapReserverT PrimaryMaps = {}; // The minimum size of pushed blocks that we will try to release the pages in // that size class. uptr SmallerBlockReleasePageDelta = 0; @@ -728,14 +737,18 @@ Region->Exhausted = true; return false; } - if (MappedUser == 0) - Region->Data = Data; - if (UNLIKELY(!map( - reinterpret_cast(RegionBeg + MappedUser), MapSize, - "scudo:primary", + // TODO: Consider allocating MemMap in init(). + if (!Region->MemMap.isAllocated()) { + Region->MemMap = PrimaryMaps.extractMemory( + getRegionBaseByClassId(ClassId), RegionSize); + } + DCHECK(Region->MemMap.isAllocated()); + + if (UNLIKELY(!Region->MemMap.remap( + RegionBeg + MappedUser, MapSize, "scudo:primary", MAP_ALLOWNOMEM | MAP_RESIZABLE | - (useMemoryTagging(Options.load()) ? MAP_MEMTAG : 0), - &Region->Data))) { + (useMemoryTagging(Options.load()) ? MAP_MEMTAG + : 0)))) { return false; } Region->MappedUser += MapSize; @@ -991,7 +1004,8 @@ const uptr ReleaseRangeSize = ReleaseEnd - ReleaseBase; const uptr ReleaseOffset = ReleaseBase - Region->RegionBeg; - ReleaseRecorder Recorder(Region->RegionBeg, ReleaseOffset, &Region->Data); + RegionReleaseRecorder Recorder(&Region->MemMap, Region->RegionBeg, + ReleaseOffset); PageReleaseContext Context(BlockSize, /*NumberOfRegions=*/1U, ReleaseRangeSize, ReleaseOffset); diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h --- a/compiler-rt/lib/scudo/standalone/release.h +++ b/compiler-rt/lib/scudo/standalone/release.h @@ -11,11 +11,42 @@ #include "common.h" #include "list.h" +#include "mem_map.h" #include "mutex.h" #include "thread_annotations.h" namespace scudo { +template class RegionReleaseRecorder { +public: + RegionReleaseRecorder(MemMapT *RegionMemMap, uptr Base, uptr Offset = 0) + : RegionMemMap(RegionMemMap), Base(Base), Offset(Offset) {} + + uptr getReleasedRangesCount() const { return ReleasedRangesCount; } + + uptr getReleasedBytes() const { return ReleasedBytes; } + + uptr getBase() const { return Base; } + + // Releases [From, To) range of pages back to OS. Note that `From` and `To` + // are offseted from `Base` + Offset. + void releasePageRangeToOS(uptr From, uptr To) { + const uptr Size = To - From; + RegionMemMap->releasePagesToOS(getBase() + Offset + From, Size); + ReleasedRangesCount++; + ReleasedBytes += Size; + } + +private: + uptr ReleasedRangesCount = 0; + uptr ReleasedBytes = 0; + MemMapT *RegionMemMap = nullptr; + uptr Base = 0; + // The release offset from Base. This is used when we know a given range after + // Base will not be released. + uptr Offset = 0; +}; + class ReleaseRecorder { public: ReleaseRecorder(uptr Base, uptr Offset = 0, MapPlatformData *Data = nullptr) diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -518,6 +518,7 @@ // Tiny allocator, its Primary only serves chunks of four sizes. using SizeClassMap = scudo::FixedSizeClassMap; + using MapReserver = scudo::DefaultMapReserver; typedef scudo::SizeClassAllocator64 Primary; static const scudo::uptr PrimaryRegionSizeLog = DeathRegionSizeLog; static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp @@ -8,6 +8,7 @@ #include "tests/scudo_unit_test.h" +#include "mem_map.h" #include "primary32.h" #include "primary64.h" #include "size_class_map.h" @@ -91,6 +92,7 @@ template struct Config : public BaseConfig { using SizeClassMap = SizeClassMapT; + using MapReserver = scudo::DefaultMapReserver; }; template @@ -169,6 +171,7 @@ struct SmallRegionsConfig { using SizeClassMap = scudo::DefaultSizeClassMap; + using MapReserver = scudo::DefaultMapReserver; static const scudo::uptr PrimaryRegionSizeLog = 21U; static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;