diff --git a/compiler-rt/lib/scudo/standalone/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/CMakeLists.txt --- a/compiler-rt/lib/scudo/standalone/CMakeLists.txt +++ b/compiler-rt/lib/scudo/standalone/CMakeLists.txt @@ -71,6 +71,7 @@ list.h local_cache.h memtag.h + mem_map.h mutex.h options.h platform.h @@ -103,6 +104,7 @@ flags.cpp fuchsia.cpp linux.cpp + mem_map.cpp release.cpp report.cpp rss_limit_checker.cpp diff --git a/compiler-rt/lib/scudo/standalone/mem_map.h b/compiler-rt/lib/scudo/standalone/mem_map.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/scudo/standalone/mem_map.h @@ -0,0 +1,189 @@ +//===-- mem_map.h -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_MEM_MAP_H_ +#define SCUDO_MEM_MAP_H_ + +#include "common.h" +#include "internal_defs.h" + +// TODO: This is only used for `MapPlatformData`. Remove these includes when we +// have all three platform specific `MemMap` and `ReservedMap` implementations. +#include "fuchsia.h" +#include "linux.h" +#include "trusty.h" + +namespace scudo { + +// In Scudo, each page request from the system will be tracked by a +// platform-specific `MemMap` implementation. For example, the `DefaultMemMap` +// below records the base of the contiguous pages and its size. With these +// information, we are able to validate various page operations before it goes +// beyond the first place. + +// A derived of `MemMapBase` is supposed to be a handle of a contiguous pages +// and `MemMapBase` contains the interfaces that the derived usually uses. Note +// that this class doesn't have any data member even for the map base and its +// size. The reason for that is, different platforms may have its own data +// structure to store those things. As a result, we simply list the common +// interfaces for all platforms. This is implemented in CRTP, so for each +// implementation, they have to implement the following functions with name +// suffixing with `Impl`. +template class MemMapBase { +public: + constexpr MemMapBase() {} + + // This is used to map a new contiguous pages. Whether the holding pages + // (if nay) will be released is implementation defined. + bool map(uptr Addr, uptr Size, const char *Name, uptr Flags = 0) { + return static_cast(this)->mapImpl(Addr, Size, Name, Flags); + } + + // This is used to unmap partial/full pages from the beginning or the end. + // I.e., the result pages are expected to be still contiguous. + void unmap(uptr Addr, uptr Size, uptr Flags = 0) { + DCHECK((Addr == getMapBase()) || + (Addr + Size == getMapBase() + getMapCapacity())); + static_cast(this)->unmapImpl(Addr, Size, Flags); + } + + // This is used to remap a mapped range. For example, we have reserved several + // pages and then we want to remap them with different flags. + bool remap(uptr Addr, uptr Size, const char *Name, uptr Flags = 0) { + DCHECK(isAllocated()); + DCHECK((Addr >= getMapBase()) || + (Addr + Size <= getMapBase() + getMapCapacity())); + return static_cast(this)->remapImpl(Addr, Size, Name, Flags); + } + + // Release a contiguous physical pages back to the OS. Note that only physical + // pages are supposed to be released. Any release of virtual pages may lead + // to undefined behavior. + void releasePagesToOS(uptr From, uptr Size) { + DCHECK(isAllocated()); + DCHECK((From >= getMapBase()) || + (From + Size <= getMapBase() + getMapCapacity())); + return static_cast(this)->releasePagesToOSImpl(From, Size); + } + // This is similar to the above one except that any subsequent access to the + // released pages will return zero-filled pages. + void releaseAndZeroPagesToOS(uptr From, uptr Size) { + DCHECK(isAllocated()); + DCHECK((From >= getMapBase()) || + (From + Size <= getMapBase() + getMapCapacity())); + return static_cast(this)->releaseAndZeroPagesToOSImpl(From, + Size); + } + + uptr getMapBase() { return static_cast(this)->getMapBaseImpl(); } + uptr getMapCapacity() { + return static_cast(this)->getMapCapacityImpl(); + } + + bool isAllocated() { return getMapBase() != 0U; } +}; + +// `ReservedMap` is a special MemMap which still manages a contiguous pages. The +// difference is that it reserves a contiguous pages for later dispatch. On some +// platforms, this may be a cheaper operation than a general map() call. +// Therefore, here we add two functions, `reserve` and `allocate` to fulfill the +// use cases. This is useful if you want to ensure the MemMaps are managed in a +// given range. It can be viewed as a sub-page-allocator. +template +class ReservedMap : public MemMapBase { +public: + using Base = MemMapBase; + using MemMapT = GeneralMapT; + + // Note that `map`/`remap`/`releasePagesToOs` are supposed not to be used in + // `ReservedMap`. Leave them unimplemented intentionally here. The followings + // are the required functions and the derived has to implement them. + using Base::getMapBase; + using Base::getMapCapacity; + using Base::isAllocated; + // This is only used for test. + using Base::unmap; + + // Reserve a chunk of memory. Note that if there's already a reserved range, + // the behavior is undefined. + void reserve(uptr Addr, uptr Size, const char *Name) { + static_cast(this)->reserveImpl(Addr, Size, Name); + } + + // Prepare a MemMap from the reserved range. Note that any fragmentation of + // the reserved pages is managed by each implementation. + MemMapT allocate(uptr Addr, uptr Size) { + DCHECK(isAllocated()); + DCHECK((Addr >= getMapBase()) || + (Addr + Size <= getMapBase() + getMapCapacity())); + return static_cast(this)->allocateImpl(Addr, Size); + } +}; + +// This will be deprecated when every allocator has been supported by each +// platform's `MemMap` implementation. +class DefaultMemMap final : public MemMapBase { +public: + constexpr DefaultMemMap() = default; + DefaultMemMap(uptr Base, uptr Capacity) + : MapBase(Base), MapCapacity(Capacity) {} + + // Impls for base functions. + bool mapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags = 0); + void unmapImpl(uptr Addr, uptr Size, uptr Flags = 0); + bool remapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags = 0); + void releasePagesToOSImpl(uptr From, uptr Size) { + return releaseAndZeroPagesToOSImpl(From, Size); + } + void releaseAndZeroPagesToOSImpl(uptr From, uptr Size); + uptr getMapBaseImpl() { return MapBase; } + uptr getMapCapacityImpl() { return MapCapacity; } + + void setMapPlatformData(MapPlatformData &NewData) { Data = NewData; } + +private: + uptr MapBase = 0; + uptr MapCapacity = 0; + MapPlatformData Data = {}; +}; + +// This will be deprecated when every allocator has been supported by each +// platform's `MemMap` implementation. +class DefaultReservedMap final + : public ReservedMap { +public: + // The following threes are the Impls for function in `MemMapBase`. + uptr getMapBaseImpl() { return MapBase; } + uptr getMapCapacityImpl() { return MapCapacity; } + // This is only used for test. + void unmapImpl(uptr Addr, uptr Size, uptr Flags = 0); + + // These two are specific to `ReservedMap`. + void reserveImpl(uptr Addr, uptr Size, const char *Name); + MemMapT allocateImpl(uptr Addr, uptr Size); + +private: + uptr MapBase = 0; + uptr MapCapacity = 0; + MapPlatformData Data = {}; +}; + +#if SCUDO_LINUX +using ReservedMapT = DefaultReservedMap; +#elif SCUDO_FUCHSIA +using ReservedMapT = DefaultReservedMap; +#elif SCUDO_TRUSTY +using ReservedMapT = DefaultReservedMap; +#else +#error \ + "Unsupported platform, please implement the ReservedMap for your platform!" +#endif + +} // namespace scudo + +#endif // SCUDO_MEM_MAP_H_ diff --git a/compiler-rt/lib/scudo/standalone/mem_map.cpp b/compiler-rt/lib/scudo/standalone/mem_map.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/scudo/standalone/mem_map.cpp @@ -0,0 +1,54 @@ +//===-- mem_map.cpp ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mem_map.h" + +#include "common.h" + +namespace scudo { + +bool DefaultMemMap::mapImpl(uptr Addr, uptr Size, const char *Name, + uptr Flags) { + return ::scudo::map(reinterpret_cast(Addr), Size, Name, Flags, &Data); +} + +void DefaultMemMap::unmapImpl(uptr Addr, uptr Size, uptr Flags) { + ::scudo::unmap(reinterpret_cast(Addr), Size, Flags, &Data); +} + +bool DefaultMemMap::remapImpl(uptr Addr, uptr Size, const char *Name, + uptr Flags) { + return ::scudo::map(reinterpret_cast(Addr), Size, Name, Flags, &Data); +} + +void DefaultMemMap::releaseAndZeroPagesToOSImpl(uptr From, uptr Size) { + return ::scudo::releasePagesToOS(MapBase, From - MapBase, Size, &Data); +} + +void DefaultReservedMap::unmapImpl(uptr Addr, uptr Size, uptr Flags) { + ::scudo::unmap(reinterpret_cast(Addr), Size, Flags, &Data); +} + +void DefaultReservedMap::reserveImpl(uptr Addr, uptr Size, const char *Name) { + void *Reserved = ::scudo::map(reinterpret_cast(Addr), Size, Name, + MAP_NOACCESS, &Data); + if (Reserved == nullptr) + return; + + MapBase = reinterpret_cast(Reserved); + MapCapacity = Size; +} + +DefaultReservedMap::MemMapT DefaultReservedMap::allocateImpl(uptr Addr, + uptr Size) { + DefaultReservedMap::MemMapT NewMap(Addr, Size); + NewMap.setMapPlatformData(Data); + return NewMap; +} + +} // namespace scudo diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -13,6 +13,7 @@ #include "common.h" #include "list.h" #include "local_cache.h" +#include "mem_map.h" #include "memtag.h" #include "options.h" #include "release.h" @@ -45,6 +46,7 @@ template class SizeClassAllocator64 { public: typedef typename Config::PrimaryCompactPtrT CompactPtrT; + typedef ReservedMapT::MemMapT MemMapT; static const uptr CompactPtrScale = Config::PrimaryCompactPtrScale; static const uptr GroupSizeLog = Config::PrimaryGroupSizeLog; static const uptr GroupScale = GroupSizeLog - CompactPtrScale; @@ -67,8 +69,9 @@ DCHECK_EQ(PrimaryBase, 0U); // Reserve the space required for the Primary. - PrimaryBase = reinterpret_cast(map( - nullptr, PrimarySize, "scudo:primary_reserve", MAP_NOACCESS, &Data)); + PrimaryMaps.reserve(/*Addr=*/0U, PrimarySize, "scudo:primary_reserve"); + PrimaryBase = PrimaryMaps.getMapBase(); + DCHECK_NE(PrimaryBase, 0U); u32 Seed; const u64 Time = getMonotonicTimeFast(); @@ -134,8 +137,7 @@ *Region = {}; } if (PrimaryBase) - unmap(reinterpret_cast(PrimaryBase), PrimarySize, UNMAP_ALL, - &Data); + PrimaryMaps.unmap(PrimaryBase, PrimarySize, UNMAP_ALL); PrimaryBase = 0U; } @@ -440,7 +442,7 @@ uptr MappedUser GUARDED_BY(Mutex) = 0; // Bytes allocated for user memory. uptr AllocatedUser GUARDED_BY(Mutex) = 0; - MapPlatformData Data GUARDED_BY(Mutex) = {}; + MemMapT MemMap = {}; ReleaseToOsInfo ReleaseInfo GUARDED_BY(Mutex) = {}; bool Exhausted GUARDED_BY(Mutex) = false; }; @@ -450,11 +452,13 @@ }; static_assert(sizeof(RegionInfo) % SCUDO_CACHE_LINE_SIZE == 0, ""); + // TODO: `PrimaryBase` can be obtained from PrimaryMaps. This needs to be + // deprecated. uptr PrimaryBase = 0; + ReservedMapT PrimaryMaps = {}; // The minimum size of pushed blocks that we will try to release the pages in // that size class. uptr SmallerBlockReleasePageDelta = 0; - MapPlatformData Data = {}; atomic_s32 ReleaseToOsIntervalMs = {}; alignas(SCUDO_CACHE_LINE_SIZE) RegionInfo RegionInfoArray[NumClasses]; @@ -728,14 +732,18 @@ Region->Exhausted = true; return false; } - if (MappedUser == 0) - Region->Data = Data; - if (UNLIKELY(!map( - reinterpret_cast(RegionBeg + MappedUser), MapSize, - "scudo:primary", + // TODO: Consider allocating MemMap in init(). + if (!Region->MemMap.isAllocated()) { + Region->MemMap = + PrimaryMaps.allocate(getRegionBaseByClassId(ClassId), RegionSize); + } + DCHECK(Region->MemMap.isAllocated()); + + if (UNLIKELY(!Region->MemMap.remap( + RegionBeg + MappedUser, MapSize, "scudo:primary", MAP_ALLOWNOMEM | MAP_RESIZABLE | - (useMemoryTagging(Options.load()) ? MAP_MEMTAG : 0), - &Region->Data))) { + (useMemoryTagging(Options.load()) ? MAP_MEMTAG + : 0)))) { return false; } Region->MappedUser += MapSize; @@ -994,7 +1002,8 @@ const uptr ReleaseRangeSize = ReleaseEnd - ReleaseBase; const uptr ReleaseOffset = ReleaseBase - Region->RegionBeg; - ReleaseRecorder Recorder(Region->RegionBeg, ReleaseOffset, &Region->Data); + RegionReleaseRecorder Recorder(&Region->MemMap, Region->RegionBeg, + ReleaseOffset); PageReleaseContext Context(BlockSize, /*NumberOfRegions=*/1U, ReleaseRangeSize, ReleaseOffset); diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h --- a/compiler-rt/lib/scudo/standalone/release.h +++ b/compiler-rt/lib/scudo/standalone/release.h @@ -11,11 +11,42 @@ #include "common.h" #include "list.h" +#include "mem_map.h" #include "mutex.h" #include "thread_annotations.h" namespace scudo { +template class RegionReleaseRecorder { +public: + RegionReleaseRecorder(MemMapT *RegionMemMap, uptr Base, uptr Offset = 0) + : RegionMemMap(RegionMemMap), Base(Base), Offset(Offset) {} + + uptr getReleasedRangesCount() const { return ReleasedRangesCount; } + + uptr getReleasedBytes() const { return ReleasedBytes; } + + uptr getBase() const { return Base; } + + // Releases [From, To) range of pages back to OS. Note that `From` and `To` + // are offseted from `Base` + Offset. + void releasePageRangeToOS(uptr From, uptr To) { + const uptr Size = To - From; + RegionMemMap->releasePagesToOS(getBase() + Offset + From, Size); + ReleasedRangesCount++; + ReleasedBytes += Size; + } + +private: + uptr ReleasedRangesCount = 0; + uptr ReleasedBytes = 0; + MemMapT *RegionMemMap = nullptr; + uptr Base = 0; + // The release offset from Base. This is used when we know a given range after + // Base will not be released. + uptr Offset = 0; +}; + class ReleaseRecorder { public: ReleaseRecorder(uptr Base, uptr Offset = 0, MapPlatformData *Data = nullptr)