diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.h b/compiler-rt/lib/scudo/standalone/allocator_config.h --- a/compiler-rt/lib/scudo/standalone/allocator_config.h +++ b/compiler-rt/lib/scudo/standalone/allocator_config.h @@ -40,6 +40,12 @@ // // eg: Ptr = Base + (CompactPtr << Scale). // typedef u32 PrimaryCompactPtrT; // static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; +// // Indicates support for offsetting the start of a region by +// // a random number of pages. Only used with primary64. +// static const bool PrimaryEnableRandomOffset = true; +// // Call map for user memory with at least this size. Only used with +// // primary64. +// static const uptr PrimaryMapSizeIncrement = 1UL << 18; // // Defines the minimal & maximal release interval that can be set. // static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; // static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; @@ -61,6 +67,8 @@ static const uptr PrimaryRegionSizeLog = 32U; typedef uptr PrimaryCompactPtrT; static const uptr PrimaryCompactPtrScale = 0; + static const bool PrimaryEnableRandomOffset = true; + static const uptr PrimaryMapSizeIncrement = 1UL << 18; #else typedef SizeClassAllocator32 Primary; static const uptr PrimaryRegionSizeLog = 19U; @@ -89,6 +97,8 @@ static const uptr PrimaryRegionSizeLog = 28U; typedef u32 PrimaryCompactPtrT; static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; + static const bool PrimaryEnableRandomOffset = true; + static const uptr PrimaryMapSizeIncrement = 1UL << 18; #else typedef SizeClassAllocator32 Primary; static const uptr PrimaryRegionSizeLog = 18U; @@ -118,6 +128,8 @@ static const uptr PrimaryRegionSizeLog = 27U; typedef u32 PrimaryCompactPtrT; static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; + static const bool PrimaryEnableRandomOffset = true; + static const uptr PrimaryMapSizeIncrement = 1UL << 18; #else typedef SizeClassAllocator32 Primary; static const uptr PrimaryRegionSizeLog = 16U; @@ -146,6 +158,8 @@ typedef SizeClassAllocator64 Primary; static const uptr PrimaryRegionSizeLog = 30U; typedef u32 PrimaryCompactPtrT; + static const bool PrimaryEnableRandomOffset = true; + static const uptr PrimaryMapSizeIncrement = 1UL << 18; static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; @@ -154,12 +168,34 @@ template using TSDRegistryT = TSDRegistrySharedT; // Shared, max 8 TSDs. }; + +struct TrustyConfig { + using SizeClassMap = TrustySizeClassMap; + static const bool MaySupportMemoryTagging = false; + + typedef SizeClassAllocator64 Primary; + // Some apps have 1 page of heap total so small regions are necessary. + static const uptr PrimaryRegionSizeLog = 10U; + typedef u32 PrimaryCompactPtrT; + static const bool PrimaryEnableRandomOffset = false; + // Trusty is extremely memory-constrained so minimally round up map calls. + static const uptr PrimaryMapSizeIncrement = 1UL << 4; + static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG; + static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + + typedef MapAllocatorNoCache SecondaryCache; + template + using TSDRegistryT = TSDRegistrySharedT; // Shared, max 1 TSD. +}; #endif #if SCUDO_ANDROID typedef AndroidConfig Config; #elif SCUDO_FUCHSIA typedef FuchsiaConfig Config; +#elif SCUDO_TRUSTY +typedef TrustyConfig Config; #else typedef DefaultConfig Config; #endif diff --git a/compiler-rt/lib/scudo/standalone/common.h b/compiler-rt/lib/scudo/standalone/common.h --- a/compiler-rt/lib/scudo/standalone/common.h +++ b/compiler-rt/lib/scudo/standalone/common.h @@ -13,6 +13,7 @@ #include "fuchsia.h" #include "linux.h" +#include "trusty.h" #include #include diff --git a/compiler-rt/lib/scudo/standalone/memtag.h b/compiler-rt/lib/scudo/standalone/memtag.h --- a/compiler-rt/lib/scudo/standalone/memtag.h +++ b/compiler-rt/lib/scudo/standalone/memtag.h @@ -55,7 +55,7 @@ #endif -#if defined(__aarch64__) +#if defined(__aarch64__) && !SCUDO_TRUSTY #if SCUDO_LINUX diff --git a/compiler-rt/lib/scudo/standalone/mutex.h b/compiler-rt/lib/scudo/standalone/mutex.h --- a/compiler-rt/lib/scudo/standalone/mutex.h +++ b/compiler-rt/lib/scudo/standalone/mutex.h @@ -46,7 +46,7 @@ static constexpr u8 NumberOfTries = 8U; static constexpr u8 NumberOfYields = 8U; -#if SCUDO_LINUX +#if SCUDO_LINUX || SCUDO_TRUSTY atomic_u32 M = {}; #elif SCUDO_FUCHSIA sync_mutex_t M = {}; diff --git a/compiler-rt/lib/scudo/standalone/platform.h b/compiler-rt/lib/scudo/standalone/platform.h --- a/compiler-rt/lib/scudo/standalone/platform.h +++ b/compiler-rt/lib/scudo/standalone/platform.h @@ -12,7 +12,7 @@ // Transitive includes of stdint.h specify some of the defines checked below. #include -#if defined(__linux__) +#if defined(__linux__) && !defined(__TRUSTY__) #define SCUDO_LINUX 1 #else #define SCUDO_LINUX 0 @@ -31,6 +31,12 @@ #define SCUDO_FUCHSIA 0 #endif +#if defined(__TRUSTY__) +#define SCUDO_TRUSTY 1 +#else +#define SCUDO_TRUSTY 0 +#endif + #if __LP64__ #define SCUDO_WORDSIZE 64U #else diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -64,6 +64,9 @@ if (SCUDO_FUCHSIA) reportError("SizeClassAllocator32 is not supported on Fuchsia"); + if (SCUDO_TRUSTY) + reportError("SizeClassAllocator32 is not supported on Trusty"); + PossibleRegions.init(); u32 Seed; diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -25,8 +25,9 @@ // // It starts by reserving NumClasses * 2^RegionSizeLog bytes, equally divided in // Regions, specific to each size class. Note that the base of that mapping is -// random (based to the platform specific map() capabilities), and that each -// Region actually starts at a random offset from its base. +// random (based to the platform specific map() capabilities). If +// PrimaryEnableRandomOffset is set, each Region actually starts at a random +// offset from its base. // // Regions are mapped incrementally on demand to fulfill allocation requests, // those mappings being split into equally sized Blocks based on the size class @@ -70,9 +71,11 @@ const uptr PageSize = getPageSizeCached(); for (uptr I = 0; I < NumClasses; I++) { RegionInfo *Region = getRegionInfo(I); - // The actual start of a region is offseted by a random number of pages. - Region->RegionBeg = - getRegionBaseByClassId(I) + (getRandomModN(&Seed, 16) + 1) * PageSize; + // The actual start of a region is offset by a random number of pages + // when PrimaryEnableRandomOffset is set. + Region->RegionBeg = getRegionBaseByClassId(I) + + (Config::PrimaryEnableRandomOffset + ? ((getRandomModN(&Seed, 16) + 1) * PageSize) : 0); Region->RandState = getRandomU32(&Seed); Region->ReleaseInfo.LastReleaseAtNs = Time; } @@ -267,8 +270,7 @@ static const uptr NumClasses = SizeClassMap::NumClasses; static const uptr PrimarySize = RegionSize * NumClasses; - // Call map for user memory with at least this size. - static const uptr MapSizeIncrement = 1UL << 18; + static const uptr MapSizeIncrement = Config::PrimaryMapSizeIncrement; // Fill at most this number of batches from the newly map'd memory. static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U; diff --git a/compiler-rt/lib/scudo/standalone/size_class_map.h b/compiler-rt/lib/scudo/standalone/size_class_map.h --- a/compiler-rt/lib/scudo/standalone/size_class_map.h +++ b/compiler-rt/lib/scudo/standalone/size_class_map.h @@ -308,6 +308,20 @@ typedef FixedSizeClassMap SvelteSizeClassMap; +// Trusty is configured to only have one region containing blocks of size +// 2^7 bytes. +struct TrustySizeClassConfig { + static const uptr NumBits = 1; + static const uptr MinSizeLog = 7; + static const uptr MidSizeLog = 7; + static const uptr MaxSizeLog = 7; + static const u32 MaxNumCachedHint = 8; + static const uptr MaxBytesCachedLog = 10; + static const uptr SizeDelta = 0; +}; + +typedef FixedSizeClassMap TrustySizeClassMap; + template inline void printMap() { ScopedString Buffer(1024); uptr PrevS = 0; diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -524,6 +524,8 @@ static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; typedef scudo::uptr PrimaryCompactPtrT; static const scudo::uptr PrimaryCompactPtrScale = 0; + static const bool PrimaryEnableRandomOffset = true; + static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18; typedef scudo::MapAllocatorNoCache SecondaryCache; template using TSDRegistryT = scudo::TSDRegistrySharedT; diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp --- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp @@ -29,6 +29,8 @@ static const bool MaySupportMemoryTagging = false; typedef scudo::uptr PrimaryCompactPtrT; static const scudo::uptr PrimaryCompactPtrScale = 0; + static const bool PrimaryEnableRandomOffset = true; + static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18; }; struct TestConfig2 { @@ -43,6 +45,8 @@ static const bool MaySupportMemoryTagging = false; typedef scudo::uptr PrimaryCompactPtrT; static const scudo::uptr PrimaryCompactPtrScale = 0; + static const bool PrimaryEnableRandomOffset = true; + static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18; }; struct TestConfig3 { @@ -57,6 +61,8 @@ static const bool MaySupportMemoryTagging = true; typedef scudo::uptr PrimaryCompactPtrT; static const scudo::uptr PrimaryCompactPtrScale = 0; + static const bool PrimaryEnableRandomOffset = true; + static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18; }; template @@ -145,6 +151,8 @@ static const bool MaySupportMemoryTagging = false; typedef scudo::uptr PrimaryCompactPtrT; static const scudo::uptr PrimaryCompactPtrScale = 0; + static const bool PrimaryEnableRandomOffset = true; + static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18; }; // The 64-bit SizeClassAllocator can be easily OOM'd with small region sizes. diff --git a/compiler-rt/lib/scudo/standalone/trusty.h b/compiler-rt/lib/scudo/standalone/trusty.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/scudo/standalone/trusty.h @@ -0,0 +1,24 @@ +//===-- trusty.h -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_TRUSTY_H_ +#define SCUDO_TRUSTY_H_ + +#include "platform.h" + +#if SCUDO_TRUSTY + +namespace scudo { +// MapPlatformData is unused on Trusty, define it as a minimially sized +// structure. +struct MapPlatformData {}; +} // namespace scudo + +#endif // SCUDO_TRUSTY + +#endif // SCUDO_TRUSTY_H_ diff --git a/compiler-rt/lib/scudo/standalone/trusty.cpp b/compiler-rt/lib/scudo/standalone/trusty.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/scudo/standalone/trusty.cpp @@ -0,0 +1,99 @@ +//===-- trusty.cpp ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "platform.h" + +#if SCUDO_TRUSTY + +#include "common.h" +#include "mutex.h" +#include "string_utils.h" +#include "trusty.h" + +#include // for errno +#include // for printf() +#include // for getenv() +#include // for getauxval() +#include // for clock_gettime() +#include // for _trusty_brk() + +#define SBRK_ALIGN 32 + +namespace scudo { + +uptr getPageSize() { return getauxval(AT_PAGESZ); } + +void NORETURN die() { abort(); } + +void *map(UNUSED void *Addr, uptr Size, UNUSED const char *Name, uptr Flags, + UNUSED MapPlatformData *Data) { + // Calling _trusty_brk(0) returns the current program break. + uptr ProgramBreak = reinterpret_cast(_trusty_brk(0)); + uptr Start; + uptr End; + + Start = roundUpTo(ProgramBreak, SBRK_ALIGN); + // Don't actually extend the heap if MAP_NOACCESS flag is set since this is + // the case where Scudo tries to reserve a memory region without mapping + // physical pages. + if (Flags & MAP_NOACCESS) + return reinterpret_cast(Start); + + // Attempt to extend the heap by Size bytes using _trusty_brk. + End = roundUpTo(Start + Size, SBRK_ALIGN); + ProgramBreak = + reinterpret_cast(_trusty_brk(reinterpret_cast(End))); + if (ProgramBreak < End) { + dieOnMapUnmapError(errno == ENOMEM); + return nullptr; + } + return reinterpret_cast(Start); // Base of new reserved region. +} + +// Unmap is a no-op since Trusty uses sbrk instead of memory mapping. +void unmap(UNUSED void *Addr, UNUSED uptr Size, UNUSED uptr Flags, + UNUSED MapPlatformData *Data) {} + +void setMemoryPermission(UNUSED uptr Addr, UNUSED uptr Size, UNUSED uptr Flags, + UNUSED MapPlatformData *Data) {} + +void releasePagesToOS(UNUSED uptr BaseAddress, UNUSED uptr Offset, + UNUSED uptr Size, UNUSED MapPlatformData *Data) {} + +const char *getEnv(const char *Name) { return getenv(Name); } + +// All mutex operations are a no-op since Trusty doesn't currently support +// threads. +bool HybridMutex::tryLock() { return true; } + +void HybridMutex::lockSlow() {} + +void HybridMutex::unlock() {} + +u64 getMonotonicTime() { + timespec TS; + clock_gettime(CLOCK_MONOTONIC, &TS); + return static_cast(TS.tv_sec) * (1000ULL * 1000 * 1000) + + static_cast(TS.tv_nsec); +} + +u32 getNumberOfCPUs() { return 0; } + +u32 getThreadID() { return 0; } + +bool getRandom(UNUSED void *Buffer, UNUSED uptr Length, UNUSED bool Blocking) { + return false; +} + +void outputRaw(const char *Buffer) { printf("%s", Buffer); } + +void setAbortMessage(UNUSED const char *Message) {} + +} // namespace scudo + +#endif // SCUDO_TRUSTY diff --git a/compiler-rt/lib/scudo/standalone/vector.h b/compiler-rt/lib/scudo/standalone/vector.h --- a/compiler-rt/lib/scudo/standalone/vector.h +++ b/compiler-rt/lib/scudo/standalone/vector.h @@ -82,8 +82,12 @@ void reallocate(uptr NewCapacity) { DCHECK_GT(NewCapacity, 0); DCHECK_LE(Size, NewCapacity); - const uptr NewCapacityBytes = - roundUpTo(NewCapacity * sizeof(T), getPageSizeCached()); + // By default, NewCapacityBytes is rounded up to the nearest page. + // Because Trusty is extremely memory-constrained, NewCapacityBytes is + // only minimally rounded up. + const uptr NewCapacityBytes = roundUpTo( + NewCapacity * sizeof(T), + SCUDO_TRUSTY ? 1U << SCUDO_MIN_ALIGNMENT_LOG : getPageSizeCached()); T *NewData = reinterpret_cast(map(nullptr, NewCapacityBytes, "scudo:vector")); if (Data) {