diff --git a/compiler-rt/lib/scudo/CMakeLists.txt b/compiler-rt/lib/scudo/CMakeLists.txt deleted file mode 100644 --- a/compiler-rt/lib/scudo/CMakeLists.txt +++ /dev/null @@ -1,160 +0,0 @@ -add_compiler_rt_component(scudo) - -include_directories(..) - -set(SCUDO_CFLAGS ${SANITIZER_COMMON_CFLAGS}) -# SANITIZER_COMMON_CFLAGS include -fno-builtin, but we actually want builtins! -list(APPEND SCUDO_CFLAGS -fbuiltin) -append_rtti_flag(OFF SCUDO_CFLAGS) - -# Too many existing bugs, needs cleanup. -append_list_if(COMPILER_RT_HAS_WNO_FORMAT -Wno-format SCUDO_CFLAGS) - -set(SCUDO_MINIMAL_DYNAMIC_LIBS ${SANITIZER_COMMON_LINK_LIBS}) -append_list_if(COMPILER_RT_HAS_LIBDL dl SCUDO_MINIMAL_DYNAMIC_LIBS) -append_list_if(COMPILER_RT_HAS_LIBRT rt SCUDO_MINIMAL_DYNAMIC_LIBS) -append_list_if(COMPILER_RT_HAS_LIBPTHREAD pthread SCUDO_MINIMAL_DYNAMIC_LIBS) -append_list_if(COMPILER_RT_HAS_LIBLOG log SCUDO_MINIMAL_DYNAMIC_LIBS) -append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer - SCUDO_CFLAGS) - -set(SCUDO_DYNAMIC_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS}) -# Use gc-sections by default to avoid unused code being pulled in. -list(APPEND SCUDO_DYNAMIC_LINK_FLAGS -Wl,--gc-sections) - -if(ANDROID) -# Put most Sanitizer shared libraries in the global group. For more details, see -# android-changes-for-ndk-developers.md#changes-to-library-search-order - if (COMPILER_RT_HAS_Z_GLOBAL) - list(APPEND SCUDO_DYNAMIC_LINK_FLAGS -Wl,-z,global) - endif() -endif() - -# The minimal Scudo runtime does not include the UBSan runtime. -set(SCUDO_MINIMAL_OBJECT_LIBS - RTSanitizerCommonNoTermination - RTSanitizerCommonLibc - RTInterception) - -if (COMPILER_RT_HAS_GWP_ASAN) - list(APPEND SCUDO_MINIMAL_OBJECT_LIBS - RTGwpAsan RTGwpAsanOptionsParser RTGwpAsanBacktraceLibc - RTGwpAsanSegvHandler) - list(APPEND SCUDO_CFLAGS -DGWP_ASAN_HOOKS) -endif() - -set(SCUDO_OBJECT_LIBS ${SCUDO_MINIMAL_OBJECT_LIBS}) -set(SCUDO_DYNAMIC_LIBS ${SCUDO_MINIMAL_DYNAMIC_LIBS}) - -if (FUCHSIA) - list(APPEND SCUDO_CFLAGS -nostdinc++) - list(APPEND SCUDO_DYNAMIC_LINK_FLAGS -nostdlib++) -else() - list(APPEND SCUDO_DYNAMIC_LIBS - ${COMPILER_RT_UNWINDER_LINK_LIBS} - ${SANITIZER_CXX_ABI_LIBRARIES}) - list(APPEND SCUDO_OBJECT_LIBS - RTSanitizerCommonCoverage - RTSanitizerCommonSymbolizer - RTUbsan) -endif() - -set(SCUDO_SOURCES - scudo_allocator.cpp - scudo_crc32.cpp - scudo_errors.cpp - scudo_flags.cpp - scudo_malloc.cpp - scudo_termination.cpp - scudo_tsd_exclusive.cpp - scudo_tsd_shared.cpp - scudo_utils.cpp) - -set(SCUDO_CXX_SOURCES - scudo_new_delete.cpp) - -set(SCUDO_HEADERS - scudo_allocator.h - scudo_allocator_combined.h - scudo_allocator_secondary.h - scudo_crc32.h - scudo_errors.h - scudo_flags.h - scudo_flags.inc - scudo_interface_internal.h - scudo_platform.h - scudo_tsd.h - scudo_tsd_exclusive.inc - scudo_tsd_shared.inc - scudo_utils.h) - -# Enable the necessary instruction set for scudo_crc32.cpp, if available. -# Newer compiler versions use -mcrc32 rather than -msse4.2. -if (COMPILER_RT_HAS_MCRC32_FLAG) - set_source_files_properties(scudo_crc32.cpp PROPERTIES COMPILE_FLAGS -mcrc32) -elseif (COMPILER_RT_HAS_MSSE4_2_FLAG) - set_source_files_properties(scudo_crc32.cpp PROPERTIES COMPILE_FLAGS -msse4.2) -endif() - -# Enable the AArch64 CRC32 feature for scudo_crc32.cpp, if available. -# Note that it is enabled by default starting with armv8.1-a. -if (COMPILER_RT_HAS_MCRC_FLAG) - set_source_files_properties(scudo_crc32.cpp PROPERTIES COMPILE_FLAGS -mcrc) -endif() - -if(COMPILER_RT_HAS_SCUDO) - add_compiler_rt_runtime(clang_rt.scudo_minimal - STATIC - ARCHS ${SCUDO_SUPPORTED_ARCH} - SOURCES ${SCUDO_SOURCES} - ADDITIONAL_HEADERS ${SCUDO_HEADERS} - OBJECT_LIBS ${SCUDO_MINIMAL_OBJECT_LIBS} - CFLAGS ${SCUDO_CFLAGS} - PARENT_TARGET scudo) - add_compiler_rt_runtime(clang_rt.scudo_cxx_minimal - STATIC - ARCHS ${SCUDO_SUPPORTED_ARCH} - SOURCES ${SCUDO_CXX_SOURCES} - ADDITIONAL_HEADERS ${SCUDO_HEADERS} - CFLAGS ${SCUDO_CFLAGS} - PARENT_TARGET scudo) - - add_compiler_rt_runtime(clang_rt.scudo - STATIC - ARCHS ${SCUDO_SUPPORTED_ARCH} - SOURCES ${SCUDO_SOURCES} - ADDITIONAL_HEADERS ${SCUDO_HEADERS} - OBJECT_LIBS ${SCUDO_OBJECT_LIBS} - CFLAGS ${SCUDO_CFLAGS} - PARENT_TARGET scudo) - add_compiler_rt_runtime(clang_rt.scudo_cxx - STATIC - ARCHS ${SCUDO_SUPPORTED_ARCH} - SOURCES ${SCUDO_CXX_SOURCES} - ADDITIONAL_HEADERS ${SCUDO_HEADERS} - OBJECT_LIBS RTUbsan_cxx - CFLAGS ${SCUDO_CFLAGS} - PARENT_TARGET scudo) - - add_compiler_rt_runtime(clang_rt.scudo_minimal - SHARED - ARCHS ${SCUDO_SUPPORTED_ARCH} - SOURCES ${SCUDO_SOURCES} ${SCUDO_CXX_SOURCES} - ADDITIONAL_HEADERS ${SCUDO_HEADERS} - OBJECT_LIBS ${SCUDO_MINIMAL_OBJECT_LIBS} - CFLAGS ${SCUDO_CFLAGS} - LINK_FLAGS ${SCUDO_DYNAMIC_LINK_FLAGS} - LINK_LIBS ${SCUDO_MINIMAL_DYNAMIC_LIBS} - PARENT_TARGET scudo) - - add_compiler_rt_runtime(clang_rt.scudo - SHARED - ARCHS ${SCUDO_SUPPORTED_ARCH} - SOURCES ${SCUDO_SOURCES} ${SCUDO_CXX_SOURCES} - ADDITIONAL_HEADERS ${SCUDO_HEADERS} - OBJECT_LIBS ${SCUDO_OBJECT_LIBS} - CFLAGS ${SCUDO_CFLAGS} - LINK_FLAGS ${SCUDO_DYNAMIC_LINK_FLAGS} - LINK_LIBS ${SCUDO_DYNAMIC_LIBS} - PARENT_TARGET scudo) -endif() diff --git a/compiler-rt/lib/scudo/scudo_allocator.h b/compiler-rt/lib/scudo/scudo_allocator.h deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_allocator.h +++ /dev/null @@ -1,125 +0,0 @@ -//===-- scudo_allocator.h ---------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Header for scudo_allocator.cpp. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_ALLOCATOR_H_ -#define SCUDO_ALLOCATOR_H_ - -#include "scudo_platform.h" - -namespace __scudo { - -enum AllocType : u8 { - FromMalloc = 0, // Memory block came from malloc, realloc, calloc, etc. - FromNew = 1, // Memory block came from operator new. - FromNewArray = 2, // Memory block came from operator new []. - FromMemalign = 3, // Memory block came from memalign, posix_memalign, etc. -}; - -enum ChunkState : u8 { - ChunkAvailable = 0, - ChunkAllocated = 1, - ChunkQuarantine = 2 -}; - -// Our header requires 64 bits of storage. Having the offset saves us from -// using functions such as GetBlockBegin, that is fairly costly. Our first -// implementation used the MetaData as well, which offers the advantage of -// being stored away from the chunk itself, but accessing it was costly as -// well. The header will be atomically loaded and stored. -typedef u64 PackedHeader; -struct UnpackedHeader { - u64 Checksum : 16; - u64 ClassId : 8; - u64 SizeOrUnusedBytes : 20; // Size for Primary backed allocations, amount of - // unused bytes in the chunk for Secondary ones. - u64 State : 2; // available, allocated, or quarantined - u64 AllocType : 2; // malloc, new, new[], or memalign - u64 Offset : 16; // Offset from the beginning of the backend - // allocation to the beginning of the chunk - // itself, in multiples of MinAlignment. See - // comment about its maximum value and in init(). -}; - -typedef atomic_uint64_t AtomicPackedHeader; -COMPILER_CHECK(sizeof(UnpackedHeader) == sizeof(PackedHeader)); - -// Minimum alignment of 8 bytes for 32-bit, 16 for 64-bit -const uptr MinAlignmentLog = FIRST_32_SECOND_64(3, 4); -const uptr MaxAlignmentLog = 24; // 16 MB -const uptr MinAlignment = 1 << MinAlignmentLog; -const uptr MaxAlignment = 1 << MaxAlignmentLog; - -// constexpr version of __sanitizer::RoundUp without the extraneous CHECK. -// This way we can use it in constexpr variables and functions declarations. -constexpr uptr RoundUpTo(uptr Size, uptr Boundary) { - return (Size + Boundary - 1) & ~(Boundary - 1); -} - -namespace Chunk { - constexpr uptr getHeaderSize() { - return RoundUpTo(sizeof(PackedHeader), MinAlignment); - } -} - -#if SANITIZER_CAN_USE_ALLOCATOR64 -const uptr AllocatorSpace = ~0ULL; -struct AP64 { - static const uptr kSpaceBeg = AllocatorSpace; - static const uptr kSpaceSize = AllocatorSize; - static const uptr kMetadataSize = 0; - typedef __scudo::SizeClassMap SizeClassMap; - typedef NoOpMapUnmapCallback MapUnmapCallback; - static const uptr kFlags = - SizeClassAllocator64FlagMasks::kRandomShuffleChunks; - using AddressSpaceView = LocalAddressSpaceView; -}; -typedef SizeClassAllocator64 PrimaryT; -#else -struct AP32 { - static const uptr kSpaceBeg = 0; - static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; - static const uptr kMetadataSize = 0; - typedef __scudo::SizeClassMap SizeClassMap; - static const uptr kRegionSizeLog = RegionSizeLog; - using AddressSpaceView = LocalAddressSpaceView; - typedef NoOpMapUnmapCallback MapUnmapCallback; - static const uptr kFlags = - SizeClassAllocator32FlagMasks::kRandomShuffleChunks | - SizeClassAllocator32FlagMasks::kUseSeparateSizeClassForBatch; -}; -typedef SizeClassAllocator32 PrimaryT; -#endif // SANITIZER_CAN_USE_ALLOCATOR64 - -#include "scudo_allocator_secondary.h" - -typedef LargeMmapAllocator SecondaryT; - -#include "scudo_allocator_combined.h" - -typedef CombinedAllocator BackendT; -typedef CombinedAllocator::AllocatorCache AllocatorCacheT; - -void initScudo(); - -void *scudoAllocate(uptr Size, uptr Alignment, AllocType Type); -void scudoDeallocate(void *Ptr, uptr Size, uptr Alignment, AllocType Type); -void *scudoRealloc(void *Ptr, uptr Size); -void *scudoCalloc(uptr NMemB, uptr Size); -void *scudoValloc(uptr Size); -void *scudoPvalloc(uptr Size); -int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size); -void *scudoAlignedAlloc(uptr Alignment, uptr Size); -uptr scudoMallocUsableSize(void *Ptr); - -} // namespace __scudo - -#endif // SCUDO_ALLOCATOR_H_ diff --git a/compiler-rt/lib/scudo/scudo_allocator.cpp b/compiler-rt/lib/scudo/scudo_allocator.cpp deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_allocator.cpp +++ /dev/null @@ -1,831 +0,0 @@ -//===-- scudo_allocator.cpp -------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Scudo Hardened Allocator implementation. -/// It uses the sanitizer_common allocator as a base and aims at mitigating -/// heap corruption vulnerabilities. It provides a checksum-guarded chunk -/// header, a delayed free list, and additional sanity checks. -/// -//===----------------------------------------------------------------------===// - -#include "scudo_allocator.h" -#include "scudo_crc32.h" -#include "scudo_errors.h" -#include "scudo_flags.h" -#include "scudo_interface_internal.h" -#include "scudo_tsd.h" -#include "scudo_utils.h" - -#include "sanitizer_common/sanitizer_allocator_checks.h" -#include "sanitizer_common/sanitizer_allocator_interface.h" -#include "sanitizer_common/sanitizer_quarantine.h" - -#ifdef GWP_ASAN_HOOKS -# include "gwp_asan/guarded_pool_allocator.h" -# include "gwp_asan/optional/backtrace.h" -# include "gwp_asan/optional/options_parser.h" -#include "gwp_asan/optional/segv_handler.h" -#endif // GWP_ASAN_HOOKS - -#include -#include - -namespace __scudo { - -// Global static cookie, initialized at start-up. -static u32 Cookie; - -// We default to software CRC32 if the alternatives are not supported, either -// at compilation or at runtime. -static atomic_uint8_t HashAlgorithm = { CRC32Software }; - -inline u32 computeCRC32(u32 Crc, uptr Value, uptr *Array, uptr ArraySize) { - // If the hardware CRC32 feature is defined here, it was enabled everywhere, - // as opposed to only for scudo_crc32.cpp. This means that other hardware - // specific instructions were likely emitted at other places, and as a - // result there is no reason to not use it here. -#if defined(__CRC32__) || defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) - Crc = CRC32_INTRINSIC(Crc, Value); - for (uptr i = 0; i < ArraySize; i++) - Crc = CRC32_INTRINSIC(Crc, Array[i]); - return Crc; -#else - if (atomic_load_relaxed(&HashAlgorithm) == CRC32Hardware) { - Crc = computeHardwareCRC32(Crc, Value); - for (uptr i = 0; i < ArraySize; i++) - Crc = computeHardwareCRC32(Crc, Array[i]); - return Crc; - } - Crc = computeSoftwareCRC32(Crc, Value); - for (uptr i = 0; i < ArraySize; i++) - Crc = computeSoftwareCRC32(Crc, Array[i]); - return Crc; -#endif // defined(__CRC32__) || defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) -} - -static BackendT &getBackend(); - -namespace Chunk { - static inline AtomicPackedHeader *getAtomicHeader(void *Ptr) { - return reinterpret_cast(reinterpret_cast(Ptr) - - getHeaderSize()); - } - static inline - const AtomicPackedHeader *getConstAtomicHeader(const void *Ptr) { - return reinterpret_cast( - reinterpret_cast(Ptr) - getHeaderSize()); - } - - static inline bool isAligned(const void *Ptr) { - return IsAligned(reinterpret_cast(Ptr), MinAlignment); - } - - // We can't use the offset member of the chunk itself, as we would double - // fetch it without any warranty that it wouldn't have been tampered. To - // prevent this, we work with a local copy of the header. - static inline void *getBackendPtr(const void *Ptr, UnpackedHeader *Header) { - return reinterpret_cast(reinterpret_cast(Ptr) - - getHeaderSize() - (Header->Offset << MinAlignmentLog)); - } - - // Returns the usable size for a chunk, meaning the amount of bytes from the - // beginning of the user data to the end of the backend allocated chunk. - static inline uptr getUsableSize(const void *Ptr, UnpackedHeader *Header) { - const uptr ClassId = Header->ClassId; - if (ClassId) - return PrimaryT::ClassIdToSize(ClassId) - getHeaderSize() - - (Header->Offset << MinAlignmentLog); - return SecondaryT::GetActuallyAllocatedSize( - getBackendPtr(Ptr, Header)) - getHeaderSize(); - } - - // Returns the size the user requested when allocating the chunk. - static inline uptr getSize(const void *Ptr, UnpackedHeader *Header) { - const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes; - if (Header->ClassId) - return SizeOrUnusedBytes; - return SecondaryT::GetActuallyAllocatedSize( - getBackendPtr(Ptr, Header)) - getHeaderSize() - SizeOrUnusedBytes; - } - - // Compute the checksum of the chunk pointer and its header. - static inline u16 computeChecksum(const void *Ptr, UnpackedHeader *Header) { - UnpackedHeader ZeroChecksumHeader = *Header; - ZeroChecksumHeader.Checksum = 0; - uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)]; - memcpy(&HeaderHolder, &ZeroChecksumHeader, sizeof(HeaderHolder)); - const u32 Crc = computeCRC32(Cookie, reinterpret_cast(Ptr), - HeaderHolder, ARRAY_SIZE(HeaderHolder)); - return static_cast(Crc); - } - - // Checks the validity of a chunk by verifying its checksum. It doesn't - // incur termination in the event of an invalid chunk. - static inline bool isValid(const void *Ptr) { - PackedHeader NewPackedHeader = - atomic_load_relaxed(getConstAtomicHeader(Ptr)); - UnpackedHeader NewUnpackedHeader = - bit_cast(NewPackedHeader); - return (NewUnpackedHeader.Checksum == - computeChecksum(Ptr, &NewUnpackedHeader)); - } - - // Ensure that ChunkAvailable is 0, so that if a 0 checksum is ever valid - // for a fully nulled out header, its state will be available anyway. - COMPILER_CHECK(ChunkAvailable == 0); - - // Loads and unpacks the header, verifying the checksum in the process. - static inline - void loadHeader(const void *Ptr, UnpackedHeader *NewUnpackedHeader) { - PackedHeader NewPackedHeader = - atomic_load_relaxed(getConstAtomicHeader(Ptr)); - *NewUnpackedHeader = bit_cast(NewPackedHeader); - if (UNLIKELY(NewUnpackedHeader->Checksum != - computeChecksum(Ptr, NewUnpackedHeader))) - dieWithMessage("corrupted chunk header at address %p\n", Ptr); - } - - // Packs and stores the header, computing the checksum in the process. - static inline void storeHeader(void *Ptr, UnpackedHeader *NewUnpackedHeader) { - NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader); - PackedHeader NewPackedHeader = bit_cast(*NewUnpackedHeader); - atomic_store_relaxed(getAtomicHeader(Ptr), NewPackedHeader); - } - - // Packs and stores the header, computing the checksum in the process. We - // compare the current header with the expected provided one to ensure that - // we are not being raced by a corruption occurring in another thread. - static inline void compareExchangeHeader(void *Ptr, - UnpackedHeader *NewUnpackedHeader, - UnpackedHeader *OldUnpackedHeader) { - NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader); - PackedHeader NewPackedHeader = bit_cast(*NewUnpackedHeader); - PackedHeader OldPackedHeader = bit_cast(*OldUnpackedHeader); - if (UNLIKELY(!atomic_compare_exchange_strong( - getAtomicHeader(Ptr), &OldPackedHeader, NewPackedHeader, - memory_order_relaxed))) - dieWithMessage("race on chunk header at address %p\n", Ptr); - } -} // namespace Chunk - -struct QuarantineCallback { - explicit QuarantineCallback(AllocatorCacheT *Cache) - : Cache_(Cache) {} - - // Chunk recycling function, returns a quarantined chunk to the backend, - // first making sure it hasn't been tampered with. - void Recycle(void *Ptr) { - UnpackedHeader Header; - Chunk::loadHeader(Ptr, &Header); - if (UNLIKELY(Header.State != ChunkQuarantine)) - dieWithMessage("invalid chunk state when recycling address %p\n", Ptr); - UnpackedHeader NewHeader = Header; - NewHeader.State = ChunkAvailable; - Chunk::compareExchangeHeader(Ptr, &NewHeader, &Header); - void *BackendPtr = Chunk::getBackendPtr(Ptr, &Header); - if (Header.ClassId) - getBackend().deallocatePrimary(Cache_, BackendPtr, Header.ClassId); - else - getBackend().deallocateSecondary(BackendPtr); - } - - // Internal quarantine allocation and deallocation functions. We first check - // that the batches are indeed serviced by the Primary. - // TODO(kostyak): figure out the best way to protect the batches. - void *Allocate(uptr Size) { - const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch)); - return getBackend().allocatePrimary(Cache_, BatchClassId); - } - - void Deallocate(void *Ptr) { - const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch)); - getBackend().deallocatePrimary(Cache_, Ptr, BatchClassId); - } - - AllocatorCacheT *Cache_; - COMPILER_CHECK(sizeof(QuarantineBatch) < SizeClassMap::kMaxSize); -}; - -typedef Quarantine QuarantineT; -typedef QuarantineT::Cache QuarantineCacheT; -COMPILER_CHECK(sizeof(QuarantineCacheT) <= - sizeof(ScudoTSD::QuarantineCachePlaceHolder)); - -QuarantineCacheT *getQuarantineCache(ScudoTSD *TSD) { - return reinterpret_cast(TSD->QuarantineCachePlaceHolder); -} - -#ifdef GWP_ASAN_HOOKS -static gwp_asan::GuardedPoolAllocator GuardedAlloc; -#endif // GWP_ASAN_HOOKS - -struct Allocator { - static const uptr MaxAllowedMallocSize = - FIRST_32_SECOND_64(2UL << 30, 1ULL << 40); - - BackendT Backend; - QuarantineT Quarantine; - - u32 QuarantineChunksUpToSize; - - bool DeallocationTypeMismatch; - bool ZeroContents; - bool DeleteSizeMismatch; - - bool CheckRssLimit; - uptr HardRssLimitMb; - uptr SoftRssLimitMb; - atomic_uint8_t RssLimitExceeded; - atomic_uint64_t RssLastCheckedAtNS; - - explicit Allocator(LinkerInitialized) - : Quarantine(LINKER_INITIALIZED) {} - - NOINLINE void performSanityChecks(); - - void init() { - SanitizerToolName = "Scudo"; - PrimaryAllocatorName = "ScudoPrimary"; - SecondaryAllocatorName = "ScudoSecondary"; - - initFlags(); - - performSanityChecks(); - - // Check if hardware CRC32 is supported in the binary and by the platform, - // if so, opt for the CRC32 hardware version of the checksum. - if (&computeHardwareCRC32 && hasHardwareCRC32()) - atomic_store_relaxed(&HashAlgorithm, CRC32Hardware); - - SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); - Backend.init(common_flags()->allocator_release_to_os_interval_ms); - HardRssLimitMb = common_flags()->hard_rss_limit_mb; - SoftRssLimitMb = common_flags()->soft_rss_limit_mb; - Quarantine.Init( - static_cast(getFlags()->QuarantineSizeKb) << 10, - static_cast(getFlags()->ThreadLocalQuarantineSizeKb) << 10); - QuarantineChunksUpToSize = (Quarantine.GetCacheSize() == 0) ? 0 : - getFlags()->QuarantineChunksUpToSize; - DeallocationTypeMismatch = getFlags()->DeallocationTypeMismatch; - DeleteSizeMismatch = getFlags()->DeleteSizeMismatch; - ZeroContents = getFlags()->ZeroContents; - - if (UNLIKELY(!GetRandom(reinterpret_cast(&Cookie), sizeof(Cookie), - /*blocking=*/false))) { - Cookie = static_cast((NanoTime() >> 12) ^ - (reinterpret_cast(this) >> 4)); - } - - CheckRssLimit = HardRssLimitMb || SoftRssLimitMb; - if (CheckRssLimit) - atomic_store_relaxed(&RssLastCheckedAtNS, MonotonicNanoTime()); - } - - // Helper function that checks for a valid Scudo chunk. nullptr isn't. - bool isValidPointer(const void *Ptr) { - initThreadMaybe(); - if (UNLIKELY(!Ptr)) - return false; - if (!Chunk::isAligned(Ptr)) - return false; - return Chunk::isValid(Ptr); - } - - NOINLINE bool isRssLimitExceeded(); - - // Allocates a chunk. - void * - allocate(uptr Size, uptr Alignment, AllocType Type, - bool ForceZeroContents = false) SANITIZER_NO_THREAD_SAFETY_ANALYSIS { - initThreadMaybe(); - - if (UNLIKELY(Alignment > MaxAlignment)) { - if (AllocatorMayReturnNull()) - return nullptr; - reportAllocationAlignmentTooBig(Alignment, MaxAlignment); - } - if (UNLIKELY(Alignment < MinAlignment)) - Alignment = MinAlignment; - -#ifdef GWP_ASAN_HOOKS - if (UNLIKELY(GuardedAlloc.shouldSample())) { - if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) { - if (SCUDO_CAN_USE_HOOKS && &__sanitizer_malloc_hook) - __sanitizer_malloc_hook(Ptr, Size); - return Ptr; - } - } -#endif // GWP_ASAN_HOOKS - - const uptr NeededSize = RoundUpTo(Size ? Size : 1, MinAlignment) + - Chunk::getHeaderSize(); - const uptr AlignedSize = (Alignment > MinAlignment) ? - NeededSize + (Alignment - Chunk::getHeaderSize()) : NeededSize; - if (UNLIKELY(Size >= MaxAllowedMallocSize) || - UNLIKELY(AlignedSize >= MaxAllowedMallocSize)) { - if (AllocatorMayReturnNull()) - return nullptr; - reportAllocationSizeTooBig(Size, AlignedSize, MaxAllowedMallocSize); - } - - if (CheckRssLimit && UNLIKELY(isRssLimitExceeded())) { - if (AllocatorMayReturnNull()) - return nullptr; - reportRssLimitExceeded(); - } - - // Primary and Secondary backed allocations have a different treatment. We - // deal with alignment requirements of Primary serviced allocations here, - // but the Secondary will take care of its own alignment needs. - void *BackendPtr; - uptr BackendSize; - u8 ClassId; - if (PrimaryT::CanAllocate(AlignedSize, MinAlignment)) { - BackendSize = AlignedSize; - ClassId = SizeClassMap::ClassID(BackendSize); - bool UnlockRequired; - ScudoTSD *TSD = getTSDAndLock(&UnlockRequired); - BackendPtr = Backend.allocatePrimary(&TSD->Cache, ClassId); - if (UnlockRequired) - TSD->unlock(); - } else { - BackendSize = NeededSize; - ClassId = 0; - BackendPtr = Backend.allocateSecondary(BackendSize, Alignment); - } - if (UNLIKELY(!BackendPtr)) { - SetAllocatorOutOfMemory(); - if (AllocatorMayReturnNull()) - return nullptr; - reportOutOfMemory(Size); - } - - // If requested, we will zero out the entire contents of the returned chunk. - if ((ForceZeroContents || ZeroContents) && ClassId) - memset(BackendPtr, 0, PrimaryT::ClassIdToSize(ClassId)); - - UnpackedHeader Header = {}; - uptr UserPtr = reinterpret_cast(BackendPtr) + Chunk::getHeaderSize(); - if (UNLIKELY(!IsAligned(UserPtr, Alignment))) { - // Since the Secondary takes care of alignment, a non-aligned pointer - // means it is from the Primary. It is also the only case where the offset - // field of the header would be non-zero. - DCHECK(ClassId); - const uptr AlignedUserPtr = RoundUpTo(UserPtr, Alignment); - Header.Offset = (AlignedUserPtr - UserPtr) >> MinAlignmentLog; - UserPtr = AlignedUserPtr; - } - DCHECK_LE(UserPtr + Size, reinterpret_cast(BackendPtr) + BackendSize); - Header.State = ChunkAllocated; - Header.AllocType = Type; - if (ClassId) { - Header.ClassId = ClassId; - Header.SizeOrUnusedBytes = Size; - } else { - // The secondary fits the allocations to a page, so the amount of unused - // bytes is the difference between the end of the user allocation and the - // next page boundary. - const uptr PageSize = GetPageSizeCached(); - const uptr TrailingBytes = (UserPtr + Size) & (PageSize - 1); - if (TrailingBytes) - Header.SizeOrUnusedBytes = PageSize - TrailingBytes; - } - void *Ptr = reinterpret_cast(UserPtr); - Chunk::storeHeader(Ptr, &Header); - if (SCUDO_CAN_USE_HOOKS && &__sanitizer_malloc_hook) - __sanitizer_malloc_hook(Ptr, Size); - return Ptr; - } - - // Place a chunk in the quarantine or directly deallocate it in the event of - // a zero-sized quarantine, or if the size of the chunk is greater than the - // quarantine chunk size threshold. - void quarantineOrDeallocateChunk(void *Ptr, UnpackedHeader *Header, uptr Size) - SANITIZER_NO_THREAD_SAFETY_ANALYSIS { - const bool BypassQuarantine = !Size || (Size > QuarantineChunksUpToSize); - if (BypassQuarantine) { - UnpackedHeader NewHeader = *Header; - NewHeader.State = ChunkAvailable; - Chunk::compareExchangeHeader(Ptr, &NewHeader, Header); - void *BackendPtr = Chunk::getBackendPtr(Ptr, Header); - if (Header->ClassId) { - bool UnlockRequired; - ScudoTSD *TSD = getTSDAndLock(&UnlockRequired); - getBackend().deallocatePrimary(&TSD->Cache, BackendPtr, - Header->ClassId); - if (UnlockRequired) - TSD->unlock(); - } else { - getBackend().deallocateSecondary(BackendPtr); - } - } else { - // If a small memory amount was allocated with a larger alignment, we want - // to take that into account. Otherwise the Quarantine would be filled - // with tiny chunks, taking a lot of VA memory. This is an approximation - // of the usable size, that allows us to not call - // GetActuallyAllocatedSize. - const uptr EstimatedSize = Size + (Header->Offset << MinAlignmentLog); - UnpackedHeader NewHeader = *Header; - NewHeader.State = ChunkQuarantine; - Chunk::compareExchangeHeader(Ptr, &NewHeader, Header); - bool UnlockRequired; - ScudoTSD *TSD = getTSDAndLock(&UnlockRequired); - Quarantine.Put(getQuarantineCache(TSD), QuarantineCallback(&TSD->Cache), - Ptr, EstimatedSize); - if (UnlockRequired) - TSD->unlock(); - } - } - - // Deallocates a Chunk, which means either adding it to the quarantine or - // directly returning it to the backend if criteria are met. - void deallocate(void *Ptr, uptr DeleteSize, uptr DeleteAlignment, - AllocType Type) { - // For a deallocation, we only ensure minimal initialization, meaning thread - // local data will be left uninitialized for now (when using ELF TLS). The - // fallback cache will be used instead. This is a workaround for a situation - // where the only heap operation performed in a thread would be a free past - // the TLS destructors, ending up in initialized thread specific data never - // being destroyed properly. Any other heap operation will do a full init. - initThreadMaybe(/*MinimalInit=*/true); - if (SCUDO_CAN_USE_HOOKS && &__sanitizer_free_hook) - __sanitizer_free_hook(Ptr); - if (UNLIKELY(!Ptr)) - return; - -#ifdef GWP_ASAN_HOOKS - if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) { - GuardedAlloc.deallocate(Ptr); - return; - } -#endif // GWP_ASAN_HOOKS - - if (UNLIKELY(!Chunk::isAligned(Ptr))) - dieWithMessage("misaligned pointer when deallocating address %p\n", Ptr); - UnpackedHeader Header; - Chunk::loadHeader(Ptr, &Header); - if (UNLIKELY(Header.State != ChunkAllocated)) - dieWithMessage("invalid chunk state when deallocating address %p\n", Ptr); - if (DeallocationTypeMismatch) { - // The deallocation type has to match the allocation one. - if (Header.AllocType != Type) { - // With the exception of memalign'd Chunks, that can be still be free'd. - if (Header.AllocType != FromMemalign || Type != FromMalloc) - dieWithMessage("allocation type mismatch when deallocating address " - "%p\n", Ptr); - } - } - const uptr Size = Chunk::getSize(Ptr, &Header); - if (DeleteSizeMismatch) { - if (DeleteSize && DeleteSize != Size) - dieWithMessage("invalid sized delete when deallocating address %p\n", - Ptr); - } - (void)DeleteAlignment; // TODO(kostyak): verify that the alignment matches. - quarantineOrDeallocateChunk(Ptr, &Header, Size); - } - - // Reallocates a chunk. We can save on a new allocation if the new requested - // size still fits in the chunk. - void *reallocate(void *OldPtr, uptr NewSize) { - initThreadMaybe(); - -#ifdef GWP_ASAN_HOOKS - if (UNLIKELY(GuardedAlloc.pointerIsMine(OldPtr))) { - size_t OldSize = GuardedAlloc.getSize(OldPtr); - void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc); - if (NewPtr) - memcpy(NewPtr, OldPtr, (NewSize < OldSize) ? NewSize : OldSize); - GuardedAlloc.deallocate(OldPtr); - return NewPtr; - } -#endif // GWP_ASAN_HOOKS - - if (UNLIKELY(!Chunk::isAligned(OldPtr))) - dieWithMessage("misaligned address when reallocating address %p\n", - OldPtr); - UnpackedHeader OldHeader; - Chunk::loadHeader(OldPtr, &OldHeader); - if (UNLIKELY(OldHeader.State != ChunkAllocated)) - dieWithMessage("invalid chunk state when reallocating address %p\n", - OldPtr); - if (DeallocationTypeMismatch) { - if (UNLIKELY(OldHeader.AllocType != FromMalloc)) - dieWithMessage("allocation type mismatch when reallocating address " - "%p\n", OldPtr); - } - const uptr UsableSize = Chunk::getUsableSize(OldPtr, &OldHeader); - // The new size still fits in the current chunk, and the size difference - // is reasonable. - if (NewSize <= UsableSize && - (UsableSize - NewSize) < (SizeClassMap::kMaxSize / 2)) { - UnpackedHeader NewHeader = OldHeader; - NewHeader.SizeOrUnusedBytes = - OldHeader.ClassId ? NewSize : UsableSize - NewSize; - Chunk::compareExchangeHeader(OldPtr, &NewHeader, &OldHeader); - return OldPtr; - } - // Otherwise, we have to allocate a new chunk and copy the contents of the - // old one. - void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc); - if (NewPtr) { - const uptr OldSize = OldHeader.ClassId ? OldHeader.SizeOrUnusedBytes : - UsableSize - OldHeader.SizeOrUnusedBytes; - memcpy(NewPtr, OldPtr, Min(NewSize, UsableSize)); - quarantineOrDeallocateChunk(OldPtr, &OldHeader, OldSize); - } - return NewPtr; - } - - // Helper function that returns the actual usable size of a chunk. - uptr getUsableSize(const void *Ptr) { - initThreadMaybe(); - if (UNLIKELY(!Ptr)) - return 0; - -#ifdef GWP_ASAN_HOOKS - if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) - return GuardedAlloc.getSize(Ptr); -#endif // GWP_ASAN_HOOKS - - UnpackedHeader Header; - Chunk::loadHeader(Ptr, &Header); - // Getting the usable size of a chunk only makes sense if it's allocated. - if (UNLIKELY(Header.State != ChunkAllocated)) - dieWithMessage("invalid chunk state when sizing address %p\n", Ptr); - return Chunk::getUsableSize(Ptr, &Header); - } - - void *calloc(uptr NMemB, uptr Size) { - initThreadMaybe(); - if (UNLIKELY(CheckForCallocOverflow(NMemB, Size))) { - if (AllocatorMayReturnNull()) - return nullptr; - reportCallocOverflow(NMemB, Size); - } - return allocate(NMemB * Size, MinAlignment, FromMalloc, true); - } - - void commitBack(ScudoTSD *TSD) { - Quarantine.Drain(getQuarantineCache(TSD), QuarantineCallback(&TSD->Cache)); - Backend.destroyCache(&TSD->Cache); - } - - uptr getStats(AllocatorStat StatType) { - initThreadMaybe(); - uptr stats[AllocatorStatCount]; - Backend.getStats(stats); - return stats[StatType]; - } - - bool canReturnNull() { - initThreadMaybe(); - return AllocatorMayReturnNull(); - } - - void setRssLimit(uptr LimitMb, bool HardLimit) { - if (HardLimit) - HardRssLimitMb = LimitMb; - else - SoftRssLimitMb = LimitMb; - CheckRssLimit = HardRssLimitMb || SoftRssLimitMb; - } - - void printStats() { - initThreadMaybe(); - Backend.printStats(); - } -}; - -NOINLINE void Allocator::performSanityChecks() { - // Verify that the header offset field can hold the maximum offset. In the - // case of the Secondary allocator, it takes care of alignment and the - // offset will always be 0. In the case of the Primary, the worst case - // scenario happens in the last size class, when the backend allocation - // would already be aligned on the requested alignment, which would happen - // to be the maximum alignment that would fit in that size class. As a - // result, the maximum offset will be at most the maximum alignment for the - // last size class minus the header size, in multiples of MinAlignment. - UnpackedHeader Header = {}; - const uptr MaxPrimaryAlignment = - 1 << MostSignificantSetBitIndex(SizeClassMap::kMaxSize - MinAlignment); - const uptr MaxOffset = - (MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog; - Header.Offset = MaxOffset; - if (Header.Offset != MaxOffset) - dieWithMessage("maximum possible offset doesn't fit in header\n"); - // Verify that we can fit the maximum size or amount of unused bytes in the - // header. Given that the Secondary fits the allocation to a page, the worst - // case scenario happens in the Primary. It will depend on the second to - // last and last class sizes, as well as the dynamic base for the Primary. - // The following is an over-approximation that works for our needs. - const uptr MaxSizeOrUnusedBytes = SizeClassMap::kMaxSize - 1; - Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes; - if (Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes) - dieWithMessage("maximum possible unused bytes doesn't fit in header\n"); - - const uptr LargestClassId = SizeClassMap::kLargestClassID; - Header.ClassId = LargestClassId; - if (Header.ClassId != LargestClassId) - dieWithMessage("largest class ID doesn't fit in header\n"); -} - -// Opportunistic RSS limit check. This will update the RSS limit status, if -// it can, every 250ms, otherwise it will just return the current one. -NOINLINE bool Allocator::isRssLimitExceeded() { - u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS); - const u64 CurrentCheck = MonotonicNanoTime(); - if (LIKELY(CurrentCheck < LastCheck + (250ULL * 1000000ULL))) - return atomic_load_relaxed(&RssLimitExceeded); - if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck, - CurrentCheck, memory_order_relaxed)) - return atomic_load_relaxed(&RssLimitExceeded); - // TODO(kostyak): We currently use sanitizer_common's GetRSS which reads the - // RSS from /proc/self/statm by default. We might want to - // call getrusage directly, even if it's less accurate. - const uptr CurrentRssMb = GetRSS() >> 20; - if (HardRssLimitMb && UNLIKELY(HardRssLimitMb < CurrentRssMb)) - dieWithMessage("hard RSS limit exhausted (%zdMb vs %zdMb)\n", - HardRssLimitMb, CurrentRssMb); - if (SoftRssLimitMb) { - if (atomic_load_relaxed(&RssLimitExceeded)) { - if (CurrentRssMb <= SoftRssLimitMb) - atomic_store_relaxed(&RssLimitExceeded, false); - } else { - if (CurrentRssMb > SoftRssLimitMb) { - atomic_store_relaxed(&RssLimitExceeded, true); - Printf("Scudo INFO: soft RSS limit exhausted (%zdMb vs %zdMb)\n", - SoftRssLimitMb, CurrentRssMb); - } - } - } - return atomic_load_relaxed(&RssLimitExceeded); -} - -static Allocator Instance(LINKER_INITIALIZED); - -static BackendT &getBackend() { - return Instance.Backend; -} - -void initScudo() { - Instance.init(); -#ifdef GWP_ASAN_HOOKS - gwp_asan::options::initOptions(__sanitizer::GetEnv("GWP_ASAN_OPTIONS"), - Printf); - gwp_asan::options::Options &Opts = gwp_asan::options::getOptions(); - Opts.Backtrace = gwp_asan::backtrace::getBacktraceFunction(); - GuardedAlloc.init(Opts); - - if (Opts.InstallSignalHandlers) - gwp_asan::segv_handler::installSignalHandlers( - &GuardedAlloc, __sanitizer::Printf, - gwp_asan::backtrace::getPrintBacktraceFunction(), - gwp_asan::backtrace::getSegvBacktraceFunction()); -#endif // GWP_ASAN_HOOKS -} - -void ScudoTSD::init() { - getBackend().initCache(&Cache); - memset(QuarantineCachePlaceHolder, 0, sizeof(QuarantineCachePlaceHolder)); -} - -void ScudoTSD::commitBack() { - Instance.commitBack(this); -} - -void *scudoAllocate(uptr Size, uptr Alignment, AllocType Type) { - if (Alignment && UNLIKELY(!IsPowerOfTwo(Alignment))) { - errno = EINVAL; - if (Instance.canReturnNull()) - return nullptr; - reportAllocationAlignmentNotPowerOfTwo(Alignment); - } - return SetErrnoOnNull(Instance.allocate(Size, Alignment, Type)); -} - -void scudoDeallocate(void *Ptr, uptr Size, uptr Alignment, AllocType Type) { - Instance.deallocate(Ptr, Size, Alignment, Type); -} - -void *scudoRealloc(void *Ptr, uptr Size) { - if (!Ptr) - return SetErrnoOnNull(Instance.allocate(Size, MinAlignment, FromMalloc)); - if (Size == 0) { - Instance.deallocate(Ptr, 0, 0, FromMalloc); - return nullptr; - } - return SetErrnoOnNull(Instance.reallocate(Ptr, Size)); -} - -void *scudoCalloc(uptr NMemB, uptr Size) { - return SetErrnoOnNull(Instance.calloc(NMemB, Size)); -} - -void *scudoValloc(uptr Size) { - return SetErrnoOnNull( - Instance.allocate(Size, GetPageSizeCached(), FromMemalign)); -} - -void *scudoPvalloc(uptr Size) { - const uptr PageSize = GetPageSizeCached(); - if (UNLIKELY(CheckForPvallocOverflow(Size, PageSize))) { - errno = ENOMEM; - if (Instance.canReturnNull()) - return nullptr; - reportPvallocOverflow(Size); - } - // pvalloc(0) should allocate one page. - Size = Size ? RoundUpTo(Size, PageSize) : PageSize; - return SetErrnoOnNull(Instance.allocate(Size, PageSize, FromMemalign)); -} - -int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size) { - if (UNLIKELY(!CheckPosixMemalignAlignment(Alignment))) { - if (!Instance.canReturnNull()) - reportInvalidPosixMemalignAlignment(Alignment); - return EINVAL; - } - void *Ptr = Instance.allocate(Size, Alignment, FromMemalign); - if (UNLIKELY(!Ptr)) - return ENOMEM; - *MemPtr = Ptr; - return 0; -} - -void *scudoAlignedAlloc(uptr Alignment, uptr Size) { - if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(Alignment, Size))) { - errno = EINVAL; - if (Instance.canReturnNull()) - return nullptr; - reportInvalidAlignedAllocAlignment(Size, Alignment); - } - return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMalloc)); -} - -uptr scudoMallocUsableSize(void *Ptr) { - return Instance.getUsableSize(Ptr); -} - -} // namespace __scudo - -using namespace __scudo; - -// MallocExtension helper functions - -uptr __sanitizer_get_current_allocated_bytes() { - return Instance.getStats(AllocatorStatAllocated); -} - -uptr __sanitizer_get_heap_size() { - return Instance.getStats(AllocatorStatMapped); -} - -uptr __sanitizer_get_free_bytes() { - return 1; -} - -uptr __sanitizer_get_unmapped_bytes() { - return 1; -} - -uptr __sanitizer_get_estimated_allocated_size(uptr Size) { - return Size; -} - -int __sanitizer_get_ownership(const void *Ptr) { - return Instance.isValidPointer(Ptr); -} - -uptr __sanitizer_get_allocated_size(const void *Ptr) { - return Instance.getUsableSize(Ptr); -} - -#if !SANITIZER_SUPPORTS_WEAK_HOOKS -SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook, - void *Ptr, uptr Size) { - (void)Ptr; - (void)Size; -} - -SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *Ptr) { - (void)Ptr; -} -#endif - -// Interface functions - -void __scudo_set_rss_limit(uptr LimitMb, s32 HardLimit) { - if (!SCUDO_CAN_USE_PUBLIC_INTERFACE) - return; - Instance.setRssLimit(LimitMb, !!HardLimit); -} - -void __scudo_print_stats() { - Instance.printStats(); -} diff --git a/compiler-rt/lib/scudo/scudo_allocator_combined.h b/compiler-rt/lib/scudo/scudo_allocator_combined.h deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_allocator_combined.h +++ /dev/null @@ -1,75 +0,0 @@ -//===-- scudo_allocator_combined.h ------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Scudo Combined Allocator, dispatches allocation & deallocation requests to -/// the Primary or the Secondary backend allocators. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_ALLOCATOR_COMBINED_H_ -#define SCUDO_ALLOCATOR_COMBINED_H_ - -#ifndef SCUDO_ALLOCATOR_H_ -# error "This file must be included inside scudo_allocator.h." -#endif - -class CombinedAllocator { - public: - using PrimaryAllocator = PrimaryT; - using SecondaryAllocator = SecondaryT; - using AllocatorCache = typename PrimaryAllocator::AllocatorCache; - void init(s32 ReleaseToOSIntervalMs) { - Primary.Init(ReleaseToOSIntervalMs); - Secondary.Init(); - Stats.Init(); - } - - // Primary allocations are always MinAlignment aligned, and as such do not - // require an Alignment parameter. - void *allocatePrimary(AllocatorCache *Cache, uptr ClassId) { - return Cache->Allocate(&Primary, ClassId); - } - - // Secondary allocations do not require a Cache, but do require an Alignment - // parameter. - void *allocateSecondary(uptr Size, uptr Alignment) { - return Secondary.Allocate(&Stats, Size, Alignment); - } - - void deallocatePrimary(AllocatorCache *Cache, void *Ptr, uptr ClassId) { - Cache->Deallocate(&Primary, ClassId, Ptr); - } - - void deallocateSecondary(void *Ptr) { - Secondary.Deallocate(&Stats, Ptr); - } - - void initCache(AllocatorCache *Cache) { - Cache->Init(&Stats); - } - - void destroyCache(AllocatorCache *Cache) { - Cache->Destroy(&Primary, &Stats); - } - - void getStats(AllocatorStatCounters StatType) const { - Stats.Get(StatType); - } - - void printStats() { - Primary.PrintStats(); - Secondary.PrintStats(); - } - - private: - PrimaryAllocator Primary; - SecondaryAllocator Secondary; - AllocatorGlobalStats Stats; -}; - -#endif // SCUDO_ALLOCATOR_COMBINED_H_ diff --git a/compiler-rt/lib/scudo/scudo_allocator_secondary.h b/compiler-rt/lib/scudo/scudo_allocator_secondary.h deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_allocator_secondary.h +++ /dev/null @@ -1,192 +0,0 @@ -//===-- scudo_allocator_secondary.h -----------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Scudo Secondary Allocator. -/// This services allocation that are too large to be serviced by the Primary -/// Allocator. It is directly backed by the memory mapping functions of the -/// operating system. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_ALLOCATOR_SECONDARY_H_ -#define SCUDO_ALLOCATOR_SECONDARY_H_ - -#ifndef SCUDO_ALLOCATOR_H_ -# error "This file must be included inside scudo_allocator.h." -#endif - -// Secondary backed allocations are standalone chunks that contain extra -// information stored in a LargeChunk::Header prior to the frontend's header. -// -// The secondary takes care of alignment requirements (so that it can release -// unnecessary pages in the rare event of larger alignments), and as such must -// know about the frontend's header size. -// -// Since Windows doesn't support partial releasing of a reserved memory region, -// we have to keep track of both the reserved and the committed memory. -// -// The resulting chunk resembles the following: -// -// +--------------------+ -// | Guard page(s) | -// +--------------------+ -// | Unused space* | -// +--------------------+ -// | LargeChunk::Header | -// +--------------------+ -// | {Unp,P}ackedHeader | -// +--------------------+ -// | Data (aligned) | -// +--------------------+ -// | Unused space** | -// +--------------------+ -// | Guard page(s) | -// +--------------------+ - -namespace LargeChunk { -struct Header { - ReservedAddressRange StoredRange; - uptr CommittedSize; - uptr Size; -}; -constexpr uptr getHeaderSize() { - return RoundUpTo(sizeof(Header), MinAlignment); -} -static Header *getHeader(uptr Ptr) { - return reinterpret_cast
(Ptr - getHeaderSize()); -} -static Header *getHeader(const void *Ptr) { - return getHeader(reinterpret_cast(Ptr)); -} -} // namespace LargeChunk - -class LargeMmapAllocator { - public: - void Init() { - internal_memset(this, 0, sizeof(*this)); - } - - void *Allocate(AllocatorStats *Stats, uptr Size, uptr Alignment) { - const uptr UserSize = Size - Chunk::getHeaderSize(); - // The Scudo frontend prevents us from allocating more than - // MaxAllowedMallocSize, so integer overflow checks would be superfluous. - uptr ReservedSize = Size + LargeChunk::getHeaderSize(); - if (UNLIKELY(Alignment > MinAlignment)) - ReservedSize += Alignment; - const uptr PageSize = GetPageSizeCached(); - ReservedSize = RoundUpTo(ReservedSize, PageSize); - // Account for 2 guard pages, one before and one after the chunk. - ReservedSize += 2 * PageSize; - - ReservedAddressRange AddressRange; - uptr ReservedBeg = AddressRange.Init(ReservedSize, SecondaryAllocatorName); - if (UNLIKELY(ReservedBeg == ~static_cast(0))) - return nullptr; - // A page-aligned pointer is assumed after that, so check it now. - DCHECK(IsAligned(ReservedBeg, PageSize)); - uptr ReservedEnd = ReservedBeg + ReservedSize; - // The beginning of the user area for that allocation comes after the - // initial guard page, and both headers. This is the pointer that has to - // abide by alignment requirements. - uptr CommittedBeg = ReservedBeg + PageSize; - uptr UserBeg = CommittedBeg + HeadersSize; - uptr UserEnd = UserBeg + UserSize; - uptr CommittedEnd = RoundUpTo(UserEnd, PageSize); - - // In the rare event of larger alignments, we will attempt to fit the mmap - // area better and unmap extraneous memory. This will also ensure that the - // offset and unused bytes field of the header stay small. - if (UNLIKELY(Alignment > MinAlignment)) { - if (!IsAligned(UserBeg, Alignment)) { - UserBeg = RoundUpTo(UserBeg, Alignment); - CommittedBeg = RoundDownTo(UserBeg - HeadersSize, PageSize); - const uptr NewReservedBeg = CommittedBeg - PageSize; - DCHECK_GE(NewReservedBeg, ReservedBeg); - if (!SANITIZER_WINDOWS && NewReservedBeg != ReservedBeg) { - AddressRange.Unmap(ReservedBeg, NewReservedBeg - ReservedBeg); - ReservedBeg = NewReservedBeg; - } - UserEnd = UserBeg + UserSize; - CommittedEnd = RoundUpTo(UserEnd, PageSize); - } - const uptr NewReservedEnd = CommittedEnd + PageSize; - DCHECK_LE(NewReservedEnd, ReservedEnd); - if (!SANITIZER_WINDOWS && NewReservedEnd != ReservedEnd) { - AddressRange.Unmap(NewReservedEnd, ReservedEnd - NewReservedEnd); - ReservedEnd = NewReservedEnd; - } - } - - DCHECK_LE(UserEnd, CommittedEnd); - const uptr CommittedSize = CommittedEnd - CommittedBeg; - // Actually mmap the memory, preserving the guard pages on either sides. - CHECK_EQ(CommittedBeg, AddressRange.Map(CommittedBeg, CommittedSize)); - const uptr Ptr = UserBeg - Chunk::getHeaderSize(); - LargeChunk::Header *H = LargeChunk::getHeader(Ptr); - H->StoredRange = AddressRange; - H->Size = CommittedEnd - Ptr; - H->CommittedSize = CommittedSize; - - // The primary adds the whole class size to the stats when allocating a - // chunk, so we will do something similar here. But we will not account for - // the guard pages. - { - SpinMutexLock l(&StatsMutex); - Stats->Add(AllocatorStatAllocated, CommittedSize); - Stats->Add(AllocatorStatMapped, CommittedSize); - AllocatedBytes += CommittedSize; - if (LargestSize < CommittedSize) - LargestSize = CommittedSize; - NumberOfAllocs++; - } - - return reinterpret_cast(Ptr); - } - - void Deallocate(AllocatorStats *Stats, void *Ptr) { - LargeChunk::Header *H = LargeChunk::getHeader(Ptr); - // Since we're unmapping the entirety of where the ReservedAddressRange - // actually is, copy onto the stack. - ReservedAddressRange AddressRange = H->StoredRange; - const uptr Size = H->CommittedSize; - { - SpinMutexLock l(&StatsMutex); - Stats->Sub(AllocatorStatAllocated, Size); - Stats->Sub(AllocatorStatMapped, Size); - FreedBytes += Size; - NumberOfFrees++; - } - AddressRange.Unmap(reinterpret_cast(AddressRange.base()), - AddressRange.size()); - } - - static uptr GetActuallyAllocatedSize(void *Ptr) { - return LargeChunk::getHeader(Ptr)->Size; - } - - void PrintStats() { - Printf("Stats: LargeMmapAllocator: allocated %zd times (%zd K), " - "freed %zd times (%zd K), remains %zd (%zd K) max %zd M\n", - NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees, - FreedBytes >> 10, NumberOfAllocs - NumberOfFrees, - (AllocatedBytes - FreedBytes) >> 10, LargestSize >> 20); - } - - private: - static constexpr uptr HeadersSize = - LargeChunk::getHeaderSize() + Chunk::getHeaderSize(); - - StaticSpinMutex StatsMutex; - u32 NumberOfAllocs; - u32 NumberOfFrees; - uptr AllocatedBytes; - uptr FreedBytes; - uptr LargestSize; -}; - -#endif // SCUDO_ALLOCATOR_SECONDARY_H_ diff --git a/compiler-rt/lib/scudo/scudo_crc32.h b/compiler-rt/lib/scudo/scudo_crc32.h deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_crc32.h +++ /dev/null @@ -1,104 +0,0 @@ -//===-- scudo_crc32.h -------------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Scudo chunk header checksum related definitions. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_CRC32_H_ -#define SCUDO_CRC32_H_ - -#include "sanitizer_common/sanitizer_internal_defs.h" - -// Hardware CRC32 is supported at compilation via the following: -// - for i386 & x86_64: -mcrc32 (earlier: -msse4.2) -// - for ARM & AArch64: -march=armv8-a+crc or -mcrc -// An additional check must be performed at runtime as well to make sure the -// emitted instructions are valid on the target host. - -#if defined(__CRC32__) || defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) -# if defined(__CRC32__) -// NB: clang has but GCC does not -# include -# define CRC32_INTRINSIC FIRST_32_SECOND_64(__builtin_ia32_crc32si, __builtin_ia32_crc32di) -# elif defined(__SSE4_2__) -# include -# define CRC32_INTRINSIC FIRST_32_SECOND_64(_mm_crc32_u32, _mm_crc32_u64) -# endif -# ifdef __ARM_FEATURE_CRC32 -# include -# define CRC32_INTRINSIC FIRST_32_SECOND_64(__crc32cw, __crc32cd) -# endif -#endif // defined(__CRC32__) || defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) - -namespace __scudo { - -enum : u8 { - CRC32Software = 0, - CRC32Hardware = 1, -}; - -static const u32 CRC32Table[] = { - 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, - 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, - 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, - 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, - 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, - 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, - 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, - 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, - 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, - 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, - 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, - 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, - 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, - 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, - 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, - 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, - 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, - 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, - 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, - 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, - 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, - 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, - 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, - 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, - 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, - 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, - 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, - 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, - 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, - 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, - 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, - 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, - 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, - 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, - 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, - 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, - 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, - 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, - 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, - 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, - 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, - 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, - 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d -}; - -inline u32 computeSoftwareCRC32(u32 Crc, uptr Data) { - for (uptr i = 0; i < sizeof(Data); i++) { - Crc = CRC32Table[(Crc ^ Data) & 0xff] ^ (Crc >> 8); - Data >>= 8; - } - return Crc; -} - -SANITIZER_WEAK_ATTRIBUTE u32 computeHardwareCRC32(u32 Crc, uptr Data); - -} // namespace __scudo - -#endif // SCUDO_CRC32_H_ diff --git a/compiler-rt/lib/scudo/scudo_crc32.cpp b/compiler-rt/lib/scudo/scudo_crc32.cpp deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_crc32.cpp +++ /dev/null @@ -1,24 +0,0 @@ -//===-- scudo_crc32.cpp -----------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// CRC32 function leveraging hardware specific instructions. This has to be -/// kept separated to restrict the use of compiler specific flags to this file. -/// -//===----------------------------------------------------------------------===// - -#include "scudo_crc32.h" - -namespace __scudo { - -#if defined(__CRC32__) || defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) -u32 computeHardwareCRC32(u32 Crc, uptr Data) { - return CRC32_INTRINSIC(Crc, Data); -} -#endif // defined(__CRC32__) || defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) - -} // namespace __scudo diff --git a/compiler-rt/lib/scudo/scudo_errors.h b/compiler-rt/lib/scudo/scudo_errors.h deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_errors.h +++ /dev/null @@ -1,34 +0,0 @@ -//===-- scudo_errors.h ------------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Header for scudo_errors.cpp. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_ERRORS_H_ -#define SCUDO_ERRORS_H_ - -#include "sanitizer_common/sanitizer_internal_defs.h" - -namespace __scudo { - -void NORETURN reportCallocOverflow(uptr Count, uptr Size); -void NORETURN reportPvallocOverflow(uptr Size); -void NORETURN reportAllocationAlignmentTooBig(uptr Alignment, - uptr MaxAlignment); -void NORETURN reportAllocationAlignmentNotPowerOfTwo(uptr Alignment); -void NORETURN reportInvalidPosixMemalignAlignment(uptr Alignment); -void NORETURN reportInvalidAlignedAllocAlignment(uptr Size, uptr Alignment); -void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize, - uptr MaxSize); -void NORETURN reportRssLimitExceeded(); -void NORETURN reportOutOfMemory(uptr RequestedSize); - -} // namespace __scudo - -#endif // SCUDO_ERRORS_H_ diff --git a/compiler-rt/lib/scudo/scudo_errors.cpp b/compiler-rt/lib/scudo/scudo_errors.cpp deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_errors.cpp +++ /dev/null @@ -1,77 +0,0 @@ -//===-- scudo_errors.cpp ----------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Verbose termination functions. -/// -//===----------------------------------------------------------------------===// - -#include "scudo_utils.h" - -#include "sanitizer_common/sanitizer_flags.h" - -namespace __scudo { - -void NORETURN reportCallocOverflow(uptr Count, uptr Size) { - dieWithMessage("calloc parameters overflow: count * size (%zd * %zd) cannot " - "be represented with type size_t\n", Count, Size); -} - -void NORETURN reportPvallocOverflow(uptr Size) { - dieWithMessage("pvalloc parameters overflow: size 0x%zx rounded up to system " - "page size 0x%zx cannot be represented in type size_t\n", Size, - GetPageSizeCached()); -} - -void NORETURN reportAllocationAlignmentTooBig(uptr Alignment, - uptr MaxAlignment) { - dieWithMessage("invalid allocation alignment: %zd exceeds maximum supported " - "allocation of %zd\n", Alignment, MaxAlignment); -} - -void NORETURN reportAllocationAlignmentNotPowerOfTwo(uptr Alignment) { - dieWithMessage("invalid allocation alignment: %zd, alignment must be a power " - "of two\n", Alignment); -} - -void NORETURN reportInvalidPosixMemalignAlignment(uptr Alignment) { - dieWithMessage( - "invalid alignment requested in posix_memalign: %zd, alignment" - " must be a power of two and a multiple of sizeof(void *) == %zd\n", - Alignment, sizeof(void *)); -} - -void NORETURN reportInvalidAlignedAllocAlignment(uptr Size, uptr Alignment) { -#if SANITIZER_POSIX - dieWithMessage("invalid alignment requested in aligned_alloc: %zd, alignment " - "must be a power of two and the requested size 0x%zx must be a multiple " - "of alignment\n", Alignment, Size); -#else - dieWithMessage("invalid alignment requested in aligned_alloc: %zd, the " - "requested size 0x%zx must be a multiple of alignment\n", Alignment, - Size); -#endif -} - -void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize, - uptr MaxSize) { - dieWithMessage("requested allocation size 0x%zx (0x%zx after adjustments) " - "exceeds maximum supported size of 0x%zx\n", UserSize, TotalSize, - MaxSize); -} - -void NORETURN reportRssLimitExceeded() { - dieWithMessage("specified RSS limit exceeded, currently set to " - "soft_rss_limit_mb=%zd\n", common_flags()->soft_rss_limit_mb); -} - -void NORETURN reportOutOfMemory(uptr RequestedSize) { - dieWithMessage("allocator is out of memory trying to allocate 0x%zx bytes\n", - RequestedSize); -} - -} // namespace __scudo diff --git a/compiler-rt/lib/scudo/scudo_flags.h b/compiler-rt/lib/scudo/scudo_flags.h deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_flags.h +++ /dev/null @@ -1,32 +0,0 @@ -//===-- scudo_flags.h -------------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Header for scudo_flags.cpp. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_FLAGS_H_ -#define SCUDO_FLAGS_H_ - -namespace __scudo { - -struct Flags { -#define SCUDO_FLAG(Type, Name, DefaultValue, Description) Type Name; -#include "scudo_flags.inc" -#undef SCUDO_FLAG - - void setDefaults(); -}; - -Flags *getFlags(); - -void initFlags(); - -} // namespace __scudo - -#endif // SCUDO_FLAGS_H_ diff --git a/compiler-rt/lib/scudo/scudo_flags.cpp b/compiler-rt/lib/scudo/scudo_flags.cpp deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_flags.cpp +++ /dev/null @@ -1,136 +0,0 @@ -//===-- scudo_flags.cpp -----------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Hardened Allocator flag parsing logic. -/// -//===----------------------------------------------------------------------===// - -#include "scudo_flags.h" -#include "scudo_interface_internal.h" -#include "scudo_utils.h" - -#include "sanitizer_common/sanitizer_flags.h" -#include "sanitizer_common/sanitizer_flag_parser.h" - -namespace __scudo { - -static Flags ScudoFlags; // Use via getFlags(). - -void Flags::setDefaults() { -#define SCUDO_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; -#include "scudo_flags.inc" -#undef SCUDO_FLAG -} - -static void RegisterScudoFlags(FlagParser *parser, Flags *f) { -#define SCUDO_FLAG(Type, Name, DefaultValue, Description) \ - RegisterFlag(parser, #Name, Description, &f->Name); -#include "scudo_flags.inc" -#undef SCUDO_FLAG -} - -static const char *getCompileDefinitionScudoDefaultOptions() { -#ifdef SCUDO_DEFAULT_OPTIONS - return SANITIZER_STRINGIFY(SCUDO_DEFAULT_OPTIONS); -#else - return ""; -#endif -} - -static const char *getScudoDefaultOptions() { - return (&__scudo_default_options) ? __scudo_default_options() : ""; -} - -void initFlags() { - SetCommonFlagsDefaults(); - { - CommonFlags cf; - cf.CopyFrom(*common_flags()); - cf.exitcode = 1; - OverrideCommonFlags(cf); - } - Flags *f = getFlags(); - f->setDefaults(); - - FlagParser ScudoParser; - RegisterScudoFlags(&ScudoParser, f); - RegisterCommonFlags(&ScudoParser); - - // Override from compile definition. - ScudoParser.ParseString(getCompileDefinitionScudoDefaultOptions()); - - // Override from user-specified string. - ScudoParser.ParseString(getScudoDefaultOptions()); - - // Override from environment. - ScudoParser.ParseStringFromEnv("SCUDO_OPTIONS"); - - InitializeCommonFlags(); - - // Sanity checks and default settings for the Quarantine parameters. - - if (f->QuarantineSizeMb >= 0) { - // Backward compatible logic if QuarantineSizeMb is set. - if (f->QuarantineSizeKb >= 0) { - dieWithMessage("ERROR: please use either QuarantineSizeMb (deprecated) " - "or QuarantineSizeKb, but not both\n"); - } - if (f->QuarantineChunksUpToSize >= 0) { - dieWithMessage("ERROR: QuarantineChunksUpToSize cannot be used in " - " conjunction with the deprecated QuarantineSizeMb option\n"); - } - // If everything is in order, update QuarantineSizeKb accordingly. - f->QuarantineSizeKb = f->QuarantineSizeMb * 1024; - } else { - // Otherwise proceed with the new options. - if (f->QuarantineSizeKb < 0) { - const int DefaultQuarantineSizeKb = FIRST_32_SECOND_64(64, 256); - f->QuarantineSizeKb = DefaultQuarantineSizeKb; - } - if (f->QuarantineChunksUpToSize < 0) { - const int DefaultQuarantineChunksUpToSize = FIRST_32_SECOND_64(512, 2048); - f->QuarantineChunksUpToSize = DefaultQuarantineChunksUpToSize; - } - } - - // We enforce an upper limit for the chunk quarantine threshold of 4Mb. - if (f->QuarantineChunksUpToSize > (4 * 1024 * 1024)) { - dieWithMessage("ERROR: the chunk quarantine threshold is too large\n"); - } - - // We enforce an upper limit for the quarantine size of 32Mb. - if (f->QuarantineSizeKb > (32 * 1024)) { - dieWithMessage("ERROR: the quarantine size is too large\n"); - } - - if (f->ThreadLocalQuarantineSizeKb < 0) { - const int DefaultThreadLocalQuarantineSizeKb = FIRST_32_SECOND_64(16, 64); - f->ThreadLocalQuarantineSizeKb = DefaultThreadLocalQuarantineSizeKb; - } - // And an upper limit of 8Mb for the thread quarantine cache. - if (f->ThreadLocalQuarantineSizeKb > (8 * 1024)) { - dieWithMessage("ERROR: the per thread quarantine cache size is too " - "large\n"); - } - if (f->ThreadLocalQuarantineSizeKb == 0 && f->QuarantineSizeKb > 0) { - dieWithMessage("ERROR: ThreadLocalQuarantineSizeKb can be set to 0 only " - "when QuarantineSizeKb is set to 0\n"); - } -} - -Flags *getFlags() { - return &ScudoFlags; -} - -} // namespace __scudo - -#if !SANITIZER_SUPPORTS_WEAK_HOOKS -SANITIZER_INTERFACE_WEAK_DEF(const char*, __scudo_default_options, void) { - return ""; -} -#endif diff --git a/compiler-rt/lib/scudo/scudo_flags.inc b/compiler-rt/lib/scudo/scudo_flags.inc deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_flags.inc +++ /dev/null @@ -1,48 +0,0 @@ -//===-- scudo_flags.inc -----------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Hardened Allocator runtime flags. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_FLAG -# error "Define SCUDO_FLAG prior to including this file!" -#endif - -SCUDO_FLAG(int, QuarantineSizeMb, -1, - "Deprecated. Please use QuarantineSizeKb.") - -// Default value is set in scudo_flags.cpp based on architecture. -SCUDO_FLAG(int, QuarantineSizeKb, -1, - "Size in KB of quarantine used to delay the actual deallocation of " - "chunks. Lower value may reduce memory usage but decrease the " - "effectiveness of the mitigation. Defaults to 64KB (32-bit) or " - "256KB (64-bit)") - -// Default value is set in scudo_flags.cpp based on architecture. -SCUDO_FLAG(int, ThreadLocalQuarantineSizeKb, -1, - "Size in KB of per-thread cache used to offload the global " - "quarantine. Lower value may reduce memory usage but might increase " - "the contention on the global quarantine. Defaults to 16KB (32-bit) " - "or 64KB (64-bit)") - -// Default value is set in scudo_flags.cpp based on architecture. -SCUDO_FLAG(int, QuarantineChunksUpToSize, -1, - "Size in bytes up to which chunks will be quarantined (if lower than" - "or equal to). Defaults to 256 (32-bit) or 2048 (64-bit)") - -// Disable the deallocation type check by default on Android, it causes too many -// issues with third party libraries. -SCUDO_FLAG(bool, DeallocationTypeMismatch, !SANITIZER_ANDROID, - "Report errors on malloc/delete, new/free, new/delete[], etc.") - -SCUDO_FLAG(bool, DeleteSizeMismatch, true, - "Report errors on mismatch between size of new and delete.") - -SCUDO_FLAG(bool, ZeroContents, false, - "Zero chunk contents on allocation and deallocation.") diff --git a/compiler-rt/lib/scudo/scudo_interface_internal.h b/compiler-rt/lib/scudo/scudo_interface_internal.h deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_interface_internal.h +++ /dev/null @@ -1,32 +0,0 @@ -//===-- scudo_interface_internal.h ------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Private Scudo interface header. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_INTERFACE_INTERNAL_H_ -#define SCUDO_INTERFACE_INTERNAL_H_ - -#include "sanitizer_common/sanitizer_internal_defs.h" - -using __sanitizer::uptr; -using __sanitizer::s32; - -extern "C" { -SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -const char* __scudo_default_options(); - -SANITIZER_INTERFACE_ATTRIBUTE -void __scudo_set_rss_limit(uptr LimitMb, s32 HardLimit); - -SANITIZER_INTERFACE_ATTRIBUTE -void __scudo_print_stats(); -} // extern "C" - -#endif // SCUDO_INTERFACE_INTERNAL_H_ diff --git a/compiler-rt/lib/scudo/scudo_malloc.cpp b/compiler-rt/lib/scudo/scudo_malloc.cpp deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_malloc.cpp +++ /dev/null @@ -1,84 +0,0 @@ -//===-- scudo_malloc.cpp ----------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Interceptors for malloc related functions. -/// -//===----------------------------------------------------------------------===// - -#include "scudo_allocator.h" - -#include "interception/interception.h" -#include "sanitizer_common/sanitizer_platform_interceptors.h" - -#include - -using namespace __scudo; - -extern "C" { -INTERCEPTOR_ATTRIBUTE void free(void *ptr) { - scudoDeallocate(ptr, 0, 0, FromMalloc); -} - -INTERCEPTOR_ATTRIBUTE void *malloc(size_t size) { - return scudoAllocate(size, 0, FromMalloc); -} - -INTERCEPTOR_ATTRIBUTE void *realloc(void *ptr, size_t size) { - return scudoRealloc(ptr, size); -} - -INTERCEPTOR_ATTRIBUTE void *calloc(size_t nmemb, size_t size) { - return scudoCalloc(nmemb, size); -} - -INTERCEPTOR_ATTRIBUTE void *valloc(size_t size) { - return scudoValloc(size); -} - -INTERCEPTOR_ATTRIBUTE -int posix_memalign(void **memptr, size_t alignment, size_t size) { - return scudoPosixMemalign(memptr, alignment, size); -} - -#if SANITIZER_INTERCEPT_CFREE -INTERCEPTOR_ATTRIBUTE void cfree(void *ptr) ALIAS("free"); -#endif - -#if SANITIZER_INTERCEPT_MEMALIGN -INTERCEPTOR_ATTRIBUTE void *memalign(size_t alignment, size_t size) { - return scudoAllocate(size, alignment, FromMemalign); -} - -INTERCEPTOR_ATTRIBUTE -void *__libc_memalign(size_t alignment, size_t size) ALIAS("memalign"); -#endif - -#if SANITIZER_INTERCEPT_PVALLOC -INTERCEPTOR_ATTRIBUTE void *pvalloc(size_t size) { - return scudoPvalloc(size); -} -#endif - -#if SANITIZER_INTERCEPT_ALIGNED_ALLOC -INTERCEPTOR_ATTRIBUTE void *aligned_alloc(size_t alignment, size_t size) { - return scudoAlignedAlloc(alignment, size); -} -#endif - -#if SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE -INTERCEPTOR_ATTRIBUTE size_t malloc_usable_size(void *ptr) { - return scudoMallocUsableSize(ptr); -} -#endif - -#if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO -INTERCEPTOR_ATTRIBUTE int mallopt(int cmd, int value) { - return 0; -} -#endif -} // extern "C" diff --git a/compiler-rt/lib/scudo/scudo_new_delete.cpp b/compiler-rt/lib/scudo/scudo_new_delete.cpp deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_new_delete.cpp +++ /dev/null @@ -1,107 +0,0 @@ -//===-- scudo_new_delete.cpp ------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Interceptors for operators new and delete. -/// -//===----------------------------------------------------------------------===// - -#include "scudo_allocator.h" -#include "scudo_errors.h" - -#include "interception/interception.h" - -#include - -using namespace __scudo; - -#define CXX_OPERATOR_ATTRIBUTE INTERCEPTOR_ATTRIBUTE - -// Fake std::nothrow_t to avoid including . -namespace std { -struct nothrow_t {}; -enum class align_val_t: size_t {}; -} // namespace std - -// TODO(alekseys): throw std::bad_alloc instead of dying on OOM. -#define OPERATOR_NEW_BODY_ALIGN(Type, Align, NoThrow) \ - void *Ptr = scudoAllocate(size, static_cast(Align), Type); \ - if (!NoThrow && UNLIKELY(!Ptr)) reportOutOfMemory(size); \ - return Ptr; -#define OPERATOR_NEW_BODY(Type, NoThrow) \ - OPERATOR_NEW_BODY_ALIGN(Type, 0, NoThrow) - -CXX_OPERATOR_ATTRIBUTE -void *operator new(size_t size) -{ OPERATOR_NEW_BODY(FromNew, /*NoThrow=*/false); } -CXX_OPERATOR_ATTRIBUTE -void *operator new[](size_t size) -{ OPERATOR_NEW_BODY(FromNewArray, /*NoThrow=*/false); } -CXX_OPERATOR_ATTRIBUTE -void *operator new(size_t size, std::nothrow_t const&) -{ OPERATOR_NEW_BODY(FromNew, /*NoThrow=*/true); } -CXX_OPERATOR_ATTRIBUTE -void *operator new[](size_t size, std::nothrow_t const&) -{ OPERATOR_NEW_BODY(FromNewArray, /*NoThrow=*/true); } -CXX_OPERATOR_ATTRIBUTE -void *operator new(size_t size, std::align_val_t align) -{ OPERATOR_NEW_BODY_ALIGN(FromNew, align, /*NoThrow=*/false); } -CXX_OPERATOR_ATTRIBUTE -void *operator new[](size_t size, std::align_val_t align) -{ OPERATOR_NEW_BODY_ALIGN(FromNewArray, align, /*NoThrow=*/false); } -CXX_OPERATOR_ATTRIBUTE -void *operator new(size_t size, std::align_val_t align, std::nothrow_t const&) -{ OPERATOR_NEW_BODY_ALIGN(FromNew, align, /*NoThrow=*/true); } -CXX_OPERATOR_ATTRIBUTE -void *operator new[](size_t size, std::align_val_t align, std::nothrow_t const&) -{ OPERATOR_NEW_BODY_ALIGN(FromNewArray, align, /*NoThrow=*/true); } - -#define OPERATOR_DELETE_BODY(Type) \ - scudoDeallocate(ptr, 0, 0, Type); -#define OPERATOR_DELETE_BODY_SIZE(Type) \ - scudoDeallocate(ptr, size, 0, Type); -#define OPERATOR_DELETE_BODY_ALIGN(Type) \ - scudoDeallocate(ptr, 0, static_cast(align), Type); -#define OPERATOR_DELETE_BODY_SIZE_ALIGN(Type) \ - scudoDeallocate(ptr, size, static_cast(align), Type); - -CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr) NOEXCEPT -{ OPERATOR_DELETE_BODY(FromNew); } -CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr) NOEXCEPT -{ OPERATOR_DELETE_BODY(FromNewArray); } -CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, std::nothrow_t const&) -{ OPERATOR_DELETE_BODY(FromNew); } -CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, std::nothrow_t const&) -{ OPERATOR_DELETE_BODY(FromNewArray); } -CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, size_t size) NOEXCEPT -{ OPERATOR_DELETE_BODY_SIZE(FromNew); } -CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, size_t size) NOEXCEPT -{ OPERATOR_DELETE_BODY_SIZE(FromNewArray); } -CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, std::align_val_t align) NOEXCEPT -{ OPERATOR_DELETE_BODY_ALIGN(FromNew); } -CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT -{ OPERATOR_DELETE_BODY_ALIGN(FromNewArray); } -CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&) -{ OPERATOR_DELETE_BODY_ALIGN(FromNew); } -CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, std::align_val_t align, std::nothrow_t const&) -{ OPERATOR_DELETE_BODY_ALIGN(FromNewArray); } -CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, size_t size, std::align_val_t align) NOEXCEPT -{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FromNew); } -CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, size_t size, std::align_val_t align) NOEXCEPT -{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FromNewArray); } diff --git a/compiler-rt/lib/scudo/scudo_platform.h b/compiler-rt/lib/scudo/scudo_platform.h deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_platform.h +++ /dev/null @@ -1,93 +0,0 @@ -//===-- scudo_platform.h ----------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Scudo platform specific definitions. -/// TODO(kostyak): add tests for the compile time defines. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_PLATFORM_H_ -#define SCUDO_PLATFORM_H_ - -#include "sanitizer_common/sanitizer_allocator.h" - -#if !SANITIZER_LINUX && !SANITIZER_FUCHSIA -# error "The Scudo hardened allocator is not supported on this platform." -#endif - -#define SCUDO_TSD_EXCLUSIVE_SUPPORTED (!SANITIZER_ANDROID && !SANITIZER_FUCHSIA) - -#ifndef SCUDO_TSD_EXCLUSIVE -// SCUDO_TSD_EXCLUSIVE wasn't defined, use a default TSD model for the platform. -# if SANITIZER_ANDROID || SANITIZER_FUCHSIA -// Android and Fuchsia use a pool of TSDs shared between threads. -# define SCUDO_TSD_EXCLUSIVE 0 -# elif SANITIZER_LINUX && !SANITIZER_ANDROID -// Non-Android Linux use an exclusive TSD per thread. -# define SCUDO_TSD_EXCLUSIVE 1 -# else -# error "No default TSD model defined for this platform." -# endif // SANITIZER_ANDROID || SANITIZER_FUCHSIA -#endif // SCUDO_TSD_EXCLUSIVE - -// If the exclusive TSD model is chosen, make sure the platform supports it. -#if SCUDO_TSD_EXCLUSIVE && !SCUDO_TSD_EXCLUSIVE_SUPPORTED -# error "The exclusive TSD model is not supported on this platform." -#endif - -// Maximum number of TSDs that can be created for the Shared model. -#ifndef SCUDO_SHARED_TSD_POOL_SIZE -# if SANITIZER_ANDROID -# define SCUDO_SHARED_TSD_POOL_SIZE 2U -# else -# define SCUDO_SHARED_TSD_POOL_SIZE 32U -# endif // SANITIZER_ANDROID -#endif // SCUDO_SHARED_TSD_POOL_SIZE - -// The following allows the public interface functions to be disabled. -#ifndef SCUDO_CAN_USE_PUBLIC_INTERFACE -# define SCUDO_CAN_USE_PUBLIC_INTERFACE 1 -#endif - -// Hooks in the allocation & deallocation paths can become a security concern if -// implemented improperly, or if overwritten by an attacker. Use with caution. -#ifndef SCUDO_CAN_USE_HOOKS -# if SANITIZER_FUCHSIA -# define SCUDO_CAN_USE_HOOKS 1 -# else -# define SCUDO_CAN_USE_HOOKS 0 -# endif // SANITIZER_FUCHSIA -#endif // SCUDO_CAN_USE_HOOKS - -namespace __scudo { - -#if SANITIZER_CAN_USE_ALLOCATOR64 -# if defined(__aarch64__) && SANITIZER_ANDROID -const uptr AllocatorSize = 0x4000000000ULL; // 256G. -# elif defined(__aarch64__) -const uptr AllocatorSize = 0x10000000000ULL; // 1T. -# else -const uptr AllocatorSize = 0x40000000000ULL; // 4T. -# endif -#else -const uptr RegionSizeLog = SANITIZER_ANDROID ? 19 : 20; -#endif // SANITIZER_CAN_USE_ALLOCATOR64 - -#if !defined(SCUDO_SIZE_CLASS_MAP) -# define SCUDO_SIZE_CLASS_MAP Dense -#endif - -#define SIZE_CLASS_MAP_TYPE SIZE_CLASS_MAP_TYPE_(SCUDO_SIZE_CLASS_MAP) -#define SIZE_CLASS_MAP_TYPE_(T) SIZE_CLASS_MAP_TYPE__(T) -#define SIZE_CLASS_MAP_TYPE__(T) T##SizeClassMap - -typedef SIZE_CLASS_MAP_TYPE SizeClassMap; - -} // namespace __scudo - -#endif // SCUDO_PLATFORM_H_ diff --git a/compiler-rt/lib/scudo/scudo_termination.cpp b/compiler-rt/lib/scudo/scudo_termination.cpp deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_termination.cpp +++ /dev/null @@ -1,41 +0,0 @@ -//===-- scudo_termination.cpp -----------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// This file contains bare-bones termination functions to replace the -/// __sanitizer ones, in order to avoid any potential abuse of the callbacks -/// functionality. -/// -//===----------------------------------------------------------------------===// - -#include "scudo_utils.h" - -#include "sanitizer_common/sanitizer_common.h" - -namespace __sanitizer { - -bool AddDieCallback(DieCallbackType Callback) { return true; } - -bool RemoveDieCallback(DieCallbackType Callback) { return true; } - -void SetUserDieCallback(DieCallbackType Callback) {} - -void NORETURN Die() { - if (common_flags()->abort_on_error) - Abort(); - internal__exit(common_flags()->exitcode); -} - -void SetCheckUnwindCallback(void (*callback)()) {} - -void NORETURN CheckFailed(const char *File, int Line, const char *Condition, - u64 Value1, u64 Value2) { - __scudo::dieWithMessage("CHECK failed at %s:%d %s (%lld, %lld)\n", - File, Line, Condition, Value1, Value2); -} - -} // namespace __sanitizer diff --git a/compiler-rt/lib/scudo/scudo_tsd.h b/compiler-rt/lib/scudo/scudo_tsd.h deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_tsd.h +++ /dev/null @@ -1,65 +0,0 @@ -//===-- scudo_tsd.h ---------------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Scudo thread specific data definition. -/// Implementation will differ based on the thread local storage primitives -/// offered by the underlying platform. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TSD_H_ -#define SCUDO_TSD_H_ - -#include "scudo_allocator.h" -#include "scudo_utils.h" - -#include - -namespace __scudo { - -struct ALIGNED(SANITIZER_CACHE_LINE_SIZE) ScudoTSD { - AllocatorCacheT Cache; - uptr QuarantineCachePlaceHolder[4]; - - void init(); - void commitBack(); - - inline bool tryLock() SANITIZER_TRY_ACQUIRE(true, Mutex) { - if (Mutex.TryLock()) { - atomic_store_relaxed(&Precedence, 0); - return true; - } - if (atomic_load_relaxed(&Precedence) == 0) - atomic_store_relaxed(&Precedence, static_cast( - MonotonicNanoTime() >> FIRST_32_SECOND_64(16, 0))); - return false; - } - - inline void lock() SANITIZER_ACQUIRE(Mutex) { - atomic_store_relaxed(&Precedence, 0); - Mutex.Lock(); - } - - inline void unlock() SANITIZER_RELEASE(Mutex) { Mutex.Unlock(); } - - inline uptr getPrecedence() { return atomic_load_relaxed(&Precedence); } - - private: - StaticSpinMutex Mutex; - atomic_uintptr_t Precedence; -}; - -void initThread(bool MinimalInit); - -// TSD model specific fastpath functions definitions. -#include "scudo_tsd_exclusive.inc" -#include "scudo_tsd_shared.inc" - -} // namespace __scudo - -#endif // SCUDO_TSD_H_ diff --git a/compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp b/compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp +++ /dev/null @@ -1,67 +0,0 @@ -//===-- scudo_tsd_exclusive.cpp ---------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Scudo exclusive TSD implementation. -/// -//===----------------------------------------------------------------------===// - -#include "scudo_tsd.h" - -#if SCUDO_TSD_EXCLUSIVE - -namespace __scudo { - -static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT; -static pthread_key_t PThreadKey; - -__attribute__((tls_model("initial-exec"))) -THREADLOCAL ThreadState ScudoThreadState = ThreadNotInitialized; -__attribute__((tls_model("initial-exec"))) -THREADLOCAL ScudoTSD TSD; - -// Fallback TSD for when the thread isn't initialized yet or is torn down. It -// can be shared between multiple threads and as such must be locked. -ScudoTSD FallbackTSD; - -static void teardownThread(void *Ptr) { - uptr I = reinterpret_cast(Ptr); - // The glibc POSIX thread-local-storage deallocation routine calls user - // provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS. - // We want to be called last since other destructors might call free and the - // like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the - // quarantine and swallowing the cache. - if (I > 1) { - // If pthread_setspecific fails, we will go ahead with the teardown. - if (LIKELY(pthread_setspecific(PThreadKey, - reinterpret_cast(I - 1)) == 0)) - return; - } - TSD.commitBack(); - ScudoThreadState = ThreadTornDown; -} - - -static void initOnce() { - CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread), 0); - initScudo(); - FallbackTSD.init(); -} - -void initThread(bool MinimalInit) { - CHECK_EQ(pthread_once(&GlobalInitialized, initOnce), 0); - if (UNLIKELY(MinimalInit)) - return; - CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast( - GetPthreadDestructorIterations())), 0); - TSD.init(); - ScudoThreadState = ThreadInitialized; -} - -} // namespace __scudo - -#endif // SCUDO_TSD_EXCLUSIVE diff --git a/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc b/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_tsd_exclusive.inc +++ /dev/null @@ -1,47 +0,0 @@ -//===-- scudo_tsd_exclusive.inc ---------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Scudo exclusive TSD fastpath functions implementation. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TSD_H_ -#error "This file must be included inside scudo_tsd.h." -#endif // SCUDO_TSD_H_ - -#if SCUDO_TSD_EXCLUSIVE - -enum ThreadState : u8 { - ThreadNotInitialized = 0, - ThreadInitialized, - ThreadTornDown, -}; -__attribute__(( - tls_model("initial-exec"))) extern THREADLOCAL ThreadState ScudoThreadState; -__attribute__((tls_model("initial-exec"))) extern THREADLOCAL ScudoTSD TSD; - -extern ScudoTSD FallbackTSD; - -ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) { - if (LIKELY(ScudoThreadState != ThreadNotInitialized)) - return; - initThread(MinimalInit); -} - -ALWAYS_INLINE ScudoTSD * -getTSDAndLock(bool *UnlockRequired) SANITIZER_NO_THREAD_SAFETY_ANALYSIS { - if (UNLIKELY(ScudoThreadState != ThreadInitialized)) { - FallbackTSD.lock(); - *UnlockRequired = true; - return &FallbackTSD; - } - *UnlockRequired = false; - return &TSD; -} - -#endif // SCUDO_TSD_EXCLUSIVE diff --git a/compiler-rt/lib/scudo/scudo_tsd_shared.cpp b/compiler-rt/lib/scudo/scudo_tsd_shared.cpp deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_tsd_shared.cpp +++ /dev/null @@ -1,107 +0,0 @@ -//===-- scudo_tsd_shared.cpp ------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Scudo shared TSD implementation. -/// -//===----------------------------------------------------------------------===// - -#include "scudo_tsd.h" - -#if !SCUDO_TSD_EXCLUSIVE - -namespace __scudo { - -static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT; -pthread_key_t PThreadKey; - -static atomic_uint32_t CurrentIndex; -static ScudoTSD *TSDs; -static u32 NumberOfTSDs; -static u32 CoPrimes[SCUDO_SHARED_TSD_POOL_SIZE]; -static u32 NumberOfCoPrimes = 0; - -#if SANITIZER_LINUX && !SANITIZER_ANDROID -__attribute__((tls_model("initial-exec"))) -THREADLOCAL ScudoTSD *CurrentTSD; -#endif - -static void initOnce() { - CHECK_EQ(pthread_key_create(&PThreadKey, NULL), 0); - initScudo(); - NumberOfTSDs = Min(Max(1U, GetNumberOfCPUsCached()), - static_cast(SCUDO_SHARED_TSD_POOL_SIZE)); - TSDs = reinterpret_cast( - MmapOrDie(sizeof(ScudoTSD) * NumberOfTSDs, "ScudoTSDs")); - for (u32 I = 0; I < NumberOfTSDs; I++) { - TSDs[I].init(); - u32 A = I + 1; - u32 B = NumberOfTSDs; - while (B != 0) { const u32 T = A; A = B; B = T % B; } - if (A == 1) - CoPrimes[NumberOfCoPrimes++] = I + 1; - } -} - -ALWAYS_INLINE void setCurrentTSD(ScudoTSD *TSD) { -#if SANITIZER_ANDROID - *get_android_tls_ptr() = reinterpret_cast(TSD); -#elif SANITIZER_LINUX - CurrentTSD = TSD; -#else - CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast(TSD)), 0); -#endif // SANITIZER_ANDROID -} - -void initThread(bool MinimalInit) { - pthread_once(&GlobalInitialized, initOnce); - // Initial context assignment is done in a plain round-robin fashion. - u32 Index = atomic_fetch_add(&CurrentIndex, 1, memory_order_relaxed); - setCurrentTSD(&TSDs[Index % NumberOfTSDs]); -} - -ScudoTSD *getTSDAndLockSlow(ScudoTSD *TSD) SANITIZER_NO_THREAD_SAFETY_ANALYSIS { - if (NumberOfTSDs > 1) { - // Use the Precedence of the current TSD as our random seed. Since we are in - // the slow path, it means that tryLock failed, and as a result it's very - // likely that said Precedence is non-zero. - u32 RandState = static_cast(TSD->getPrecedence()); - const u32 R = Rand(&RandState); - const u32 Inc = CoPrimes[R % NumberOfCoPrimes]; - u32 Index = R % NumberOfTSDs; - uptr LowestPrecedence = UINTPTR_MAX; - ScudoTSD *CandidateTSD = nullptr; - // Go randomly through at most 4 contexts and find a candidate. - for (u32 I = 0; I < Min(4U, NumberOfTSDs); I++) { - if (TSDs[Index].tryLock()) { - setCurrentTSD(&TSDs[Index]); - return &TSDs[Index]; - } - const uptr Precedence = TSDs[Index].getPrecedence(); - // A 0 precedence here means another thread just locked this TSD. - if (Precedence && Precedence < LowestPrecedence) { - CandidateTSD = &TSDs[Index]; - LowestPrecedence = Precedence; - } - Index += Inc; - if (Index >= NumberOfTSDs) - Index -= NumberOfTSDs; - } - if (CandidateTSD) { - CandidateTSD->lock(); - setCurrentTSD(CandidateTSD); - return CandidateTSD; - } - } - // Last resort, stick with the current one. - TSD->lock(); - return TSD; -} - -} // namespace __scudo - -#endif // !SCUDO_TSD_EXCLUSIVE diff --git a/compiler-rt/lib/scudo/scudo_tsd_shared.inc b/compiler-rt/lib/scudo/scudo_tsd_shared.inc deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_tsd_shared.inc +++ /dev/null @@ -1,56 +0,0 @@ -//===-- scudo_tsd_shared.inc ------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Scudo shared TSD fastpath functions implementation. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TSD_H_ -# error "This file must be included inside scudo_tsd.h." -#endif // SCUDO_TSD_H_ - -#if !SCUDO_TSD_EXCLUSIVE - -extern pthread_key_t PThreadKey; - -#if SANITIZER_LINUX && !SANITIZER_ANDROID -__attribute__((tls_model("initial-exec"))) -extern THREADLOCAL ScudoTSD *CurrentTSD; -#endif - -ALWAYS_INLINE ScudoTSD* getCurrentTSD() { -#if SANITIZER_ANDROID - return reinterpret_cast(*get_android_tls_ptr()); -#elif SANITIZER_LINUX - return CurrentTSD; -#else - return reinterpret_cast(pthread_getspecific(PThreadKey)); -#endif // SANITIZER_ANDROID -} - -ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) { - if (LIKELY(getCurrentTSD())) - return; - initThread(MinimalInit); -} - -ScudoTSD *getTSDAndLockSlow(ScudoTSD *TSD); - -ALWAYS_INLINE ScudoTSD * -getTSDAndLock(bool *UnlockRequired) SANITIZER_NO_THREAD_SAFETY_ANALYSIS { - ScudoTSD *TSD = getCurrentTSD(); - DCHECK(TSD && "No TSD associated with the current thread!"); - *UnlockRequired = true; - // Try to lock the currently associated context. - if (TSD->tryLock()) - return TSD; - // If it failed, go the slow path. - return getTSDAndLockSlow(TSD); -} - -#endif // !SCUDO_TSD_EXCLUSIVE diff --git a/compiler-rt/lib/scudo/scudo_utils.h b/compiler-rt/lib/scudo/scudo_utils.h deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_utils.h +++ /dev/null @@ -1,36 +0,0 @@ -//===-- scudo_utils.h -------------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Header for scudo_utils.cpp. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_UTILS_H_ -#define SCUDO_UTILS_H_ - -#include "sanitizer_common/sanitizer_common.h" - -#include - -namespace __scudo { - -template -inline Dest bit_cast(const Source& source) { - static_assert(sizeof(Dest) == sizeof(Source), "Sizes are not equal!"); - Dest dest; - memcpy(&dest, &source, sizeof(dest)); - return dest; -} - -void dieWithMessage(const char *Format, ...) NORETURN FORMAT(1, 2); - -bool hasHardwareCRC32(); - -} // namespace __scudo - -#endif // SCUDO_UTILS_H_ diff --git a/compiler-rt/lib/scudo/scudo_utils.cpp b/compiler-rt/lib/scudo/scudo_utils.cpp deleted file mode 100644 --- a/compiler-rt/lib/scudo/scudo_utils.cpp +++ /dev/null @@ -1,145 +0,0 @@ -//===-- scudo_utils.cpp -----------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// Platform specific utility functions. -/// -//===----------------------------------------------------------------------===// - -#include "scudo_utils.h" - -#if defined(__x86_64__) || defined(__i386__) -# include -#elif defined(__arm__) || defined(__aarch64__) -# include "sanitizer_common/sanitizer_getauxval.h" -# if SANITIZER_FUCHSIA -# include -# include -# elif SANITIZER_POSIX -# include "sanitizer_common/sanitizer_posix.h" -# include -# endif -#endif - -#include - -// TODO(kostyak): remove __sanitizer *Printf uses in favor for our own less -// complicated string formatting code. The following is a -// temporary workaround to be able to use __sanitizer::VSNPrintf. -namespace __sanitizer { - -extern int VSNPrintf(char *buff, int buff_length, const char *format, - va_list args); - -} // namespace __sanitizer - -namespace __scudo { - -void dieWithMessage(const char *Format, ...) { - static const char ScudoError[] = "Scudo ERROR: "; - static constexpr uptr PrefixSize = sizeof(ScudoError) - 1; - // Our messages are tiny, 256 characters is more than enough. - char Message[256]; - va_list Args; - va_start(Args, Format); - internal_memcpy(Message, ScudoError, PrefixSize); - VSNPrintf(Message + PrefixSize, sizeof(Message) - PrefixSize, Format, Args); - va_end(Args); - LogMessageOnPrintf(Message); - if (common_flags()->abort_on_error) - SetAbortMessage(Message); - RawWrite(Message); - Die(); -} - -#if defined(__x86_64__) || defined(__i386__) -// i386 and x86_64 specific code to detect CRC32 hardware support via CPUID. -// CRC32 requires the SSE 4.2 instruction set. -# ifndef bit_SSE4_2 -# define bit_SSE4_2 bit_SSE42 // clang and gcc have different defines. -# endif - -#ifndef signature_HYGON_ebx // They are not defined in gcc. -// HYGON: "HygonGenuine". -#define signature_HYGON_ebx 0x6f677948 -#define signature_HYGON_edx 0x6e65476e -#define signature_HYGON_ecx 0x656e6975 -#endif - -bool hasHardwareCRC32() { - u32 Eax, Ebx, Ecx, Edx; - __get_cpuid(0, &Eax, &Ebx, &Ecx, &Edx); - const bool IsIntel = (Ebx == signature_INTEL_ebx) && - (Edx == signature_INTEL_edx) && - (Ecx == signature_INTEL_ecx); - const bool IsAMD = (Ebx == signature_AMD_ebx) && - (Edx == signature_AMD_edx) && - (Ecx == signature_AMD_ecx); - const bool IsHygon = (Ebx == signature_HYGON_ebx) && - (Edx == signature_HYGON_edx) && - (Ecx == signature_HYGON_ecx); - if (!IsIntel && !IsAMD && !IsHygon) - return false; - __get_cpuid(1, &Eax, &Ebx, &Ecx, &Edx); - return !!(Ecx & bit_SSE4_2); -} -#elif defined(__arm__) || defined(__aarch64__) -// For ARM and AArch64, hardware CRC32 support is indicated in the AT_HWCAP -// auxiliary vector. -# ifndef AT_HWCAP -# define AT_HWCAP 16 -# endif -# ifndef HWCAP_CRC32 -# define HWCAP_CRC32 (1 << 7) // HWCAP_CRC32 is missing on older platforms. -# endif -# if SANITIZER_POSIX -bool hasHardwareCRC32ARMPosix() { - uptr F = internal_open("/proc/self/auxv", O_RDONLY); - if (internal_iserror(F)) - return false; - struct { uptr Tag; uptr Value; } Entry = { 0, 0 }; - for (;;) { - uptr N = internal_read(F, &Entry, sizeof(Entry)); - if (internal_iserror(N) || N != sizeof(Entry) || - (Entry.Tag == 0 && Entry.Value == 0) || Entry.Tag == AT_HWCAP) - break; - } - internal_close(F); - return (Entry.Tag == AT_HWCAP && (Entry.Value & HWCAP_CRC32) != 0); -} -# else -bool hasHardwareCRC32ARMPosix() { return false; } -# endif // SANITIZER_POSIX - -// Bionic doesn't initialize its globals early enough. This causes issues when -// trying to access them from a preinit_array (b/25751302) or from another -// constructor called before the libc one (b/68046352). __progname is -// initialized after the other globals, so we can check its value to know if -// calling getauxval is safe. -extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname; -inline bool areBionicGlobalsInitialized() { - return !SANITIZER_ANDROID || (&__progname && __progname); -} - -bool hasHardwareCRC32() { -#if SANITIZER_FUCHSIA - u32 HWCap; - zx_status_t Status = zx_system_get_features(ZX_FEATURE_KIND_CPU, &HWCap); - if (Status != ZX_OK || (HWCap & ZX_ARM64_FEATURE_ISA_CRC32) == 0) - return false; - return true; -#else - if (&getauxval && areBionicGlobalsInitialized()) - return !!(getauxval(AT_HWCAP) & HWCAP_CRC32); - return hasHardwareCRC32ARMPosix(); -#endif // SANITIZER_FUCHSIA -} -#else -bool hasHardwareCRC32() { return false; } -#endif // defined(__x86_64__) || defined(__i386__) - -} // namespace __scudo