This is an archive of the discontinued LLVM Phabricator instance.

	add_compiler_rt_component(scudo)

	include_directories(..)

	set(SCUDO_CFLAGS ${SANITIZER_COMMON_CFLAGS})
	# SANITIZER_COMMON_CFLAGS include -fno-builtin, but we actually want builtins!
	list(APPEND SCUDO_CFLAGS -fbuiltin)
	append_rtti_flag(OFF SCUDO_CFLAGS)

	# Too many existing bugs, needs cleanup.
	append_list_if(COMPILER_RT_HAS_WNO_FORMAT -Wno-format SCUDO_CFLAGS)

	set(SCUDO_MINIMAL_DYNAMIC_LIBS ${SANITIZER_COMMON_LINK_LIBS})
	append_list_if(COMPILER_RT_HAS_LIBDL dl SCUDO_MINIMAL_DYNAMIC_LIBS)
	append_list_if(COMPILER_RT_HAS_LIBRT rt SCUDO_MINIMAL_DYNAMIC_LIBS)
	append_list_if(COMPILER_RT_HAS_LIBPTHREAD pthread SCUDO_MINIMAL_DYNAMIC_LIBS)
	append_list_if(COMPILER_RT_HAS_LIBLOG log SCUDO_MINIMAL_DYNAMIC_LIBS)
	append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer
	SCUDO_CFLAGS)

	set(SCUDO_DYNAMIC_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS})
	# Use gc-sections by default to avoid unused code being pulled in.
	list(APPEND SCUDO_DYNAMIC_LINK_FLAGS -Wl,--gc-sections)

	if(ANDROID)
	# Put most Sanitizer shared libraries in the global group. For more details, see
	# android-changes-for-ndk-developers.md#changes-to-library-search-order
	if (COMPILER_RT_HAS_Z_GLOBAL)
	list(APPEND SCUDO_DYNAMIC_LINK_FLAGS -Wl,-z,global)
	endif()
	endif()

	# The minimal Scudo runtime does not include the UBSan runtime.
	set(SCUDO_MINIMAL_OBJECT_LIBS
	RTSanitizerCommonNoTermination
	RTSanitizerCommonLibc
	RTInterception)

	if (COMPILER_RT_HAS_GWP_ASAN)
	list(APPEND SCUDO_MINIMAL_OBJECT_LIBS
	RTGwpAsan RTGwpAsanOptionsParser RTGwpAsanBacktraceLibc
	RTGwpAsanSegvHandler)
	list(APPEND SCUDO_CFLAGS -DGWP_ASAN_HOOKS)
	endif()

	set(SCUDO_OBJECT_LIBS ${SCUDO_MINIMAL_OBJECT_LIBS})
	set(SCUDO_DYNAMIC_LIBS ${SCUDO_MINIMAL_DYNAMIC_LIBS})

	if (FUCHSIA)
	list(APPEND SCUDO_CFLAGS -nostdinc++)
	list(APPEND SCUDO_DYNAMIC_LINK_FLAGS -nostdlib++)
	else()
	list(APPEND SCUDO_DYNAMIC_LIBS
	${COMPILER_RT_UNWINDER_LINK_LIBS}
	${SANITIZER_CXX_ABI_LIBRARIES})
	list(APPEND SCUDO_OBJECT_LIBS
	RTSanitizerCommonCoverage
	RTSanitizerCommonSymbolizer
	RTUbsan)
	endif()

	set(SCUDO_SOURCES
	scudo_allocator.cpp
	scudo_crc32.cpp
	scudo_errors.cpp
	scudo_flags.cpp
	scudo_malloc.cpp
	scudo_termination.cpp
	scudo_tsd_exclusive.cpp
	scudo_tsd_shared.cpp
	scudo_utils.cpp)

	set(SCUDO_CXX_SOURCES
	scudo_new_delete.cpp)

	set(SCUDO_HEADERS
	scudo_allocator.h
	scudo_allocator_combined.h
	scudo_allocator_secondary.h
	scudo_crc32.h
	scudo_errors.h
	scudo_flags.h
	scudo_flags.inc
	scudo_interface_internal.h
	scudo_platform.h
	scudo_tsd.h
	scudo_tsd_exclusive.inc
	scudo_tsd_shared.inc
	scudo_utils.h)

	# Enable the necessary instruction set for scudo_crc32.cpp, if available.
	# Newer compiler versions use -mcrc32 rather than -msse4.2.
	if (COMPILER_RT_HAS_MCRC32_FLAG)
	set_source_files_properties(scudo_crc32.cpp PROPERTIES COMPILE_FLAGS -mcrc32)
	elseif (COMPILER_RT_HAS_MSSE4_2_FLAG)
	set_source_files_properties(scudo_crc32.cpp PROPERTIES COMPILE_FLAGS -msse4.2)
	endif()

	# Enable the AArch64 CRC32 feature for scudo_crc32.cpp, if available.
	# Note that it is enabled by default starting with armv8.1-a.
	if (COMPILER_RT_HAS_MCRC_FLAG)
	set_source_files_properties(scudo_crc32.cpp PROPERTIES COMPILE_FLAGS -mcrc)
	endif()

	if(COMPILER_RT_HAS_SCUDO)
	add_compiler_rt_runtime(clang_rt.scudo_minimal
	STATIC
	ARCHS ${SCUDO_SUPPORTED_ARCH}
	SOURCES ${SCUDO_SOURCES}
	ADDITIONAL_HEADERS ${SCUDO_HEADERS}
	OBJECT_LIBS ${SCUDO_MINIMAL_OBJECT_LIBS}
	CFLAGS ${SCUDO_CFLAGS}
	PARENT_TARGET scudo)
	add_compiler_rt_runtime(clang_rt.scudo_cxx_minimal
	STATIC
	ARCHS ${SCUDO_SUPPORTED_ARCH}
	SOURCES ${SCUDO_CXX_SOURCES}
	ADDITIONAL_HEADERS ${SCUDO_HEADERS}
	CFLAGS ${SCUDO_CFLAGS}
	PARENT_TARGET scudo)

	add_compiler_rt_runtime(clang_rt.scudo
	STATIC
	ARCHS ${SCUDO_SUPPORTED_ARCH}
	SOURCES ${SCUDO_SOURCES}
	ADDITIONAL_HEADERS ${SCUDO_HEADERS}
	OBJECT_LIBS ${SCUDO_OBJECT_LIBS}
	CFLAGS ${SCUDO_CFLAGS}
	PARENT_TARGET scudo)
	add_compiler_rt_runtime(clang_rt.scudo_cxx
	STATIC
	ARCHS ${SCUDO_SUPPORTED_ARCH}
	SOURCES ${SCUDO_CXX_SOURCES}
	ADDITIONAL_HEADERS ${SCUDO_HEADERS}
	OBJECT_LIBS RTUbsan_cxx
	CFLAGS ${SCUDO_CFLAGS}
	PARENT_TARGET scudo)

	add_compiler_rt_runtime(clang_rt.scudo_minimal
	SHARED
	ARCHS ${SCUDO_SUPPORTED_ARCH}
	SOURCES ${SCUDO_SOURCES} ${SCUDO_CXX_SOURCES}
	ADDITIONAL_HEADERS ${SCUDO_HEADERS}
	OBJECT_LIBS ${SCUDO_MINIMAL_OBJECT_LIBS}
	CFLAGS ${SCUDO_CFLAGS}
	LINK_FLAGS ${SCUDO_DYNAMIC_LINK_FLAGS}
	LINK_LIBS ${SCUDO_MINIMAL_DYNAMIC_LIBS}
	PARENT_TARGET scudo)

	add_compiler_rt_runtime(clang_rt.scudo
	SHARED
	ARCHS ${SCUDO_SUPPORTED_ARCH}
	SOURCES ${SCUDO_SOURCES} ${SCUDO_CXX_SOURCES}
	ADDITIONAL_HEADERS ${SCUDO_HEADERS}
	OBJECT_LIBS ${SCUDO_OBJECT_LIBS}
	CFLAGS ${SCUDO_CFLAGS}
	LINK_FLAGS ${SCUDO_DYNAMIC_LINK_FLAGS}
	LINK_LIBS ${SCUDO_DYNAMIC_LIBS}
	PARENT_TARGET scudo)
	endif()

compiler-rt/lib/scudo/scudo_allocator.h

This file was deleted.

	//===-- scudo_allocator.h ---------------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Header for scudo_allocator.cpp.
	///
	//===----------------------------------------------------------------------===//

	#ifndef SCUDO_ALLOCATOR_H_
	#define SCUDO_ALLOCATOR_H_

	#include "scudo_platform.h"

	namespace __scudo {

	enum AllocType : u8 {
	FromMalloc = 0, // Memory block came from malloc, realloc, calloc, etc.
	FromNew = 1, // Memory block came from operator new.
	FromNewArray = 2, // Memory block came from operator new [].
	FromMemalign = 3, // Memory block came from memalign, posix_memalign, etc.
	};

	enum ChunkState : u8 {
	ChunkAvailable = 0,
	ChunkAllocated = 1,
	ChunkQuarantine = 2
	};

	// Our header requires 64 bits of storage. Having the offset saves us from
	// using functions such as GetBlockBegin, that is fairly costly. Our first
	// implementation used the MetaData as well, which offers the advantage of
	// being stored away from the chunk itself, but accessing it was costly as
	// well. The header will be atomically loaded and stored.
	typedef u64 PackedHeader;
	struct UnpackedHeader {
	u64 Checksum : 16;
	u64 ClassId : 8;
	u64 SizeOrUnusedBytes : 20; // Size for Primary backed allocations, amount of
	// unused bytes in the chunk for Secondary ones.
	u64 State : 2; // available, allocated, or quarantined
	u64 AllocType : 2; // malloc, new, new[], or memalign
	u64 Offset : 16; // Offset from the beginning of the backend
	// allocation to the beginning of the chunk
	// itself, in multiples of MinAlignment. See
	// comment about its maximum value and in init().
	};

	typedef atomic_uint64_t AtomicPackedHeader;
	COMPILER_CHECK(sizeof(UnpackedHeader) == sizeof(PackedHeader));

	// Minimum alignment of 8 bytes for 32-bit, 16 for 64-bit
	const uptr MinAlignmentLog = FIRST_32_SECOND_64(3, 4);
	const uptr MaxAlignmentLog = 24; // 16 MB
	const uptr MinAlignment = 1 << MinAlignmentLog;
	const uptr MaxAlignment = 1 << MaxAlignmentLog;

	// constexpr version of __sanitizer::RoundUp without the extraneous CHECK.
	// This way we can use it in constexpr variables and functions declarations.
	constexpr uptr RoundUpTo(uptr Size, uptr Boundary) {
	return (Size + Boundary - 1) & ~(Boundary - 1);
	}

	namespace Chunk {
	constexpr uptr getHeaderSize() {
	return RoundUpTo(sizeof(PackedHeader), MinAlignment);
	}
	}

	#if SANITIZER_CAN_USE_ALLOCATOR64
	const uptr AllocatorSpace = ~0ULL;
	struct AP64 {
	static const uptr kSpaceBeg = AllocatorSpace;
	static const uptr kSpaceSize = AllocatorSize;
	static const uptr kMetadataSize = 0;
	typedef __scudo::SizeClassMap SizeClassMap;
	typedef NoOpMapUnmapCallback MapUnmapCallback;
	static const uptr kFlags =
	SizeClassAllocator64FlagMasks::kRandomShuffleChunks;
	using AddressSpaceView = LocalAddressSpaceView;
	};
	typedef SizeClassAllocator64<AP64> PrimaryT;
	#else
	struct AP32 {
	static const uptr kSpaceBeg = 0;
	static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
	static const uptr kMetadataSize = 0;
	typedef __scudo::SizeClassMap SizeClassMap;
	static const uptr kRegionSizeLog = RegionSizeLog;
	using AddressSpaceView = LocalAddressSpaceView;
	typedef NoOpMapUnmapCallback MapUnmapCallback;
	static const uptr kFlags =
	SizeClassAllocator32FlagMasks::kRandomShuffleChunks \|
	SizeClassAllocator32FlagMasks::kUseSeparateSizeClassForBatch;
	};
	typedef SizeClassAllocator32<AP32> PrimaryT;
	#endif // SANITIZER_CAN_USE_ALLOCATOR64

	#include "scudo_allocator_secondary.h"

	typedef LargeMmapAllocator SecondaryT;

	#include "scudo_allocator_combined.h"

	typedef CombinedAllocator BackendT;
	typedef CombinedAllocator::AllocatorCache AllocatorCacheT;

	void initScudo();

	void *scudoAllocate(uptr Size, uptr Alignment, AllocType Type);
	void scudoDeallocate(void *Ptr, uptr Size, uptr Alignment, AllocType Type);
	void scudoRealloc(void Ptr, uptr Size);
	void *scudoCalloc(uptr NMemB, uptr Size);
	void *scudoValloc(uptr Size);
	void *scudoPvalloc(uptr Size);
	int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size);
	void *scudoAlignedAlloc(uptr Alignment, uptr Size);
	uptr scudoMallocUsableSize(void *Ptr);

	} // namespace __scudo

	#endif // SCUDO_ALLOCATOR_H_

compiler-rt/lib/scudo/scudo_allocator.cpp

This file was deleted.

	//===-- scudo_allocator.cpp -------------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Scudo Hardened Allocator implementation.
	/// It uses the sanitizer_common allocator as a base and aims at mitigating
	/// heap corruption vulnerabilities. It provides a checksum-guarded chunk
	/// header, a delayed free list, and additional sanity checks.
	///
	//===----------------------------------------------------------------------===//

	#include "scudo_allocator.h"
	#include "scudo_crc32.h"
	#include "scudo_errors.h"
	#include "scudo_flags.h"
	#include "scudo_interface_internal.h"
	#include "scudo_tsd.h"
	#include "scudo_utils.h"

	#include "sanitizer_common/sanitizer_allocator_checks.h"
	#include "sanitizer_common/sanitizer_allocator_interface.h"
	#include "sanitizer_common/sanitizer_quarantine.h"

	#ifdef GWP_ASAN_HOOKS
	# include "gwp_asan/guarded_pool_allocator.h"
	# include "gwp_asan/optional/backtrace.h"
	# include "gwp_asan/optional/options_parser.h"
	#include "gwp_asan/optional/segv_handler.h"
	#endif // GWP_ASAN_HOOKS

	#include <errno.h>
	#include <string.h>

	namespace __scudo {

	// Global static cookie, initialized at start-up.
	static u32 Cookie;

	// We default to software CRC32 if the alternatives are not supported, either
	// at compilation or at runtime.
	static atomic_uint8_t HashAlgorithm = { CRC32Software };

	inline u32 computeCRC32(u32 Crc, uptr Value, uptr *Array, uptr ArraySize) {
	// If the hardware CRC32 feature is defined here, it was enabled everywhere,
	// as opposed to only for scudo_crc32.cpp. This means that other hardware
	// specific instructions were likely emitted at other places, and as a
	// result there is no reason to not use it here.
	#if defined(__CRC32__) \|\| defined(__SSE4_2__) \|\| defined(__ARM_FEATURE_CRC32)
	Crc = CRC32_INTRINSIC(Crc, Value);
	for (uptr i = 0; i < ArraySize; i++)
	Crc = CRC32_INTRINSIC(Crc, Array[i]);
	return Crc;
	#else
	if (atomic_load_relaxed(&HashAlgorithm) == CRC32Hardware) {
	Crc = computeHardwareCRC32(Crc, Value);
	for (uptr i = 0; i < ArraySize; i++)
	Crc = computeHardwareCRC32(Crc, Array[i]);
	return Crc;
	}
	Crc = computeSoftwareCRC32(Crc, Value);
	for (uptr i = 0; i < ArraySize; i++)
	Crc = computeSoftwareCRC32(Crc, Array[i]);
	return Crc;
	#endif // defined(__CRC32__) \|\| defined(__SSE4_2__) \|\| defined(__ARM_FEATURE_CRC32)
	}

	static BackendT &getBackend();

	namespace Chunk {
	static inline AtomicPackedHeader getAtomicHeader(void Ptr) {
	return reinterpret_cast<AtomicPackedHeader *>(reinterpret_cast<uptr>(Ptr) -
	getHeaderSize());
	}
	static inline
	const AtomicPackedHeader getConstAtomicHeader(const void Ptr) {
	return reinterpret_cast<const AtomicPackedHeader *>(
	reinterpret_cast<uptr>(Ptr) - getHeaderSize());
	}

	static inline bool isAligned(const void *Ptr) {
	return IsAligned(reinterpret_cast<uptr>(Ptr), MinAlignment);
	}

	// We can't use the offset member of the chunk itself, as we would double
	// fetch it without any warranty that it wouldn't have been tampered. To
	// prevent this, we work with a local copy of the header.
	static inline void getBackendPtr(const void Ptr, UnpackedHeader *Header) {
	return reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
	getHeaderSize() - (Header->Offset << MinAlignmentLog));
	}

	// Returns the usable size for a chunk, meaning the amount of bytes from the
	// beginning of the user data to the end of the backend allocated chunk.
	static inline uptr getUsableSize(const void Ptr, UnpackedHeader Header) {
	const uptr ClassId = Header->ClassId;
	if (ClassId)
	return PrimaryT::ClassIdToSize(ClassId) - getHeaderSize() -
	(Header->Offset << MinAlignmentLog);
	return SecondaryT::GetActuallyAllocatedSize(
	getBackendPtr(Ptr, Header)) - getHeaderSize();
	}

	// Returns the size the user requested when allocating the chunk.
	static inline uptr getSize(const void Ptr, UnpackedHeader Header) {
	const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
	if (Header->ClassId)
	return SizeOrUnusedBytes;
	return SecondaryT::GetActuallyAllocatedSize(
	getBackendPtr(Ptr, Header)) - getHeaderSize() - SizeOrUnusedBytes;
	}

	// Compute the checksum of the chunk pointer and its header.
	static inline u16 computeChecksum(const void Ptr, UnpackedHeader Header) {
	UnpackedHeader ZeroChecksumHeader = *Header;
	ZeroChecksumHeader.Checksum = 0;
	uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)];
	memcpy(&HeaderHolder, &ZeroChecksumHeader, sizeof(HeaderHolder));
	const u32 Crc = computeCRC32(Cookie, reinterpret_cast<uptr>(Ptr),
	HeaderHolder, ARRAY_SIZE(HeaderHolder));
	return static_cast<u16>(Crc);
	}

	// Checks the validity of a chunk by verifying its checksum. It doesn't
	// incur termination in the event of an invalid chunk.
	static inline bool isValid(const void *Ptr) {
	PackedHeader NewPackedHeader =
	atomic_load_relaxed(getConstAtomicHeader(Ptr));
	UnpackedHeader NewUnpackedHeader =
	bit_cast<UnpackedHeader>(NewPackedHeader);
	return (NewUnpackedHeader.Checksum ==
	computeChecksum(Ptr, &NewUnpackedHeader));
	}

	// Ensure that ChunkAvailable is 0, so that if a 0 checksum is ever valid
	// for a fully nulled out header, its state will be available anyway.
	COMPILER_CHECK(ChunkAvailable == 0);

	// Loads and unpacks the header, verifying the checksum in the process.
	static inline
	void loadHeader(const void Ptr, UnpackedHeader NewUnpackedHeader) {
	PackedHeader NewPackedHeader =
	atomic_load_relaxed(getConstAtomicHeader(Ptr));
	*NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader);
	if (UNLIKELY(NewUnpackedHeader->Checksum !=
	computeChecksum(Ptr, NewUnpackedHeader)))
	dieWithMessage("corrupted chunk header at address %p\n", Ptr);
	}

	// Packs and stores the header, computing the checksum in the process.
	static inline void storeHeader(void Ptr, UnpackedHeader NewUnpackedHeader) {
	NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader);
	PackedHeader NewPackedHeader = bit_cast<PackedHeader>(*NewUnpackedHeader);
	atomic_store_relaxed(getAtomicHeader(Ptr), NewPackedHeader);
	}

	// Packs and stores the header, computing the checksum in the process. We
	// compare the current header with the expected provided one to ensure that
	// we are not being raced by a corruption occurring in another thread.
	static inline void compareExchangeHeader(void *Ptr,
	UnpackedHeader *NewUnpackedHeader,
	UnpackedHeader *OldUnpackedHeader) {
	NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader);
	PackedHeader NewPackedHeader = bit_cast<PackedHeader>(*NewUnpackedHeader);
	PackedHeader OldPackedHeader = bit_cast<PackedHeader>(*OldUnpackedHeader);
	if (UNLIKELY(!atomic_compare_exchange_strong(
	getAtomicHeader(Ptr), &OldPackedHeader, NewPackedHeader,
	memory_order_relaxed)))
	dieWithMessage("race on chunk header at address %p\n", Ptr);
	}
	} // namespace Chunk

	struct QuarantineCallback {
	explicit QuarantineCallback(AllocatorCacheT *Cache)
	: Cache_(Cache) {}

	// Chunk recycling function, returns a quarantined chunk to the backend,
	// first making sure it hasn't been tampered with.
	void Recycle(void *Ptr) {
	UnpackedHeader Header;
	Chunk::loadHeader(Ptr, &Header);
	if (UNLIKELY(Header.State != ChunkQuarantine))
	dieWithMessage("invalid chunk state when recycling address %p\n", Ptr);
	UnpackedHeader NewHeader = Header;
	NewHeader.State = ChunkAvailable;
	Chunk::compareExchangeHeader(Ptr, &NewHeader, &Header);
	void *BackendPtr = Chunk::getBackendPtr(Ptr, &Header);
	if (Header.ClassId)
	getBackend().deallocatePrimary(Cache_, BackendPtr, Header.ClassId);
	else
	getBackend().deallocateSecondary(BackendPtr);
	}

	// Internal quarantine allocation and deallocation functions. We first check
	// that the batches are indeed serviced by the Primary.
	// TODO(kostyak): figure out the best way to protect the batches.
	void *Allocate(uptr Size) {
	const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch));
	return getBackend().allocatePrimary(Cache_, BatchClassId);
	}

	void Deallocate(void *Ptr) {
	const uptr BatchClassId = SizeClassMap::ClassID(sizeof(QuarantineBatch));
	getBackend().deallocatePrimary(Cache_, Ptr, BatchClassId);
	}

	AllocatorCacheT *Cache_;
	COMPILER_CHECK(sizeof(QuarantineBatch) < SizeClassMap::kMaxSize);
	};

	typedef Quarantine<QuarantineCallback, void> QuarantineT;
	typedef QuarantineT::Cache QuarantineCacheT;
	COMPILER_CHECK(sizeof(QuarantineCacheT) <=
	sizeof(ScudoTSD::QuarantineCachePlaceHolder));

	QuarantineCacheT getQuarantineCache(ScudoTSD TSD) {
	return reinterpret_cast<QuarantineCacheT *>(TSD->QuarantineCachePlaceHolder);
	}

	#ifdef GWP_ASAN_HOOKS
	static gwp_asan::GuardedPoolAllocator GuardedAlloc;
	#endif // GWP_ASAN_HOOKS

	struct Allocator {
	static const uptr MaxAllowedMallocSize =
	FIRST_32_SECOND_64(2UL << 30, 1ULL << 40);

	BackendT Backend;
	QuarantineT Quarantine;

	u32 QuarantineChunksUpToSize;

	bool DeallocationTypeMismatch;
	bool ZeroContents;
	bool DeleteSizeMismatch;

	bool CheckRssLimit;
	uptr HardRssLimitMb;
	uptr SoftRssLimitMb;
	atomic_uint8_t RssLimitExceeded;
	atomic_uint64_t RssLastCheckedAtNS;

	explicit Allocator(LinkerInitialized)
	: Quarantine(LINKER_INITIALIZED) {}

	NOINLINE void performSanityChecks();

	void init() {
	SanitizerToolName = "Scudo";
	PrimaryAllocatorName = "ScudoPrimary";
	SecondaryAllocatorName = "ScudoSecondary";

	initFlags();

	performSanityChecks();

	// Check if hardware CRC32 is supported in the binary and by the platform,
	// if so, opt for the CRC32 hardware version of the checksum.
	if (&computeHardwareCRC32 && hasHardwareCRC32())
	atomic_store_relaxed(&HashAlgorithm, CRC32Hardware);

	SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
	Backend.init(common_flags()->allocator_release_to_os_interval_ms);
	HardRssLimitMb = common_flags()->hard_rss_limit_mb;
	SoftRssLimitMb = common_flags()->soft_rss_limit_mb;
	Quarantine.Init(
	static_cast<uptr>(getFlags()->QuarantineSizeKb) << 10,
	static_cast<uptr>(getFlags()->ThreadLocalQuarantineSizeKb) << 10);
	QuarantineChunksUpToSize = (Quarantine.GetCacheSize() == 0) ? 0 :
	getFlags()->QuarantineChunksUpToSize;
	DeallocationTypeMismatch = getFlags()->DeallocationTypeMismatch;
	DeleteSizeMismatch = getFlags()->DeleteSizeMismatch;
	ZeroContents = getFlags()->ZeroContents;

	if (UNLIKELY(!GetRandom(reinterpret_cast<void *>(&Cookie), sizeof(Cookie),
	/blocking=/false))) {
	Cookie = static_cast<u32>((NanoTime() >> 12) ^
	(reinterpret_cast<uptr>(this) >> 4));
	}

	CheckRssLimit = HardRssLimitMb \|\| SoftRssLimitMb;
	if (CheckRssLimit)
	atomic_store_relaxed(&RssLastCheckedAtNS, MonotonicNanoTime());
	}

	// Helper function that checks for a valid Scudo chunk. nullptr isn't.
	bool isValidPointer(const void *Ptr) {
	initThreadMaybe();
	if (UNLIKELY(!Ptr))
	return false;
	if (!Chunk::isAligned(Ptr))
	return false;
	return Chunk::isValid(Ptr);
	}

	NOINLINE bool isRssLimitExceeded();

	// Allocates a chunk.
	void *
	allocate(uptr Size, uptr Alignment, AllocType Type,
	bool ForceZeroContents = false) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	initThreadMaybe();

	if (UNLIKELY(Alignment > MaxAlignment)) {
	if (AllocatorMayReturnNull())
	return nullptr;
	reportAllocationAlignmentTooBig(Alignment, MaxAlignment);
	}
	if (UNLIKELY(Alignment < MinAlignment))
	Alignment = MinAlignment;

	#ifdef GWP_ASAN_HOOKS
	if (UNLIKELY(GuardedAlloc.shouldSample())) {
	if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) {
	if (SCUDO_CAN_USE_HOOKS && &__sanitizer_malloc_hook)
	__sanitizer_malloc_hook(Ptr, Size);
	return Ptr;
	}
	}
	#endif // GWP_ASAN_HOOKS

	const uptr NeededSize = RoundUpTo(Size ? Size : 1, MinAlignment) +
	Chunk::getHeaderSize();
	const uptr AlignedSize = (Alignment > MinAlignment) ?
	NeededSize + (Alignment - Chunk::getHeaderSize()) : NeededSize;
	if (UNLIKELY(Size >= MaxAllowedMallocSize) \|\|
	UNLIKELY(AlignedSize >= MaxAllowedMallocSize)) {
	if (AllocatorMayReturnNull())
	return nullptr;
	reportAllocationSizeTooBig(Size, AlignedSize, MaxAllowedMallocSize);
	}

	if (CheckRssLimit && UNLIKELY(isRssLimitExceeded())) {
	if (AllocatorMayReturnNull())
	return nullptr;
	reportRssLimitExceeded();
	}

	// Primary and Secondary backed allocations have a different treatment. We
	// deal with alignment requirements of Primary serviced allocations here,
	// but the Secondary will take care of its own alignment needs.
	void *BackendPtr;
	uptr BackendSize;
	u8 ClassId;
	if (PrimaryT::CanAllocate(AlignedSize, MinAlignment)) {
	BackendSize = AlignedSize;
	ClassId = SizeClassMap::ClassID(BackendSize);
	bool UnlockRequired;
	ScudoTSD *TSD = getTSDAndLock(&UnlockRequired);
	BackendPtr = Backend.allocatePrimary(&TSD->Cache, ClassId);
	if (UnlockRequired)
	TSD->unlock();
	} else {
	BackendSize = NeededSize;
	ClassId = 0;
	BackendPtr = Backend.allocateSecondary(BackendSize, Alignment);
	}
	if (UNLIKELY(!BackendPtr)) {
	SetAllocatorOutOfMemory();
	if (AllocatorMayReturnNull())
	return nullptr;
	reportOutOfMemory(Size);
	}

	// If requested, we will zero out the entire contents of the returned chunk.
	if ((ForceZeroContents \|\| ZeroContents) && ClassId)
	memset(BackendPtr, 0, PrimaryT::ClassIdToSize(ClassId));

	UnpackedHeader Header = {};
	uptr UserPtr = reinterpret_cast<uptr>(BackendPtr) + Chunk::getHeaderSize();
	if (UNLIKELY(!IsAligned(UserPtr, Alignment))) {
	// Since the Secondary takes care of alignment, a non-aligned pointer
	// means it is from the Primary. It is also the only case where the offset
	// field of the header would be non-zero.
	DCHECK(ClassId);
	const uptr AlignedUserPtr = RoundUpTo(UserPtr, Alignment);
	Header.Offset = (AlignedUserPtr - UserPtr) >> MinAlignmentLog;
	UserPtr = AlignedUserPtr;
	}
	DCHECK_LE(UserPtr + Size, reinterpret_cast<uptr>(BackendPtr) + BackendSize);
	Header.State = ChunkAllocated;
	Header.AllocType = Type;
	if (ClassId) {
	Header.ClassId = ClassId;
	Header.SizeOrUnusedBytes = Size;
	} else {
	// The secondary fits the allocations to a page, so the amount of unused
	// bytes is the difference between the end of the user allocation and the
	// next page boundary.
	const uptr PageSize = GetPageSizeCached();
	const uptr TrailingBytes = (UserPtr + Size) & (PageSize - 1);
	if (TrailingBytes)
	Header.SizeOrUnusedBytes = PageSize - TrailingBytes;
	}
	void Ptr = reinterpret_cast<void >(UserPtr);
	Chunk::storeHeader(Ptr, &Header);
	if (SCUDO_CAN_USE_HOOKS && &__sanitizer_malloc_hook)
	__sanitizer_malloc_hook(Ptr, Size);
	return Ptr;
	}

	// Place a chunk in the quarantine or directly deallocate it in the event of
	// a zero-sized quarantine, or if the size of the chunk is greater than the
	// quarantine chunk size threshold.
	void quarantineOrDeallocateChunk(void Ptr, UnpackedHeader Header, uptr Size)
	SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	const bool BypassQuarantine = !Size \|\| (Size > QuarantineChunksUpToSize);
	if (BypassQuarantine) {
	UnpackedHeader NewHeader = *Header;
	NewHeader.State = ChunkAvailable;
	Chunk::compareExchangeHeader(Ptr, &NewHeader, Header);
	void *BackendPtr = Chunk::getBackendPtr(Ptr, Header);
	if (Header->ClassId) {
	bool UnlockRequired;
	ScudoTSD *TSD = getTSDAndLock(&UnlockRequired);
	getBackend().deallocatePrimary(&TSD->Cache, BackendPtr,
	Header->ClassId);
	if (UnlockRequired)
	TSD->unlock();
	} else {
	getBackend().deallocateSecondary(BackendPtr);
	}
	} else {
	// If a small memory amount was allocated with a larger alignment, we want
	// to take that into account. Otherwise the Quarantine would be filled
	// with tiny chunks, taking a lot of VA memory. This is an approximation
	// of the usable size, that allows us to not call
	// GetActuallyAllocatedSize.
	const uptr EstimatedSize = Size + (Header->Offset << MinAlignmentLog);
	UnpackedHeader NewHeader = *Header;
	NewHeader.State = ChunkQuarantine;
	Chunk::compareExchangeHeader(Ptr, &NewHeader, Header);
	bool UnlockRequired;
	ScudoTSD *TSD = getTSDAndLock(&UnlockRequired);
	Quarantine.Put(getQuarantineCache(TSD), QuarantineCallback(&TSD->Cache),
	Ptr, EstimatedSize);
	if (UnlockRequired)
	TSD->unlock();
	}
	}

	// Deallocates a Chunk, which means either adding it to the quarantine or
	// directly returning it to the backend if criteria are met.
	void deallocate(void *Ptr, uptr DeleteSize, uptr DeleteAlignment,
	AllocType Type) {
	// For a deallocation, we only ensure minimal initialization, meaning thread
	// local data will be left uninitialized for now (when using ELF TLS). The
	// fallback cache will be used instead. This is a workaround for a situation
	// where the only heap operation performed in a thread would be a free past
	// the TLS destructors, ending up in initialized thread specific data never
	// being destroyed properly. Any other heap operation will do a full init.
	initThreadMaybe(/MinimalInit=/true);
	if (SCUDO_CAN_USE_HOOKS && &__sanitizer_free_hook)
	__sanitizer_free_hook(Ptr);
	if (UNLIKELY(!Ptr))
	return;

	#ifdef GWP_ASAN_HOOKS
	if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) {
	GuardedAlloc.deallocate(Ptr);
	return;
	}
	#endif // GWP_ASAN_HOOKS

	if (UNLIKELY(!Chunk::isAligned(Ptr)))
	dieWithMessage("misaligned pointer when deallocating address %p\n", Ptr);
	UnpackedHeader Header;
	Chunk::loadHeader(Ptr, &Header);
	if (UNLIKELY(Header.State != ChunkAllocated))
	dieWithMessage("invalid chunk state when deallocating address %p\n", Ptr);
	if (DeallocationTypeMismatch) {
	// The deallocation type has to match the allocation one.
	if (Header.AllocType != Type) {
	// With the exception of memalign'd Chunks, that can be still be free'd.
	if (Header.AllocType != FromMemalign \|\| Type != FromMalloc)
	dieWithMessage("allocation type mismatch when deallocating address "
	"%p\n", Ptr);
	}
	}
	const uptr Size = Chunk::getSize(Ptr, &Header);
	if (DeleteSizeMismatch) {
	if (DeleteSize && DeleteSize != Size)
	dieWithMessage("invalid sized delete when deallocating address %p\n",
	Ptr);
	}
	(void)DeleteAlignment; // TODO(kostyak): verify that the alignment matches.
	quarantineOrDeallocateChunk(Ptr, &Header, Size);
	}

	// Reallocates a chunk. We can save on a new allocation if the new requested
	// size still fits in the chunk.
	void reallocate(void OldPtr, uptr NewSize) {
	initThreadMaybe();

	#ifdef GWP_ASAN_HOOKS
	if (UNLIKELY(GuardedAlloc.pointerIsMine(OldPtr))) {
	size_t OldSize = GuardedAlloc.getSize(OldPtr);
	void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc);
	if (NewPtr)
	memcpy(NewPtr, OldPtr, (NewSize < OldSize) ? NewSize : OldSize);
	GuardedAlloc.deallocate(OldPtr);
	return NewPtr;
	}
	#endif // GWP_ASAN_HOOKS

	if (UNLIKELY(!Chunk::isAligned(OldPtr)))
	dieWithMessage("misaligned address when reallocating address %p\n",
	OldPtr);
	UnpackedHeader OldHeader;
	Chunk::loadHeader(OldPtr, &OldHeader);
	if (UNLIKELY(OldHeader.State != ChunkAllocated))
	dieWithMessage("invalid chunk state when reallocating address %p\n",
	OldPtr);
	if (DeallocationTypeMismatch) {
	if (UNLIKELY(OldHeader.AllocType != FromMalloc))
	dieWithMessage("allocation type mismatch when reallocating address "
	"%p\n", OldPtr);
	}
	const uptr UsableSize = Chunk::getUsableSize(OldPtr, &OldHeader);
	// The new size still fits in the current chunk, and the size difference
	// is reasonable.
	if (NewSize <= UsableSize &&
	(UsableSize - NewSize) < (SizeClassMap::kMaxSize / 2)) {
	UnpackedHeader NewHeader = OldHeader;
	NewHeader.SizeOrUnusedBytes =
	OldHeader.ClassId ? NewSize : UsableSize - NewSize;
	Chunk::compareExchangeHeader(OldPtr, &NewHeader, &OldHeader);
	return OldPtr;
	}
	// Otherwise, we have to allocate a new chunk and copy the contents of the
	// old one.
	void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc);
	if (NewPtr) {
	const uptr OldSize = OldHeader.ClassId ? OldHeader.SizeOrUnusedBytes :
	UsableSize - OldHeader.SizeOrUnusedBytes;
	memcpy(NewPtr, OldPtr, Min(NewSize, UsableSize));
	quarantineOrDeallocateChunk(OldPtr, &OldHeader, OldSize);
	}
	return NewPtr;
	}

	// Helper function that returns the actual usable size of a chunk.
	uptr getUsableSize(const void *Ptr) {
	initThreadMaybe();
	if (UNLIKELY(!Ptr))
	return 0;

	#ifdef GWP_ASAN_HOOKS
	if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr)))
	return GuardedAlloc.getSize(Ptr);
	#endif // GWP_ASAN_HOOKS

	UnpackedHeader Header;
	Chunk::loadHeader(Ptr, &Header);
	// Getting the usable size of a chunk only makes sense if it's allocated.
	if (UNLIKELY(Header.State != ChunkAllocated))
	dieWithMessage("invalid chunk state when sizing address %p\n", Ptr);
	return Chunk::getUsableSize(Ptr, &Header);
	}

	void *calloc(uptr NMemB, uptr Size) {
	initThreadMaybe();
	if (UNLIKELY(CheckForCallocOverflow(NMemB, Size))) {
	if (AllocatorMayReturnNull())
	return nullptr;
	reportCallocOverflow(NMemB, Size);
	}
	return allocate(NMemB * Size, MinAlignment, FromMalloc, true);
	}

	void commitBack(ScudoTSD *TSD) {
	Quarantine.Drain(getQuarantineCache(TSD), QuarantineCallback(&TSD->Cache));
	Backend.destroyCache(&TSD->Cache);
	}

	uptr getStats(AllocatorStat StatType) {
	initThreadMaybe();
	uptr stats[AllocatorStatCount];
	Backend.getStats(stats);
	return stats[StatType];
	}

	bool canReturnNull() {
	initThreadMaybe();
	return AllocatorMayReturnNull();
	}

	void setRssLimit(uptr LimitMb, bool HardLimit) {
	if (HardLimit)
	HardRssLimitMb = LimitMb;
	else
	SoftRssLimitMb = LimitMb;
	CheckRssLimit = HardRssLimitMb \|\| SoftRssLimitMb;
	}

	void printStats() {
	initThreadMaybe();
	Backend.printStats();
	}
	};

	NOINLINE void Allocator::performSanityChecks() {
	// Verify that the header offset field can hold the maximum offset. In the
	// case of the Secondary allocator, it takes care of alignment and the
	// offset will always be 0. In the case of the Primary, the worst case
	// scenario happens in the last size class, when the backend allocation
	// would already be aligned on the requested alignment, which would happen
	// to be the maximum alignment that would fit in that size class. As a
	// result, the maximum offset will be at most the maximum alignment for the
	// last size class minus the header size, in multiples of MinAlignment.
	UnpackedHeader Header = {};
	const uptr MaxPrimaryAlignment =
	1 << MostSignificantSetBitIndex(SizeClassMap::kMaxSize - MinAlignment);
	const uptr MaxOffset =
	(MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog;
	Header.Offset = MaxOffset;
	if (Header.Offset != MaxOffset)
	dieWithMessage("maximum possible offset doesn't fit in header\n");
	// Verify that we can fit the maximum size or amount of unused bytes in the
	// header. Given that the Secondary fits the allocation to a page, the worst
	// case scenario happens in the Primary. It will depend on the second to
	// last and last class sizes, as well as the dynamic base for the Primary.
	// The following is an over-approximation that works for our needs.
	const uptr MaxSizeOrUnusedBytes = SizeClassMap::kMaxSize - 1;
	Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;
	if (Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes)
	dieWithMessage("maximum possible unused bytes doesn't fit in header\n");

	const uptr LargestClassId = SizeClassMap::kLargestClassID;
	Header.ClassId = LargestClassId;
	if (Header.ClassId != LargestClassId)
	dieWithMessage("largest class ID doesn't fit in header\n");
	}

	// Opportunistic RSS limit check. This will update the RSS limit status, if
	// it can, every 250ms, otherwise it will just return the current one.
	NOINLINE bool Allocator::isRssLimitExceeded() {
	u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS);
	const u64 CurrentCheck = MonotonicNanoTime();
	if (LIKELY(CurrentCheck < LastCheck + (250ULL * 1000000ULL)))
	return atomic_load_relaxed(&RssLimitExceeded);
	if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck,
	CurrentCheck, memory_order_relaxed))
	return atomic_load_relaxed(&RssLimitExceeded);
	// TODO(kostyak): We currently use sanitizer_common's GetRSS which reads the
	// RSS from /proc/self/statm by default. We might want to
	// call getrusage directly, even if it's less accurate.
	const uptr CurrentRssMb = GetRSS() >> 20;
	if (HardRssLimitMb && UNLIKELY(HardRssLimitMb < CurrentRssMb))
	dieWithMessage("hard RSS limit exhausted (%zdMb vs %zdMb)\n",
	HardRssLimitMb, CurrentRssMb);
	if (SoftRssLimitMb) {
	if (atomic_load_relaxed(&RssLimitExceeded)) {
	if (CurrentRssMb <= SoftRssLimitMb)
	atomic_store_relaxed(&RssLimitExceeded, false);
	} else {
	if (CurrentRssMb > SoftRssLimitMb) {
	atomic_store_relaxed(&RssLimitExceeded, true);
	Printf("Scudo INFO: soft RSS limit exhausted (%zdMb vs %zdMb)\n",
	SoftRssLimitMb, CurrentRssMb);
	}
	}
	}
	return atomic_load_relaxed(&RssLimitExceeded);
	}

	static Allocator Instance(LINKER_INITIALIZED);

	static BackendT &getBackend() {
	return Instance.Backend;
	}

	void initScudo() {
	Instance.init();
	#ifdef GWP_ASAN_HOOKS
	gwp_asan::options::initOptions(__sanitizer::GetEnv("GWP_ASAN_OPTIONS"),
	Printf);
	gwp_asan::options::Options &Opts = gwp_asan::options::getOptions();
	Opts.Backtrace = gwp_asan::backtrace::getBacktraceFunction();
	GuardedAlloc.init(Opts);

	if (Opts.InstallSignalHandlers)
	gwp_asan::segv_handler::installSignalHandlers(
	&GuardedAlloc, __sanitizer::Printf,
	gwp_asan::backtrace::getPrintBacktraceFunction(),
	gwp_asan::backtrace::getSegvBacktraceFunction());
	#endif // GWP_ASAN_HOOKS
	}

	void ScudoTSD::init() {
	getBackend().initCache(&Cache);
	memset(QuarantineCachePlaceHolder, 0, sizeof(QuarantineCachePlaceHolder));
	}

	void ScudoTSD::commitBack() {
	Instance.commitBack(this);
	}

	void *scudoAllocate(uptr Size, uptr Alignment, AllocType Type) {
	if (Alignment && UNLIKELY(!IsPowerOfTwo(Alignment))) {
	errno = EINVAL;
	if (Instance.canReturnNull())
	return nullptr;
	reportAllocationAlignmentNotPowerOfTwo(Alignment);
	}
	return SetErrnoOnNull(Instance.allocate(Size, Alignment, Type));
	}

	void scudoDeallocate(void *Ptr, uptr Size, uptr Alignment, AllocType Type) {
	Instance.deallocate(Ptr, Size, Alignment, Type);
	}

	void scudoRealloc(void Ptr, uptr Size) {
	if (!Ptr)
	return SetErrnoOnNull(Instance.allocate(Size, MinAlignment, FromMalloc));
	if (Size == 0) {
	Instance.deallocate(Ptr, 0, 0, FromMalloc);
	return nullptr;
	}
	return SetErrnoOnNull(Instance.reallocate(Ptr, Size));
	}

	void *scudoCalloc(uptr NMemB, uptr Size) {
	return SetErrnoOnNull(Instance.calloc(NMemB, Size));
	}

	void *scudoValloc(uptr Size) {
	return SetErrnoOnNull(
	Instance.allocate(Size, GetPageSizeCached(), FromMemalign));
	}

	void *scudoPvalloc(uptr Size) {
	const uptr PageSize = GetPageSizeCached();
	if (UNLIKELY(CheckForPvallocOverflow(Size, PageSize))) {
	errno = ENOMEM;
	if (Instance.canReturnNull())
	return nullptr;
	reportPvallocOverflow(Size);
	}
	// pvalloc(0) should allocate one page.
	Size = Size ? RoundUpTo(Size, PageSize) : PageSize;
	return SetErrnoOnNull(Instance.allocate(Size, PageSize, FromMemalign));
	}

	int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size) {
	if (UNLIKELY(!CheckPosixMemalignAlignment(Alignment))) {
	if (!Instance.canReturnNull())
	reportInvalidPosixMemalignAlignment(Alignment);
	return EINVAL;
	}
	void *Ptr = Instance.allocate(Size, Alignment, FromMemalign);
	if (UNLIKELY(!Ptr))
	return ENOMEM;
	*MemPtr = Ptr;
	return 0;
	}

	void *scudoAlignedAlloc(uptr Alignment, uptr Size) {
	if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(Alignment, Size))) {
	errno = EINVAL;
	if (Instance.canReturnNull())
	return nullptr;
	reportInvalidAlignedAllocAlignment(Size, Alignment);
	}
	return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMalloc));
	}

	uptr scudoMallocUsableSize(void *Ptr) {
	return Instance.getUsableSize(Ptr);
	}

	} // namespace __scudo

	using namespace __scudo;

	// MallocExtension helper functions

	uptr __sanitizer_get_current_allocated_bytes() {
	return Instance.getStats(AllocatorStatAllocated);
	}

	uptr __sanitizer_get_heap_size() {
	return Instance.getStats(AllocatorStatMapped);
	}

	uptr __sanitizer_get_free_bytes() {
	return 1;
	}

	uptr __sanitizer_get_unmapped_bytes() {
	return 1;
	}

	uptr __sanitizer_get_estimated_allocated_size(uptr Size) {
	return Size;
	}

	int __sanitizer_get_ownership(const void *Ptr) {
	return Instance.isValidPointer(Ptr);
	}

	uptr __sanitizer_get_allocated_size(const void *Ptr) {
	return Instance.getUsableSize(Ptr);
	}

	#if !SANITIZER_SUPPORTS_WEAK_HOOKS
	SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook,
	void *Ptr, uptr Size) {
	(void)Ptr;
	(void)Size;
	}

	SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *Ptr) {
	(void)Ptr;
	}
	#endif

	// Interface functions

	void __scudo_set_rss_limit(uptr LimitMb, s32 HardLimit) {
	if (!SCUDO_CAN_USE_PUBLIC_INTERFACE)
	return;
	Instance.setRssLimit(LimitMb, !!HardLimit);
	}

	void __scudo_print_stats() {
	Instance.printStats();
	}

compiler-rt/lib/scudo/scudo_allocator_combined.h

This file was deleted.

	//===-- scudo_allocator_combined.h ------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Scudo Combined Allocator, dispatches allocation & deallocation requests to
	/// the Primary or the Secondary backend allocators.
	///
	//===----------------------------------------------------------------------===//

	#ifndef SCUDO_ALLOCATOR_COMBINED_H_
	#define SCUDO_ALLOCATOR_COMBINED_H_

	#ifndef SCUDO_ALLOCATOR_H_
	# error "This file must be included inside scudo_allocator.h."
	#endif

	class CombinedAllocator {
	public:
	using PrimaryAllocator = PrimaryT;
	using SecondaryAllocator = SecondaryT;
	using AllocatorCache = typename PrimaryAllocator::AllocatorCache;
	void init(s32 ReleaseToOSIntervalMs) {
	Primary.Init(ReleaseToOSIntervalMs);
	Secondary.Init();
	Stats.Init();
	}

	// Primary allocations are always MinAlignment aligned, and as such do not
	// require an Alignment parameter.
	void allocatePrimary(AllocatorCache Cache, uptr ClassId) {
	return Cache->Allocate(&Primary, ClassId);
	}

	// Secondary allocations do not require a Cache, but do require an Alignment
	// parameter.
	void *allocateSecondary(uptr Size, uptr Alignment) {
	return Secondary.Allocate(&Stats, Size, Alignment);
	}

	void deallocatePrimary(AllocatorCache Cache, void Ptr, uptr ClassId) {
	Cache->Deallocate(&Primary, ClassId, Ptr);
	}

	void deallocateSecondary(void *Ptr) {
	Secondary.Deallocate(&Stats, Ptr);
	}

	void initCache(AllocatorCache *Cache) {
	Cache->Init(&Stats);
	}

	void destroyCache(AllocatorCache *Cache) {
	Cache->Destroy(&Primary, &Stats);
	}

	void getStats(AllocatorStatCounters StatType) const {
	Stats.Get(StatType);
	}

	void printStats() {
	Primary.PrintStats();
	Secondary.PrintStats();
	}

	private:
	PrimaryAllocator Primary;
	SecondaryAllocator Secondary;
	AllocatorGlobalStats Stats;
	};

	#endif // SCUDO_ALLOCATOR_COMBINED_H_

compiler-rt/lib/scudo/scudo_allocator_secondary.h

This file was deleted.

	//===-- scudo_allocator_secondary.h ------------------------------ C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Scudo Secondary Allocator.
	/// This services allocation that are too large to be serviced by the Primary
	/// Allocator. It is directly backed by the memory mapping functions of the
	/// operating system.
	///
	//===----------------------------------------------------------------------===//

	#ifndef SCUDO_ALLOCATOR_SECONDARY_H_
	#define SCUDO_ALLOCATOR_SECONDARY_H_

	#ifndef SCUDO_ALLOCATOR_H_
	# error "This file must be included inside scudo_allocator.h."
	#endif

	// Secondary backed allocations are standalone chunks that contain extra
	// information stored in a LargeChunk::Header prior to the frontend's header.
	//
	// The secondary takes care of alignment requirements (so that it can release
	// unnecessary pages in the rare event of larger alignments), and as such must
	// know about the frontend's header size.
	//
	// Since Windows doesn't support partial releasing of a reserved memory region,
	// we have to keep track of both the reserved and the committed memory.
	//
	// The resulting chunk resembles the following:
	//
	// +--------------------+
	// \| Guard page(s) \|
	// +--------------------+
	// \| Unused space* \|
	// +--------------------+
	// \| LargeChunk::Header \|
	// +--------------------+
	// \| {Unp,P}ackedHeader \|
	// +--------------------+
	// \| Data (aligned) \|
	// +--------------------+
	// \| Unused space** \|
	// +--------------------+
	// \| Guard page(s) \|
	// +--------------------+

	namespace LargeChunk {
	struct Header {
	ReservedAddressRange StoredRange;
	uptr CommittedSize;
	uptr Size;
	};
	constexpr uptr getHeaderSize() {
	return RoundUpTo(sizeof(Header), MinAlignment);
	}
	static Header *getHeader(uptr Ptr) {
	return reinterpret_cast<Header *>(Ptr - getHeaderSize());
	}
	static Header getHeader(const void Ptr) {
	return getHeader(reinterpret_cast<uptr>(Ptr));
	}
	} // namespace LargeChunk

	class LargeMmapAllocator {
	public:
	void Init() {
	internal_memset(this, 0, sizeof(*this));
	}

	void Allocate(AllocatorStats Stats, uptr Size, uptr Alignment) {
	const uptr UserSize = Size - Chunk::getHeaderSize();
	// The Scudo frontend prevents us from allocating more than
	// MaxAllowedMallocSize, so integer overflow checks would be superfluous.
	uptr ReservedSize = Size + LargeChunk::getHeaderSize();
	if (UNLIKELY(Alignment > MinAlignment))
	ReservedSize += Alignment;
	const uptr PageSize = GetPageSizeCached();
	ReservedSize = RoundUpTo(ReservedSize, PageSize);
	// Account for 2 guard pages, one before and one after the chunk.
	ReservedSize += 2 * PageSize;

	ReservedAddressRange AddressRange;
	uptr ReservedBeg = AddressRange.Init(ReservedSize, SecondaryAllocatorName);
	if (UNLIKELY(ReservedBeg == ~static_cast<uptr>(0)))
	return nullptr;
	// A page-aligned pointer is assumed after that, so check it now.
	DCHECK(IsAligned(ReservedBeg, PageSize));
	uptr ReservedEnd = ReservedBeg + ReservedSize;
	// The beginning of the user area for that allocation comes after the
	// initial guard page, and both headers. This is the pointer that has to
	// abide by alignment requirements.
	uptr CommittedBeg = ReservedBeg + PageSize;
	uptr UserBeg = CommittedBeg + HeadersSize;
	uptr UserEnd = UserBeg + UserSize;
	uptr CommittedEnd = RoundUpTo(UserEnd, PageSize);

	// In the rare event of larger alignments, we will attempt to fit the mmap
	// area better and unmap extraneous memory. This will also ensure that the
	// offset and unused bytes field of the header stay small.
	if (UNLIKELY(Alignment > MinAlignment)) {
	if (!IsAligned(UserBeg, Alignment)) {
	UserBeg = RoundUpTo(UserBeg, Alignment);
	CommittedBeg = RoundDownTo(UserBeg - HeadersSize, PageSize);
	const uptr NewReservedBeg = CommittedBeg - PageSize;
	DCHECK_GE(NewReservedBeg, ReservedBeg);
	if (!SANITIZER_WINDOWS && NewReservedBeg != ReservedBeg) {
	AddressRange.Unmap(ReservedBeg, NewReservedBeg - ReservedBeg);
	ReservedBeg = NewReservedBeg;
	}
	UserEnd = UserBeg + UserSize;
	CommittedEnd = RoundUpTo(UserEnd, PageSize);
	}
	const uptr NewReservedEnd = CommittedEnd + PageSize;
	DCHECK_LE(NewReservedEnd, ReservedEnd);
	if (!SANITIZER_WINDOWS && NewReservedEnd != ReservedEnd) {
	AddressRange.Unmap(NewReservedEnd, ReservedEnd - NewReservedEnd);
	ReservedEnd = NewReservedEnd;
	}
	}

	DCHECK_LE(UserEnd, CommittedEnd);
	const uptr CommittedSize = CommittedEnd - CommittedBeg;
	// Actually mmap the memory, preserving the guard pages on either sides.
	CHECK_EQ(CommittedBeg, AddressRange.Map(CommittedBeg, CommittedSize));
	const uptr Ptr = UserBeg - Chunk::getHeaderSize();
	LargeChunk::Header *H = LargeChunk::getHeader(Ptr);
	H->StoredRange = AddressRange;
	H->Size = CommittedEnd - Ptr;
	H->CommittedSize = CommittedSize;

	// The primary adds the whole class size to the stats when allocating a
	// chunk, so we will do something similar here. But we will not account for
	// the guard pages.
	{
	SpinMutexLock l(&StatsMutex);
	Stats->Add(AllocatorStatAllocated, CommittedSize);
	Stats->Add(AllocatorStatMapped, CommittedSize);
	AllocatedBytes += CommittedSize;
	if (LargestSize < CommittedSize)
	LargestSize = CommittedSize;
	NumberOfAllocs++;
	}

	return reinterpret_cast<void *>(Ptr);
	}

	void Deallocate(AllocatorStats Stats, void Ptr) {
	LargeChunk::Header *H = LargeChunk::getHeader(Ptr);
	// Since we're unmapping the entirety of where the ReservedAddressRange
	// actually is, copy onto the stack.
	ReservedAddressRange AddressRange = H->StoredRange;
	const uptr Size = H->CommittedSize;
	{
	SpinMutexLock l(&StatsMutex);
	Stats->Sub(AllocatorStatAllocated, Size);
	Stats->Sub(AllocatorStatMapped, Size);
	FreedBytes += Size;
	NumberOfFrees++;
	}
	AddressRange.Unmap(reinterpret_cast<uptr>(AddressRange.base()),
	AddressRange.size());
	}

	static uptr GetActuallyAllocatedSize(void *Ptr) {
	return LargeChunk::getHeader(Ptr)->Size;
	}

	void PrintStats() {
	Printf("Stats: LargeMmapAllocator: allocated %zd times (%zd K), "
	"freed %zd times (%zd K), remains %zd (%zd K) max %zd M\n",
	NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees,
	FreedBytes >> 10, NumberOfAllocs - NumberOfFrees,
	(AllocatedBytes - FreedBytes) >> 10, LargestSize >> 20);
	}

	private:
	static constexpr uptr HeadersSize =
	LargeChunk::getHeaderSize() + Chunk::getHeaderSize();

	StaticSpinMutex StatsMutex;
	u32 NumberOfAllocs;
	u32 NumberOfFrees;
	uptr AllocatedBytes;
	uptr FreedBytes;
	uptr LargestSize;
	};

	#endif // SCUDO_ALLOCATOR_SECONDARY_H_

compiler-rt/lib/scudo/scudo_crc32.h

This file was deleted.

	//===-- scudo_crc32.h -------------------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Scudo chunk header checksum related definitions.
	///
	//===----------------------------------------------------------------------===//

	#ifndef SCUDO_CRC32_H_
	#define SCUDO_CRC32_H_

	#include "sanitizer_common/sanitizer_internal_defs.h"

	// Hardware CRC32 is supported at compilation via the following:
	// - for i386 & x86_64: -mcrc32 (earlier: -msse4.2)
	// - for ARM & AArch64: -march=armv8-a+crc or -mcrc
	// An additional check must be performed at runtime as well to make sure the
	// emitted instructions are valid on the target host.

	#if defined(__CRC32__) \|\| defined(__SSE4_2__) \|\| defined(__ARM_FEATURE_CRC32)
	# if defined(__CRC32__)
	// NB: clang has <crc32intrin.h> but GCC does not
	# include <smmintrin.h>
	# define CRC32_INTRINSIC FIRST_32_SECOND_64(__builtin_ia32_crc32si, __builtin_ia32_crc32di)
	# elif defined(__SSE4_2__)
	# include <smmintrin.h>
	# define CRC32_INTRINSIC FIRST_32_SECOND_64(_mm_crc32_u32, _mm_crc32_u64)
	# endif
	# ifdef __ARM_FEATURE_CRC32
	# include <arm_acle.h>
	# define CRC32_INTRINSIC FIRST_32_SECOND_64(__crc32cw, __crc32cd)
	# endif
	#endif // defined(__CRC32__) \|\| defined(__SSE4_2__) \|\| defined(__ARM_FEATURE_CRC32)

	namespace __scudo {

	enum : u8 {
	CRC32Software = 0,
	CRC32Hardware = 1,
	};

	static const u32 CRC32Table[] = {
	0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f,
	0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
	0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2,
	0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
	0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9,
	0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
	0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c,
	0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
	0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423,
	0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
	0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106,
	0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
	0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d,
	0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
	0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
	0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
	0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7,
	0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
	0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa,
	0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
	0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81,
	0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
	0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84,
	0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
	0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
	0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
	0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e,
	0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
	0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55,
	0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
	0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28,
	0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
	0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f,
	0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
	0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
	0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
	0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69,
	0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
	0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc,
	0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
	0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693,
	0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
	0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
	};

	inline u32 computeSoftwareCRC32(u32 Crc, uptr Data) {
	for (uptr i = 0; i < sizeof(Data); i++) {
	Crc = CRC32Table[(Crc ^ Data) & 0xff] ^ (Crc >> 8);
	Data >>= 8;
	}
	return Crc;
	}

	SANITIZER_WEAK_ATTRIBUTE u32 computeHardwareCRC32(u32 Crc, uptr Data);

	} // namespace __scudo

	#endif // SCUDO_CRC32_H_

compiler-rt/lib/scudo/scudo_crc32.cpp

This file was deleted.

	//===-- scudo_crc32.cpp ------------------------------------------ C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// CRC32 function leveraging hardware specific instructions. This has to be
	/// kept separated to restrict the use of compiler specific flags to this file.
	///
	//===----------------------------------------------------------------------===//

	#include "scudo_crc32.h"

	namespace __scudo {

	#if defined(__CRC32__) \|\| defined(__SSE4_2__) \|\| defined(__ARM_FEATURE_CRC32)
	u32 computeHardwareCRC32(u32 Crc, uptr Data) {
	return CRC32_INTRINSIC(Crc, Data);
	}
	#endif // defined(__CRC32__) \|\| defined(__SSE4_2__) \|\| defined(__ARM_FEATURE_CRC32)

	} // namespace __scudo

compiler-rt/lib/scudo/scudo_errors.h

This file was deleted.

	//===-- scudo_errors.h ------------------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Header for scudo_errors.cpp.
	///
	//===----------------------------------------------------------------------===//

	#ifndef SCUDO_ERRORS_H_
	#define SCUDO_ERRORS_H_

	#include "sanitizer_common/sanitizer_internal_defs.h"

	namespace __scudo {

	void NORETURN reportCallocOverflow(uptr Count, uptr Size);
	void NORETURN reportPvallocOverflow(uptr Size);
	void NORETURN reportAllocationAlignmentTooBig(uptr Alignment,
	uptr MaxAlignment);
	void NORETURN reportAllocationAlignmentNotPowerOfTwo(uptr Alignment);
	void NORETURN reportInvalidPosixMemalignAlignment(uptr Alignment);
	void NORETURN reportInvalidAlignedAllocAlignment(uptr Size, uptr Alignment);
	void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize,
	uptr MaxSize);
	void NORETURN reportRssLimitExceeded();
	void NORETURN reportOutOfMemory(uptr RequestedSize);

	} // namespace __scudo

	#endif // SCUDO_ERRORS_H_

compiler-rt/lib/scudo/scudo_errors.cpp

This file was deleted.

	//===-- scudo_errors.cpp ----------------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Verbose termination functions.
	///
	//===----------------------------------------------------------------------===//

	#include "scudo_utils.h"

	#include "sanitizer_common/sanitizer_flags.h"

	namespace __scudo {

	void NORETURN reportCallocOverflow(uptr Count, uptr Size) {
	dieWithMessage("calloc parameters overflow: count * size (%zd * %zd) cannot "
	"be represented with type size_t\n", Count, Size);
	}

	void NORETURN reportPvallocOverflow(uptr Size) {
	dieWithMessage("pvalloc parameters overflow: size 0x%zx rounded up to system "
	"page size 0x%zx cannot be represented in type size_t\n", Size,
	GetPageSizeCached());
	}

	void NORETURN reportAllocationAlignmentTooBig(uptr Alignment,
	uptr MaxAlignment) {
	dieWithMessage("invalid allocation alignment: %zd exceeds maximum supported "
	"allocation of %zd\n", Alignment, MaxAlignment);
	}

	void NORETURN reportAllocationAlignmentNotPowerOfTwo(uptr Alignment) {
	dieWithMessage("invalid allocation alignment: %zd, alignment must be a power "
	"of two\n", Alignment);
	}

	void NORETURN reportInvalidPosixMemalignAlignment(uptr Alignment) {
	dieWithMessage(
	"invalid alignment requested in posix_memalign: %zd, alignment"
	" must be a power of two and a multiple of sizeof(void *) == %zd\n",
	Alignment, sizeof(void *));
	}

	void NORETURN reportInvalidAlignedAllocAlignment(uptr Size, uptr Alignment) {
	#if SANITIZER_POSIX
	dieWithMessage("invalid alignment requested in aligned_alloc: %zd, alignment "
	"must be a power of two and the requested size 0x%zx must be a multiple "
	"of alignment\n", Alignment, Size);
	#else
	dieWithMessage("invalid alignment requested in aligned_alloc: %zd, the "
	"requested size 0x%zx must be a multiple of alignment\n", Alignment,
	Size);
	#endif
	}

	void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize,
	uptr MaxSize) {
	dieWithMessage("requested allocation size 0x%zx (0x%zx after adjustments) "
	"exceeds maximum supported size of 0x%zx\n", UserSize, TotalSize,
	MaxSize);
	}

	void NORETURN reportRssLimitExceeded() {
	dieWithMessage("specified RSS limit exceeded, currently set to "
	"soft_rss_limit_mb=%zd\n", common_flags()->soft_rss_limit_mb);
	}

	void NORETURN reportOutOfMemory(uptr RequestedSize) {
	dieWithMessage("allocator is out of memory trying to allocate 0x%zx bytes\n",
	RequestedSize);
	}

	} // namespace __scudo

compiler-rt/lib/scudo/scudo_flags.h

This file was deleted.

	//===-- scudo_flags.h -------------------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Header for scudo_flags.cpp.
	///
	//===----------------------------------------------------------------------===//

	#ifndef SCUDO_FLAGS_H_
	#define SCUDO_FLAGS_H_

	namespace __scudo {

	struct Flags {
	#define SCUDO_FLAG(Type, Name, DefaultValue, Description) Type Name;
	#include "scudo_flags.inc"
	#undef SCUDO_FLAG

	void setDefaults();
	};

	Flags *getFlags();

	void initFlags();

	} // namespace __scudo

	#endif // SCUDO_FLAGS_H_

compiler-rt/lib/scudo/scudo_flags.cpp

This file was deleted.

	//===-- scudo_flags.cpp ------------------------------------------ C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Hardened Allocator flag parsing logic.
	///
	//===----------------------------------------------------------------------===//

	#include "scudo_flags.h"
	#include "scudo_interface_internal.h"
	#include "scudo_utils.h"

	#include "sanitizer_common/sanitizer_flags.h"
	#include "sanitizer_common/sanitizer_flag_parser.h"

	namespace __scudo {

	static Flags ScudoFlags; // Use via getFlags().

	void Flags::setDefaults() {
	#define SCUDO_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
	#include "scudo_flags.inc"
	#undef SCUDO_FLAG
	}

	static void RegisterScudoFlags(FlagParser parser, Flags f) {
	#define SCUDO_FLAG(Type, Name, DefaultValue, Description) \
	RegisterFlag(parser, #Name, Description, &f->Name);
	#include "scudo_flags.inc"
	#undef SCUDO_FLAG
	}

	static const char *getCompileDefinitionScudoDefaultOptions() {
	#ifdef SCUDO_DEFAULT_OPTIONS
	return SANITIZER_STRINGIFY(SCUDO_DEFAULT_OPTIONS);
	#else
	return "";
	#endif
	}

	static const char *getScudoDefaultOptions() {
	return (&__scudo_default_options) ? __scudo_default_options() : "";
	}

	void initFlags() {
	SetCommonFlagsDefaults();
	{
	CommonFlags cf;
	cf.CopyFrom(*common_flags());
	cf.exitcode = 1;
	OverrideCommonFlags(cf);
	}
	Flags *f = getFlags();
	f->setDefaults();

	FlagParser ScudoParser;
	RegisterScudoFlags(&ScudoParser, f);
	RegisterCommonFlags(&ScudoParser);

	// Override from compile definition.
	ScudoParser.ParseString(getCompileDefinitionScudoDefaultOptions());

	// Override from user-specified string.
	ScudoParser.ParseString(getScudoDefaultOptions());

	// Override from environment.
	ScudoParser.ParseStringFromEnv("SCUDO_OPTIONS");

	InitializeCommonFlags();

	// Sanity checks and default settings for the Quarantine parameters.

	if (f->QuarantineSizeMb >= 0) {
	// Backward compatible logic if QuarantineSizeMb is set.
	if (f->QuarantineSizeKb >= 0) {
	dieWithMessage("ERROR: please use either QuarantineSizeMb (deprecated) "
	"or QuarantineSizeKb, but not both\n");
	}
	if (f->QuarantineChunksUpToSize >= 0) {
	dieWithMessage("ERROR: QuarantineChunksUpToSize cannot be used in "
	" conjunction with the deprecated QuarantineSizeMb option\n");
	}
	// If everything is in order, update QuarantineSizeKb accordingly.
	f->QuarantineSizeKb = f->QuarantineSizeMb * 1024;
	} else {
	// Otherwise proceed with the new options.
	if (f->QuarantineSizeKb < 0) {
	const int DefaultQuarantineSizeKb = FIRST_32_SECOND_64(64, 256);
	f->QuarantineSizeKb = DefaultQuarantineSizeKb;
	}
	if (f->QuarantineChunksUpToSize < 0) {
	const int DefaultQuarantineChunksUpToSize = FIRST_32_SECOND_64(512, 2048);
	f->QuarantineChunksUpToSize = DefaultQuarantineChunksUpToSize;
	}
	}

	// We enforce an upper limit for the chunk quarantine threshold of 4Mb.
	if (f->QuarantineChunksUpToSize > (4 * 1024 * 1024)) {
	dieWithMessage("ERROR: the chunk quarantine threshold is too large\n");
	}

	// We enforce an upper limit for the quarantine size of 32Mb.
	if (f->QuarantineSizeKb > (32 * 1024)) {
	dieWithMessage("ERROR: the quarantine size is too large\n");
	}

	if (f->ThreadLocalQuarantineSizeKb < 0) {
	const int DefaultThreadLocalQuarantineSizeKb = FIRST_32_SECOND_64(16, 64);
	f->ThreadLocalQuarantineSizeKb = DefaultThreadLocalQuarantineSizeKb;
	}
	// And an upper limit of 8Mb for the thread quarantine cache.
	if (f->ThreadLocalQuarantineSizeKb > (8 * 1024)) {
	dieWithMessage("ERROR: the per thread quarantine cache size is too "
	"large\n");
	}
	if (f->ThreadLocalQuarantineSizeKb == 0 && f->QuarantineSizeKb > 0) {
	dieWithMessage("ERROR: ThreadLocalQuarantineSizeKb can be set to 0 only "
	"when QuarantineSizeKb is set to 0\n");
	}
	}

	Flags *getFlags() {
	return &ScudoFlags;
	}

	} // namespace __scudo

	#if !SANITIZER_SUPPORTS_WEAK_HOOKS
	SANITIZER_INTERFACE_WEAK_DEF(const char*, __scudo_default_options, void) {
	return "";
	}
	#endif

compiler-rt/lib/scudo/scudo_flags.inc

This file was deleted.

	//===-- scudo_flags.inc ------------------------------------------ C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Hardened Allocator runtime flags.
	///
	//===----------------------------------------------------------------------===//

	#ifndef SCUDO_FLAG
	# error "Define SCUDO_FLAG prior to including this file!"
	#endif

	SCUDO_FLAG(int, QuarantineSizeMb, -1,
	"Deprecated. Please use QuarantineSizeKb.")

	// Default value is set in scudo_flags.cpp based on architecture.
	SCUDO_FLAG(int, QuarantineSizeKb, -1,
	"Size in KB of quarantine used to delay the actual deallocation of "
	"chunks. Lower value may reduce memory usage but decrease the "
	"effectiveness of the mitigation. Defaults to 64KB (32-bit) or "
	"256KB (64-bit)")

	// Default value is set in scudo_flags.cpp based on architecture.
	SCUDO_FLAG(int, ThreadLocalQuarantineSizeKb, -1,
	"Size in KB of per-thread cache used to offload the global "
	"quarantine. Lower value may reduce memory usage but might increase "
	"the contention on the global quarantine. Defaults to 16KB (32-bit) "
	"or 64KB (64-bit)")

	// Default value is set in scudo_flags.cpp based on architecture.
	SCUDO_FLAG(int, QuarantineChunksUpToSize, -1,
	"Size in bytes up to which chunks will be quarantined (if lower than"
	"or equal to). Defaults to 256 (32-bit) or 2048 (64-bit)")

	// Disable the deallocation type check by default on Android, it causes too many
	// issues with third party libraries.
	SCUDO_FLAG(bool, DeallocationTypeMismatch, !SANITIZER_ANDROID,
	"Report errors on malloc/delete, new/free, new/delete[], etc.")

	SCUDO_FLAG(bool, DeleteSizeMismatch, true,
	"Report errors on mismatch between size of new and delete.")

	SCUDO_FLAG(bool, ZeroContents, false,
	"Zero chunk contents on allocation and deallocation.")

compiler-rt/lib/scudo/scudo_interface_internal.h

This file was deleted.

	//===-- scudo_interface_internal.h ------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Private Scudo interface header.
	///
	//===----------------------------------------------------------------------===//

	#ifndef SCUDO_INTERFACE_INTERNAL_H_
	#define SCUDO_INTERFACE_INTERNAL_H_

	#include "sanitizer_common/sanitizer_internal_defs.h"

	using __sanitizer::uptr;
	using __sanitizer::s32;

	extern "C" {
	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
	const char* __scudo_default_options();

	SANITIZER_INTERFACE_ATTRIBUTE
	void __scudo_set_rss_limit(uptr LimitMb, s32 HardLimit);

	SANITIZER_INTERFACE_ATTRIBUTE
	void __scudo_print_stats();
	} // extern "C"

	#endif // SCUDO_INTERFACE_INTERNAL_H_

compiler-rt/lib/scudo/scudo_malloc.cpp

This file was deleted.

	//===-- scudo_malloc.cpp ----------------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Interceptors for malloc related functions.
	///
	//===----------------------------------------------------------------------===//

	#include "scudo_allocator.h"

	#include "interception/interception.h"
	#include "sanitizer_common/sanitizer_platform_interceptors.h"

	#include <stddef.h>

	using namespace __scudo;

	extern "C" {
	INTERCEPTOR_ATTRIBUTE void free(void *ptr) {
	scudoDeallocate(ptr, 0, 0, FromMalloc);
	}

	INTERCEPTOR_ATTRIBUTE void *malloc(size_t size) {
	return scudoAllocate(size, 0, FromMalloc);
	}

	INTERCEPTOR_ATTRIBUTE void realloc(void ptr, size_t size) {
	return scudoRealloc(ptr, size);
	}

	INTERCEPTOR_ATTRIBUTE void *calloc(size_t nmemb, size_t size) {
	return scudoCalloc(nmemb, size);
	}

	INTERCEPTOR_ATTRIBUTE void *valloc(size_t size) {
	return scudoValloc(size);
	}

	INTERCEPTOR_ATTRIBUTE
	int posix_memalign(void **memptr, size_t alignment, size_t size) {
	return scudoPosixMemalign(memptr, alignment, size);
	}

	#if SANITIZER_INTERCEPT_CFREE
	INTERCEPTOR_ATTRIBUTE void cfree(void *ptr) ALIAS("free");
	#endif

	#if SANITIZER_INTERCEPT_MEMALIGN
	INTERCEPTOR_ATTRIBUTE void *memalign(size_t alignment, size_t size) {
	return scudoAllocate(size, alignment, FromMemalign);
	}

	INTERCEPTOR_ATTRIBUTE
	void *__libc_memalign(size_t alignment, size_t size) ALIAS("memalign");
	#endif

	#if SANITIZER_INTERCEPT_PVALLOC
	INTERCEPTOR_ATTRIBUTE void *pvalloc(size_t size) {
	return scudoPvalloc(size);
	}
	#endif

	#if SANITIZER_INTERCEPT_ALIGNED_ALLOC
	INTERCEPTOR_ATTRIBUTE void *aligned_alloc(size_t alignment, size_t size) {
	return scudoAlignedAlloc(alignment, size);
	}
	#endif

	#if SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE
	INTERCEPTOR_ATTRIBUTE size_t malloc_usable_size(void *ptr) {
	return scudoMallocUsableSize(ptr);
	}
	#endif

	#if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
	INTERCEPTOR_ATTRIBUTE int mallopt(int cmd, int value) {
	return 0;
	}
	#endif
	} // extern "C"

compiler-rt/lib/scudo/scudo_new_delete.cpp

This file was deleted.

	//===-- scudo_new_delete.cpp ------------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Interceptors for operators new and delete.
	///
	//===----------------------------------------------------------------------===//

	#include "scudo_allocator.h"
	#include "scudo_errors.h"

	#include "interception/interception.h"

	#include <stddef.h>

	using namespace __scudo;

	#define CXX_OPERATOR_ATTRIBUTE INTERCEPTOR_ATTRIBUTE

	// Fake std::nothrow_t to avoid including <new>.
	namespace std {
	struct nothrow_t {};
	enum class align_val_t: size_t {};
	} // namespace std

	// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
	#define OPERATOR_NEW_BODY_ALIGN(Type, Align, NoThrow) \
	void *Ptr = scudoAllocate(size, static_cast<uptr>(Align), Type); \
	if (!NoThrow && UNLIKELY(!Ptr)) reportOutOfMemory(size); \
	return Ptr;
	#define OPERATOR_NEW_BODY(Type, NoThrow) \
	OPERATOR_NEW_BODY_ALIGN(Type, 0, NoThrow)

	CXX_OPERATOR_ATTRIBUTE
	void *operator new(size_t size)
	{ OPERATOR_NEW_BODY(FromNew, /NoThrow=/false); }
	CXX_OPERATOR_ATTRIBUTE
	void *operator new[](size_t size)
	{ OPERATOR_NEW_BODY(FromNewArray, /NoThrow=/false); }
	CXX_OPERATOR_ATTRIBUTE
	void *operator new(size_t size, std::nothrow_t const&)
	{ OPERATOR_NEW_BODY(FromNew, /NoThrow=/true); }
	CXX_OPERATOR_ATTRIBUTE
	void *operator new[](size_t size, std::nothrow_t const&)
	{ OPERATOR_NEW_BODY(FromNewArray, /NoThrow=/true); }
	CXX_OPERATOR_ATTRIBUTE
	void *operator new(size_t size, std::align_val_t align)
	{ OPERATOR_NEW_BODY_ALIGN(FromNew, align, /NoThrow=/false); }
	CXX_OPERATOR_ATTRIBUTE
	void *operator new[](size_t size, std::align_val_t align)
	{ OPERATOR_NEW_BODY_ALIGN(FromNewArray, align, /NoThrow=/false); }
	CXX_OPERATOR_ATTRIBUTE
	void *operator new(size_t size, std::align_val_t align, std::nothrow_t const&)
	{ OPERATOR_NEW_BODY_ALIGN(FromNew, align, /NoThrow=/true); }
	CXX_OPERATOR_ATTRIBUTE
	void *operator new[](size_t size, std::align_val_t align, std::nothrow_t const&)
	{ OPERATOR_NEW_BODY_ALIGN(FromNewArray, align, /NoThrow=/true); }

	#define OPERATOR_DELETE_BODY(Type) \
	scudoDeallocate(ptr, 0, 0, Type);
	#define OPERATOR_DELETE_BODY_SIZE(Type) \
	scudoDeallocate(ptr, size, 0, Type);
	#define OPERATOR_DELETE_BODY_ALIGN(Type) \
	scudoDeallocate(ptr, 0, static_cast<uptr>(align), Type);
	#define OPERATOR_DELETE_BODY_SIZE_ALIGN(Type) \
	scudoDeallocate(ptr, size, static_cast<uptr>(align), Type);

	CXX_OPERATOR_ATTRIBUTE
	void operator delete(void *ptr) NOEXCEPT
	{ OPERATOR_DELETE_BODY(FromNew); }
	CXX_OPERATOR_ATTRIBUTE
	void operator delete[](void *ptr) NOEXCEPT
	{ OPERATOR_DELETE_BODY(FromNewArray); }
	CXX_OPERATOR_ATTRIBUTE
	void operator delete(void *ptr, std::nothrow_t const&)
	{ OPERATOR_DELETE_BODY(FromNew); }
	CXX_OPERATOR_ATTRIBUTE
	void operator delete[](void *ptr, std::nothrow_t const&)
	{ OPERATOR_DELETE_BODY(FromNewArray); }
	CXX_OPERATOR_ATTRIBUTE
	void operator delete(void *ptr, size_t size) NOEXCEPT
	{ OPERATOR_DELETE_BODY_SIZE(FromNew); }
	CXX_OPERATOR_ATTRIBUTE
	void operator delete[](void *ptr, size_t size) NOEXCEPT
	{ OPERATOR_DELETE_BODY_SIZE(FromNewArray); }
	CXX_OPERATOR_ATTRIBUTE
	void operator delete(void *ptr, std::align_val_t align) NOEXCEPT
	{ OPERATOR_DELETE_BODY_ALIGN(FromNew); }
	CXX_OPERATOR_ATTRIBUTE
	void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT
	{ OPERATOR_DELETE_BODY_ALIGN(FromNewArray); }
	CXX_OPERATOR_ATTRIBUTE
	void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&)
	{ OPERATOR_DELETE_BODY_ALIGN(FromNew); }
	CXX_OPERATOR_ATTRIBUTE
	void operator delete[](void *ptr, std::align_val_t align, std::nothrow_t const&)
	{ OPERATOR_DELETE_BODY_ALIGN(FromNewArray); }
	CXX_OPERATOR_ATTRIBUTE
	void operator delete(void *ptr, size_t size, std::align_val_t align) NOEXCEPT
	{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FromNew); }
	CXX_OPERATOR_ATTRIBUTE
	void operator delete[](void *ptr, size_t size, std::align_val_t align) NOEXCEPT
	{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FromNewArray); }

compiler-rt/lib/scudo/scudo_platform.h

This file was deleted.

	//===-- scudo_platform.h ----------------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Scudo platform specific definitions.
	/// TODO(kostyak): add tests for the compile time defines.
	///
	//===----------------------------------------------------------------------===//

	#ifndef SCUDO_PLATFORM_H_
	#define SCUDO_PLATFORM_H_

	#include "sanitizer_common/sanitizer_allocator.h"

	#if !SANITIZER_LINUX && !SANITIZER_FUCHSIA
	# error "The Scudo hardened allocator is not supported on this platform."
	#endif

	#define SCUDO_TSD_EXCLUSIVE_SUPPORTED (!SANITIZER_ANDROID && !SANITIZER_FUCHSIA)

	#ifndef SCUDO_TSD_EXCLUSIVE
	// SCUDO_TSD_EXCLUSIVE wasn't defined, use a default TSD model for the platform.
	# if SANITIZER_ANDROID \|\| SANITIZER_FUCHSIA
	// Android and Fuchsia use a pool of TSDs shared between threads.
	# define SCUDO_TSD_EXCLUSIVE 0
	# elif SANITIZER_LINUX && !SANITIZER_ANDROID
	// Non-Android Linux use an exclusive TSD per thread.
	# define SCUDO_TSD_EXCLUSIVE 1
	# else
	# error "No default TSD model defined for this platform."
	# endif // SANITIZER_ANDROID \|\| SANITIZER_FUCHSIA
	#endif // SCUDO_TSD_EXCLUSIVE

	// If the exclusive TSD model is chosen, make sure the platform supports it.
	#if SCUDO_TSD_EXCLUSIVE && !SCUDO_TSD_EXCLUSIVE_SUPPORTED
	# error "The exclusive TSD model is not supported on this platform."
	#endif

	// Maximum number of TSDs that can be created for the Shared model.
	#ifndef SCUDO_SHARED_TSD_POOL_SIZE
	# if SANITIZER_ANDROID
	# define SCUDO_SHARED_TSD_POOL_SIZE 2U
	# else
	# define SCUDO_SHARED_TSD_POOL_SIZE 32U
	# endif // SANITIZER_ANDROID
	#endif // SCUDO_SHARED_TSD_POOL_SIZE

	// The following allows the public interface functions to be disabled.
	#ifndef SCUDO_CAN_USE_PUBLIC_INTERFACE
	# define SCUDO_CAN_USE_PUBLIC_INTERFACE 1
	#endif

	// Hooks in the allocation & deallocation paths can become a security concern if
	// implemented improperly, or if overwritten by an attacker. Use with caution.
	#ifndef SCUDO_CAN_USE_HOOKS
	# if SANITIZER_FUCHSIA
	# define SCUDO_CAN_USE_HOOKS 1
	# else
	# define SCUDO_CAN_USE_HOOKS 0
	# endif // SANITIZER_FUCHSIA
	#endif // SCUDO_CAN_USE_HOOKS

	namespace __scudo {

	#if SANITIZER_CAN_USE_ALLOCATOR64
	# if defined(__aarch64__) && SANITIZER_ANDROID
	const uptr AllocatorSize = 0x4000000000ULL; // 256G.
	# elif defined(__aarch64__)
	const uptr AllocatorSize = 0x10000000000ULL; // 1T.
	# else
	const uptr AllocatorSize = 0x40000000000ULL; // 4T.
	# endif
	#else
	const uptr RegionSizeLog = SANITIZER_ANDROID ? 19 : 20;
	#endif // SANITIZER_CAN_USE_ALLOCATOR64

	#if !defined(SCUDO_SIZE_CLASS_MAP)
	# define SCUDO_SIZE_CLASS_MAP Dense
	#endif

	#define SIZE_CLASS_MAP_TYPE SIZE_CLASS_MAP_TYPE_(SCUDO_SIZE_CLASS_MAP)
	#define SIZE_CLASS_MAP_TYPE_(T) SIZE_CLASS_MAP_TYPE__(T)
	#define SIZE_CLASS_MAP_TYPE__(T) T##SizeClassMap

	typedef SIZE_CLASS_MAP_TYPE SizeClassMap;

	} // namespace __scudo

	#endif // SCUDO_PLATFORM_H_

compiler-rt/lib/scudo/scudo_termination.cpp

This file was deleted.

	//===-- scudo_termination.cpp ------------------------------------ C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// This file contains bare-bones termination functions to replace the
	/// __sanitizer ones, in order to avoid any potential abuse of the callbacks
	/// functionality.
	///
	//===----------------------------------------------------------------------===//

	#include "scudo_utils.h"

	#include "sanitizer_common/sanitizer_common.h"

	namespace __sanitizer {

	bool AddDieCallback(DieCallbackType Callback) { return true; }

	bool RemoveDieCallback(DieCallbackType Callback) { return true; }

	void SetUserDieCallback(DieCallbackType Callback) {}

	void NORETURN Die() {
	if (common_flags()->abort_on_error)
	Abort();
	internal__exit(common_flags()->exitcode);
	}

	void SetCheckUnwindCallback(void (*callback)()) {}

	void NORETURN CheckFailed(const char File, int Line, const char Condition,
	u64 Value1, u64 Value2) {
	__scudo::dieWithMessage("CHECK failed at %s:%d %s (%lld, %lld)\n",
	File, Line, Condition, Value1, Value2);
	}

	} // namespace __sanitizer

compiler-rt/lib/scudo/scudo_tsd.h

This file was deleted.

	//===-- scudo_tsd.h ---------------------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Scudo thread specific data definition.
	/// Implementation will differ based on the thread local storage primitives
	/// offered by the underlying platform.
	///
	//===----------------------------------------------------------------------===//

	#ifndef SCUDO_TSD_H_
	#define SCUDO_TSD_H_

	#include "scudo_allocator.h"
	#include "scudo_utils.h"

	#include <pthread.h>

	namespace __scudo {

	struct ALIGNED(SANITIZER_CACHE_LINE_SIZE) ScudoTSD {
	AllocatorCacheT Cache;
	uptr QuarantineCachePlaceHolder[4];

	void init();
	void commitBack();

	inline bool tryLock() SANITIZER_TRY_ACQUIRE(true, Mutex) {
	if (Mutex.TryLock()) {
	atomic_store_relaxed(&Precedence, 0);
	return true;
	}
	if (atomic_load_relaxed(&Precedence) == 0)
	atomic_store_relaxed(&Precedence, static_cast<uptr>(
	MonotonicNanoTime() >> FIRST_32_SECOND_64(16, 0)));
	return false;
	}

	inline void lock() SANITIZER_ACQUIRE(Mutex) {
	atomic_store_relaxed(&Precedence, 0);
	Mutex.Lock();
	}

	inline void unlock() SANITIZER_RELEASE(Mutex) { Mutex.Unlock(); }

	inline uptr getPrecedence() { return atomic_load_relaxed(&Precedence); }

	private:
	StaticSpinMutex Mutex;
	atomic_uintptr_t Precedence;
	};

	void initThread(bool MinimalInit);

	// TSD model specific fastpath functions definitions.
	#include "scudo_tsd_exclusive.inc"
	#include "scudo_tsd_shared.inc"

	} // namespace __scudo

	#endif // SCUDO_TSD_H_

compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp

This file was deleted.

	//===-- scudo_tsd_exclusive.cpp ---------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Scudo exclusive TSD implementation.
	///
	//===----------------------------------------------------------------------===//

	#include "scudo_tsd.h"

	#if SCUDO_TSD_EXCLUSIVE

	namespace __scudo {

	static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT;
	static pthread_key_t PThreadKey;

	__attribute__((tls_model("initial-exec")))
	THREADLOCAL ThreadState ScudoThreadState = ThreadNotInitialized;
	__attribute__((tls_model("initial-exec")))
	THREADLOCAL ScudoTSD TSD;

	// Fallback TSD for when the thread isn't initialized yet or is torn down. It
	// can be shared between multiple threads and as such must be locked.
	ScudoTSD FallbackTSD;

	static void teardownThread(void *Ptr) {
	uptr I = reinterpret_cast<uptr>(Ptr);
	// The glibc POSIX thread-local-storage deallocation routine calls user
	// provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS.
	// We want to be called last since other destructors might call free and the
	// like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the
	// quarantine and swallowing the cache.
	if (I > 1) {
	// If pthread_setspecific fails, we will go ahead with the teardown.
	if (LIKELY(pthread_setspecific(PThreadKey,
	reinterpret_cast<void *>(I - 1)) == 0))
	return;
	}
	TSD.commitBack();
	ScudoThreadState = ThreadTornDown;
	}


	static void initOnce() {
	CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread), 0);
	initScudo();
	FallbackTSD.init();
	}

	void initThread(bool MinimalInit) {
	CHECK_EQ(pthread_once(&GlobalInitialized, initOnce), 0);
	if (UNLIKELY(MinimalInit))
	return;
	CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast<void *>(
	GetPthreadDestructorIterations())), 0);
	TSD.init();
	ScudoThreadState = ThreadInitialized;
	}

	} // namespace __scudo

	#endif // SCUDO_TSD_EXCLUSIVE

compiler-rt/lib/scudo/scudo_tsd_exclusive.inc

This file was deleted.

	//===-- scudo_tsd_exclusive.inc ---------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Scudo exclusive TSD fastpath functions implementation.
	///
	//===----------------------------------------------------------------------===//

	#ifndef SCUDO_TSD_H_
	#error "This file must be included inside scudo_tsd.h."
	#endif // SCUDO_TSD_H_

	#if SCUDO_TSD_EXCLUSIVE

	enum ThreadState : u8 {
	ThreadNotInitialized = 0,
	ThreadInitialized,
	ThreadTornDown,
	};
	__attribute__((
	tls_model("initial-exec"))) extern THREADLOCAL ThreadState ScudoThreadState;
	__attribute__((tls_model("initial-exec"))) extern THREADLOCAL ScudoTSD TSD;

	extern ScudoTSD FallbackTSD;

	ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
	if (LIKELY(ScudoThreadState != ThreadNotInitialized))
	return;
	initThread(MinimalInit);
	}

	ALWAYS_INLINE ScudoTSD *
	getTSDAndLock(bool *UnlockRequired) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	if (UNLIKELY(ScudoThreadState != ThreadInitialized)) {
	FallbackTSD.lock();
	*UnlockRequired = true;
	return &FallbackTSD;
	}
	*UnlockRequired = false;
	return &TSD;
	}

	#endif // SCUDO_TSD_EXCLUSIVE

compiler-rt/lib/scudo/scudo_tsd_shared.cpp

This file was deleted.

	//===-- scudo_tsd_shared.cpp ------------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Scudo shared TSD implementation.
	///
	//===----------------------------------------------------------------------===//

	#include "scudo_tsd.h"

	#if !SCUDO_TSD_EXCLUSIVE

	namespace __scudo {

	static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT;
	pthread_key_t PThreadKey;

	static atomic_uint32_t CurrentIndex;
	static ScudoTSD *TSDs;
	static u32 NumberOfTSDs;
	static u32 CoPrimes[SCUDO_SHARED_TSD_POOL_SIZE];
	static u32 NumberOfCoPrimes = 0;

	#if SANITIZER_LINUX && !SANITIZER_ANDROID
	__attribute__((tls_model("initial-exec")))
	THREADLOCAL ScudoTSD *CurrentTSD;
	#endif

	static void initOnce() {
	CHECK_EQ(pthread_key_create(&PThreadKey, NULL), 0);
	initScudo();
	NumberOfTSDs = Min(Max(1U, GetNumberOfCPUsCached()),
	static_cast<u32>(SCUDO_SHARED_TSD_POOL_SIZE));
	TSDs = reinterpret_cast<ScudoTSD *>(
	MmapOrDie(sizeof(ScudoTSD) * NumberOfTSDs, "ScudoTSDs"));
	for (u32 I = 0; I < NumberOfTSDs; I++) {
	TSDs[I].init();
	u32 A = I + 1;
	u32 B = NumberOfTSDs;
	while (B != 0) { const u32 T = A; A = B; B = T % B; }
	if (A == 1)
	CoPrimes[NumberOfCoPrimes++] = I + 1;
	}
	}

	ALWAYS_INLINE void setCurrentTSD(ScudoTSD *TSD) {
	#if SANITIZER_ANDROID
	*get_android_tls_ptr() = reinterpret_cast<uptr>(TSD);
	#elif SANITIZER_LINUX
	CurrentTSD = TSD;
	#else
	CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast<void *>(TSD)), 0);
	#endif // SANITIZER_ANDROID
	}

	void initThread(bool MinimalInit) {
	pthread_once(&GlobalInitialized, initOnce);
	// Initial context assignment is done in a plain round-robin fashion.
	u32 Index = atomic_fetch_add(&CurrentIndex, 1, memory_order_relaxed);
	setCurrentTSD(&TSDs[Index % NumberOfTSDs]);
	}

	ScudoTSD getTSDAndLockSlow(ScudoTSD TSD) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	if (NumberOfTSDs > 1) {
	// Use the Precedence of the current TSD as our random seed. Since we are in
	// the slow path, it means that tryLock failed, and as a result it's very
	// likely that said Precedence is non-zero.
	u32 RandState = static_cast<u32>(TSD->getPrecedence());
	const u32 R = Rand(&RandState);
	const u32 Inc = CoPrimes[R % NumberOfCoPrimes];
	u32 Index = R % NumberOfTSDs;
	uptr LowestPrecedence = UINTPTR_MAX;
	ScudoTSD *CandidateTSD = nullptr;
	// Go randomly through at most 4 contexts and find a candidate.
	for (u32 I = 0; I < Min(4U, NumberOfTSDs); I++) {
	if (TSDs[Index].tryLock()) {
	setCurrentTSD(&TSDs[Index]);
	return &TSDs[Index];
	}
	const uptr Precedence = TSDs[Index].getPrecedence();
	// A 0 precedence here means another thread just locked this TSD.
	if (Precedence && Precedence < LowestPrecedence) {
	CandidateTSD = &TSDs[Index];
	LowestPrecedence = Precedence;
	}
	Index += Inc;
	if (Index >= NumberOfTSDs)
	Index -= NumberOfTSDs;
	}
	if (CandidateTSD) {
	CandidateTSD->lock();
	setCurrentTSD(CandidateTSD);
	return CandidateTSD;
	}
	}
	// Last resort, stick with the current one.
	TSD->lock();
	return TSD;
	}

	} // namespace __scudo

	#endif // !SCUDO_TSD_EXCLUSIVE

compiler-rt/lib/scudo/scudo_tsd_shared.inc

This file was deleted.

	//===-- scudo_tsd_shared.inc ------------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Scudo shared TSD fastpath functions implementation.
	///
	//===----------------------------------------------------------------------===//

	#ifndef SCUDO_TSD_H_
	# error "This file must be included inside scudo_tsd.h."
	#endif // SCUDO_TSD_H_

	#if !SCUDO_TSD_EXCLUSIVE

	extern pthread_key_t PThreadKey;

	#if SANITIZER_LINUX && !SANITIZER_ANDROID
	__attribute__((tls_model("initial-exec")))
	extern THREADLOCAL ScudoTSD *CurrentTSD;
	#endif

	ALWAYS_INLINE ScudoTSD* getCurrentTSD() {
	#if SANITIZER_ANDROID
	return reinterpret_cast<ScudoTSD >(get_android_tls_ptr());
	#elif SANITIZER_LINUX
	return CurrentTSD;
	#else
	return reinterpret_cast<ScudoTSD *>(pthread_getspecific(PThreadKey));
	#endif // SANITIZER_ANDROID
	}

	ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
	if (LIKELY(getCurrentTSD()))
	return;
	initThread(MinimalInit);
	}

	ScudoTSD getTSDAndLockSlow(ScudoTSD TSD);

	ALWAYS_INLINE ScudoTSD *
	getTSDAndLock(bool *UnlockRequired) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	ScudoTSD *TSD = getCurrentTSD();
	DCHECK(TSD && "No TSD associated with the current thread!");
	*UnlockRequired = true;
	// Try to lock the currently associated context.
	if (TSD->tryLock())
	return TSD;
	// If it failed, go the slow path.
	return getTSDAndLockSlow(TSD);
	}

	#endif // !SCUDO_TSD_EXCLUSIVE

compiler-rt/lib/scudo/scudo_utils.h

This file was deleted.

	//===-- scudo_utils.h -------------------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Header for scudo_utils.cpp.
	///
	//===----------------------------------------------------------------------===//

	#ifndef SCUDO_UTILS_H_
	#define SCUDO_UTILS_H_

	#include "sanitizer_common/sanitizer_common.h"

	#include <string.h>

	namespace __scudo {

	template <class Dest, class Source>
	inline Dest bit_cast(const Source& source) {
	static_assert(sizeof(Dest) == sizeof(Source), "Sizes are not equal!");
	Dest dest;
	memcpy(&dest, &source, sizeof(dest));
	return dest;
	}

	void dieWithMessage(const char *Format, ...) NORETURN FORMAT(1, 2);

	bool hasHardwareCRC32();

	} // namespace __scudo

	#endif // SCUDO_UTILS_H_

compiler-rt/lib/scudo/scudo_utils.cpp

This file was deleted.

	//===-- scudo_utils.cpp ------------------------------------------ C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	///
	/// Platform specific utility functions.
	///
	//===----------------------------------------------------------------------===//

	#include "scudo_utils.h"

	#if defined(__x86_64__) \|\| defined(__i386__)
	# include <cpuid.h>
	#elif defined(__arm__) \|\| defined(__aarch64__)
	# include "sanitizer_common/sanitizer_getauxval.h"
	# if SANITIZER_FUCHSIA
	# include <zircon/syscalls.h>
	# include <zircon/features.h>
	# elif SANITIZER_POSIX
	# include "sanitizer_common/sanitizer_posix.h"
	# include <fcntl.h>
	# endif
	#endif

	#include <stdarg.h>

	// TODO(kostyak): remove __sanitizer *Printf uses in favor for our own less
	// complicated string formatting code. The following is a
	// temporary workaround to be able to use __sanitizer::VSNPrintf.
	namespace __sanitizer {

	extern int VSNPrintf(char buff, int buff_length, const char format,
	va_list args);

	} // namespace __sanitizer

	namespace __scudo {

	void dieWithMessage(const char *Format, ...) {
	static const char ScudoError[] = "Scudo ERROR: ";
	static constexpr uptr PrefixSize = sizeof(ScudoError) - 1;
	// Our messages are tiny, 256 characters is more than enough.
	char Message[256];
	va_list Args;
	va_start(Args, Format);
	internal_memcpy(Message, ScudoError, PrefixSize);
	VSNPrintf(Message + PrefixSize, sizeof(Message) - PrefixSize, Format, Args);
	va_end(Args);
	LogMessageOnPrintf(Message);
	if (common_flags()->abort_on_error)
	SetAbortMessage(Message);
	RawWrite(Message);
	Die();
	}

	#if defined(__x86_64__) \|\| defined(__i386__)
	// i386 and x86_64 specific code to detect CRC32 hardware support via CPUID.
	// CRC32 requires the SSE 4.2 instruction set.
	# ifndef bit_SSE4_2
	# define bit_SSE4_2 bit_SSE42 // clang and gcc have different defines.
	# endif

	#ifndef signature_HYGON_ebx // They are not defined in gcc.
	// HYGON: "HygonGenuine".
	#define signature_HYGON_ebx 0x6f677948
	#define signature_HYGON_edx 0x6e65476e
	#define signature_HYGON_ecx 0x656e6975
	#endif

	bool hasHardwareCRC32() {
	u32 Eax, Ebx, Ecx, Edx;
	__get_cpuid(0, &Eax, &Ebx, &Ecx, &Edx);
	const bool IsIntel = (Ebx == signature_INTEL_ebx) &&
	(Edx == signature_INTEL_edx) &&
	(Ecx == signature_INTEL_ecx);
	const bool IsAMD = (Ebx == signature_AMD_ebx) &&
	(Edx == signature_AMD_edx) &&
	(Ecx == signature_AMD_ecx);
	const bool IsHygon = (Ebx == signature_HYGON_ebx) &&
	(Edx == signature_HYGON_edx) &&
	(Ecx == signature_HYGON_ecx);
	if (!IsIntel && !IsAMD && !IsHygon)
	return false;
	__get_cpuid(1, &Eax, &Ebx, &Ecx, &Edx);
	return !!(Ecx & bit_SSE4_2);
	}
	#elif defined(__arm__) \|\| defined(__aarch64__)
	// For ARM and AArch64, hardware CRC32 support is indicated in the AT_HWCAP
	// auxiliary vector.
	# ifndef AT_HWCAP
	# define AT_HWCAP 16
	# endif
	# ifndef HWCAP_CRC32
	# define HWCAP_CRC32 (1 << 7) // HWCAP_CRC32 is missing on older platforms.
	# endif
	# if SANITIZER_POSIX
	bool hasHardwareCRC32ARMPosix() {
	uptr F = internal_open("/proc/self/auxv", O_RDONLY);
	if (internal_iserror(F))
	return false;
	struct { uptr Tag; uptr Value; } Entry = { 0, 0 };
	for (;;) {
	uptr N = internal_read(F, &Entry, sizeof(Entry));
	if (internal_iserror(N) \|\| N != sizeof(Entry) \|\|
	(Entry.Tag == 0 && Entry.Value == 0) \|\| Entry.Tag == AT_HWCAP)
	break;
	}
	internal_close(F);
	return (Entry.Tag == AT_HWCAP && (Entry.Value & HWCAP_CRC32) != 0);
	}
	# else
	bool hasHardwareCRC32ARMPosix() { return false; }
	# endif // SANITIZER_POSIX

	// Bionic doesn't initialize its globals early enough. This causes issues when
	// trying to access them from a preinit_array (b/25751302) or from another
	// constructor called before the libc one (b/68046352). __progname is
	// initialized after the other globals, so we can check its value to know if
	// calling getauxval is safe.
	extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname;
	inline bool areBionicGlobalsInitialized() {
	return !SANITIZER_ANDROID \|\| (&__progname && __progname);
	}

	bool hasHardwareCRC32() {
	#if SANITIZER_FUCHSIA
	u32 HWCap;
	zx_status_t Status = zx_system_get_features(ZX_FEATURE_KIND_CPU, &HWCap);
	if (Status != ZX_OK \|\| (HWCap & ZX_ARM64_FEATURE_ISA_CRC32) == 0)
	return false;
	return true;
	#else
	if (&getauxval && areBionicGlobalsInitialized())
	return !!(getauxval(AT_HWCAP) & HWCAP_CRC32);
	return hasHardwareCRC32ARMPosix();
	#endif // SANITIZER_FUCHSIA
	}
	#else
	bool hasHardwareCRC32() { return false; }
	#endif // defined(__x86_64__) \|\| defined(__i386__)

	} // namespace __scudo

This is an archive of the discontinued LLVM Phabricator instance.

Delete sanitizer_common-based ('old') scudo: o7ClosedPublic

Details

Diff Detail

Event Timeline

Revision Contents

Diff 482272

compiler-rt/lib/scudo/CMakeLists.txt

compiler-rt/lib/scudo/scudo_allocator.h

compiler-rt/lib/scudo/scudo_allocator.cpp

compiler-rt/lib/scudo/scudo_allocator_combined.h

compiler-rt/lib/scudo/scudo_allocator_secondary.h

compiler-rt/lib/scudo/scudo_crc32.h

compiler-rt/lib/scudo/scudo_crc32.cpp

compiler-rt/lib/scudo/scudo_errors.h

compiler-rt/lib/scudo/scudo_errors.cpp

compiler-rt/lib/scudo/scudo_flags.h

compiler-rt/lib/scudo/scudo_flags.cpp

compiler-rt/lib/scudo/scudo_flags.inc

compiler-rt/lib/scudo/scudo_interface_internal.h

compiler-rt/lib/scudo/scudo_malloc.cpp

compiler-rt/lib/scudo/scudo_new_delete.cpp

compiler-rt/lib/scudo/scudo_platform.h

compiler-rt/lib/scudo/scudo_termination.cpp

compiler-rt/lib/scudo/scudo_tsd.h

compiler-rt/lib/scudo/scudo_tsd_exclusive.cpp

compiler-rt/lib/scudo/scudo_tsd_exclusive.inc

compiler-rt/lib/scudo/scudo_tsd_shared.cpp

compiler-rt/lib/scudo/scudo_tsd_shared.inc

compiler-rt/lib/scudo/scudo_utils.h

compiler-rt/lib/scudo/scudo_utils.cpp

Delete sanitizer_common-based ('old') scudo: o7
ClosedPublic