Index: lib/scudo/CMakeLists.txt =================================================================== --- lib/scudo/CMakeLists.txt +++ lib/scudo/CMakeLists.txt @@ -14,8 +14,8 @@ scudo_interceptors.cpp scudo_new_delete.cpp scudo_termination.cpp - scudo_tls_android.cpp - scudo_tls_linux.cpp + scudo_tsd_exclusive.cpp + scudo_tsd_shared.cpp scudo_utils.cpp) # Enable the SSE 4.2 instruction set for scudo_crc32.cpp, if available. Index: lib/scudo/scudo_allocator.h =================================================================== --- lib/scudo/scudo_allocator.h +++ lib/scudo/scudo_allocator.h @@ -14,11 +14,7 @@ #ifndef SCUDO_ALLOCATOR_H_ #define SCUDO_ALLOCATOR_H_ -#include "sanitizer_common/sanitizer_allocator.h" - -#if !SANITIZER_LINUX -# error "The Scudo hardened allocator is currently only supported on Linux." -#endif +#include "scudo_platform.h" namespace __scudo { @@ -70,14 +66,6 @@ #if SANITIZER_CAN_USE_ALLOCATOR64 const uptr AllocatorSpace = ~0ULL; -# if defined(__aarch64__) && SANITIZER_ANDROID -const uptr AllocatorSize = 0x4000000000ULL; // 256G. -# elif defined(__aarch64__) -const uptr AllocatorSize = 0x10000000000ULL; // 1T. -# else -const uptr AllocatorSize = 0x40000000000ULL; // 4T. -# endif -typedef DefaultSizeClassMap SizeClassMap; struct AP64 { static const uptr kSpaceBeg = AllocatorSpace; static const uptr kSpaceSize = AllocatorSize; @@ -92,14 +80,12 @@ // Currently, the 32-bit Sanitizer allocator has not yet benefited from all the // security improvements brought to the 64-bit one. This makes the 32-bit // version of Scudo slightly less toughened. -static const uptr RegionSizeLog = 20; static const uptr NumRegions = SANITIZER_MMAP_RANGE_SIZE >> RegionSizeLog; # if SANITIZER_WORDSIZE == 32 typedef FlatByteMap ByteMap; # elif SANITIZER_WORDSIZE == 64 typedef TwoLevelByteMap<(NumRegions >> 12), 1 << 12> ByteMap; # endif // SANITIZER_WORDSIZE -typedef DefaultSizeClassMap SizeClassMap; struct AP32 { static const uptr kSpaceBeg = 0; static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; Index: lib/scudo/scudo_allocator.cpp =================================================================== --- lib/scudo/scudo_allocator.cpp +++ lib/scudo/scudo_allocator.cpp @@ -17,7 +17,7 @@ #include "scudo_allocator.h" #include "scudo_crc32.h" #include "scudo_flags.h" -#include "scudo_tls.h" +#include "scudo_tsd.h" #include "scudo_utils.h" #include "sanitizer_common/sanitizer_allocator_checks.h" Index: lib/scudo/scudo_platform.h =================================================================== --- /dev/null +++ lib/scudo/scudo_platform.h @@ -0,0 +1,58 @@ +//===-- scudo_platform.h ----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Scudo platform specific definitions. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_PLATFORM_H_ +#define SCUDO_PLATFORM_H_ + +#include "sanitizer_common/sanitizer_allocator.h" + +#if !SANITIZER_LINUX && !SANITIZER_FUCHSIA +# error "The Scudo hardened allocator is currently not supported this platform." +#endif + +// Android and Fuchsia use a pool of TSDs shared between threads. +#if SANITIZER_ANDROID || SANITIZER_FUCHSIA +# define SCUDO_TSD_EXCLUSIVE 0 +#endif // SANITIZER_FUCHSIA || SANITIZER_FUCHSIA + +// Non-Android Linux use an exclusive TSD per thread. +#if SANITIZER_LINUX && !SANITIZER_ANDROID +# define SCUDO_TSD_EXCLUSIVE 1 +#endif // SANITIZER_LINUX && !SANITIZER_ANDROID + +namespace __scudo { + +#if SANITIZER_CAN_USE_ALLOCATOR64 +# if defined(__aarch64__) && SANITIZER_ANDROID +const uptr AllocatorSize = 0x2000000000ULL; // 128G. +typedef VeryCompactSizeClassMap SizeClassMap; +# elif defined(__aarch64__) +const uptr AllocatorSize = 0x10000000000ULL; // 1T. +typedef CompactSizeClassMap SizeClassMap; +# else +const uptr AllocatorSize = 0x40000000000ULL; // 4T. +typedef CompactSizeClassMap SizeClassMap; +# endif +#else +# if SANITIZER_ANDROID +static const uptr RegionSizeLog = 19; +typedef VeryCompactSizeClassMap SizeClassMap; +# else +static const uptr RegionSizeLog = 20; +typedef CompactSizeClassMap SizeClassMap; +# endif +#endif // SANITIZER_CAN_USE_ALLOCATOR64 + +} // namespace __scudo + +#endif // SCUDO_PLATFORM_H_ Index: lib/scudo/scudo_tls.h =================================================================== --- lib/scudo/scudo_tls.h +++ lib/scudo/scudo_tls.h @@ -1,74 +0,0 @@ -//===-- scudo_tls.h ---------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Scudo thread local structure definition. -/// Implementation will differ based on the thread local storage primitives -/// offered by the underlying platform. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TLS_H_ -#define SCUDO_TLS_H_ - -#include "scudo_allocator.h" -#include "scudo_utils.h" - -#include "sanitizer_common/sanitizer_linux.h" -#include "sanitizer_common/sanitizer_platform.h" - -namespace __scudo { - -struct ALIGNED(64) ScudoTSD { - AllocatorCache Cache; - ScudoPrng Prng; - uptr QuarantineCachePlaceHolder[4]; - - void init(bool Shared); - void commitBack(); - - INLINE bool tryLock() { - if (Mutex.TryLock()) { - atomic_store_relaxed(&Precedence, 0); - return true; - } - if (atomic_load_relaxed(&Precedence) == 0) - atomic_store_relaxed(&Precedence, NanoTime()); - return false; - } - - INLINE void lock() { - Mutex.Lock(); - atomic_store_relaxed(&Precedence, 0); - } - - INLINE void unlock() { - if (!UnlockRequired) - return; - Mutex.Unlock(); - } - - INLINE u64 getPrecedence() { - return atomic_load_relaxed(&Precedence); - } - - private: - bool UnlockRequired; - StaticSpinMutex Mutex; - atomic_uint64_t Precedence; -}; - -void initThread(bool MinimalInit); - -// Platform specific fastpath functions definitions. -#include "scudo_tls_android.inc" -#include "scudo_tls_linux.inc" - -} // namespace __scudo - -#endif // SCUDO_TLS_H_ Index: lib/scudo/scudo_tls_android.cpp =================================================================== --- lib/scudo/scudo_tls_android.cpp +++ lib/scudo/scudo_tls_android.cpp @@ -1,98 +0,0 @@ -//===-- scudo_tls_android.cpp -----------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Scudo thread local structure implementation for Android. -/// -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_platform.h" - -#if SANITIZER_LINUX && SANITIZER_ANDROID - -#include "scudo_tls.h" - -#include - -namespace __scudo { - -static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT; -static pthread_key_t PThreadKey; - -static atomic_uint32_t CurrentIndex; -static ScudoTSD *TSDs; -static u32 NumberOfTSDs; - -// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used as they allocate memory. -static uptr getNumberOfCPUs() { - cpu_set_t CPUs; - CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0); - return CPU_COUNT(&CPUs); -} - -static void initOnce() { - // Hack: TLS_SLOT_TSAN was introduced in N. To be able to use it on M for - // testing, we create an unused key. Since the key_data array follows the tls - // array, it basically gives us the extra entry we need. - // TODO(kostyak): remove and restrict to N and above. - CHECK_EQ(pthread_key_create(&PThreadKey, NULL), 0); - initScudo(); - NumberOfTSDs = getNumberOfCPUs(); - if (NumberOfTSDs == 0) - NumberOfTSDs = 1; - if (NumberOfTSDs > 32) - NumberOfTSDs = 32; - TSDs = reinterpret_cast( - MmapOrDie(sizeof(ScudoTSD) * NumberOfTSDs, "ScudoTSDs")); - for (u32 i = 0; i < NumberOfTSDs; i++) - TSDs[i].init(/*Shared=*/true); -} - -void initThread(bool MinimalInit) { - pthread_once(&GlobalInitialized, initOnce); - // Initial context assignment is done in a plain round-robin fashion. - u32 Index = atomic_fetch_add(&CurrentIndex, 1, memory_order_relaxed); - ScudoTSD *TSD = &TSDs[Index % NumberOfTSDs]; - *get_android_tls_ptr() = reinterpret_cast(TSD); -} - -ScudoTSD *getTSDAndLockSlow() { - ScudoTSD *TSD; - if (NumberOfTSDs > 1) { - // Go through all the contexts and find the first unlocked one. - for (u32 i = 0; i < NumberOfTSDs; i++) { - TSD = &TSDs[i]; - if (TSD->tryLock()) { - *get_android_tls_ptr() = reinterpret_cast(TSD); - return TSD; - } - } - // No luck, find the one with the lowest Precedence, and slow lock it. - u64 LowestPrecedence = UINT64_MAX; - for (u32 i = 0; i < NumberOfTSDs; i++) { - u64 Precedence = TSDs[i].getPrecedence(); - if (Precedence && Precedence < LowestPrecedence) { - TSD = &TSDs[i]; - LowestPrecedence = Precedence; - } - } - if (LIKELY(LowestPrecedence != UINT64_MAX)) { - TSD->lock(); - *get_android_tls_ptr() = reinterpret_cast(TSD); - return TSD; - } - } - // Last resort, stick with the current one. - TSD = reinterpret_cast(*get_android_tls_ptr()); - TSD->lock(); - return TSD; -} - -} // namespace __scudo - -#endif // SANITIZER_LINUX && SANITIZER_ANDROID Index: lib/scudo/scudo_tls_android.inc =================================================================== --- lib/scudo/scudo_tls_android.inc +++ lib/scudo/scudo_tls_android.inc @@ -1,38 +0,0 @@ -//===-- scudo_tls_android.inc -----------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Scudo thread local structure fastpath functions implementation for Android. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TLS_H_ -# error "This file must be included inside scudo_tls.h." -#endif // SCUDO_TLS_H_ - -#if SANITIZER_LINUX && SANITIZER_ANDROID - -ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) { - if (LIKELY(*get_android_tls_ptr())) - return; - initThread(MinimalInit); -} - -ScudoTSD *getTSDAndLockSlow(); - -ALWAYS_INLINE ScudoTSD *getTSDAndLock() { - ScudoTSD *TSD = reinterpret_cast(*get_android_tls_ptr()); - CHECK(TSD && "No TSD associated with the current thread!"); - // Try to lock the currently associated context. - if (TSD->tryLock()) - return TSD; - // If it failed, go the slow path. - return getTSDAndLockSlow(); -} - -#endif // SANITIZER_LINUX && SANITIZER_ANDROID Index: lib/scudo/scudo_tls_linux.cpp =================================================================== --- lib/scudo/scudo_tls_linux.cpp +++ lib/scudo/scudo_tls_linux.cpp @@ -1,73 +0,0 @@ -//===-- scudo_tls_linux.cpp -------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Scudo thread local structure implementation for platforms supporting -/// thread_local. -/// -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_platform.h" - -#if SANITIZER_LINUX && !SANITIZER_ANDROID - -#include "scudo_tls.h" - -#include - -namespace __scudo { - -static pthread_once_t GlobalInitialized = PTHREAD_ONCE_INIT; -static pthread_key_t PThreadKey; - -__attribute__((tls_model("initial-exec"))) -THREADLOCAL ThreadState ScudoThreadState = ThreadNotInitialized; -__attribute__((tls_model("initial-exec"))) -THREADLOCAL ScudoTSD TSD; - -// Fallback TSD for when the thread isn't initialized yet or is torn down. It -// can be shared between multiple threads and as such must be locked. -ScudoTSD FallbackTSD; - -static void teardownThread(void *Ptr) { - uptr I = reinterpret_cast(Ptr); - // The glibc POSIX thread-local-storage deallocation routine calls user - // provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS. - // We want to be called last since other destructors might call free and the - // like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the - // quarantine and swallowing the cache. - if (I > 1) { - // If pthread_setspecific fails, we will go ahead with the teardown. - if (LIKELY(pthread_setspecific(PThreadKey, - reinterpret_cast(I - 1)) == 0)) - return; - } - TSD.commitBack(); - ScudoThreadState = ThreadTornDown; -} - - -static void initOnce() { - CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread), 0); - initScudo(); - FallbackTSD.init(/*Shared=*/true); -} - -void initThread(bool MinimalInit) { - CHECK_EQ(pthread_once(&GlobalInitialized, initOnce), 0); - if (UNLIKELY(MinimalInit)) - return; - CHECK_EQ(pthread_setspecific(PThreadKey, reinterpret_cast( - GetPthreadDestructorIterations())), 0); - TSD.init(/*Shared=*/false); - ScudoThreadState = ThreadInitialized; -} - -} // namespace __scudo - -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID Index: lib/scudo/scudo_tls_linux.inc =================================================================== --- lib/scudo/scudo_tls_linux.inc +++ lib/scudo/scudo_tls_linux.inc @@ -1,47 +0,0 @@ -//===-- scudo_tls_linux.inc -------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Scudo thread local structure fastpath functions implementation for platforms -/// supporting thread_local. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_TLS_H_ -# error "This file must be included inside scudo_tls.h." -#endif // SCUDO_TLS_H_ - -#if SANITIZER_LINUX && !SANITIZER_ANDROID - -enum ThreadState : u8 { - ThreadNotInitialized = 0, - ThreadInitialized, - ThreadTornDown, -}; -__attribute__((tls_model("initial-exec"))) -extern THREADLOCAL ThreadState ScudoThreadState; -__attribute__((tls_model("initial-exec"))) -extern THREADLOCAL ScudoTSD TSD; - -extern ScudoTSD FallbackTSD; - -ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) { - if (LIKELY(ScudoThreadState != ThreadNotInitialized)) - return; - initThread(MinimalInit); -} - -ALWAYS_INLINE ScudoTSD *getTSDAndLock() { - if (UNLIKELY(ScudoThreadState != ThreadInitialized)) { - FallbackTSD.lock(); - return &FallbackTSD; - } - return &TSD; -} - -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID Index: lib/scudo/scudo_tsd.h =================================================================== --- lib/scudo/scudo_tsd.h +++ lib/scudo/scudo_tsd.h @@ -1,4 +1,4 @@ -//===-- scudo_tls.h ---------------------------------------------*- C++ -*-===// +//===-- scudo_tsd.h ---------------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,21 +7,18 @@ // //===----------------------------------------------------------------------===// /// -/// Scudo thread local structure definition. +/// Scudo thread specific data definition. /// Implementation will differ based on the thread local storage primitives /// offered by the underlying platform. /// //===----------------------------------------------------------------------===// -#ifndef SCUDO_TLS_H_ -#define SCUDO_TLS_H_ +#ifndef SCUDO_TSD_H_ +#define SCUDO_TSD_H_ #include "scudo_allocator.h" #include "scudo_utils.h" -#include "sanitizer_common/sanitizer_linux.h" -#include "sanitizer_common/sanitizer_platform.h" - namespace __scudo { struct ALIGNED(64) ScudoTSD { @@ -66,9 +63,9 @@ void initThread(bool MinimalInit); // Platform specific fastpath functions definitions. -#include "scudo_tls_android.inc" -#include "scudo_tls_linux.inc" +#include "scudo_tsd_exclusive.inc" +#include "scudo_tsd_shared.inc" } // namespace __scudo -#endif // SCUDO_TLS_H_ +#endif // SCUDO_TSD_H_ Index: lib/scudo/scudo_tsd_exclusive.cpp =================================================================== --- lib/scudo/scudo_tsd_exclusive.cpp +++ lib/scudo/scudo_tsd_exclusive.cpp @@ -1,4 +1,4 @@ -//===-- scudo_tls_linux.cpp -------------------------------------*- C++ -*-===// +//===-- scudo_tsd_exclusive.cpp ---------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,16 +7,13 @@ // //===----------------------------------------------------------------------===// /// -/// Scudo thread local structure implementation for platforms supporting -/// thread_local. +/// Scudo exclusive TSD implementation. /// //===----------------------------------------------------------------------===// -#include "sanitizer_common/sanitizer_platform.h" +#include "scudo_tsd.h" -#if SANITIZER_LINUX && !SANITIZER_ANDROID - -#include "scudo_tls.h" +#if SCUDO_TSD_EXCLUSIVE #include @@ -70,4 +67,4 @@ } // namespace __scudo -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID +#endif // SCUDO_TSD_EXCLUSIVE Index: lib/scudo/scudo_tsd_exclusive.inc =================================================================== --- lib/scudo/scudo_tsd_exclusive.inc +++ lib/scudo/scudo_tsd_exclusive.inc @@ -1,4 +1,4 @@ -//===-- scudo_tls_linux.inc -------------------------------------*- C++ -*-===// +//===-- scudo_tsd_exclusive.inc ---------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,16 +7,15 @@ // //===----------------------------------------------------------------------===// /// -/// Scudo thread local structure fastpath functions implementation for platforms -/// supporting thread_local. +/// Scudo exclusive TSD fastpath functions implementation. /// //===----------------------------------------------------------------------===// -#ifndef SCUDO_TLS_H_ -# error "This file must be included inside scudo_tls.h." -#endif // SCUDO_TLS_H_ +#ifndef SCUDO_TSD_H_ +# error "This file must be included inside scudo_tsd.h." +#endif // SCUDO_TSD_H_ -#if SANITIZER_LINUX && !SANITIZER_ANDROID +#if SCUDO_TSD_EXCLUSIVE enum ThreadState : u8 { ThreadNotInitialized = 0, @@ -44,4 +43,4 @@ return &TSD; } -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID +#endif // SCUDO_TSD_EXCLUSIVE Index: lib/scudo/scudo_tsd_shared.cpp =================================================================== --- lib/scudo/scudo_tsd_shared.cpp +++ lib/scudo/scudo_tsd_shared.cpp @@ -1,4 +1,4 @@ -//===-- scudo_tls_android.cpp -----------------------------------*- C++ -*-===// +//===-- scudo_tsd_shared.cpp ------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,15 +7,13 @@ // //===----------------------------------------------------------------------===// /// -/// Scudo thread local structure implementation for Android. +/// Scudo shared TSD implementation. /// //===----------------------------------------------------------------------===// -#include "sanitizer_common/sanitizer_platform.h" +#include "scudo_tsd.h" -#if SANITIZER_LINUX && SANITIZER_ANDROID - -#include "scudo_tls.h" +#if !SCUDO_TSD_EXCLUSIVE #include @@ -95,4 +93,4 @@ } // namespace __scudo -#endif // SANITIZER_LINUX && SANITIZER_ANDROID +#endif // !SCUDO_TSD_EXCLUSIVE Index: lib/scudo/scudo_tsd_shared.inc =================================================================== --- lib/scudo/scudo_tsd_shared.inc +++ lib/scudo/scudo_tsd_shared.inc @@ -1,4 +1,4 @@ -//===-- scudo_tls_android.inc -----------------------------------*- C++ -*-===// +//===-- scudo_tsd_shared.inc ------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,15 +7,15 @@ // //===----------------------------------------------------------------------===// /// -/// Scudo thread local structure fastpath functions implementation for Android. +/// Scudo shared TSD fastpath functions implementation. /// //===----------------------------------------------------------------------===// -#ifndef SCUDO_TLS_H_ -# error "This file must be included inside scudo_tls.h." -#endif // SCUDO_TLS_H_ +#ifndef SCUDO_TSD_H_ +# error "This file must be included inside scudo_tsd.h." +#endif // SCUDO_TSD_H_ -#if SANITIZER_LINUX && SANITIZER_ANDROID +#if !SCUDO_TSD_EXCLUSIVE ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) { if (LIKELY(*get_android_tls_ptr())) @@ -35,4 +35,4 @@ return getTSDAndLockSlow(); } -#endif // SANITIZER_LINUX && SANITIZER_ANDROID +#endif // !SCUDO_TSD_EXCLUSIVE