diff --git a/compiler-rt/lib/gwp_asan/CMakeLists.txt b/compiler-rt/lib/gwp_asan/CMakeLists.txt --- a/compiler-rt/lib/gwp_asan/CMakeLists.txt +++ b/compiler-rt/lib/gwp_asan/CMakeLists.txt @@ -22,6 +22,7 @@ mutex.h options.h options.inc + platform_specific/guarded_pool_allocator_tls.h stack_trace_compressor.h utilities.h ) diff --git a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h --- a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h +++ b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h @@ -13,6 +13,7 @@ #include "gwp_asan/definitions.h" #include "gwp_asan/mutex.h" #include "gwp_asan/options.h" +#include "gwp_asan/platform_specific/guarded_pool_allocator_tls.h" #include "gwp_asan/stack_trace_compressor.h" #include @@ -77,12 +78,12 @@ // class must be valid when zero-initialised, and we wish to sample as // infrequently as possible when this is the case, hence we underflow to // UINT32_MAX. - if (GWP_ASAN_UNLIKELY(ThreadLocals.NextSampleCounter == 0)) - ThreadLocals.NextSampleCounter = + if (GWP_ASAN_UNLIKELY(getThreadLocals()->NextSampleCounter == 0)) + getThreadLocals()->NextSampleCounter = ((getRandomUnsigned32() % (AdjustedSampleRatePlusOne - 1)) + 1) & ThreadLocalPackedVariables::NextSampleCounterMask; - return GWP_ASAN_UNLIKELY(--ThreadLocals.NextSampleCounter == 0); + return GWP_ASAN_UNLIKELY(--getThreadLocals()->NextSampleCounter == 0); } // Returns whether the provided pointer is a current sampled allocation that @@ -206,37 +207,10 @@ // the sample rate. uint32_t AdjustedSampleRatePlusOne = 0; - // Pack the thread local variables into a struct to ensure that they're in - // the same cache line for performance reasons. These are the most touched - // variables in GWP-ASan. - struct alignas(8) ThreadLocalPackedVariables { - constexpr ThreadLocalPackedVariables() - : RandomState(0xff82eb50), NextSampleCounter(0), RecursiveGuard(false) { - } - // Initialised to a magic constant so that an uninitialised GWP-ASan won't - // regenerate its sample counter for as long as possible. The xorshift32() - // algorithm used below results in getRandomUnsigned32(0xff82eb50) == - // 0xfffffea4. - uint32_t RandomState; - // Thread-local decrementing counter that indicates that a given allocation - // should be sampled when it reaches zero. - uint32_t NextSampleCounter : 31; - // The mask is needed to silence conversion errors. - static const uint32_t NextSampleCounterMask = (1U << 31) - 1; - // Guard against recursivity. Unwinders often contain complex behaviour that - // may not be safe for the allocator (i.e. the unwinder calls dlopen(), - // which calls malloc()). When recursive behaviour is detected, we will - // automatically fall back to the supporting allocator to supply the - // allocation. - bool RecursiveGuard : 1; - }; - static_assert(sizeof(ThreadLocalPackedVariables) == sizeof(uint64_t), - "thread local data does not fit in a uint64_t"); - class ScopedRecursiveGuard { public: - ScopedRecursiveGuard() { ThreadLocals.RecursiveGuard = true; } - ~ScopedRecursiveGuard() { ThreadLocals.RecursiveGuard = false; } + ScopedRecursiveGuard() { getThreadLocals()->RecursiveGuard = true; } + ~ScopedRecursiveGuard() { getThreadLocals()->RecursiveGuard = false; } }; // Initialise the PRNG, platform-specific. @@ -245,8 +219,6 @@ // xorshift (32-bit output), extremely fast PRNG that uses arithmetic // operations only. Seeded using platform-specific mechanisms by initPRNG(). uint32_t getRandomUnsigned32(); - - static GWP_ASAN_TLS_INITIAL_EXEC ThreadLocalPackedVariables ThreadLocals; }; } // namespace gwp_asan diff --git a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp --- a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp +++ b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp @@ -37,6 +37,14 @@ // referenced by users outside this translation unit, in order to avoid // init-order-fiasco. GuardedPoolAllocator *SingletonPtr = nullptr; + +size_t roundUpTo(size_t Size, size_t Boundary) { + return (Size + Boundary - 1) & ~(Boundary - 1); +} + +uintptr_t getPageAddr(uintptr_t Ptr, uintptr_t PageSize) { + return Ptr & ~(PageSize - 1); +} } // anonymous namespace // Gets the singleton implementation of this class. Thread-compatible until @@ -45,10 +53,6 @@ return SingletonPtr; } -static size_t roundUpTo(size_t Size, size_t Boundary) { - return (Size + Boundary - 1) & ~(Boundary - 1); -} - void GuardedPoolAllocator::init(const options::Options &Opts) { // Note: We return from the constructor here if GWP-ASan is not available. // This will stop heap-allocation of class members, as well as mmap() of the @@ -99,7 +103,7 @@ AdjustedSampleRatePlusOne = 2; initPRNG(); - ThreadLocals.NextSampleCounter = + getThreadLocals()->NextSampleCounter = ((getRandomUnsigned32() % (AdjustedSampleRatePlusOne - 1)) + 1) & ThreadLocalPackedVariables::NextSampleCounterMask; @@ -146,22 +150,18 @@ } } -static uintptr_t getPageAddr(uintptr_t Ptr, uintptr_t PageSize) { - return Ptr & ~(PageSize - 1); -} - void *GuardedPoolAllocator::allocate(size_t Size) { // GuardedPagePoolEnd == 0 when GWP-ASan is disabled. If we are disabled, fall // back to the supporting allocator. if (State.GuardedPagePoolEnd == 0) { - ThreadLocals.NextSampleCounter = + getThreadLocals()->NextSampleCounter = (AdjustedSampleRatePlusOne - 1) & ThreadLocalPackedVariables::NextSampleCounterMask; return nullptr; } // Protect against recursivity. - if (ThreadLocals.RecursiveGuard) + if (getThreadLocals()->RecursiveGuard) return nullptr; ScopedRecursiveGuard SRG; @@ -212,7 +212,7 @@ } void GuardedPoolAllocator::stop() { - ThreadLocals.RecursiveGuard = true; + getThreadLocals()->RecursiveGuard = true; PoolMutex.tryLock(); } @@ -243,7 +243,7 @@ // Ensure that the unwinder is not called if the recursive flag is set, // otherwise non-reentrant unwinders may deadlock. - if (!ThreadLocals.RecursiveGuard) { + if (!getThreadLocals()->RecursiveGuard) { ScopedRecursiveGuard SRG; Meta->DeallocationTrace.RecordBacktrace(Backtrace); } @@ -290,15 +290,11 @@ } uint32_t GuardedPoolAllocator::getRandomUnsigned32() { - uint32_t RandomState = ThreadLocals.RandomState; + uint32_t RandomState = getThreadLocals()->RandomState; RandomState ^= RandomState << 13; RandomState ^= RandomState >> 17; RandomState ^= RandomState << 5; - ThreadLocals.RandomState = RandomState; + getThreadLocals()->RandomState = RandomState; return RandomState; } - -GWP_ASAN_TLS_INITIAL_EXEC -GuardedPoolAllocator::ThreadLocalPackedVariables - GuardedPoolAllocator::ThreadLocals; } // namespace gwp_asan diff --git a/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp b/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp --- a/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp +++ b/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp @@ -38,7 +38,7 @@ namespace gwp_asan { void GuardedPoolAllocator::initPRNG() { - ThreadLocals.RandomState = + getThreadLocals()->RandomState = static_cast(time(nullptr) + getThreadID()); } diff --git a/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_tls.h b/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_tls.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_tls.h @@ -0,0 +1,53 @@ +//===-- guarded_pool_allocator_tls.h ----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef GWP_ASAN_GUARDED_POOL_ALLOCATOR_TLS_H_ +#define GWP_ASAN_GUARDED_POOL_ALLOCATOR_TLS_H_ + +#include "gwp_asan/definitions.h" + +#include + +namespace gwp_asan { +// Pack the thread local variables into a struct to ensure that they're in +// the same cache line for performance reasons. These are the most touched +// variables in GWP-ASan. +struct ThreadLocalPackedVariables { + constexpr ThreadLocalPackedVariables() + : RandomState(0xacd979ce), NextSampleCounter(0), RecursiveGuard(false) {} + // Initialised to a magic constant so that an uninitialised GWP-ASan won't + // regenerate its sample counter for as long as possible. The xorshift32() + // algorithm used below results in getRandomUnsigned32(0xacd979ce) == + // 0xfffffffe. + uint32_t RandomState; + // Thread-local decrementing counter that indicates that a given allocation + // should be sampled when it reaches zero. + uint32_t NextSampleCounter : 31; + // The mask is needed to silence conversion errors. + static const uint32_t NextSampleCounterMask = (1U << 31) - 1; + // Guard against recursivity. Unwinders often contain complex behaviour that + // may not be safe for the allocator (i.e. the unwinder calls dlopen(), + // which calls malloc()). When recursive behaviour is detected, we will + // automatically fall back to the supporting allocator to supply the + // allocation. + bool RecursiveGuard : 1; +}; +static_assert(sizeof(ThreadLocalPackedVariables) == sizeof(uint64_t), + "thread local data does not fit in a uint64_t"); + +#ifdef GWP_ASAN_PLATFORM_TLS_HEADER +#include GWP_ASAN_PLATFORM_TLS_HEADER +#else +inline ThreadLocalPackedVariables *getThreadLocals() { + alignas(8) static GWP_ASAN_TLS_INITIAL_EXEC ThreadLocalPackedVariables Locals; + return &Locals; +} +#endif +} // namespace gwp_asan + +#endif // GWP_ASAN_GUARDED_POOL_ALLOCATOR_TLS_H_ diff --git a/compiler-rt/lib/gwp_asan/platform_specific/utilities_posix.cpp b/compiler-rt/lib/gwp_asan/platform_specific/utilities_posix.cpp --- a/compiler-rt/lib/gwp_asan/platform_specific/utilities_posix.cpp +++ b/compiler-rt/lib/gwp_asan/platform_specific/utilities_posix.cpp @@ -24,7 +24,7 @@ if (&android_set_abort_message != nullptr) android_set_abort_message(Message); abort(); -#else // __BIONIC__ +#else // __BIONIC__ fprintf(stderr, "%s", Message); __builtin_trap(); #endif // __BIONIC__ diff --git a/compiler-rt/lib/gwp_asan/utilities.cpp b/compiler-rt/lib/gwp_asan/utilities.cpp --- a/compiler-rt/lib/gwp_asan/utilities.cpp +++ b/compiler-rt/lib/gwp_asan/utilities.cpp @@ -37,7 +37,7 @@ #ifdef __BIONIC__ static constexpr AlignmentStrategy PlatformDefaultAlignment = AlignmentStrategy::BIONIC; -#else // __BIONIC__ +#else // __BIONIC__ static constexpr AlignmentStrategy PlatformDefaultAlignment = AlignmentStrategy::POWER_OF_TWO; #endif // __BIONIC__