diff --git a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h --- a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h +++ b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h @@ -15,6 +15,15 @@ #include "gwp_asan/options.h" #include "gwp_asan/stack_trace_compressor.h" +#if GWP_ASAN_HAS_PLATFORM_TLS_SLOT +// This is a platform-provided header that needs to be on the include path when +// GWP-ASan is compiled. It must declare a function with the prototype: +// uint64_t *getPlatformGwpAsanTlsSlot() +// that returns the address of a thread-local word of storage reserved for +// GWP-ASan, that should be initialized to 0xacd979ce in newly created threads. +#include "gwp_asan_platform_tls_slot.h" +#endif + #include #include @@ -77,12 +86,12 @@ // class must be valid when zero-initialised, and we wish to sample as // infrequently as possible when this is the case, hence we underflow to // UINT32_MAX. - if (GWP_ASAN_UNLIKELY(ThreadLocals.NextSampleCounter == 0)) - ThreadLocals.NextSampleCounter = + if (GWP_ASAN_UNLIKELY(getThreadLocals()->NextSampleCounter == 0)) + getThreadLocals()->NextSampleCounter = ((getRandomUnsigned32() % (AdjustedSampleRatePlusOne - 1)) + 1) & ThreadLocalPackedVariables::NextSampleCounterMask; - return GWP_ASAN_UNLIKELY(--ThreadLocals.NextSampleCounter == 0); + return GWP_ASAN_UNLIKELY(--getThreadLocals()->NextSampleCounter == 0); } // Returns whether the provided pointer is a current sampled allocation that @@ -211,12 +220,12 @@ // variables in GWP-ASan. struct alignas(8) ThreadLocalPackedVariables { constexpr ThreadLocalPackedVariables() - : RandomState(0xff82eb50), NextSampleCounter(0), RecursiveGuard(false) { + : RandomState(0xacd979ce), NextSampleCounter(0), RecursiveGuard(false) { } // Initialised to a magic constant so that an uninitialised GWP-ASan won't // regenerate its sample counter for as long as possible. The xorshift32() - // algorithm used below results in getRandomUnsigned32(0xff82eb50) == - // 0xfffffea4. + // algorithm used below results in getRandomUnsigned32(0xacd979ce) == + // 0xfffffffe. uint32_t RandomState; // Thread-local decrementing counter that indicates that a given allocation // should be sampled when it reaches zero. @@ -233,10 +242,28 @@ static_assert(sizeof(ThreadLocalPackedVariables) == sizeof(uint64_t), "thread local data does not fit in a uint64_t"); + ThreadLocalPackedVariables *getThreadLocals() { +#if GWP_ASAN_HAS_PLATFORM_TLS_SLOT + return reinterpret_cast( + getPlatformGwpAsanTlsSlot()); +#else + alignas( + 8) static GWP_ASAN_TLS_INITIAL_EXEC ThreadLocalPackedVariables Locals; + return &Locals; +#endif + } + class ScopedRecursiveGuard { public: - ScopedRecursiveGuard() { ThreadLocals.RecursiveGuard = true; } - ~ScopedRecursiveGuard() { ThreadLocals.RecursiveGuard = false; } + ScopedRecursiveGuard(GuardedPoolAllocator *GPA) : Instance(GPA) { + Instance->getThreadLocals()->RecursiveGuard = true; + } + ~ScopedRecursiveGuard() { + Instance->getThreadLocals()->RecursiveGuard = false; + } + + private: + GuardedPoolAllocator *Instance; }; // Initialise the PRNG, platform-specific. @@ -245,8 +272,6 @@ // xorshift (32-bit output), extremely fast PRNG that uses arithmetic // operations only. Seeded using platform-specific mechanisms by initPRNG(). uint32_t getRandomUnsigned32(); - - static GWP_ASAN_TLS_INITIAL_EXEC ThreadLocalPackedVariables ThreadLocals; }; } // namespace gwp_asan diff --git a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp --- a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp +++ b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp @@ -37,6 +37,14 @@ // referenced by users outside this translation unit, in order to avoid // init-order-fiasco. GuardedPoolAllocator *SingletonPtr = nullptr; + +size_t roundUpTo(size_t Size, size_t Boundary) { + return (Size + Boundary - 1) & ~(Boundary - 1); +} + +uintptr_t getPageAddr(uintptr_t Ptr, uintptr_t PageSize) { + return Ptr & ~(PageSize - 1); +} } // anonymous namespace // Gets the singleton implementation of this class. Thread-compatible until @@ -45,10 +53,6 @@ return SingletonPtr; } -static size_t roundUpTo(size_t Size, size_t Boundary) { - return (Size + Boundary - 1) & ~(Boundary - 1); -} - void GuardedPoolAllocator::init(const options::Options &Opts) { // Note: We return from the constructor here if GWP-ASan is not available. // This will stop heap-allocation of class members, as well as mmap() of the @@ -99,7 +103,7 @@ AdjustedSampleRatePlusOne = 2; initPRNG(); - ThreadLocals.NextSampleCounter = + getThreadLocals()->NextSampleCounter = ((getRandomUnsigned32() % (AdjustedSampleRatePlusOne - 1)) + 1) & ThreadLocalPackedVariables::NextSampleCounterMask; @@ -146,24 +150,20 @@ } } -static uintptr_t getPageAddr(uintptr_t Ptr, uintptr_t PageSize) { - return Ptr & ~(PageSize - 1); -} - void *GuardedPoolAllocator::allocate(size_t Size) { // GuardedPagePoolEnd == 0 when GWP-ASan is disabled. If we are disabled, fall // back to the supporting allocator. if (State.GuardedPagePoolEnd == 0) { - ThreadLocals.NextSampleCounter = + getThreadLocals()->NextSampleCounter = (AdjustedSampleRatePlusOne - 1) & ThreadLocalPackedVariables::NextSampleCounterMask; return nullptr; } // Protect against recursivity. - if (ThreadLocals.RecursiveGuard) + if (getThreadLocals()->RecursiveGuard) return nullptr; - ScopedRecursiveGuard SRG; + ScopedRecursiveGuard SRG(this); if (Size == 0 || Size > State.maximumAllocationSize()) return nullptr; @@ -212,7 +212,7 @@ } void GuardedPoolAllocator::stop() { - ThreadLocals.RecursiveGuard = true; + getThreadLocals()->RecursiveGuard = true; PoolMutex.tryLock(); } @@ -243,8 +243,8 @@ // Ensure that the unwinder is not called if the recursive flag is set, // otherwise non-reentrant unwinders may deadlock. - if (!ThreadLocals.RecursiveGuard) { - ScopedRecursiveGuard SRG; + if (!getThreadLocals()->RecursiveGuard) { + ScopedRecursiveGuard SRG(this); Meta->DeallocationTrace.RecordBacktrace(Backtrace); } } @@ -290,15 +290,11 @@ } uint32_t GuardedPoolAllocator::getRandomUnsigned32() { - uint32_t RandomState = ThreadLocals.RandomState; + uint32_t RandomState = getThreadLocals()->RandomState; RandomState ^= RandomState << 13; RandomState ^= RandomState >> 17; RandomState ^= RandomState << 5; - ThreadLocals.RandomState = RandomState; + getThreadLocals()->RandomState = RandomState; return RandomState; } - -GWP_ASAN_TLS_INITIAL_EXEC -GuardedPoolAllocator::ThreadLocalPackedVariables - GuardedPoolAllocator::ThreadLocals; } // namespace gwp_asan diff --git a/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp b/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp --- a/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp +++ b/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp @@ -38,7 +38,7 @@ namespace gwp_asan { void GuardedPoolAllocator::initPRNG() { - ThreadLocals.RandomState = + getThreadLocals()->RandomState = static_cast(time(nullptr) + getThreadID()); }