diff --git a/compiler-rt/lib/hwasan/CMakeLists.txt b/compiler-rt/lib/hwasan/CMakeLists.txt --- a/compiler-rt/lib/hwasan/CMakeLists.txt +++ b/compiler-rt/lib/hwasan/CMakeLists.txt @@ -7,6 +7,7 @@ hwasan_allocation_functions.cpp hwasan_dynamic_shadow.cpp hwasan_exceptions.cpp + hwasan_fuchsia.cpp hwasan_globals.cpp hwasan_interceptors.cpp hwasan_interceptors_vfork.S diff --git a/compiler-rt/lib/hwasan/hwasan.cpp b/compiler-rt/lib/hwasan/hwasan.cpp --- a/compiler-rt/lib/hwasan/hwasan.cpp +++ b/compiler-rt/lib/hwasan/hwasan.cpp @@ -276,7 +276,6 @@ } InitThreads(); - hwasanThreadList().CreateCurrentThread(); hwasan_instrumentation_inited = 1; } diff --git a/compiler-rt/lib/hwasan/hwasan_fuchsia.cpp b/compiler-rt/lib/hwasan/hwasan_fuchsia.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/hwasan/hwasan_fuchsia.cpp @@ -0,0 +1,214 @@ +//===-- hwasan_fuchsia.cpp --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file is a part of HWAddressSanitizer and contains Fuchsia-specific +/// code. +/// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_fuchsia.h" +#if SANITIZER_FUCHSIA + +#include "hwasan.h" +#include "hwasan_interface_internal.h" +#include "hwasan_report.h" +#include "hwasan_thread.h" +#include "hwasan_thread_list.h" + +// This TLS variable contains the location of the stack ring buffer and can be +// used to always find the hwasan thread object associated with the current +// running thread. +SANITIZER_INTERFACE_ATTRIBUTE +THREADLOCAL uptr __hwasan_tls; + +namespace __hwasan { + +// These are known parameters passed to the hwasan runtime on thread creation +// when creating a thread. +struct Thread::InitOptions { + uptr stack_bottom, stack_top; +}; + +static void FinishThreadInitialization(Thread *thread); + +void InitThreads() { + uptr alloc_size = UINT64_C(1) << kShadowBaseAlignment; + uptr thread_start = reinterpret_cast( + MmapAlignedOrDieOnFatalError(alloc_size, alloc_size, __func__)); + + InitThreadList(thread_start, alloc_size); + + // Create the hwasan thread object for the current (main) thread. Stack info + // for this thread is known from information passed via + // __sanitizer_startup_hook. + const Thread::InitOptions options = { + .stack_bottom = __sanitizer::MainThreadStackBase, + .stack_top = + __sanitizer::MainThreadStackBase + __sanitizer::MainThreadStackSize, + }; + FinishThreadInitialization(hwasanThreadList().CreateCurrentThread(&options)); +} + +uptr *GetCurrentThreadLongPtr() { return &__hwasan_tls; } + +Thread *GetCurrentThread() { + uptr *ThreadLongPtr = GetCurrentThreadLongPtr(); + if (UNLIKELY(*ThreadLongPtr == 0)) + return nullptr; + auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr; + return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next()); +} + +// This is called from the parent thread before the new thread is created. Here +// we can propagate known info like the stack bounds to Thread::Init before +// jumping into the thread. We cannot initialize the stack ring buffer yet since +// we have not entered the new thread. +static void *BeforeThreadCreateHook(uptr user_id, bool detached, + const char *name, uptr stack_bottom, + uptr stack_size) { + const Thread::InitOptions options = { + .stack_bottom = stack_bottom, + .stack_top = stack_bottom + stack_size, + }; + return hwasanThreadList().CreateCurrentThread(&options); +} + +// Unlike the linux implementation which requires Thread::Init to be called +// while in the newly created thread, we can set the stack bounds because they +// are known before thread creation. However, we cannot setup the stack ring +// buffer just yet because it is stored in the global __hwasan_tls, which we can +// only correctly access once in the new thread. This will be set up in the +// ThreadStartHook where we can safely reference __hwasan_tls while in the new +// thread. +// +// This function is called through HwasanThreadList::CreateCurrentThread. +void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size, + const InitOptions *options) { + CHECK_EQ(0, unique_id_); // try to catch bad stack reuse + CHECK_EQ(0, stack_top_); + CHECK_EQ(0, stack_bottom_); + + static u64 unique_id; + unique_id_ = unique_id++; + if (auto sz = flags()->heap_history_size) + heap_allocations_ = HeapAllocationsRingBuffer::New(sz); + + CHECK_NE(options->stack_bottom, 0); + CHECK_NE(options->stack_top, 0); + stack_bottom_ = options->stack_bottom; + stack_top_ = options->stack_top; + tls_end_ = tls_begin_ = 0; +} + +// This is called after creating a new thread with the pointer returned by +// BeforeThreadCreateHook. We are still in the creating thread and should check +// if it was actually created correctly. +static void ThreadCreateHook(void *hook, bool aborted) { + Thread *thread = static_cast(hook); + if (!aborted) { + // The thread was created successfully. + // ThreadStartHook can already be running in the new thread. + } else { + // The thread wasn't created after all. + // Clean up everything we set up in BeforeThreadCreateHook. + atomic_signal_fence(memory_order_seq_cst); + hwasanThreadList().ReleaseThread(thread); + } +} + +// This is called in the newly-created thread before it runs anything else, +// with the pointer returned by BeforeThreadCreateHook (above). Here we can +// setup the stack ring buffer. +static void ThreadStartHook(void *hook, thrd_t self) { + Thread *thread = static_cast(hook); + FinishThreadInitialization(thread); +} + +// This is the actual function that (1) sets up the stack ring buffer and (2) +// initializes the random state of the thread. This function should only be +// called while IN the thread of the passed hwasan thread object so the right +// __hwasan_tls can be referenced. +static void FinishThreadInitialization(Thread *thread) { + CHECK_NE(thread, nullptr); + + // The following implicitly sets up the thread object as the current thread. + // That is, we can always get the appropriate thread object from the ring + // buffer using __hwasan_tls. The ring buffer is located immediately before + // the thread object. + uptr stack_buffer_size = hwasanThreadList().GetRingBufferSize(); + uptr stack_buffer_start = reinterpret_cast(thread) - stack_buffer_size; + uptr *ThreadLong = GetCurrentThreadLongPtr(); + thread->InitStackAllocations(new (ThreadLong) StackAllocationsRingBuffer( + (void *)stack_buffer_start, stack_buffer_size)); + CHECK_EQ(GetCurrentThread(), thread); + + // ScopedTaggingDisable needs GetCurrentThread to be set up. This disables + // random tag generation until we exit this function. We do this because we + // still need to initialize the random state of the thread. This is only + // relevant if tags are generated with calls to the runtime. + ScopedTaggingDisabler disabler; + + // Sanity checks to make sure we have the correct stack bounds. + CHECK_NE(thread->stack_bottom(), 0); + CHECK_NE(thread->stack_top(), 0); + int local; + CHECK(thread->AddrIsInStack((uptr)&local)); + + // TODO(leonardchan): Uncomment these once MemIsApp is implemented. + // CHECK(MemIsApp(thread->stack_bottom())); + // CHECK(MemIsApp(thread->stack_top() - 1)); + + if (flags()->verbose_threads) { + if (thread->IsMainThread()) { + Printf("sizeof(Thread): %zd sizeof(HeapRB): %zd sizeof(StackRB): %zd\n", + sizeof(Thread), thread->heap_allocations()->SizeInBytes(), + thread->stack_allocations()->size() * sizeof(uptr)); + } + Printf("Creating : "); + } + + // Initialize a random state. This sets a random starting position in the + // stack ring buffer and is used for making random tags if tags are generated + // by calls to the runtime. + thread->InitRandomState(); +} + +static void ThreadExitHook(void *hook, thrd_t self) { + Thread *thread = static_cast(hook); + atomic_signal_fence(memory_order_seq_cst); + hwasanThreadList().ReleaseThread(thread); +} + +} // namespace __hwasan + +extern "C" { + +void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached, + const char *name, void *stack_base, + size_t stack_size) { + return __hwasan::BeforeThreadCreateHook( + reinterpret_cast(thread), detached, name, + reinterpret_cast(stack_base), stack_size); +} + +void __sanitizer_thread_create_hook(void *hook, thrd_t thread, int error) { + __hwasan::ThreadCreateHook(hook, error != thrd_success); +} + +void __sanitizer_thread_start_hook(void *hook, thrd_t self) { + __hwasan::ThreadStartHook(hook, reinterpret_cast(self)); +} + +void __sanitizer_thread_exit_hook(void *hook, thrd_t self) { + __hwasan::ThreadExitHook(hook, self); +} + +} // extern "C" + +#endif // SANITIZER_FUCHSIA diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cpp b/compiler-rt/lib/hwasan/hwasan_linux.cpp --- a/compiler-rt/lib/hwasan/hwasan_linux.cpp +++ b/compiler-rt/lib/hwasan/hwasan_linux.cpp @@ -250,6 +250,8 @@ ProtectGap(thread_space_end, __hwasan_shadow_memory_dynamic_address - thread_space_end); InitThreadList(thread_space_start, thread_space_end - thread_space_start); + + hwasanThreadList().CreateCurrentThread(); } bool MemIsApp(uptr p) { @@ -427,6 +429,55 @@ HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr); } +// This must be called from the thread itself. +void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size, + const InitOptions *options) { + CHECK_EQ(0, unique_id_); // try to catch bad stack reuse + CHECK_EQ(0, stack_top_); + CHECK_EQ(0, stack_bottom_); + + static u64 unique_id; + unique_id_ = unique_id++; + if (auto sz = flags()->heap_history_size) + heap_allocations_ = HeapAllocationsRingBuffer::New(sz); + + HwasanTSDThreadInit(); // Only needed with interceptors. + uptr *ThreadLong = GetCurrentThreadLongPtr(); + // The following implicitly sets (this) as the current thread. + stack_allocations_ = new (ThreadLong) + StackAllocationsRingBuffer((void *)stack_buffer_start, stack_buffer_size); + // Check that it worked. + CHECK_EQ(GetCurrentThread(), this); + + // ScopedTaggingDisable needs GetCurrentThread to be set up. + ScopedTaggingDisabler disabler; + + // The options should not be passed in this implementation. Stack and TLS info + // are retrieved via GetThreadStackAndTls. + DCHECK_EQ(options, nullptr); + uptr tls_size; + uptr stack_size; + GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_, + &tls_size); + stack_top_ = stack_bottom_ + stack_size; + tls_end_ = tls_begin_ + tls_size; + + if (stack_bottom_) { + int local; + CHECK(AddrIsInStack((uptr)&local)); + CHECK(MemIsApp(stack_bottom_)); + CHECK(MemIsApp(stack_top_ - 1)); + } + + if (flags()->verbose_threads) { + if (IsMainThread()) { + Printf("sizeof(Thread): %zd sizeof(HeapRB): %zd sizeof(StackRB): %zd\n", + sizeof(Thread), heap_allocations_->SizeInBytes(), + stack_allocations_->size() * sizeof(uptr)); + } + Print("Creating : "); + } +} } // namespace __hwasan diff --git a/compiler-rt/lib/hwasan/hwasan_thread.h b/compiler-rt/lib/hwasan/hwasan_thread.h --- a/compiler-rt/lib/hwasan/hwasan_thread.h +++ b/compiler-rt/lib/hwasan/hwasan_thread.h @@ -23,7 +23,17 @@ class Thread { public: - void Init(uptr stack_buffer_start, uptr stack_buffer_size); // Must be called from the thread itself. + // These are optional parameters that can be passed to Init. + struct InitOptions; + + void Init(uptr stack_buffer_start, uptr stack_buffer_size, + const InitOptions *options = nullptr); + + void InitStackAllocations(StackAllocationsRingBuffer *allocations) { + CHECK_EQ(stack_allocations_, nullptr); + stack_allocations_ = allocations; + } + void InitRandomState(); void Destroy(); @@ -72,7 +82,7 @@ AllocatorCache allocator_cache_; HeapAllocationsRingBuffer *heap_allocations_; - StackAllocationsRingBuffer *stack_allocations_; + StackAllocationsRingBuffer *stack_allocations_ = nullptr; u64 unique_id_; // counting from zero. diff --git a/compiler-rt/lib/hwasan/hwasan_thread.cpp b/compiler-rt/lib/hwasan/hwasan_thread.cpp --- a/compiler-rt/lib/hwasan/hwasan_thread.cpp +++ b/compiler-rt/lib/hwasan/hwasan_thread.cpp @@ -34,51 +34,6 @@ stack_allocations_->push(0); } -void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) { - CHECK_EQ(0, unique_id_); // try to catch bad stack reuse - CHECK_EQ(0, stack_top_); - CHECK_EQ(0, stack_bottom_); - - static u64 unique_id; - unique_id_ = unique_id++; - if (auto sz = flags()->heap_history_size) - heap_allocations_ = HeapAllocationsRingBuffer::New(sz); - - HwasanTSDThreadInit(); // Only needed with interceptors. - uptr *ThreadLong = GetCurrentThreadLongPtr(); - // The following implicitly sets (this) as the current thread. - stack_allocations_ = new (ThreadLong) - StackAllocationsRingBuffer((void *)stack_buffer_start, stack_buffer_size); - // Check that it worked. - CHECK_EQ(GetCurrentThread(), this); - - // ScopedTaggingDisable needs GetCurrentThread to be set up. - ScopedTaggingDisabler disabler; - - uptr tls_size; - uptr stack_size; - GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_, - &tls_size); - stack_top_ = stack_bottom_ + stack_size; - tls_end_ = tls_begin_ + tls_size; - - if (stack_bottom_) { - int local; - CHECK(AddrIsInStack((uptr)&local)); - CHECK(MemIsApp(stack_bottom_)); - CHECK(MemIsApp(stack_top_ - 1)); - } - - if (flags()->verbose_threads) { - if (IsMainThread()) { - Printf("sizeof(Thread): %zd sizeof(HeapRB): %zd sizeof(StackRB): %zd\n", - sizeof(Thread), heap_allocations_->SizeInBytes(), - stack_allocations_->size() * sizeof(uptr)); - } - Print("Creating : "); - } -} - void Thread::ClearShadowForThreadStackAndTLS() { if (stack_top_ != stack_bottom_) TagMemory(stack_bottom_, stack_top_ - stack_bottom_, 0); diff --git a/compiler-rt/lib/hwasan/hwasan_thread_list.h b/compiler-rt/lib/hwasan/hwasan_thread_list.h --- a/compiler-rt/lib/hwasan/hwasan_thread_list.h +++ b/compiler-rt/lib/hwasan/hwasan_thread_list.h @@ -85,7 +85,7 @@ RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2); } - Thread *CreateCurrentThread() { + Thread *CreateCurrentThread(const Thread::InitOptions *options = nullptr) { Thread *t = nullptr; { SpinMutexLock l(&free_list_mutex_); @@ -104,7 +104,7 @@ SpinMutexLock l(&live_list_mutex_); live_list_.push_back(t); } - t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_); + t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_, options); AddThreadStats(t); return t; } @@ -171,6 +171,9 @@ return stats_; } + // Reuse this value so external users don't need to call RingBufferSize(). + uptr GetRingBufferSize() const { return ring_buffer_size_; } + private: Thread *AllocThread() { SpinMutexLock l(&free_space_mutex_);