Index: compiler-rt/CMakeLists.txt =================================================================== --- compiler-rt/CMakeLists.txt +++ compiler-rt/CMakeLists.txt @@ -59,6 +59,9 @@ -D${COMPILER_RT_ASAN_SHADOW_SCALE_DEFINITION}) endif() +set(COMPILER_RT_HWASAN_WITH_INTERCEPTORS ON CACHE BOOLEAN + "Enable libc interceptors in HWASan (testing mode)") + set(COMPILER_RT_BAREMETAL_BUILD OFF CACHE BOOLEAN "Build for a bare-metal target.") Index: compiler-rt/include/sanitizer/hwasan_interface.h =================================================================== --- compiler-rt/include/sanitizer/hwasan_interface.h +++ compiler-rt/include/sanitizer/hwasan_interface.h @@ -47,6 +47,14 @@ // does would cause false reports. void __hwasan_handle_longjmp(const void *sp_dst); + // Libc hook for thread creation. Should be called in the child thread before + // any instrumented code. + void __hwasan_thread_enter(); + + // Libc hook for thread destruction. No instrumented code should run after + // this call. + void __hwasan_thread_exit(); + // Print shadow and origin for the memory range to stderr in a human-readable // format. void __hwasan_print_shadow(const volatile void *x, size_t size); Index: compiler-rt/lib/hwasan/CMakeLists.txt =================================================================== --- compiler-rt/lib/hwasan/CMakeLists.txt +++ compiler-rt/lib/hwasan/CMakeLists.txt @@ -28,6 +28,9 @@ hwasan_thread.h) +set(HWASAN_DEFINITIONS) +append_list_if(COMPILER_RT_HWASAN_WITH_INTERCEPTORS HWASAN_WITH_INTERCEPTORS=1 HWASAN_DEFINITIONS) + set(HWASAN_RTL_CFLAGS ${SANITIZER_COMMON_CFLAGS}) append_rtti_flag(OFF HWASAN_RTL_CFLAGS) append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC HWASAN_RTL_CFLAGS) @@ -55,23 +58,27 @@ ARCHS ${HWASAN_SUPPORTED_ARCH} SOURCES ${HWASAN_RTL_SOURCES} ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS} - CFLAGS ${HWASAN_RTL_CFLAGS}) + CFLAGS ${HWASAN_RTL_CFLAGS} + DEFS ${HWASAN_DEFINITIONS}) add_compiler_rt_object_libraries(RTHwasan_cxx ARCHS ${HWASAN_SUPPORTED_ARCH} SOURCES ${HWASAN_RTL_CXX_SOURCES} ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS} - CFLAGS ${HWASAN_RTL_CFLAGS}) + CFLAGS ${HWASAN_RTL_CFLAGS} + DEFS ${HWASAN_DEFINITIONS}) add_compiler_rt_object_libraries(RTHwasan_dynamic ARCHS ${HWASAN_SUPPORTED_ARCH} SOURCES ${HWASAN_RTL_SOURCES} ${HWASAN_RTL_CXX_SOURCES} ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS} - CFLAGS ${HWASAN_DYNAMIC_CFLAGS}) + CFLAGS ${HWASAN_DYNAMIC_CFLAGS} + DEFS ${HWASAN_DEFINITIONS}) file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cc "") add_compiler_rt_object_libraries(RTHwasan_dynamic_version_script_dummy ARCHS ${HWASAN_SUPPORTED_ARCH} SOURCES ${CMAKE_CURRENT_BINARY_DIR}/dummy.cc - CFLAGS ${HWASAN_DYNAMIC_CFLAGS}) + CFLAGS ${HWASAN_DYNAMIC_CFLAGS} + DEFS ${HWASAN_DEFINITIONS}) foreach(arch ${HWASAN_SUPPORTED_ARCH}) add_compiler_rt_runtime(clang_rt.hwasan Index: compiler-rt/lib/hwasan/hwasan.h =================================================================== --- compiler-rt/lib/hwasan/hwasan.h +++ compiler-rt/lib/hwasan/hwasan.h @@ -30,6 +30,10 @@ # define HWASAN_CONTAINS_UBSAN CAN_SANITIZE_UB #endif +#ifndef HWASAN_WITH_INTERCEPTORS +#define HWASAN_WITH_INTERCEPTORS 0 +#endif + typedef u8 tag_t; // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address @@ -136,10 +140,7 @@ u64 va_arg_overflow_size_tls; }; -void HwasanTSDInit(void (*destructor)(void *tsd)); -void *HwasanTSDGet(); -void HwasanTSDSet(void *tsd); -void HwasanTSDDtor(void *tsd); +void HwasanTSDInit(); void HwasanOnDeadlySignal(int signo, void *info, void *context); Index: compiler-rt/lib/hwasan/hwasan.cc =================================================================== --- compiler-rt/lib/hwasan/hwasan.cc +++ compiler-rt/lib/hwasan/hwasan.cc @@ -209,13 +209,13 @@ InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); - HwasanTSDInit(HwasanTSDDtor); + HwasanTSDInit(); HwasanAllocatorInit(); HwasanThread *main_thread = HwasanThread::Create(nullptr, nullptr); SetCurrentThread(main_thread); - main_thread->ThreadStart(); + main_thread->Init(); #if HWASAN_CONTAINS_UBSAN __ubsan::InitAsPlugin(); Index: compiler-rt/lib/hwasan/hwasan_interceptors.cc =================================================================== --- compiler-rt/lib/hwasan/hwasan_interceptors.cc +++ compiler-rt/lib/hwasan/hwasan_interceptors.cc @@ -225,7 +225,7 @@ asm volatile("mov %0,x8" : "=r" (r8)); sret = reinterpret_cast<__sanitizer_struct_mallinfo*>(r8); #endif - REAL(memset)(sret, 0, sizeof(*sret)); + internal_memset(sret, 0, sizeof(*sret)); } #define HWASAN_MAYBE_INTERCEPT_MALLINFO INTERCEPT_FUNCTION(mallinfo) #else @@ -287,27 +287,13 @@ return hwasan_malloc(size, &stack); } -template -static void *mmap_interceptor(Mmap real_mmap, void *addr, SIZE_T sz, int prot, - int flags, int fd, OFF64_T off) { - if (addr && !MEM_IS_APP(addr)) { - if (flags & map_fixed) { - errno = errno_EINVAL; - return (void *)-1; - } else { - addr = nullptr; - } - } - return real_mmap(addr, sz, prot, flags, fd, off); -} - +#if HWASAN_WITH_INTERCEPTORS extern "C" int pthread_attr_init(void *attr); extern "C" int pthread_attr_destroy(void *attr); static void *HwasanThreadStartFunc(void *arg) { - HwasanThread *t = (HwasanThread *)arg; - SetCurrentThread(t); - return t->ThreadStart(); + __hwasan_thread_enter(); + return ((HwasanThread *)arg)->ThreadStart(); } INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), @@ -329,6 +315,7 @@ pthread_attr_destroy(&myattr); return res; } +#endif // HWASAN_WITH_INTERCEPTORS static void BeforeFork() { StackDepotLockAll(); @@ -360,134 +347,14 @@ } // namespace __hwasan -// A version of CHECK_UNPOISONED using a saved scope value. Used in common -// interceptors. -#define CHECK_UNPOISONED_CTX(ctx, x, n) \ - do { \ - if (!((HwasanInterceptorContext *)ctx)->in_interceptor_scope) \ - CHECK_UNPOISONED_0(x, n); \ - } while (0) - -#define HWASAN_INTERCEPT_FUNC(name) \ - do { \ - if ((!INTERCEPT_FUNCTION(name) || !REAL(name))) \ - VReport(1, "HWAddressSanitizer: failed to intercept '" #name "'\n"); \ - } while (0) - -#define HWASAN_INTERCEPT_FUNC_VER(name, ver) \ - do { \ - if ((!INTERCEPT_FUNCTION_VER(name, ver) || !REAL(name))) \ - VReport( \ - 1, "HWAddressSanitizer: failed to intercept '" #name "@@" #ver "'\n"); \ - } while (0) - -#define COMMON_INTERCEPT_FUNCTION(name) HWASAN_INTERCEPT_FUNC(name) -#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ - HWASAN_INTERCEPT_FUNC_VER(name, ver) -#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ - CHECK_UNPOISONED_CTX(ctx, ptr, size) -#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ - CHECK_UNPOISONED_CTX(ctx, ptr, size) -#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(ptr, size) \ - HWASAN_WRITE_RANGE(ctx, ptr, size) -#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ - if (hwasan_init_is_running) return REAL(func)(__VA_ARGS__); \ - ENSURE_HWASAN_INITED(); \ - HwasanInterceptorContext hwasan_ctx = {IsInInterceptorScope()}; \ - ctx = (void *)&hwasan_ctx; \ - (void)ctx; \ - InterceptorScope interceptor_scope; -#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ - do { \ - } while (false) -#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ - do { \ - } while (false) // FIXME -#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ - do { \ - } while (false) // FIXME -#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) -#define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit() - -#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ - if (HwasanThread *t = GetCurrentThread()) { \ - *begin = t->tls_begin(); \ - *end = t->tls_end(); \ - } else { \ - *begin = *end = 0; \ - } - -// AArch64 has TBI and can (and must!) pass the pointer to system memset as-is. -// Other platforms need to remove the tag. -#if defined(__aarch64__) -#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \ - { \ - COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \ - if (common_flags()->intercept_intrin && \ - MEM_IS_APP(GetAddressFromPointer(dst))) \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ - return REAL(memset)(dst, v, size); \ - } -#else -#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \ - { \ - COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \ - if (common_flags()->intercept_intrin && \ - MEM_IS_APP(GetAddressFromPointer(dst))) \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ - return REAL(memset)(GetAddressFromPointer(dst), v, size); \ - } -#endif - -#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, length, prot, flags, fd, \ - offset) \ - do { \ - return mmap_interceptor(REAL(mmap), addr, length, prot, flags, fd, \ - offset); \ - } while (false) - -#include "sanitizer_common/sanitizer_platform_interceptors.h" -#include "sanitizer_common/sanitizer_common_interceptors.inc" -#include "sanitizer_common/sanitizer_signal_interceptors.inc" - -#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) CHECK_UNPOISONED(p, s) -#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ - do { \ - (void)(p); \ - (void)(s); \ - } while (false) -#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ - do { \ - (void)(p); \ - (void)(s); \ - } while (false) -#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ - do { \ - (void)(p); \ - (void)(s); \ - } while (false) -#include "sanitizer_common/sanitizer_common_syscalls.inc" -#include "sanitizer_common/sanitizer_syscalls_netbsd.inc" - - - namespace __hwasan { void InitializeInterceptors() { static int inited = 0; CHECK_EQ(inited, 0); - InitializeCommonInterceptors(); - InitializeSignalInterceptors(); + // FIXME: disable interceptors for allocator functions, keep only __sanitizer + // aliases unless HWASAN_WITH_INTERCEPTORS=1. INTERCEPT_FUNCTION(posix_memalign); HWASAN_MAYBE_INTERCEPT_MEMALIGN; INTERCEPT_FUNCTION(__libc_memalign); @@ -502,9 +369,12 @@ HWASAN_MAYBE_INTERCEPT_MALLINFO; HWASAN_MAYBE_INTERCEPT_MALLOPT; HWASAN_MAYBE_INTERCEPT_MALLOC_STATS; - INTERCEPT_FUNCTION(pthread_create); INTERCEPT_FUNCTION(fork); +#if HWASAN_WITH_INTERCEPTORS + INTERCEPT_FUNCTION(pthread_create); +#endif + inited = 1; } } // namespace __hwasan Index: compiler-rt/lib/hwasan/hwasan_interface_internal.h =================================================================== --- compiler-rt/lib/hwasan/hwasan_interface_internal.h +++ compiler-rt/lib/hwasan/hwasan_interface_internal.h @@ -137,6 +137,12 @@ SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_disable_allocator_tagging(); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_thread_enter(); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_thread_exit(); + } // extern "C" #endif // HWASAN_INTERFACE_INTERNAL_H Index: compiler-rt/lib/hwasan/hwasan_linux.cc =================================================================== --- compiler-rt/lib/hwasan/hwasan_linux.cc +++ compiler-rt/lib/hwasan/hwasan_linux.cc @@ -251,17 +251,42 @@ // ---------------------- TSD ---------------- {{{1 +extern "C" void __hwasan_thread_enter() { + HwasanThread *t = HwasanThread::Create(nullptr, nullptr); + SetCurrentThread(t); + t->Init(); +} + +extern "C" void __hwasan_thread_exit() { + HwasanThread *t = GetCurrentThread(); + // Make sure that signal handler can not see a stale current thread pointer. + atomic_signal_fence(memory_order_seq_cst); + if (t) + t->Destroy(); +} + +#if HWASAN_WITH_INTERCEPTORS static pthread_key_t tsd_key; static bool tsd_key_inited = false; -void HwasanTSDInit(void (*destructor)(void *tsd)) { +void HwasanTSDDtor(void *tsd) { + HwasanThread *t = (HwasanThread*)tsd; + if (t->destructor_iterations_ > 1) { + t->destructor_iterations_--; + CHECK_EQ(0, pthread_setspecific(tsd_key, tsd)); + return; + } + __hwasan_thread_exit(); +} + +void HwasanTSDInit() { CHECK(!tsd_key_inited); tsd_key_inited = true; - CHECK_EQ(0, pthread_key_create(&tsd_key, destructor)); + CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor)); } HwasanThread *GetCurrentThread() { - return (HwasanThread*)pthread_getspecific(tsd_key); + return (HwasanThread *)pthread_getspecific(tsd_key); } void SetCurrentThread(HwasanThread *t) { @@ -271,18 +296,18 @@ CHECK_EQ(0, pthread_getspecific(tsd_key)); pthread_setspecific(tsd_key, (void *)t); } +#elif SANITIZER_ANDROID +void HwasanTSDInit() {} +HwasanThread *GetCurrentThread() { + return (HwasanThread*)*get_android_tls_ptr(); +} -void HwasanTSDDtor(void *tsd) { - HwasanThread *t = (HwasanThread*)tsd; - if (t->destructor_iterations_ > 1) { - t->destructor_iterations_--; - CHECK_EQ(0, pthread_setspecific(tsd_key, tsd)); - return; - } - // Make sure that signal handler can not see a stale current thread pointer. - atomic_signal_fence(memory_order_seq_cst); - HwasanThread::TSDDtor(tsd); +void SetCurrentThread(HwasanThread *t) { + *get_android_tls_ptr() = (uptr)t; } +#else +#error unsupported configuration !HWASAN_WITH_INTERCEPTORS && !SANITIZER_ANDROID +#endif struct AccessInfo { uptr addr; Index: compiler-rt/lib/hwasan/hwasan_thread.h =================================================================== --- compiler-rt/lib/hwasan/hwasan_thread.h +++ compiler-rt/lib/hwasan/hwasan_thread.h @@ -22,10 +22,9 @@ class HwasanThread { public: static HwasanThread *Create(thread_callback_t start_routine, void *arg); - static void TSDDtor(void *tsd); void Destroy(); - void Init(); // Should be called from the thread itself. + void Init(); thread_return_t ThreadStart(); uptr stack_top() { return stack_top_; } Index: compiler-rt/lib/hwasan/hwasan_thread.cc =================================================================== --- compiler-rt/lib/hwasan/hwasan_thread.cc +++ compiler-rt/lib/hwasan/hwasan_thread.cc @@ -65,11 +65,6 @@ } } -void HwasanThread::TSDDtor(void *tsd) { - HwasanThread *t = (HwasanThread*)tsd; - t->Destroy(); -} - void HwasanThread::ClearShadowForThreadStackAndTLS() { if (stack_top_ != stack_bottom_) TagMemory(stack_bottom_, stack_top_ - stack_bottom_, 0); @@ -86,18 +81,7 @@ } thread_return_t HwasanThread::ThreadStart() { - Init(); - - if (!start_routine_) { - // start_routine_ == 0 if we're on the main thread or on one of the - // OS X libdispatch worker threads. But nobody is supposed to call - // ThreadStart() for the worker threads. - return 0; - } - - thread_return_t res = start_routine_(arg_); - - return res; + return start_routine_(arg_); } static u32 xorshift(u32 state) { Index: compiler-rt/test/hwasan/TestCases/longjmp.c =================================================================== --- compiler-rt/test/hwasan/TestCases/longjmp.c +++ compiler-rt/test/hwasan/TestCases/longjmp.c @@ -9,8 +9,8 @@ __attribute__((noinline)) int f(void *caller_frame) { - char z[32] = {}; - char *volatile p = z; + int z = 0; + int *volatile p = &z; // Tag of local is never zero. assert(__hwasan_tag_pointer(p, 0) != p); #ifndef NEGATIVE