diff --git a/compiler-rt/lib/asan/asan_malloc_linux.cpp b/compiler-rt/lib/asan/asan_malloc_linux.cpp --- a/compiler-rt/lib/asan/asan_malloc_linux.cpp +++ b/compiler-rt/lib/asan/asan_malloc_linux.cpp @@ -21,108 +21,66 @@ # include "asan_interceptors.h" # include "asan_internal.h" # include "asan_stack.h" +# include "lsan/lsan_common.h" # include "sanitizer_common/sanitizer_allocator_checks.h" +# include "sanitizer_common/sanitizer_allocator_dlsym.h" # include "sanitizer_common/sanitizer_errno.h" # include "sanitizer_common/sanitizer_tls_get_addr.h" // ---------------------- Replacement functions ---------------- {{{1 using namespace __asan; -static uptr allocated_for_dlsym; -static uptr last_dlsym_alloc_size_in_words; -static const uptr kDlsymAllocPoolSize = 1024; -static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; - -static inline bool IsInDlsymAllocPool(const void *ptr) { - uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]); -} - -static void *AllocateFromLocalPool(uptr size_in_bytes) { - uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; - void *mem = (void*)&alloc_memory_for_dlsym[allocated_for_dlsym]; - last_dlsym_alloc_size_in_words = size_in_words; - allocated_for_dlsym += size_in_words; - CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); - return mem; -} - -static void DeallocateFromLocalPool(const void *ptr) { - // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store - // error messages and instead uses malloc followed by free. To avoid pool - // exhaustion due to long object filenames, handle that special case here. - uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words; - void *prev_mem = (void*)&alloc_memory_for_dlsym[prev_offset]; - if (prev_mem == ptr) { - REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize); - allocated_for_dlsym = prev_offset; - last_dlsym_alloc_size_in_words = 0; +struct DlsymAlloc : public DlSymAllocator { + static bool UseImpl() { return asan_init_is_running; } + static void OnAllocate(const void *ptr, uptr size) { +# if CAN_SANITIZE_LEAKS + // Suppress leaks from dlerror(). Previously dlsym hack on global array was + // used by leak sanitizer as a root region. + __lsan_register_root_region(ptr, size); +# endif } -} - -static inline bool MaybeInDlsym() { - // Fuchsia doesn't use dlsym-based interceptors. - return !SANITIZER_FUCHSIA && asan_init_is_running; -} - -static inline bool UseLocalPool() { return MaybeInDlsym(); } - -static void *ReallocFromLocalPool(void *ptr, uptr size) { - const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); - void *new_ptr; - if (UNLIKELY(UseLocalPool())) { - new_ptr = AllocateFromLocalPool(size); - } else { - ENSURE_ASAN_INITED(); - GET_STACK_TRACE_MALLOC; - new_ptr = asan_malloc(size, &stack); + static void OnFree(const void *ptr, uptr size) { +# if CAN_SANITIZE_LEAKS + __lsan_unregister_root_region(ptr, size); +# endif } - internal_memcpy(new_ptr, ptr, copy_size); - return new_ptr; -} +}; INTERCEPTOR(void, free, void *ptr) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) { - DeallocateFromLocalPool(ptr); - return; - } + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_STACK_TRACE_FREE; asan_free(ptr, &stack, FROM_MALLOC); } #if SANITIZER_INTERCEPT_CFREE INTERCEPTOR(void, cfree, void *ptr) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) - return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_STACK_TRACE_FREE; asan_free(ptr, &stack, FROM_MALLOC); } #endif // SANITIZER_INTERCEPT_CFREE INTERCEPTOR(void*, malloc, uptr size) { - if (UNLIKELY(UseLocalPool())) - // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. - return AllocateFromLocalPool(size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Allocate(size); ENSURE_ASAN_INITED(); GET_STACK_TRACE_MALLOC; return asan_malloc(size, &stack); } INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) { - if (UNLIKELY(UseLocalPool())) - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - return AllocateFromLocalPool(nmemb * size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); ENSURE_ASAN_INITED(); GET_STACK_TRACE_MALLOC; return asan_calloc(nmemb, size, &stack); } INTERCEPTOR(void*, realloc, void *ptr, uptr size) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) - return ReallocFromLocalPool(ptr, size); - if (UNLIKELY(UseLocalPool())) - return AllocateFromLocalPool(size); + if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Realloc(ptr, size); ENSURE_ASAN_INITED(); GET_STACK_TRACE_MALLOC; return asan_realloc(ptr, size, &stack); diff --git a/compiler-rt/lib/dfsan/dfsan_interceptors.cpp b/compiler-rt/lib/dfsan/dfsan_interceptors.cpp --- a/compiler-rt/lib/dfsan/dfsan_interceptors.cpp +++ b/compiler-rt/lib/dfsan/dfsan_interceptors.cpp @@ -17,6 +17,7 @@ #include "dfsan/dfsan.h" #include "dfsan/dfsan_thread.h" #include "interception/interception.h" +#include "sanitizer_common/sanitizer_allocator_dlsym.h" #include "sanitizer_common/sanitizer_allocator_interface.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_errno.h" @@ -26,11 +27,11 @@ using namespace __sanitizer; -namespace { +static bool interceptors_initialized; -bool interceptors_initialized; - -} // namespace +struct DlsymAlloc : public DlSymAllocator { + static bool UseImpl() { return !__dfsan::dfsan_inited; } +}; INTERCEPTOR(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size) { return __dfsan::dfsan_reallocarray(ptr, nmemb, size); @@ -47,63 +48,37 @@ return __dfsan::dfsan_aligned_alloc(alignment, size); } -static uptr allocated_for_dlsym; -static const uptr kDlsymAllocPoolSize = 1024; -static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; - -static bool IsInDlsymAllocPool(const void *ptr) { - uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - return off < sizeof(alloc_memory_for_dlsym); -} - -static void *AllocateFromLocalPool(uptr size_in_bytes) { - uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; - void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; - allocated_for_dlsym += size_in_words; - CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); - return mem; -} - INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) { - if (UNLIKELY(!__dfsan::dfsan_inited)) - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - return AllocateFromLocalPool(nmemb * size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); return __dfsan::dfsan_calloc(nmemb, size); } INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) { - uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); - void *new_ptr; - if (UNLIKELY(!__dfsan::dfsan_inited)) { - new_ptr = AllocateFromLocalPool(copy_size); - } else { - copy_size = size; - new_ptr = __dfsan::dfsan_malloc(copy_size); - } - internal_memcpy(new_ptr, ptr, copy_size); - return new_ptr; - } + if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Realloc(ptr, size); return __dfsan::dfsan_realloc(ptr, size); } INTERCEPTOR(void *, malloc, SIZE_T size) { - if (UNLIKELY(!__dfsan::dfsan_inited)) - // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. - return AllocateFromLocalPool(size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Allocate(size); return __dfsan::dfsan_malloc(size); } INTERCEPTOR(void, free, void *ptr) { - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) + if (!ptr) return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); return __dfsan::dfsan_deallocate(ptr); } INTERCEPTOR(void, cfree, void *ptr) { - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) + if (!ptr) return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); return __dfsan::dfsan_deallocate(ptr); } diff --git a/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp b/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp --- a/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp +++ b/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp @@ -14,6 +14,7 @@ #include "hwasan.h" #include "interception/interception.h" +#include "sanitizer_common/sanitizer_allocator_dlsym.h" #include "sanitizer_common/sanitizer_allocator_interface.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" @@ -21,22 +22,9 @@ using namespace __hwasan; -static uptr allocated_for_dlsym; -static const uptr kDlsymAllocPoolSize = 1024; -static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; - -static bool IsInDlsymAllocPool(const void *ptr) { - uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - return off < sizeof(alloc_memory_for_dlsym); -} - -static void *AllocateFromLocalPool(uptr size_in_bytes) { - uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; - void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; - allocated_for_dlsym += size_in_words; - CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); - return mem; -} +struct DlsymAlloc : public DlSymAllocator { + static bool UseImpl() { return !hwasan_inited; } +}; extern "C" { @@ -83,16 +71,20 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_free(void *ptr) { - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) + if (!ptr) return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_MALLOC_STACK_TRACE; hwasan_free(ptr, &stack); } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cfree(void *ptr) { - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) + if (!ptr) return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_MALLOC_STACK_TRACE; hwasan_free(ptr, &stack); } @@ -119,29 +111,16 @@ SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_calloc(uptr nmemb, uptr size) { - if (UNLIKELY(!hwasan_inited)) - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - return AllocateFromLocalPool(nmemb * size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); GET_MALLOC_STACK_TRACE; return hwasan_calloc(nmemb, size, &stack); } SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_realloc(void *ptr, uptr size) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) { - uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); - void *new_ptr; - if (UNLIKELY(!hwasan_inited)) { - new_ptr = AllocateFromLocalPool(copy_size); - } else { - copy_size = size; - GET_MALLOC_STACK_TRACE; - new_ptr = hwasan_malloc(copy_size, &stack); - } - internal_memcpy(new_ptr, ptr, copy_size); - return new_ptr; - } + if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Realloc(ptr, size); GET_MALLOC_STACK_TRACE; return hwasan_realloc(ptr, size, &stack); } @@ -156,9 +135,8 @@ void *__sanitizer_malloc(uptr size) { if (UNLIKELY(!hwasan_init_is_running)) ENSURE_HWASAN_INITED(); - if (UNLIKELY(!hwasan_inited)) - // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. - return AllocateFromLocalPool(size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Allocate(size); GET_MALLOC_STACK_TRACE; return hwasan_malloc(size, &stack); } diff --git a/compiler-rt/lib/lsan/lsan_common.h b/compiler-rt/lib/lsan/lsan_common.h --- a/compiler-rt/lib/lsan/lsan_common.h +++ b/compiler-rt/lib/lsan/lsan_common.h @@ -280,6 +280,13 @@ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char *__lsan_default_suppressions(); + +SANITIZER_INTERFACE_ATTRIBUTE +void __lsan_register_root_region(const void *p, __lsan::uptr size); + +SANITIZER_INTERFACE_ATTRIBUTE +void __lsan_unregister_root_region(const void *p, __lsan::uptr size); + } // extern "C" #endif // LSAN_COMMON_H diff --git a/compiler-rt/lib/lsan/lsan_interceptors.cpp b/compiler-rt/lib/lsan/lsan_interceptors.cpp --- a/compiler-rt/lib/lsan/lsan_interceptors.cpp +++ b/compiler-rt/lib/lsan/lsan_interceptors.cpp @@ -13,6 +13,7 @@ #include "interception/interception.h" #include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_dlsym.h" #include "sanitizer_common/sanitizer_allocator_report.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" @@ -43,6 +44,22 @@ int pthread_setspecific(unsigned key, const void *v); } +struct DlsymAlloc : DlSymAllocator { + static bool UseImpl() { return lsan_init_is_running; } + static void OnAllocate(const void *ptr, uptr size) { +#if CAN_SANITIZE_LEAKS + // Suppress leaks from dlerror(). Previously dlsym hack on global array was + // used by leak sanitizer as a root region. + __lsan_register_root_region(ptr, size); +#endif + } + static void OnFree(const void *ptr, uptr size) { +#if CAN_SANITIZE_LEAKS + __lsan_unregister_root_region(ptr, size); +#endif + } +}; + ///// Malloc/free interceptors. ///// namespace std { @@ -52,41 +69,34 @@ #if !SANITIZER_MAC INTERCEPTOR(void*, malloc, uptr size) { + if (DlsymAlloc::Use()) + return DlsymAlloc::Allocate(size); ENSURE_LSAN_INITED; GET_STACK_TRACE_MALLOC; return lsan_malloc(size, stack); } INTERCEPTOR(void, free, void *p) { + if (DlsymAlloc::PointerIsMine(p)) + return DlsymAlloc::Free(p); ENSURE_LSAN_INITED; lsan_free(p); } INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) { - // This hack is not required for Fuchsia because there are no dlsym calls - // involved in setting up interceptors. -#if !SANITIZER_FUCHSIA - if (lsan_init_is_running) { - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - const uptr kCallocPoolSize = 1024; - static uptr calloc_memory_for_dlsym[kCallocPoolSize]; - static uptr allocated; - uptr size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize; - void *mem = (void*)&calloc_memory_for_dlsym[allocated]; - allocated += size_in_words; - CHECK(allocated < kCallocPoolSize); - return mem; - } -#endif // !SANITIZER_FUCHSIA + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); ENSURE_LSAN_INITED; GET_STACK_TRACE_MALLOC; return lsan_calloc(nmemb, size, stack); } -INTERCEPTOR(void*, realloc, void *q, uptr size) { +INTERCEPTOR(void *, realloc, void *ptr, uptr size) { + if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Realloc(ptr, size); ENSURE_LSAN_INITED; GET_STACK_TRACE_MALLOC; - return lsan_realloc(q, size, stack); + return lsan_realloc(ptr, size, stack); } INTERCEPTOR(void*, reallocarray, void *q, uptr nmemb, uptr size) { diff --git a/compiler-rt/lib/memprof/memprof_malloc_linux.cpp b/compiler-rt/lib/memprof/memprof_malloc_linux.cpp --- a/compiler-rt/lib/memprof/memprof_malloc_linux.cpp +++ b/compiler-rt/lib/memprof/memprof_malloc_linux.cpp @@ -23,104 +23,52 @@ #include "memprof_internal.h" #include "memprof_stack.h" #include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_allocator_dlsym.h" #include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" // ---------------------- Replacement functions ---------------- {{{1 using namespace __memprof; -static uptr allocated_for_dlsym; -static uptr last_dlsym_alloc_size_in_words; -static const uptr kDlsymAllocPoolSize = 1024; -static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; - -static inline bool IsInDlsymAllocPool(const void *ptr) { - uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]); -} - -static void *AllocateFromLocalPool(uptr size_in_bytes) { - uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; - void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; - last_dlsym_alloc_size_in_words = size_in_words; - allocated_for_dlsym += size_in_words; - CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); - return mem; -} - -static void DeallocateFromLocalPool(const void *ptr) { - // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store - // error messages and instead uses malloc followed by free. To avoid pool - // exhaustion due to long object filenames, handle that special case here. - uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words; - void *prev_mem = (void *)&alloc_memory_for_dlsym[prev_offset]; - if (prev_mem == ptr) { - REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize); - allocated_for_dlsym = prev_offset; - last_dlsym_alloc_size_in_words = 0; - } -} - -static inline bool MaybeInDlsym() { return memprof_init_is_running; } - -static inline bool UseLocalPool() { return MaybeInDlsym(); } - -static void *ReallocFromLocalPool(void *ptr, uptr size) { - const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); - void *new_ptr; - if (UNLIKELY(UseLocalPool())) { - new_ptr = AllocateFromLocalPool(size); - } else { - ENSURE_MEMPROF_INITED(); - GET_STACK_TRACE_MALLOC; - new_ptr = memprof_malloc(size, &stack); - } - internal_memcpy(new_ptr, ptr, copy_size); - return new_ptr; -} +struct DlsymAlloc : public DlSymAllocator { + static bool UseImpl() { return memprof_init_is_running; } +}; INTERCEPTOR(void, free, void *ptr) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) { - DeallocateFromLocalPool(ptr); - return; - } + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_STACK_TRACE_FREE; memprof_free(ptr, &stack, FROM_MALLOC); } #if SANITIZER_INTERCEPT_CFREE INTERCEPTOR(void, cfree, void *ptr) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) - return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_STACK_TRACE_FREE; memprof_free(ptr, &stack, FROM_MALLOC); } #endif // SANITIZER_INTERCEPT_CFREE INTERCEPTOR(void *, malloc, uptr size) { - if (UNLIKELY(UseLocalPool())) - // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. - return AllocateFromLocalPool(size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Allocate(size); ENSURE_MEMPROF_INITED(); GET_STACK_TRACE_MALLOC; return memprof_malloc(size, &stack); } INTERCEPTOR(void *, calloc, uptr nmemb, uptr size) { - if (UNLIKELY(UseLocalPool())) - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - return AllocateFromLocalPool(nmemb * size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); ENSURE_MEMPROF_INITED(); GET_STACK_TRACE_MALLOC; return memprof_calloc(nmemb, size, &stack); } INTERCEPTOR(void *, realloc, void *ptr, uptr size) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) - return ReallocFromLocalPool(ptr, size); - if (UNLIKELY(UseLocalPool())) - return AllocateFromLocalPool(size); + if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Realloc(ptr, size); ENSURE_MEMPROF_INITED(); GET_STACK_TRACE_MALLOC; return memprof_realloc(ptr, size, &stack); diff --git a/compiler-rt/lib/msan/msan_interceptors.cpp b/compiler-rt/lib/msan/msan_interceptors.cpp --- a/compiler-rt/lib/msan/msan_interceptors.cpp +++ b/compiler-rt/lib/msan/msan_interceptors.cpp @@ -22,8 +22,8 @@ #include "msan_report.h" #include "msan_thread.h" #include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_dlsym.h" #include "sanitizer_common/sanitizer_allocator_interface.h" -#include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_errno.h" @@ -75,22 +75,9 @@ return in_interceptor_scope; } -static uptr allocated_for_dlsym; -static const uptr kDlsymAllocPoolSize = 1024; -static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; - -static bool IsInDlsymAllocPool(const void *ptr) { - uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - return off < sizeof(alloc_memory_for_dlsym); -} - -static void *AllocateFromLocalPool(uptr size_in_bytes) { - uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; - void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; - allocated_for_dlsym += size_in_words; - CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); - return mem; -} +struct DlsymAlloc : public DlSymAllocator { + static bool UseImpl() { return !msan_inited; } +}; #define ENSURE_MSAN_INITED() do { \ CHECK(!msan_init_is_running); \ @@ -221,14 +208,20 @@ #endif INTERCEPTOR(void, free, void *ptr) { - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; + if (UNLIKELY(!ptr)) + return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_MALLOC_STACK_TRACE; MsanDeallocate(&stack, ptr); } #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(void, cfree, void *ptr) { - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; + if (UNLIKELY(!ptr)) + return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_MALLOC_STACK_TRACE; MsanDeallocate(&stack, ptr); } @@ -879,27 +872,14 @@ INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) { GET_MALLOC_STACK_TRACE; - if (UNLIKELY(!msan_inited)) - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - return AllocateFromLocalPool(nmemb * size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); return msan_calloc(nmemb, size, &stack); } INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) { - uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); - void *new_ptr; - if (UNLIKELY(!msan_inited)) { - new_ptr = AllocateFromLocalPool(copy_size); - } else { - copy_size = size; - GET_MALLOC_STACK_TRACE; - new_ptr = msan_malloc(copy_size, &stack); - } - internal_memcpy(new_ptr, ptr, copy_size); - return new_ptr; - } + if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Realloc(ptr, size); GET_MALLOC_STACK_TRACE; return msan_realloc(ptr, size, &stack); } @@ -910,9 +890,8 @@ } INTERCEPTOR(void *, malloc, SIZE_T size) { - if (UNLIKELY(!msan_inited)) - // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. - return AllocateFromLocalPool(size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Allocate(size); GET_MALLOC_STACK_TRACE; return msan_malloc(size, &stack); } diff --git a/compiler-rt/lib/sanitizer_common/CMakeLists.txt b/compiler-rt/lib/sanitizer_common/CMakeLists.txt --- a/compiler-rt/lib/sanitizer_common/CMakeLists.txt +++ b/compiler-rt/lib/sanitizer_common/CMakeLists.txt @@ -100,6 +100,7 @@ sanitizer_allocator.h sanitizer_allocator_checks.h sanitizer_allocator_combined.h + sanitizer_allocator_dlsym.h sanitizer_allocator_interface.h sanitizer_allocator_internal.h sanitizer_allocator_local_cache.h diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h @@ -0,0 +1,79 @@ +//===-- sanitizer_allocator_dlsym.h -----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Hack: Sanitizer initializer calls dlsym which may need to allocate and call +// back into uninitialized sanitizer. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_ALLOCATOR_DLSYM_H +#define SANITIZER_ALLOCATOR_DLSYM_H + +#include "sanitizer_allocator_internal.h" + +namespace __sanitizer { + +template +struct DlSymAllocator { + static bool Use() { + // Fuchsia doesn't use dlsym-based interceptors. + return !SANITIZER_FUCHSIA && UNLIKELY(Details::UseImpl()); + } + + static bool PointerIsMine(const void *ptr) { + // Fuchsia doesn't use dlsym-based interceptors. + return !SANITIZER_FUCHSIA && + UNLIKELY(internal_allocator()->FromPrimary(ptr)); + } + + static void *Allocate(uptr size_in_bytes) { + void *ptr = InternalAlloc(size_in_bytes, nullptr, kWordSize); + CHECK(internal_allocator()->FromPrimary(ptr)); + Details::OnAllocate(ptr, + internal_allocator()->GetActuallyAllocatedSize(ptr)); + return ptr; + } + + static void *Callocate(SIZE_T nmemb, SIZE_T size) { + void *ptr = InternalCalloc(nmemb, size); + CHECK(internal_allocator()->FromPrimary(ptr)); + Details::OnAllocate(ptr, + internal_allocator()->GetActuallyAllocatedSize(ptr)); + return ptr; + } + + static void Free(void *ptr) { + uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr); + Details::OnFree(ptr, size); + InternalFree(ptr); + } + + static void *Realloc(void *ptr, uptr new_size) { + if (!ptr) + return Allocate(new_size); + CHECK(internal_allocator()->FromPrimary(ptr)); + if (!new_size) { + Free(ptr); + return nullptr; + } + uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr); + uptr memcpy_size = Min(new_size, size); + void *new_ptr = Allocate(new_size); + if (new_ptr) + internal_memcpy(new_ptr, ptr, memcpy_size); + Free(ptr); + return new_ptr; + } + + static void OnAllocate(const void *ptr, uptr size) {} + static void OnFree(const void *ptr, uptr size) {} +}; + +} // namespace __sanitizer + +#endif // SANITIZER_ALLOCATOR_DLSYM_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc b/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc --- a/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc +++ b/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc @@ -23,6 +23,7 @@ #include #include "interception/interception.h" +#include "sanitizer_common/sanitizer_allocator_dlsym.h" #include "sanitizer_common/sanitizer_mac.h" // Similar code is used in Google Perftools, @@ -192,20 +193,15 @@ return p; } +struct DlsymAlloc : public DlSymAllocator { + static bool UseImpl() { return !COMMON_MALLOC_SANITIZER_INITIALIZED; } +}; + extern "C" SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_mz_calloc(malloc_zone_t *zone, size_t nmemb, size_t size) { - if (UNLIKELY(!COMMON_MALLOC_SANITIZER_INITIALIZED)) { - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - const size_t kCallocPoolSize = 1024; - static uptr calloc_memory_for_dlsym[kCallocPoolSize]; - static size_t allocated; - size_t size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize; - void *mem = (void*)&calloc_memory_for_dlsym[allocated]; - allocated += size_in_words; - CHECK(allocated < kCallocPoolSize); - return mem; - } + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); COMMON_MALLOC_CALLOC(nmemb, size); return p; } @@ -223,6 +219,8 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_mz_free(malloc_zone_t *zone, void *ptr) { if (!ptr) return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); COMMON_MALLOC_FREE(ptr); }