diff --git a/compiler-rt/lib/asan/asan_malloc_linux.cpp b/compiler-rt/lib/asan/asan_malloc_linux.cpp --- a/compiler-rt/lib/asan/asan_malloc_linux.cpp +++ b/compiler-rt/lib/asan/asan_malloc_linux.cpp @@ -22,107 +22,58 @@ # include "asan_internal.h" # include "asan_stack.h" # include "sanitizer_common/sanitizer_allocator_checks.h" +# include "sanitizer_common/sanitizer_allocator_dlsym.h" # include "sanitizer_common/sanitizer_errno.h" # include "sanitizer_common/sanitizer_tls_get_addr.h" // ---------------------- Replacement functions ---------------- {{{1 using namespace __asan; -static uptr allocated_for_dlsym; -static uptr last_dlsym_alloc_size_in_words; -static const uptr kDlsymAllocPoolSize = 1024; -static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; - -static inline bool IsInDlsymAllocPool(const void *ptr) { - uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]); -} - -static void *AllocateFromLocalPool(uptr size_in_bytes) { - uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; - void *mem = (void*)&alloc_memory_for_dlsym[allocated_for_dlsym]; - last_dlsym_alloc_size_in_words = size_in_words; - allocated_for_dlsym += size_in_words; - CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); - return mem; -} - -static void DeallocateFromLocalPool(const void *ptr) { - // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store - // error messages and instead uses malloc followed by free. To avoid pool - // exhaustion due to long object filenames, handle that special case here. - uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words; - void *prev_mem = (void*)&alloc_memory_for_dlsym[prev_offset]; - if (prev_mem == ptr) { - REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize); - allocated_for_dlsym = prev_offset; - last_dlsym_alloc_size_in_words = 0; - } -} - -static inline bool MaybeInDlsym() { - // Fuchsia doesn't use dlsym-based interceptors. - return !SANITIZER_FUCHSIA && asan_init_is_running; -} - -static inline bool UseLocalPool() { return MaybeInDlsym(); } - -static void *ReallocFromLocalPool(void *ptr, uptr size) { - const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); - void *new_ptr; - if (UNLIKELY(UseLocalPool())) { - new_ptr = AllocateFromLocalPool(size); - } else { +struct AsanDlSymAlloc { + static bool UseImpl() { return asan_init_is_running; } + static void *ReallocImpl(uptr size) { ENSURE_ASAN_INITED(); GET_STACK_TRACE_MALLOC; - new_ptr = asan_malloc(size, &stack); + return asan_realloc(nullptr, size, &stack); } - internal_memcpy(new_ptr, ptr, copy_size); - return new_ptr; -} +}; +using DlsymAlloc = DlSymAllocator; INTERCEPTOR(void, free, void *ptr) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) { - DeallocateFromLocalPool(ptr); - return; - } + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_STACK_TRACE_FREE; asan_free(ptr, &stack, FROM_MALLOC); } #if SANITIZER_INTERCEPT_CFREE INTERCEPTOR(void, cfree, void *ptr) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) - return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_STACK_TRACE_FREE; asan_free(ptr, &stack, FROM_MALLOC); } #endif // SANITIZER_INTERCEPT_CFREE INTERCEPTOR(void*, malloc, uptr size) { - if (UNLIKELY(UseLocalPool())) - // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. - return AllocateFromLocalPool(size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Allocate(size); ENSURE_ASAN_INITED(); GET_STACK_TRACE_MALLOC; return asan_malloc(size, &stack); } INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) { - if (UNLIKELY(UseLocalPool())) - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - return AllocateFromLocalPool(nmemb * size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); ENSURE_ASAN_INITED(); GET_STACK_TRACE_MALLOC; return asan_calloc(nmemb, size, &stack); } INTERCEPTOR(void*, realloc, void *ptr, uptr size) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) - return ReallocFromLocalPool(ptr, size); - if (UNLIKELY(UseLocalPool())) - return AllocateFromLocalPool(size); + if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Realloc(ptr, size); ENSURE_ASAN_INITED(); GET_STACK_TRACE_MALLOC; return asan_realloc(ptr, size, &stack); diff --git a/compiler-rt/lib/dfsan/dfsan_interceptors.cpp b/compiler-rt/lib/dfsan/dfsan_interceptors.cpp --- a/compiler-rt/lib/dfsan/dfsan_interceptors.cpp +++ b/compiler-rt/lib/dfsan/dfsan_interceptors.cpp @@ -17,6 +17,7 @@ #include "dfsan/dfsan.h" #include "dfsan/dfsan_thread.h" #include "interception/interception.h" +#include "sanitizer_common/sanitizer_allocator_dlsym.h" #include "sanitizer_common/sanitizer_allocator_interface.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_errno.h" @@ -26,11 +27,15 @@ using namespace __sanitizer; -namespace { +static bool interceptors_initialized; -bool interceptors_initialized; - -} // namespace +struct DfsanDlSymAlloc { + static bool UseImpl() { return !__dfsan::dfsan_inited; } + static void *ReallocImpl(uptr size) { + return __dfsan::dfsan_realloc(nullptr, size); + } +}; +using DlsymAlloc = DlSymAllocator; INTERCEPTOR(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size) { return __dfsan::dfsan_reallocarray(ptr, nmemb, size); @@ -47,63 +52,37 @@ return __dfsan::dfsan_aligned_alloc(alignment, size); } -static uptr allocated_for_dlsym; -static const uptr kDlsymAllocPoolSize = 1024; -static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; - -static bool IsInDlsymAllocPool(const void *ptr) { - uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - return off < sizeof(alloc_memory_for_dlsym); -} - -static void *AllocateFromLocalPool(uptr size_in_bytes) { - uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; - void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; - allocated_for_dlsym += size_in_words; - CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); - return mem; -} - INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) { - if (UNLIKELY(!__dfsan::dfsan_inited)) - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - return AllocateFromLocalPool(nmemb * size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); return __dfsan::dfsan_calloc(nmemb, size); } INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) { - uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); - void *new_ptr; - if (UNLIKELY(!__dfsan::dfsan_inited)) { - new_ptr = AllocateFromLocalPool(copy_size); - } else { - copy_size = size; - new_ptr = __dfsan::dfsan_malloc(copy_size); - } - internal_memcpy(new_ptr, ptr, copy_size); - return new_ptr; - } + if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Realloc(ptr, size); return __dfsan::dfsan_realloc(ptr, size); } INTERCEPTOR(void *, malloc, SIZE_T size) { - if (UNLIKELY(!__dfsan::dfsan_inited)) - // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. - return AllocateFromLocalPool(size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Allocate(size); return __dfsan::dfsan_malloc(size); } INTERCEPTOR(void, free, void *ptr) { - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) + if (!ptr) return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); return __dfsan::dfsan_deallocate(ptr); } INTERCEPTOR(void, cfree, void *ptr) { - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) + if (!ptr) return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); return __dfsan::dfsan_deallocate(ptr); } diff --git a/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp b/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp --- a/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp +++ b/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp @@ -14,6 +14,7 @@ #include "hwasan.h" #include "interception/interception.h" +#include "sanitizer_common/sanitizer_allocator_dlsym.h" #include "sanitizer_common/sanitizer_allocator_interface.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" @@ -21,23 +22,14 @@ using namespace __hwasan; -static uptr allocated_for_dlsym; -static const uptr kDlsymAllocPoolSize = 1024; -static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; - -static bool IsInDlsymAllocPool(const void *ptr) { - uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - return off < sizeof(alloc_memory_for_dlsym); -} - -static void *AllocateFromLocalPool(uptr size_in_bytes) { - uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; - void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; - allocated_for_dlsym += size_in_words; - CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); - return mem; -} - +struct HWasanDlsymAlloc { + static bool UseImpl() { return !hwasan_inited; } + static void *ReallocImpl(uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_realloc(nullptr, size, &stack); + } +}; +using DlsymAlloc = DlSymAllocator; extern "C" { SANITIZER_INTERFACE_ATTRIBUTE @@ -83,16 +75,20 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_free(void *ptr) { - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) + if (!ptr) return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_MALLOC_STACK_TRACE; hwasan_free(ptr, &stack); } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cfree(void *ptr) { - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) + if (!ptr) return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_MALLOC_STACK_TRACE; hwasan_free(ptr, &stack); } @@ -119,29 +115,16 @@ SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_calloc(uptr nmemb, uptr size) { - if (UNLIKELY(!hwasan_inited)) - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - return AllocateFromLocalPool(nmemb * size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); GET_MALLOC_STACK_TRACE; return hwasan_calloc(nmemb, size, &stack); } SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_realloc(void *ptr, uptr size) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) { - uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); - void *new_ptr; - if (UNLIKELY(!hwasan_inited)) { - new_ptr = AllocateFromLocalPool(copy_size); - } else { - copy_size = size; - GET_MALLOC_STACK_TRACE; - new_ptr = hwasan_malloc(copy_size, &stack); - } - internal_memcpy(new_ptr, ptr, copy_size); - return new_ptr; - } + if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Realloc(ptr, size); GET_MALLOC_STACK_TRACE; return hwasan_realloc(ptr, size, &stack); } @@ -156,9 +139,8 @@ void *__sanitizer_malloc(uptr size) { if (UNLIKELY(!hwasan_init_is_running)) ENSURE_HWASAN_INITED(); - if (UNLIKELY(!hwasan_inited)) - // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. - return AllocateFromLocalPool(size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Allocate(size); GET_MALLOC_STACK_TRACE; return hwasan_malloc(size, &stack); } diff --git a/compiler-rt/lib/lsan/lsan_interceptors.cpp b/compiler-rt/lib/lsan/lsan_interceptors.cpp --- a/compiler-rt/lib/lsan/lsan_interceptors.cpp +++ b/compiler-rt/lib/lsan/lsan_interceptors.cpp @@ -13,6 +13,7 @@ #include "interception/interception.h" #include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_dlsym.h" #include "sanitizer_common/sanitizer_allocator_report.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" @@ -43,6 +44,16 @@ int pthread_setspecific(unsigned key, const void *v); } +struct LsanDlSymAlloc { + static bool UseImpl() { return lsan_init_is_running; } + static void *ReallocImpl(uptr size) { + ENSURE_LSAN_INITED; + GET_STACK_TRACE_MALLOC; + return lsan_realloc(nullptr, size, stack); + } +}; +using DlsymAlloc = DlSymAllocator; + ///// Malloc/free interceptors. ///// namespace std { @@ -52,41 +63,34 @@ #if !SANITIZER_MAC INTERCEPTOR(void*, malloc, uptr size) { + if (DlsymAlloc::Use()) + return DlsymAlloc::Allocate(size); ENSURE_LSAN_INITED; GET_STACK_TRACE_MALLOC; return lsan_malloc(size, stack); } INTERCEPTOR(void, free, void *p) { + if (DlsymAlloc::PointerIsMine(p)) + return DlsymAlloc::Free(p); ENSURE_LSAN_INITED; lsan_free(p); } INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) { - // This hack is not required for Fuchsia because there are no dlsym calls - // involved in setting up interceptors. -#if !SANITIZER_FUCHSIA - if (lsan_init_is_running) { - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - const uptr kCallocPoolSize = 1024; - static uptr calloc_memory_for_dlsym[kCallocPoolSize]; - static uptr allocated; - uptr size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize; - void *mem = (void*)&calloc_memory_for_dlsym[allocated]; - allocated += size_in_words; - CHECK(allocated < kCallocPoolSize); - return mem; - } -#endif // !SANITIZER_FUCHSIA + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); ENSURE_LSAN_INITED; GET_STACK_TRACE_MALLOC; return lsan_calloc(nmemb, size, stack); } -INTERCEPTOR(void*, realloc, void *q, uptr size) { +INTERCEPTOR(void *, realloc, void *ptr, uptr size) { + if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Realloc(ptr, size); ENSURE_LSAN_INITED; GET_STACK_TRACE_MALLOC; - return lsan_realloc(q, size, stack); + return lsan_realloc(ptr, size, stack); } INTERCEPTOR(void*, reallocarray, void *q, uptr nmemb, uptr size) { diff --git a/compiler-rt/lib/memprof/memprof_malloc_linux.cpp b/compiler-rt/lib/memprof/memprof_malloc_linux.cpp --- a/compiler-rt/lib/memprof/memprof_malloc_linux.cpp +++ b/compiler-rt/lib/memprof/memprof_malloc_linux.cpp @@ -23,104 +23,58 @@ #include "memprof_internal.h" #include "memprof_stack.h" #include "sanitizer_common/sanitizer_allocator_checks.h" +#include "sanitizer_common/sanitizer_allocator_dlsym.h" #include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" // ---------------------- Replacement functions ---------------- {{{1 using namespace __memprof; -static uptr allocated_for_dlsym; -static uptr last_dlsym_alloc_size_in_words; -static const uptr kDlsymAllocPoolSize = 1024; -static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; - -static inline bool IsInDlsymAllocPool(const void *ptr) { - uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]); -} - -static void *AllocateFromLocalPool(uptr size_in_bytes) { - uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; - void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; - last_dlsym_alloc_size_in_words = size_in_words; - allocated_for_dlsym += size_in_words; - CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); - return mem; -} - -static void DeallocateFromLocalPool(const void *ptr) { - // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store - // error messages and instead uses malloc followed by free. To avoid pool - // exhaustion due to long object filenames, handle that special case here. - uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words; - void *prev_mem = (void *)&alloc_memory_for_dlsym[prev_offset]; - if (prev_mem == ptr) { - REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize); - allocated_for_dlsym = prev_offset; - last_dlsym_alloc_size_in_words = 0; - } -} - -static inline bool MaybeInDlsym() { return memprof_init_is_running; } - -static inline bool UseLocalPool() { return MaybeInDlsym(); } - -static void *ReallocFromLocalPool(void *ptr, uptr size) { - const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); - void *new_ptr; - if (UNLIKELY(UseLocalPool())) { - new_ptr = AllocateFromLocalPool(size); - } else { +struct MemprofDlsymAlloc { + static bool UseImpl() { return memprof_init_is_running; } + static void *ReallocImpl(uptr size) { ENSURE_MEMPROF_INITED(); - GET_STACK_TRACE_MALLOC; - new_ptr = memprof_malloc(size, &stack); + GET_STACK_TRACE_FREE; + return memprof_realloc(nullptr, size, &stack); } - internal_memcpy(new_ptr, ptr, copy_size); - return new_ptr; -} +}; +using DlsymAlloc = DlSymAllocator; INTERCEPTOR(void, free, void *ptr) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) { - DeallocateFromLocalPool(ptr); - return; - } + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_STACK_TRACE_FREE; memprof_free(ptr, &stack, FROM_MALLOC); } #if SANITIZER_INTERCEPT_CFREE INTERCEPTOR(void, cfree, void *ptr) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) - return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_STACK_TRACE_FREE; memprof_free(ptr, &stack, FROM_MALLOC); } #endif // SANITIZER_INTERCEPT_CFREE INTERCEPTOR(void *, malloc, uptr size) { - if (UNLIKELY(UseLocalPool())) - // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. - return AllocateFromLocalPool(size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Allocate(size); ENSURE_MEMPROF_INITED(); GET_STACK_TRACE_MALLOC; return memprof_malloc(size, &stack); } INTERCEPTOR(void *, calloc, uptr nmemb, uptr size) { - if (UNLIKELY(UseLocalPool())) - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - return AllocateFromLocalPool(nmemb * size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); ENSURE_MEMPROF_INITED(); GET_STACK_TRACE_MALLOC; return memprof_calloc(nmemb, size, &stack); } INTERCEPTOR(void *, realloc, void *ptr, uptr size) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) - return ReallocFromLocalPool(ptr, size); - if (UNLIKELY(UseLocalPool())) - return AllocateFromLocalPool(size); + if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Realloc(ptr, size); ENSURE_MEMPROF_INITED(); GET_STACK_TRACE_MALLOC; return memprof_realloc(ptr, size, &stack); diff --git a/compiler-rt/lib/msan/msan_interceptors.cpp b/compiler-rt/lib/msan/msan_interceptors.cpp --- a/compiler-rt/lib/msan/msan_interceptors.cpp +++ b/compiler-rt/lib/msan/msan_interceptors.cpp @@ -22,8 +22,8 @@ #include "msan_report.h" #include "msan_thread.h" #include "sanitizer_common/sanitizer_allocator.h" +#include "sanitizer_common/sanitizer_allocator_dlsym.h" #include "sanitizer_common/sanitizer_allocator_interface.h" -#include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_errno.h" @@ -75,22 +75,14 @@ return in_interceptor_scope; } -static uptr allocated_for_dlsym; -static const uptr kDlsymAllocPoolSize = 1024; -static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; - -static bool IsInDlsymAllocPool(const void *ptr) { - uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - return off < sizeof(alloc_memory_for_dlsym); -} - -static void *AllocateFromLocalPool(uptr size_in_bytes) { - uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; - void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; - allocated_for_dlsym += size_in_words; - CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); - return mem; -} +struct MsanDlsymAlloc { + static bool UseImpl() { return !msan_inited; } + static void *ReallocImpl(uptr size) { + GET_MALLOC_STACK_TRACE; + return msan_realloc(nullptr, size, &stack); + } +}; +using DlsymAlloc = DlSymAllocator; #define ENSURE_MSAN_INITED() do { \ CHECK(!msan_init_is_running); \ @@ -221,14 +213,20 @@ #endif INTERCEPTOR(void, free, void *ptr) { - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; + if (UNLIKELY(!ptr)) + return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_MALLOC_STACK_TRACE; MsanDeallocate(&stack, ptr); } #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD INTERCEPTOR(void, cfree, void *ptr) { - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; + if (UNLIKELY(!ptr)) + return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); GET_MALLOC_STACK_TRACE; MsanDeallocate(&stack, ptr); } @@ -879,27 +877,14 @@ INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) { GET_MALLOC_STACK_TRACE; - if (UNLIKELY(!msan_inited)) - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - return AllocateFromLocalPool(nmemb * size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); return msan_calloc(nmemb, size, &stack); } INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) { - if (UNLIKELY(IsInDlsymAllocPool(ptr))) { - uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); - void *new_ptr; - if (UNLIKELY(!msan_inited)) { - new_ptr = AllocateFromLocalPool(copy_size); - } else { - copy_size = size; - GET_MALLOC_STACK_TRACE; - new_ptr = msan_malloc(copy_size, &stack); - } - internal_memcpy(new_ptr, ptr, copy_size); - return new_ptr; - } + if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Realloc(ptr, size); GET_MALLOC_STACK_TRACE; return msan_realloc(ptr, size, &stack); } @@ -910,9 +895,8 @@ } INTERCEPTOR(void *, malloc, SIZE_T size) { - if (UNLIKELY(!msan_inited)) - // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. - return AllocateFromLocalPool(size); + if (DlsymAlloc::Use()) + return DlsymAlloc::Allocate(size); GET_MALLOC_STACK_TRACE; return msan_malloc(size, &stack); } diff --git a/compiler-rt/lib/sanitizer_common/CMakeLists.txt b/compiler-rt/lib/sanitizer_common/CMakeLists.txt --- a/compiler-rt/lib/sanitizer_common/CMakeLists.txt +++ b/compiler-rt/lib/sanitizer_common/CMakeLists.txt @@ -100,6 +100,7 @@ sanitizer_allocator.h sanitizer_allocator_checks.h sanitizer_allocator_combined.h + sanitizer_allocator_dlsym.h sanitizer_allocator_interface.h sanitizer_allocator_internal.h sanitizer_allocator_local_cache.h diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h @@ -0,0 +1,65 @@ +//===-- sanitizer_allocator_dlsym.h -----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Hack: Sanitizer initializer calls dlsym which may need to allocate and call +// back into uninitialized sanitizer. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_ALLOCATOR_DLSYM_H +#define SANITIZER_ALLOCATOR_DLSYM_H + +#include "sanitizer_allocator_internal.h" + +namespace __sanitizer { + +template +struct DlSymAllocator { + static bool Use() { + // Fuchsia doesn't use dlsym-based interceptors. + return !SANITIZER_FUCHSIA && UNLIKELY(Details::UseImpl()); + } + + static bool PointerIsMine(const void *ptr) { + // Fuchsia doesn't use dlsym-based interceptors. + return !SANITIZER_FUCHSIA && + UNLIKELY(internal_allocator()->FromPrimary(ptr)); + } + + static void *Allocate(uptr size_in_bytes) { + void *ptr = InternalAlloc(size_in_bytes, nullptr, kWordSize); + CHECK(internal_allocator()->FromPrimary(ptr)); + return ptr; + } + + static void *Callocate(SIZE_T nmemb, SIZE_T size) { + void *ptr = InternalCalloc(nmemb, size); + CHECK(internal_allocator()->FromPrimary(ptr)); + return ptr; + } + + static void Free(void *ptr) { InternalFree(ptr); } + + static void *Realloc(void *ptr, uptr size) { + CHECK(!ptr || internal_allocator()->FromPrimary(ptr)); + if (Use()) + return InternalRealloc(ptr, size); + void *new_ptr = Details::ReallocImpl(size); + if (ptr) { + internal_memcpy( + new_ptr, ptr, + Min(size, internal_allocator()->GetActuallyAllocatedSize(ptr))); + Free(ptr); + } + return new_ptr; + } +}; + +} // namespace __sanitizer + +#endif // SANITIZER_ALLOCATOR_DLSYM_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc b/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc --- a/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc +++ b/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc @@ -23,6 +23,7 @@ #include #include "interception/interception.h" +#include "sanitizer_common/sanitizer_allocator_dlsym.h" #include "sanitizer_common/sanitizer_mac.h" // Similar code is used in Google Perftools, @@ -192,20 +193,20 @@ return p; } +struct MzDlSymAlloc { + static bool UseImpl() { return !COMMON_MALLOC_SANITIZER_INITIALIZED; } + static void *ReallocImpl(uptr size) { + CHECK(0); + return nullptr; + } +}; +using DlsymAlloc = DlSymAllocator; + extern "C" SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_mz_calloc(malloc_zone_t *zone, size_t nmemb, size_t size) { - if (UNLIKELY(!COMMON_MALLOC_SANITIZER_INITIALIZED)) { - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - const size_t kCallocPoolSize = 1024; - static uptr calloc_memory_for_dlsym[kCallocPoolSize]; - static size_t allocated; - size_t size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize; - void *mem = (void*)&calloc_memory_for_dlsym[allocated]; - allocated += size_in_words; - CHECK(allocated < kCallocPoolSize); - return mem; - } + if (DlsymAlloc::Use()) + return DlsymAlloc::Callocate(nmemb, size); COMMON_MALLOC_CALLOC(nmemb, size); return p; } @@ -223,6 +224,8 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_mz_free(malloc_zone_t *zone, void *ptr) { if (!ptr) return; + if (DlsymAlloc::PointerIsMine(ptr)) + return DlsymAlloc::Free(ptr); COMMON_MALLOC_FREE(ptr); }