Index: compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator.cc =================================================================== --- compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator.cc +++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator.cc @@ -22,30 +22,47 @@ #if defined(SANITIZER_GO) || defined(SANITIZER_USE_MALLOC) # if SANITIZER_LINUX && !SANITIZER_ANDROID extern "C" void *__libc_malloc(uptr size); +extern "C" void *__libc_memalign(uptr alignment, uptr size); +extern "C" void *__libc_realloc(void *ptr, uptr size); extern "C" void __libc_free(void *ptr); -# define LIBC_MALLOC __libc_malloc -# define LIBC_FREE __libc_free # else # include -# define LIBC_MALLOC malloc -# define LIBC_FREE free +# define __libc_malloc malloc +static void *__libc_memalign(uptr alignment, uptr size) { + void *p; + uptr error = posix_memalign(&p, alignment, size); + if (error) return nullptr; + return p; +} +# define __libc_realloc realloc +# define __libc_free free # endif -static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache) { +static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, + uptr alignment) { + (void)cache; + if (alignment == 0) + return __libc_malloc(size); + else + return __libc_memalign(alignment, size); +} + +static void *RawInternalRealloc(void *ptr, uptr size, + InternalAllocatorCache *cache) { (void)cache; - return LIBC_MALLOC(size); + return __libc_realloc(ptr, size); } static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { (void)cache; - LIBC_FREE(ptr); + __libc_free(ptr); } InternalAllocator *internal_allocator() { return 0; } -#else // SANITIZER_GO +#else // defined(SANITIZER_GO) || defined(SANITIZER_USE_MALLOC) static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)]; static atomic_uint8_t internal_allocator_initialized; @@ -68,13 +85,26 @@ return internal_allocator_instance; } -static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache) { +static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, + uptr alignment) { + if (alignment == 0) alignment = 8; if (cache == 0) { SpinMutexLock l(&internal_allocator_cache_mu); - return internal_allocator()->Allocate(&internal_allocator_cache, size, 8, - false); + return internal_allocator()->Allocate(&internal_allocator_cache, size, + alignment, false); } - return internal_allocator()->Allocate(cache, size, 8, false); + return internal_allocator()->Allocate(cache, size, alignment, false); +} + +static void *RawInternalRealloc(void *ptr, uptr size, + InternalAllocatorCache *cache) { + uptr alignment = 8; + if (cache == 0) { + SpinMutexLock l(&internal_allocator_cache_mu); + return internal_allocator()->Reallocate(&internal_allocator_cache, ptr, + size, alignment); + } + return internal_allocator()->Reallocate(cache, ptr, size, alignment); } static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { @@ -85,20 +115,42 @@ internal_allocator()->Deallocate(cache, ptr); } -#endif // SANITIZER_GO +#endif // defined(SANITIZER_GO) || defined(SANITIZER_USE_MALLOC) const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull; -void *InternalAlloc(uptr size, InternalAllocatorCache *cache) { +void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) { if (size + sizeof(u64) < size) return nullptr; - void *p = RawInternalAlloc(size + sizeof(u64), cache); + void *p = RawInternalAlloc(size + sizeof(u64), cache, alignment); if (!p) return nullptr; ((u64*)p)[0] = kBlockMagic; return (char*)p + sizeof(u64); } +void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) { + if (!addr) + return InternalAlloc(size, cache); + if (size + sizeof(u64) < size) + return nullptr; + addr = (char*)addr - sizeof(u64); + size = size + sizeof(u64); + CHECK_EQ(kBlockMagic, ((u64*)addr)[0]); + void *p = RawInternalRealloc(addr, size, cache); + if (!p) + return nullptr; + return (char*)p + sizeof(u64); +} + +void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) { + if (CallocShouldReturnNullDueToOverflow(count, size)) + return internal_allocator()->ReturnNullOrDie(); + void *p = InternalAlloc(count * size, cache); + if (p) internal_memset(p, 0, count * size); + return p; +} + void InternalFree(void *addr, InternalAllocatorCache *cache) { if (!addr) return; Index: compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_internal.h =================================================================== --- compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_internal.h +++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_allocator_internal.h @@ -45,7 +45,12 @@ typedef CombinedAllocator > InternalAllocator; -void *InternalAlloc(uptr size, InternalAllocatorCache *cache = nullptr); +void *InternalAlloc(uptr size, InternalAllocatorCache *cache = nullptr, + uptr alignment = 0); +void *InternalRealloc(void *p, uptr size, + InternalAllocatorCache *cache = nullptr); +void *InternalCalloc(uptr countr, uptr size, + InternalAllocatorCache *cache = nullptr); void InternalFree(void *p, InternalAllocatorCache *cache = nullptr); InternalAllocator *internal_allocator(); Index: compiler-rt/trunk/lib/tsan/dd/dd_interceptors.cc =================================================================== --- compiler-rt/trunk/lib/tsan/dd/dd_interceptors.cc +++ compiler-rt/trunk/lib/tsan/dd/dd_interceptors.cc @@ -15,9 +15,6 @@ using namespace __dsan; -extern "C" void *__libc_malloc(uptr size); -extern "C" void __libc_free(void *ptr); - __attribute__((tls_model("initial-exec"))) static __thread Thread *thr; __attribute__((tls_model("initial-exec"))) Index: compiler-rt/trunk/lib/tsan/rtl/tsan_interceptors.h =================================================================== --- compiler-rt/trunk/lib/tsan/rtl/tsan_interceptors.h +++ compiler-rt/trunk/lib/tsan/rtl/tsan_interceptors.h @@ -46,12 +46,4 @@ #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) -#if SANITIZER_FREEBSD -#define __libc_free __free -#define __libc_malloc __malloc -#endif - -extern "C" void __libc_free(void *ptr); -extern "C" void *__libc_malloc(uptr size); - #endif // TSAN_INTERCEPTORS_H Index: compiler-rt/trunk/lib/tsan/rtl/tsan_interceptors.cc =================================================================== --- compiler-rt/trunk/lib/tsan/rtl/tsan_interceptors.cc +++ compiler-rt/trunk/lib/tsan/rtl/tsan_interceptors.cc @@ -41,20 +41,8 @@ #define stderr __stderrp #endif -#if SANITIZER_FREEBSD -#define __libc_realloc __realloc -#define __libc_calloc __calloc -#elif SANITIZER_MAC -#define __libc_malloc REAL(malloc) -#define __libc_realloc REAL(realloc) -#define __libc_calloc REAL(calloc) -#define __libc_free REAL(free) -#elif SANITIZER_ANDROID +#if SANITIZER_ANDROID #define __errno_location __errno -#define __libc_malloc REAL(malloc) -#define __libc_realloc REAL(realloc) -#define __libc_calloc REAL(calloc) -#define __libc_free REAL(free) #define mallopt(a, b) #endif @@ -109,10 +97,6 @@ extern "C" void _exit(int status); extern "C" int *__errno_location(); extern "C" int fileno_unlocked(void *stream); -#if !SANITIZER_ANDROID -extern "C" void *__libc_calloc(uptr size, uptr n); -extern "C" void *__libc_realloc(void *ptr, uptr size); -#endif extern "C" int dirfd(void *dirp); #if !SANITIZER_FREEBSD && !SANITIZER_ANDROID extern "C" int mallopt(int param, int value); @@ -396,7 +380,7 @@ Acquire(thr, pc, (uptr)arg); AtExitCtx *ctx = (AtExitCtx*)arg; ((void(*)(void *arg))ctx->f)(ctx->arg); - __libc_free(ctx); + InternalFree(ctx); } static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), @@ -422,7 +406,7 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), void *arg, void *dso) { - AtExitCtx *ctx = (AtExitCtx*)__libc_malloc(sizeof(AtExitCtx)); + AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); ctx->f = f; ctx->arg = arg; Release(thr, pc, (uptr)ctx); @@ -441,14 +425,14 @@ Acquire(thr, pc, (uptr)arg); AtExitCtx *ctx = (AtExitCtx*)arg; ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg); - __libc_free(ctx); + InternalFree(ctx); } TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { if (cur_thread()->in_symbolizer) return 0; SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); - AtExitCtx *ctx = (AtExitCtx*)__libc_malloc(sizeof(AtExitCtx)); + AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); ctx->f = (void(*)())f; ctx->arg = arg; Release(thr, pc, (uptr)ctx); @@ -601,7 +585,7 @@ #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, malloc, uptr size) { if (cur_thread()->in_symbolizer) - return __libc_malloc(size); + return InternalAlloc(size); void *p = 0; { SCOPED_INTERCEPTOR_RAW(malloc, size); @@ -618,7 +602,7 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { if (cur_thread()->in_symbolizer) - return __libc_calloc(size, n); + return InternalCalloc(size, n); void *p = 0; { SCOPED_INTERCEPTOR_RAW(calloc, size, n); @@ -630,7 +614,7 @@ TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { if (cur_thread()->in_symbolizer) - return __libc_realloc(p, size); + return InternalRealloc(p, size); if (p) invoke_free_hook(p); { @@ -645,7 +629,7 @@ if (p == 0) return; if (cur_thread()->in_symbolizer) - return __libc_free(p); + return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(free, p); user_free(thr, pc, p); @@ -655,7 +639,7 @@ if (p == 0) return; if (cur_thread()->in_symbolizer) - return __libc_free(p); + return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(cfree, p); user_free(thr, pc, p); Index: compiler-rt/trunk/lib/tsan/rtl/tsan_malloc_mac.cc =================================================================== --- compiler-rt/trunk/lib/tsan/rtl/tsan_malloc_mac.cc +++ compiler-rt/trunk/lib/tsan/rtl/tsan_malloc_mac.cc @@ -27,33 +27,28 @@ #define COMMON_MALLOC_MEMALIGN(alignment, size) \ void *p = \ user_alloc(cur_thread(), StackTrace::GetCurrentPc(), size, alignment) -#define COMMON_MALLOC_MALLOC(size) \ - if (cur_thread()->in_symbolizer) \ - return REAL(malloc)(size); \ - SCOPED_INTERCEPTOR_RAW(malloc, size); \ +#define COMMON_MALLOC_MALLOC(size) \ + if (cur_thread()->in_symbolizer) return InternalAlloc(size); \ + SCOPED_INTERCEPTOR_RAW(malloc, size); \ void *p = user_alloc(thr, pc, size) -#define COMMON_MALLOC_REALLOC(ptr, size) \ - if (cur_thread()->in_symbolizer) \ - return REAL(realloc)(ptr, size); \ - SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \ +#define COMMON_MALLOC_REALLOC(ptr, size) \ + if (cur_thread()->in_symbolizer) return InternalRealloc(ptr, size); \ + SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \ void *p = user_realloc(thr, pc, ptr, size) -#define COMMON_MALLOC_CALLOC(count, size) \ - if (cur_thread()->in_symbolizer) \ - return REAL(calloc)(count, size); \ - SCOPED_INTERCEPTOR_RAW(calloc, size, count); \ +#define COMMON_MALLOC_CALLOC(count, size) \ + if (cur_thread()->in_symbolizer) return InternalCalloc(count, size); \ + SCOPED_INTERCEPTOR_RAW(calloc, size, count); \ void *p = user_calloc(thr, pc, size, count) -#define COMMON_MALLOC_VALLOC(size) \ - if (cur_thread()->in_symbolizer) \ - return REAL(valloc)(size); \ - SCOPED_INTERCEPTOR_RAW(valloc, size); \ +#define COMMON_MALLOC_VALLOC(size) \ + if (cur_thread()->in_symbolizer) \ + return InternalAlloc(size, nullptr, GetPageSizeCached()); \ + SCOPED_INTERCEPTOR_RAW(valloc, size); \ void *p = user_alloc(thr, pc, size, GetPageSizeCached()) -#define COMMON_MALLOC_FREE(ptr) \ - if (cur_thread()->in_symbolizer) \ - return REAL(free)(ptr); \ - SCOPED_INTERCEPTOR_RAW(free, ptr); \ +#define COMMON_MALLOC_FREE(ptr) \ + if (cur_thread()->in_symbolizer) return InternalFree(ptr); \ + SCOPED_INTERCEPTOR_RAW(free, ptr); \ user_free(thr, pc, ptr) -#define COMMON_MALLOC_SIZE(ptr) \ - uptr size = user_alloc_usable_size(ptr); +#define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr); #define COMMON_MALLOC_FILL_STATS(zone, stats) #define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \ (void)zone_name; \ Index: compiler-rt/trunk/lib/tsan/rtl/tsan_new_delete.cc =================================================================== --- compiler-rt/trunk/lib/tsan/rtl/tsan_new_delete.cc +++ compiler-rt/trunk/lib/tsan/rtl/tsan_new_delete.cc @@ -23,14 +23,10 @@ DECLARE_REAL(void *, malloc, uptr size) DECLARE_REAL(void, free, void *ptr) -#if SANITIZER_MAC || SANITIZER_ANDROID -#define __libc_malloc REAL(malloc) -#define __libc_free REAL(free) -#endif #define OPERATOR_NEW_BODY(mangled_name) \ if (cur_thread()->in_symbolizer) \ - return __libc_malloc(size); \ + return InternalAlloc(size); \ void *p = 0; \ { \ SCOPED_INTERCEPTOR_RAW(mangled_name, size); \ @@ -66,7 +62,7 @@ #define OPERATOR_DELETE_BODY(mangled_name) \ if (ptr == 0) return; \ if (cur_thread()->in_symbolizer) \ - return __libc_free(ptr); \ + return InternalFree(ptr); \ invoke_free_hook(ptr); \ SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \ user_free(thr, pc, ptr);