[tsan] Switch to InternalAlloc everywhere __libc_malloc is currently used

[kubamracek]

Summary

This patch replaces all uses of __libc_malloc and friends with the internal allocator.

It seems that the only reason why we have calls to __libc_malloc in the first place was the lack of the internal allocator at the time. Using the internal allocator will also make sure that the system allocator is never used (this is the same behavior as ASan), and we don’t have to worry about working with unknown pointers coming from the system allocator.

[dvyukov]

LGTM

Thanks!

lib/sanitizer_common/sanitizer_allocator.cc

65

while you are here:

defined(SANITIZER_GO) || defined(SANITIZER_USE_MALLOC)

118

while you are here:

defined(SANITIZER_GO) || defined(SANITIZER_USE_MALLOC)

lib/sanitizer_common/sanitizer_allocator.cc

 #include "sanitizer_common.h"
 
 namespace __sanitizer {
 
 // ThreadSanitizer for Go uses libc malloc/free.
 #if defined(SANITIZER_GO) || defined(SANITIZER_USE_MALLOC)
 # if SANITIZER_LINUX && !SANITIZER_ANDROID
 extern "C" void *__libc_malloc(uptr size);
+extern "C" void *__libc_memalign(uptr alignment, uptr size);
+extern "C" void *__libc_realloc(void *ptr, uptr size);
 extern "C" void __libc_free(void *ptr);
-#  define LIBC_MALLOC __libc_malloc
-#  define LIBC_FREE __libc_free
 # else
 #  include <stdlib.h>
-#  define LIBC_MALLOC malloc
-#  define LIBC_FREE free
+#  define __libc_malloc malloc
+static void *__libc_memalign(uptr alignment, uptr size) {
+  void *p;
+  uptr error = posix_memalign(&p, alignment, size);
+  if (error) return nullptr;
+  return p;
+}
+#  define __libc_realloc realloc
+#  define __libc_free free
 # endif
 
-static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache) {
+static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache,
+                              uptr alignment) {
+  (void)cache;
+  if (alignment == 0)
+    return __libc_malloc(size);
+  else
+    return __libc_memalign(alignment, size);
+}
+
+static void *RawInternalRealloc(void *ptr, uptr size,
+                                InternalAllocatorCache *cache) {
   (void)cache;
-  return LIBC_MALLOC(size);
+  return __libc_realloc(ptr, size);
 }
 
 static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) {
   (void)cache;
-  LIBC_FREE(ptr);
+  __libc_free(ptr);
 }
 
 InternalAllocator *internal_allocator() {
   return 0;
 }
 
 #else // SANITIZER_GO
 
 static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)];
 static atomic_uint8_t internal_allocator_initialized;
 static StaticSpinMutex internal_alloc_init_mu;
 
 static InternalAllocatorCache internal_allocator_cache;
 static StaticSpinMutex internal_allocator_cache_mu;
 
 InternalAllocator *internal_allocator() {
   InternalAllocator *internal_allocator_instance =
       reinterpret_cast<InternalAllocator *>(&internal_alloc_placeholder);
   if (atomic_load(&internal_allocator_initialized, memory_order_acquire) == 0) {
     SpinMutexLock l(&internal_alloc_init_mu);
     if (atomic_load(&internal_allocator_initialized, memory_order_relaxed) ==
         0) {
       internal_allocator_instance->Init(/* may_return_null*/ false);
       atomic_store(&internal_allocator_initialized, 1, memory_order_release);
     }
   }
   return internal_allocator_instance;
 }
 
-static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache) {
+static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache,
+                              uptr alignment) {
+  if (alignment == 0) alignment = 8;
   if (cache == 0) {
     SpinMutexLock l(&internal_allocator_cache_mu);
-    return internal_allocator()->Allocate(&internal_allocator_cache, size, 8,
-                                          false);
+    return internal_allocator()->Allocate(&internal_allocator_cache, size,
+                                          alignment, false);
   }
-  return internal_allocator()->Allocate(cache, size, 8, false);
+  return internal_allocator()->Allocate(cache, size, alignment, false);
+}
+
+static void *RawInternalRealloc(void *ptr, uptr size,
+                                InternalAllocatorCache *cache) {
+  uptr alignment = 8;
+  if (cache == 0) {
+    SpinMutexLock l(&internal_allocator_cache_mu);
+    return internal_allocator()->Reallocate(&internal_allocator_cache, ptr,
+                                            size, alignment);
+  }
+  return internal_allocator()->Reallocate(cache, ptr, size, alignment);
 }
 
 static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) {
   if (!cache) {
     SpinMutexLock l(&internal_allocator_cache_mu);
     return internal_allocator()->Deallocate(&internal_allocator_cache, ptr);
   }
   internal_allocator()->Deallocate(cache, ptr);
 }
 
 #endif // SANITIZER_GO
 
 const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull;
 
-void *InternalAlloc(uptr size, InternalAllocatorCache *cache) {
+void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) {
   if (size + sizeof(u64) < size)
     return nullptr;
-  void *p = RawInternalAlloc(size + sizeof(u64), cache);
+  void *p = RawInternalAlloc(size + sizeof(u64), cache, alignment);
   if (!p)
     return nullptr;
   ((u64*)p)[0] = kBlockMagic;
   return (char*)p + sizeof(u64);
 }
 
+void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) {
+  if (!addr)
+    return InternalAlloc(size, cache);
+  if (size + sizeof(u64) < size)
+    return nullptr;
+  addr = (char*)addr - sizeof(u64);
+  size = size + sizeof(u64);
+  CHECK_EQ(kBlockMagic, ((u64*)addr)[0]);
+  void *p = RawInternalRealloc(addr, size, cache);
+  if (!p)
+    return nullptr;
+  return (char*)p + sizeof(u64);
+}
+
+void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) {
+  if (CallocShouldReturnNullDueToOverflow(count, size))
+    return internal_allocator()->ReturnNullOrDie();
+  void *p = InternalAlloc(count * size, cache);
+  if (p) internal_memset(p, 0, count * size);
+  return p;
+}
+
 void InternalFree(void *addr, InternalAllocatorCache *cache) {
   if (!addr)
     return;
   addr = (char*)addr - sizeof(u64);
   CHECK_EQ(kBlockMagic, ((u64*)addr)[0]);
   ((u64*)addr)[0] = 0;
   RawInternalFree(addr, cache);
 }

lib/sanitizer_common/sanitizer_allocator_internal.h

typedef SizeClassAllocator32<
     kInternalAllocatorRegionSizeLog, ByteMap> PrimaryInternalAllocator;
 
 typedef SizeClassAllocatorLocalCache<PrimaryInternalAllocator>
     InternalAllocatorCache;
 
 typedef CombinedAllocator<PrimaryInternalAllocator, InternalAllocatorCache,
                           LargeMmapAllocator<> > InternalAllocator;
 
-void *InternalAlloc(uptr size, InternalAllocatorCache *cache = nullptr);
+void *InternalAlloc(uptr size, InternalAllocatorCache *cache = nullptr,
+                    uptr alignment = 0);
+void *InternalRealloc(void *p, uptr size,
+                      InternalAllocatorCache *cache = nullptr);
+void *InternalCalloc(uptr countr, uptr size,
+                     InternalAllocatorCache *cache = nullptr);
 void InternalFree(void *p, InternalAllocatorCache *cache = nullptr);
 InternalAllocator *internal_allocator();
 
 enum InternalAllocEnum {
   INTERNAL_ALLOC
 };
 
 } // namespace __sanitizer
 
 inline void *operator new(__sanitizer::operator_new_size_type size,
                           InternalAllocEnum) {
   return InternalAlloc(size);
 }
 
 #endif // SANITIZER_ALLOCATOR_INTERNAL_H

lib/tsan/dd/dd_interceptors.cc

 #include "dd_rtl.h"
 #include "interception/interception.h"
 #include "sanitizer_common/sanitizer_procmaps.h"
 #include <pthread.h>
 #include <stdlib.h>
 
 using namespace __dsan;
 
-extern "C" void *__libc_malloc(uptr size);
-extern "C" void __libc_free(void *ptr);
-
 __attribute__((tls_model("initial-exec")))
 static __thread Thread *thr;
 __attribute__((tls_model("initial-exec")))
 static __thread volatile int initing;
 static bool inited;
 static uptr g_data_start;
 static uptr g_data_end;
 

lib/tsan/rtl/tsan_interceptors.h

 #define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \
     si.UserCallbackStart();
 
 #define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() \
     si.UserCallbackEnd();
 
 #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
 
-#if SANITIZER_FREEBSD
-#define __libc_free __free
-#define __libc_malloc __malloc
-#endif
-
-extern "C" void __libc_free(void *ptr);
-extern "C" void *__libc_malloc(uptr size);
-
 #endif  // TSAN_INTERCEPTORS_H

lib/tsan/rtl/tsan_interceptors.cc

 using namespace __tsan;  // NOLINT
 
 #if SANITIZER_FREEBSD || SANITIZER_MAC
 #define __errno_location __error
 #define stdout __stdoutp
 #define stderr __stderrp
 #endif
 
-#if SANITIZER_FREEBSD
-#define __libc_realloc __realloc
-#define __libc_calloc __calloc
-#elif SANITIZER_MAC
-#define __libc_malloc REAL(malloc)
-#define __libc_realloc REAL(realloc)
-#define __libc_calloc REAL(calloc)
-#define __libc_free REAL(free)
-#elif SANITIZER_ANDROID
+#if SANITIZER_ANDROID
 #define __errno_location __errno
-#define __libc_malloc REAL(malloc)
-#define __libc_realloc REAL(realloc)
-#define __libc_calloc REAL(calloc)
-#define __libc_free REAL(free)
 #define mallopt(a, b)
 #endif
 
 #if SANITIZER_LINUX || SANITIZER_FREEBSD
 #define PTHREAD_CREATE_DETACHED 1
 #elif SANITIZER_MAC
 #define PTHREAD_CREATE_DETACHED 2
 #endif
extern "C" int pthread_sigmask(int how, const __sanitizer_sigset_t *set,
                                __sanitizer_sigset_t *oldset);
 DECLARE_REAL(int, fflush, __sanitizer_FILE *fp)
 DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)
 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
 extern "C" void *pthread_self();
 extern "C" void _exit(int status);
 extern "C" int *__errno_location();
 extern "C" int fileno_unlocked(void *stream);
-#if !SANITIZER_ANDROID
-extern "C" void *__libc_calloc(uptr size, uptr n);
-extern "C" void *__libc_realloc(void *ptr, uptr size);
-#endif
 extern "C" int dirfd(void *dirp);
 #if !SANITIZER_FREEBSD && !SANITIZER_ANDROID
 extern "C" int mallopt(int param, int value);
 #endif
 extern __sanitizer_FILE *stdout, *stderr;
 #if !SANITIZER_FREEBSD && !SANITIZER_MAC
 const int PTHREAD_MUTEX_RECURSIVE = 1;
 const int PTHREAD_MUTEX_RECURSIVE_NP = 1;

 };
 
 static void at_exit_wrapper(void *arg) {
   ThreadState *thr = cur_thread();
   uptr pc = 0;
   Acquire(thr, pc, (uptr)arg);
   AtExitCtx *ctx = (AtExitCtx*)arg;
   ((void(*)(void *arg))ctx->f)(ctx->arg);
-  __libc_free(ctx);
+  InternalFree(ctx);
 }
 
 static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(),
       void *arg, void *dso);
 
 #if !SANITIZER_ANDROID
 TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
   if (cur_thread()->in_symbolizer)
TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) {
   if (cur_thread()->in_symbolizer)
     return 0;
   SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso);
   return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso);
 }
 
 static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(),
       void *arg, void *dso) {
-  AtExitCtx *ctx = (AtExitCtx*)__libc_malloc(sizeof(AtExitCtx));
+  AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx));
   ctx->f = f;
   ctx->arg = arg;
   Release(thr, pc, (uptr)ctx);
   // Memory allocation in __cxa_atexit will race with free during exit,
   // because we do not see synchronization around atexit callback list.
   ThreadIgnoreBegin(thr, pc);
   int res = REAL(__cxa_atexit)(at_exit_wrapper, ctx, dso);
   ThreadIgnoreEnd(thr, pc);
   return res;
 }
 
 #if !SANITIZER_MAC
 static void on_exit_wrapper(int status, void *arg) {
   ThreadState *thr = cur_thread();
   uptr pc = 0;
   Acquire(thr, pc, (uptr)arg);
   AtExitCtx *ctx = (AtExitCtx*)arg;
   ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg);
-  __libc_free(ctx);
+  InternalFree(ctx);
 }
 
 TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) {
   if (cur_thread()->in_symbolizer)
     return 0;
   SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg);
-  AtExitCtx *ctx = (AtExitCtx*)__libc_malloc(sizeof(AtExitCtx));
+  AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx));
   ctx->f = (void(*)())f;
   ctx->arg = arg;
   Release(thr, pc, (uptr)ctx);
   // Memory allocation in __cxa_atexit will race with free during exit,
   // because we do not see synchronization around atexit callback list.
   ThreadIgnoreBegin(thr, pc);
   int res = REAL(on_exit)(on_exit_wrapper, ctx);
   ThreadIgnoreEnd(thr, pc);
TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) {
   }
   LongJmp(cur_thread(), env);
   REAL(siglongjmp)(env, val);
 }
 
 #if !SANITIZER_MAC
 TSAN_INTERCEPTOR(void*, malloc, uptr size) {
   if (cur_thread()->in_symbolizer)
-    return __libc_malloc(size);
+    return InternalAlloc(size);
   void *p = 0;
   {
     SCOPED_INTERCEPTOR_RAW(malloc, size);
     p = user_alloc(thr, pc, size);
   }
   invoke_malloc_hook(p, size);
   return p;
 }
 
 TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
   SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz);
   return user_alloc(thr, pc, sz, align);
 }
 
 TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
   if (cur_thread()->in_symbolizer)
-    return __libc_calloc(size, n);
+    return InternalCalloc(size, n);
   void *p = 0;
   {
     SCOPED_INTERCEPTOR_RAW(calloc, size, n);
     p = user_calloc(thr, pc, size, n);
   }
   invoke_malloc_hook(p, n * size);
   return p;
 }
 
 TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) {
   if (cur_thread()->in_symbolizer)
-    return __libc_realloc(p, size);
+    return InternalRealloc(p, size);
   if (p)
     invoke_free_hook(p);
   {
     SCOPED_INTERCEPTOR_RAW(realloc, p, size);
     p = user_realloc(thr, pc, p, size);
   }
   invoke_malloc_hook(p, size);
   return p;
 }
 
 TSAN_INTERCEPTOR(void, free, void *p) {
   if (p == 0)
     return;
   if (cur_thread()->in_symbolizer)
-    return __libc_free(p);
+    return InternalFree(p);
   invoke_free_hook(p);
   SCOPED_INTERCEPTOR_RAW(free, p);
   user_free(thr, pc, p);
 }
 
 TSAN_INTERCEPTOR(void, cfree, void *p) {
   if (p == 0)
     return;
   if (cur_thread()->in_symbolizer)
-    return __libc_free(p);
+    return InternalFree(p);
   invoke_free_hook(p);
   SCOPED_INTERCEPTOR_RAW(cfree, p);
   user_free(thr, pc, p);
 }
 
 TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) {
   SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p);
   return user_alloc_usable_size(p);

lib/tsan/rtl/tsan_malloc_mac.cc

 #define COMMON_MALLOC_ZONE_NAME "tsan"
 #define COMMON_MALLOC_ENTER()
 #define COMMON_MALLOC_SANITIZER_INITIALIZED (cur_thread()->is_inited)
 #define COMMON_MALLOC_FORCE_LOCK()
 #define COMMON_MALLOC_FORCE_UNLOCK()
 #define COMMON_MALLOC_MEMALIGN(alignment, size) \
   void *p =                                     \
       user_alloc(cur_thread(), StackTrace::GetCurrentPc(), size, alignment)
-#define COMMON_MALLOC_MALLOC(size)      \
-  if (cur_thread()->in_symbolizer)      \
-    return REAL(malloc)(size);          \
-  SCOPED_INTERCEPTOR_RAW(malloc, size); \
+#define COMMON_MALLOC_MALLOC(size)                             \
+  if (cur_thread()->in_symbolizer) return InternalAlloc(size); \
+  SCOPED_INTERCEPTOR_RAW(malloc, size);                        \
   void *p = user_alloc(thr, pc, size)
-#define COMMON_MALLOC_REALLOC(ptr, size)      \
-  if (cur_thread()->in_symbolizer)            \
-    return REAL(realloc)(ptr, size);          \
-  SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \
+#define COMMON_MALLOC_REALLOC(ptr, size)                              \
+  if (cur_thread()->in_symbolizer) return InternalRealloc(ptr, size); \
+  SCOPED_INTERCEPTOR_RAW(realloc, ptr, size);                         \
   void *p = user_realloc(thr, pc, ptr, size)
-#define COMMON_MALLOC_CALLOC(count, size)      \
-  if (cur_thread()->in_symbolizer)             \
-    return REAL(calloc)(count, size);          \
-  SCOPED_INTERCEPTOR_RAW(calloc, size, count); \
+#define COMMON_MALLOC_CALLOC(count, size)                              \
+  if (cur_thread()->in_symbolizer) return InternalCalloc(count, size); \
+  SCOPED_INTERCEPTOR_RAW(calloc, size, count);                         \
   void *p = user_calloc(thr, pc, size, count)
-#define COMMON_MALLOC_VALLOC(size)                          \
-  if (cur_thread()->in_symbolizer)                          \
-    return REAL(valloc)(size);                              \
-  SCOPED_INTERCEPTOR_RAW(valloc, size);                     \
+#define COMMON_MALLOC_VALLOC(size)                            \
+  if (cur_thread()->in_symbolizer)                            \
+    return InternalAlloc(size, nullptr, GetPageSizeCached()); \
+  SCOPED_INTERCEPTOR_RAW(valloc, size);                       \
   void *p = user_alloc(thr, pc, size, GetPageSizeCached())
-#define COMMON_MALLOC_FREE(ptr)      \
-  if (cur_thread()->in_symbolizer)   \
-    return REAL(free)(ptr);          \
-  SCOPED_INTERCEPTOR_RAW(free, ptr); \
+#define COMMON_MALLOC_FREE(ptr)                              \
+  if (cur_thread()->in_symbolizer) return InternalFree(ptr); \
+  SCOPED_INTERCEPTOR_RAW(free, ptr);                         \
   user_free(thr, pc, ptr)
-#define COMMON_MALLOC_SIZE(ptr) \
-  uptr size = user_alloc_usable_size(ptr);
+#define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr);
 #define COMMON_MALLOC_FILL_STATS(zone, stats)
 #define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \
   (void)zone_name; \
   Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr);
 #define COMMON_MALLOC_NAMESPACE __tsan
 
 #include "sanitizer_common/sanitizer_malloc_mac.inc"
 
 #endif

lib/tsan/rtl/tsan_new_delete.cc

 using namespace __tsan;  // NOLINT
 
 namespace std {
 struct nothrow_t {};
 }  // namespace std
 
 DECLARE_REAL(void *, malloc, uptr size)
 DECLARE_REAL(void, free, void *ptr)
-#if SANITIZER_MAC || SANITIZER_ANDROID
-#define __libc_malloc REAL(malloc)
-#define __libc_free REAL(free)
-#endif
 
 #define OPERATOR_NEW_BODY(mangled_name) \
   if (cur_thread()->in_symbolizer) \
-    return __libc_malloc(size); \
+    return InternalAlloc(size); \
   void *p = 0; \
   {  \
     SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
     p = user_alloc(thr, pc, size); \
   }  \
   invoke_malloc_hook(p, size);  \
   return p;
 

 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&);
 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) {
   OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t);
 }
 
 #define OPERATOR_DELETE_BODY(mangled_name) \
   if (ptr == 0) return;  \
   if (cur_thread()->in_symbolizer) \
-    return __libc_free(ptr); \
+    return InternalFree(ptr); \
   invoke_free_hook(ptr);  \
   SCOPED_INTERCEPTOR_RAW(mangled_name, ptr);  \
   user_free(thr, pc, ptr);
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void operator delete(void *ptr) NOEXCEPT;
 void operator delete(void *ptr) NOEXCEPT {
   OPERATOR_DELETE_BODY(_ZdlPv);