[sanitizer] Switch dlsym hack to internal_allocator

[vitalybuka]

Summary

Since glibc 2.34, dlsym does

1. malloc 1
2. malloc 2
3. free pointer from malloc 1
4. free pointer from malloc 2

These sequence was not handled by trivial dlsym hack.

This fixes https://bugs.llvm.org/show_bug.cgi?id=52278

[vitalybuka]

compiler-rt/lib/asan/asan_malloc_linux.cpp

33

I realy don't like were we are going. What if we need 3 soon?

Any idea why we can't use internal_allocator()?

if we can't use internal allocator here I propose the following to simplify this.

1. AllocateFromLocalPool store size at (return value - 1) location.
2. so DeallocateFromLocalPool can find size from there.
3. To create byte(or bit) map to mark used words.
4. AllocateFromLocalPool will use last_dlsym_alloc_size_in_words_index to walkback and find start of unused bytest (or maybe just remove last_dlsym_alloc_size_in_words_index and scan full range)

[vitalybuka]

compiler-rt/lib/asan/asan_malloc_linux.cpp

33

Please let me know if you are going to implement suggested changes.
If if not, you are fixing a real issue, and I will do that myself.

[hjl.tools]

compiler-rt/lib/asan/asan_malloc_linux.cpp

33

Please let me know if you are going to implement suggested changes.
If if not, you are fixing a real issue, and I will do that myself.

I am not familiar with the code. Could you please fix it? Thanks.

[vitalybuka]

upload

[vitalybuka]

remove global

[morehouse]

compiler-rt/lib/memprof/memprof_malloc_linux.cpp

37–38

Shouldn't this be GET_STACK_TRACE_MALLOC?

[eugenis]

compiler-rt/lib/asan/asan_malloc_linux.cpp

37

Why do we need to specialize ReallocImpl? And do we even need stack traces here?

[vitalybuka]

Add root region hooks

[vitalybuka]

compiler-rt/lib/asan/asan_malloc_linux.cpp

37

This one is annoying. To keep behavioral closer to original I use root regions.
What I see is that "dlsym hack" allocations contains pointers to later regular dlerror allocations.

it would be nice to investigate if dlerror is special. If so we can remove regions in followup patches
and add dlerror interceptors to lsan and asan.

[eugenis]

compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h

68

I'd rather call OnFree(p) before deallocating, even if it does not matter for lsan.
Or does it?

[vitalybuka]

compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h

68

we do on like 52

[vitalybuka]

compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h

68

if the question about Details::OnAllocate Details::OnFree order then
the point to keep at least one of buffers, ptr or new_ptr, in lsan roots.

[vitalybuka]

redo Realloc

[eugenis]

LGTM

compiler-rt/lib/asan/asan_malloc_linux.cpp

 #if SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX || \
     SANITIZER_NETBSD || SANITIZER_SOLARIS
 
 #  include "asan_allocator.h"
 #  include "asan_interceptors.h"
 #  include "asan_internal.h"
 #  include "asan_stack.h"
 #  include "sanitizer_common/sanitizer_allocator_checks.h"
+#  include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #  include "sanitizer_common/sanitizer_errno.h"
 #  include "sanitizer_common/sanitizer_tls_get_addr.h"
 
 // ---------------------- Replacement functions ---------------- {{{1
 using namespace __asan;
 
-static uptr allocated_for_dlsym;
-static uptr last_dlsym_alloc_size_in_words;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static inline bool IsInDlsymAllocPool(const void *ptr) {
-  uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-  return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
-  uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
-  void *mem = (void*)&alloc_memory_for_dlsym[allocated_for_dlsym];
-  last_dlsym_alloc_size_in_words = size_in_words;
-  allocated_for_dlsym += size_in_words;
-  CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
-  return mem;
-}
-
-static void DeallocateFromLocalPool(const void *ptr) {
-  // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store
-  // error messages and instead uses malloc followed by free. To avoid pool
-  // exhaustion due to long object filenames, handle that special case here.
-  uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words;
-  void *prev_mem = (void*)&alloc_memory_for_dlsym[prev_offset];
-  if (prev_mem == ptr) {
-    REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize);
-    allocated_for_dlsym = prev_offset;
-    last_dlsym_alloc_size_in_words = 0;
-  }
-}
-
-static inline bool MaybeInDlsym() {
-  // Fuchsia doesn't use dlsym-based interceptors.
-  return !SANITIZER_FUCHSIA && asan_init_is_running;
-}
-
-static inline bool UseLocalPool() { return MaybeInDlsym(); }
-
-static void *ReallocFromLocalPool(void *ptr, uptr size) {
-  const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-  const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
-  void *new_ptr;
-  if (UNLIKELY(UseLocalPool())) {
-    new_ptr = AllocateFromLocalPool(size);
-  } else {
+struct AsanDlSymAllocator : public DlSymAllocator<AsanDlSymAllocator> {
+  static bool UseImpl() { return asan_init_is_running; }
+  static void *ReallocImpl(uptr size) {
     ENSURE_ASAN_INITED();
     GET_STACK_TRACE_MALLOC;
-    new_ptr = asan_malloc(size, &stack);
-  }
-  internal_memcpy(new_ptr, ptr, copy_size);
-  return new_ptr;
+    return asan_realloc(nullptr, size, &stack);
   }
+};
+
+static AsanDlSymAllocator dlsym_allocator;
 
 INTERCEPTOR(void, free, void *ptr) {
-  if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
-    DeallocateFromLocalPool(ptr);
-    return;
-  }
+  if (dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Free(ptr);
   GET_STACK_TRACE_FREE;
   asan_free(ptr, &stack, FROM_MALLOC);
 }
 
 #if SANITIZER_INTERCEPT_CFREE
 INTERCEPTOR(void, cfree, void *ptr) {
-  if (UNLIKELY(IsInDlsymAllocPool(ptr)))
-    return;
+  if (dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Free(ptr);
   GET_STACK_TRACE_FREE;
   asan_free(ptr, &stack, FROM_MALLOC);
 }
 #endif // SANITIZER_INTERCEPT_CFREE
 
 INTERCEPTOR(void*, malloc, uptr size) {
-  if (UNLIKELY(UseLocalPool()))
-    // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(size);
+  if (dlsym_allocator.Use())
+    return dlsym_allocator.Allocate(size);
   ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
   return asan_malloc(size, &stack);
 }
 
 INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
-  if (UNLIKELY(UseLocalPool()))
-    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(nmemb * size);
+  if (dlsym_allocator.Use())
+    return dlsym_allocator.Callocate(nmemb, size);
   ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
   return asan_calloc(nmemb, size, &stack);
 }
 
 INTERCEPTOR(void*, realloc, void *ptr, uptr size) {
-  if (UNLIKELY(IsInDlsymAllocPool(ptr)))
-    return ReallocFromLocalPool(ptr, size);
-  if (UNLIKELY(UseLocalPool()))
-    return AllocateFromLocalPool(size);
+  if (dlsym_allocator.Use() || dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Realloc(ptr, size);
   ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
   return asan_realloc(ptr, size, &stack);
 }
 
 #if SANITIZER_INTERCEPT_REALLOCARRAY
 INTERCEPTOR(void*, reallocarray, void *ptr, uptr nmemb, uptr size) {
   ENSURE_ASAN_INITED();

compiler-rt/lib/dfsan/dfsan_interceptors.cpp

 //===----------------------------------------------------------------------===//
 
 #include <sys/syscall.h>
 #include <unistd.h>
 
 #include "dfsan/dfsan.h"
 #include "dfsan/dfsan_thread.h"
 #include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
 #include "sanitizer_common/sanitizer_posix.h"
 #include "sanitizer_common/sanitizer_tls_get_addr.h"
 
 using namespace __sanitizer;
 
-namespace {
+static bool interceptors_initialized;
 
-bool interceptors_initialized;
+struct DfsanDlSymAllocator : public DlSymAllocator<DfsanDlSymAllocator> {
+  static bool UseImpl() { return !__dfsan::dfsan_inited; }
+  static void *ReallocImpl(uptr size) {
+    return __dfsan::dfsan_realloc(nullptr, size);
+  }
+};
 
-}  // namespace
+static DfsanDlSymAllocator dlsym_allocator;
 
 INTERCEPTOR(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size) {
   return __dfsan::dfsan_reallocarray(ptr, nmemb, size);
 }
 
 INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) {
   void *ptr = __dfsan::dfsan_memalign(alignment, size);
   if (ptr)
     DTLS_on_libc_memalign(ptr, size);
   return ptr;
 }
 
 INTERCEPTOR(void *, aligned_alloc, SIZE_T alignment, SIZE_T size) {
   return __dfsan::dfsan_aligned_alloc(alignment, size);
 }
 
-static uptr allocated_for_dlsym;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static bool IsInDlsymAllocPool(const void *ptr) {
-  uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-  return off < sizeof(alloc_memory_for_dlsym);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
-  uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
-  void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
-  allocated_for_dlsym += size_in_words;
-  CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
-  return mem;
-}
-
 INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) {
-  if (UNLIKELY(!__dfsan::dfsan_inited))
-    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(nmemb * size);
+  if (dlsym_allocator.Use())
+    return dlsym_allocator.Callocate(nmemb, size);
   return __dfsan::dfsan_calloc(nmemb, size);
 }
 
 INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) {
-  if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
-    uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-    uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
-    void *new_ptr;
-    if (UNLIKELY(!__dfsan::dfsan_inited)) {
-      new_ptr = AllocateFromLocalPool(copy_size);
-    } else {
-      copy_size = size;
-      new_ptr = __dfsan::dfsan_malloc(copy_size);
-    }
-    internal_memcpy(new_ptr, ptr, copy_size);
-    return new_ptr;
-  }
+  if (dlsym_allocator.Use() || dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Realloc(ptr, size);
   return __dfsan::dfsan_realloc(ptr, size);
 }
 
 INTERCEPTOR(void *, malloc, SIZE_T size) {
-  if (UNLIKELY(!__dfsan::dfsan_inited))
-    // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(size);
+  if (dlsym_allocator.Use())
+    return dlsym_allocator.Allocate(size);
   return __dfsan::dfsan_malloc(size);
 }
 
 INTERCEPTOR(void, free, void *ptr) {
-  if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+  if (!ptr)
     return;
+  if (dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Free(ptr);
   return __dfsan::dfsan_deallocate(ptr);
 }
 
 INTERCEPTOR(void, cfree, void *ptr) {
-  if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+  if (!ptr)
     return;
+  if (dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Free(ptr);
   return __dfsan::dfsan_deallocate(ptr);
 }
 
 INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) {
   CHECK_NE(memptr, 0);
   int res = __dfsan::dfsan_posix_memalign(memptr, alignment, size);
   if (!res)
     dfsan_set_label(0, memptr, sizeof(*memptr));

compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp

 //===-- hwasan_allocation_functions.cpp -----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of HWAddressSanitizer.
 //
 // Definitions for __sanitizer allocation functions.
 //
 //===----------------------------------------------------------------------===//
 
 #include "hwasan.h"
 #include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
 #include "sanitizer_common/sanitizer_tls_get_addr.h"
 
 #if !SANITIZER_FUCHSIA
 
 using namespace __hwasan;
 
-static uptr allocated_for_dlsym;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static bool IsInDlsymAllocPool(const void *ptr) {
-  uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-  return off < sizeof(alloc_memory_for_dlsym);
+struct HWasanDlSymAllocator : public DlSymAllocator<HWasanDlSymAllocator> {
+  static bool UseImpl() { return !hwasan_inited; }
+  static void *ReallocImpl(uptr size) {
+    GET_MALLOC_STACK_TRACE;
+    return hwasan_realloc(nullptr, size, &stack);
   }
+};
 
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
-  uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
-  void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
-  allocated_for_dlsym += size_in_words;
-  CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
-  return mem;
-}
+static HWasanDlSymAllocator dlsym_allocator;
 
 extern "C" {
 
 SANITIZER_INTERFACE_ATTRIBUTE
 int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) {
   GET_MALLOC_STACK_TRACE;
   CHECK_NE(memptr, 0);
   int res = hwasan_posix_memalign(memptr, alignment, size, &stack);

 SANITIZER_INTERFACE_ATTRIBUTE
 void *__sanitizer_pvalloc(uptr size) {
   GET_MALLOC_STACK_TRACE;
   return hwasan_pvalloc(size, &stack);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void __sanitizer_free(void *ptr) {
-  if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+  if (!ptr)
     return;
+  if (dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Free(ptr);
   GET_MALLOC_STACK_TRACE;
   hwasan_free(ptr, &stack);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void __sanitizer_cfree(void *ptr) {
-  if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+  if (!ptr)
     return;
+  if (dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Free(ptr);
   GET_MALLOC_STACK_TRACE;
   hwasan_free(ptr, &stack);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 uptr __sanitizer_malloc_usable_size(const void *ptr) {
   return __sanitizer_get_allocated_size(ptr);
 }

 
 SANITIZER_INTERFACE_ATTRIBUTE
 void __sanitizer_malloc_stats(void) {
   // FIXME: implement, but don't call REAL(malloc_stats)!
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__sanitizer_calloc(uptr nmemb, uptr size) {
-  if (UNLIKELY(!hwasan_inited))
-    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(nmemb * size);
+  if (dlsym_allocator.Use())
+    return dlsym_allocator.Callocate(nmemb, size);
   GET_MALLOC_STACK_TRACE;
   return hwasan_calloc(nmemb, size, &stack);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__sanitizer_realloc(void *ptr, uptr size) {
-  if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
-    uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-    uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
-    void *new_ptr;
-    if (UNLIKELY(!hwasan_inited)) {
-      new_ptr = AllocateFromLocalPool(copy_size);
-    } else {
-      copy_size = size;
-      GET_MALLOC_STACK_TRACE;
-      new_ptr = hwasan_malloc(copy_size, &stack);
-    }
-    internal_memcpy(new_ptr, ptr, copy_size);
-    return new_ptr;
-  }
+  if (dlsym_allocator.Use() || dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Realloc(ptr, size);
   GET_MALLOC_STACK_TRACE;
   return hwasan_realloc(ptr, size, &stack);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) {
   GET_MALLOC_STACK_TRACE;
   return hwasan_reallocarray(ptr, nmemb, size, &stack);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__sanitizer_malloc(uptr size) {
   if (UNLIKELY(!hwasan_init_is_running))
     ENSURE_HWASAN_INITED();
-  if (UNLIKELY(!hwasan_inited))
-    // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(size);
+  if (dlsym_allocator.Use())
+    return dlsym_allocator.Allocate(size);
   GET_MALLOC_STACK_TRACE;
   return hwasan_malloc(size, &stack);
 }
 
 }  // extern "C"
 
 #if HWASAN_WITH_INTERCEPTORS
 #  define INTERCEPTOR_ALIAS(RET, FN, ARGS...)                                 \

compiler-rt/lib/lsan/lsan_interceptors.cpp

 //=-- lsan_interceptors.cpp -----------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of LeakSanitizer.
 // Interceptors for standalone LSan.
 //
 //===----------------------------------------------------------------------===//
 
 #include "interception/interception.h"
 #include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #include "sanitizer_common/sanitizer_allocator_report.h"
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_linux.h"
 #include "sanitizer_common/sanitizer_platform_interceptors.h"
 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h"

 extern "C" {
 int pthread_attr_init(void *attr);
 int pthread_attr_destroy(void *attr);
 int pthread_attr_getdetachstate(void *attr, int *v);
 int pthread_key_create(unsigned *key, void (*destructor)(void* v));
 int pthread_setspecific(unsigned key, const void *v);
 }
 
+struct LsanDlSymAllocator : public DlSymAllocator<LsanDlSymAllocator> {
+  static bool UseImpl() { return lsan_init_is_running; }
+  static void *ReallocImpl(uptr size) {
+    ENSURE_LSAN_INITED;
+    GET_STACK_TRACE_MALLOC;
+    return lsan_realloc(nullptr, size, stack);
+  }
+};
+
+static LsanDlSymAllocator dlsym_allocator;
+
 ///// Malloc/free interceptors. /////
 
 namespace std {
   struct nothrow_t;
   enum class align_val_t: size_t;
 }
 
 #if !SANITIZER_MAC
 INTERCEPTOR(void*, malloc, uptr size) {
+  if (dlsym_allocator.Use())
+    return dlsym_allocator.Allocate(size);
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
   return lsan_malloc(size, stack);
 }
 
 INTERCEPTOR(void, free, void *p) {
+  if (dlsym_allocator.PointerIsMine(p))
+    return dlsym_allocator.Free(p);
   ENSURE_LSAN_INITED;
   lsan_free(p);
 }
 
 INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
-  // This hack is not required for Fuchsia because there are no dlsym calls
-  // involved in setting up interceptors.
-#if !SANITIZER_FUCHSIA
-  if (lsan_init_is_running) {
-    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
-    const uptr kCallocPoolSize = 1024;
-    static uptr calloc_memory_for_dlsym[kCallocPoolSize];
-    static uptr allocated;
-    uptr size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
-    void *mem = (void*)&calloc_memory_for_dlsym[allocated];
-    allocated += size_in_words;
-    CHECK(allocated < kCallocPoolSize);
-    return mem;
-  }
-#endif  // !SANITIZER_FUCHSIA
+  if (dlsym_allocator.Use())
+    return dlsym_allocator.Callocate(nmemb, size);
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
   return lsan_calloc(nmemb, size, stack);
 }
 
-INTERCEPTOR(void*, realloc, void *q, uptr size) {
+INTERCEPTOR(void *, realloc, void *ptr, uptr size) {
+  if (dlsym_allocator.Use() || dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Realloc(ptr, size);
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
-  return lsan_realloc(q, size, stack);
+  return lsan_realloc(ptr, size, stack);
 }
 
 INTERCEPTOR(void*, reallocarray, void *q, uptr nmemb, uptr size) {
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
   return lsan_reallocarray(q, nmemb, size, stack);
 }
 

compiler-rt/lib/memprof/memprof_malloc_linux.cpp

 #error Unsupported OS
 #endif
 
 #include "memprof_allocator.h"
 #include "memprof_interceptors.h"
 #include "memprof_internal.h"
 #include "memprof_stack.h"
 #include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_tls_get_addr.h"
 
 // ---------------------- Replacement functions ---------------- {{{1
 using namespace __memprof;
 
-static uptr allocated_for_dlsym;
-static uptr last_dlsym_alloc_size_in_words;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static inline bool IsInDlsymAllocPool(const void *ptr) {
-  uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-  return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
-  uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
-  void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
-  last_dlsym_alloc_size_in_words = size_in_words;
-  allocated_for_dlsym += size_in_words;
-  CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
-  return mem;
-}
-
-static void DeallocateFromLocalPool(const void *ptr) {
-  // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store
-  // error messages and instead uses malloc followed by free. To avoid pool
-  // exhaustion due to long object filenames, handle that special case here.
-  uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words;
-  void *prev_mem = (void *)&alloc_memory_for_dlsym[prev_offset];
-  if (prev_mem == ptr) {
-    REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize);
-    allocated_for_dlsym = prev_offset;
-    last_dlsym_alloc_size_in_words = 0;
-  }
-}
-
-static inline bool MaybeInDlsym() { return memprof_init_is_running; }
-
-static inline bool UseLocalPool() { return MaybeInDlsym(); }
-
-static void *ReallocFromLocalPool(void *ptr, uptr size) {
-  const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-  const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
-  void *new_ptr;
-  if (UNLIKELY(UseLocalPool())) {
-    new_ptr = AllocateFromLocalPool(size);
-  } else {
+struct MemprofDlSymAllocator : public DlSymAllocator<MemprofDlSymAllocator> {
+  static bool UseImpl() { return memprof_init_is_running; }
+  static void *ReallocImpl(uptr size) {
     ENSURE_MEMPROF_INITED();
-    GET_STACK_TRACE_MALLOC;
-    new_ptr = memprof_malloc(size, &stack);
-  }
-  internal_memcpy(new_ptr, ptr, copy_size);
-  return new_ptr;
+    GET_STACK_TRACE_FREE;
+    return memprof_realloc(nullptr, size, &stack);
   }
+};
+
+static MemprofDlSymAllocator dlsym_allocator;
 
 INTERCEPTOR(void, free, void *ptr) {
-  if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
-    DeallocateFromLocalPool(ptr);
-    return;
-  }
+  if (dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Free(ptr);
   GET_STACK_TRACE_FREE;
   memprof_free(ptr, &stack, FROM_MALLOC);
 }
 
 #if SANITIZER_INTERCEPT_CFREE
 INTERCEPTOR(void, cfree, void *ptr) {
-  if (UNLIKELY(IsInDlsymAllocPool(ptr)))
-    return;
+  if (dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Free(ptr);
   GET_STACK_TRACE_FREE;
   memprof_free(ptr, &stack, FROM_MALLOC);
 }
 #endif // SANITIZER_INTERCEPT_CFREE
 
 INTERCEPTOR(void *, malloc, uptr size) {
-  if (UNLIKELY(UseLocalPool()))
-    // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(size);
+  if (dlsym_allocator.Use())
+    return dlsym_allocator.Allocate(size);
   ENSURE_MEMPROF_INITED();
   GET_STACK_TRACE_MALLOC;
   return memprof_malloc(size, &stack);
 }
 
 INTERCEPTOR(void *, calloc, uptr nmemb, uptr size) {
-  if (UNLIKELY(UseLocalPool()))
-    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(nmemb * size);
+  if (dlsym_allocator.Use())
+    return dlsym_allocator.Callocate(nmemb, size);
   ENSURE_MEMPROF_INITED();
   GET_STACK_TRACE_MALLOC;
   return memprof_calloc(nmemb, size, &stack);
 }
 
 INTERCEPTOR(void *, realloc, void *ptr, uptr size) {
-  if (UNLIKELY(IsInDlsymAllocPool(ptr)))
-    return ReallocFromLocalPool(ptr, size);
-  if (UNLIKELY(UseLocalPool()))
-    return AllocateFromLocalPool(size);
+  if (dlsym_allocator.Use() || dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Realloc(ptr, size);
   ENSURE_MEMPROF_INITED();
   GET_STACK_TRACE_MALLOC;
   return memprof_realloc(ptr, size, &stack);
 }
 
 #if SANITIZER_INTERCEPT_REALLOCARRAY
 INTERCEPTOR(void *, reallocarray, void *ptr, uptr nmemb, uptr size) {
   ENSURE_MEMPROF_INITED();

compiler-rt/lib/msan/msan_interceptors.cpp

 #include "interception/interception.h"
 #include "msan.h"
 #include "msan_chained_origin_depot.h"
 #include "msan_origin.h"
 #include "msan_poisoning.h"
 #include "msan_report.h"
 #include "msan_thread.h"
 #include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
-#include "sanitizer_common/sanitizer_allocator_internal.h"
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_errno_codes.h"
 #include "sanitizer_common/sanitizer_glibc_version.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_linux.h"
 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
struct InterceptorScope {
   InterceptorScope() { ++in_interceptor_scope; }
   ~InterceptorScope() { --in_interceptor_scope; }
 };
 
 bool IsInInterceptorScope() {
   return in_interceptor_scope;
 }
 
-static uptr allocated_for_dlsym;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static bool IsInDlsymAllocPool(const void *ptr) {
-  uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-  return off < sizeof(alloc_memory_for_dlsym);
+struct MsanDlSymAllocator : public DlSymAllocator<MsanDlSymAllocator> {
+  static bool UseImpl() { return !msan_inited; }
+  static void *ReallocImpl(uptr size) {
+    GET_MALLOC_STACK_TRACE;
+    return msan_realloc(nullptr, size, &stack);
   }
+};
 
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
-  uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
-  void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
-  allocated_for_dlsym += size_in_words;
-  CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
-  return mem;
-}
+static MsanDlSymAllocator dlsym_allocator;
 
 #define ENSURE_MSAN_INITED() do { \
   CHECK(!msan_init_is_running); \
   if (!msan_inited) { \
     __msan_init(); \
   } \
 } while (0)
 
INTERCEPTOR(void *, pvalloc, SIZE_T size) {
   return msan_pvalloc(size, &stack);
 }
 #define MSAN_MAYBE_INTERCEPT_PVALLOC INTERCEPT_FUNCTION(pvalloc)
 #else
 #define MSAN_MAYBE_INTERCEPT_PVALLOC
 #endif
 
 INTERCEPTOR(void, free, void *ptr) {
-  if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
+  if (UNLIKELY(!ptr))
+    return;
+  if (dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Free(ptr);
   GET_MALLOC_STACK_TRACE;
   MsanDeallocate(&stack, ptr);
 }
 
 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
 INTERCEPTOR(void, cfree, void *ptr) {
-  if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
+  if (UNLIKELY(!ptr))
+    return;
+  if (dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Free(ptr);
   GET_MALLOC_STACK_TRACE;
   MsanDeallocate(&stack, ptr);
 }
 #  define MSAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree)
 #else
 #define MSAN_MAYBE_INTERCEPT_CFREE
 #endif
 

 }
 #define MSAN_MAYBE_INTERCEPT_EPOLL_PWAIT INTERCEPT_FUNCTION(epoll_pwait)
 #else
 #define MSAN_MAYBE_INTERCEPT_EPOLL_PWAIT
 #endif
 
 INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) {
   GET_MALLOC_STACK_TRACE;
-  if (UNLIKELY(!msan_inited))
-    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(nmemb * size);
+  if (dlsym_allocator.Use())
+    return dlsym_allocator.Callocate(nmemb, size);
   return msan_calloc(nmemb, size, &stack);
 }
 
 INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) {
-  if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
-    uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-    uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
-    void *new_ptr;
-    if (UNLIKELY(!msan_inited)) {
-      new_ptr = AllocateFromLocalPool(copy_size);
-    } else {
-      copy_size = size;
-      GET_MALLOC_STACK_TRACE;
-      new_ptr = msan_malloc(copy_size, &stack);
-    }
-    internal_memcpy(new_ptr, ptr, copy_size);
-    return new_ptr;
-  }
+  if (dlsym_allocator.Use() || dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Realloc(ptr, size);
   GET_MALLOC_STACK_TRACE;
   return msan_realloc(ptr, size, &stack);
 }
 
 INTERCEPTOR(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size) {
   GET_MALLOC_STACK_TRACE;
   return msan_reallocarray(ptr, nmemb, size, &stack);
 }
 
 INTERCEPTOR(void *, malloc, SIZE_T size) {
-  if (UNLIKELY(!msan_inited))
-    // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(size);
+  if (dlsym_allocator.Use())
+    return dlsym_allocator.Allocate(size);
   GET_MALLOC_STACK_TRACE;
   return msan_malloc(size, &stack);
 }
 
 void __msan_allocated_memory(const void *data, uptr size) {
   if (flags()->poison_in_malloc) {
     GET_MALLOC_STACK_TRACE;
     stack.tag = STACK_TRACE_TAG_POISON;

compiler-rt/lib/sanitizer_common/CMakeLists.txt

 # headers when building our custom unit tests.
 set(SANITIZER_IMPL_HEADERS
   sancov_flags.h
   sancov_flags.inc
   sanitizer_addrhashmap.h
   sanitizer_allocator.h
   sanitizer_allocator_checks.h
   sanitizer_allocator_combined.h
+  sanitizer_allocator_dlsym.h
   sanitizer_allocator_interface.h
   sanitizer_allocator_internal.h
   sanitizer_allocator_local_cache.h
   sanitizer_allocator_primary32.h
   sanitizer_allocator_primary64.h
   sanitizer_allocator_report.h
   sanitizer_allocator_secondary.h
   sanitizer_allocator_size_class_map.h

compiler-rt/lib/sanitizer_common/sanitizer_allocator_dlsym.h

+//===-- sanitizer_allocator_dlsym.h -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Hack: Sanitizer initializer calls dlsym which may need to allocate and call
+// back into uninitialized sanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ALLOCATOR_DLSYM_H
+#define SANITIZER_ALLOCATOR_DLSYM_H
+
+#include "sanitizer_allocator_internal.h"
+
+namespace __sanitizer {
+
+template <typename Details>
+struct DlSymAllocator {
+  static bool Use() {
+    // Fuchsia doesn't use dlsym-based interceptors.
+    return !SANITIZER_FUCHSIA && UNLIKELY(Details::UseImpl());
+  }
+
+  static bool PointerIsMine(const void *ptr) {
+    // Fuchsia doesn't use dlsym-based interceptors.
+    return !SANITIZER_FUCHSIA &&
+           UNLIKELY(internal_allocator()->FromPrimary(ptr));
+  }
+
+  static void *Allocate(uptr size_in_bytes) {
+    void *ptr = InternalAlloc(size_in_bytes, nullptr, kWordSize);
+    CHECK(internal_allocator()->FromPrimary(ptr));
+    return ptr;
+  }
+
+  static void *Callocate(SIZE_T nmemb, SIZE_T size) {
+    void *ptr = InternalCalloc(nmemb, size);
+    CHECK(internal_allocator()->FromPrimary(ptr));
+    return ptr;
+  }
+
+  static void Free(void *ptr) { InternalFree(ptr); }
+
+  static void *Realloc(void *ptr, uptr size) {
+    CHECK(!ptr || internal_allocator()->FromPrimary(ptr));
+    if (Use())
+      return InternalRealloc(ptr, size);
+    void *new_ptr = Details::ReallocImpl(size);
+    if (ptr) {
+      internal_memcpy(
+          new_ptr, ptr,
+          Min(size, internal_allocator()->GetActuallyAllocatedSize(ptr)));
+      Free(ptr);
+    }
+    return new_ptr;
+  }
+};
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_ALLOCATOR_DLSYM_H

compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc

 
 #include <AvailabilityMacros.h>
 #include <CoreFoundation/CFBase.h>
 #include <dlfcn.h>
 #include <malloc/malloc.h>
 #include <sys/mman.h>
 
 #include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #include "sanitizer_common/sanitizer_mac.h"
 
 // Similar code is used in Google Perftools,
 // https://github.com/gperftools/gperftools.
 
 namespace __sanitizer {
 
 extern malloc_zone_t sanitizer_zone;

 extern "C"
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__sanitizer_mz_malloc(malloc_zone_t *zone, uptr size) {
   COMMON_MALLOC_ENTER();
   COMMON_MALLOC_MALLOC(size);
   return p;
 }
 
+struct MzDlSymAllocator : public DlSymAllocator<LsanDlSymAllocator> {
+  static bool UseImpl() { return !COMMON_MALLOC_SANITIZER_INITIALIZED; }
+  static void *ReallocImpl(uptr size) {
+    CHECK(0);
+    return nullptr;
+  }
+};
+
+static MzDlSymAllocator dlsym_allocator;
+
 extern "C"
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__sanitizer_mz_calloc(malloc_zone_t *zone, size_t nmemb, size_t size) {
-  if (UNLIKELY(!COMMON_MALLOC_SANITIZER_INITIALIZED)) {
-    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
-    const size_t kCallocPoolSize = 1024;
-    static uptr calloc_memory_for_dlsym[kCallocPoolSize];
-    static size_t allocated;
-    size_t size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
-    void *mem = (void*)&calloc_memory_for_dlsym[allocated];
-    allocated += size_in_words;
-    CHECK(allocated < kCallocPoolSize);
-    return mem;
-  }
+  if (dlsym_allocator.Use())
+    return dlsym_allocator.Callocate(nmemb, size);
   COMMON_MALLOC_CALLOC(nmemb, size);
   return p;
 }
 
 extern "C"
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__sanitizer_mz_valloc(malloc_zone_t *zone, size_t size) {
   COMMON_MALLOC_ENTER();
   COMMON_MALLOC_VALLOC(size);
   return p;
 }
 
 // TODO(glider): the allocation callbacks need to be refactored.
 extern "C"
 SANITIZER_INTERFACE_ATTRIBUTE
 void __sanitizer_mz_free(malloc_zone_t *zone, void *ptr) {
   if (!ptr) return;
+  if (dlsym_allocator.PointerIsMine(ptr))
+    return dlsym_allocator.Free(ptr);
   COMMON_MALLOC_FREE(ptr);
 }
 
 #define GET_ZONE_FOR_PTR(ptr) \
   malloc_zone_t *zone_ptr = WRAP(malloc_zone_from_ptr)(ptr); \
   const char *zone_name = (zone_ptr == 0) ? 0 : zone_ptr->zone_name
 
 extern "C"