[esan] Fix ESan test failure on Debian Sid bot

[zhaoqin]

Summary

Handles multiple early allocation requests from dlsym.

Existing code handles single early allocation request from dlsym.
This CL extends the existing code to handle multiple requests with a larger local static buffer.

[bruening]

Handles early allocation from dlsym by allocating memory from a local
static buffer.

The existing code already attempts to handle early allocation, using a local static buffer: so this commit message seems to describe the CL from months ago. Please describe what's *new* in the commit message.

[zhaoqin]

Update the commit message.

[bruening]

lib/esan/esan_interceptors.cpp

464 ↗

(On Diff #73290)

Variable name style violation.

The consensus is to match LLVM style in general. The params to the interceptors here do not to match the common interceptors. It seems like this should either be under_score_style, or it and the 2 below should be CamelCase.

469 ↗

(On Diff #73290)

Hmm, unsigned underflow is defined, though it feels a little hacky

477 ↗

(On Diff #73290)

Spelling

512 ↗

(On Diff #73290)

No need for the "res" var

[zhaoqin]

PTAL

[zhaoqin]

remove prefix k_

compiler-rt/trunk/lib/esan/esan_interceptors.cpp

 #else
 #define ESAN_MAYBE_INTERCEPT_PTHREAD_SIGMASK
 #endif
 
 //===----------------------------------------------------------------------===//
 // Malloc interceptors
 //===----------------------------------------------------------------------===//
 
-static char early_alloc_buf[128];
-static bool used_early_alloc_buf;
+static const uptr early_alloc_buf_size = 1024;
+static uptr allocated_bytes;
+static char early_alloc_buf[early_alloc_buf_size];
+
+static bool isInEarlyAllocBuf(const void *ptr) {
+  return ((uptr)ptr >= (uptr)early_alloc_buf &&
+          ((uptr)ptr - (uptr)early_alloc_buf) < sizeof(early_alloc_buf));
+}
 
 static void *handleEarlyAlloc(uptr size) {
   // If esan is initialized during an interceptor (which happens with some
   // tcmalloc implementations that call pthread_mutex_lock), the call from
-  // dlsym to calloc will deadlock.  There is only one such calloc (dlsym
-  // allocates a single pthread key), so we work around it by using a
-  // static buffer for the calloc request.  The loader currently needs
-  // 32 bytes but we size at 128 to allow for future changes.
+  // dlsym to calloc will deadlock.
+  // dlsym may also call malloc before REAL(malloc) is retrieved from dlsym.
+  // We work around it by using a static buffer for the early malloc/calloc
+  // requests.
   // This solution will also allow us to deliberately intercept malloc & family
   // in the future (to perform tool actions on each allocation, without
   // replacing the allocator), as it also solves the problem of intercepting
   // calloc when it will itself be called before its REAL pointer is
   // initialized.
-  CHECK(!used_early_alloc_buf && size < sizeof(early_alloc_buf));
   // We do not handle multiple threads here.  This only happens at process init
   // time, and while it's possible for a shared library to create early threads
   // that race here, we consider that to be a corner case extreme enough that
   // it's not worth the effort to handle.
-  used_early_alloc_buf = true;
-  return (void *)early_alloc_buf;
+  void *mem = (void *)&early_alloc_buf[allocated_bytes];
+  allocated_bytes += size;
+  CHECK_LT(allocated_bytes, early_alloc_buf_size);
+  return mem;
 }
 
 INTERCEPTOR(void*, calloc, uptr size, uptr n) {
   if (EsanDuringInit && REAL(calloc) == nullptr)
     return handleEarlyAlloc(size * n);
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, calloc, size, n);
   void *res = REAL(calloc)(size, n);
   // The memory is zeroed and thus is all written.
   COMMON_INTERCEPTOR_WRITE_RANGE(nullptr, (uptr)res, size * n);
   return res;
 }
 
+INTERCEPTOR(void*, malloc, uptr size) {
+  if (EsanDuringInit && REAL(malloc) == nullptr)
+    return handleEarlyAlloc(size);
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, malloc, size);
+  return REAL(malloc)(size);
+}
+
 INTERCEPTOR(void, free, void *p) {
   void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, free, p);
-  if (p == (void *)early_alloc_buf) {
-    // We expect just a singleton use but we clear this for cleanliness.
-    used_early_alloc_buf = false;
+  // There are only a few early allocation requests, so we simply skip the free.
+  if (isInEarlyAllocBuf(p))
     return;
-  }
+  COMMON_INTERCEPTOR_ENTER(ctx, free, p);
   REAL(free)(p);
 }
 
 namespace __esan {
 
 void initializeInterceptors() {
   InitializeCommonInterceptors();
 
void initializeInterceptors() {
   ESAN_MAYBE_INTERCEPT_MMAP64;
 
   ESAN_MAYBE_INTERCEPT_SIGNAL;
   ESAN_MAYBE_INTERCEPT_SIGACTION;
   ESAN_MAYBE_INTERCEPT_SIGPROCMASK;
   ESAN_MAYBE_INTERCEPT_PTHREAD_SIGMASK;
 
   INTERCEPT_FUNCTION(calloc);
+  INTERCEPT_FUNCTION(malloc);
   INTERCEPT_FUNCTION(free);
 
   // TODO(bruening): intercept routines that other sanitizers intercept that
   // are not in the common pool or here yet, ideally by adding to the common
   // pool.  Examples include wcslen and bcopy.
 
   // TODO(bruening): there are many more libc routines that read or write data
   // structures that no sanitizer is intercepting: sigaction, strtol, etc.
 }
 
 } // namespace __esan