[esan] Initialize runtime during early interceptors

[bruening]

Summary

Adds initialization of esan's runtime library during any early interceptors
that are sometimes called prior to the official __esan_init() invocation
(we see this with apps using tcmalloc).

Adds handling of interceptors called during interceptor initialization.

[aizatsky]

lib/esan/esan_interceptors.cpp

56 ↗

(On Diff #59588)

If performance is a concern, I suggest trying to make this look like:

if (UNLIKELY(flag1)) {
  // 2 branches here
}

instead of two consequent ones. I am not sure how two UNLIKELY work though.

But this will have only one memory read on the fast path.

[bruening]

Eliminate one branch on the interceptor entry common path.

compiler-rt/trunk/lib/esan/esan.h

 
 #include "interception/interception.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "esan_interface_internal.h"
 
 namespace __esan {
 
 extern bool EsanIsInitialized;
+extern bool EsanDuringInit;
 
 void initializeLibrary(ToolType Tool);
 int finalizeLibrary();
 // Esan creates the variable per tool per compilation unit at compile time
 // and passes its pointer Ptr to the runtime library.
 void processCompilationUnitInit(void *Ptr);
 void processCompilationUnitExit(void *Ptr);
 void processRangeAccess(uptr PC, uptr Addr, int Size, bool IsWrite);

compiler-rt/trunk/lib/esan/esan.cpp

 // See comment below.
 extern "C" {
 extern void __cxa_atexit(void (*function)(void));
 }
 
 namespace __esan {
 
 bool EsanIsInitialized;
+bool EsanDuringInit;
 ShadowMapping Mapping;
 
 // Different tools use different scales within the same shadow mapping scheme.
 // The scale used here must match that used by the compiler instrumentation.
 // This array is indexed by the ToolType enum.
 static const uptr ShadowScale[] = {
   0, // ESAN_None.
   2, // ESAN_CacheFrag: 4B:1B, so 4 to 1 == >>2.
for (int i = 0; getShadowRegion(i, &ShadowStart, &ShadowEnd); ++i) {
     if (common_flags()->use_madv_dontdump)
       DontDumpShadowMemory(ShadowStart, ShadowEnd - ShadowStart);
 
     // TODO: Call MmapNoAccess() on in-between regions.
   }
 }
 
 void initializeLibrary(ToolType Tool) {
-  // We assume there is only one thread during init.
-  if (EsanIsInitialized) {
-    CHECK(Tool == __esan_which_tool);
+  // We assume there is only one thread during init, but we need to
+  // guard against double-init when we're (re-)called from an
+  // early interceptor.
+  if (EsanIsInitialized || EsanDuringInit)
     return;
-  }
+  EsanDuringInit = true;
+  CHECK(Tool == __esan_which_tool);
   SanitizerToolName = "EfficiencySanitizer";
   CacheBinaryName();
   initializeFlags();
 
   // Intercepting libc _exit or exit via COMMON_INTERCEPTOR_ON_EXIT only
   // finalizes on an explicit exit call by the app.  To handle a normal
   // exit we register an atexit handler.
   ::__cxa_atexit((void (*)())finalizeLibrary);
void initializeLibrary(ToolType Tool) {
 
   if (__esan_which_tool == ESAN_CacheFrag) {
     initializeCacheFrag();
   } else if (__esan_which_tool == ESAN_WorkingSet) {
     initializeWorkingSet();
   }
 
   EsanIsInitialized = true;
+  EsanDuringInit = false;
 }
 
 int finalizeLibrary() {
   VPrintf(1, "in esan::%s\n", __FUNCTION__);
   if (__esan_which_tool == ESAN_CacheFrag) {
     return finalizeCacheFrag();
   } else if (__esan_which_tool == ESAN_WorkingSet) {
     return finalizeWorkingSet();

compiler-rt/trunk/lib/esan/esan_interceptors.cpp

 #undef SANITIZER_INTERCEPT_REALPATH
 
 #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!EsanIsInitialized)
 
 #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
 #define COMMON_INTERCEPT_FUNCTION_VER(name, ver)                          \
   INTERCEPT_FUNCTION_VER(name, ver)
 
+// We must initialize during early interceptors, to support tcmalloc.
+// This means that for some apps we fully initialize prior to
+// __esan_init() being called.
 // We currently do not use ctx.
 #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...)                               \
   do {                                                                         \
     if (UNLIKELY(COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)) {                 \
+      if (!UNLIKELY(EsanDuringInit))                                           \
+        initializeLibrary(__esan_which_tool);                                  \
       return REAL(func)(__VA_ARGS__);                                          \
     }                                                                          \
     ctx = nullptr;                                                             \
     (void)ctx;                                                                 \
   } while (false)
 
 #define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...)                      \
   COMMON_INTERCEPTOR_ENTER(ctx, func, __VA_ARGS__)

 // Shadow-related interceptors
 //===----------------------------------------------------------------------===//
 
 // These are candidates for sharing with all sanitizers if shadow memory
 // support is also standardized.
 
 INTERCEPTOR(void *, mmap, void *addr, SIZE_T sz, int prot, int flags,
                  int fd, OFF_T off) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, mmap, addr, sz, prot, flags, fd, off);
   if (!fixMmapAddr(&addr, sz, flags))
     return (void *)-1;
   void *result = REAL(mmap)(addr, sz, prot, flags, fd, off);
   return (void *)checkMmapResult((uptr)result, sz);
 }
 
 #if SANITIZER_LINUX
 INTERCEPTOR(void *, mmap64, void *addr, SIZE_T sz, int prot, int flags,
                  int fd, OFF64_T off) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, mmap64, addr, sz, prot, flags, fd, off);
   if (!fixMmapAddr(&addr, sz, flags))
     return (void *)-1;
   void *result = REAL(mmap64)(addr, sz, prot, flags, fd, off);
   return (void *)checkMmapResult((uptr)result, sz);
 }
 #define ESAN_MAYBE_INTERCEPT_MMAP64 INTERCEPT_FUNCTION(mmap64)
 #else
 #define ESAN_MAYBE_INTERCEPT_MMAP64