Allow initialization of Asan interceptors before the general Asan initialization takes place on FreeBSD

[kutuzov.viktor.84]

Here's how things currently look during initialization on FreeBSD:

#0  AsanInitInternal () at /usr/home/ik/llvm/llvm.current/projects/compiler-rt/lib/asan/asan_rtl.cc:695
#1  0x0000000000411d7a in __interceptor_strncmp () at sanitizer_common_interceptors.inc:178
#2  0x000000080109e35e in getenv () from /lib/libc.so.7
#3  0x0000000801302d03 in pthread_timedjoin_np () from /lib/libthr.so.3
#4  0x00000008013055d2 in __error () from /lib/libthr.so.3
#5  0x00000008012f9bd6 in .init () from /lib/libthr.so.3
#6  0x00007fffffffd0d0 in ?? ()
#7  0x000000080065d611 in r_debug_state () from /libexec/ld-elf.so.1
#8  0x000000080065cc97 in __tls_get_addr () from /libexec/ld-elf.so.1
#9  0x000000080065b089 in .text () from /libexec/ld-elf.so.1
#10 0x0000000000000000 in ?? ()

Note that an intercepted function is called before Asan has a chance to initialize itself.

The fix eliminates mass failures on Asan tests--both unit and lit ones.

[kcc]

This overcomplicates things that are already overcomplicated to my taste.
I especially dislike extra if(FREEBSD).

What is wrong with the stack you've shown?

[kutuzov.viktor.84]

What is wrong with the stack you've shown?

The problem is that strncmp() is called before REAL(strncmp) is initialized so we have to do InitializeAsanInterceptors() before we can refer to the real function. Calling AsanInitInternal() cannot proceed correctly as it actually called from within initialization of libthr while AsanInitInternal() itself tries to initialize the thread subsystem what lead us to recursive initialization of libthr.

[eugenis]

Can you use ENSURE_ASAN_INITED?

[kutuzov.viktor.84]

Can you use ENSURE_ASAN_INITED?

It would cause calling AsanInitInternal() which, in turn, will attempt to (recursively) initialize threads.

[eugenis]

OK.
So, InitializeAsanInterceptors() here is OK, and AsanInit() is not. Could this be solved by initializing interceptors earlier in AsanInit? This initialization sequence is kind of fragile, and I don't like that we arbitrarily move part of it upwards in a platform-dependent way.

[kutuzov.viktor.84]

Could this be solved by initializing interceptors earlier in AsanInit? This initialization sequence is kind of fragile, and I don't like that we arbitrarily move part of it upwards in a platform-dependent way.

By the time when the first of the intercepted functions is called, the libthr (the library Asan's thread rely on on FreeBSD) is being initialized already. AsanInitInternal() initializes both the interceptors and threads unconditionally and thus causes recursive initialization of libthr.

The overall process looks like this:

• the loader calls libthr's .init();
• .init() calls getenv() during initialization of the library;
• genenv() relies on strncmp();
• strncmp() is intercepted, but at this phase the REAL(strncmp) is not yet initialized;
• since Asan is not initialized, AsanInitInternal() takes place;
• AsanInitInternal() initializes the references to the real functions and then call AsanTSDInit();
• AsanTSDInit() calls pthread_key_create();
• pthread_key_create() sees that libthr is not yet initialized (what means its initialization is not yet completed) and launches initialization of libthr.

So the answer to your question is: unfortunately, it doesn't matter how early interceptors get initialized in AsanInitInternal(); the point is that on FreeBSD the initialization of interceptors and initialization of threads are separate phases that cannot be tied together.

[kcc]

nice loop.
Maybe we could break the loop by calling internal_strncmp?

INTERCEPTOR(int, strncmp, const char *s1, const char *s2, uptr size) {

if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
   return internal_strncmp(s1, s2, size);

we have a couple of similar hacks elsewhere

[kutuzov.viktor.84]

Yes, it worked. Thanks, Kostya. Updated.

[kcc]

LGTM with one small nit

lib/sanitizer_common/sanitizer_common_interceptors.inc

95 ↗

(On Diff #11575)

why new line?

[kutuzov.viktor.84]

Closed by commit rL213941 (authored by vkutuzov).

lib/asan/asan_interceptors.cc

 // OS X interceptors don't need to be initialized with INTERCEPT_FUNCTION.
 #define ASAN_INTERCEPT_FUNC(name)
 #endif  // SANITIZER_MAC
 
 #define COMMON_INTERCEPT_FUNCTION(name) ASAN_INTERCEPT_FUNC(name)
 #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
   ASAN_WRITE_RANGE(ptr, size)
 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) ASAN_READ_RANGE(ptr, size)
+// On FreeBSD intercepted function may be called before the references to the
+// real functions are initialized.
 #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...)                               \
   do {                                                                         \
+    if (SANITIZER_FREEBSD && UNLIKELY(!asan_interceptors_inited))              \
+      InitializeAsanInterceptors();                                            \
     if (asan_init_is_running)                                                  \
       return REAL(func)(__VA_ARGS__);                                          \
     ctx = 0;                                                                   \
     (void) ctx;                                                                \
-    if (SANITIZER_MAC && UNLIKELY(!asan_inited))                               \
+    if ((SANITIZER_MAC != 0 || SANITIZER_FREEBSD != 0) &&                      \
+        UNLIKELY(!asan_inited))                                                \
       return REAL(func)(__VA_ARGS__);                                          \
     ENSURE_ASAN_INITED();                                                      \
   } while (false)
 #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
   do {                                         \
   } while (false)
 #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
   do {                                         \

 }
 
 }  // namespace __asan
 #endif
 
 // ---------------------- InitializeAsanInterceptors ---------------- {{{1
 namespace __asan {
 void InitializeAsanInterceptors() {
-  static bool was_called_once;
-  CHECK(was_called_once == false);
-  was_called_once = true;
+  if (asan_interceptors_inited) return;
+
   InitializeCommonInterceptors();
 
   // Intercept mem* functions.
   ASAN_INTERCEPT_FUNC(memcmp);
   ASAN_INTERCEPT_FUNC(memmove);
   ASAN_INTERCEPT_FUNC(memset);
   if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) {
     ASAN_INTERCEPT_FUNC(memcpy);
#if ASAN_INTERCEPT_FORK
   ASAN_INTERCEPT_FUNC(fork);
 #endif
 
   // Some Windows-specific interceptors.
 #if SANITIZER_WINDOWS
   InitializeWindowsInterceptors();
 #endif
 
+  asan_interceptors_inited = true;
+
   VReport(1, "AddressSanitizer: libc interceptors initialized\n");
 }
 
 }  // namespace __asan

lib/asan/asan_internal.h

#define ASAN_FREE_HOOK(ptr) \
   if (&__asan_free_hook) __asan_free_hook(ptr); \
   if (&__sanitizer_free_hook) __sanitizer_free_hook(ptr)
 #define ASAN_ON_ERROR() \
   if (&__asan_on_error) __asan_on_error()
 
 extern int asan_inited;
 // Used to avoid infinite recursion in __asan_init().
 extern bool asan_init_is_running;
+extern bool asan_interceptors_inited;
 extern void (*death_callback)(void);
 
 // These magic values are written to shadow for better error reporting.
 const int kAsanHeapLeftRedzoneMagic = 0xfa;
 const int kAsanHeapRightRedzoneMagic = 0xfb;
 const int kAsanHeapFreeMagic = 0xfd;
 const int kAsanStackLeftRedzoneMagic = 0xf1;
 const int kAsanStackMidRedzoneMagic = 0xf2;

lib/asan/asan_rtl.cc

void ParseExtraActivationFlags() {
   ParseFlagsFromString(flags(), buf);
   if (buf[0] != '\0')
     VReport(1, "Extra activation flags: %s\n", buf);
 }
 
 // -------------------------- Globals --------------------- {{{1
 int asan_inited;
 bool asan_init_is_running;
+bool asan_interceptors_inited;
 void (*death_callback)(void);
 
 #if !ASAN_FIXED_MAPPING
 uptr kHighMemEnd, kMidMemBeg, kMidMemEnd;
 #endif
 
 // -------------------------- Misc ---------------- {{{1
 void ShowStatsAndAbort() {