[compiler-rt] [sanitizers] Move CheckVMASize after flag initialization

[samsonov]

lib/asan/asan_rtl.cc

380

Fine, but please remove it from __asan_init

lib/tsan/rtl/tsan_rtl.cc

316

So.. where is it now?

[zatrazz]

Update patch with CheckVMASize remove from __asan_init and add the tsan one.

[samsonov]

LGTM

lib/asan/asan_rtl.cc

static void AsanInitInternal() {
   asan_init_is_running = true;
 
   CacheBinaryName();
 
   // Initialize flags. This must be done early, because most of the
   // initialization steps look at flags().
   InitializeFlags();
 
+  CheckVMASize();
+
   AsanCheckIncompatibleRT();
   AsanCheckDynamicRTPrereqs();
 
   SetCanPoisonMemory(flags()->poison_heap);
   SetMallocContextSize(common_flags()->malloc_context_size);
 
   InitializeHighMemEnd();
 

 
 void NOINLINE __asan_set_death_callback(void (*callback)(void)) {
   SetUserDieCallback(callback);
 }
 
 // Initialize as requested from instrumented application code.
 // We use this call as a trigger to wake up ASan from deactivated state.
 void __asan_init() {
-  CheckVMASize();
   AsanActivate();
   AsanInitInternal();
 }
 
 void __asan_version_mismatch_check() {
   // Do nothing.
 }

lib/dfsan/dfsan.cc

if (internal_strcmp(flags().dump_labels_at_exit, "") != 0) {
     Report("INFO: DataFlowSanitizer: dumping labels to %s\n",
            flags().dump_labels_at_exit);
     dfsan_dump_labels(fd);
     CloseFile(fd);
   }
 }
 
 static void dfsan_init(int argc, char **argv, char **envp) {
+  InitializeFlags();
+
   CheckVMASize();
 
   MmapFixedNoReserve(kShadowAddr, kUnusedAddr - kShadowAddr);
 
   // Protect the region of memory we don't use, to preserve the one-to-one
   // mapping from application to shadow memory. But if ASLR is disabled, Linux
   // will load our executable in the middle of our unused region. This mostly
   // works so long as the program doesn't use too much memory. We support this
   // case by disabling memory protection when ASLR is disabled.
   uptr init_addr = (uptr)&dfsan_init;
   if (!(init_addr >= kUnusedAddr && init_addr < kAppAddr))
     MmapNoAccess(kUnusedAddr, kAppAddr - kUnusedAddr);
 
-  InitializeFlags();
   InitializeInterceptors();
 
   // Register the fini callback to run when the program terminates successfully
   // or it is killed by the runtime.
   Atexit(dfsan_fini);
   AddDieCallback(dfsan_fini);
 
   __dfsan_label_info[kInitializingLabel].desc = "<init label>";
 }
 
 #if SANITIZER_CAN_USE_PREINIT_ARRAY
 __attribute__((section(".preinit_array"), used))
 static void (*dfsan_init_ptr)(int, char **, char **) = dfsan_init;
 #endif

lib/msan/msan.cc

 }
 
 void __msan_init() {
   CHECK(!msan_init_is_running);
   if (msan_inited) return;
   msan_init_is_running = 1;
   SanitizerToolName = "MemorySanitizer";
 
-  CheckVMASize();
-
   InitTlsSize();
 
   CacheBinaryName();
   InitializeFlags();
+
+  CheckVMASize();
+
   __sanitizer_set_report_path(common_flags()->log_path);
 
   InitializeInterceptors();
   InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
 
   DisableCoreDumperIfNecessary();
   if (StackSizeIsUnlimited()) {
     VPrintf(1, "Unlimited stack, doing reexec\n");

lib/tsan/rtl/tsan_rtl.cc

 }
 
 void Initialize(ThreadState *thr) {
   // Thread safe because done before all threads exist.
   static bool is_initialized = false;
   if (is_initialized)
     return;
   is_initialized = true;
-
-  CheckVMASize();
-
   // We are not ready to handle interceptors yet.
   ScopedIgnoreInterceptors ignore;
   SanitizerToolName = "ThreadSanitizer";
   // Install tool-specific callbacks in sanitizer_common.
   SetCheckFailedCallback(TsanCheckFailed);
 
   ctx = new(ctx_placeholder) Context;
   const char *options = GetEnv(kTsanOptionsEnv);
   CacheBinaryName();
   InitializeFlags(&ctx->flags, options);
+  CheckVMASize();
 #ifndef SANITIZER_GO
   InitializeAllocator();
 #endif
   InitializeInterceptors();
   CheckShadowMapping();
   InitializePlatform();
   InitializeMutex();
   InitializeDynamicAnnotations();