[compiler-rt] [xray] Detect MPROTECT and error out when it's enabled (on NetBSD)

[mgorny]

Summary

Add a CheckMPROTECT() routine to detect when pax MPROTECT is enabled
on NetBSD, and error xray out when it is. The solution is adapted
from existing CheckASLR().

[krytarowski]

lib/xray/xray_init.cc

70

PaX

lib/sanitizer_common/sanitizer_common.h

 void DumpProcessMap();
 void PrintModuleMap();
 const char *GetEnv(const char *name);
 bool SetEnv(const char *name, const char *value);
 
 u32 GetUid();
 void ReExec();
 void CheckASLR();
+void CheckMPROTECT();
 char **GetArgv();
 char **GetEnviron();
 void PrintCmdline();
 bool StackSizeIsUnlimited();
 uptr GetStackSizeLimitInBytes();
 void SetStackSizeLimitInBytes(uptr limit);
 bool AddressSpaceIsUnlimited();
 void SetAddressSpaceUnlimited();

lib/sanitizer_common/sanitizer_fuchsia.cc

void GetThreadStackTopAndBottom(bool, uptr *stack_top, uptr *stack_bottom) {
 
   *stack_bottom = reinterpret_cast<uptr>(base);
   *stack_top = *stack_bottom + size;
 }
 
 void InitializePlatformEarly() {}
 void MaybeReexec() {}
 void CheckASLR() {}
+void CheckMPROTECT() {}
 void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {}
 void DisableCoreDumperIfNecessary() {}
 void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
 void SetAlternateSignalStack() {}
 void UnsetAlternateSignalStack() {}
 void InitTlsSize() {}
 
 void PrintModuleMap() {}

lib/sanitizer_common/sanitizer_linux.cc

if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
     CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1);
     ReExec();
   }
 #else
   // Do nothing
 #endif
 }
 
+void CheckMPROTECT() {
+#if SANITIZER_NETBSD
+  int mib[3];
+  int paxflags;
+  uptr len = sizeof(paxflags);
+
+  mib[0] = CTL_PROC;
+  mib[1] = internal_getpid();
+  mib[2] = PROC_PID_PAXFLAGS;
+
+  if (UNLIKELY(internal_sysctl(mib, 3, &paxflags, &len, NULL, 0) == -1)) {
+    Printf("sysctl failed\n");
+    Die();
+  }
+
+  if (UNLIKELY(paxflags & CTL_PROC_PAXFLAGS_MPROTECT)) {
+    Printf("This sanitizer is not compatible with enabled MPROTECT\n");
+    Die();
+  }
+#else
+  // Do nothing
+#endif
+}
+
 void PrintModuleMap() { }
 
 void CheckNoDeepBind(const char *filename, int flag) {
 #ifdef RTLD_DEEPBIND
   if (flag & RTLD_DEEPBIND) {
     Report(
         "You are trying to dlopen a %s shared library with RTLD_DEEPBIND flag"
         " which is incompatibe with sanitizer runtime "

lib/sanitizer_common/sanitizer_mac.cc

 void ReExec() {
   UNIMPLEMENTED();
 }
 
 void CheckASLR() {
   // Do nothing
 }
 
+void CheckMPROTECT() {
+  // Do nothing
+}
+
 uptr GetPageSize() {
   return sysconf(_SC_PAGESIZE);
 }
 
 extern "C" unsigned malloc_num_zones;
 extern "C" malloc_zone_t **malloc_zones;
 malloc_zone_t sanitizer_zone;
 

lib/sanitizer_common/sanitizer_rtems.cc

void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
   *stk_addr = stack_bottom;
   *stk_size = stack_top - stack_bottom;
   *tls_addr = *tls_size = 0;
 }
 
 void InitializePlatformEarly() {}
 void MaybeReexec() {}
 void CheckASLR() {}
+void CheckMPROTECT() {}
 void DisableCoreDumperIfNecessary() {}
 void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
 void SetAlternateSignalStack() {}
 void UnsetAlternateSignalStack() {}
 void InitTlsSize() {}
 
 void PrintModuleMap() {}
 

lib/sanitizer_common/sanitizer_win.cc

 void MaybeReexec() {
   // No need to re-exec on Windows.
 }
 
 void CheckASLR() {
   // Do nothing
 }
 
+void CheckMPROTECT() {
+  // Do nothing
+}
+
 char **GetArgv() {
   // FIXME: Actually implement this function.
   return 0;
 }
 
 char **GetEnviron() {
   // FIXME: Actually implement this function.
   return 0;

lib/xray/xray_init.cc

 // __xray_init() will do the actual loading of the current process' memory map
 // and then proceed to look for the .xray_instr_map section/segment.
 void __xray_init() XRAY_NEVER_INSTRUMENT {
   SpinMutexLock Guard(&XRayInitMutex);
   // Short-circuit if we've already initialized XRay before.
   if (atomic_load(&XRayInitialized, memory_order_acquire))
     return;
 
+  // XRAY is not compatible with pax MPROTECT
+  CheckMPROTECT();
+
   if (!atomic_load(&XRayFlagsInitialized, memory_order_acquire)) {
     initializeFlags();
     atomic_store(&XRayFlagsInitialized, true, memory_order_release);
   }
 
   if (__start_xray_instr_map == nullptr) {
     if (Verbosity())
       Report("XRay instrumentation map missing. Not initializing XRay.\n");