[tsan] Adjust setjmp/longjmp handling on Darwin for macOS Mojave

[kubamracek]

Summary

On macOS Mojave, the OS started using the XOR-by-a-secret-key scheme (same as glibc is alread doing) for storing the SP value in setjmp environment. We need to adjust for that to keep supporting setjmp/longjmp on latest Darwin. The patch is basically doing the same what we're already doing for glibc.

rdar://problem/43542596

[kubamracek]

I'm going to land this patch now, even though it's only been in review for a couple hours as there currently tests failing and known crashes (basically anything that does setjmp + longjmp) on macOS. I'll be happy to address post-commit comments.

[george.karpenkov]

lib/sanitizer_common/sanitizer_mac.cc

530

I'm a bit confused how this bit is relevant to your patch. Probably should be removed.

[kubamracek]

Removing the check for '2x' versions, as that's not really relevant here.

[delcypher]

lib/tsan/rtl/tsan_platform_mac.cc

260

This seems slightly odd. Using pthread_getspecific suggests the value you're requested could be different between threads. However the value you read here is global is being written to a global in the TSan runtime and thus will be shared by all threads.

At this point during init there is only one thread so there's only one value you can retrieve this early on but I do wonder if you'd get a different value on another thread. Perhaps this key should be stored per thread instead and initialized on thread creation?

If we expect the key to not change between threads perhaps we should add a CHECK_EQ() call that on thread creation checks that the key retrieved for the thread matches the global value stored during init?

261

On the else branch here we could add CHECK_EQ(__tsan_darwin_setjmp_xor_key, 0); because we require this value for the key so that the xor operation on older OS versions is a no-op.

lib/tsan/rtl/tsan_rtl_amd64.S

199

Could you jog my memory. Why is %rip involved in the xor calculation? ___tsan_darwin_setjmp_xor_key is being used as an offset here so presumably the offset is instruction relative?

[kubamracek]

lib/tsan/rtl/tsan_platform_mac.cc

260

You're right, this is a bit weird, but I've confirmed with the Libc folks that currently, the value is process-global and doesn't change between threads. This is something that can potentially change in the future.

Adding the check sounds reasonable, I'll try to do that.

261

Hm, that seems superfluous. __tsan_darwin_setjmp_xor_key is initialized to zero a couple lines above.

lib/tsan/rtl/tsan_rtl_amd64.S

199

Sure, this is to emit %rip-relative reference to ___tsan_darwin_setjmp_xor_key. Position-independent code requires that, IMHO.

lib/sanitizer_common/sanitizer_mac.h

enum MacosVersion {
   MACOS_VERSION_UNINITIALIZED = 0,
   MACOS_VERSION_UNKNOWN,
   MACOS_VERSION_LEOPARD,
   MACOS_VERSION_SNOW_LEOPARD,
   MACOS_VERSION_LION,
   MACOS_VERSION_MOUNTAIN_LION,
   MACOS_VERSION_MAVERICKS,
   MACOS_VERSION_YOSEMITE,
+  MACOS_VERSION_EL_CAPITAN,
+  MACOS_VERSION_SIERRA,
+  MACOS_VERSION_HIGH_SIERRA,
+  MACOS_VERSION_MOJAVE,
   MACOS_VERSION_UNKNOWN_NEWER
 };
 
 MacosVersion GetMacosVersion();
 
 char **GetEnviron();
 
 void RestrictMemoryToMaxAddress(uptr max_address);

lib/sanitizer_common/sanitizer_mac.cc

switch (version[0]) {
     case '9': return MACOS_VERSION_LEOPARD;
     case '1': {
       switch (version[1]) {
         case '0': return MACOS_VERSION_SNOW_LEOPARD;
         case '1': return MACOS_VERSION_LION;
         case '2': return MACOS_VERSION_MOUNTAIN_LION;
         case '3': return MACOS_VERSION_MAVERICKS;
         case '4': return MACOS_VERSION_YOSEMITE;
+        case '5': return MACOS_VERSION_EL_CAPITAN;
+        case '6': return MACOS_VERSION_SIERRA;
+        case '7': return MACOS_VERSION_HIGH_SIERRA;
+        case '8': return MACOS_VERSION_MOJAVE;
         default:
           if (IsDigit(version[1]))
             return MACOS_VERSION_UNKNOWN_NEWER;
           else
             return MACOS_VERSION_UNKNOWN;
       }
     }
     default: return MACOS_VERSION_UNKNOWN;
   }
 }
 
 MacosVersion GetMacosVersion() {
   atomic_uint32_t *cache =
       reinterpret_cast<atomic_uint32_t*>(&cached_macos_version);
   MacosVersion result =
       static_cast<MacosVersion>(atomic_load(cache, memory_order_acquire));
   if (result == MACOS_VERSION_UNINITIALIZED) {
     result = GetMacosVersionInternal();
     atomic_store(cache, result, memory_order_release);
   }
   return result;

lib/tsan/rtl/tsan_interceptors.cc

 #ifdef __powerpc__
   uptr mangled_sp = env[0];
 #elif SANITIZER_FREEBSD
   uptr mangled_sp = env[2];
 #elif SANITIZER_NETBSD
   uptr mangled_sp = env[6];
 #elif SANITIZER_MAC
 # ifdef __aarch64__
-    uptr mangled_sp = env[13];
+  uptr mangled_sp =
+      (GetMacosVersion() >= MACOS_VERSION_MOJAVE) ? env[12] : env[13];
 # else
     uptr mangled_sp = env[2];
 # endif
 #elif SANITIZER_LINUX
 # ifdef __aarch64__
   uptr mangled_sp = env[13];
 # elif defined(__mips64)
   uptr mangled_sp = env[1];

lib/tsan/rtl/tsan_platform_mac.cc

#if defined(__aarch64__)
   if (max_vm != Mapping::kHiAppMemEnd) {
     Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
            max_vm, Mapping::kHiAppMemEnd);
     Die();
   }
 #endif
 }
 
+static const uptr kPthreadSetjmpXorKeySlot = 0x7;
+extern "C" uptr __tsan_darwin_setjmp_xor_key = 0;
+
 void InitializePlatform() {
   DisableCoreDumperIfNecessary();
 #if !SANITIZER_GO
   CheckAndProtect();
 
   CHECK_EQ(main_thread_identity, 0);
   main_thread_identity = (uptr)pthread_self();
 
   prev_pthread_introspection_hook =
       pthread_introspection_hook_install(&my_pthread_introspection_hook);
 #endif
+
+  if (GetMacosVersion() >= MACOS_VERSION_MOJAVE) {
+    __tsan_darwin_setjmp_xor_key =
+        (uptr)pthread_getspecific(kPthreadSetjmpXorKeySlot);
+  }
 }
 
 #if !SANITIZER_GO
 void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
   // The pointer to the ThreadState object is stored in the shadow memory
   // of the tls.
   uptr tls_end = tls_addr + tls_size;
   ThreadState **thr_state_loc = cur_thread_location();

lib/tsan/rtl/tsan_rtl_aarch64.S

 
 #if !defined(__APPLE__)
   // SP pointer mangling (see glibc setjmp)
   adrp    x2, __tsan_pointer_chk_guard
   ldr     x2, [x2, #:lo12:__tsan_pointer_chk_guard]
   add     x0, x29, 32
   eor     x1, x2, x0
 #else
+  adrp    x2, ___tsan_darwin_setjmp_xor_key@page
+  ldr     x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff]
   add     x0, x29, 32
-  mov     x1, x0
+  eor     x1, x2, x0
 #endif
 
   // call tsan interceptor
   bl      ASM_SYMBOL(__tsan_setjmp)
 
   // restore env parameter
   mov     x0, x19
   ldr     x19, [sp, 16]

 
 #if !defined(__APPLE__)
   // SP pointer mangling (see glibc setjmp)
   adrp    x2, __tsan_pointer_chk_guard
   ldr     x2, [x2, #:lo12:__tsan_pointer_chk_guard]
   add     x0, x29, 32
   eor     x1, x2, x0
 #else
+  adrp    x2, ___tsan_darwin_setjmp_xor_key@page
+  ldr     x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff]
   add     x0, x29, 32
-  mov     x1, x0
+  eor     x1, x2, x0
 #endif
 
   // call tsan interceptor
   bl      ASM_SYMBOL(__tsan_setjmp)
 
   // Restore jmp_buf parameter
   mov     x0, x19
   ldr     x19, [sp, 16]

 
 #if !defined(__APPLE__)
   // SP pointer mangling (see glibc setjmp)
   adrp    x2, __tsan_pointer_chk_guard
   ldr     x2, [x2, #:lo12:__tsan_pointer_chk_guard]
   add     x0, x29, 32
   eor     x1, x2, x0
 #else
+  adrp    x2, ___tsan_darwin_setjmp_xor_key@page
+  ldr     x2, [x2, ___tsan_darwin_setjmp_xor_key@pageoff]
   add     x0, x29, 32
-  mov     x1, x0
+  eor     x1, x2, x0
 #endif
 
   // call tsan interceptor
   bl      ASM_SYMBOL(__tsan_setjmp)
 
   // restore env parameter
   mov     w1, w20
   mov     x0, x19

lib/tsan/rtl/tsan_rtl_amd64.S

#endif
   CFI_REL_OFFSET(%rdi, 0)
   // obtain %rsp
 #if defined(__FreeBSD__) || defined(__NetBSD__)
   lea 8(%rsp), %rdi
   mov %rdi, %rsi
 #elif defined(__APPLE__)
   lea 16(%rsp), %rdi
   mov %rdi, %rsi
+  xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi
 #elif defined(__linux__)
   lea 16(%rsp), %rdi
   mov %rdi, %rsi
   xor %fs:0x30, %rsi  // magic mangling of rsp (see libc setjmp)
   rol $0x11, %rsi
 #else
 # error "Unknown platform"
 #endif
ASM_SYMBOL_INTERCEPTOR(_setjmp):
   CFI_REL_OFFSET(%rdi, 0)
   // obtain %rsp
 #if defined(__FreeBSD__) || defined(__NetBSD__)
   lea 8(%rsp), %rdi
   mov %rdi, %rsi
 #elif defined(__APPLE__)
   lea 16(%rsp), %rdi
   mov %rdi, %rsi
+  xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi
 #elif defined(__linux__)
   lea 16(%rsp), %rdi
   mov %rdi, %rsi
   xor %fs:0x30, %rsi  // magic mangling of rsp (see libc setjmp)
   rol $0x11, %rsi
 #else
 # error "Unknown platform"
 #endif
#endif
   CFI_ADJUST_CFA_OFFSET(8)
   // obtain %rsp
 #if defined(__FreeBSD__) || defined(__NetBSD__)
   lea 24(%rsp), %rdi
   mov %rdi, %rsi
 #elif defined(__APPLE__)
   lea 32(%rsp), %rdi
   mov %rdi, %rsi
+  xorq ___tsan_darwin_setjmp_xor_key(%rip), %rsi
 #elif defined(__linux__)
   lea 32(%rsp), %rdi
   mov %rdi, %rsi
   xor %fs:0x30, %rsi  // magic mangling of rsp (see libc setjmp)
   rol $0x11, %rsi
 #else
 # error "Unknown platform"
 #endif