[tsan] Use direct syscalls for internal_mmap and internal_munmap on OS X

[kubamracek]

Summary

On OS X, internal_mmap just uses mmap, which can invoke callbacks into libmalloc (e.g. when MallocStackLogging is enabled). This can subsequently call other intercepted functions, and this breaks our Darwin-specific ThreadState initialization. Let's use direct syscalls in internal_mmap and internal_munmap. Added a testcase.

[glider]

compiler-rt/trunk/test/tsan/Darwin/malloc-stack-logging.cc

1 ↗

(On Diff #51537)

It's not clear from the test case what it is testing.

[kubamracek]

compiler-rt/trunk/test/tsan/Darwin/malloc-stack-logging.cc

1 ↗

(On Diff #51537)

Added a comment in r264271.

lib/sanitizer_common/sanitizer_mac.cc

extern "C" {
   extern char ***_NSGetArgv(void);
   extern char ***_NSGetEnviron(void);
 }
 
 namespace __sanitizer {
 
 #include "sanitizer_syscall_generic.inc"
 
+// Direct syscalls, don't call libmalloc hooks.
+extern "C" void *__mmap(void *addr, size_t len, int prot, int flags, int fildes,
+                        off_t off);
+extern "C" int __munmap(void *, size_t);
+
 // ---------------------- sanitizer_libc.h
 uptr internal_mmap(void *addr, size_t length, int prot, int flags,
                    int fd, u64 offset) {
   if (fd == -1) fd = VM_MAKE_TAG(VM_MEMORY_ANALYSIS_TOOL);
-  return (uptr)mmap(addr, length, prot, flags, fd, offset);
+  return (uptr)__mmap(addr, length, prot, flags, fd, offset);
 }
 
 uptr internal_munmap(void *addr, uptr length) {
-  return munmap(addr, length);
+  return __munmap(addr, length);
 }
 
 int internal_mprotect(void *addr, uptr length, int prot) {
   return mprotect(addr, length, prot);
 }
 
 uptr internal_close(fd_t fd) {
   return close(fd);

test/tsan/Darwin/malloc-stack-logging.cc

+// RUN: %clangxx_tsan -O1 %s -o %t
+// RUN: MallocStackLogging=1 %run %t 2>&1 | FileCheck %s
+#include <pthread.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+void *foo(void *p) {
+    return NULL;
+}
+
+int main() {
+    pthread_t t;
+    pthread_create(&t, NULL, foo, NULL);
+    pthread_join(t, NULL);
+    fprintf(stderr, "Done.\n");
+    return 0;
+}
+
+// CHECK: Done.