[sanitizer] Implement internal_fork and internal_forkpty for OS X

[kubamracek]

Summary

On Linux, we have internal_fork that forks without invoking user's pthread_atfork handlers, which is important for spawning external symbolizers. Let's implement this for OS X as well (using __fork). This patch also adds internal_forkpty which re-implements forkpty and uses __fork in it as well.

[dvyukov]

LGTM with a nit

lib/sanitizer_common/sanitizer_mac.cc

162 ↗

(On Diff #40766)

add:

close(master);
close(slave);

compiler-rt/trunk/lib/sanitizer_common/sanitizer_mac.cc

 #include <signal.h>
 #include <stdlib.h>
 #include <sys/mman.h>
 #include <sys/resource.h>
 #include <sys/stat.h>
 #include <sys/sysctl.h>
 #include <sys/types.h>
 #include <unistd.h>
+#include <util.h>
 
 namespace __sanitizer {
 
 #include "sanitizer_syscall_generic.inc"
 
 // ---------------------- sanitizer_libc.h
 uptr internal_mmap(void *addr, size_t length, int prot, int flags,
                    int fd, u64 offset) {

 
 void internal_sigfillset(__sanitizer_sigset_t *set) { sigfillset(set); }
 
 uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
                           __sanitizer_sigset_t *oldset) {
   return sigprocmask(how, set, oldset);
 }
 
+// Doesn't call pthread_atfork() handlers.
+extern "C" pid_t __fork(void);
+
 int internal_fork() {
-  // TODO(glider): this may call user's pthread_atfork() handlers which is bad.
-  return fork();
+  return __fork();
+}
+
+int internal_forkpty(int *amaster) {
+  int master, slave;
+  if (openpty(&master, &slave, nullptr, nullptr, nullptr) == -1) return -1;
+  int pid = __fork();
+  if (pid == -1) {
+    close(master);
+    close(slave);
+    return -1;
+  }
+  if (pid == 0) {
+    close(master);
+    CHECK_EQ(login_tty(slave), 0);
+  } else {
+    *amaster = master;
+    close(slave);
+  }
+  return pid;
 }
 
 uptr internal_rename(const char *oldpath, const char *newpath) {
   return rename(oldpath, newpath);
 }
 
 uptr internal_ftruncate(fd_t fd, uptr size) {
   return ftruncate(fd, size);

compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix.h

 uptr internal_unlink(const char *path);
 uptr internal_rename(const char *oldpath, const char *newpath);
 uptr internal_lseek(fd_t fd, OFF_T offset, int whence);
 
 uptr internal_ptrace(int request, int pid, void *addr, void *data);
 uptr internal_waitpid(int pid, int *status, int options);
 
 int internal_fork();
+int internal_forkpty(int *amaster);
 
 // These functions call appropriate pthread_ functions directly, bypassing
 // the interceptor. They are weak and may not be present in some tools.
 SANITIZER_WEAK_ATTRIBUTE
 int real_pthread_create(void *th, void *attr, void *(*callback)(void *),
                         void *param);
 SANITIZER_WEAK_ATTRIBUTE
 int real_pthread_join(void *th, void **ret);

compiler-rt/trunk/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc

if (!FileExists(path_)) {
     return false;
   }
 
   int pid;
   if (use_forkpty_) {
 #if SANITIZER_MAC
     fd_t fd = kInvalidFd;
     // Use forkpty to disable buffering in the new terminal.
-    pid = forkpty(&fd, 0, 0, 0);
+    pid = internal_forkpty(&fd);
     if (pid == -1) {
       // forkpty() failed.
       Report("WARNING: failed to fork external symbolizer (errno: %d)\n",
              errno);
       return false;
     } else if (pid == 0) {
       // Child subprocess.
       const char *argv[kArgVMax];