[TSan] Add interceptors for mach_vm_[de]allocate

[yln]

Summary

I verified that the test is red without the interceptors.

rdar://40334350

[yln]

Fix warning in test.

[kubamracek]

compiler-rt/lib/tsan/rtl/tsan_interceptors.cpp

748–749 ↗

(On Diff #216696)

is this related? or just a co-incidental change?

compiler-rt/lib/tsan/rtl/tsan_interceptors_mach_vm.cpp

42–45 ↗

(On Diff #216696)

This is copied from mmap, right? Can it be extracted into a common function instead?

58–62 ↗

(On Diff #216696)

ditto here

[yln]

compiler-rt/lib/tsan/rtl/tsan_interceptors.cpp

748–749 ↗

(On Diff #216696)

Related. It took me a while to understand what it is doing, so I left a comment.

void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset);

1) addr=0, flags=... -> kernel picks address, best most portable option
2) addr=0x123, flags=... -> kernel picks address, 'addr' is treated as a hint, no guarantees
3) addr=0x123, flag=MAP_FIXED -> mmap fails if allocation cannot be placed at 'addr'

This function ensures that we never call the real mmap with an addr that points into the shadow. For 2) it discards the hint, for 3) we return with failure without forwarding to the real mmap.

vm_mach_allocate only has cases 1) and 3). I named the function intersects_with_shadow

compiler-rt/lib/tsan/rtl/tsan_interceptors_mach_vm.cpp

42–45 ↗

(On Diff #216696)

I tried extracting a function and then parameterizing it for most of the interceptor, but that turned messy really quickly. I can certainly do it for the above 4 lines.

[yln]

Extract UnmapShadow and MemoryRangeImitateWriteOrResetRange
helper functions to reduce code duplication.

[kubamracek]

Looks good to me. @dvyukov ?

[yln]

@dvyukov, do you want to weigh in here?

[yln]

*ping*
Any comments before I ask Kuba to waive this through?

[vitalybuka]

compiler-rt/lib/tsan/rtl/tsan_interceptors_mach_vm.cpp

34 ↗

(On Diff #217213)

maybe drop {} on one-liners for consistency

[yln]

Drop {} on one-liners for consistency.

compiler-rt/trunk/lib/tsan/CMakeLists.txt

 
 set(TSAN_CXX_SOURCES
   rtl/tsan_new_delete.cpp
   )
 
 if(APPLE)
   list(APPEND TSAN_SOURCES
     rtl/tsan_interceptors_mac.cpp
+    rtl/tsan_interceptors_mach_vm.cpp
     rtl/tsan_platform_mac.cpp
     rtl/tsan_platform_posix.cpp
     )
 elseif(UNIX)
   # Assume Linux
   list(APPEND TSAN_SOURCES
     rtl/tsan_platform_linux.cpp
     rtl/tsan_platform_posix.cpp

compiler-rt/trunk/lib/tsan/rtl/tsan_interceptors.cpp

 }
 
 TSAN_INTERCEPTOR(char*, strdup, const char *str) {
   SCOPED_TSAN_INTERCEPTOR(strdup, str);
   // strdup will call malloc, so no instrumentation is required here.
   return REAL(strdup)(str);
 }
 
+// Zero out addr if it points into shadow memory and was provided as a hint
+// only, i.e., MAP_FIXED is not set.
 static bool fix_mmap_addr(void **addr, long_t sz, int flags) {
   if (*addr) {
     if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) {
       if (flags & MAP_FIXED) {
         errno = errno_EINVAL;
         return false;
       } else {
         *addr = 0;
       }
     }
   }
   return true;
 }
 
 template <class Mmap>
 static void *mmap_interceptor(ThreadState *thr, uptr pc, Mmap real_mmap,
                               void *addr, SIZE_T sz, int prot, int flags,
                               int fd, OFF64_T off) {
   if (!fix_mmap_addr(&addr, sz, flags)) return MAP_FAILED;
   void *res = real_mmap(addr, sz, prot, flags, fd, off);
   if (res != MAP_FAILED) {
     if (fd > 0) FdAccess(thr, pc, fd);
-    if (thr->ignore_reads_and_writes == 0)
-      MemoryRangeImitateWrite(thr, pc, (uptr)res, sz);
-    else
-      MemoryResetRange(thr, pc, (uptr)res, sz);
+    MemoryRangeImitateWriteOrResetRange(thr, pc, (uptr)res, sz);
   }
   return res;
 }
 
 TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) {
   SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz);
-  if (sz != 0) {
-    // If sz == 0, munmap will return EINVAL and don't unmap any memory.
-    DontNeedShadowFor((uptr)addr, sz);
-    ScopedGlobalProcessor sgp;
-    ctx->metamap.ResetRange(thr->proc(), (uptr)addr, (uptr)sz);
-  }
+  UnmapShadow(thr, (uptr)addr, sz);
   int res = REAL(munmap)(addr, sz);
   return res;
 }
 
 #if SANITIZER_LINUX
 TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
   SCOPED_INTERCEPTOR_RAW(memalign, align, sz);
   return user_memalign(thr, pc, align, sz);

compiler-rt/trunk/lib/tsan/rtl/tsan_interceptors_mach_vm.cpp

+//===-- tsan_interceptors_mach_vm.cpp -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Interceptors for mach_vm_* user space memory routines on Darwin.
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "tsan_interceptors.h"
+#include "tsan_platform.h"
+
+#include <mach/mach.h>
+
+namespace __tsan {
+
+static bool intersects_with_shadow(mach_vm_address_t *address,
+                                   mach_vm_size_t size, int flags) {
+  // VM_FLAGS_FIXED is 0x0, so we have to test for VM_FLAGS_ANYWHERE.
+  if (flags & VM_FLAGS_ANYWHERE) return false;
+  uptr ptr = *address;
+  return !IsAppMem(ptr) || !IsAppMem(ptr + size - 1);
+}
+
+TSAN_INTERCEPTOR(kern_return_t, mach_vm_allocate, vm_map_t target,
+                 mach_vm_address_t *address, mach_vm_size_t size, int flags) {
+  SCOPED_TSAN_INTERCEPTOR(mach_vm_allocate, target, address, size, flags);
+  if (target != mach_task_self())
+    return REAL(mach_vm_allocate)(target, address, size, flags);
+  if (intersects_with_shadow(address, size, flags))
+    return KERN_NO_SPACE;
+  kern_return_t res = REAL(mach_vm_allocate)(target, address, size, flags);
+  if (res == KERN_SUCCESS)
+    MemoryRangeImitateWriteOrResetRange(thr, pc, *address, size);
+  return res;
+}
+
+TSAN_INTERCEPTOR(kern_return_t, mach_vm_deallocate, vm_map_t target,
+                 mach_vm_address_t address, mach_vm_size_t size) {
+  SCOPED_TSAN_INTERCEPTOR(mach_vm_deallocate, target, address, size);
+  if (target != mach_task_self())
+    return REAL(mach_vm_deallocate)(target, address, size);
+  UnmapShadow(thr, address, size);
+  return REAL(mach_vm_deallocate)(target, address, size);
+}
+
+}  // namespace __tsan

compiler-rt/trunk/lib/tsan/rtl/tsan_rtl.h

 #if TSAN_COLLECT_STATS
   thr->stat[typ] = n;
 #endif
 }
 
 void MapShadow(uptr addr, uptr size);
 void MapThreadTrace(uptr addr, uptr size, const char *name);
 void DontNeedShadowFor(uptr addr, uptr size);
+void UnmapShadow(ThreadState *thr, uptr addr, uptr size);
 void InitializeShadowMemory();
 void InitializeInterceptors();
 void InitializeLibIgnore();
 void InitializeDynamicAnnotations();
 
 void ForkBefore(ThreadState *thr, uptr pc);
 void ForkParentAfter(ThreadState *thr, uptr pc);
 void ForkChildAfter(ThreadState *thr, uptr pc);

 void ALWAYS_INLINE MemoryWriteAtomic(ThreadState *thr, uptr pc,
                                             uptr addr, int kAccessSizeLog) {
   MemoryAccess(thr, pc, addr, kAccessSizeLog, true, true);
 }
 
 void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
 void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
 void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
+void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
+                                         uptr size);
 
 void ThreadIgnoreBegin(ThreadState *thr, uptr pc, bool save_stack = true);
 void ThreadIgnoreEnd(ThreadState *thr, uptr pc);
 void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc, bool save_stack = true);
 void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc);
 
 void FuncEntry(ThreadState *thr, uptr pc);
 void FuncExit(ThreadState *thr);

compiler-rt/trunk/lib/tsan/rtl/tsan_rtl.cpp

 }
 #endif
 #endif
 
 void DontNeedShadowFor(uptr addr, uptr size) {
   ReleaseMemoryPagesToOS(MemToShadow(addr), MemToShadow(addr + size));
 }
 
+#if !SANITIZER_GO
+void UnmapShadow(ThreadState *thr, uptr addr, uptr size) {
+  if (size == 0) return;
+  DontNeedShadowFor(addr, size);
+  ScopedGlobalProcessor sgp;
+  ctx->metamap.ResetRange(thr->proc(), addr, size);
+}
+#endif
+
 void MapShadow(uptr addr, uptr size) {
   // Global data is not 64K aligned, but there are no adjacent mappings,
   // so we can get away with unaligned mapping.
   // CHECK_EQ(addr, addr & ~((64 << 10) - 1));  // windows wants 64K alignment
   const uptr kPageSize = GetPageSizeCached();
   uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize);
   uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize);
   if (!MmapFixedNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"))
void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) {
   }
   Shadow s(thr->fast_state);
   s.ClearIgnoreBit();
   s.SetWrite(true);
   s.SetAddr0AndSizeLog(0, 3);
   MemoryRangeSet(thr, pc, addr, size, s.raw());
 }
 
+void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
+                                         uptr size) {
+  if (thr->ignore_reads_and_writes == 0)
+    MemoryRangeImitateWrite(thr, pc, addr, size);
+  else
+    MemoryResetRange(thr, pc, addr, size);
+}
+
 ALWAYS_INLINE USED
 void FuncEntry(ThreadState *thr, uptr pc) {
   StatInc(thr, StatFuncEnter);
   DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc);
   if (kCollectHistory) {
     thr->fast_state.IncrementEpoch();
     TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc);
   }

compiler-rt/trunk/test/tsan/Darwin/mach_vm_allocate.c

+// Test that mach_vm_[de]allocate resets shadow memory status.
+//
+// RUN: %clang_tsan %s -o %t
+// RUN: %run %t 2>&1 | FileCheck %s --implicit-check-not='ThreadSanitizer'
+
+#include <mach/mach.h>
+#include <mach/mach_vm.h>
+#include <pthread.h>
+#include <assert.h>
+#include <stdio.h>
+
+#include "../test.h"
+
+void AnnotateIgnoreReadsBegin(const char *f, int l);
+void AnnotateIgnoreReadsEnd(const char *f, int l);
+void AnnotateIgnoreWritesBegin(const char *f, int l);
+void AnnotateIgnoreWritesEnd(const char *f, int l);
+
+static int *global_ptr;
+const mach_vm_size_t alloc_size = sizeof(int);
+
+static int *alloc() {
+  mach_vm_address_t addr;
+  kern_return_t res =
+      mach_vm_allocate(mach_task_self(), &addr, alloc_size, VM_FLAGS_ANYWHERE);
+  assert(res == KERN_SUCCESS);
+  return (int *)addr;
+}
+
+static void alloc_fixed(int *ptr) {
+  mach_vm_address_t addr = (mach_vm_address_t)ptr;
+  kern_return_t res =
+      mach_vm_allocate(mach_task_self(), &addr, alloc_size, VM_FLAGS_FIXED);
+  assert(res == KERN_SUCCESS);
+}
+
+static void dealloc(int *ptr) {
+  kern_return_t res =
+      mach_vm_deallocate(mach_task_self(), (mach_vm_address_t)ptr, alloc_size);
+  assert(res == KERN_SUCCESS);
+}
+
+static void *Thread(void *arg) {
+  *global_ptr = 7;  // Assignment 1
+
+  // We want to test that TSan does not report a race between the two
+  // assignments to global_ptr when memory is re-allocated here. The calls to
+  // the API itself are racy though, so ignore them.
+  AnnotateIgnoreWritesBegin(__FILE__, __LINE__);
+  dealloc(global_ptr);
+  alloc_fixed(global_ptr);
+  AnnotateIgnoreWritesEnd(__FILE__, __LINE__);
+
+  barrier_wait(&barrier);
+  return NULL;;
+}
+
+int main(int argc, const char *argv[]) {
+  barrier_init(&barrier, 2);
+  global_ptr = alloc();
+  pthread_t t;
+  pthread_create(&t, NULL, Thread, NULL);
+
+  barrier_wait(&barrier);
+  *global_ptr = 8;  // Assignment 2
+
+  pthread_join(t, NULL);
+  dealloc(global_ptr);
+  printf("Done.\n");
+  return 0;
+}
+
+// CHECK: Done.