[tsan] Add TSan unit test support for OS X

[kubamracek]

Summary

This patch enables building and running TSan unit tests on OS X. There are several changes:

• Move the detection of OSX_SYSROOT_FLAG to config-ix.cmake so it becomes a global-scope variable.
• Add building a static version of the runtime and link it with the unit tests.
• Modify tsan_test_util_linux.cc to be Darwin-compatible and add it into the OS X build. In a subsequent commit, I'll rename the file to tsan_test_util_posix.cc.
• Use symbolize=false:abort_on_error=0 as default TSAN_OPTIONS to speed up the unit tests significantly.
• In tsan_clock_test.cc, replace CLOCK_MONOTONIC with gettimeofday, which is available on OS X.

[dvyukov]

LGTM with a nit

lib/tsan/tests/rtl/tsan_test_util_linux.cc

143

Tests can rely on the fact that this is an actual mutex that provides actual synchronization. You turn it into no-op. It's probably not super important (as no tests fail and we do not add any more of these tests), but I think it is better to define Spin to Normal on mac:

enum Type {
  Normal,
  RW,

#ifndef APPLE

Spin

#else

// because ...
Spin = Normal

#endif

};

This way it is still a mutex from point of view of both execution and tsan.

cmake/Modules/CompilerRTCompile.cmake

foreach(flag ${msvc_flags})
     elseif ("${flag}" MATCHES "^-O[2x]")
       # Canonicalize normal optimization flags to -O2.
       list(APPEND clang_flags "-O2")
     endif()
   endforeach()
   set(${out_flags} "${clang_flags}" PARENT_SCOPE)
 endfunction()
 
-if (APPLE)
-  # On Darwin if /usr/include doesn't exist, the user probably has Xcode but not
-  # the command line tools. If this is the case, we need to find the OS X
-  # sysroot to pass to clang.
-  if(NOT EXISTS /usr/include)
-    execute_process(COMMAND xcodebuild -version -sdk macosx Path
-       OUTPUT_VARIABLE OSX_SYSROOT
-       ERROR_QUIET
-       OUTPUT_STRIP_TRAILING_WHITESPACE)
-    set(OSX_SYSROOT_FLAG "-isysroot${OSX_SYSROOT}")
-  endif()
-endif()
-
 # Compile a source into an object file with COMPILER_RT_TEST_COMPILER using
 # a provided compile flags and dependenices.
 # clang_compile(<object> <source>
 #               CFLAGS <list of compile flags>
 #               DEPS <list of dependencies>)
 macro(clang_compile object_file source)
   cmake_parse_arguments(SOURCE "" "" "CFLAGS;DEPS" ${ARGN})
   get_filename_component(source_rpath ${source} REALPATH)

cmake/config-ix.cmake

 set(ALL_TSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64})
 set(ALL_UBSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64}
     ${MIPS32} ${MIPS64} ${PPC64})
 set(ALL_SAFESTACK_SUPPORTED_ARCH ${X86} ${X86_64})
 
 if(APPLE)
   include(CompilerRTDarwinUtils)
 
+  # On Darwin if /usr/include doesn't exist, the user probably has Xcode but not
+  # the command line tools. If this is the case, we need to find the OS X
+  # sysroot to pass to clang.
+  if(NOT EXISTS /usr/include)
+    execute_process(COMMAND xcodebuild -version -sdk macosx Path
+       OUTPUT_VARIABLE OSX_SYSROOT
+       ERROR_QUIET
+       OUTPUT_STRIP_TRAILING_WHITESPACE)
+    set(OSX_SYSROOT_FLAG "-isysroot${OSX_SYSROOT}")
+  endif()
+
   option(COMPILER_RT_ENABLE_IOS "Enable building for iOS - Experimental" Off)
 
   find_darwin_sdk_dir(DARWIN_osx_SYSROOT macosx)
   find_darwin_sdk_dir(DARWIN_iossim_SYSROOT iphonesimulator)
   find_darwin_sdk_dir(DARWIN_ios_SYSROOT iphoneos)
 
   # Note: In order to target x86_64h on OS X the minimum deployment target must
   # be 10.8 or higher.

lib/tsan/CMakeLists.txt

set(TSAN_HEADERS
   rtl/tsan_sync.h
   rtl/tsan_trace.h
   rtl/tsan_update_shadow_word_inl.h
   rtl/tsan_vector.h)
 
 set(TSAN_RUNTIME_LIBRARIES)
 add_custom_target(tsan)
 
+add_compiler_rt_object_libraries(RTTsan_dynamic 
+  OS ${TSAN_SUPPORTED_OS}
+  ARCHS ${TSAN_SUPPORTED_ARCH}
+  SOURCES ${TSAN_SOURCES} ${TSAN_CXX_SOURCES}
+  CFLAGS ${TSAN_RTL_CFLAGS})
+
 if(APPLE)
   add_compiler_rt_runtime(clang_rt.tsan
     SHARED
     OS ${TSAN_SUPPORTED_OS}
     ARCHS ${TSAN_SUPPORTED_ARCH}
     SOURCES ${TSAN_SOURCES} ${TSAN_CXX_SOURCES}
     OBJECT_LIBS RTInterception
                 RTSanitizerCommon

lib/tsan/tests/CMakeLists.txt

macro(tsan_compile obj_list source arch)
   endif()
   clang_compile(${output_obj} ${source}
           CFLAGS ${TSAN_UNITTEST_CFLAGS} ${TARGET_CFLAGS}
           DEPS ${COMPILE_DEPS})
   list(APPEND ${obj_list} ${output_obj})
 endmacro()
 
 macro(add_tsan_unittest testname)
-  # Build unit tests only for 64-bit Linux.
-  if(UNIX AND NOT APPLE)
+  if(UNIX)
     foreach(arch ${TSAN_SUPPORTED_ARCH})
       cmake_parse_arguments(TEST "" "" "SOURCES;HEADERS" ${ARGN})
       set(TEST_OBJECTS)
       foreach(SOURCE ${TEST_SOURCES} ${COMPILER_RT_GTEST_SOURCE})
         tsan_compile(TEST_OBJECTS ${SOURCE} ${arch} ${TEST_HEADERS})
       endforeach()
       get_target_flags_for_arch(${arch} TARGET_LINK_FLAGS)
       set(TEST_DEPS ${TEST_OBJECTS})
       if(NOT COMPILER_RT_STANDALONE_BUILD)
         list(APPEND TEST_DEPS tsan)
       endif()
+      if(NOT APPLE)
         # FIXME: Looks like we should link TSan with just-built runtime,
         # and not rely on -fsanitize=thread, as these tests are essentially
         # unit tests.
         add_compiler_rt_test(TsanUnitTests ${testname}
                 OBJECTS ${TEST_OBJECTS}
                 DEPS ${TEST_DEPS}
                 LINK_FLAGS ${TARGET_LINK_FLAGS}
                            -fsanitize=thread
                            -lstdc++ -lm)
+      else()
+        set(TSAN_TEST_RUNTIME_OBJECTS
+          $<TARGET_OBJECTS:RTTsan_dynamic.osx>
+          $<TARGET_OBJECTS:RTInterception.osx>
+          $<TARGET_OBJECTS:RTSanitizerCommon.osx>
+          $<TARGET_OBJECTS:RTSanitizerCommonLibc.osx>
+          $<TARGET_OBJECTS:RTUbsan.osx>)
+        set(TSAN_TEST_RUNTIME RTTsanTest.${testname}.${arch})
+        add_library(${TSAN_TEST_RUNTIME} STATIC ${TSAN_TEST_RUNTIME_OBJECTS})
+        set_target_properties(${TSAN_TEST_RUNTIME} PROPERTIES
+          ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+        list(APPEND TEST_OBJECTS lib${TSAN_TEST_RUNTIME}.a)
+        list(APPEND TEST_DEPS ${TSAN_TEST_RUNTIME})
+        # Intentionally do *not* link with `-fsanitize=thread`. We already link
+        # against a static version of the runtime, and we don't want the dynamic
+        # one.
+        add_compiler_rt_test(TsanUnitTests "${testname}-${arch}-Test"
+                OBJECTS ${TEST_OBJECTS}
+                DEPS ${TEST_DEPS}
+                LINK_FLAGS ${TARGET_LINK_FLAGS}
+                           -lc++)
+      endif()
     endforeach()
   endif()
 endmacro()
 
 if(COMPILER_RT_CAN_EXECUTE_TESTS)
   add_subdirectory(rtl)
   add_subdirectory(unit)
 endif()

lib/tsan/tests/rtl/CMakeLists.txt

 set(TSAN_RTL_TEST_SOURCES
   tsan_bench.cc
   tsan_mop.cc
   tsan_mutex.cc
   tsan_posix.cc
   tsan_string.cc
   tsan_test.cc
   tsan_thread.cc)
 
-if(UNIX AND NOT APPLE)
+if(UNIX)
   list(APPEND TSAN_RTL_TEST_SOURCES tsan_test_util_linux.cc)
 endif()
 
 set(TSAN_RTL_TEST_HEADERS
   tsan_test_util.h)
 
 add_tsan_unittest(TsanRtlTest
   SOURCES ${TSAN_RTL_TEST_SOURCES}
   HEADERS ${TSAN_RTL_TEST_HEADERS})

lib/tsan/tests/rtl/tsan_test.cc

int run_tests(int argc, char **argv) {
 
   __tsan_func_exit();
   __tsan_func_exit();
   return res;
 }
 
 const char *argv0;
 
+#ifdef __APPLE__
+// On Darwin, turns off symbolication and crash logs to make tests faster.
+extern "C" const char* __tsan_default_options() {
+  return "symbolize=false:abort_on_error=0";
+}
+#endif
+
 int main(int argc, char **argv) {
   argv0 = argv[0];
   return run_tests(argc, argv);
 }

lib/tsan/tests/rtl/tsan_test_util_linux.cc

-
 //===-- tsan_test_util_linux.cc -------------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of ThreadSanitizer (TSan), a race detector.
 //
-// Test utils, Linux and FreeBSD implementation.
+// Test utils, Linux, FreeBSD and Darwin implementation.
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "tsan_interface.h"
 #include "tsan_test_util.h"
 #include "tsan_report.h"
 
 #include "gtest/gtest.h"
 
 #include <assert.h>
 #include <pthread.h>
 #include <stdio.h>
 #include <stdint.h>
 #include <string.h>
 #include <unistd.h>
 #include <errno.h>
 
 using namespace __tsan;  // NOLINT
 
 static __thread bool expect_report;
 static __thread bool expect_report_reported;
 static __thread ReportType expect_report_type;
 
-extern "C" void *__interceptor_memcpy(void*, const void*, uptr);
-extern "C" void *__interceptor_memset(void*, int, uptr);
+#ifdef __APPLE__
+#define __interceptor_memcpy wrap_memcpy
+#define __interceptor_memset wrap_memset
+#define __interceptor_pthread_create wrap_pthread_create
+#define __interceptor_pthread_join wrap_pthread_join
+#endif
+
+extern "C" void *__interceptor_memcpy(void *, const void *, uptr);
+extern "C" void *__interceptor_memset(void *, int, uptr);
+extern "C" int __interceptor_pthread_create(pthread_t *thread,
+                                            const pthread_attr_t *attr,
+                                            void *(*start_routine)(void *),
+                                            void *arg);
+extern "C" int __interceptor_pthread_join(pthread_t thread, void **value_ptr);
 
 static void *BeforeInitThread(void *param) {
   (void)param;
   return 0;
 }
 
 static void AtExit() {
 }
 
 void TestMutexBeforeInit() {
   // Mutexes must be usable before __tsan_init();
   pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER;
   pthread_mutex_lock(&mtx);
   pthread_mutex_unlock(&mtx);
   pthread_mutex_destroy(&mtx);
   pthread_t thr;
-  pthread_create(&thr, 0, BeforeInitThread, 0);
-  pthread_join(thr, 0);
+  __interceptor_pthread_create(&thr, 0, BeforeInitThread, 0);
+  __interceptor_pthread_join(thr, 0);
   atexit(AtExit);
 }
 
 namespace __tsan {
 bool OnReport(const ReportDesc *rep, bool suppressed) {
   if (expect_report) {
     if (rep->typ != expect_report_type) {
       printf("Expected report of type %d, got type %d\n",
Mutex::~Mutex() {
   CHECK(!alive_);
 }
 
 void Mutex::Init() {
   CHECK(!alive_);
   alive_ = true;
   if (type_ == Normal)
     CHECK_EQ(pthread_mutex_init((pthread_mutex_t*)mtx_, 0), 0);
+#ifndef __APPLE__
   else if (type_ == Spin)
     CHECK_EQ(pthread_spin_init((pthread_spinlock_t*)mtx_, 0), 0);
+#endif
   else if (type_ == RW)
     CHECK_EQ(pthread_rwlock_init((pthread_rwlock_t*)mtx_, 0), 0);
   else
     CHECK(0);
 }
 
 void Mutex::StaticInit() {
   CHECK(!alive_);
   CHECK(type_ == Normal);
   alive_ = true;
   pthread_mutex_t tmp = PTHREAD_MUTEX_INITIALIZER;
   memcpy(mtx_, &tmp, sizeof(tmp));
 }
 
 void Mutex::Destroy() {
   CHECK(alive_);
   alive_ = false;
   if (type_ == Normal)
     CHECK_EQ(pthread_mutex_destroy((pthread_mutex_t*)mtx_), 0);
+#ifndef __APPLE__
   else if (type_ == Spin)
     CHECK_EQ(pthread_spin_destroy((pthread_spinlock_t*)mtx_), 0);
+#endif
   else if (type_ == RW)
     CHECK_EQ(pthread_rwlock_destroy((pthread_rwlock_t*)mtx_), 0);
 }
 
 void Mutex::Lock() {
   CHECK(alive_);
   if (type_ == Normal)
     CHECK_EQ(pthread_mutex_lock((pthread_mutex_t*)mtx_), 0);
+#ifndef __APPLE__
   else if (type_ == Spin)
     CHECK_EQ(pthread_spin_lock((pthread_spinlock_t*)mtx_), 0);
+#endif
   else if (type_ == RW)
     CHECK_EQ(pthread_rwlock_wrlock((pthread_rwlock_t*)mtx_), 0);
 }
 
 bool Mutex::TryLock() {
   CHECK(alive_);
   if (type_ == Normal)
     return pthread_mutex_trylock((pthread_mutex_t*)mtx_) == 0;
+#ifndef __APPLE__
   else if (type_ == Spin)
     return pthread_spin_trylock((pthread_spinlock_t*)mtx_) == 0;
+#endif
   else if (type_ == RW)
     return pthread_rwlock_trywrlock((pthread_rwlock_t*)mtx_) == 0;
   return false;
 }
 
 void Mutex::Unlock() {
   CHECK(alive_);
   if (type_ == Normal)
     CHECK_EQ(pthread_mutex_unlock((pthread_mutex_t*)mtx_), 0);
+#ifndef __APPLE__
   else if (type_ == Spin)
     CHECK_EQ(pthread_spin_unlock((pthread_spinlock_t*)mtx_), 0);
+#endif
   else if (type_ == RW)
     CHECK_EQ(pthread_rwlock_unlock((pthread_rwlock_t*)mtx_), 0);
 }
 
 void Mutex::ReadLock() {
   CHECK(alive_);
   CHECK(type_ == RW);
   CHECK_EQ(pthread_rwlock_rdlock((pthread_rwlock_t*)mtx_), 0);
if (ev->type == Event::READ) {
         case 1: tsan_mop = __tsan_write1; break;
         case 2: tsan_mop = __tsan_write2; break;
         case 4: tsan_mop = __tsan_write4; break;
         case 8: tsan_mop = __tsan_write8; break;
         case 16: tsan_mop = __tsan_write16; break;
       }
     }
     CHECK_NE(tsan_mop, 0);
-#if defined(__FreeBSD__)
+#if defined(__FreeBSD__) || defined(__APPLE__)
     const int ErrCode = ESOCKTNOSUPPORT;
 #else
     const int ErrCode = ECHRNG;
 #endif
     errno = ErrCode;
     tsan_mop(ev->ptr);
     CHECK_EQ(ErrCode, errno);  // In no case must errno be changed.
     break;

 }
 
 void *ScopedThread::Impl::ScopedThreadCallback(void *arg) {
   __tsan_func_entry(__builtin_return_address(0));
   Impl *impl = (Impl*)arg;
   for (;;) {
     Event* ev = (Event*)atomic_load(&impl->event, memory_order_acquire);
     if (ev == 0) {
-      pthread_yield();
+      sched_yield();
       continue;
     }
     if (ev->type == Event::SHUTDOWN) {
       atomic_store(&impl->event, 0, memory_order_release);
       break;
     }
     impl->HandleEvent(ev);
     atomic_store(&impl->event, 0, memory_order_release);
   }
   __tsan_func_exit();
   return 0;
 }
 
 void ScopedThread::Impl::send(Event *e) {
   if (main) {
     HandleEvent(e);
   } else {
     CHECK_EQ(atomic_load(&event, memory_order_relaxed), 0);
     atomic_store(&event, (uintptr_t)e, memory_order_release);
     while (atomic_load(&event, memory_order_acquire) != 0)
-      pthread_yield();
+      sched_yield();
   }
 }
 
 ScopedThread::ScopedThread(bool detached, bool main) {
   impl_ = new Impl;
   impl_->main = main;
   impl_->detached = detached;
   atomic_store(&impl_->event, 0, memory_order_relaxed);
   if (!main) {
     pthread_attr_t attr;
     pthread_attr_init(&attr);
     pthread_attr_setdetachstate(&attr, detached);
     pthread_attr_setstacksize(&attr, 64*1024);
-    pthread_create(&impl_->thread, &attr,
+    __interceptor_pthread_create(&impl_->thread, &attr,
         ScopedThread::Impl::ScopedThreadCallback, impl_);
   }
 }
 
 ScopedThread::~ScopedThread() {
   if (!impl_->main) {
     Event event(Event::SHUTDOWN);
     impl_->send(&event);
     if (!impl_->detached)
-      pthread_join(impl_->thread, 0);
+      __interceptor_pthread_join(impl_->thread, 0);
   }
   delete impl_;
 }
 
 void ScopedThread::Detach() {
   CHECK(!impl_->main);
   CHECK(!impl_->detached);
   impl_->detached = true;

lib/tsan/tests/unit/tsan_clock_test.cc

 //===-- tsan_clock_test.cc ------------------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of ThreadSanitizer (TSan), a race detector.
 //
 //===----------------------------------------------------------------------===//
 #include "tsan_clock.h"
 #include "tsan_rtl.h"
 #include "gtest/gtest.h"
+#include <sys/time.h>
 #include <time.h>
 
 namespace __tsan {
 
 ClockCache cache;
 
 TEST(Clock, VectorBasic) {
   ThreadClock clk(0);
static bool ClockFuzzer(bool printing) {
 
   for (unsigned i = 0; i < kClocks; i++) {
     sync1[i]->Reset(&cache);
   }
   return true;
 }
 
 TEST(Clock, Fuzzer) {
-  timespec ts;
-  clock_gettime(CLOCK_MONOTONIC, &ts);
-  int seed = ts.tv_sec + ts.tv_nsec;
+  struct timeval tv;
+  gettimeofday(&tv, NULL);
+  int seed = tv.tv_sec + tv.tv_usec;
   printf("seed=%d\n", seed);
   srand(seed);
   if (!ClockFuzzer(false)) {
     // Redo the test with the same seed, but logging operations.
     srand(seed);
     ClockFuzzer(true);
     ASSERT_TRUE(false);
   }
 }
 
 }  // namespace __tsan