[CMake][TSAN] Add build dependency against libcxx

[avogelsgesang]

Summary

Our internal OSX build machines were hitting race conditions while
building the runtime libraries during a bootstrap build, i.e. building
both libcxx and TSAN in the same build. TSAN includes parts of libcxx
(errno.h, ...) but the CMakeLists.txt didn't declare this dependency.

With this change, our (Tableau's) bootstrapping on OSX works as expected.

[avogelsgesang]

Adding reviewers based on CODE_OWNERS.TXT

[avogelsgesang]

Adding phosek based on his commit 6db314e86b26741c2e29ce51d88a6a5dca35336c

I think this is the commit which broke our Apple build...
My proposed change here undoes poarts of his changes.

But I admittedly don't understand LLVM's CMake deeply enough to know whether my proposed fix is indeed correct, or if there is a better way to do so

[avogelsgesang]

Some more context:

Without this patch, our build system failed with the output:

[12:12:49] :					 [clang-build-stage1] [ 68%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/clang_rt.tsan_iossim_dynamic.dir/rtl/tsan_external.cpp.o
[12:12:49] :					 [clang-build-stage1] [ 68%] Copying CXX header __support/xlocale/__strtonum_fallback.h
[12:12:49] :					 [clang-build-stage1] [ 68%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/RTTsan_dynamic.iossim.dir/rtl/tsan_rtl_thread.cpp.o
[12:12:49] :					 [clang-build-stage1] [ 68%] Copying CXX header system_error
[12:12:49] :					 [clang-build-stage1] [ 68%] Building CXX object compiler-rt/lib/asan/CMakeFiles/RTAsan_dynamic.osx.dir/asan_new_delete.cpp.o
[12:12:49] :					 [clang-build-stage1] [ 68%] Copying CXX header tgmath.h
[12:12:49] :					 [clang-build-stage1] [ 68%] Copying CXX header thread
[12:12:49] :					 [clang-build-stage1] [ 68%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/RTTsan_dynamic.osx.dir/rtl/tsan_rtl.cpp.o
[12:12:49] :					 [clang-build-stage1] [ 68%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/RTTsan_dynamic.osx.dir/rtl/tsan_rtl_mutex.cpp.o
[12:12:49] :					 [clang-build-stage1] [ 69%] Copying CXX header tuple
[12:12:49] :					 [clang-build-stage1] In file included from /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/compiler-rt/lib/asan/asan_new_delete.cpp:22:
[12:12:49] :					 [clang-build-stage1] /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/build/stage1_build/./bin/../include/c++/v1/stddef.h:39:10: fatal error: '__config' file not found
[12:12:49] :					 [clang-build-stage1] #include <__config>
[12:12:49] :					 [clang-build-stage1]          ^~~~~~~~~~
[12:12:49] :					 [clang-build-stage1] 1 error generated.
[12:12:49] :					 [clang-build-stage1] make[5]: *** [compiler-rt/lib/asan/CMakeFiles/RTAsan_dynamic.osx.dir/asan_new_delete.cpp.o] Error 1
[12:12:49] :					 [clang-build-stage1] make[4]: *** [compiler-rt/lib/asan/CMakeFiles/RTAsan_dynamic.osx.dir/all] Error 2
[12:12:49] :					 [clang-build-stage1] [ 69%] Copying CXX header type_traits
[12:12:49] :					 [clang-build-stage1] [ 69%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/clang_rt.tsan_osx_dynamic.dir/rtl/tsan_fd.cpp.o
[12:12:49] :					 [clang-build-stage1] [ 69%] Copying CXX header typeindex
[12:12:49] :					 [clang-build-stage1] [ 69%] Copying CXX header typeinfo
[12:12:50] :					 [clang-build-stage1] [ 69%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/clang_rt.tsan_iossim_dynamic.dir/rtl/tsan_fd.cpp.o
[12:12:50] :					 [clang-build-stage1] [ 69%] Copying CXX header unordered_map
[12:12:50] :					 [clang-build-stage1] [ 69%] Copying CXX header unordered_set
[12:12:50] :					 [clang-build-stage1] [ 69%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/clang_rt.tsan_iossim_dynamic.dir/rtl/tsan_flags.cpp.o
[12:12:50] :					 [clang-build-stage1] [ 69%] Copying CXX header utility
[12:12:50] :					 [clang-build-stage1] [ 69%] Copying CXX header valarray
[12:12:50] :					 [clang-build-stage1] [ 69%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/RTTsan_dynamic.osx.dir/rtl/tsan_rtl_proc.cpp.o
[12:12:50] :					 [clang-build-stage1] [ 69%] Copying CXX header variant
[12:12:50] :					 [clang-build-stage1] [ 69%] Copying CXX header vector
[12:12:50] :					 [clang-build-stage1] [ 69%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/clang_rt.tsan_iossim_dynamic.dir/rtl/tsan_ignoreset.cpp.o
[12:12:50] :					 [clang-build-stage1] [ 69%] Copying CXX header version
[12:12:50] :					 [clang-build-stage1] [ 69%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/RTTsan_dynamic.osx.dir/rtl/tsan_rtl_report.cpp.o
[12:12:50] :					 [clang-build-stage1] [ 69%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/clang_rt.tsan_osx_dynamic.dir/rtl/tsan_flags.cpp.o
[12:12:50] :					 [clang-build-stage1] [ 69%] Copying CXX header wchar.h
[12:12:50] :					 [clang-build-stage1] [ 70%] Copying CXX header wctype.h
[12:12:50] :					 [clang-build-stage1] [ 70%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/clang_rt.tsan_iossim_dynamic.dir/rtl/tsan_interceptors_posix.cpp.o
[12:12:50] :					 [clang-build-stage1] [ 71%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/RTTsan_dynamic.iossim.dir/rtl/tsan_stack_trace.cpp.o
[12:12:50] :					 [clang-build-stage1] [ 71%] Copying CXX __config
[12:12:50] :					 [clang-build-stage1] [ 71%] Built target generate-cxx-headers

Note how the missing include __config occurred at 12:12:49, and "Copying CXX __config" took place 1 second later, at 12:12:50

[phosek]

In D106143#2883083, @avogelsgesang wrote:

[12:12:49] : [clang-build-stage1] In file included from /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/compiler-rt/lib/asan/asan_new_delete.cpp:22:
[12:12:49] : [clang-build-stage1] /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/build/stage1_build/./bin/../include/c++/v1/stddef.h:39:10: fatal error: 'config' file not found
[12:12:49] : [clang-build-stage1] #include <config>
[12:12:49] : [clang-build-stage1] ^~~~~~~~~~
[12:12:49] : [clang-build-stage1] 1 error generated.
[12:12:49] : [clang-build-stage1] make[5]: * [compiler-rt/lib/asan/CMakeFiles/RTAsan_dynamic.osx.dir/asan_new_delete.cpp.o] Error 1
[12:12:49] : [clang-build-stage1] make[4]: * [compiler-rt/lib/asan/CMakeFiles/RTAsan_dynamic.osx.dir/all] Error 2

That error shows issue when compiling ASan, not TSan, so I'm still not sure why this change is needed?

compiler-rt/lib/tsan/CMakeLists.txt

117

Headers like errno.h or stdio.h are provided by libc, not libc++, do you know which libc++ headers are needed by TSan?

[avogelsgesang]

That error shows issue when compiling ASan, not TSan, so I'm still not sure why this change is needed?

Good catch!
I actually executed the build multiple times, and the first time it was failing in TSAN.
I made the incorrect assumption that all errors would be due to the TSAN, but didn't check all instances.
Above, I just copied the one variant which showed the race condition most impressively (i.e. with only 1 second between failed cpp compilation error and and copying of __config).
I completely overlooked that not all errors were the same, but that they were for different components in the build.

From a different build log (showing errors in lsan/sanitizer_common, ASAN and TSAN):

[12:08:24] :					 [clang-build-stage1] [ 64%] Copying CXX header stdio.h
[12:08:24] :					 [clang-build-stage1] In file included from /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/compiler-rt/lib/lsan/lsan_malloc_mac.cpp:57:
[12:08:24] :					 [clang-build-stage1] In file included from /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/compiler-rt/lib/lsan/../sanitizer_common/sanitizer_malloc_mac.inc:20:
[12:08:24] :					 [clang-build-stage1] In file included from /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk/System/Library/Frameworks/CoreFoundation.framework/Headers/CFBase.h:67:
[12:08:24] :					 [clang-build-stage1] /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/build/stage1_build/./bin/../include/c++/v1/stdint.h:106:10: fatal error: '__config' file not found
[12:08:24] :					 [clang-build-stage1] #include <__config>
[12:08:24] :					 [clang-build-stage1]          ^~~~~~~~~~
[12:08:24] :					 [clang-build-stage1] [ 64%] Copying CXX header stdlib.h
[12:08:24] :					 [clang-build-stage1] [ 64%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/RTTsan_dynamic.iossim.dir/rtl/tsan_rtl_mutex.cpp.o
[12:08:24] :					 [clang-build-stage1] [ 64%] Building CXX object compiler-rt/lib/lsan/CMakeFiles/clang_rt.lsan_osx_dynamic.dir/lsan_thread.cpp.o
[12:08:24] :					 [clang-build-stage1] [ 64%] Copying CXX header streambuf
[12:08:24] :					 [clang-build-stage1] In file included from /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/compiler-rt/lib/asan/asan_new_delete.cpp:22:
[12:08:24] :					 [clang-build-stage1] /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/build/stage1_build/./bin/../include/c++/v1/stddef.h:39:10: fatal error: '__config' file not found
[12:08:24] :					 [clang-build-stage1] #include <__config>
[12:08:24] :					 [clang-build-stage1]          ^~~~~~~~~~
[12:08:24] :					 [clang-build-stage1] 1 error generated.
.....
[12:08:27] :					 [clang-build-stage1] [ 69%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/RTTsan_dynamic.iossim.dir/rtl/tsan_interceptors_mac.cpp.o
[12:08:27] :					 [clang-build-stage1] [ 69%] Copying CXX header valarray
[12:08:27] :					 [clang-build-stage1] [ 69%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/RTTsan_dynamic.iossim.dir/rtl/tsan_interceptors_mach_vm.cpp.o
[12:08:27] :					 [clang-build-stage1] [ 69%] Copying CXX header variant
[12:08:27] :					 [clang-build-stage1] [ 69%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/RTTsan_dynamic.iossim.dir/rtl/tsan_platform_mac.cpp.o
[12:08:27] :					 [clang-build-stage1] [ 69%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/RTTsan_dynamic.iossim.dir/rtl/tsan_platform_posix.cpp.o
[12:08:27] :					 [clang-build-stage1] [ 69%] Copying CXX header vector
[12:08:27] :					 [clang-build-stage1] [ 69%] Copying CXX header version
[12:08:27] :					 [clang-build-stage1] In file included from /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cpp:23:
[12:08:27] :					 [clang-build-stage1] /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/build/stage1_build/./bin/../include/c++/v1/errno.h:25:10: fatal error: '__config' file not found
[12:08:27] :					 [clang-build-stage1] #include <__config>
[12:08:27] :					 [clang-build-stage1]          ^~~~~~~~~~
[12:08:27] :					 [clang-build-stage1] [ 69%] Copying CXX header wchar.h
[12:08:27] :					 [clang-build-stage1] In file included from /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/compiler-rt/lib/tsan/rtl/tsan_interceptors_mach_vm.cpp:18:
[12:08:27] :					 [clang-build-stage1] In file included from /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator12.1.sdk/usr/include/mach/mach.h:65:
[12:08:27] :					 [clang-build-stage1] In file included from /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator12.1.sdk/usr/include/mach/std_types.h:66:
[12:08:27] :					 [clang-build-stage1] /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/build/stage1_build/./bin/../include/c++/v1/stdint.h:106:10: fatal error: '__config' file not found
[12:08:27] :					 [clang-build-stage1] #include <__config>
[12:08:27] :					 [clang-build-stage1]          ^~~~~~~~~~
[12:08:27] :					 [clang-build-stage1] [ 69%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/RTTsan_dynamic.iossim.dir/rtl/tsan_interceptors_libdispatch.cpp.o
[12:08:27] :					 [clang-build-stage1] [ 70%] Copying CXX header wctype.h
[12:08:27] :					 [clang-build-stage1] [ 70%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/RTTsan_dynamic.iossim.dir/rtl/tsan_new_delete.cpp.o
[12:08:27] :					 [clang-build-stage1] [ 70%] Building CXX object compiler-rt/lib/tsan/CMakeFiles/RTTsan_dynamic.osx.dir/rtl/tsan_sync.cpp.o
[12:08:27] :					 [clang-build-stage1] [ 70%] Copying CXX __config
[12:08:27] :					 [clang-build-stage1] In file included from /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp:28:
[12:08:27] :					 [clang-build-stage1] In file included from /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator12.1.sdk/usr/include/mach/mach.h:65:
[12:08:27] :					 [clang-build-stage1] In file included from /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator12.1.sdk/usr/include/mach/std_types.h:66:
[12:08:27] :					 [clang-build-stage1] /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/build/stage1_build/./bin/../include/c++/v1/stdint.h:106:10: fatal error: '__config' file not found
[12:08:27] :					 [clang-build-stage1] #include <__config>
[12:08:27] :					 [clang-build-stage1]          ^~~~~~~~~~
[12:08:27] :					 [clang-build-stage1] 1 error generated.

[avogelsgesang]

Oh, and I should probably add: I observed those build issues on the 12.0.1 release. I didn't try to build HEAD, yet.
So it might be that in the meantime this race condition was fixed in some other, probably better way than my proposed patch - not sure.
You probably have a better overview over LLVM than I do, and maybe something comes to your mind which might have fixed this already...

[avogelsgesang]

compiler-rt/lib/tsan/CMakeLists.txt

117

my understanding is that the build included errno.h from libc++ (https://github.com/llvm/llvm-project/blob/main/libcxx/include/errno.h)

In file included from /Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cpp:23:
/Volumes/build/tc/473843e25ae443f4/release/src/clang-osx10.11-x86_64-release-master_avogelsgesang-llvm12.12.0.1.c2458596.r74f7019d/build/stage1_build/./bin/../include/c++/v1/errno.h:25:10: fatal error: '__config' file not found
#include <__config>
         ^~~~~~~~~~

Not sure if that is intentional, or if something is going wrong in a more fundamental way here...

compiler-rt/lib/tsan/CMakeLists.txt

set(TSAN_HEADERS
   rtl/tsan_sync.h
   rtl/tsan_trace.h
   rtl/tsan_update_shadow_word_inl.h
   )
 
 set(TSAN_RUNTIME_LIBRARIES)
 add_compiler_rt_component(tsan)
 
+# TSAN uses headers provided by libcxx (errno.h, stdio.h, ...).
+if (TARGET cxx-headers OR HAVE_LIBCXX)
+  set(TSAN_DEPS cxx-headers)
+endif()
+
 if("${CMAKE_C_FLAGS}" MATCHES "-Wno-(error=)?unused-command-line-argument")
   set(EXTRA_CFLAGS "-Wno-error=unused-command-line-argument ${EXTRA_CFLAGS}")
 endif()
 
 if(APPLE)
   # Ideally we would check the SDK version for the actual platform we are
   # building for here.  To make our lifes easier we assume the host SDK setup is
   # sane and use the macOS SDK version as a proxy for aligned SDKs.
OBJECT_LIBS RTInterception
                 RTSanitizerCommon
                 RTSanitizerCommonLibc
                 RTSanitizerCommonCoverage
                 RTSanitizerCommonSymbolizer
                 RTUbsan
     CFLAGS ${TSAN_RTL_CFLAGS}
     LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
     LINK_LIBS ${TSAN_LINK_LIBS} objc
-    PARENT_TARGET tsan)
+    PARENT_TARGET tsan
+    DEPS ${TSAN_DEPS})
   add_compiler_rt_object_libraries(RTTsan_dynamic
     OS ${TSAN_SUPPORTED_OS}
     ARCHS ${TSAN_SUPPORTED_ARCH}
     SOURCES ${TSAN_SOURCES} ${TSAN_CXX_SOURCES} ${TSAN_ASM_SOURCES}
     ADDITIONAL_HEADERS ${TSAN_HEADERS}
-    CFLAGS ${TSAN_RTL_CFLAGS})
+    CFLAGS ${TSAN_RTL_CFLAGS}
+    DEPS ${TSAN_DEPS})
 
   # Build and check Go runtime.
   set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh)
   add_custom_target(GotsanRuntimeCheck
     COMMAND env "CC=${CMAKE_C_COMPILER} ${OSX_SYSROOT_FLAG}"
             EXTRA_CFLAGS=${EXTRA_CFLAGS}
             IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
     DEPENDS tsan ${BUILDGO_SCRIPT}
add_compiler_rt_runtime(clang_rt.tsan
               $<TARGET_OBJECTS:RTInterception.${arch}>
               $<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
               $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
               $<TARGET_OBJECTS:RTSanitizerCommonCoverage.${arch}>
               $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>
               $<TARGET_OBJECTS:RTUbsan.${arch}>
       ADDITIONAL_HEADERS ${TSAN_HEADERS}
       CFLAGS ${TSAN_RTL_CFLAGS}
-      PARENT_TARGET tsan)
+      PARENT_TARGET tsan
+      DEPS ${TSAN_DEPS})
     add_compiler_rt_runtime(clang_rt.tsan_cxx
       STATIC
       ARCHS ${arch}
       SOURCES ${TSAN_CXX_SOURCES}
               $<TARGET_OBJECTS:RTUbsan_cxx.${arch}>
       ADDITIONAL_HEADERS ${TSAN_HEADERS}
       CFLAGS ${TSAN_RTL_CFLAGS}
-      PARENT_TARGET tsan)
+      PARENT_TARGET tsan
+      DEPS ${TSAN_DEPS})
     list(APPEND TSAN_RUNTIME_LIBRARIES clang_rt.tsan-${arch}
                                        clang_rt.tsan_cxx-${arch})
     add_sanitizer_rt_symbols(clang_rt.tsan
       ARCHS ${arch}
       EXTRA rtl/tsan.syms.extra)
     add_sanitizer_rt_symbols(clang_rt.tsan_cxx
       ARCHS ${arch}
       EXTRA rtl/tsan.syms.extra)