[runtimes] Use a response file for runtimes test suites

[phosek]

Summary

We don't know which test suites are going to be included by runtimes
builds so we cannot include those before running the sub-build, but
that's not possible during the LLVM build configuration. We instead use
a response file that's populated by the runtimes build as a level of
indirection.

This addresses the issue described in:
https://discourse.llvm.org/t/cmake-regeneration-is-broken/62788

[thieta]

I tested this on macOS with -DLLVM_ENABLE_RUNTIMES=compiler-rt and ninja check-all:

Without the patch on release/15.x

Testing Time: 1337.62s
  Skipped          :    92
  Unsupported      : 24132
  Passed           : 58680
  Expectedly Failed:    90

With the patch on top of release/15.x:

Testing Time: 2626.51s
  Skipped          :    92
  Unsupported      : 25144
  Passed           : 60152
  Expectedly Failed:   115

So it seems to be working to me.

[thieta]

@phosek I think we should merge this now and get it into -rc3. Do you mind landing it? Otherwise I'll do it tonight EU time.

[phosek]

In D132438#3744820, @thieta wrote:

@phosek I think we should merge this now and get it into -rc3. Do you mind landing it? Otherwise I'll do it tonight EU time.

I'm still waiting for LGTM on D132437 which is a dependency of this change.

[uabelho]

Hello,

Anyone else seeing testcase failures after this patch?
I seem to rather consistently get

Timed Out Tests (1):
  llvm-libc++-shared.cfg.in :: libcxx/modules_include.sh.cpp

but I've also seen e.g. this one fail

13:54:58 ******************** TEST 'llvm-libc++-shared.cfg.in :: std/utilities/meta/meta.trans/meta.trans.other/result_of.pass.cpp' FAILED ********************

[thieta]

In D132438#3748743, @uabelho wrote:

Hello,

Anyone else seeing testcase failures after this patch?
I seem to rather consistently get

Timed Out Tests (1):
  llvm-libc++-shared.cfg.in :: libcxx/modules_include.sh.cpp

but I've also seen e.g. this one fail

13:54:58 ******************** TEST 'llvm-libc++-shared.cfg.in :: std/utilities/meta/meta.trans/meta.trans.other/result_of.pass.cpp' FAILED ********************

If you where building libc++ with LLVM_ENABLE_RUNTIMES then these tests might not have been executed before. So they probably have failed for some time but you haven't seen it until this patch was merged.

[hans]

This broke Chromium's build of android runtimes, see https://crbug.com/1357638

The problem is that we build with -DRUNTIMES_aarch64-linux-android21_LLVM_INCLUDE_TESTS=OFF which means no rsp file gets written, and the lit fails to reference it when we run check-all:

lit: error: [Errno 2] No such file or directory: '/work/llvm-project/build.yy/runtimes/aarch64-linux-android21/Tests.rsp'

Perhaps runtimes/CMakeLists.txt should be writing an empty Tests.rsp for the case when LLVM_INCLUDE_TESTS is false?

[aeubanks]

$ cmake -GNinja -DLLVM_ENABLE_RUNTIMES=compiler-rt -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS='clang' -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_ENABLE_LLD=ON -DRUNTIMES_i386-unknown-linux-gnu_LLVM_ENABLE_RUNTIMES=compiler-rt -DRUNTIMES_i386-unknown-linux-gnu_LLVM_INCLUDE_TESTS=OFF -DLLVM_RUNTIME_TARGETS=i386-unknown-linux-gnu ../../llvm

$ ninja check-all
lit: error: [Errno 2] No such file or directory: '/usr/local/google/home/aeubanks/repos/llvm-project2/build/cmake/runtimes/i386-unknown-linux-gnu/Tests.rsp'

I've reverted this for now

[amyk]

I figured I would also report the behaviour I am seeing with this patch.
When check-all is ran on the PowerPC target (for reference, compiler-rt, libcxx and libcxxabi are built as runtimes), there a bunch of tsan failures:

Failed Tests (19):
  ThreadSanitizer-powerpc64le :: atexit5.cpp
  ThreadSanitizer-powerpc64le :: compare_exchange.cpp
  ThreadSanitizer-powerpc64le :: custom_mutex0.cpp
  ThreadSanitizer-powerpc64le :: custom_mutex1.cpp
  ThreadSanitizer-powerpc64le :: custom_mutex2.cpp
  ThreadSanitizer-powerpc64le :: custom_mutex3.cpp
  ThreadSanitizer-powerpc64le :: custom_mutex4.cpp
  ThreadSanitizer-powerpc64le :: custom_mutex5.cpp
  ThreadSanitizer-powerpc64le :: deadlock_detector_stress_test.cpp
  ThreadSanitizer-powerpc64le :: dl_iterate_phdr.cpp
  ThreadSanitizer-powerpc64le :: dlclose.cpp
  ThreadSanitizer-powerpc64le :: ignore_lib0.cpp
  ThreadSanitizer-powerpc64le :: ignore_lib2.cpp
  ThreadSanitizer-powerpc64le :: ignore_lib3.cpp
  ThreadSanitizer-powerpc64le :: ignore_lib4.cpp
  ThreadSanitizer-powerpc64le :: ignored-interceptors-mmap.cpp
  ThreadSanitizer-powerpc64le :: libcxx/std_shared_ptr.cpp
  ThreadSanitizer-powerpc64le :: load_shared_lib.cpp
  ThreadSanitizer-powerpc64le :: real_deadlock_detector_stress_test.cpp

And all of them involve not being able to find the tsan related headers,

FAIL: ThreadSanitizer-powerpc64le :: atexit5.cpp (71517 of 82996)
******************** TEST 'ThreadSanitizer-powerpc64le :: atexit5.cpp' FAILED ********************
Script:
--
: 'RUN: at line 1';      /home/amyk/testbuild/./bin/clang  --driver-mode=g++ -fsanitize=thread -Wall  -m64 -fno-function-sections   -gline-tables-only -I/data/amyk/compiler/community/llvm-project/compiler-rt/test/tsan/../ -std=c++11 -I/data/amyk/compiler/community/llvm-project/compiler-rt/test/tsan/../ -nostdinc++ -I/home/amyk/testbuild/runtimes/runtimes-bins/compiler-rt/lib/tsan/libcxx_tsan_powerpc64le/include/c++/v1 -O1 -fno-inline-functions /data/amyk/compiler/community/llvm-project/compiler-rt/test/tsan/atexit5.cpp -o /home/amyk/testbuild/runtimes/runtimes-bins/compiler-rt/test/tsan/POWERPC64LEConfig/Output/atexit5.cpp.tmp && /data/amyk/compiler/community/llvm-project/compiler-rt/test/tsan/deflake.bash 10  /home/amyk/testbuild/runtimes/runtimes-bins/compiler-rt/test/tsan/POWERPC64LEConfig/Output/atexit5.cpp.tmp | FileCheck /data/amyk/compiler/community/llvm-project/compiler-rt/test/tsan/atexit5.cpp
--
Exit Code: 1

Command Output (stderr):
--
/data/amyk/compiler/community/llvm-project/compiler-rt/test/tsan/atexit5.cpp:3:10: fatal error: 'memory' file not found
#include <memory>
         ^~~~~~~~
1 error generated.

Or, that the libcxx tsan library was not built and the tests could not find them:

FAIL: ThreadSanitizer-powerpc64le :: compare_exchange.cpp (71529 of 82996)
******************** TEST 'ThreadSanitizer-powerpc64le :: compare_exchange.cpp' FAILED ********************
Script:
--
: 'RUN: at line 1';      /home/amyk/testbuild/./bin/clang  --driver-mode=g++ -fsanitize=thread -Wall  -m64 -fno-function-sections   -gline-tables-only -I/data/amyk/compiler/community/llvm-project/compiler-rt/test/tsan/../ -std=c++11 -I/data/amyk/compiler/community/llvm-project/compiler-rt/test/tsan/../ -nostdinc++ -I/home/amyk/testbuild/runtimes/runtimes-bins/compiler-rt/lib/tsan/libcxx_tsan_powerpc64le/include/c++/v1 -O1 /data/amyk/compiler/community/llvm-project/compiler-rt/test/tsan/compare_exchange.cpp /home/amyk/testbuild/runtimes/runtimes-bins/compiler-rt/lib/tsan/libcxx_tsan_powerpc64le/lib/libc++.a -o /home/amyk/testbuild/runtimes/runtimes-bins/compiler-rt/test/tsan/POWERPC64LEConfig/Output/compare_exchange.cpp.tmp && /data/amyk/compiler/community/llvm-project/compiler-rt/test/tsan/deflake.bash 10 env TSAN_OPTIONS=atexit_sleep_ms=0:atexit_sleep_ms=50  /home/amyk/testbuild/runtimes/runtimes-bins/compiler-rt/test/tsan/POWERPC64LEConfig/Output/compare_exchange.cpp.tmp 2>&1 | FileCheck --check-prefix=CHECK-REPORT /data/amyk/compiler/community/llvm-project/compiler-rt/test/tsan/compare_exchange.cpp
--
Exit Code: 1

Command Output (stderr):
--
clang-15: error: no such file or directory: '/home/amyk/testbuild/runtimes/runtimes-bins/compiler-rt/lib/tsan/libcxx_tsan_powerpc64le/lib/libc++.a'

What I did notice though, is if I do check-runtimes and then check-all, the tests pass since the check-runtimes step builds the missing tsan library. I am not sure if anyone else if experiencing this issue.

[aeubanks]

I'm also seeing the missing tsan library when directly running ninja check-runtimes, not sure where the missing dependency is

[phosek]

In D132438#3859050, @aeubanks wrote:

I'm also seeing the missing tsan library when directly running ninja check-runtimes, not sure where the missing dependency is

What's your build configuration? I'd like to try and reproduce this locally.

[aeubanks]

In D132438#3859300, @phosek wrote:

In D132438#3859050, @aeubanks wrote:

I'm also seeing the missing tsan library when directly running ninja check-runtimes, not sure where the missing dependency is

What's your build configuration? I'd like to try and reproduce this locally.

nothing fancy

$ cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PROJECTS='clang;lld' -DLLVM_ENABLE_RUNTIMES=compiler-rt '-DLLVM_TARGETS_TO_BUILD=X86' -DLLVM_ENABLE_LLD=ON  ../../llvm
# plus the specific C/C++ compiler and a compiler launcher that shouldn't matter
$ ninja check-runtimes
[553/854] Generating TsanRtlTest-x86_64-Test
FAILED: compiler-rt/lib/tsan/tests/rtl/TsanRtlTest-x86_64-Test /usr/local/google/home/aeubanks/repos/llvm-project2/build/cmake/runtimes/runtimes-bins/compiler-rt/lib/tsan/tests/rtl/TsanRtlTest-x86_64-Test 
cd /usr/local/google/home/aeubanks/repos/llvm-project2/build/cmake/runtimes/runtimes-bins/compiler-rt/lib/tsan/tests/rtl && /usr/local/google/home/aeubanks/repos/llvm-project2/build/cmake/./bin/clang++ TsanUnitTestsObjects.tsan_bench.cpp.x86_64.o TsanUnitTestsObjects.tsan_mop.cpp.x86_64.o TsanUnitTestsObjects.tsan_mutex.cpp.x86_64.o TsanUnitTestsObjects.tsan_posix.cpp.x86_64.o TsanUnitTestsObjects.tsan_string.cpp.x86_64.o TsanUnitTestsObjects.tsan_test.cpp.x86_64.o TsanUnitTestsObjects.tsan_thread.cpp.x86_64.o TsanUnitTestsObjects.tsan_test_util_posix.cpp.x86_64.o TsanUnitTestsObjects.gtest-all.cc.x86_64.o -o /usr/local/google/home/aeubanks/repos/llvm-project2/build/cmake/runtimes/runtimes-bins/compiler-rt/lib/tsan/tests/rtl/./TsanRtlTest-x86_64-Test -fuse-ld=lld -Wl,--color-diagnostics -Wthread-safety -Wthread-safety-reference -Wthread-safety-beta -lstdc++ -fsanitize=thread -lm -m64
ld.lld: error: cannot open /usr/local/google/home/aeubanks/repos/llvm-project2/build/cmake/lib/clang/16.0.0/lib/linux/libclang_rt.tsan-x86_64.a: No such file or directory
ld.lld: error: cannot open /usr/local/google/home/aeubanks/repos/llvm-project2/build/cmake/lib/clang/16.0.0/lib/linux/libclang_rt.tsan_cxx-x86_64.a: No such file or directory

[aeubanks]

separately, over in chrome land we've had to disable a bunch of tests that have now started running: https://crbug.com/1374690
we cross build arm and i386 on x64. not sure if the arm tests should automatically be disabled? but the i386 compiler-rt tests aren't specifying various flags like --sysroot and even --target

llvm/runtimes/CMakeLists.txt

function(runtime_default_target)
   endforeach()
   foreach(component ${LLVM_RUNTIME_DISTRIBUTION_COMPONENTS})
     if(NOT ${component} IN_LIST SUB_COMPONENTS)
       list(APPEND extra_targets install-${component} install-${component}-stripped)
     endif()
   endforeach()
 
   if(LLVM_INCLUDE_TESTS)
-    include(${LLVM_BINARY_DIR}/runtimes/Tests.cmake OPTIONAL RESULT_VARIABLE have_tests)
-    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${LLVM_BINARY_DIR}/runtimes/Tests.cmake)
-    if(have_tests)
-      set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES ${SUB_LIT_TESTSUITES})
-      set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_PARAMS ${SUB_LIT_PARAMS})
-      set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_EXTRA_ARGS ${SUB_LIT_EXTRA_ARGS})
-    endif()
+    set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-bins/lit.tests")
     list(APPEND test_targets runtimes-test-depends check-runtimes)
   endif()
 
   set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR_default ON)
   # AIX should fold 32-bit & 64-bit arch libraries into a single archive.
   if (LLVM_TARGET_TRIPLE MATCHES "aix")
     set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR_default OFF)
   endif()
function(runtime_register_target name target)
   foreach(component ${LLVM_RUNTIME_DISTRIBUTION_COMPONENTS})
     set(${component}-${name} ${component})
     set(install-${component}-${name} install-${component})
     set(install-${component}-${name}-stripped install-${component}-stripped)
     list(APPEND ${name}_extra_targets ${component}-${name} install-${component}-${name} install-${component}-${name}-stripped)
   endforeach()
 
   if(LLVM_INCLUDE_TESTS)
-    include(${LLVM_BINARY_DIR}/runtimes/${name}/Tests.cmake OPTIONAL RESULT_VARIABLE have_tests)
-    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${LLVM_BINARY_DIR}/runtimes/${name}/Tests.cmake)
-    if(have_tests)
-      set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES ${SUB_LIT_TESTSUITES})
-      set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_PARAMS ${SUB_LIT_PARAMS})
-      set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_EXTRA_ARGS ${SUB_LIT_EXTRA_ARGS})
-    endif()
+    set_property(GLOBAL APPEND PROPERTY LLVM_ALL_LIT_TESTSUITES "@${LLVM_BINARY_DIR}/runtimes/runtimes-${name}-bins/lit.tests")
     set(runtimes-test-depends-${name} runtimes-test-depends)
     set(check-runtimes-${name} check-runtimes)
     list(APPEND ${name}_test_targets runtimes-test-depends-${name} check-runtimes-${name})
     list(APPEND test_targets ${${name}_test_targets})
     foreach(target_name IN LISTS SUB_CHECK_TARGETS)
       set(${target_name}-${name} ${target_name})
       list(APPEND ${name}_test_targets ${target_name}-${name})
       list(APPEND test_targets ${target_name}-${name})
set(RUNTIMES_TEST_DEPENDS
         sancov
         sanstats
         llvm_gtest_main
         llvm_gtest
       )
     foreach(target ${test_targets} ${SUB_CHECK_TARGETS})
       add_dependencies(${target} ${RUNTIMES_TEST_DEPENDS})
     endforeach()
+
+    set_property(GLOBAL APPEND PROPERTY LLVM_ALL_ADDITIONAL_TEST_TARGETS runtimes ${RUNTIMES_TEST_DEPENDS})
   endif()
 endif()

runtimes/CMakeLists.txt

if(LLVM_INCLUDE_TESTS)
   if (NOT HAVE_LLVM_LIT)
     # If built by manually invoking cmake on this directory, we don't have
     # llvm-lit. If invoked via llvm/runtimes, the toplevel llvm cmake
     # invocation already generated the llvm-lit script.
     add_subdirectory(${LLVM_MAIN_SRC_DIR}/utils/llvm-lit
                      ${CMAKE_CURRENT_BINARY_DIR}/llvm-lit)
   endif()
 
-  if(LLVM_RUNTIMES_TARGET)
-    configure_file(
-      ${CMAKE_CURRENT_SOURCE_DIR}/Tests.cmake.in
-      ${LLVM_BINARY_DIR}/runtimes/${LLVM_RUNTIMES_TARGET}/Tests.cmake)
-  else()
-    configure_file(
-      ${CMAKE_CURRENT_SOURCE_DIR}/Tests.cmake.in
-      ${LLVM_BINARY_DIR}/runtimes/Tests.cmake)
-  endif()
+  get_property(LLVM_RUNTIMES_LIT_TESTSUITES GLOBAL PROPERTY LLVM_RUNTIMES_LIT_TESTSUITES)
+  string(REPLACE ";" "\n" LLVM_RUNTIMES_LIT_TESTSUITES "${LLVM_RUNTIMES_LIT_TESTSUITES}")
+  file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/lit.tests ${LLVM_RUNTIMES_LIT_TESTSUITES})
+  # Create empty files so the parent build can use these unconditionally.
+  file(TOUCH ${CMAKE_CURRENT_BINARY_DIR}/lit.tests)
 endif()
 
 get_property(SUB_COMPONENTS GLOBAL PROPERTY SUB_COMPONENTS)
 if(SUB_COMPONENTS)
   list(REMOVE_DUPLICATES SUB_COMPONENTS)
   foreach(component ${SUB_COMPONENTS})
     if(NOT TARGET ${component})
       message(SEND_ERROR "Missing target for runtime component ${component}!")

runtimes/Tests.cmake.in

-set(SUB_LIT_TESTSUITES @LLVM_RUNTIMES_LIT_TESTSUITES@)
-set(SUB_LIT_PARAMS @LLVM_RUNTIMES_LIT_PARAMS@)
-set(SUB_LIT_EXTRA_ARGS @LLVM_RUNTIMES_LIT_EXTRA_ARGS@)