[libc++] Make the libc++ test-suite standalone

[Jake-Egan]

Summary

This patch enables the use of the libc++ test-suite on its own (without needing to build the runtime). To do this, use the option LIBCXX_TEST_ONLY. The header and library location of an existing runtime needs to be specified with LIBCXX_EXTERNAL_HEADER_DIR and LIBCXX_EXTERNAL_LIBRARY_DIR. Example for configuring and running the test-suite:

$ cmake -G Ninja -DCMAKE_CXX_COMPILER=/some/c++ -DLLVM_ENABLE_RUNTIMES=libcxx -DLIBCXX_TEST_CONFIG=ibm-libc++-shared.cfg.in -DLIBCXX_TEST_ONLY=ON -DLIBCXX_EXTERNAL_HEADER_DIR=/path/to/build/include/c++/v1 -DLIBCXX_EXTERNAL_LIBRARY_DIR=/path/to/build/lib
$ ninja check-cxx

Benefits of this change include:

• Allows for a different compiler (different from the compiler used to build the runtime) to be specified for testing (with CMAKE_CXX_COMPILER)
• Allows platforms to run only the test-suite (if they don’t care about building the runtime).

[EricWF]

I'm not convinced this is something that we want to add the complexity for.

There was a lot of work done to remove similar functionality from the test suite in recent years in order to simplify the test harness and make it more maintainable.

Are we sure we want to re-add this?
Is there a way we can add this functionality out-of-band? It seems to me passing LIT the additional configuration lines as part of its invocation should serve your purpose, no? Is there really a need to make this a CMake configurable option if you can just point LIT at the tests?

[Jake-Egan]

Looking into an out-of-band solution.

libcxx/CMakeLists.txt

if (WIN32 AND NOT MINGW)
   # functions, that currently are provided by libgcc or compiler_rt builtins.
   # These are available in MinGW environments, but not currently in MSVC
   # environments.
   set(ENABLE_FILESYSTEM_DEFAULT OFF)
 endif()
 option(LIBCXX_ENABLE_FILESYSTEM "Build filesystem as part of the main libc++ library"
     ${ENABLE_FILESYSTEM_DEFAULT})
 option(LIBCXX_INCLUDE_TESTS "Build the libc++ tests." ${LLVM_INCLUDE_TESTS})
+option(LIBCXX_TEST_ONLY "Only run the libc++ test-suite without building the runtime." OFF)
 option(LIBCXX_ENABLE_PARALLEL_ALGORITHMS "Enable the parallel algorithms library. This requires the PSTL to be available." OFF)
 option(LIBCXX_ENABLE_DEBUG_MODE
   "Whether to build libc++ with the debug mode enabled.
    By default, this is turned off. Turning it on results in a different ABI (additional
    symbols but also potentially different layouts of types), and one should not mix code
    built against a dylib that has debug mode and code built against a regular dylib." OFF)
 option(LIBCXX_ENABLE_RANDOM_DEVICE
   "Whether to include support for std::random_device in the library. Disabling
set(LIBCXX_TEST_CONFIG "${LIBCXX_DEFAULT_TEST_CONFIG}" CACHE STRING
     "The path to the Lit testing configuration to use when running the tests.
      If a relative path is provided, it is assumed to be relative to '<monorepo>/libcxx/test/configs'.")
 if (NOT IS_ABSOLUTE "${LIBCXX_TEST_CONFIG}")
   set(LIBCXX_TEST_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/test/configs/${LIBCXX_TEST_CONFIG}")
 endif()
 message(STATUS "Using libc++ testing configuration: ${LIBCXX_TEST_CONFIG}")
 set(LIBCXX_TEST_PARAMS "" CACHE STRING
     "A list of parameters to run the Lit test suite with.")
+set(LIBCXX_EXTERNAL_HEADER_DIR "${LIBCXX_EXTERNAL_HEADER_DIR}" CACHE PATH
+    "The path to an external header directory to be used with the test-suite.")
+set(LIBCXX_EXTERNAL_LIBRARY_DIR "${LIBCXX_EXTERNAL_LIBRARY_DIR}" CACHE PATH
+    "The path to an external library directory to be used with the test-suite.")
 
 # Benchmark options -----------------------------------------------------------
 option(LIBCXX_INCLUDE_BENCHMARKS "Build the libc++ benchmarks and their dependencies" ON)
 
 set(LIBCXX_BENCHMARK_TEST_ARGS_DEFAULT --benchmark_min_time=0.01)
 set(LIBCXX_BENCHMARK_TEST_ARGS "${LIBCXX_BENCHMARK_TEST_ARGS_DEFAULT}" CACHE STRING
     "Arguments to pass when running the benchmarks using check-cxx-benchmarks")
 
function(cxx_add_common_build_flags target)
   cxx_add_rtti_flags(${target})
   cxx_add_module_flags(${target})
   cxx_link_system_libraries(${target})
 endfunction()
 
 #===============================================================================
 # Setup Source Code And Tests
 #===============================================================================
+if (NOT LIBCXX_TEST_ONLY)
   add_subdirectory(include)
   add_subdirectory(src)
+endif()
 add_subdirectory(utils)
 
 set(LIBCXX_TEST_DEPS "cxx_experimental")
 
 if (LIBCXX_BUILD_EXTERNAL_THREAD_LIBRARY)
   list(APPEND LIBCXX_TEST_DEPS cxx_external_threads)
 endif()
 

libcxx/test/CMakeLists.txt

configure_file("${CMAKE_CURRENT_SOURCE_DIR}/configs/cmake-bridge.cfg.in"
                  "${CMAKE_CURRENT_BINARY_DIR}/cmake-bridge.cfg"
                  @ONLY)
 
   configure_lit_site_cfg(
     "${LIBCXX_TEST_CONFIG}"
     ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
     MAIN_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py")
 
+  if (LIBCXX_TEST_ONLY)
+    add_lit_testsuite(check-cxx
+      "Running libcxx tests"
+    ${CMAKE_CURRENT_BINARY_DIR})
+  else()
     add_custom_target(cxx-test-depends
       DEPENDS cxx ${LIBCXX_TEST_DEPS}
       COMMENT "Builds dependencies required to run the test suite.")
 
     add_lit_testsuite(check-cxx
       "Running libcxx tests"
       ${CMAKE_CURRENT_BINARY_DIR}
       DEPENDS cxx-test-depends)
    endif()
+endif()
 
 if (LIBCXX_GENERATE_COVERAGE)
   include(CodeCoverage)
   set(output_dir "${CMAKE_CURRENT_BINARY_DIR}/coverage")
   set(capture_dirs
       "${LIBCXX_LIB_CMAKEFILES_DIR}/cxx_objects.dir/"
       "${LIBCXX_LIB_CMAKEFILES_DIR}/cxx.dir/"
       "${LIBCXX_LIB_CMAKEFILES_DIR}/cxx_experimental.dir/"

libcxx/test/configs/cmake-bridge.cfg.in

 config.test_format = libcxx.test.format.CxxStandardLibraryTest()
 config.recursiveExpansionLimit = 10
 config.test_exec_root = os.path.join('@CMAKE_BINARY_DIR@', 'test')
 
 # Add substitutions for bootstrapping the test suite configuration
 import shlex
 config.substitutions.append(('%{cxx}', shlex.quote('@CMAKE_CXX_COMPILER@')))
 config.substitutions.append(('%{libcxx}', '@LIBCXX_SOURCE_DIR@'))
-config.substitutions.append(('%{include}', '@LIBCXX_GENERATED_INCLUDE_DIR@'))
 config.substitutions.append(('%{target-include}', '@LIBCXX_GENERATED_INCLUDE_TARGET_DIR@'))
-config.substitutions.append(('%{lib}', '@LIBCXX_LIBRARY_DIR@'))
 config.substitutions.append(('%{executor}', '@LIBCXX_EXECUTOR@'))
 config.substitutions.append(('%{test-tools}', '@LIBCXX_TEST_TOOLS_PATH@'))
+
+if '@LIBCXX_TEST_ONLY@'.lower() == 'on' :
+    config.substitutions.append(('%{include}', '@LIBCXX_EXTERNAL_HEADER_DIR@'))
+    config.substitutions.append(('%{lib}', '@LIBCXX_EXTERNAL_LIBRARY_DIR@'))
+else:
+    config.substitutions.append(('%{include}', '@LIBCXX_GENERATED_INCLUDE_DIR@'))
+    config.substitutions.append(('%{lib}', '@LIBCXX_LIBRARY_DIR@'))

libcxxabi/CMakeLists.txt

set(LIBCXXABI_TEST_CONFIG "${LIBCXXABI_DEFAULT_TEST_CONFIG}" CACHE STRING
    If a relative path is provided, it is assumed to be relative to '<monorepo>/libcxxabi/test/configs'.")
 if (NOT IS_ABSOLUTE "${LIBCXXABI_TEST_CONFIG}")
   set(LIBCXXABI_TEST_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/test/configs/${LIBCXXABI_TEST_CONFIG}")
 endif()
 message(STATUS "Using libc++abi testing configuration: ${LIBCXXABI_TEST_CONFIG}")
 set(LIBCXXABI_TEST_PARAMS "" CACHE STRING
     "A list of parameters to run the Lit test suite with.")
 
+set(LIBCXXABI_EXTERNAL_HEADER_DIR "${LIBCXX_EXTERNAL_HEADER_DIR}")
+set(LIBCXXABI_EXTERNAL_LIBRARY_DIR "${LIBCXX_EXTERNAL_LIBRARY_DIR}")
+
 #===============================================================================
 # Configure System
 #===============================================================================
 
 # Add path for custom modules
 set(CMAKE_MODULE_PATH
   "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
   "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules"
if (LIBCXXABI_USE_LLVM_UNWINDER OR LLVM_NATIVE_ARCH MATCHES ARM)
 
   if (NOT LIBCXXABI_LIBUNWIND_INCLUDES_INTERNAL STREQUAL "")
     include_directories("${LIBCXXABI_LIBUNWIND_INCLUDES_INTERNAL}")
   endif()
 endif()
 
 # Add source code. This also contains all of the logic for deciding linker flags
 # soname, etc...
+if (NOT LIBCXX_TEST_ONLY)
   add_subdirectory(include)
   add_subdirectory(src)
+endif()
 
 if (LIBCXXABI_INCLUDE_TESTS)
   add_subdirectory(test)
   add_subdirectory(fuzz)
 endif()

libcxxabi/test/CMakeLists.txt

configure_file("${CMAKE_CURRENT_SOURCE_DIR}/configs/cmake-bridge.cfg.in"
                "${CMAKE_CURRENT_BINARY_DIR}/cmake-bridge.cfg"
                @ONLY)
 
 configure_lit_site_cfg(
   "${LIBCXXABI_TEST_CONFIG}"
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
   MAIN_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py")
 
+if (LIBCXX_TEST_ONLY)
+  add_lit_testsuite(check-cxxabi "Running libcxxabi tests"
+    ${CMAKE_CURRENT_BINARY_DIR})
+else()
   add_lit_testsuite(check-cxxabi "Running libcxxabi tests"
     ${CMAKE_CURRENT_BINARY_DIR}
     DEPENDS ${LIBCXXABI_TEST_DEPS})
+endif()

libcxxabi/test/configs/cmake-bridge.cfg.in

 config.recursiveExpansionLimit = 10
 config.test_exec_root = '@CMAKE_BINARY_DIR@'
 
 # TODO: This is a non-standard Lit attribute and we should have another way of accessing this.
 config.host_triple = '@LLVM_HOST_TRIPLE@'
 
 config.substitutions.append(('%{cxx}', '@CMAKE_CXX_COMPILER@'))
 config.substitutions.append(('%{libcxx}', '@LIBCXXABI_LIBCXX_PATH@'))
-config.substitutions.append(('%{include}', '@LIBCXXABI_SOURCE_DIR@/include'))
 config.substitutions.append(('%{cxx-include}', '@LIBCXXABI_HEADER_DIR@/include/c++/v1'))
 config.substitutions.append(('%{cxx-target-include}', '@LIBCXXABI_HEADER_DIR@/include/%{triple}/c++/v1'))
-config.substitutions.append(('%{lib}', '@LIBCXXABI_LIBRARY_DIR@'))
 config.substitutions.append(('%{executor}', '@LIBCXXABI_EXECUTOR@'))
 
+if '@LIBCXX_TEST_ONLY@'.lower() == 'on':
+    config.substitutions.append(('%{include}', '@LIBCXXABI_EXTERNAL_HEADER_DIR@'))
+    config.substitutions.append(('%{lib}', '@LIBCXXABI_EXTERNAL_LIBRARY_DIR@'))
+else:
+    config.substitutions.append(('%{include}', '@LIBCXXABI_SOURCE_DIR@/include'))
+    config.substitutions.append(('%{lib}', '@LIBCXXABI_LIBRARY_DIR@'))
+
 if @LIBCXXABI_USE_LLVM_UNWINDER@:
     config.substitutions.append(('%{maybe-include-libunwind}', '-I "@LIBCXXABI_LIBUNWIND_INCLUDES_INTERNAL@"'))
 else:
     config.substitutions.append(('%{maybe-include-libunwind}', ''))

libunwind/CMakeLists.txt

 if (WIN32)
   set(LIBUNWIND_DEFAULT_HIDE_SYMBOLS TRUE)
 else()
   set(LIBUNWIND_DEFAULT_HIDE_SYMBOLS FALSE)
 endif()
 option(LIBUNWIND_HIDE_SYMBOLS
   "Do not export any symbols from the static library." ${LIBUNWIND_DEFAULT_HIDE_SYMBOLS})
 
+set(LIBUNWIND_EXTERNAL_HEADER_DIR "${LIBCXX_EXTERNAL_HEADER_DIR}")
+set(LIBUNWIND_EXTERNAL_LIBRARY_DIR "${LIBCXX_EXTERNAL_LIBRARY_DIR}")
+
 #===============================================================================
 # Configure System
 #===============================================================================
 
 # Add path for custom modules
 set(CMAKE_MODULE_PATH
     "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
     ${CMAKE_MODULE_PATH})
if (LIBUNWIND_HAS_PTHREAD_LIB)
     add_definitions(-D_LIBUNWIND_LINK_PTHREAD_LIB)
   endif()
 endif()
 
 #===============================================================================
 # Setup Source Code
 #===============================================================================
 
+if (NOT LIBCXX_TEST_ONLY)
   add_subdirectory(include)
-
   add_subdirectory(src)
+endif()
 
 if (LIBUNWIND_INCLUDE_DOCS)
   add_subdirectory(docs)
 endif()
 
 if (LIBUNWIND_INCLUDE_TESTS AND EXISTS ${LLVM_CMAKE_DIR})
   add_subdirectory(test)
 endif()

libunwind/test/CMakeLists.txt

configure_file("${CMAKE_CURRENT_SOURCE_DIR}/configs/cmake-bridge.cfg.in"
                "${CMAKE_CURRENT_BINARY_DIR}/cmake-bridge.cfg"
                @ONLY)
 
 configure_lit_site_cfg(
   "${LIBUNWIND_TEST_CONFIG}"
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
   MAIN_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py")
 
+if (LIBCXX_TEST_ONLY)
+  add_lit_testsuite(check-unwind "Running libunwind tests"
+    ${CMAKE_CURRENT_BINARY_DIR})
+else()
   add_lit_testsuite(check-unwind "Running libunwind tests"
     ${CMAKE_CURRENT_BINARY_DIR}
     DEPENDS unwind ${LIBUNWIND_TEST_DEPS})
+endif()

libunwind/test/configs/cmake-bridge.cfg.in

if not @LIBUNWIND_ENABLE_THREADS@:
     config.available_features.add('libunwind-no-threads')
 
 # Add substitutions for bootstrapping the test suite configuration
 import shlex
 config.substitutions.append(('%{cxx}', shlex.quote('@CMAKE_CXX_COMPILER@')))
 config.substitutions.append(('%{executor}', '@LIBUNWIND_EXECUTOR@'))
 config.substitutions.append(('%{include}', '@LIBUNWIND_SOURCE_DIR@/include'))
 config.substitutions.append(('%{lib}', '@LIBUNWIND_LIBRARY_DIR@'))
+
+if '@LIBCXX_TEST_ONLY@'.lower() == 'on' :
+    config.substitutions.append(('%{include}', '@LIBUNWIND_EXTERNAL_HEADER_DIR@'))
+    config.substitutions.append(('%{lib}', '@LIBUNWIND_EXTERNAL_LIBRARY_DIR@'))
+else:
+    config.substitutions.append(('%{include}', '@LIBUNWIND_SOURCE_DIR@/include'))
+    config.substitutions.append(('%{lib}', '@LIBUNWIND_LIBRARY_DIR@'))