[libc++] Simplify writing testing config files

[ldionne]

Summary

Reduce code duplication by sharing most of the test suite setup across
the different from-scratch configs.

[ldionne]

Fix issues with Python

[ldionne]

The only CI failure is on the Apple-system configuration, and it's caused by a typo. I fixed it locally and verified it worked. I'll ship this without re-triggering CI to save some resources.

libcxx/test/CMakeLists.txt

 
 if (NOT DEFINED LIBCXX_TEST_DEPS)
   message(FATAL_ERROR "Expected LIBCXX_TEST_DEPS to be defined")
 endif()
 
 if (LIBCXX_INCLUDE_TESTS)
   include(AddLLVM) # for configure_lit_site_cfg and add_lit_testsuite
 
+  configure_file("${CMAKE_CURRENT_SOURCE_DIR}/configs/cmake-bridge.cfg.in"
+                 "${CMAKE_CURRENT_BINARY_DIR}/cmake-bridge.cfg"
+                 @ONLY)
+
   configure_lit_site_cfg(
     "${LIBCXX_TEST_CONFIG}"
     ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
     MAIN_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py")
 
   add_custom_target(cxx-test-depends
     DEPENDS cxx ${LIBCXX_TEST_DEPS}
     COMMENT "Builds dependencies required to run the test suite.")

libcxx/test/configs/apple-libc++-shared.cfg.in

-@AUTO_GEN_COMMENT@
-
-@SERIALIZED_LIT_PARAMS@
-
-#
 # Testing configuration for Apple's system libc++.
 #
 # This configuration differs from a normal LLVM shared library configuration in
 # that we must use DYLD_LIBRARY_PATH to run the tests against the just-built library,
 # since Apple's libc++ has an absolute install_name.
 #
 # We also don't use a per-target include directory layout, so we have only one
 # include directory for the libc++ headers.
-#
-
-LIBCXX_ROOT = "@LIBCXX_SOURCE_DIR@"
-INSTALL_ROOT = "@CMAKE_BINARY_DIR@"
-COMPILER = "@CMAKE_CXX_COMPILER@"
-EXEC_ROOT = "@LIBCXX_BINARY_DIR@"
-CMAKE_OSX_SYSROOT = "@CMAKE_OSX_SYSROOT@"
-INCLUDE_DIR = "@LIBCXX_INSTALL_INCLUDE_DIR@"
-LIBRARY_DIR = "@LIBCXX_INSTALL_LIBRARY_DIR@"
 
-import os
-import pipes
-import site
 import sys
-site.addsitedir(os.path.join(LIBCXX_ROOT, 'utils'))
-import libcxx.test.features
-import libcxx.test.format
-import libcxx.test.newconfig
-import libcxx.test.params
 
-# Configure basic properties of the test suite
-config.name = 'libcxx-trunk-shared'
-config.test_source_root = os.path.join(LIBCXX_ROOT, 'test')
-config.test_format = libcxx.test.format.CxxStandardLibraryTest()
-config.recursiveExpansionLimit = 10
-config.test_exec_root = EXEC_ROOT
+lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
 
-# Configure basic substitutions
-runPy = os.path.join(LIBCXX_ROOT, 'utils', 'run.py')
-config.substitutions.append(('%{cxx}', COMPILER))
 config.substitutions.append(('%{flags}',
-    '-isysroot {}'.format(CMAKE_OSX_SYSROOT) if CMAKE_OSX_SYSROOT else ''
+    '-isysroot {}'.format('@CMAKE_OSX_SYSROOT@') if '@CMAKE_OSX_SYSROOT@' else ''
 ))
 config.substitutions.append(('%{compile_flags}',
-    '-nostdinc++ -isystem {} -I {}'.format(
-        os.path.join(INSTALL_ROOT, INCLUDE_DIR),
-        os.path.join(LIBCXX_ROOT, 'test', 'support'))
+    '-nostdinc++ -isystem %{install}/include/c++/v1 -I %{libcxx}/test/support'
 ))
 config.substitutions.append(('%{link_flags}',
-    '-nostdlib++ -L {} -lc++'.format(
-        os.path.join(INSTALL_ROOT, LIBRARY_DIR))
+    '-nostdlib++ -L %{install}/lib -lc++'
 ))
 config.substitutions.append(('%{exec}',
-    '{} {} --execdir %T --env DYLD_LIBRARY_PATH={} -- '.format(
-        pipes.quote(sys.executable),
-        pipes.quote(runPy),
-        pipes.quote(os.path.join(INSTALL_ROOT, LIBRARY_DIR)))
+    '{} %{{libcxx}}/utils/run.py --execdir %T --env DYLD_LIBRARY_PATH=%{{install}}/lib -- '.format(sys.executable)
 ))
-config.substitutions.append(('%{install}', INSTALL_ROOT))
 
-# Add parameters and features to the config
+import os, site
+site.addsitedir(os.path.join('@LIBCXX_SOURCE_DIR@', 'utils'))
+import libcxx.test.params, libcxx.test.newconfig, libcxx.test.newconfig
 libcxx.test.newconfig.configure(
     libcxx.test.params.DEFAULT_PARAMETERS,
     libcxx.test.features.DEFAULT_FEATURES,
     config,
     lit_config
 )

libcxx/test/configs/cmake-bridge.cfg.in

+@AUTO_GEN_COMMENT@
+
+@SERIALIZED_LIT_PARAMS@
+
+#
+# This file performs the bridge between the CMake configuration and the Lit
+# configuration files by setting up the LitConfig object and various Lit
+# substitutions from CMake variables.
+#
+# Individual configuration files can take advantage of this bridge by
+# loading the file and then setting up the remaining Lit substitutions.
+#
+
+import os, site
+site.addsitedir(os.path.join('@LIBCXX_SOURCE_DIR@', 'utils'))
+import libcxx.test.format
+
+# Basic configuration of the test suite
+config.name = os.path.basename('@LIBCXX_TEST_CONFIG@')
+config.test_source_root = os.path.join('@LIBCXX_SOURCE_DIR@', 'test')
+config.test_format = libcxx.test.format.CxxStandardLibraryTest()
+config.recursiveExpansionLimit = 10
+config.test_exec_root = '@CMAKE_BINARY_DIR@'
+
+# Add substitutions for bootstrapping the test suite configuration
+config.substitutions.append(('%{cxx}', '@CMAKE_CXX_COMPILER@'))
+config.substitutions.append(('%{libcxx}', '@LIBCXX_SOURCE_DIR@'))
+config.substitutions.append(('%{install}', '@CMAKE_BINARY_DIR@'))
+config.substitutions.append(('%{include}', '%{install}/@LIBCXX_INSTALL_INCLUDE_DIR@'))
+config.substitutions.append(('%{target-include}', '%{install}/@LIBCXX_INSTALL_INCLUDE_TARGET_DIR@'))
+config.substitutions.append(('%{lib}', '%{install}/@LIBCXX_INSTALL_LIBRARY_DIR@'))

libcxx/test/configs/llvm-libc++-shared.cfg.in

-@AUTO_GEN_COMMENT@
-
-@SERIALIZED_LIT_PARAMS@
-
-#
 # This testing configuration handles running the test suite against LLVM's libc++
 # using a shared library.
-#
-
-LIBCXX_ROOT = "@LIBCXX_SOURCE_DIR@"
-INSTALL_ROOT = "@CMAKE_BINARY_DIR@"
-COMPILER = "@CMAKE_CXX_COMPILER@"
-EXEC_ROOT = "@LIBCXX_BINARY_DIR@"
-CMAKE_OSX_SYSROOT = "@CMAKE_OSX_SYSROOT@"
-INCLUDE_DIR = "@LIBCXX_INSTALL_INCLUDE_DIR@"
-INCLUDE_TARGET_DIR = "@LIBCXX_INSTALL_INCLUDE_TARGET_DIR@"
-LIBRARY_DIR = "@LIBCXX_INSTALL_LIBRARY_DIR@"
 
-import os
-import pipes
-import site
 import sys
-site.addsitedir(os.path.join(LIBCXX_ROOT, 'utils'))
-import libcxx.test.features
-import libcxx.test.format
-import libcxx.test.newconfig
-import libcxx.test.params
 
-# Configure basic properties of the test suite
-config.name = 'llvm-libc++-shared'
-config.test_source_root = os.path.join(LIBCXX_ROOT, 'test')
-config.test_format = libcxx.test.format.CxxStandardLibraryTest()
-config.recursiveExpansionLimit = 10
-config.test_exec_root = EXEC_ROOT
+lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
 
-# Configure basic substitutions
-runPy = os.path.join(LIBCXX_ROOT, 'utils', 'run.py')
-config.substitutions.append(('%{cxx}', COMPILER))
 config.substitutions.append(('%{flags}',
-    '-isysroot {}'.format(CMAKE_OSX_SYSROOT) if CMAKE_OSX_SYSROOT else ''
+    '-isysroot {}'.format('@CMAKE_OSX_SYSROOT@') if '@CMAKE_OSX_SYSROOT@' else ''
 ))
 config.substitutions.append(('%{compile_flags}',
-    '-nostdinc++ -isystem {} -isystem {} -I {}'.format(
-        os.path.join(INSTALL_ROOT, INCLUDE_DIR),
-        os.path.join(INSTALL_ROOT, INCLUDE_TARGET_DIR),
-        os.path.join(LIBCXX_ROOT, 'test', 'support'))
+    '-nostdinc++ -isystem %{include} -isystem %{target-include} -I %{libcxx}/test/support'
 ))
 config.substitutions.append(('%{link_flags}',
-    '-nostdlib++ -L {0} -lc++ -Wl,-rpath,{0} -pthread'.format(
-        os.path.join(INSTALL_ROOT, LIBRARY_DIR))
+    '-nostdlib++ -L %{lib} -Wl,-rpath,%{lib} -lc++ -pthread'
 ))
 config.substitutions.append(('%{exec}',
-    '{} {} --execdir %T -- '.format(
-        pipes.quote(sys.executable),
-        pipes.quote(runPy))
+    '{} %{{libcxx}}/utils/run.py --execdir %T -- '.format(sys.executable)
 ))
 
-# Add parameters and features to the config
+import os, site
+site.addsitedir(os.path.join('@LIBCXX_SOURCE_DIR@', 'utils'))
+import libcxx.test.params, libcxx.test.newconfig, libcxx.test.newconfig
 libcxx.test.newconfig.configure(
     libcxx.test.params.DEFAULT_PARAMETERS,
     libcxx.test.features.DEFAULT_FEATURES,
     config,
     lit_config
 )

libcxx/test/configs/llvm-libc++-static.cfg.in

-@AUTO_GEN_COMMENT@
-
-@SERIALIZED_LIT_PARAMS@
-
-#
 # This testing configuration handles running the test suite against LLVM's libc++
 # using a static library.
-#
-
-LIBCXX_ROOT = "@LIBCXX_SOURCE_DIR@"
-INSTALL_ROOT = "@CMAKE_BINARY_DIR@"
-COMPILER = "@CMAKE_CXX_COMPILER@"
-EXEC_ROOT = "@LIBCXX_BINARY_DIR@"
-CMAKE_OSX_SYSROOT = "@CMAKE_OSX_SYSROOT@"
-INCLUDE_DIR = "@LIBCXX_INSTALL_INCLUDE_DIR@"
-INCLUDE_TARGET_DIR = "@LIBCXX_INSTALL_INCLUDE_TARGET_DIR@"
-LIBRARY_DIR = "@LIBCXX_INSTALL_LIBRARY_DIR@"
 
-import os
-import pipes
-import site
 import sys
-site.addsitedir(os.path.join(LIBCXX_ROOT, 'utils'))
-import libcxx.test.features
-import libcxx.test.format
-import libcxx.test.newconfig
-import libcxx.test.params
 
-# Configure basic properties of the test suite
-config.name = 'llvm-libc++-static'
-config.test_source_root = os.path.join(LIBCXX_ROOT, 'test')
-config.test_format = libcxx.test.format.CxxStandardLibraryTest()
-config.recursiveExpansionLimit = 10
-config.test_exec_root = EXEC_ROOT
+lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
 
-# Configure basic substitutions
-runPy = os.path.join(LIBCXX_ROOT, 'utils', 'run.py')
-config.substitutions.append(('%{cxx}', COMPILER))
 config.substitutions.append(('%{flags}',
-    '-isysroot {}'.format(CMAKE_OSX_SYSROOT) if CMAKE_OSX_SYSROOT else ''
+    '-isysroot {}'.format('@CMAKE_OSX_SYSROOT@') if '@CMAKE_OSX_SYSROOT@' else ''
 ))
 config.substitutions.append(('%{compile_flags}',
-    '-nostdinc++ -isystem {} -isystem {} -I {}'.format(
-        os.path.join(INSTALL_ROOT, INCLUDE_DIR),
-        os.path.join(INSTALL_ROOT, INCLUDE_TARGET_DIR),
-        os.path.join(LIBCXX_ROOT, 'test', 'support'))
+    '-nostdinc++ -isystem %{include} -isystem %{target-include} -I %{libcxx}/test/support'
 ))
 config.substitutions.append(('%{link_flags}',
-    '-nostdlib++ -L {} -lc++ -lc++abi -pthread'.format(
-        os.path.join(INSTALL_ROOT, LIBRARY_DIR))
+    '-nostdlib++ -L %{lib} -lc++ -lc++abi -pthread'
 ))
 config.substitutions.append(('%{exec}',
-    '{} {} --execdir %T -- '.format(
-        pipes.quote(sys.executable),
-        pipes.quote(runPy))
+    '{} %{{libcxx}}/utils/run.py --execdir %T -- '.format(sys.executable)
 ))
 
-# Add parameters and features to the config
+import os, site
+site.addsitedir(os.path.join('@LIBCXX_SOURCE_DIR@', 'utils'))
+import libcxx.test.params, libcxx.test.newconfig, libcxx.test.newconfig
 libcxx.test.newconfig.configure(
     libcxx.test.params.DEFAULT_PARAMETERS,
     libcxx.test.features.DEFAULT_FEATURES,
     config,
     lit_config
 )

libcxx/utils/libcxx/test/params.py

 
 DEFAULT_PARAMETERS = [
   Parameter(name='target_triple', type=str,
             help="The target triple to compile the test suite for. This must be "
                  "compatible with the target that the tests will be run on.",
             actions=lambda triple: filter(None, [
               AddFeature('target={}'.format(triple)),
               AddFlagIfSupported('--target={}'.format(triple)),
+              AddSubstitution('%{triple}', triple)
             ])),
 
   Parameter(name='std', choices=_allStandards, type=str,
             help="The version of the standard to compile the test suite with.",
             default=lambda cfg: next(s for s in reversed(_allStandards) if getStdFlag(cfg, s)),
             actions=lambda std: [
               AddFeature(std),
               AddCompileFlag(lambda cfg: getStdFlag(cfg, std)),