[compiler-rt][asan][fuzzer][profile][ubsan] add lit multicfg test suite support

[ychen]

Summary

llvm-lit /path/to/test/src should now invoke tests across all configurations. If this looks helpful, I could port other components.

For example

➜  bin/llvm-lit -vv ~/Src/u/compiler-rt/test/asan/TestCases/alloca_big_alignment.cpp
-- Testing: 4 tests, 4 workers --
PASS: AddressSanitizer-i386-linux-dynamic :: TestCases/alloca_big_alignment.cpp (1 of 4)
PASS: AddressSanitizer-x86_64-linux-dynamic :: TestCases/alloca_big_alignment.cpp (2 of 4)
PASS: AddressSanitizer-i386-linux :: TestCases/alloca_big_alignment.cpp (3 of 4)
PASS: AddressSanitizer-x86_64-linux :: TestCases/alloca_big_alignment.cpp (4 of 4)

Testing Time: 0.21s
  Expected Passes    : 4

[merge_guards_bot]

Unit tests: unknown.

clang-tidy: pass.

clang-format: pass.

Build artifacts: diff.json, clang-tidy.txt, clang-format.patch, CMakeCache.txt, console-log.txt

Pre-merge checks is in beta. Report issue. Please join beta or enable it for your project.

[lebedev.ri]

Thank you for this!
Please add UBsan, this has always been a pain point for me when doing sanitizer changes.

[ychen]

In D73981#1857537, @lebedev.ri wrote:

Thank you for this!
Please add UBsan, this has always been a pain point for me when doing sanitizer changes.

:) Same here while working a little on Fuzzer. Will do.

[ychen]

add ubsan
move sub-suite adding part to be inside configure_lit_site_cfg

[rnk]

lgtm, thanks!

compiler-rt/test/asan/CMakeLists.txt

 if(APPLE)
   darwin_filter_host_archs(ASAN_SUPPORTED_ARCH ASAN_TEST_ARCH)
 endif()
 list(REMOVE_ITEM ASAN_TEST_ARCH sparc sparcv9)
 if(OS_NAME MATCHES "SunOS")
   list(REMOVE_ITEM ASAN_TEST_ARCH x86_64)
 endif()
 
+set(LIT_SITE_MULTI_CFG_ASAN "CompilerRT ASAN")
 foreach(arch ${ASAN_TEST_ARCH})
   set(ASAN_TEST_TARGET_ARCH ${arch})
   set(ASAN_TEST_APPLE_PLATFORM "osx")
   string(TOLOWER "-${arch}-${OS_NAME}" ASAN_TEST_CONFIG_SUFFIX)
   get_bits_for_arch(${arch} ASAN_TEST_BITS)
   get_test_cc_for_arch(${arch} ASAN_TEST_TARGET_CC ASAN_TEST_TARGET_CFLAGS)
   if(ANDROID OR APPLE)
     set(ASAN_TEST_DYNAMIC True)
   else()
     set(ASAN_TEST_DYNAMIC False)
   endif()
   string(TOUPPER ${arch} ARCH_UPPER_CASE)
   set(CONFIG_NAME ${ARCH_UPPER_CASE}${OS_NAME}Config)
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
     ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py
+    MULTI_CONFIG LIT_SITE_MULTI_CFG_ASAN
     )
   list(APPEND ASAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
 
   if(COMPILER_RT_ASAN_HAS_STATIC_RUNTIME)
     string(TOLOWER "-${arch}-${OS_NAME}-dynamic" ASAN_TEST_CONFIG_SUFFIX)
     set(ASAN_TEST_DYNAMIC True)
     set(CONFIG_NAME ${ARCH_UPPER_CASE}${OS_NAME}DynamicConfig)
     configure_lit_site_cfg(
       ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
-      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py)
+      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py
+      MULTI_CONFIG LIT_SITE_MULTI_CFG_ASAN
+      )
     list(APPEND ASAN_DYNAMIC_TESTSUITES
       ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
   endif()
 endforeach()
 
+configure_lit_site_multi_cfg(${LIT_SITE_MULTI_CFG_ASAN}
+                             ${CMAKE_CURRENT_BINARY_DIR}/lit.site.multi.cfg
+                             ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)
+
 # iOS and iOS simulator test suites
 # These are not added into "check-all", in order to run these tests, use
 # "check-asan-iossim-x86_64" and similar. They also require that an extra env
 # variable to select which iOS device or simulator to use, e.g.:
 # SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER="iPhone 6"
 if(APPLE)
   set(EXCLUDE_FROM_ALL ON)
   set(ASAN_TEST_TARGET_CC ${COMPILER_RT_TEST_COMPILER})

compiler-rt/test/asan/lit.site.cfg.py.in

 @LIT_SITE_CFG_IN_HEADER@
 
 # Tool-specific config options.
 config.name_suffix = "@ASAN_TEST_CONFIG_SUFFIX@"
 config.target_cflags = "@ASAN_TEST_TARGET_CFLAGS@"
 config.clang = "@ASAN_TEST_TARGET_CC@"
 config.bits = "@ASAN_TEST_BITS@"
 config.apple_platform = "@ASAN_TEST_APPLE_PLATFORM@"
 config.asan_dynamic = @ASAN_TEST_DYNAMIC@
 config.target_arch = "@ASAN_TEST_TARGET_ARCH@"
+config.test_exec_root = os.path.dirname(__file__)
 
 # Load common config for all compiler-rt lit tests.
 lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured")
 
 # Load tool-specific config that would do the real work.
 lit_config.load_config(config, "@ASAN_LIT_SOURCE_DIR@/lit.cfg.py")

compiler-rt/test/fuzzer/CMakeLists.txt

if (APPLE)
   darwin_filter_host_archs(FUZZER_SUPPORTED_ARCH FUZZER_TEST_ARCH)
 endif()
 
 if(COMPILER_RT_INCLUDE_TESTS)
   list(APPEND LIBFUZZER_TEST_DEPS FuzzerUnitTests)
   list(APPEND LIBFUZZER_TEST_DEPS FuzzedDataProviderUnitTests)
 endif()
 
-add_custom_target(check-fuzzer)
+add_lit_testsuite(check-fuzzer
+  "Running libFuzzer tests for all configurations"
+  ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS ${LIBFUZZER_TEST_DEPS})
+if(TEST_DEPS)
+  add_dependencies(check-fuzzer ${TEST_DEPS})
+endif()
 
 if(COMPILER_RT_INCLUDE_TESTS)
   # libFuzzer unit tests.
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.site.cfg.py.in
     ${CMAKE_CURRENT_BINARY_DIR}/unit/lit.site.cfg.py)
   add_lit_testsuite(check-fuzzer-unit "Running Fuzzer unit tests"
     ${CMAKE_CURRENT_BINARY_DIR}/unit
     DEPENDS ${LIBFUZZER_TEST_DEPS})
   set_target_properties(check-fuzzer-unit PROPERTIES FOLDER "Compiler-RT Tests")
   add_dependencies(check-fuzzer check-fuzzer-unit)
 endif()
 
+set(LIT_SITE_MULTI_CFG_FUZZER "CompilerRT fuzzer")
 macro(test_fuzzer stdlib)
   cmake_parse_arguments(TEST "" "" "DEPS" ${ARGN})
   string(REPLACE "+" "x" stdlib_name ${stdlib})
   string(REPLACE "-" ";" stdlib_list ${stdlib_name})
   set(STDLIB_CAPITALIZED "")
   foreach(part IN LISTS stdlib_list)
     string(SUBSTRING ${part} 0 1 first_letter)
     string(TOUPPER ${first_letter} first_letter)
     string(REGEX REPLACE "^.(.*)" "${first_letter}\\1" part "${part}")
     set(STDLIB_CAPITALIZED "${STDLIB_CAPITALIZED}${part}")
   endforeach()
   foreach(arch ${FUZZER_TEST_ARCH})
     set(LIBFUZZER_TEST_COMPILER ${COMPILER_RT_TEST_COMPILER})
     get_test_cc_for_arch(${arch} LIBFUZZER_TEST_COMPILER LIBFUZZER_TEST_FLAGS)
 
-    set(LIBFUZZER_TEST_TARGET_ARCH ${arch})
+    set(LIBFUZZER_TEST_CONFIG_SUFFIX "-${arch}-${STDLIB_CAPITALIZED}")
     set(LIBFUZZER_TEST_APPLE_PLATFORM "osx")
-
+    set(LIBFUZZER_TEST_TARGET_ARCH ${arch})
     set(LIBFUZZER_TEST_STDLIB ${stdlib})
 
     string(TOUPPER ${arch} ARCH_UPPER_CASE)
     set(CONFIG_NAME ${ARCH_UPPER_CASE}${STDLIB_CAPITALIZED}${OS_NAME}Config)
+    set(LIBFUZZER_TEST_EXEC_DIR ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
 
     # LIT-based libFuzzer tests.
     configure_lit_site_cfg(
       ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
-      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py
+      ${LIBFUZZER_TEST_EXEC_DIR}/lit.site.cfg.py
+      MULTI_CONFIG LIT_SITE_MULTI_CFG_FUZZER
       )
 
     add_lit_testsuite(check-fuzzer-${stdlib_name}-${arch}
       "Running libFuzzer ${stdlib} tests for arch ${arch}"
-      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/
+      ${LIBFUZZER_TEST_EXEC_DIR}
       DEPENDS ${LIBFUZZER_TEST_DEPS})
     if(TEST_DEPS)
       add_dependencies(check-fuzzer-${stdlib_name}-${arch} ${TEST_DEPS})
     endif()
     set_target_properties(check-fuzzer-${stdlib_name}-${arch}
         PROPERTIES FOLDER "Compiler-RT Tests")
-    add_dependencies(check-fuzzer check-fuzzer-${stdlib_name}-${arch})
   endforeach()
 endmacro()
 
 test_fuzzer("default")
 if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
   if(TARGET cxx_shared)
     test_fuzzer("libc++" DEPS cxx_shared)
   endif()
   if(TARGET cxx_static)
     test_fuzzer("static-libc++" DEPS cxx_static)
   endif()
 endif()
 
+configure_lit_site_multi_cfg(${LIT_SITE_MULTI_CFG_FUZZER}
+                             ${CMAKE_CURRENT_BINARY_DIR}/lit.site.multi.cfg
+                             ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)
+
 if (APPLE)
   set(EXCLUDE_FROM_ALL ON)
   set(LIBFUZZER_TEST_COMPILER ${COMPILER_RT_TEST_COMPILER})
   set(FUZZER_APPLE_PLATFORMS ${FUZZER_SUPPORTED_OS})
   foreach(platform ${FUZZER_APPLE_PLATFORMS})
     if ("${platform}" STREQUAL "osx")
       # Skip macOS because it's handled by the code above that builds tests for the host machine.
       continue()

compiler-rt/test/fuzzer/lit.site.cfg.py.in

 @LIT_SITE_CFG_IN_HEADER@
 
 config.cpp_compiler = "@LIBFUZZER_TEST_COMPILER@"
 config.target_flags = "@LIBFUZZER_TEST_FLAGS@"
 config.c_compiler = "@LIBFUZZER_TEST_COMPILER@"
 config.stdlib = "@LIBFUZZER_TEST_STDLIB@"
 config.apple_platform = "@LIBFUZZER_TEST_APPLE_PLATFORM@"
 config.name_suffix = "@LIBFUZZER_TEST_CONFIG_SUFFIX@"
+config.test_exec_root = os.path.dirname(__file__)
 
 config.osx_sysroot_flag = "@OSX_SYSROOT_FLAG@"
 config.cmake_binary_dir = "@CMAKE_BINARY_DIR@"
 config.llvm_library_dir = "@LLVM_LIBRARY_DIR@"
 config.target_triple = "@TARGET_TRIPLE@"
 config.target_arch = "@LIBFUZZER_TEST_TARGET_ARCH@"
 
 # Load common config for all compiler-rt lit tests.

compiler-rt/test/profile/CMakeLists.txt

-set(PROFILE_LIT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
-set(PROFILE_LIT_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
-
-set(PROFILE_TESTSUITES)
 set(PROFILE_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS})
 if(NOT COMPILER_RT_STANDALONE_BUILD)
   list(APPEND PROFILE_TEST_DEPS profile llvm-profdata llvm-cov)
 endif()
 
 set(PROFILE_TEST_ARCH ${PROFILE_SUPPORTED_ARCH})
 if(APPLE)
   darwin_filter_host_archs(PROFILE_SUPPORTED_ARCH PROFILE_TEST_ARCH)
 endif()
 
+set(LIT_SITE_MULTI_CFG_PROFILE "Compiler-RT profile")
 foreach(arch ${PROFILE_TEST_ARCH})
   set(PROFILE_TEST_TARGET_ARCH ${arch})
   get_test_cc_for_arch(${arch} PROFILE_TEST_TARGET_CC PROFILE_TEST_TARGET_CFLAGS)
-  set(CONFIG_NAME Profile-${arch})
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
-    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py
+    ${CMAKE_CURRENT_BINARY_DIR}/${arch}/lit.site.cfg.py
+    MULTI_CONFIG LIT_SITE_MULTI_CFG_PROFILE
    )
-  list(APPEND PROFILE_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
 endforeach()
 
 add_lit_testsuite(check-profile "Running the profile tests"
-  ${PROFILE_TESTSUITES}
+  ${CMAKE_CURRENT_BINARY_DIR}
   DEPENDS ${PROFILE_TEST_DEPS})
 set_target_properties(check-profile PROPERTIES FOLDER "Compiler-RT Misc")
+configure_lit_site_multi_cfg(${LIT_SITE_MULTI_CFG_PROFILE}
+                             ${CMAKE_CURRENT_BINARY_DIR}/lit.site.multi.cfg
+                             ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)

compiler-rt/test/profile/lit.cfg.py

def get_required_attr(config, attr_name):
   return attr_value
 
 # Setup config name.
 config.name = 'Profile-' + config.target_arch
 
 # Setup source root.
 config.test_source_root = os.path.dirname(__file__)
 
-# Setup executable root.
-if hasattr(config, 'profile_lit_binary_dir') and \
-        config.profile_lit_binary_dir is not None:
-    config.test_exec_root = os.path.join(config.profile_lit_binary_dir, config.name)
-
 if config.host_os in ['Linux']:
   extra_link_flags = ["-ldl"]
 elif config.host_os in ['Windows']:
   # InstrProf is incompatible with incremental linking. Disable it as a
   # workaround.
   extra_link_flags = ["-Wl,-incremental:no"]
 else:
   extra_link_flags = []

compiler-rt/test/profile/lit.site.cfg.py.in

 @LIT_SITE_CFG_IN_HEADER@
 
 # Tool-specific config options.
-config.profile_lit_binary_dir = "@PROFILE_LIT_BINARY_DIR@"
 config.target_cflags = "@PROFILE_TEST_TARGET_CFLAGS@"
 config.target_arch = "@PROFILE_TEST_TARGET_ARCH@"
+config.test_exec_root = os.path.dirname(__file__)
 
 # Load common config for all compiler-rt lit tests.
 lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured")
 
 # Load tool-specific config that would do the real work.
-lit_config.load_config(config, "@PROFILE_LIT_SOURCE_DIR@/lit.cfg.py")
+lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py")

compiler-rt/test/ubsan/CMakeLists.txt

 set(UBSAN_LIT_TESTS_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 set(UBSAN_TESTSUITES)
 set(UBSAN_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS})
 
+set(LIT_SITE_MULTI_CFG_UBSAN "Compiler-RT ubsan")
+
 macro(add_ubsan_testsuite test_mode sanitizer arch lld thinlto)
   set(UBSAN_LIT_TEST_MODE "${test_mode}")
   set(CONFIG_NAME ${UBSAN_LIT_TEST_MODE})
   if (${lld})
     set(CONFIG_NAME ${CONFIG_NAME}-lld)
     if (TARGET lld)
       list(APPEND UBSAN_TEST_DEPS lld)
     endif()
   endif()
   if (${thinlto})
     set(CONFIG_NAME ${CONFIG_NAME}-thinlto)
     list(APPEND UBSAN_TEST_DEPS LTO)
   endif()
   set(UBSAN_TEST_USE_LLD ${lld})
   set(UBSAN_TEST_USE_THINLTO ${thinlto})
   set(CONFIG_NAME ${CONFIG_NAME}-${arch})
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
-    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py)
+    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py
+    MULTI_CONFIG
+    LIT_SITE_MULTI_CFG_UBSAN
+    )
   list(APPEND UBSAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
   if(NOT COMPILER_RT_STANDALONE_BUILD)
     list(APPEND UBSAN_TEST_DEPS ${sanitizer})
   endif()
 endmacro()
 
 macro(add_ubsan_testsuites test_mode sanitizer arch)
   add_ubsan_testsuite(${test_mode} ${sanitizer} ${arch} False False)
foreach(arch ${UBSAN_TEST_ARCH})
   if(COMPILER_RT_HAS_MSAN AND ";${MSAN_SUPPORTED_ARCH};" MATCHES ";${arch};")
     add_ubsan_testsuites("MemorySanitizer" msan ${arch})
   endif()
   if(COMPILER_RT_HAS_TSAN AND ";${TSAN_SUPPORTED_ARCH};" MATCHES ";${arch};" AND NOT ANDROID)
     add_ubsan_testsuites("ThreadSanitizer" tsan ${arch})
   endif()
 endforeach()
 
+configure_lit_site_multi_cfg(${LIT_SITE_MULTI_CFG_UBSAN}
+                             ${CMAKE_CURRENT_BINARY_DIR}/lit.site.multi.cfg
+                             ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)
+
 macro(add_ubsan_device_testsuite test_mode sanitizer platform arch)
   # Note we expect the caller to have already set UBSAN_TEST_TARGET_CFLAGS
   set(UBSAN_LIT_TEST_MODE "${test_mode}")
   set(CONFIG_NAME ${UBSAN_LIT_TEST_MODE}-${platform}-${arch})
   set(UBSAN_TEST_TARGET_ARCH ${arch})
   set(UBSAN_TEST_USE_LLD "False")
   set(UBSAN_TEST_USE_THINLTO "False")
   if (APPLE)

compiler-rt/test/ubsan/lit.cfg.py

 # -*- Python -*-
 
 import os
 
 def get_required_attr(config, attr_name):
   attr_value = getattr(config, attr_name, None)
   if attr_value == None:
     lit_config.fatal(
       "No attribute %r in test configuration! You may need to run "
       "tests from your build directory or add this attribute "
       "to lit.site.cfg.py " % attr_name)
   return attr_value
 
 # Setup config name.
 config.name = 'UBSan-' + config.name_suffix
 
 # Setup source root.
 config.test_source_root = os.path.dirname(__file__)
 
 default_ubsan_opts = list(config.default_sanitizer_opts)
 # Choose between standalone and UBSan+ASan modes.
 ubsan_lit_test_mode = get_required_attr(config, 'ubsan_lit_test_mode')
 if ubsan_lit_test_mode == "Standalone":
   config.available_features.add("ubsan-standalone")
   clang_ubsan_cflags = []
 elif ubsan_lit_test_mode == "StandaloneStatic":
   config.available_features.add("ubsan-standalone-static")
   clang_ubsan_cflags = ['-static-libsan']
 elif ubsan_lit_test_mode == "AddressSanitizer":
   config.available_features.add("ubsan-asan")
   clang_ubsan_cflags = ["-fsanitize=address"]
   default_ubsan_opts += ['detect_leaks=0']
 elif ubsan_lit_test_mode == "MemorySanitizer":
   config.available_features.add("ubsan-msan")
   clang_ubsan_cflags = ["-fsanitize=memory"]
 elif ubsan_lit_test_mode == "ThreadSanitizer":
   config.available_features.add("ubsan-tsan")
   clang_ubsan_cflags = ["-fsanitize=thread"]
 else:
   lit_config.fatal("Unknown UBSan test mode: %r" % ubsan_lit_test_mode)
 
 # Platform-specific default for lit tests.
 if config.target_arch == 's390x':
   # On SystemZ we need -mbackchain to make the fast unwinder work.
   clang_ubsan_cflags.append("-mbackchain")
 
 default_ubsan_opts_str = ':'.join(default_ubsan_opts)
 if default_ubsan_opts_str:
   config.environment['UBSAN_OPTIONS'] = default_ubsan_opts_str
   default_ubsan_opts_str += ':'
 # Substitution to setup UBSAN_OPTIONS in portable way.
 config.substitutions.append(('%env_ubsan_opts=',
                              'env UBSAN_OPTIONS=' + default_ubsan_opts_str))
 
 def build_invocation(compile_flags):
   return " " + " ".join([config.clang] + compile_flags) + " "
 
 target_cflags = [get_required_attr(config, "target_cflags")]
 clang_ubsan_cflags += target_cflags
 clang_ubsan_cxxflags = config.cxx_mode_flags + clang_ubsan_cflags
 
 # Define %clang and %clangxx substitutions to use in test RUN lines.
 config.substitutions.append( ("%clang ", build_invocation(clang_ubsan_cflags)) )
 config.substitutions.append( ("%clangxx ", build_invocation(clang_ubsan_cxxflags)) )
 config.substitutions.append( ("%gmlt ", " ".join(config.debug_info_flags) + " ") )
 
 # Default test suffixes.
 config.suffixes = ['.c', '.cpp', '.m']
 
 # Check that the host supports UndefinedBehaviorSanitizer tests
 if config.host_os not in ['Linux', 'Darwin', 'FreeBSD', 'Windows', 'NetBSD', 'SunOS', 'OpenBSD']:
   config.unsupported = True
 
 config.available_features.add('arch=' + config.target_arch)
 
 config.excludes = ['Inputs']
 
 if ubsan_lit_test_mode in ['AddressSanitizer', 'MemorySanitizer', 'ThreadSanitizer']:
   if not config.parallelism_group:
     config.parallelism_group = 'shadow-memory'
   if config.host_os == 'NetBSD':
     config.substitutions.insert(0, ('%run', config.netbsd_noaslr_prefix))

compiler-rt/test/ubsan/lit.site.cfg.py.in

 @LIT_SITE_CFG_IN_HEADER@
 
 # Tool-specific config options.
 config.name_suffix = "@CONFIG_NAME@"
 config.ubsan_lit_test_mode = "@UBSAN_LIT_TEST_MODE@"
 config.target_cflags = "@UBSAN_TEST_TARGET_CFLAGS@"
 config.target_arch = "@UBSAN_TEST_TARGET_ARCH@"
 config.use_lld = @UBSAN_TEST_USE_LLD@
 config.use_thinlto = @UBSAN_TEST_USE_THINLTO@
 config.apple_platform = "@UBSAN_TEST_APPLE_PLATFORM@"
+config.test_exec_root = os.path.dirname(__file__)
 
 # Load common config for all compiler-rt lit tests.
 lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured")
 
 # Load tool-specific config that would do the real work.
-lit_config.load_config(config, "@UBSAN_LIT_TESTS_DIR@/lit.common.cfg.py")
+lit_config.load_config(config, "@UBSAN_LIT_TESTS_DIR@/lit.cfg.py")

compiler-rt/test/ubsan_minimal/CMakeLists.txt

 set(UBSAN_LIT_TESTS_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 set(UBSAN_TEST_ARCH ${UBSAN_SUPPORTED_ARCH})
 if(APPLE)
   darwin_filter_host_archs(UBSAN_SUPPORTED_ARCH UBSAN_TEST_ARCH)
 endif()
 
 set(UBSAN_TESTSUITES)
 set(UBSAN_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS})
 if(NOT COMPILER_RT_STANDALONE_BUILD)
   list(APPEND UBSAN_TEST_DEPS ubsan-minimal)
 endif()
 
+set(LIT_SITE_MULTI_CFG_UBSAN_MINIMAL "Compiler-RT usban minimal")
 foreach(arch ${UBSAN_TEST_ARCH})
   get_test_cc_for_arch(${arch} UBSAN_TEST_TARGET_CC UBSAN_TEST_TARGET_CFLAGS)
   set(CONFIG_NAME ${arch})
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
-    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py)
+    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py
+    MULTI_CONFIG LIT_SITE_MULTI_CFG_UBSAN_MINIMAL
+    )
   list(APPEND UBSAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
 endforeach()
 
 add_lit_testsuite(check-ubsan-minimal "Running UndefinedBehaviorSanitizerMinimal tests"
   ${UBSAN_TESTSUITES}
   DEPENDS ${UBSAN_TEST_DEPS})
 set_target_properties(check-ubsan-minimal PROPERTIES FOLDER "Compiler-RT Misc")
+configure_lit_site_multi_cfg(${LIT_SITE_MULTI_CFG_UBSAN_MINIMAL}
+                             ${CMAKE_CURRENT_BINARY_DIR}/lit.site.multi.cfg
+                             ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)

compiler-rt/test/ubsan_minimal/lit.cfg.py

 # -*- Python -*-
 
 import os
 
 def get_required_attr(config, attr_name):
   attr_value = getattr(config, attr_name, None)
   if attr_value == None:
     lit_config.fatal(
       "No attribute %r in test configuration! You may need to run "
       "tests from your build directory or add this attribute "
       "to lit.site.cfg.py " % attr_name)
   return attr_value
 
 # Setup source root.
 config.test_source_root = os.path.dirname(__file__)
 config.name = 'UBSan-Minimal-' + config.target_arch
 
 def build_invocation(compile_flags):
   return " " + " ".join([config.clang] + compile_flags) + " "
 
 target_cflags = [get_required_attr(config, "target_cflags")]
 clang_ubsan_cflags = ["-fsanitize-minimal-runtime"] + target_cflags
 clang_ubsan_cxxflags = config.cxx_mode_flags + clang_ubsan_cflags
 
 # Define %clang and %clangxx substitutions to use in test RUN lines.
 config.substitutions.append( ("%clang ", build_invocation(clang_ubsan_cflags)) )
 config.substitutions.append( ("%clangxx ", build_invocation(clang_ubsan_cxxflags)) )
 
 # Default test suffixes.
 config.suffixes = ['.c', '.cpp']
 
 # Check that the host supports UndefinedBehaviorSanitizerMinimal tests
 if config.host_os not in ['Linux', 'FreeBSD', 'NetBSD', 'Darwin', 'OpenBSD']: # TODO: Windows
   config.unsupported = True
 
 # Don't target x86_64h if the test machine can't execute x86_64h binaries.
 if '-arch x86_64h' in target_cflags and 'x86_64h' not in config.available_features:
   config.unsupported = True
 
 config.available_features.add('arch=' + config.target_arch)

compiler-rt/test/ubsan_minimal/lit.site.cfg.py.in

 @LIT_SITE_CFG_IN_HEADER@
 
 # Tool-specific config options.
 config.target_cflags = "@UBSAN_TEST_TARGET_CFLAGS@"
 config.target_arch = "@UBSAN_TEST_TARGET_ARCH@"
+config.test_exec_root = os.path.dirname(__file__)
 
 # Load common config for all compiler-rt lit tests.
 lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured")
 
 # Load tool-specific config that would do the real work.
-lit_config.load_config(config, "@UBSAN_LIT_TESTS_DIR@/lit.common.cfg.py")
+lit_config.load_config(config, "@UBSAN_LIT_TESTS_DIR@/lit.cfg.py")

llvm/cmake/modules/AddLLVM.cmake

macro(set_llvm_build_mode)
   # Configuration-time: See Unit/lit.site.cfg.in
   if (CMAKE_CFG_INTDIR STREQUAL ".")
     set(LLVM_BUILD_MODE ".")
   else ()
     set(LLVM_BUILD_MODE "%(build_mode)s")
   endif ()
 endmacro()
 
+function(add_entry_to_lit_config_map key val)
+  set_property(GLOBAL APPEND_STRING
+               PROPERTY LLVM_LIT_CONFIG_MAP "map_config('${key}', '${val}')\n")
+endfunction()
+
 # This function provides an automatic way to 'configure'-like generate a file
 # based on a set of common and custom variables, specifically targeting the
 # variables needed for the 'lit.site.cfg' files. This function bundles the
 # common variables that any Lit instance is likely to need, and custom
 # variables can be passed in.
 function(configure_lit_site_cfg site_in site_out)
-  cmake_parse_arguments(ARG "" "" "MAIN_CONFIG;OUTPUT_MAPPING" ${ARGN})
+  cmake_parse_arguments(ARG "" "" "MAIN_CONFIG;OUTPUT_MAPPING;MULTI_CONFIG" ${ARGN})
 
   if ("${ARG_MAIN_CONFIG}" STREQUAL "")
     get_filename_component(INPUT_DIR ${site_in} DIRECTORY)
     set(ARG_MAIN_CONFIG "${INPUT_DIR}/lit.cfg")
   endif()
   if ("${ARG_OUTPUT_MAPPING}" STREQUAL "")
     set(ARG_OUTPUT_MAPPING "${site_out}")
   endif()
 
+  if (NOT "${ARG_MULTI_CONFIG}" STREQUAL "")
+    string(APPEND ${ARG_MULTI_CONFIG} "\n${site_out}")
+    set(${ARG_MULTI_CONFIG} ${${ARG_MULTI_CONFIG}} PARENT_SCOPE)
+  endif()
+
   foreach(c ${LLVM_TARGETS_TO_BUILD})
     set(TARGETS_BUILT "${TARGETS_BUILT} ${c}")
   endforeach(c)
   set(TARGETS_TO_BUILD ${TARGETS_BUILT})
 
   set(SHLIBEXT "${LTDL_SHLIB_EXT}")
 
   set_llvm_build_mode()
string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}\n\n"
       )
 
     # This is expanded to; config.target_triple = ""+config.target_triple+""
     set(TARGET_TRIPLE "\"+config.target_triple+\"")
   endif()
 
   configure_file(${site_in} ${site_out} @ONLY)
   if (EXISTS "${ARG_MAIN_CONFIG}")
-    set(PYTHON_STATEMENT "map_config('${ARG_MAIN_CONFIG}', '${site_out}')")
-    get_property(LLVM_LIT_CONFIG_MAP GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP)
-    set(LLVM_LIT_CONFIG_MAP "${LLVM_LIT_CONFIG_MAP}\n${PYTHON_STATEMENT}")
-    set_property(GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP ${LLVM_LIT_CONFIG_MAP})
+    add_entry_to_lit_config_map(${ARG_MAIN_CONFIG} ${site_out})
   endif()
 endfunction()
 
+function(configure_lit_site_multi_cfg site_outs site_multi_out main_config)
+  # Generate test suite multi-config file for 'config_map'
+  file(GENERATE OUTPUT ${site_multi_out} CONTENT "${site_outs}")
+  add_entry_to_lit_config_map(${main_config} ${site_multi_out})
+endfunction()
+
 function(dump_all_cmake_variables)
   get_cmake_property(_variableNames VARIABLES)
   foreach (_variableName ${_variableNames})
     message(STATUS "${_variableName}=${${_variableName}}")
   endforeach()
 endfunction()
 
 function(get_llvm_lit_path base_dir file_name)