[compiler-rt][asan][fuzzer][profile][ubsan] add lit multicfg test suite support

[ychen]

Summary

llvm-lit /path/to/test/src should now invoke tests across all configurations. If this looks helpful, I could port other components.

For example

➜  bin/llvm-lit -vv ~/Src/u/compiler-rt/test/asan/TestCases/alloca_big_alignment.cpp
-- Testing: 4 tests, 4 workers --
PASS: AddressSanitizer-i386-linux-dynamic :: TestCases/alloca_big_alignment.cpp (1 of 4)
PASS: AddressSanitizer-x86_64-linux-dynamic :: TestCases/alloca_big_alignment.cpp (2 of 4)
PASS: AddressSanitizer-i386-linux :: TestCases/alloca_big_alignment.cpp (3 of 4)
PASS: AddressSanitizer-x86_64-linux :: TestCases/alloca_big_alignment.cpp (4 of 4)

Testing Time: 0.21s
  Expected Passes    : 4

[merge_guards_bot]

Unit tests: unknown.

clang-tidy: pass.

clang-format: pass.

Build artifacts: diff.json, clang-tidy.txt, clang-format.patch, CMakeCache.txt, console-log.txt

Pre-merge checks is in beta. Report issue. Please join beta or enable it for your project.

[lebedev.ri]

Thank you for this!
Please add UBsan, this has always been a pain point for me when doing sanitizer changes.

[ychen]

In D73981#1857537, @lebedev.ri wrote:

Thank you for this!
Please add UBsan, this has always been a pain point for me when doing sanitizer changes.

:) Same here while working a little on Fuzzer. Will do.

[ychen]

add ubsan
move sub-suite adding part to be inside configure_lit_site_cfg

[rnk]

lgtm, thanks!

compiler-rt/test/asan/CMakeLists.txt

 if(APPLE)
   darwin_filter_host_archs(ASAN_SUPPORTED_ARCH ASAN_TEST_ARCH)
 endif()
 list(REMOVE_ITEM ASAN_TEST_ARCH sparc sparcv9)
 if(OS_NAME MATCHES "SunOS")
   list(REMOVE_ITEM ASAN_TEST_ARCH x86_64)
 endif()
 
+set(LIT_SITE_MULTI_CFG_ASAN "CompilerRT ASAN")
 foreach(arch ${ASAN_TEST_ARCH})
   set(ASAN_TEST_TARGET_ARCH ${arch})
   set(ASAN_TEST_APPLE_PLATFORM "osx")
   string(TOLOWER "-${arch}-${OS_NAME}" ASAN_TEST_CONFIG_SUFFIX)
   get_bits_for_arch(${arch} ASAN_TEST_BITS)
   get_test_cc_for_arch(${arch} ASAN_TEST_TARGET_CC ASAN_TEST_TARGET_CFLAGS)
   if(ANDROID OR APPLE)
     set(ASAN_TEST_DYNAMIC True)
   else()
     set(ASAN_TEST_DYNAMIC False)
   endif()
   string(TOUPPER ${arch} ARCH_UPPER_CASE)
   set(CONFIG_NAME ${ARCH_UPPER_CASE}${OS_NAME}Config)
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
     ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py
     )
   list(APPEND ASAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
+  string(APPEND LIT_SITE_MULTI_CFG_ASAN
+         "\n${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py")
 
   if(COMPILER_RT_ASAN_HAS_STATIC_RUNTIME)
     string(TOLOWER "-${arch}-${OS_NAME}-dynamic" ASAN_TEST_CONFIG_SUFFIX)
     set(ASAN_TEST_DYNAMIC True)
     set(CONFIG_NAME ${ARCH_UPPER_CASE}${OS_NAME}DynamicConfig)
     configure_lit_site_cfg(
       ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
       ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py)
     list(APPEND ASAN_DYNAMIC_TESTSUITES
       ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
+    string(APPEND LIT_SITE_MULTI_CFG_ASAN
+           "\n${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py")
   endif()
 endforeach()
 
+configure_lit_site_multi_cfg(${LIT_SITE_MULTI_CFG_ASAN}
+                             ${CMAKE_CURRENT_BINARY_DIR}/lit.site.multi.cfg
+                             ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)
+
 # iOS and iOS simulator test suites
 # These are not added into "check-all", in order to run these tests, use
 # "check-asan-iossim-x86_64" and similar. They also require that an extra env
 # variable to select which iOS device or simulator to use, e.g.:
 # SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER="iPhone 6"
 if(APPLE)
   set(EXCLUDE_FROM_ALL ON)
   set(ASAN_TEST_TARGET_CC ${COMPILER_RT_TEST_COMPILER})

compiler-rt/test/asan/lit.site.cfg.py.in

 @LIT_SITE_CFG_IN_HEADER@
 
 # Tool-specific config options.
 config.name_suffix = "@ASAN_TEST_CONFIG_SUFFIX@"
 config.target_cflags = "@ASAN_TEST_TARGET_CFLAGS@"
 config.clang = "@ASAN_TEST_TARGET_CC@"
 config.bits = "@ASAN_TEST_BITS@"
 config.apple_platform = "@ASAN_TEST_APPLE_PLATFORM@"
 config.asan_dynamic = @ASAN_TEST_DYNAMIC@
 config.target_arch = "@ASAN_TEST_TARGET_ARCH@"
+config.test_exec_root = os.path.dirname(__file__)
 
 # Load common config for all compiler-rt lit tests.
 lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured")
 
 # Load tool-specific config that would do the real work.
 lit_config.load_config(config, "@ASAN_LIT_SOURCE_DIR@/lit.cfg.py")

compiler-rt/test/fuzzer/CMakeLists.txt

if (APPLE)
   darwin_filter_host_archs(FUZZER_SUPPORTED_ARCH FUZZER_TEST_ARCH)
 endif()
 
 if(COMPILER_RT_INCLUDE_TESTS)
   list(APPEND LIBFUZZER_TEST_DEPS FuzzerUnitTests)
   list(APPEND LIBFUZZER_TEST_DEPS FuzzedDataProviderUnitTests)
 endif()
 
-add_custom_target(check-fuzzer)
+add_lit_testsuite(check-fuzzer
+  "Running libFuzzer tests for all configurations"
+  ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS ${LIBFUZZER_TEST_DEPS})
+if(TEST_DEPS)
+  add_dependencies(check-fuzzer ${TEST_DEPS})
+endif()
 
 if(COMPILER_RT_INCLUDE_TESTS)
   # libFuzzer unit tests.
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.site.cfg.py.in
     ${CMAKE_CURRENT_BINARY_DIR}/unit/lit.site.cfg.py)
   add_lit_testsuite(check-fuzzer-unit "Running Fuzzer unit tests"
     ${CMAKE_CURRENT_BINARY_DIR}/unit
     DEPENDS ${LIBFUZZER_TEST_DEPS})
   set_target_properties(check-fuzzer-unit PROPERTIES FOLDER "Compiler-RT Tests")
   add_dependencies(check-fuzzer check-fuzzer-unit)
 endif()
 
+set(LIT_SITE_MULTI_CFG_FUZZER "CompilerRT fuzzer")
 macro(test_fuzzer stdlib)
   cmake_parse_arguments(TEST "" "" "DEPS" ${ARGN})
   string(REPLACE "+" "x" stdlib_name ${stdlib})
   string(REPLACE "-" ";" stdlib_list ${stdlib_name})
   set(STDLIB_CAPITALIZED "")
   foreach(part IN LISTS stdlib_list)
     string(SUBSTRING ${part} 0 1 first_letter)
     string(TOUPPER ${first_letter} first_letter)
     string(REGEX REPLACE "^.(.*)" "${first_letter}\\1" part "${part}")
     set(STDLIB_CAPITALIZED "${STDLIB_CAPITALIZED}${part}")
   endforeach()
   foreach(arch ${FUZZER_TEST_ARCH})
     set(LIBFUZZER_TEST_COMPILER ${COMPILER_RT_TEST_COMPILER})
     get_test_cc_for_arch(${arch} LIBFUZZER_TEST_COMPILER LIBFUZZER_TEST_FLAGS)
 
-    set(LIBFUZZER_TEST_TARGET_ARCH ${arch})
+    set(LIBFUZZER_TEST_CONFIG_SUFFIX "-${arch}-${STDLIB_CAPITALIZED}")
     set(LIBFUZZER_TEST_APPLE_PLATFORM "osx")
-
+    set(LIBFUZZER_TEST_TARGET_ARCH ${arch})
     set(LIBFUZZER_TEST_STDLIB ${stdlib})
 
     string(TOUPPER ${arch} ARCH_UPPER_CASE)
     set(CONFIG_NAME ${ARCH_UPPER_CASE}${STDLIB_CAPITALIZED}${OS_NAME}Config)
+    set(LIBFUZZER_TEST_EXEC_DIR ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
 
     # LIT-based libFuzzer tests.
     configure_lit_site_cfg(
       ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
-      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py
+      ${LIBFUZZER_TEST_EXEC_DIR}/lit.site.cfg.py
       )
+    string(APPEND LIT_SITE_MULTI_CFG_FUZZER
+           "\n${LIBFUZZER_TEST_EXEC_DIR}/lit.site.cfg.py")
 
     add_lit_testsuite(check-fuzzer-${stdlib_name}-${arch}
       "Running libFuzzer ${stdlib} tests for arch ${arch}"
-      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/
+      ${LIBFUZZER_TEST_EXEC_DIR}
       DEPENDS ${LIBFUZZER_TEST_DEPS})
     if(TEST_DEPS)
       add_dependencies(check-fuzzer-${stdlib_name}-${arch} ${TEST_DEPS})
     endif()
     set_target_properties(check-fuzzer-${stdlib_name}-${arch}
         PROPERTIES FOLDER "Compiler-RT Tests")
-    add_dependencies(check-fuzzer check-fuzzer-${stdlib_name}-${arch})
   endforeach()
 endmacro()
 
 test_fuzzer("default")
 if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
   if(TARGET cxx_shared)
     test_fuzzer("libc++" DEPS cxx_shared)
   endif()
   if(TARGET cxx_static)
     test_fuzzer("static-libc++" DEPS cxx_static)
   endif()
 endif()
 
+configure_lit_site_multi_cfg(${LIT_SITE_MULTI_CFG_FUZZER}
+                             ${CMAKE_CURRENT_BINARY_DIR}/lit.site.multi.cfg
+                             ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)
+
 if (APPLE)
   set(EXCLUDE_FROM_ALL ON)
   set(LIBFUZZER_TEST_COMPILER ${COMPILER_RT_TEST_COMPILER})
   set(FUZZER_APPLE_PLATFORMS ${FUZZER_SUPPORTED_OS})
   foreach(platform ${FUZZER_APPLE_PLATFORMS})
     if ("${platform}" STREQUAL "osx")
       # Skip macOS because it's handled by the code above that builds tests for the host machine.
       continue()

compiler-rt/test/fuzzer/lit.site.cfg.py.in

 @LIT_SITE_CFG_IN_HEADER@
 
 config.cpp_compiler = "@LIBFUZZER_TEST_COMPILER@"
 config.target_flags = "@LIBFUZZER_TEST_FLAGS@"
 config.c_compiler = "@LIBFUZZER_TEST_COMPILER@"
 config.stdlib = "@LIBFUZZER_TEST_STDLIB@"
 config.apple_platform = "@LIBFUZZER_TEST_APPLE_PLATFORM@"
 config.name_suffix = "@LIBFUZZER_TEST_CONFIG_SUFFIX@"
+config.test_exec_root = os.path.dirname(__file__)
 
 config.osx_sysroot_flag = "@OSX_SYSROOT_FLAG@"
 config.cmake_binary_dir = "@CMAKE_BINARY_DIR@"
 config.llvm_library_dir = "@LLVM_LIBRARY_DIR@"
 config.target_triple = "@TARGET_TRIPLE@"
 config.target_arch = "@LIBFUZZER_TEST_TARGET_ARCH@"
 
 # Load common config for all compiler-rt lit tests.

compiler-rt/test/profile/CMakeLists.txt

-set(PROFILE_LIT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
-set(PROFILE_LIT_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
-
-set(PROFILE_TESTSUITES)
 set(PROFILE_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS})
 if(NOT COMPILER_RT_STANDALONE_BUILD)
   list(APPEND PROFILE_TEST_DEPS profile llvm-profdata llvm-cov)
 endif()
 
 set(PROFILE_TEST_ARCH ${PROFILE_SUPPORTED_ARCH})
 if(APPLE)
   darwin_filter_host_archs(PROFILE_SUPPORTED_ARCH PROFILE_TEST_ARCH)
 endif()
 
+set(LIT_SITE_MULTI_CFG_PROFILE "Compiler-RT profile")
 foreach(arch ${PROFILE_TEST_ARCH})
   set(PROFILE_TEST_TARGET_ARCH ${arch})
   get_test_cc_for_arch(${arch} PROFILE_TEST_TARGET_CC PROFILE_TEST_TARGET_CFLAGS)
-  set(CONFIG_NAME Profile-${arch})
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
-    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py
+    ${CMAKE_CURRENT_BINARY_DIR}/${arch}/lit.site.cfg.py
    )
-  list(APPEND PROFILE_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
+  string(APPEND LIT_SITE_MULTI_CFG_PROFILE
+         "\n${CMAKE_CURRENT_BINARY_DIR}/${arch}/lit.site.cfg.py")
 endforeach()
 
 add_lit_testsuite(check-profile "Running the profile tests"
-  ${PROFILE_TESTSUITES}
+  ${CMAKE_CURRENT_BINARY_DIR}
   DEPENDS ${PROFILE_TEST_DEPS})
 set_target_properties(check-profile PROPERTIES FOLDER "Compiler-RT Misc")
+configure_lit_site_multi_cfg(${LIT_SITE_MULTI_CFG_PROFILE}
+                             ${CMAKE_CURRENT_BINARY_DIR}/lit.site.multi.cfg
+                             ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)

compiler-rt/test/profile/lit.cfg.py

def get_required_attr(config, attr_name):
   return attr_value
 
 # Setup config name.
 config.name = 'Profile-' + config.target_arch
 
 # Setup source root.
 config.test_source_root = os.path.dirname(__file__)
 
-# Setup executable root.
-if hasattr(config, 'profile_lit_binary_dir') and \
-        config.profile_lit_binary_dir is not None:
-    config.test_exec_root = os.path.join(config.profile_lit_binary_dir, config.name)
-
 if config.host_os in ['Linux']:
   extra_link_flags = ["-ldl"]
 elif config.host_os in ['Windows']:
   # InstrProf is incompatible with incremental linking. Disable it as a
   # workaround.
   extra_link_flags = ["-Wl,-incremental:no"]
 else:
   extra_link_flags = []

compiler-rt/test/profile/lit.site.cfg.py.in

 @LIT_SITE_CFG_IN_HEADER@
 
 # Tool-specific config options.
-config.profile_lit_binary_dir = "@PROFILE_LIT_BINARY_DIR@"
 config.target_cflags = "@PROFILE_TEST_TARGET_CFLAGS@"
 config.target_arch = "@PROFILE_TEST_TARGET_ARCH@"
+config.test_exec_root = os.path.dirname(__file__)
 
 # Load common config for all compiler-rt lit tests.
 lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured")
 
 # Load tool-specific config that would do the real work.
-lit_config.load_config(config, "@PROFILE_LIT_SOURCE_DIR@/lit.cfg.py")
+lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py")

llvm/cmake/modules/AddLLVM.cmake

macro(set_llvm_build_mode)
   # Configuration-time: See Unit/lit.site.cfg.in
   if (CMAKE_CFG_INTDIR STREQUAL ".")
     set(LLVM_BUILD_MODE ".")
   else ()
     set(LLVM_BUILD_MODE "%(build_mode)s")
   endif ()
 endmacro()
 
+function(add_entry_to_lit_config_map key val)
+  set_property(GLOBAL APPEND_STRING
+               PROPERTY LLVM_LIT_CONFIG_MAP "map_config('${key}', '${val}')\n")
+endfunction()
+
 # This function provides an automatic way to 'configure'-like generate a file
 # based on a set of common and custom variables, specifically targeting the
 # variables needed for the 'lit.site.cfg' files. This function bundles the
 # common variables that any Lit instance is likely to need, and custom
 # variables can be passed in.
 function(configure_lit_site_cfg site_in site_out)
   cmake_parse_arguments(ARG "" "" "MAIN_CONFIG;OUTPUT_MAPPING" ${ARGN})
 
string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}\n\n"
       )
 
     # This is expanded to; config.target_triple = ""+config.target_triple+""
     set(TARGET_TRIPLE "\"+config.target_triple+\"")
   endif()
 
   configure_file(${site_in} ${site_out} @ONLY)
   if (EXISTS "${ARG_MAIN_CONFIG}")
-    set(PYTHON_STATEMENT "map_config('${ARG_MAIN_CONFIG}', '${site_out}')")
-    get_property(LLVM_LIT_CONFIG_MAP GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP)
-    set(LLVM_LIT_CONFIG_MAP "${LLVM_LIT_CONFIG_MAP}\n${PYTHON_STATEMENT}")
-    set_property(GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP ${LLVM_LIT_CONFIG_MAP})
+    add_entry_to_lit_config_map(${ARG_MAIN_CONFIG} ${site_out})
   endif()
 endfunction()
 
+function(configure_lit_site_multi_cfg site_outs site_multi_out main_config)
+  # Generate test suite multi-config file for 'config_map'
+  file(GENERATE OUTPUT ${site_multi_out} CONTENT "${site_outs}")
+  add_entry_to_lit_config_map(${main_config} ${site_multi_out})
+endfunction()
+
 function(dump_all_cmake_variables)
   get_cmake_property(_variableNames VARIABLES)
   foreach (_variableName ${_variableNames})
     message(STATUS "${_variableName}=${${_variableName}}")
   endforeach()
 endfunction()
 
 function(get_llvm_lit_path base_dir file_name)