[lldb] Make the api, shell and unit tests independent lit test suites

[JDevlieghere]

Summary

Make the api, shell and unit tests independent lit test suites. This allows us to specify different dependencies and skip rebuilding all the unit test (which is particularly expensive) when running check-lldb-api or check-lldb-shell. This does not change the autogenerated targets such as check-lldb-shell-driver or the top level check-lldb target, which all continue to work as before.

[stella.stamenova]

Looks good as long as it doesn't break windows :O

[teemperor]

Just gonna post a comment to point out that this apparently adds all tests twice to check-all which makes the tests go kaputt (but I only messaged Jonas directly about that, hence the lack of detailed errors here).

[stella.stamenova]

In D98842#2634902, @teemperor wrote:

Just gonna post a comment to point out that this apparently adds all tests twice to check-all which makes the tests go kaputt (but I only messaged Jonas directly about that, hence the lack of detailed errors here).

Oh, I have a vague recollection of this happening when he tried to split the targets before.

[JDevlieghere]

Add EXCLUDE_FROM_CHECK_ALL to the new targets to avoid tests getting added twice.

[JDevlieghere]

Rebase

[teemperor]

Clean builds for all the test targets work now and this also looks good, so let's ship it.

(also thanks for working on that, the unit test builds where a big annoyance when testing API tests)

[stella.stamenova]

This did end up breaking the windows build bot...

https://lab.llvm.org/buildbot/#/builders/83/builds/4808

[JDevlieghere]

In D98842#2638929, @stella.stamenova wrote:

This did end up breaking the windows build bot...

https://lab.llvm.org/buildbot/#/builders/83/builds/4808

Thanks, the error message suggests the tests are run twice and the log shows every test failing twice. Let me try adding EXCLUDE_FROM_CHECK_ALL to add_lit_testsuites as well. It didn't change anything on macOS, but maybe it does on Windows. If that doesn't fix it I'll revert to turn the bot green again over the weekend.

lldb/test/API/CMakeLists.txt

+add_custom_target(lldb-api-test-deps)
+add_dependencies(lldb-api-test-deps lldb-test-deps)
+
+add_lit_testsuites(LLDB-API
+  ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS lldb-api-test-deps)
+
 function(add_python_test_target name test_script args comment)
   set(PYTHON_TEST_COMMAND
     ${Python3_EXECUTABLE}
     ${test_script}
     ${args}
     )
 
   add_custom_target(${name}

 
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_DOTEST_ARGS "${LLDB_DOTEST_ARGS}")
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_SOURCE_DIR "${LLDB_SOURCE_DIR}")
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_BUILD_DIRECTORY "${LLDB_TEST_BUILD_DIRECTORY}")
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_EXECUTABLE "${LLDB_TEST_EXECUTABLE}")
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_COMPILER "${LLDB_TEST_COMPILER}")
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_DSYMUTIL "${LLDB_TEST_DSYMUTIL}")
 
-# Configure the API test suite.
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)
 
-if (CMAKE_GENERATOR STREQUAL "Xcode")
-  # Xcode does not get the auto-generated targets. We need to create
-  # check-lldb-api manually.
-  add_lit_testsuite(check-lldb-api "Running lldb api test suite"
-    ${CMAKE_CURRENT_BINARY_DIR}
-    DEPENDS lldb-test-deps)
-endif()
-
 # Targets for running the test suite on the different Apple simulators.
 add_lit_testsuite(check-lldb-simulator-ios
   "Running lldb test suite on the iOS simulator"
   ${CMAKE_CURRENT_BINARY_DIR}
   PARAMS "lldb-run-with-simulator=ios"
   EXCLUDE_FROM_CHECK_ALL
-  DEPENDS lldb-test-deps)
+  DEPENDS lldb-api-test-deps)
 
 add_lit_testsuite(check-lldb-simulator-watchos
   "Running lldb test suite on the watchOS simulator"
   ${CMAKE_CURRENT_BINARY_DIR}
   PARAMS "lldb-run-with-simulator=watchos"
   EXCLUDE_FROM_CHECK_ALL
-  DEPENDS lldb-test-deps)
+  DEPENDS lldb-api-test-deps)
 
 add_lit_testsuite(check-lldb-simulator-tvos
   "Running lldb test suite on the tvOS simulator"
   ${CMAKE_CURRENT_BINARY_DIR}
   PARAMS "lldb-run-with-simulator=tvos"
   EXCLUDE_FROM_CHECK_ALL
-  DEPENDS lldb-test-deps)
+  DEPENDS lldb-api-test-deps)
+
+add_lit_testsuite(check-lldb-api "Running lldb api test suite"
+  ${CMAKE_CURRENT_BINARY_DIR}
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS lldb-api-test-deps)

lldb/test/API/lit.cfg.py

 
 # name: The name of this test suite.
 config.name = 'lldb-api'
 
 # suffixes: A list of file extensions to treat as test files.
 config.suffixes = ['.py']
 
 # test_source_root: The root path where tests are located.
-# test_exec_root: The root path where tests should be run.
 config.test_source_root = os.path.dirname(__file__)
-config.test_exec_root = config.test_source_root
+
+# test_exec_root: The root path where tests should be run.
+config.test_exec_root = os.path.join(config.lldb_obj_root, 'test')
 
 
 def mkdir_p(path):
   import errno
   try:
     os.makedirs(path)
   except OSError as e:
     if e.errno != errno.EEXIST:

lldb/test/API/lit.site.cfg.py.in

 @LIT_SITE_CFG_IN_HEADER@
 
-config.test_exec_root = "@LLDB_BINARY_DIR@"
 config.llvm_src_root = "@LLVM_SOURCE_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
 config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
 config.llvm_shlib_dir = "@SHLIBDIR@"
 config.llvm_build_mode = "@LLVM_BUILD_MODE@"
 config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
 config.lldb_obj_root = "@LLDB_BINARY_DIR@"

lldb/test/CMakeLists.txt

 
 # Configure the top level test suite.
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)
 
-add_lit_testsuites(LLDB
-  ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS lldb-test-deps)
-
-add_lit_testsuite(check-lldb-lit "Running lldb lit test suite"
+add_lit_testsuite(check-lldb "Running lldb lit test suite"
   ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS lldb-test-deps)
-set_target_properties(check-lldb-lit PROPERTIES FOLDER "lldb tests")
-
-add_custom_target(check-lldb)
-add_dependencies(check-lldb lldb-test-deps)
-set_target_properties(check-lldb PROPERTIES FOLDER "lldb misc")
-add_dependencies(check-lldb check-lldb-lit)
+  DEPENDS
+    lldb-api-test-deps
+    lldb-shell-test-deps
+    lldb-unit-test-deps)
+set_target_properties(check-lldb PROPERTIES FOLDER "lldb tests")
 
 # Add a lit test suite that runs the API & shell test while capturing a
 # reproducer.
 add_lit_testsuite(check-lldb-reproducers-capture
   "Running lldb test suite with reproducer capture"
   ${CMAKE_CURRENT_BINARY_DIR}/API
   ${CMAKE_CURRENT_BINARY_DIR}/Shell
   PARAMS "lldb-run-with-repro=capture"

lldb/test/Shell/CMakeLists.txt

-# Configure the Shell test suite.
+add_custom_target(lldb-shell-test-deps)
+add_dependencies(lldb-shell-test-deps lldb-test-deps)
+
+add_lit_testsuites(LLDB-SHELL
+  ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS lldb-shell-test-deps)
+
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)
 configure_file(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit-lldb-init.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit-lldb-init)
 
-if (CMAKE_GENERATOR STREQUAL "Xcode")
-  # Xcode does not get the auto-generated targets. We need to create
-  # check-lldb-shell manually.
-  add_lit_testsuite(check-lldb-shell "Running lldb shell test suite"
-    ${CMAKE_CURRENT_BINARY_DIR}
-    DEPENDS lldb-test-deps)
-endif()
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS lldb-shell-test-deps)

lldb/test/Unit/CMakeLists.txt

-# Configure the Unit test suite.
+add_custom_target(lldb-unit-test-deps)
+add_dependencies(lldb-unit-test-deps lldb-test-deps)
+
+add_lit_testsuites(LLDB-UNIT
+  ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS lldb-unit-test-deps)
+
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py)
 
+add_lit_testsuite(check-lldb-unit "Running lldb unit test suite"
+  ${CMAKE_CURRENT_BINARY_DIR}
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS lldb-unit-test-deps)

lldb/unittests/CMakeLists.txt

 add_custom_target(LLDBUnitTests)
 set_target_properties(LLDBUnitTests PROPERTIES FOLDER "lldb tests")
-add_dependencies(lldb-test-deps LLDBUnitTests)
+
+add_dependencies(lldb-unit-test-deps LLDBUnitTests)
 
 include_directories(${LLDB_SOURCE_ROOT})
 include_directories(${LLDB_PROJECT_ROOT}/unittests)
 
 if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
   add_compile_options("-Wno-suggest-override")
 endif()