[lit] Avoid copying llvm/utils/lit/tests/Inputs with lit site configs

[rnk]

Summary

This is an alternative solution to running the lit test suite on bots
without polluting the source directory. Each input test suite gets an
auto-generated site config in the build directory that points back to
the test input source directory.

This adds some cmake comlexity, but now we don't need to remove and
re-copy the test input directory before every test.

[modocache]

LGTM, thanks!

[mgorny]

Well, I can confirm that it doesn't break running in place ;-).

[modocache]

@rnk After this patch, the check-lit target leaves untracked files in the source tree:

utils/lit/tests/Inputs/max-failures/Output/
utils/lit/tests/Inputs/shtest-format/external_shell/Output/
utils/lit/tests/Inputs/shtest-output-printing/Output/
utils/lit/tests/Inputs/shtest-shell/Output/

How would you feel if I reverted this for now? I think this change is the right way forward but it might need some tweaking to prevent the above artifacts from being generated.

[rnk]

In D36026#829224, @modocache wrote:

@rnk After this patch, the check-lit target leaves untracked files in the source tree:

utils/lit/tests/Inputs/max-failures/Output/
utils/lit/tests/Inputs/shtest-format/external_shell/Output/
utils/lit/tests/Inputs/shtest-output-printing/Output/
utils/lit/tests/Inputs/shtest-shell/Output/

How would you feel if I reverted this for now? I think this change is the right way forward but it might need some tweaking to prevent the above artifacts from being generated.

Yeah, let's do that, it's not working as intended, and it doesn't look easy to fix.

[rnk]

Reverted in rL309833

[modocache]

Thanks!

llvm/CMakeLists.txt

if( LLVM_INCLUDE_TESTS )
   if(EXISTS ${LLVM_MAIN_SRC_DIR}/projects/test-suite AND TARGET clang)
     include(LLVMExternalProjectUtils)
     llvm_ExternalProject_Add(test-suite ${LLVM_MAIN_SRC_DIR}/projects/test-suite
       USE_TOOLCHAIN
       EXCLUDE_FROM_ALL
       NO_INSTALL
       ALWAYS_CLEAN)
   endif()
-  add_subdirectory(utils/lit)
+  add_subdirectory(utils/lit/tests)
   add_subdirectory(test)
   add_subdirectory(unittests)
   if( LLVM_INCLUDE_UTILS )
     add_subdirectory(utils/unittest)
   endif()
 
   if (WIN32)
     # This utility is used to prevent crashing tests from calling Dr. Watson on

llvm/utils/lit/CMakeLists.txt

-# The configured file is not placed in the correct location
-# until the tests are run as we need to copy it into
-# a copy of the tests folder
-configure_file("tests/lit.site.cfg.in" "lit.site.cfg" @ONLY)
-
-# Lit's test suite creates output files next to the sources which makes the
-# source tree dirty. This is undesirable because we do out of source builds.
-# To work around this the tests and the configuration file are copied into the
-# build directory just before running them. The tests are not copied over at
-# configure time (i.e. `file(COPY ...)`) because this could lead to stale
-# tests being run.
-add_custom_target(prepare-check-lit
-  COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_CURRENT_BINARY_DIR}/tests"
-  COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_CURRENT_SOURCE_DIR}/tests" "${CMAKE_CURRENT_BINARY_DIR}/tests"
-  COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg" "${CMAKE_CURRENT_BINARY_DIR}/tests"
-  COMMENT "Preparing lit tests"
-)
-
-# Add rules for lit's own test suite
-add_lit_testsuite(check-lit "Running lit's tests"
-  ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS "FileCheck" "not" "prepare-check-lit"
-)
-
-# For IDEs
-set_target_properties(check-lit PROPERTIES FOLDER "Tests")
-set_target_properties(prepare-check-lit PROPERTIES FOLDER "Tests")

llvm/utils/lit/tests/CMakeLists.txt

+# The configured file is not placed in the correct location
+# until the tests are run as we need to copy it into
+# a copy of the tests folder
+configure_file("lit.site.cfg.in" "lit.site.cfg" @ONLY)
+
+# For every lit.cfg in the Inputs tree, create a lit.site.cfg that points at
+# the tests back in the source directory. Running the tests this way keeps the
+# temporary output files in the build directory.
+file(GLOB inputs_suites "Inputs/*/lit.cfg")
+foreach(lit_cfg ${inputs_suites})
+  get_filename_component(LIT_TEST_SRC_DIR "${lit_cfg}" DIRECTORY)
+  file(RELATIVE_PATH LIT_TEST_BIN_DIR "${CMAKE_CURRENT_SOURCE_DIR}" "${LIT_TEST_SRC_DIR}")
+  set(LIT_TEST_BIN_DIR "${CMAKE_CURRENT_BINARY_DIR}/${LIT_TEST_BIN_DIR}")
+  configure_file("Inputs/lit.site.cfg.in" "${LIT_TEST_BIN_DIR}/lit.site.cfg" @ONLY)
+endforeach()
+
+# Add rules for lit's own test suite
+add_lit_testsuite(check-lit "Running lit's tests"
+  ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS "FileCheck" "not"
+)
+
+# For IDEs
+set_target_properties(check-lit PROPERTIES FOLDER "Tests")

llvm/utils/lit/tests/Inputs/lit.site.cfg.in

+lit_config.load_config(config, "@LIT_TEST_SRC_DIR@/lit.cfg")
+config.test_source_root = "@LIT_TEST_SRC_DIR@"
+config.test_exec_root = "@LIT_TEST_BIN_DIR@"

llvm/utils/lit/tests/lit.site.cfg.in

 ## Autogenerated by LLVM/Clang configuration.
 # Do not edit!
 config.llvm_src_root = "@LLVM_SOURCE_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
 config.llvm_tools_dir = "@LLVM_TOOLS_BINARY_DIR@"
 
 # Let the main config do the real work.
-lit_config.load_config(config, "@LLVM_BINARY_DIR@/utils/lit/tests/lit.cfg")
+lit_config.load_config(config, "@LLVM_SOURCE_DIR@/utils/lit/tests/lit.cfg")
+config.test_source_root = "@CMAKE_CURRENT_SOURCE_DIR@"
+config.test_exec_root = "@CMAKE_CURRENT_BINARY_DIR@"