Move the MLIR integration tests as a subdirectory of test (NFC)

[mehdi_amini]

Summary

This does not change the behavior directly: the tests only run when
-DMLIR_INCLUDE_INTEGRATION_TESTS=ON is configured. However running
ninja check-mlir will not run all the tests within a single
lit invocation. The previous behavior would wait for all the integration
tests to complete before starting to run the first regular test. The
test results were also reported separately. This change is unifying all
of this and allow concurrent execution of the integration tests with
regular non-regression and unit-tests.

[jpienaar]

So this will avoid that long pause at the end?

[mehdi_amini]

In D97241#2580465, @jpienaar wrote:

So this will avoid that long pause at the end?

No the long pause is usually due to the "standalone" test in the examples which runs CMake before building. You can avoid this by not configuring with -DLLVM_INCLUDE_EXAMPLES=ON

[mehdi_amini]

What this changes in practice is this:

$ ninja check-mlir
4.494 [3/3/215] Running the MLIR integration tests

Testing Time: 45.39s
  Unsupported:  1
  Passed     : 80
60.002 [0/1/220] Running the MLIR regression tests

Testing Time: 5.42s
  Unsupported:  45
  Passed     : 742

Becomes :

$ ninja check-mlir 
0.000 [0/1/0] Running the MLIR regression tests

Testing Time: 44.86s
  Unsupported:  46
  Passed     : 822

[aartbik]

Would you please post a note on this on the original discourse thread in which I proposed adding the integration tests (the discussion there was in favor of moving this out of test), just so that when people find that thread, they are not confused why it is no longer there.

https://llvm.discourse.group/t/vectorops-rfc-add-suite-of-integration-tests-for-vector-dialect-operations/1213

[mehdi_amini]

Sure, done! Can you confirm you're OK with the change though Aart?

[aartbik]

In D97241#2580629, @mehdi_amini wrote:

Sure, done! Can you confirm you're OK with the change though Aart?

Thanks for posting on discourse. Yes, this change makes a lot of sense, unifying the test (when configured).
Thanks for making the change!

I just had submitted two minor changes to the integration test, that probably conflicts with this.
Sorry about that, let me know if you need a pointer on what I changed.

mlir/CMakeLists.txt

if (MLIR_INCLUDE_TESTS)
   add_custom_target(MLIRUnitTests)
   if (EXISTS ${LLVM_MAIN_SRC_DIR}/utils/unittest/googletest/include/gtest/gtest.h)
     add_subdirectory(unittests)
   else()
     message(WARNING "gtest not found, unittests will not be available")
   endif()
   add_subdirectory(test)
 endif()
-if (MLIR_INCLUDE_INTEGRATION_TESTS)
-  set(INTEL_SDE_EXECUTABLE "" CACHE STRING
-      "If set, arch-specific integration tests are run with Intel SDE.")
-  option(MLIR_RUN_AVX512_TESTS "Run AVX512 tests.")
-  add_definitions(-DMLIR_INCLUDE_INTEGRATION_TESTS)
-  add_subdirectory(integration_test)
-endif()
 # Tools needs to come late to ensure that MLIR_ALL_LIBS is populated.
 # Generally things after this point may depend on MLIR_ALL_LIBS or libMLIR.so.
 add_subdirectory(tools)
 
 if( LLVM_INCLUDE_EXAMPLES )
   add_subdirectory(examples)
 endif()
 

mlir/integration_test/CMakeLists.txt

-# Passed to lit.site.cfg.py.in to set up the path where to find the libraries.
-set(MLIR_INTEGRATION_TEST_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
-
-set(MLIR_INTEGRATION_TEST_DEPENDS
-  FileCheck count not
-  lli
-  mlir-cpu-runner
-  mlir-opt
-  mlir_runner_utils
-  mlir_c_runner_utils
-  )
-
-configure_lit_site_cfg(
-  ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
-  ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
-  ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
-  )
-
-add_lit_testsuite(check-mlir-integration "Running the MLIR integration tests"
-  ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS ${MLIR_INTEGRATION_TEST_DEPENDS}
-  )
-set_target_properties(check-mlir-integration PROPERTIES FOLDER "integration_test")
-
-# Let target "check-mlir" imply the target "check-mlir-integration" as well,
-# provided that MLIR_INCLUDE_INTEGRATION_TESTS=ON during the initial setup.
-add_dependencies(check-mlir check-mlir-integration)
-
-add_lit_testsuites(MLIR_INTEGRATION ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS ${MLIR_INTEGRATION_TEST_DEPENDS}
-  )
-
-# Copy test data over.
-file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/data/test.mtx
-          ${CMAKE_CURRENT_SOURCE_DIR}/data/test.tns
-        DESTINATION ${MLIR_INTEGRATION_TEST_DIR}/data/)

mlir/integration_test/lit.cfg.py

-# -*- Python -*-
-
-import os
-import platform
-import re
-import subprocess
-import tempfile
-
-import lit.formats
-import lit.util
-
-from lit.llvm import llvm_config
-from lit.llvm.subst import ToolSubst
-
-# Configuration file for the 'lit' integration test runner.
-
-# name: The name of this integration test suite.
-config.name = 'MLIR_INTEGRATION'
-
-config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
-
-# suffixes: A list of file extensions to treat as integration test files.
-config.suffixes = ['.mlir']
-
-# test_source_root: The root path where integration tests are located.
-config.test_source_root = os.path.dirname(__file__)
-
-# test_exec_root: The root path where integration tests should be run.
-config.test_exec_root = os.path.join(config.mlir_obj_root, 'integration_test')
-
-config.substitutions.append(('%PATH%', config.environment['PATH']))
-config.substitutions.append(('%shlibext', config.llvm_shlib_ext))
-config.substitutions.append(('%mlir_src_root', config.mlir_src_root))
-
-llvm_config.with_system_environment(['HOME', 'INCLUDE', 'LIB', 'TMP', 'TEMP'])
-
-llvm_config.use_default_substitutions()
-
-# excludes: A list of directories to exclude from the integration testsuite.
-config.excludes = ['CMakeLists.txt', 'README.txt', 'LICENSE.txt']
-
-# Tweak the PATH to include the tools dir.
-llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)
-tool_dirs = [config.mlir_tools_dir, config.llvm_tools_dir]
-tools = [
-    'mlir-opt',
-    'mlir-cpu-runner',
-]
-
-# The following tools are optional.
-tools.extend([
-    ToolSubst(
-        '%mlir_integration_test_dir',
-        config.mlir_integration_test_dir,
-        unresolved='ignore'),
-])
-
-llvm_config.add_tool_substitutions(tools, tool_dirs)

mlir/integration_test/lit.site.cfg.py.in

-@LIT_SITE_CFG_IN_HEADER@
-
-import sys
-
-config.host_triple = "@LLVM_HOST_TRIPLE@"
-config.target_triple = "@TARGET_TRIPLE@"
-config.llvm_src_root = "@LLVM_SOURCE_DIR@"
-config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
-config.llvm_lib_dir = "@LLVM_LIBS_DIR@"
-config.llvm_shlib_dir = "@SHLIBDIR@"
-config.llvm_shlib_ext = "@SHLIBEXT@"
-config.llvm_exe_ext = "@EXEEXT@"
-config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
-config.python_executable = "@PYTHON_EXECUTABLE@"
-config.gold_executable = "@GOLD_EXECUTABLE@"
-config.ld64_executable = "@LD64_EXECUTABLE@"
-config.enable_shared = @ENABLE_SHARED@
-config.enable_assertions = @ENABLE_ASSERTIONS@
-config.targets_to_build = "@TARGETS_TO_BUILD@"
-config.native_target = "@LLVM_NATIVE_ARCH@"
-config.llvm_bindings = "@LLVM_BINDINGS@".split(' ')
-config.host_os = "@HOST_OS@"
-config.host_cc = "@HOST_CC@"
-config.host_cxx = "@HOST_CXX@"
-config.host_cmake = "@CMAKE_COMMAND@"
-# Note: ldflags can contain double-quoted paths, so must use single quotes here.
-config.host_ldflags = '@HOST_LDFLAGS@'
-config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
-config.llvm_host_triple = '@LLVM_HOST_TRIPLE@'
-config.host_arch = "@HOST_ARCH@"
-config.mlir_src_root = "@MLIR_SOURCE_DIR@"
-config.mlir_obj_root = "@MLIR_BINARY_DIR@"
-config.mlir_tools_dir = "@MLIR_TOOLS_DIR@"
-config.mlir_integration_test_dir = "@MLIR_INTEGRATION_TEST_DIR@"
-config.intel_sde_executable = "@INTEL_SDE_EXECUTABLE@"
-config.mlir_run_avx512_tests = "@MLIR_RUN_AVX512_TESTS@"
-
-# Support substitution of the tools_dir with user parameters. This is
-# used when we can't determine the tool dir at configuration time.
-try:
-    config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
-    config.llvm_lib_dir = config.llvm_lib_dir % lit_config.params
-    config.llvm_shlib_dir = config.llvm_shlib_dir % lit_config.params
-except KeyError:
-    e = sys.exc_info()[1]
-    key, = e.args
-    lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
-
-import lit.llvm
-lit.llvm.initialize(lit_config, config)
-
-# Let the main config do the real work.
-lit_config.load_config(config, "@MLIR_SOURCE_DIR@/integration_test/lit.cfg.py")

mlir/test/CMakeLists.txt

 
 # Passed to lit.site.cfg.py.in to set up the path where to find the libraries
 # for the mlir cuda / rocm / spirv / vulkan runner tests.
 set(MLIR_CUDA_WRAPPER_LIBRARY_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 set(MLIR_ROCM_WRAPPER_LIBRARY_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 set(MLIR_SPIRV_WRAPPER_LIBRARY_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 set(MLIR_VULKAN_WRAPPER_LIBRARY_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 
+if (MLIR_INCLUDE_INTEGRATION_TESTS)
+  set(INTEL_SDE_EXECUTABLE "" CACHE STRING
+      "If set, arch-specific integration tests are run with Intel SDE.")
+  option(MLIR_RUN_AVX512_TESTS "Run AVX512 tests.")
+  # Passed to lit.site.cfg.py.in to set up the path where to find the libraries.
+  set(MLIR_INTEGRATION_TEST_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
+
+  # Copy test data over.
+  file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/Integration/data/test.mtx
+            ${CMAKE_CURRENT_SOURCE_DIR}/Integration/data/test.tns
+          DESTINATION ${MLIR_INTEGRATION_TEST_DIR}/data/)
+endif()
+
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
   )
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in

mlir/test/Integration/lit.local.cfg

+if config.mlir_include_integration_tests != 'ON':
+    config.unsupported = True

mlir/test/lit.cfg.py

tools.extend([
     ToolSubst('toy-ch4', unresolved='ignore'),
     ToolSubst('toy-ch5', unresolved='ignore'),
     ToolSubst('%cuda_wrapper_library_dir', config.cuda_wrapper_library_dir, unresolved='ignore'),
     ToolSubst('%linalg_test_lib_dir', config.linalg_test_lib_dir, unresolved='ignore'),
     ToolSubst('%mlir_runner_utils_dir', config.mlir_runner_utils_dir, unresolved='ignore'),
     ToolSubst('%rocm_wrapper_library_dir', config.rocm_wrapper_library_dir, unresolved='ignore'),
     ToolSubst('%spirv_wrapper_library_dir', config.spirv_wrapper_library_dir, unresolved='ignore'),
     ToolSubst('%vulkan_wrapper_library_dir', config.vulkan_wrapper_library_dir, unresolved='ignore'),
+    ToolSubst('%mlir_integration_test_dir', config.mlir_integration_test_dir, unresolved='ignore'),
 ])
-
 llvm_config.add_tool_substitutions(tools, tool_dirs)
 
 
 # FileCheck -enable-var-scope is enabled by default in MLIR test
 # This option avoids to accidentally reuse variable across -LABEL match,
 # it can be explicitly opted-in by prefixing the variable name with $
 config.environment['FILECHECK_OPTS'] = "-enable-var-scope --allow-unused-prefixes=false"
 

mlir/test/lit.site.cfg.py.in

 config.run_rocm_tests = @MLIR_ROCM_CONVERSIONS_ENABLED@
 config.rocm_wrapper_library_dir = "@MLIR_ROCM_WRAPPER_LIBRARY_DIR@"
 config.enable_rocm_runner = @MLIR_ROCM_RUNNER_ENABLED@
 config.spirv_wrapper_library_dir = "@MLIR_SPIRV_WRAPPER_LIBRARY_DIR@"
 config.enable_spirv_cpu_runner = @MLIR_SPIRV_CPU_RUNNER_ENABLED@
 config.vulkan_wrapper_library_dir = "@MLIR_VULKAN_WRAPPER_LIBRARY_DIR@"
 config.enable_vulkan_runner = @MLIR_VULKAN_RUNNER_ENABLED@
 config.enable_bindings_python = @MLIR_BINDINGS_PYTHON_ENABLED@
+config.mlir_integration_test_dir = "@MLIR_INTEGRATION_TEST_DIR@"
+config.intel_sde_executable = "@INTEL_SDE_EXECUTABLE@"
+config.mlir_run_avx512_tests = "@MLIR_RUN_AVX512_TESTS@"
+config.mlir_include_integration_tests = "@MLIR_INCLUDE_INTEGRATION_TESTS@"
 
 # Support substitution of the tools_dir with user parameters. This is
 # used when we can't determine the tool dir at configuration time.
 try:
     config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
     config.llvm_lib_dir = config.llvm_lib_dir % lit_config.params
     config.llvm_shlib_dir = config.llvm_shlib_dir % lit_config.params
 except KeyError: