[flang] Run non-gtest unit tests with lit.

[DavidTruby]

Summary

As a corrollary, these tests are now run as part of the check-flang
target.

[CarolineConcatto]

Thank you David for these patch.
Do you mind to explain why you had to create this new non-gtest unit in the patch.

flang/test/NonGtestUnit/lit.cfg.py

14

why here I don't need to tweak the path like it is done in flang/test/Unit/lit.cfg.py?

flang/unittests/Decimal/CMakeLists.txt

12

Should this be removed?

flang/unittests/Evaluate/CMakeLists.txt

7

Should this be removed?

26

where are the tests Leadz and PopPar? Are they running?

flang/unittests/Runtime/CMakeLists.txt

33

Why this one was not removed like the others?

[DavidTruby]

Do you mind to explain why you had to create this new non-gtest unit in the patch.

Assuming you mean the new folder; lit requires a separate folder in /test for each test configuration you want to use. Since I'm adding a new configuration here to simply run a binary and report the error code, we need a separate folder for that than the Unit folder.

flang/test/NonGtestUnit/lit.cfg.py

14

I believe in the case of the gtest tests, gtest needs to know where those tests are so they are added to PATH. We don't need that here as we just run the test binaries directly.

flang/unittests/Decimal/CMakeLists.txt

12

This test is currently not run even with ctest. I've left it out of the port here for that reason.

15

This should be removed though!

flang/unittests/Evaluate/CMakeLists.txt

7

This library is common to the upcoming tests so it needs to stay here.

26

They're added above (lines 11 and 16) and are just called leading-zero-bit-count.test and bit-population-count.test under the new testing framework.

flang/unittests/Runtime/CMakeLists.txt

33

Again, this test is not currently run under the old ctest framework so I've left it alone.

[DavidTruby]

@sscalpone do you have any input as to whether the external-hello-world and thorough-test binaries should be run as unit tests? I'm not sure what to do with those binaries as currently they are built but never automatically tested.

[sscalpone]

external-hello-world

The external-hello-world requires input. Does lit handle that well?

thorough-test

The thorough-test runs for a long time. I say keep it, make sure it links, but don't run it as part of unit tests.

[DavidTruby]

Added comments on tests that aren't run by default.

flang/CMakeLists.txt

 option(FLANG_BUILD_TOOLS
   "Build the Flang tools. If OFF, just generate build targets." ON)
 if (FLANG_BUILD_TOOLS)
   add_subdirectory(tools)
 endif()
 add_subdirectory(runtime)
 
 if (FLANG_INCLUDE_TESTS)
-  enable_testing()
   add_subdirectory(test)
   if (FLANG_GTEST_AVAIL)
     add_subdirectory(unittests)
   endif ()
 endif()
 
 option(FLANG_INCLUDE_DOCS "Generate build targets for the Flang docs."
        ${LLVM_INCLUDE_DOCS})

flang/test/CMakeLists.txt

 
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.cfg.py
 )
 
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/NonGtestUnit/lit.site.cfg.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/NonGtestUnit/lit.site.cfg.py
+  MAIN_CONFIG
+  ${CMAKE_CURRENT_SOURCE_DIR}/NonGtestUnit/lit.cfg.py
+)
+
 set(FLANG_TEST_PARAMS
   flang_site_config=${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py)
 
 set(FLANG_TEST_DEPENDS
   f18 FileCheck count not module_files
 )
 
 if (LINK_WITH_FIR)

flang/test/NonGtestUnit/lit.cfg.py

+import os
+
+import lit.Test
+
+config.name = 'flang-OldUnit'
+
+config.suffixes = [".test"]
+
+config.test_source_root = os.path.join(config.flang_obj_root, 'unittests')
+config.test_exec_root = config.test_source_root
+
+config.test_format = lit.formats.ExecutableTest()
+
+path = os.path.pathsep.join((config.flang_libs_dir, config.llvm_libs_dir,
+                              config.environment.get('LD_LIBRARY_PATH','')))
+config.environment['LD_LIBRARY_PATH'] = path

flang/test/NonGtestUnit/lit.site.cfg.py.in

+@LIT_SITE_CFG_IN_HEADER@
+
+config.llvm_src_root = "@LLVM_SOURCE_DIR@"
+config.llvm_obj_root = "@LLVM_BINARY_DIR@"
+config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
+config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
+config.llvm_build_mode = "@LLVM_BUILD_MODE@"
+config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
+config.flang_obj_root = "@FLANG_BINARY_DIR@"
+config.flang_src_root = "@FLANG_SOURCE_DIR@"
+config.flang_libs_dir = "@LLVM_LIBRARY_OUTPUT_INTDIR@"
+config.flang_tools_dir = "@LLVM_RUNTIME_OUTPUT_INTDIR@"
+config.target_triple = "@TARGET_TRIPLE@"
+config.python_executable = "@Python3_EXECUTABLE@"
+
+# Support substitution of the tools and libs dirs with user parameters. This is
+# used when we can't determine the tool dir at configuration time.
+try:
+    config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
+    config.llvm_libs_dir = config.llvm_libs_dir % lit_config.params
+    config.llvm_build_mode = config.llvm_build_mode % lit_config.params
+except KeyError as e:
+    key, = e.args
+    lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
+
+# Let the main config do the real work.
+lit_config.load_config(config, "@FLANG_SOURCE_DIR@/test/NonGtestUnit/lit.cfg.py")

flang/unittests/CMakeLists.txt

 add_custom_target(FlangUnitTests)
 set_target_properties(FlangUnitTests PROPERTIES FOLDER "Flang Unit Tests")
 
 function(add_flang_unittest test_dirname)
   add_unittest(FlangUnitTests ${test_dirname} ${ARGN})
 endfunction()
 
 if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
   add_compile_options("-Wno-suggest-override")
 endif()
 
+function(add_flang_nongtest_unittest test_name)
+  add_executable(${test_name}.test ${test_name}.cpp)
+
+  target_link_libraries(${test_name}.test ${ARGN})
+
+  add_dependencies(FlangUnitTests ${test_name}.test)
+endfunction()
+
 add_subdirectory(Optimizer)
 add_subdirectory(Decimal)
 add_subdirectory(Evaluate)
 add_subdirectory(Runtime)
 add_subdirectory(Lower)

flang/unittests/Decimal/CMakeLists.txt

 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-add_executable(quick-sanity-test
-  quick-sanity-test.cpp
-)
-
-target_link_libraries(quick-sanity-test
+add_flang_nongtest_unittest(quick-sanity-test
   FortranDecimal
   LLVMSupport
 )
 
+# This test is not run by default as it takes a long time to execute
 add_executable(thorough-test
   thorough-test.cpp
 )
 
 target_link_libraries(thorough-test
   FortranDecimal
   LLVMSupport
 )
-
-add_test(NAME Sanity COMMAND quick-sanity-test)

flang/unittests/Evaluate/CMakeLists.txt

 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 add_library(FortranEvaluateTesting
   testing.cpp
   fp-testing.cpp
 )
 
 target_link_libraries(FortranEvaluateTesting
   LLVMSupport
 )
 
-add_executable(leading-zero-bit-count-test
-  leading-zero-bit-count.cpp
-)
-
-target_link_libraries(leading-zero-bit-count-test
+add_flang_nongtest_unittest(leading-zero-bit-count
   FortranEvaluateTesting
   LLVMSupport
 )
 
-add_executable(bit-population-count-test
-  bit-population-count.cpp
-)
-
-target_link_libraries(bit-population-count-test
+add_flang_nongtest_unittest(bit-population-count
   FortranEvaluateTesting
   LLVMSupport
 )
 
-add_executable(uint128-test
-  uint128.cpp
-)
-
-target_link_libraries(uint128-test
+add_flang_nongtest_unittest(uint128
   FortranEvaluateTesting
   LLVMSupport
 )
 
-# These routines live in lib/Common but we test them here.
-add_test(UINT128 uint128-test)
-add_test(Leadz leading-zero-bit-count-test)
-add_test(PopPar bit-population-count-test)
-
-add_executable(expression-test
-  expression.cpp
-)
-
-target_link_libraries(expression-test
+add_flang_nongtest_unittest(expression
   FortranCommon
   FortranEvaluateTesting
   FortranEvaluate
   FortranSemantics
   FortranParser
   LLVMSupport
 )
 
-add_executable(integer-test
-  integer.cpp
-)
-
-target_link_libraries(integer-test
+add_flang_nongtest_unittest(integer
   FortranEvaluateTesting
   FortranEvaluate
   FortranSemantics
   LLVMSupport
 )
 
-add_executable(intrinsics-test
-  intrinsics.cpp
-)
-
-target_link_libraries(intrinsics-test
+add_flang_nongtest_unittest(intrinsics
   FortranCommon
   FortranEvaluateTesting
   FortranEvaluate
   FortranDecimal
   FortranSemantics
   FortranParser
   FortranRuntime
   LLVMSupport
 )
 
-add_executable(logical-test
-  logical.cpp
-)
-
-target_link_libraries(logical-test
+add_flang_nongtest_unittest(logical
   FortranEvaluateTesting
   FortranEvaluate
   FortranSemantics
   LLVMSupport
 )
 
 # GCC -fno-exceptions breaks the fenv.h interfaces needed to capture
 # IEEE exception flags (different use of the word "exception")
 # in the actual hardware floating-point status register, so ensure that
 # C++ exceptions are enabled for this test.
 set(LLVM_REQUIRES_EH ON)
 set(LLVM_REQUIRES_RTTI ON)
-add_executable(real-test
-  real.cpp
-)
-llvm_update_compile_flags(real-test)
-
-target_link_libraries(real-test
+add_flang_nongtest_unittest(real
   FortranEvaluateTesting
   FortranEvaluate
   FortranDecimal
   FortranSemantics
   LLVMSupport
 )
+llvm_update_compile_flags(real.test)
 
-add_executable(reshape-test
-  reshape.cpp
-)
-
-target_link_libraries(reshape-test
+add_flang_nongtest_unittest(reshape
   FortranEvaluateTesting
   FortranSemantics
   FortranEvaluate
   FortranRuntime
   LLVMSupport
 )
 
-add_executable(ISO-Fortran-binding-test
-  ISO-Fortran-binding.cpp
-)
-
-target_link_libraries(ISO-Fortran-binding-test
+add_flang_nongtest_unittest(ISO-Fortran-binding
   FortranEvaluateTesting
   FortranEvaluate
   FortranSemantics
   FortranRuntime
   LLVMSupport
 )
 
-add_executable(folding-test
-  folding.cpp
-)
-
-target_link_libraries(folding-test
+add_flang_nongtest_unittest(folding
   FortranCommon
   FortranEvaluateTesting
   FortranEvaluate
   FortranSemantics
   LLVMSupport
 )
-
-add_test(Expression expression-test)
-add_test(Integer integer-test)
-add_test(Intrinsics intrinsics-test)
-add_test(Logical logical-test)
-add_test(Real real-test)
-add_test(RESHAPE reshape-test)
-add_test(ISO-binding ISO-Fortran-binding-test)
-add_test(folding folding-test)

flang/unittests/Runtime/CMakeLists.txt

 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 
 # RuntimeTesting needs exceptions enabled
 set(LLVM_REQUIRES_EH ON)
 set(LLVM_REQUIRES_RTTI ON)
 add_library(RuntimeTesting
   testing.cpp
 )
 llvm_update_compile_flags(RuntimeTesting)
 
 target_link_libraries(RuntimeTesting
   FortranRuntime
   LLVMSupport
 )
 
-add_executable(format-test
-  format.cpp
-)
-
-target_link_libraries(format-test
+add_flang_nongtest_unittest(format
   RuntimeTesting
   FortranRuntime
   LLVMSupport
 )
 
-add_test(NAME Format COMMAND format-test)
-
-add_executable(hello-world
-  hello.cpp
-)
-
-target_link_libraries(hello-world
+add_flang_nongtest_unittest(hello
   RuntimeTesting
   FortranRuntime
   LLVMSupport
 )
 
-add_test(NAME HelloWorld COMMAND hello-world)
-
+# This test is not run by default as it requires input.
 add_executable(external-hello-world
   external-hello.cpp
 )
 
 target_link_libraries(external-hello-world
   FortranRuntime
   LLVMSupport
 )
 
-add_executable(external-io
-  external-io.cpp
-)
-
-target_link_libraries(external-io
+add_flang_nongtest_unittest(external-io
   RuntimeTesting
   FortranRuntime
   LLVMSupport
 )
 
-add_test(NAME ExternalIO COMMAND external-io)
-
-add_executable(list-input-test
-  list-input.cpp
-)
-
-target_link_libraries(list-input-test
+add_flang_nongtest_unittest(list-input
   RuntimeTesting
   FortranRuntime
   LLVMSupport
 )
 
-add_test(NAME ListInput COMMAND list-input-test)
-
-add_executable(character-test
-  character.cpp
-)
-
-target_link_libraries(character-test
+add_flang_nongtest_unittest(character
   RuntimeTesting
   FortranRuntime
   LLVMSupport
 )
-
-add_test(NAME CharacterTest COMMAND character-test)

llvm/utils/lit/lit/formats/__init__.py

 from lit.formats.base import (  # noqa: F401
     TestFormat,
     FileBasedTest,
-    OneCommandPerFileTest
+    OneCommandPerFileTest,
+    ExecutableTest
 )
 
 from lit.formats.googletest import GoogleTest  # noqa: F401
 from lit.formats.shtest import ShTest  # noqa: F401

llvm/utils/lit/lit/formats/base.py

def execute(self, test, litConfig):
         report = """Command: %s\n""" % ' '.join(["'%s'" % a
                                                  for a in cmd])
         if self.useTempInput:
             report += """Temporary File: %s\n""" % tmp.name
             report += "--\n%s--\n""" % open(tmp.name).read()
         report += """Output:\n--\n%s--""" % diags
 
         return lit.Test.FAIL, report
+
+
+###
+
+# Check exit code of a simple executable with no input
+class ExecutableTest(FileBasedTest):
+    def execute(self, test, litConfig):
+        if test.config.unsupported:
+            return lit.Test.UNSUPPORTED
+
+        out, err, exitCode = lit.util.executeCommand(test.getSourcePath())
+
+        if not exitCode:
+            return lit.Test.PASS, ''
+
+        return lit.Test.FAIL, out+err
+