[OpenMP][libomptarget][test] Separate lit tests for different offloading targets

[protze.joachim]

Summary

This patch creates a separate test directory for each offloading target to be tested.
This allows to test multiple architectures in one configuration, while still see all failing tests separately.
The lit test names include the target triple, so that it will be easier to spot the failing target.

The new %...-generic substitutions are preferred over the specialized rules. The specialized rules should only be used, if different action is required for different targets. I only updated mapping/private_mapping.c to demonstrate the necessary change.

This patch also allows to mark expected failing tests based on the target-triple, the currently used triple is added to the "features":

%XFAIL: target-triple

[JonChesterfield]

Nice!

Does the permit one line for 'generic' and one for 'nvptx', where generic would run everything except for nvptx?

Maybe add an example of the target specific XFAIL trick?

[protze.joachim]

The majority of tests has 5 identical RUN lines, which just differ in the triple-suffix. These 5 identical lines fold into the generic RUN line like I showed for private_mappings.c
The code I added in lit.cfg effectively makes the generic RUN line an alias for the selected specific RUN line.
I'm not sure, what special handling for nvptx you have in mind.

Tests in unified_shared_memory have 4 RUN lines and omit a nvptx line.
In these tests, I would also use the generic RUN line and mark nvptx64-nvidia-cuda as UNSUPPORTED or XFAIL. I just added api.c and close_enter_exit.c to showcase the changes.
This demonstrates, how you would use UNSUPPORTED / XFAIL for specific target triples.

When I run the libomptarget-nvptx64-nvidia-cuda tests, the output will contain:

$ make check-libomptarget-nvptx64-nvidia-cuda
Unsupported Tests (1):
  libomptarget :: nvptx64-nvidia-cuda :: unified_shared_memory/api.c

********************
Expectedly Failed Tests (1):
  libomptarget :: nvptx64-nvidia-cuda :: unified_shared_memory/close_enter_exit.c

UNSUPPORTED will not even try to run the test for the specified target. XFAIL will try and expect to fail.
I prefer XFAIL if there is a chance, that the test might succeed in the future, so that we remember to activate a test once the test case is supported.
I use UNSUPPORTED in two cases

• if there is no chance, the test will ever work with this feature (like some clauses/directives are just not available in older clang).
• if a specific test was fixed between minor releases (e.g. fail with clang/11.0, succeeds with clang/11.1)

[protze.joachim]

I moved the RUN line changes into D101326, which updates all tests.

[JonChesterfield]

That works exactly like I hoped it would. Thanks!

openmp/libomptarget/test/CMakeLists.txt

 # CMakeLists.txt file for unit testing OpenMP offloading runtime library.
 if(NOT OPENMP_TEST_COMPILER_ID STREQUAL "Clang" OR
    OPENMP_TEST_COMPILER_VERSION VERSION_LESS 6.0.0)
   libomptarget_say("Can only test with Clang compiler in version 6.0.0 or later.")
   libomptarget_warning_say("The check-libomptarget target will not be available!")
   return()
 endif()
 
 if(LIBOMPTARGET_ENABLE_DEBUG)
   set(LIBOMPTARGET_DEBUG True)
 else()
   set(LIBOMPTARGET_DEBUG False)
 endif()
 
-add_openmp_testsuite(check-libomptarget "Running libomptarget tests" ${CMAKE_CURRENT_BINARY_DIR} DEPENDS omptarget omp ${LIBOMPTARGET_TESTED_PLUGINS})
+string(REGEX MATCHALL "([^\ ]+\ |[^\ ]+$)" SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}")
+foreach(CURRENT_TARGET IN LISTS SYSTEM_TARGETS)
+  string(STRIP "${CURRENT_TARGET}" CURRENT_TARGET)
+  add_openmp_testsuite(check-libomptarget-${CURRENT_TARGET} "Running libomptarget tests" ${CMAKE_CURRENT_BINARY_DIR}/${CURRENT_TARGET} DEPENDS omptarget omp ${LIBOMPTARGET_TESTED_PLUGINS})
+  list(APPEND LIBOMPTARGET_LIT_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CURRENT_TARGET})
 
   # Configure the lit.site.cfg.in file
   set(AUTO_GEN_COMMENT "## Autogenerated by libomptarget configuration.\n# Do not edit!")
-configure_file(lit.site.cfg.in lit.site.cfg @ONLY)
+  configure_file(lit.site.cfg.in ${CURRENT_TARGET}/lit.site.cfg @ONLY)
+endforeach()
+
+add_openmp_testsuite(check-libomptarget "Running libomptarget tests" ${LIBOMPTARGET_LIT_TESTSUITES} EXCLUDE_FROM_CHECK_ALL DEPENDS omptarget omp ${LIBOMPTARGET_TESTED_PLUGINS})
 No newline at end of file

openmp/libomptarget/test/lit.cfg

 
 def append_dynamic_library_path(name, value, sep):
     if name in config.environment:
         config.environment[name] = value + sep + config.environment[name]
     else:
         config.environment[name] = value
 
 # name: The name of this test suite.
-config.name = 'libomptarget'
+config.name = 'libomptarget :: ' + config.libomptarget_current_target
 
 # suffixes: A list of file extensions to treat as test files.
 config.suffixes = ['.c', '.cpp', '.cc']
 
 # test_source_root: The root path where tests are located.
 config.test_source_root = os.path.dirname(__file__)
 
 # test_exec_root: The root object directory where output is placed

 # Allow REQUIRES / UNSUPPORTED / XFAIL to work
 config.target_triple = [ ]
 for feature in config.test_compiler_features:
     config.available_features.add(feature)
 
 if config.libomptarget_debug:
   config.available_features.add('libomptarget-debug')
 
+config.available_features.add(config.libomptarget_current_target)
+
 # Setup environment to find dynamic library at runtime
 if config.operating_system == 'Windows':
     append_dynamic_library_path('PATH', config.library_dir, ";")
     append_dynamic_library_path('PATH', config.omp_host_rtl_directory, ";")
 elif config.operating_system == 'Darwin':
     append_dynamic_library_path('DYLD_LIBRARY_PATH', config.library_dir, ":")
     append_dynamic_library_path('DYLD_LIBRARY_PATH', \
         config.omp_host_rtl_directory, ";")

 # - for targets that exist in the system create the actual command.
 # - for valid targets that do not exist in the system, return false, so that the
 #   same test can be used for different targets.
 
 # Scan all the valid targets.
 for libomptarget_target in config.libomptarget_all_targets:
     # Is this target in the current system? If so create a compile, run and test
     # command. Otherwise create command that return false.
-    if libomptarget_target in config.libomptarget_system_targets:
+    if libomptarget_target == config.libomptarget_current_target:
+        config.substitutions.append(("%libomptarget-compilexx-run-and-check-generic", 
+            "%libomptarget-compilexx-run-and-check-" + libomptarget_target))
+        config.substitutions.append(("%libomptarget-compile-run-and-check-generic",
+            "%libomptarget-compile-run-and-check-" + libomptarget_target))
+        config.substitutions.append(("%libomptarget-compilexx-and-run-generic",
+            "%libomptarget-compilexx-and-run-" + libomptarget_target))
+        config.substitutions.append(("%libomptarget-compile-and-run-generic",
+            "%libomptarget-compile-and-run-" + libomptarget_target))
+        config.substitutions.append(("%libomptarget-compilexx-generic",
+            "%libomptarget-compilexx-" + libomptarget_target))
+        config.substitutions.append(("%libomptarget-compile-generic",
+            "%libomptarget-compile-" + libomptarget_target))
+        config.substitutions.append(("%libomptarget-run-generic",
+            "%libomptarget-run-" + libomptarget_target))
+        config.substitutions.append(("%libomptarget-run-fail-generic",
+            "%libomptarget-run-fail-" + libomptarget_target))
+        config.substitutions.append(("%clangxx-generic",
+            "%clangxx-" + libomptarget_target))
+        config.substitutions.append(("%clang-generic",
+            "%clang-" + libomptarget_target))
+        config.substitutions.append(("%fcheck-generic",
+            config.libomptarget_filecheck + " %s"))
+
+
         config.substitutions.append(("%libomptarget-compilexx-run-and-check-" + \
             libomptarget_target, \
             "%libomptarget-compilexx-and-run-" + libomptarget_target + \
             " | " + config.libomptarget_filecheck + " %s"))
         config.substitutions.append(("%libomptarget-compile-run-and-check-" + \
             libomptarget_target, \
             "%libomptarget-compile-and-run-" + libomptarget_target + \
             " | " + config.libomptarget_filecheck + " %s"))

openmp/libomptarget/test/lit.site.cfg.in

 @AUTO_GEN_COMMENT@
 
 config.test_c_compiler = "@OPENMP_TEST_C_COMPILER@"
 config.test_cxx_compiler = "@OPENMP_TEST_CXX_COMPILER@"
 config.test_compiler_features = @OPENMP_TEST_COMPILER_FEATURES@
 config.test_openmp_flags = "@OPENMP_TEST_OPENMP_FLAGS@"
 config.test_extra_flags = "@OPENMP_TEST_FLAGS@"
-config.libomptarget_obj_root = "@CMAKE_CURRENT_BINARY_DIR@"
+config.libomptarget_obj_root = "@CMAKE_CURRENT_BINARY_DIR@/@CURRENT_TARGET@"
 config.library_dir = "@LIBOMPTARGET_LIBRARY_DIR@"
 config.omp_header_directory = "@LIBOMPTARGET_OPENMP_HEADER_FOLDER@"
 config.omp_host_rtl_directory = "@LIBOMPTARGET_OPENMP_HOST_RTL_FOLDER@"
 config.operating_system = "@CMAKE_SYSTEM_NAME@"
 config.libomptarget_all_targets = "@LIBOMPTARGET_ALL_TARGETS@".split()
-config.libomptarget_system_targets = "@LIBOMPTARGET_SYSTEM_TARGETS@".split()
+config.libomptarget_current_target = "@CURRENT_TARGET@"
 config.libomptarget_filecheck = "@OPENMP_FILECHECK_EXECUTABLE@"
 config.libomptarget_not = "@OPENMP_NOT_EXECUTABLE@"
 config.libomptarget_debug = @LIBOMPTARGET_DEBUG@
 
 # Let the main config do the real work.
 lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg")