[openmp] Fix flang detection for offloading test

[elmcdonough]

Summary

This patch fixes the flang detection in the openmp fortran offloading test.

[jsjodin]

This change works for me, so the test is run and fails with:

ld.lld: error: undefined symbol: increment_at
>>>> referenced by /tmp/lit-tmp-hsj2_6gq/basic_array.c.tmp.amdgcn-amd-amdhsa.gfx90a-465f86.o:(__omp_offloading_802_ae2379e_increment_array_l17)
>>>> referenced by /tmp/lit-tmp-hsj2_6gq/basic_array.c.tmp.amdgcn-amd-amdhsa.gfx90a-465f86.o:(__omp_offloading_802_ae2379e_increment_array_l17)
clang: error: ld.lld command failed with exit code 1 (use -v to see invocation)
/home/jsjodin/git/trunk/llvm-project/build/./bin/clang-linker-wrapper: error: 'clang' failed
clang: error: linker command failed with exit code 1 (use -v to see invocation)

error: command failed with exit status: 1

[elmcdonough]

In D158546#4610557, @jsjodin wrote:

This change works for me, so the test is run and fails with:

ld.lld: error: undefined symbol: increment_at
>>>> referenced by /tmp/lit-tmp-hsj2_6gq/basic_array.c.tmp.amdgcn-amd-amdhsa.gfx90a-465f86.o:(__omp_offloading_802_ae2379e_increment_array_l17)
>>>> referenced by /tmp/lit-tmp-hsj2_6gq/basic_array.c.tmp.amdgcn-amd-amdhsa.gfx90a-465f86.o:(__omp_offloading_802_ae2379e_increment_array_l17)
clang: error: ld.lld command failed with exit code 1 (use -v to see invocation)
/home/jsjodin/git/trunk/llvm-project/build/./bin/clang-linker-wrapper: error: 'clang' failed
clang: error: linker command failed with exit code 1 (use -v to see invocation)

error: command failed with exit status: 1

Yeah, that test fails at the moment. Right now it seems that increment_at isn't registered as a symbol in the LLVM bitcode for the embedded object. I can confirm that this used to work, but I'm still trying to figure out why it doesn't anymore.

[elmcdonough]

Fixes the actual test. Adding !$omp declare target to the offloaded function prevents the OMPFunctionFiltering pass from optimizing it out. I'm not sure if we want public functions to be hidden from object files targeting GPUs, but I think that might be outside the scope of this patch.

[jsjodin]

In D158546#4623296, @elmcdonough wrote:

Fixes the actual test. Adding !$omp declare target to the offloaded function prevents the OMPFunctionFiltering pass from optimizing it out. I'm not sure if we want public functions to be hidden from object files targeting GPUs, but I think that might be outside the scope of this patch.

LGTM, the test passes for me.

I'm not sure what the correct behavior is. The standard says:
"If a procedure is explicitly or implicitly referenced in a target construct that does not specify a device clause in which the ancestor device-modifier appears then that procedure is treated as if its name had appeared in an enter clause on a declare target directive"
It doesn't say it has to be in the same compilation unit, but it also doesn't seem unreasonable to have to specify declare target if it is outside the compilation unit. @jdoerfert, do you know correct behavior is?

[jdoerfert]

This unfortunately picks up a random flang-new, if there is one on the path.

openmp/CMakeLists.txt

if (OPENMP_STANDALONE_BUILD)
   # Do not use OPENMP_LIBDIR_SUFFIX directly, use OPENMP_INSTALL_LIBDIR.
   set(OPENMP_INSTALL_LIBDIR "lib${OPENMP_LIBDIR_SUFFIX}")
 
   # Group test settings.
   set(OPENMP_TEST_C_COMPILER ${CMAKE_C_COMPILER} CACHE STRING
     "C compiler to use for testing OpenMP runtime libraries.")
   set(OPENMP_TEST_CXX_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING
     "C++ compiler to use for testing OpenMP runtime libraries.")
+  set(OPENMP_TEST_Fortran_COMPILER ${CMAKE_Fortran_COMPILER} CACHE STRING
+    "FORTRAN compiler to use for testing OpenMP runtime libraries.")
   set(OPENMP_LLVM_TOOLS_DIR "" CACHE PATH "Path to LLVM tools for testing.")
 
   set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to conform to")
   set(CMAKE_CXX_STANDARD_REQUIRED NO)
   set(CMAKE_CXX_EXTENSIONS NO)
 else()
   set(OPENMP_ENABLE_WERROR ${LLVM_ENABLE_WERROR})
   # If building in tree, we honor the same install suffix LLVM uses.
   set(OPENMP_INSTALL_LIBDIR "lib${LLVM_LIBDIR_SUFFIX}")
 
   if (NOT MSVC)
     set(OPENMP_TEST_C_COMPILER ${LLVM_RUNTIME_OUTPUT_INTDIR}/clang)
     set(OPENMP_TEST_CXX_COMPILER ${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++)
   else()
     set(OPENMP_TEST_C_COMPILER ${LLVM_RUNTIME_OUTPUT_INTDIR}/clang.exe)
     set(OPENMP_TEST_CXX_COMPILER ${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++.exe)
   endif()
 
+  find_program(OPENMP_TEST_Fortran_COMPILER flang-new PATHS LLVM_RUNTIME_OUTPUT_INTDIR)
+  if (NOT OPENMP_TEST_Fortran_COMPILER)
+    unset(OPENMP_TEST_Fortran_COMPILER CACHE)
+  endif()
+
   # If not standalone, set CMAKE_CXX_STANDARD but don't set the global cache value,
   # only set it locally for OpenMP.
   set(CMAKE_CXX_STANDARD 17)
   set(CMAKE_CXX_STANDARD_REQUIRED NO)
   set(CMAKE_CXX_EXTENSIONS NO)
 endif()
 
 # Check and set up common compiler flags.

openmp/README.rst

 **OPENMP_TEST_C_COMPILER** = ``${CMAKE_C_COMPILER}``
   Compiler to use for testing. Defaults to the compiler that was also used for
   building.
 
 **OPENMP_TEST_CXX_COMPILER** = ``${CMAKE_CXX_COMPILER}``
   Compiler to use for testing. Defaults to the compiler that was also used for
   building.
 
+**OPENMP_TEST_Fortran_COMPILER** = ``${CMAKE_Fortran_COMPILER}``
+  Compiler to use for testing. Defaults to the compiler that was also used for
+  building. Will default to flang if build is in-tree.
+
 **OPENMP_LLVM_TOOLS_DIR** = ``/path/to/built/llvm/tools``
   Additional path to search for LLVM tools needed by tests.
 
 **OPENMP_LLVM_LIT_EXECUTABLE** = ``/path/to/llvm-lit``
   Specify full path to ``llvm-lit`` executable for running tests.  The default
   is to search the ``PATH`` and the directory in **OPENMP_LLVM_TOOLS_DIR**.
 
 **OPENMP_FILECHECK_EXECUTABLE** = ``/path/to/FileCheck``
**LIBOMP_ENABLE_SHARED** = ``ON|OFF``
 
   .. note::
 
     Static libraries are not supported on Windows*.
 
 **LIBOMP_FORTRAN_MODULES** = ``OFF|ON``
   Create the Fortran modules (requires Fortran compiler).
 
-  .. note::
-
-  If libomptarget is built in-tree with both flang and openmp in
-  `LLVM_ENABLE_PROJECTS`, flang will be used for Fortran offloading
-  tests.
-
 macOS* Fat Libraries
 """"""""""""""""""""
 On macOS* machines, it is possible to build universal (or fat) libraries which
 include both i386 and x86_64 architecture objects in a single archive.
 
 .. code-block:: console
 
   $ cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_OSX_ARCHITECTURES='i386;x86_64' ..

openmp/libomptarget/test/CMakeLists.txt

foreach(CURRENT_TARGET IN LISTS SYSTEM_TARGETS)
   list(APPEND LIBOMPTARGET_LIT_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CURRENT_TARGET})
 
   # Configure the lit.site.cfg.in file
   set(AUTO_GEN_COMMENT "## Autogenerated by libomptarget configuration.\n# Do not edit!")
   get_filename_component(CUDA_LIBDIR "${CUDA_cudart_static_LIBRARY}" DIRECTORY)
   configure_file(lit.site.cfg.in ${CURRENT_TARGET}/lit.site.cfg @ONLY)
 endforeach()
 
-if ("flang" IN_LIST LLVM_ENABLE_PROJECTS)
-  SET(FORTRAN_TEST_DEPS flang-new)
-endif()
 
 add_openmp_testsuite(check-libomptarget
   "Running libomptarget tests"
   ${LIBOMPTARGET_LIT_TESTSUITES}
   EXCLUDE_FROM_CHECK_ALL
-  DEPENDS omptarget omp ${LIBOMPTARGET_TESTED_PLUGINS} ${FORTRAN_TEST_DEPS}
+  DEPENDS omptarget omp ${LIBOMPTARGET_TESTED_PLUGINS}
   ARGS ${LIBOMPTARGET_LIT_ARG_LIST})

openmp/libomptarget/test/lit.cfg

 # -*- Python -*- vim: set ft=python ts=4 sw=4 expandtab tw=79:
 # Configuration file for the 'lit' test runner.
 
 import os
 import lit.formats
 
-from lit.llvm import llvm_config
-from lit.llvm.subst import ToolSubst
-from lit.llvm.subst import FindTool
-
 # Tell pylint that we know config and lit_config exist somewhere.
 if 'PYLINT_IMPORT' in os.environ:
     config = object()
     lit_config = object()
 
 # Use the CUDA device as suggested by the env
 if 'CUDA_VISIBLE_DEVICES' in os.environ:
     config.environment['CUDA_VISIBLE_DEVICES'] = os.environ['CUDA_VISIBLE_DEVICES']

 config.suffixes = ['.c', '.cpp', '.cc']
 
 # test_source_root: The root path where tests are located.
 config.test_source_root = os.path.dirname(__file__)
 
 # test_exec_root: The root object directory where output is placed
 config.test_exec_root = config.libomptarget_obj_root
 
-tools = []
-
 # test format
 config.test_format = lit.formats.ShTest()
 
 # compiler flags
 config.test_flags = " -I " + config.test_source_root + \
     " -I " + config.omp_header_directory + \
     " -L " + config.library_dir + \
     " -L " + config.llvm_lib_directory

 if config.libomptarget_debug:
   config.available_features.add('libomptarget-debug')
 
 if config.has_libomptarget_ompt:
   config.available_features.add('ompt')
 
 config.available_features.add(config.libomptarget_current_target)
 
-if 'flang' in config.llvm_enabled_projects:
-  config.available_features.add('flang')
-  tools.append(ToolSubst('%flang', command=FindTool('flang-new'), unresolved='fatal'))
-
 if config.libomptarget_has_libc:
   config.available_features.add('libc')
 
 # Determine whether the test system supports unified memory.
 # For CUDA, this is the case with compute capability 70 (Volta) or higher.
 # For all other targets, we currently assume it is.
 supports_unified_shared_memory = True
 if config.libomptarget_current_target.startswith('nvptx'):
else:
             "echo ignored-command"))
         config.substitutions.append(("%clangxx-" + libomptarget_target, \
             "echo ignored-command"))
         config.substitutions.append(("%fcheck-" + libomptarget_target, \
             "echo ignored-command"))
 
 config.substitutions.append(("%clangxx", config.test_cxx_compiler))
 config.substitutions.append(("%clang", config.test_c_compiler))
+
+if config.test_fortran_compiler:
+    config.available_features.add('flang')
+    config.substitutions.append(("%flang", config.test_fortran_compiler))
+
 config.substitutions.append(("%openmp_flags", config.test_openmp_flags))
 if config.libomptarget_current_target.startswith('nvptx') and config.cuda_path:
     config.substitutions.append(("%cuda_flags", "--cuda-path=" + config.cuda_path))
 else:
     config.substitutions.append(("%cuda_flags", ""))
 config.substitutions.append(("%flags", config.test_flags))
 config.substitutions.append(("%not", config.libomptarget_not))
-llvm_config.add_tool_substitutions(tools, config.bin_llvm_tools_dir)

openmp/libomptarget/test/lit.site.cfg.in

 @AUTO_GEN_COMMENT@
 
 config.bin_llvm_tools_dir = "@CMAKE_BINARY_DIR@/bin"
 config.test_c_compiler = "@OPENMP_TEST_C_COMPILER@"
 config.test_cxx_compiler = "@OPENMP_TEST_CXX_COMPILER@"
+config.test_fortran_compiler="@OPENMP_TEST_Fortran_COMPILER@"
 config.test_compiler_features = @OPENMP_TEST_COMPILER_FEATURES@
 config.test_openmp_flags = "@OPENMP_TEST_OPENMP_FLAGS@"
 config.test_extra_flags = "@OPENMP_TEST_FLAGS@"
 config.cuda_path = "@CUDA_TOOLKIT_ROOT_DIR@"
 config.cuda_libdir = "@CUDA_LIBDIR@"
 config.cuda_test_arch = "@LIBOMPTARGET_DEP_CUDA_ARCH@"
 config.libomptarget_obj_root = "@CMAKE_CURRENT_BINARY_DIR@/@CURRENT_TARGET@"
 config.library_dir = "@LIBOMPTARGET_LIBRARY_DIR@"
 config.omp_header_directory = "@LIBOMPTARGET_OPENMP_HEADER_FOLDER@"
 config.omp_host_rtl_directory = "@LIBOMPTARGET_OPENMP_HOST_RTL_FOLDER@"
 config.llvm_lib_directory = "@LIBOMPTARGET_LLVM_LIBRARY_DIR@"
 config.operating_system = "@CMAKE_SYSTEM_NAME@"
 config.libomptarget_all_targets = "@LIBOMPTARGET_ALL_TARGETS@".split()
 config.libomptarget_current_target = "@CURRENT_TARGET@"
 config.libomptarget_filecheck = "@OPENMP_FILECHECK_EXECUTABLE@"
 config.libomptarget_not = "@OPENMP_NOT_EXECUTABLE@"
 config.libomptarget_debug = @LIBOMPTARGET_DEBUG@
 config.has_libomptarget_ompt = @LIBOMPTARGET_OMPT_SUPPORT@
-config.llvm_enabled_projects = "@LLVM_ENABLE_PROJECTS@".split(";")
 config.libomptarget_has_libc = @LIBOMPTARGET_GPU_LIBC_SUPPORT@
 
-import lit.llvm
-lit.llvm.initialize(lit_config, config)
-
 # Let the main config do the real work.
 lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg")

openmp/libomptarget/test/offloading/fortran/basic_array.f90

 subroutine increment_at(c_index, arr) bind(C, name="increment_at")
     use ISO_C_BINDING
+    !$omp declare target
     integer (C_INT), dimension(*), intent(inout) :: arr
     integer (C_INT), value :: c_index
     arr(c_index+1) = arr(c_index+1) + 1
 end subroutine