[test-suite] Add sollve_vv external project.

[Meinersbur]

Summary

sollve_vv is the OpenMP Validation & Verification suite to test conformance to with the OpenMP specification.

It is included as an external benchmark, i.e. some work is required before it is enabled, for instance setting TEST_SUITE_SOLLVEVV_ROOT to the downloaded source repository. OpenMP is detected using CMake's find_package(OpenMP).

Unfortunately, it does not detect additional flags required for offloading which have to be set manually. The current defaults are the ones I used to successfully offload to NVidia's V100 GPU with clang. I am open to discuss what the default should be or whether we should have a compiler/platform introspection. sollve_vv's own makefile use a default set of flags per compiler executable name.

[homerdin]

Thanks for patch! It will be great to have this setup available.

Could you add a short README with what the sollve suite is and a quick example for building and controlling the offload flags.

External/CMakeLists.txt

7

From what I understand the sollve benchmarks are less useful for performance benchmarking, please consider wrapping this add_subdirectory with if(NOT TEST_SUITE_BENCHMARKING_ONLY) unless I'm mistaken.

External/sollve_vv/CMakeLists.txt

54

It would be nice to be able to checkout the suite into this directory and have it picked up automatically. I don't know enough about the other external suites to know if this should be put into the llvm_externals_find logic. We could add

if( NOT TEST_SUITE_SOLLVEVV_ROOT)
  if( IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/sollve_vv")
    set(TEST_SUITE_SOLLVEVV_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/sollve_vv")
  endif()
endif()

[Meinersbur]

• Address @bhomerdin's review:
  • Add README
  • Respect TEST_SUITE_BENCHMARKING_ONLY

External/sollve_vv/CMakeLists.txt

54

The test-suite has a standard way to look for external source files. See the command llvm_externals_find in cmake/modules/External.cmake. I also added it to the README.

[homerdin]

LGTM, thanks.

[Meinersbur]

Committed in https://github.com/llvm/llvm-test-suite/commit/9a9a976cb5cc73a411df8e6f1a1e2d49c804b858

CMakeLists.txt

if (TEST_SUITE_CXX_ABI STREQUAL "default")
   endif()
 else()
   STRING(REGEX REPLACE "^lib" "" TEST_SUITE_CXX_ABI_LIBNAME ${TEST_SUITE_CXX_ABI})
 endif()
 
 include(TestSuite)
 include(SingleMultiSource)
 find_package(TCL)
+find_package(OpenMP)
 
 if(NOT DEFINED TARGET_OS)
   message(STATUS "Check target operating system - ${CMAKE_SYSTEM_NAME}")
   set(TARGET_OS ${CMAKE_SYSTEM_NAME})
 endif()
 if(NOT DEFINED ARCH)
   include(DetectArchitecture)
   detect_architecture(ARCH)

External/CMakeLists.txt

 add_subdirectory(CUDA)
 add_subdirectory(HMMER)
 add_subdirectory(Nurbs)
 add_subdirectory(Povray)
 add_subdirectory(SPEC)
 add_subdirectory(skidmarks10)
+add_subdirectory(sollve_vv)

External/sollve_vv/CMakeLists.txt

+# SOLLVE OpenMP Offloading Validation & Verification Suite
+# https://crpl.cis.udel.edu/ompvvsollve/
+
+set(TEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS "-fopenmp-targets=nvptx64-nvidia-cuda;-Xopenmp-target;-march=sm_70" CACHE STRING "Compiler arguments for OpenMP offloading")
+set(TEST_SUITE_SOLLVEVV_OFFLOADING_LDFLAGS "-fopenmp-targets=nvptx64-nvidia-cuda;-Xopenmp-target;-march=sm_70;-lopenmptarget" CACHE STRING "Linker arguments for OpenMP offloading")
+
+function (add_sollvevv LANG)
+  set(_includedir "${TEST_SUITE_SOLLVEVV_ROOT}/ompvv" )
+  
+  if (NOT OPENMP_${LANG}_FOUND) 
+    message(FATAL_ERROR "OpenMP for ${LANG} not found")
+    return ()
+  endif ()
+  
+  #if (OpenMP_${LANG}_VERSION VERSION_LESS "4.5")
+  #  message(FATAL_ERROR "OpenMP version ${OpenMP_${LANG}_VERSION} too old")
+  #endif ()
+  
+  if ("${LANG}" STREQUAL "C")
+    set(_langext ".c")
+  elseif ("${LANG}" STREQUAL "CXX")
+    set(_langext ".cpp")
+  else ()
+    message(FATAL_ERROR "Unsupported languge ${LANG}")
+  endif ()
+  
+  file(GLOB_RECURSE _tests_sources RELATIVE "${TEST_SUITE_SOLLVEVV_ROOT}/tests" "${TEST_SUITE_SOLLVEVV_ROOT}/tests/*${_langext}" )
+  foreach (_file IN LISTS _tests_sources)
+    get_filename_component(_ext "${_file}" EXT)
+    get_filename_component(_basename "${_file}" NAME_WE)
+    get_filename_component(_directory "${_file}" DIRECTORY)
+    string(REPLACE "." "" _ext "${_ext}")
+    string(REPLACE "/" "_" _directory "${_directory}")
+    string(REPLACE "." "" _directory "${_directory}")
+    set(_name "omptargetvv-${_basename}-${_directory}-${_ext}")
+
+    llvm_test_run()
+    
+    llvm_test_executable(${_name} "${TEST_SUITE_SOLLVEVV_ROOT}/tests/${_file}")
+    target_include_directories(${_name} PRIVATE "${_includedir}")
+    target_link_libraries(${_name} PUBLIC OpenMP::OpenMP_${_lang})
+
+    # Add -fopenmp to linker command line; for some reason this is not done by target_link_libraries.
+    target_link_options(${_name} PRIVATE ${OpenMP_${LANG}_FLAGS})
+
+    # CMake's find_package(OpenMP) currently does not not introspect flags necessary for offloading.
+    target_compile_options(${_name} PUBLIC ${TEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS})
+    target_link_options(${_name} PUBLIC ${TEST_SUITE_SOLLVEVV_OFFLOADING_LDFLAGS})
+  endforeach ()
+endfunction ()
+
+
+llvm_externals_find(TEST_SUITE_SOLLVEVV_ROOT "sollve_vv" "OpenMP Offloading Validation & Verification Suite")
+if (TEST_SUITE_SOLLVEVV_ROOT AND NOT TEST_SUITE_BENCHMARKING_ONLY)
+  if (OPENMP_FOUND)
+    message(STATUS "Adding OpenMP Offloading Validiation & Verification")
+  else ()
+    message(STATUS "NOT using OpenMP Validiation & Verification because OpenMP was not found")
+    return ()
+  endif ()
+  
+  foreach (_lang in C CXX) 
+    if (CMAKE_${_lang}_COMPILER)
+      add_sollvevv(${_lang})
+    endif()
+  endforeach ()
+endif ()

External/sollve_vv/README

+SOLLVE OpenMP Offloading Validation & Verification Suite
+https://crpl.cis.udel.edu/ompvvsollve/
+
+This directory contains a CMakeLists.txt for the SOLLVE OpenMP
+Offloading Validation and Verification Suite so it can be built as part
+of the LLVM test-suite. Its sources are not part of the test-suite but
+have to be fetched separately from https://github.com/SOLLVE/sollve_vv.
+
+The sources are expected either in ${TEST_SUITE_SOLLVEVV_ROOT} or
+${TEST_SUITE_EXTERNALS_DIR}/sollve_vv, where TEST_SUITE_SOLLVEVV_ROOT
+and TEST_SUITE_EXTERNALS_DIR are CMake configure variables. If none of
+them are set, it will look into
+${CMAKE_SOURCE_DIR}/test-suite-externals/sollve_vv where
+CMAKE_SOURCE_DIR is the root directory of the test-suite sources.
+
+The CMakeLists.txt will search for all C and C++ source files of the
+OpenMP V&V suite, compile and run them. That is, running llvm-lit
+(or "make check") will require a compatible accelerator on the running
+machine.
+
+OpenMP support is autodetected by CMake, but clang requires additional
+flags to enable offloading. An example run is:
+
+$ cmake ../llvm-test-suite -GNinja -DCMAKE_BUILD_TYPE=Release \
+  -DTEST_SUITE_SOLLVEVV_ROOT=${HOME}/src/sollve_vv \
+  -DTEST_SUITE_LIT=${HOME}/build/llvm-project/release/bin/llvm-lit \
+  -DCMAKE_C_COMPILER=${HOME}/install/llvm-project/release/bin/clang \
+  -DCMAKE_CXX_COMPILER=${HOME}/install/llvm-project/release/bin/clang++ \
+  -DTEST_SUITE_SOLLVEVV_OFFLOADING_CFLAGS=-fopenmp-targets=nvptx64-nvidia-cuda;--cuda-path=/soft/compilers/cuda/cuda-10.1.243;-Xopenmp-target;-march=sm_70 \
+  -DTEST_SUITE_SOLLVEVV_OFFLOADING_LDFLAGS=-fopenmp-targets=nvptx64-nvidia-cuda;--cuda-path=/soft/compilers/cuda/cuda-10.1.243;-Xopenmp-target;-march=sm_70;-lopenmptarget
+
+$ LD_LIBRARY_PATH=${HOME}/install/llvm-project/release/lib ninja check 
+
+
+Clang also needs to be compiled with enabled offloading for the chosen
+target. A configuration compatible for the commands above is:
+
+$ cmake ../llvm-project/llvm -GNinja -DCMAKE_BUILD_TYPE=Release   \
+  -DCMAKE_INSTALL_PREFIX=${HOME}/install/llvm-project/release     \  
+  -DLLVM_ENABLE_PROJECTS=clang;openmp                             \
+  -CUDA_TOOLKIT_ROOT_DIR=/soft/compilers/cuda/cuda-10.1.243       \
+  -DLIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES=70
+
+$ cmake install
+
+
+In this example, Clang is not installed into a default search path such
+that the paths have to be specified explicitly.