[Libomptarget] Build the device runtime as a static library

[jhuber6]

Summary

This patch adds the necessary CMake configuration to build a static
library version of the device runtime, libomptarget.devicertl.a.
Various improvements in how we handle static libraries and generating
offloading code should allow us to treat the device library as a regular
project without needing to invoke the clang front-end directly. Here we
generate a job for each offloading architecture supported. Each
offloading architecture will be embedded into the static library and
used as-needed by the host.

This library will primarily be used to replace the bitcode library when
performing LTO. Currently, we need to manually pass in the bitcode
library which requires foreknowledge of the offloading architecture.
This approach lets us handle that in the linker wrapper instead.
Furthermore this should improve our interface to the device runtime. We
can now build it fully under a release build and have all the expected
entry points, as well as supporting debug builds.

Depends on D125265 D125256 D125260 D125314 D125563

[tianshilei1992]

This design is much better because we don't need to have one bc file for each target. What's more, it uses CMake command to compile the library, which has better dependence tracking. Just one comment on the order stuff.

openmp/libomptarget/CMakeLists.txt

41

The order here is critical because both openmp/libomptarget/include and openmp/libomptarget/DeviceRTL/include has Debug.h. However, I don't think that's a good practice to use order to guard that. IMO, we need to move the global include to where it is actually being used.

[jhuber6]

openmp/libomptarget/CMakeLists.txt

41

We need to change the include_directories to target_include_directories and specify it for each target. I wasn't sure if I should bother doing that for this patch.

[tianshilei1992]

openmp/libomptarget/CMakeLists.txt

41

That is the better way. No need to do it here but worth another patch. However, using order is not a good idea.

[jhuber6]

Rebase to not change the include directories.

[tianshilei1992]

LGTM

[nikic]

This change has broken the build of libomp with gcc, as it appears to be using clang-specific options:

g++: error: unrecognized command-line option '-Xopenmp-target=nvptx64-nvidia-cuda'
g++: error: unrecognized command-line option '--cuda-feature=+ptx61'
g++: error: unrecognized command-line option '-nocudalib'; did you mean '-nostdlib'?
g++: error: unrecognized command-line option '-nogpulib'
g++: error: unrecognized command-line option '--offload-arch=sm_35'
g++: error: unrecognized command-line option '--offload-arch=sm_37'
g++: error: unrecognized command-line option '--offload-arch=sm_50'
g++: error: unrecognized command-line option '--offload-arch=sm_52'
g++: error: unrecognized command-line option '--offload-arch=sm_53'
g++: error: unrecognized command-line option '--offload-arch=sm_60'; did you mean '--offload-abi=lp64'?
g++: error: unrecognized command-line option '--offload-arch=sm_61'; did you mean '--offload-abi=lp64'?
g++: error: unrecognized command-line option '--offload-arch=sm_62'
g++: error: unrecognized command-line option '--offload-arch=sm_70'
g++: error: unrecognized command-line option '--offload-arch=sm_72'
g++: error: unrecognized command-line option '--offload-arch=sm_75'
g++: error: unrecognized command-line option '--offload-arch=sm_80'
g++: error: unrecognized command-line option '--offload-arch=sm_86'
g++: error: unrecognized command-line option '--offload-arch=gfx700'
g++: error: unrecognized command-line option '--offload-arch=gfx701'
g++: error: unrecognized command-line option '--offload-arch=gfx801'
g++: error: unrecognized command-line option '--offload-arch=gfx803'
g++: error: unrecognized command-line option '--offload-arch=gfx900'
g++: error: unrecognized command-line option '--offload-arch=gfx902'
g++: error: unrecognized command-line option '--offload-arch=gfx906'
g++: error: unrecognized command-line option '--offload-arch=gfx908'
g++: error: unrecognized command-line option '--offload-arch=gfx90a'
g++: error: unrecognized command-line option '--offload-arch=gfx90c'
g++: error: unrecognized command-line option '--offload-arch=gfx940'
g++: error: unrecognized command-line option '--offload-arch=gfx1010'
g++: error: unrecognized command-line option '--offload-arch=gfx1030'
g++: error: unrecognized command-line option '--offload-arch=gfx1031'
g++: error: unrecognized command-line option '--offload-arch=gfx1032'
g++: error: unrecognized command-line option '--offload-arch=gfx1033'
g++: error: unrecognized command-line option '--offload-arch=gfx1034'
g++: error: unrecognized command-line option '--offload-arch=gfx1035'
g++: error: unrecognized command-line option '--offload-arch=gfx1036'

[jhuber6]

In D125315#3515223, @nikic wrote:

This change has broken the build of libomp with gcc, as it appears to be using clang-specific options:

g++: error: unrecognized command-line option '-Xopenmp-target=nvptx64-nvidia-cuda'
g++: error: unrecognized command-line option '--cuda-feature=+ptx61'
g++: error: unrecognized command-line option '-nocudalib'; did you mean '-nostdlib'?
g++: error: unrecognized command-line option '-nogpulib'
g++: error: unrecognized command-line option '--offload-arch=sm_35'
g++: error: unrecognized command-line option '--offload-arch=sm_37'
g++: error: unrecognized command-line option '--offload-arch=sm_50'
g++: error: unrecognized command-line option '--offload-arch=sm_52'
g++: error: unrecognized command-line option '--offload-arch=sm_53'
g++: error: unrecognized command-line option '--offload-arch=sm_60'; did you mean '--offload-abi=lp64'?
g++: error: unrecognized command-line option '--offload-arch=sm_61'; did you mean '--offload-abi=lp64'?
g++: error: unrecognized command-line option '--offload-arch=sm_62'
g++: error: unrecognized command-line option '--offload-arch=sm_70'
g++: error: unrecognized command-line option '--offload-arch=sm_72'
g++: error: unrecognized command-line option '--offload-arch=sm_75'
g++: error: unrecognized command-line option '--offload-arch=sm_80'
g++: error: unrecognized command-line option '--offload-arch=sm_86'
g++: error: unrecognized command-line option '--offload-arch=gfx700'
g++: error: unrecognized command-line option '--offload-arch=gfx701'
g++: error: unrecognized command-line option '--offload-arch=gfx801'
g++: error: unrecognized command-line option '--offload-arch=gfx803'
g++: error: unrecognized command-line option '--offload-arch=gfx900'
g++: error: unrecognized command-line option '--offload-arch=gfx902'
g++: error: unrecognized command-line option '--offload-arch=gfx906'
g++: error: unrecognized command-line option '--offload-arch=gfx908'
g++: error: unrecognized command-line option '--offload-arch=gfx90a'
g++: error: unrecognized command-line option '--offload-arch=gfx90c'
g++: error: unrecognized command-line option '--offload-arch=gfx940'
g++: error: unrecognized command-line option '--offload-arch=gfx1010'
g++: error: unrecognized command-line option '--offload-arch=gfx1030'
g++: error: unrecognized command-line option '--offload-arch=gfx1031'
g++: error: unrecognized command-line option '--offload-arch=gfx1032'
g++: error: unrecognized command-line option '--offload-arch=gfx1033'
g++: error: unrecognized command-line option '--offload-arch=gfx1034'
g++: error: unrecognized command-line option '--offload-arch=gfx1035'
g++: error: unrecognized command-line option '--offload-arch=gfx1036'

The OpenMP offloading runtime was always intended to be built with a newly-built Clang, either through a two-phase build or with -DLLVM_ENABLE_RUNTIMES=openmp. This patch added some code that uses standard CMake to build the library, rather than locating the clang binary directly. I could add some code to skip building this static library, or the device runtime entirely, if the compiler isn't an up-to-date Clang. I'm not sure what the best solution to this is, since we always required that this was to be built with Clang, this patch just made it a more strict requirement.

openmp/libomptarget/CMakeLists.txt

 
 # LLVM source tree is required at build time for libomptarget
 if (NOT LIBOMPTARGET_LLVM_INCLUDE_DIRS)
   message(FATAL_ERROR "Missing definition for LIBOMPTARGET_LLVM_INCLUDE_DIRS")
 endif()
 
 include_directories(${LIBOMPTARGET_LLVM_INCLUDE_DIRS})
 
+# Build the device RTL before we set the libomptarget include directories.
+add_subdirectory(DeviceRTL)
+
 # This is a list of all the targets that are supported/tested right now.
 set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} aarch64-unknown-linux-gnu")
 set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} aarch64-unknown-linux-gnu-oldDriver")
 set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} amdgcn-amd-amdhsa")
 set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} amdgcn-amd-amdhsa-oldDriver")
 set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} powerpc64le-ibm-linux-gnu")
 set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} powerpc64le-ibm-linux-gnu-oldDriver")
 set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} powerpc64-ibm-linux-gnu")

 set(LIBOMPTARGET_OPENMP_HEADER_FOLDER "${LIBOMP_INCLUDE_DIR}" CACHE STRING
   "Path to folder containing omp.h")
 set(LIBOMPTARGET_OPENMP_HOST_RTL_FOLDER "${LIBOMP_LIBRARY_DIR}" CACHE STRING
   "Path to folder containing libomp.so, and libLLVMSupport.so with profiling enabled")
 
 
 # Build offloading plugins and device RTLs if they are available.
 add_subdirectory(plugins)
-add_subdirectory(DeviceRTL)
 add_subdirectory(tools)
 
 # Add tests.
 add_subdirectory(test)

openmp/libomptarget/DeviceRTL/CMakeLists.txt

 foreach(sm ${nvptx_sm_list})
   compileDeviceRTLLibrary(sm_${sm} nvptx -target nvptx64-nvidia-cuda -Xclang -target-feature -Xclang +ptx61 "-D__CUDA_ARCH__=${sm}0")
 endforeach()
 
 add_custom_target(omptarget.devicertl.amdgpu)
 foreach(mcpu ${amdgpu_mcpus})
   compileDeviceRTLLibrary(${mcpu} amdgpu -target amdgcn-amd-amdhsa -D__AMDGCN__ -nogpulib)
 endforeach()
+
+# Set the flags to build the device runtime from clang.
+set(clang_lib_flags -fopenmp -fopenmp-cuda-mode -foffload-lto -fvisibility=hidden -mllvm -openmp-opt-disable -nogpulib -nostdinc -DSHARED_SCRATCHPAD_SIZE=512)
+foreach(arch ${nvptx_sm_list})
+  set(clang_lib_flags ${clang_lib_flags} --offload-arch=sm_${arch})
+endforeach()
+foreach(arch ${amdgpu_mcpus})
+  set(clang_lib_flags ${clang_lib_flags} --offload-arch=${arch})
+endforeach()
+
+# Build the static library version of the device runtime.
+add_library(omptarget.devicertl STATIC)
+target_compile_features(omptarget.devicertl PRIVATE cxx_std_17)
+target_include_directories(omptarget.devicertl PRIVATE ${include_directory} ${devicertl_base_directory}/../include)
+target_compile_options(omptarget.devicertl PRIVATE ${clang_lib_flags})
+
+install(TARGETS omptarget.devicertl ARCHIVE DESTINATION ${OPENMP_INSTALL_LIBDIR})
+
+add_subdirectory(src)

openmp/libomptarget/DeviceRTL/src/CMakeLists.txt

+target_sources(omptarget.devicertl PRIVATE
+  Configuration.cpp
+  Debug.cpp
+  Kernel.cpp
+  Mapping.cpp
+  Misc.cpp
+  Parallelism.cpp
+  Reduction.cpp
+  State.cpp
+  Synchronization.cpp
+  Tasking.cpp
+  Utils.cpp
+  Workshare.cpp)