[Libomptarget] Make the plugins link as LLVM libraries

[jhuber6]

Summary

Previously we made libomptarget link as an LLVM library so we have
access to the LLVM core libraries. After the initial patch stuck we can
now apply the same changes to the plugins. This will allow us to use
LLVM in all of libomptarget when we have uses for them. In the future
this should allow us to remove the dependencies on libelf, libffi,
and dl.

[JonChesterfield]

LGTM, thanks! Hopefully CI will be easier this time around.

[tianshilei1992]

openmp/libomptarget/plugins/ve/CMakeLists.txt

43

why do we need NO_INSTALL_RPATH here but later set its property separately?

[jhuber6]

openmp/libomptarget/plugins/ve/CMakeLists.txt

43

I'm not exactly sure, that was a configuration the AMD people gave me that finally made it stop breaking on their CI system. I think we need to manually set it to the libomptarget install as well whereas that one would set it to the one that goes in /build/lib. Ideally in the future we'd just put everything in the same library instead of duplicating it, but getting these configurations right is pretty finicky.

[tianshilei1992]

openmp/libomptarget/plugins/ve/CMakeLists.txt

43

That doesn't look good. Is there any argument provided by add_llvm_library could do something similar, instead of having completely contrary setting?

[jhuber6]

openmp/libomptarget/plugins/ve/CMakeLists.txt

43

Not sure, I think the NO_INSTALL_RPATH only applies to the one installed under the /lib directory and not the one we manually install. it worked on my machine without the rpath / installation business but the only buildbot we have didn't like it. If I had access to the machine I could try to figure it out but I'm not really interested in pushing and reverting commits until it's happy.

[tianshilei1992]

openmp/libomptarget/plugins/ve/CMakeLists.txt

43

The script used by those systems are available right?

openmp/libomptarget/plugins/CMakeLists.txt

if(LIBOMPTARGET_DEP_LIBFFI_FOUND)
       libomptarget_say("Building ${tmachine_name} offloading plugin.")
 
       # Define macro to be used as prefix of the runtime messages for this target.
       add_definitions("-DTARGET_NAME=${tmachine_name}")
 
       # Define macro with the ELF ID for this target.
       add_definitions("-DTARGET_ELF_ID=${elf_machine_id}")
 
-      add_library("omptarget.rtl.${tmachine_libname}" SHARED
-        ${CMAKE_CURRENT_SOURCE_DIR}/../generic-elf-64bit/src/rtl.cpp)
+      add_llvm_library("omptarget.rtl.${tmachine_libname}"
+        SHARED
 
-      # Install plugin under the lib destination folder.
-      install(TARGETS "omptarget.rtl.${tmachine_libname}"
-        LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
+        ${CMAKE_CURRENT_SOURCE_DIR}/../generic-elf-64bit/src/rtl.cpp
 
-      target_include_directories( "omptarget.rtl.${tmachine_libname}" PRIVATE
+        ADDITIONAL_HEADER_DIRS
         ${LIBOMPTARGET_INCLUDE_DIR}
         ${LIBOMPTARGET_DEP_LIBFFI_INCLUDE_DIR}
-        ${LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIR})
+        ${LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIR}
 
-      target_link_libraries(
-        "omptarget.rtl.${tmachine_libname}"
+        LINK_LIBS 
+        PRIVATE
         elf_common
         ${LIBOMPTARGET_DEP_LIBFFI_LIBRARIES}
         ${LIBOMPTARGET_DEP_LIBELF_LIBRARIES}
         dl
         ${OPENMP_PTHREAD_LIB}
-        "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports")
+        "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports"
+
+        NO_INSTALL_RPATH
+      )
+
+      # Install plugin under the lib destination folder.
+      install(TARGETS "omptarget.rtl.${tmachine_libname}"
+        LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
+      set_target_properties("omptarget.rtl.${tmachine_libname}" PROPERTIES 
+        INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
+
+      target_include_directories( "omptarget.rtl.${tmachine_libname}" PRIVATE
+        ${LIBOMPTARGET_INCLUDE_DIR}
+        ${LIBOMPTARGET_DEP_LIBFFI_INCLUDE_DIR}
+        ${LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIR})
 
       list(APPEND LIBOMPTARGET_TESTED_PLUGINS
         "omptarget.rtl.${tmachine_libname}")
 
       # Report to the parent scope that we are building a plugin.
       set(LIBOMPTARGET_SYSTEM_TARGETS
         "${LIBOMPTARGET_SYSTEM_TARGETS} ${tmachine_triple} ${tmachine_triple}-oldDriver" PARENT_SCOPE)
       set(LIBOMPTARGET_SYSTEM_TARGETS

openmp/libomptarget/plugins/amdgpu/CMakeLists.txt

if (${hsa-runtime64_FOUND} AND NOT LIBOMPTARGET_FORCE_DLOPEN_LIBHSA)
   set(LIBOMPTARGET_DEP_LIBRARIES hsa-runtime64::hsa-runtime64)
 else()
   libomptarget_say("Building AMDGPU plugin for dlopened libhsa")
   include_directories(dynamic_hsa)
   set(LIBOMPTARGET_EXTRA_SOURCE dynamic_hsa/hsa.cpp)
   set(LIBOMPTARGET_DEP_LIBRARIES)
 endif()
 
-add_library(omptarget.rtl.amdgpu SHARED
-      impl/impl.cpp
-      impl/interop_hsa.cpp
-      impl/data.cpp
-      impl/get_elf_mach_gfx_name.cpp
-      impl/system.cpp
-      impl/msgpack.cpp
-      src/rtl.cpp
-      ${LIBOMPTARGET_EXTRA_SOURCE}
-      )
-add_dependencies(omptarget.rtl.amdgpu omptarget.devicertl.amdgpu)
-
-# Install plugin under the lib destination folder.
-# When we build for debug, OPENMP_LIBDIR_SUFFIX get set to -debug
-install(TARGETS omptarget.rtl.amdgpu LIBRARY DESTINATION "lib${OPENMP_LIBDIR_SUFFIX}")
-set_property(TARGET omptarget.rtl.amdgpu PROPERTY INSTALL_RPATH_USE_LINK_PATH ON)
-
 if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
   # On FreeBSD, the 'environ' symbol is undefined at link time, but resolved by
   # the dynamic linker at runtime. Therefore, allow the symbol to be undefined
   # when creating a shared library.
   set(LDFLAGS_UNDEFINED "-Wl,--allow-shlib-undefined")
 else()
   set(LDFLAGS_UNDEFINED "-Wl,-z,defs")
 endif()
 
-target_include_directories(
-  omptarget.rtl.amdgpu
-  PRIVATE
+add_llvm_library(omptarget.rtl.amdgpu SHARED
+  impl/impl.cpp
+  impl/interop_hsa.cpp
+  impl/data.cpp
+  impl/get_elf_mach_gfx_name.cpp
+  impl/system.cpp
+  impl/msgpack.cpp
+  src/rtl.cpp
+  ${LIBOMPTARGET_EXTRA_SOURCE}
+
+  ADDITIONAL_HEADER_DIRS
   ${LIBOMPTARGET_INCLUDE_DIR}
   ${CMAKE_CURRENT_SOURCE_DIR}/impl
-)
 
-target_link_libraries(
-  omptarget.rtl.amdgpu
+  LINK_LIBS 
   PRIVATE
   elf_common
   ${LIBOMPTARGET_DEP_LIBRARIES}
   ${CMAKE_DL_LIBS}
   ${LIBOMPTARGET_DEP_LIBELF_LIBRARIES}
   ${OPENMP_PTHREAD_LIB}
   "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports"
   ${LDFLAGS_UNDEFINED}
-  )
+
+  NO_INSTALL_RPATH
+add_dependencies(omptarget.rtl.amdgpu omptarget.devicertl.amdgpu)
+
+target_include_directories(
+  omptarget.rtl.amdgpu
+  PRIVATE
+  ${LIBOMPTARGET_INCLUDE_DIR}
+  ${CMAKE_CURRENT_SOURCE_DIR}/impl
+)
+
+
+# Install plugin under the lib destination folder.
+install(TARGETS omptarget.rtl.amdgpu LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
+set_target_properties(omptarget.rtl.amdgpu PROPERTIES 
+  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
 
 # in case of amdgcn, skip running tests if amdgpu-arch was not built or fails
 if (NOT TARGET amdgpu-arch)
   libomptarget_say("Not generating amdgcn test targets as amdgpu-arch is not found")
   return()
 endif()
 
 get_property(AMDGPU_ARCH_COMMAND TARGET amdgpu-arch PROPERTY LOCATION)

openmp/libomptarget/plugins/cuda/CMakeLists.txt

 set(LIBOMPTARGET_CAN_LINK_LIBCUDA FALSE)
 if (LIBOMPTARGET_DEP_CUDA_FOUND AND LIBOMPTARGET_DEP_CUDA_DRIVER_FOUND)
   set(LIBOMPTARGET_CAN_LINK_LIBCUDA TRUE)
 endif()
 
 if (LIBOMPTARGET_CAN_LINK_LIBCUDA AND NOT LIBOMPTARGET_FORCE_DLOPEN_LIBCUDA)
   libomptarget_say("Building CUDA plugin linked against libcuda")
   include_directories(${LIBOMPTARGET_DEP_CUDA_INCLUDE_DIRS})
-  add_library(omptarget.rtl.cuda SHARED src/rtl.cpp)
-  set (LIBOMPTARGET_DEP_LIBRARIES ${LIBOMPTARGET_DEP_CUDA_DRIVER_LIBRARIES})
+  add_llvm_library(omptarget.rtl.cuda SHARED
+
+    src/rtl.cpp
+
+    ADDITIONAL_HEADER_DIRS
+    ${LIBOMPTARGET_INCLUDE_DIR}
+    ${LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIRS}
+
+    LINK_LIBS
+    PRIVATE
+    elf_common
+    MemoryManager
+    ${LIBOMPTARGET_DEP_CUDA_DRIVER_LIBRARIES}
+    ${LIBOMPTARGET_DEP_LIBELF_LIBRARIES}
+    ${OPENMP_PTHREAD_LIB}
+    "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports"
+    "-Wl,-z,defs"
+
+    NO_INSTALL_RPATH
+  )
 else()
   libomptarget_say("Building CUDA plugin for dlopened libcuda")
   include_directories(dynamic_cuda)
-  add_library(omptarget.rtl.cuda SHARED src/rtl.cpp dynamic_cuda/cuda.cpp)
-  set (LIBOMPTARGET_DEP_LIBRARIES ${CMAKE_DL_LIBS})
-endif()
-add_dependencies(omptarget.rtl.cuda omptarget.devicertl.nvptx)
+  add_llvm_library(omptarget.rtl.cuda 
+    SHARED
 
-# Install plugin under the lib destination folder.
-install(TARGETS omptarget.rtl.cuda LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
+    src/rtl.cpp
+    dynamic_cuda/cuda.cpp
 
-target_include_directories(omptarget.rtl.cuda PRIVATE
+    ADDITIONAL_HEADER_DIRS
     ${LIBOMPTARGET_INCLUDE_DIR}
     ${LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIRS}
-)
 
-target_link_libraries(omptarget.rtl.cuda
+    LINK_LIBS
+    PRIVATE
     elf_common
     MemoryManager
-  ${LIBOMPTARGET_DEP_LIBRARIES}
+    ${CMAKE_DL_LIBS}
     ${LIBOMPTARGET_DEP_LIBELF_LIBRARIES}
     ${OPENMP_PTHREAD_LIB}
     "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports"
-  "-Wl,-z,defs")
+    "-Wl,-z,defs"
+
+    NO_INSTALL_RPATH
+  )
+endif()
+add_dependencies(omptarget.rtl.cuda omptarget.devicertl.nvptx)
+
+# Install plugin under the lib destination folder.
+install(TARGETS omptarget.rtl.cuda LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
+set_target_properties(omptarget.rtl.cuda PROPERTIES 
+  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
+
+target_include_directories(omptarget.rtl.cuda PRIVATE
+  ${LIBOMPTARGET_INCLUDE_DIR}
+  ${LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIRS}
+)
 
 # Report to the parent scope that we are building a plugin for CUDA.
 # This controls whether tests are run for the nvptx offloading target
 # Run them if libcuda is available, or if the user explicitly asked for dlopen
 # Otherwise this plugin is being built speculatively and there may be no cuda available
 if (LIBOMPTARGET_CAN_LINK_LIBCUDA OR LIBOMPTARGET_FORCE_DLOPEN_LIBCUDA)
   libomptarget_say("Enable tests using CUDA plugin")
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS} nvptx64-nvidia-cuda nvptx64-nvidia-cuda-oldDriver" PARENT_SCOPE)
   set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS} nvptx64-nvidia-cuda nvptx64-nvidia-cuda-LTO" PARENT_SCOPE)
   list(APPEND LIBOMPTARGET_TESTED_PLUGINS "omptarget.rtl.cuda")
   set(LIBOMPTARGET_TESTED_PLUGINS "${LIBOMPTARGET_TESTED_PLUGINS}" PARENT_SCOPE)
 else()
   libomptarget_say("Disabling tests using CUDA plugin as cuda may not be available")
 endif()

openmp/libomptarget/plugins/ve/CMakeLists.txt

if(${LIBOMPTARGET_DEP_VEO_FOUND})
   set(elf_machine_id 251)
 
   # Define macro to be used as prefix of the runtime messages for this target.
   add_definitions("-DTARGET_NAME=${tmachine_name}")
 
   # Define macro with the ELF ID for this target.
   add_definitions("-DTARGET_ELF_ID=${elf_machine_id}")
 
-  add_library("omptarget.rtl.${tmachine_libname}" SHARED
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/rtl.cpp)
+  add_llvm_library("omptarget.rtl.${tmachine_libname}" 
+    SHARED
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/rtl.cpp
+
+    ADDITIONAL_HEADER_DIRS
+    ${LIBOMPTARGET_INCLUDE_DIR}
+    ${LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIR}
+    ${LIBOMPTARGET_DEP_VEO_INCLUDE_DIR}
+
+    LINK_LIBS
+    PRIVATE
+    elf_common
+    ${LIBOMPTARGET_DEP_LIBFFI_LIBRARIES}
+    ${LIBOMPTARGET_DEP_LIBELF_LIBRARIES}
+    ${additional_libs}
+    "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports -Wl,-z,defs"
+
+    NO_INSTALL_RPATH
+  )
 
   # Install plugin under the lib destination folder.
-  install(TARGETS "omptarget.rtl.${tmachine_libname}"
-    LIBRARY DESTINATION lib${OPENMP_LIBDIR_SUFFIX})
+  install(TARGETS  "omptarget.rtl.${tmachine_libname}" LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
+  set_target_properties("omptarget.rtl.${tmachine_libname}" PROPERTIES 
+    INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
 
   target_include_directories("omptarget.rtl.${tmachine_libname}" PRIVATE
     ${LIBOMPTARGET_INCLUDE_DIR}
     ${LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIR}
     ${LIBOMPTARGET_DEP_VEO_INCLUDE_DIR})
 
   target_link_libraries(
     "omptarget.rtl.${tmachine_libname}"

openmp/libomptarget/src/CMakeLists.txt

add_llvm_library(omptarget
 
   ADDITIONAL_HEADER_DIRS
   ${LIBOMPTARGET_INCLUDE_DIR}
 
   LINK_COMPONENTS
   Support
   Object
 
-  LINK_LIBS
+  LINK_LIBS 
+  PRIVATE
   ${CMAKE_DL_LIBS}
   "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports"
   NO_INSTALL_RPATH
 )
 target_include_directories(omptarget PRIVATE ${LIBOMPTARGET_INCLUDE_DIR})
 
 # libomptarget.so needs to be aware of where the plugins live as they
 # are now separated in the build directory.
 set_target_properties(omptarget PROPERTIES INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
 install(TARGETS omptarget LIBRARY COMPONENT omptarget DESTINATION "${OPENMP_INSTALL_LIBDIR}")