[Libomptarget] Update handling of architectures for DeviceRTL

[jhuber6]

Summary

The support for enabling and disabling certain architectures for the
OpenMP device RTL is different between AMD and Nvidia. This patch
updates the logic to make it common. This supports the auto format
more generally via the nvptx-arch and amdgpu-arch options. (These
are not availible at CMake time without a runtimes build, or another
install somewhere. But that only prevents users from using auto).

[jdoerfert]

openmp/libomptarget/DeviceRTL/CMakeLists.txt

64

you are dropping 89 and 90?

[jhuber6]

openmp/libomptarget/DeviceRTL/CMakeLists.txt

64

Didn't notice those were missing, went off an old list.

[jhuber6]

adding sm_89 and sm_90

[jhuber6]

GPU -> DEVICE

[ye-luo]

openmp/libomptarget/DeviceRTL/CMakeLists.txt

68

Prefer default all instead of the full list.
When doing an incremental build and having new arch being listed. all continue to build everything but cached list doesn't

74

Are both tool always built together?

[jhuber6]

openmp/libomptarget/DeviceRTL/CMakeLists.txt

68

Good point

74

Yes, they are always built by clang.

[jhuber6]

Default value to all

[ye-luo]

openmp/libomptarget/DeviceRTL/CMakeLists.txt

74

Did you mean they are always built by the clang target outside openmp project?
If I don't build NVPTX or AMDGPU backend, are they still being built?
Can it be better if we do this.
if nvptx-arch exists, add its list to the LIBOMPTARGET_DEVICE_ARCHITECTURES.
if it doesn't exists, print a warning but don't error out.
do the same for amdgpu-arch.
if LIBOMPTARGET_DEVICE_ARCHITECTURES is still empty, error out.

openmp/libomptarget/cmake/Modules/LibomptargetGetDependencies.cmake

121–122

There are a few variables related to ARCH. Could you rename this to LIBOMPTARGET_NVPTX_ARCH_DETECTED_LIST.
Also the AMDGPU one.

[jhuber6]

openmp/libomptarget/DeviceRTL/CMakeLists.txt

74

nvptx-arch and amdgpu-arch are clang tools. They are built unconditionally whenever you build clang, they will be available here during a runtimes build, or if the user has the binary somewhere else. This is only necessary for auto support, which I'm pretty sure already used this transitively after I made a previous change deprecating the use of findCUDA in favor of nvptx-arch.

This logic only runs on auto which I feel is fair to require some additional handling to support. Or are you suggesting we make the default only supporting the user's architecture?

openmp/libomptarget/cmake/Modules/LibomptargetGetDependencies.cmake

121–122

Sure

[jhuber6]

Adding detected to variable name

[ye-luo]

openmp/libomptarget/DeviceRTL/CMakeLists.txt

74

What I described above is intended for inside the logic block "auto". I don't support having auto as the default.

[jhuber6]

openmp/libomptarget/DeviceRTL/CMakeLists.txt

74

I think this is fine since there's no situation where the user would have one but not the other unless they manually deleted it.

[ye-luo]

openmp/libomptarget/DeviceRTL/CMakeLists.txt

74

Then what we have now is sufficient. Could you extend the error message saying current auto detection only support NV and AMD.

openmp/libomptarget/DeviceRTL/CMakeLists.txt

else()
   libomptarget_say("Not building DeviceRTL. No appropriate clang found")
   return()
 endif()
 
 set(devicertl_base_directory ${CMAKE_CURRENT_SOURCE_DIR})
 set(include_directory ${devicertl_base_directory}/include)
 set(source_directory ${devicertl_base_directory}/src)
 
-set(all_capabilities 35 37 50 52 53 60 61 62 70 72 75 80 86 89 90)
-
-set(LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES ${all_capabilities} CACHE STRING
-  "List of CUDA Compute Capabilities to be used to compile the NVPTX DeviceRTL.")
-string(TOLOWER ${LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES} LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES)
-
-if (LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES STREQUAL "all")
-  set(nvptx_sm_list ${all_capabilities})
-elseif(LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES STREQUAL "auto")
-  if (NOT LIBOMPTARGET_DEP_CUDA_FOUND)
-    libomptarget_error_say("[NVPTX] Cannot auto detect compute capability as CUDA not found.")
-  endif()
-  set(nvptx_sm_list ${LIBOMPTARGET_DEP_CUDA_ARCH})
-else()
-  string(REPLACE "," ";" nvptx_sm_list "${LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES}")
+set(all_amdgpu_architectures "gfx700;gfx701;gfx801;gfx803;gfx900;gfx902;gfx906"
+                             "gfx908;gfx90a;gfx90c;gfx940;gfx1010;gfx1030"
+                             "gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036"
+                             "gfx1100;gfx1101;gfx1102;gfx1103")
+set(all_nvptx_architectures "sm_35;sm_37;sm_50;sm_52;sm_53;sm_60;sm_61;sm_62"
+                            "sm_70;sm_72;sm_75;sm_80;sm_86;sm_89;sm_90")
+set(all_gpu_architectures
+    "${all_amdgpu_architectures};${all_nvptx_architectures}")
+
+set(LIBOMPTARGET_DEVICE_ARCHITECTURES "all" CACHE STRING
+    "List of device architectures to be used to compile the OpenMP DeviceRTL.")
+
+if(LIBOMPTARGET_DEVICE_ARCHITECTURES STREQUAL "all")
+  set(LIBOMPTARGET_DEVICE_ARCHITECTURES ${all_gpu_architectures})
+elseif(LIBOMPTARGET_DEVICE_ARCHITECTURES STREQUAL "auto")
+  if(NOT LIBOMPTARGET_NVPTX_ARCH AND NOT LIBOMPTARGET_AMDGPU_ARCH)
+    libomptarget_error_say(
+      "Could not find 'amdgpu-arch' and 'nvptx-arch' tools required for 'auto'")
+  elseif(NOT LIBOMPTARGET_FOUND_NVIDIA_GPU AND NOT LIBOMPTARGET_FOUND_AMDGPU_GPU)
+    libomptarget_error_say("No AMD or Nvidia found on the system when using 'auto'")
   endif()
-
-# Check all SM values
-foreach(sm ${nvptx_sm_list})
-  if (NOT ${sm} IN_LIST all_capabilities)
-    libomptarget_warning_say("[NVPTX] Compute capability ${sm} is not supported. Make sure clang can work with it.")
-  endif()
-endforeach()
-
-set(amdgpu_mcpus gfx700 gfx701 gfx801 gfx803 gfx900 gfx902 gfx906 gfx908 gfx90a gfx90c gfx940 gfx1010 gfx1030 gfx1031 gfx1032 gfx1033 gfx1034 gfx1035 gfx1036 gfx1100 gfx1101 gfx1102 gfx1103)
-if (DEFINED LIBOMPTARGET_AMDGCN_GFXLIST)
-  set(amdgpu_mcpus ${LIBOMPTARGET_AMDGCN_GFXLIST})
+  set(LIBOMPTARGET_DEVICE_ARCHITECTURES
+      "${LIBOMPTARGET_NVPTX_DETECTED_ARCH_LIST};${LIBOMPTARGET_AMDGPU_DETECTED_ARCH_LIST}")
 endif()
 
 set(include_files
   ${include_directory}/Configuration.h
   ${include_directory}/Debug.h
   ${include_directory}/Interface.h
   ${include_directory}/LibC.h
   ${include_directory}/Mapping.h
add_custom_command(OUTPUT ${output_name}
       DEPENDS clang
       APPEND)
   endif()
 
   set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${output_name})
   set_property(TARGET omptarget.devicertl.all_objs APPEND PROPERTY IMPORTED_OBJECTS ${output_name})
 endfunction()
 
-# Generate a Bitcode library for all the compute capabilities the user requested
+# Generate a Bitcode library for all the gpu architectures the user requested.
 add_custom_target(omptarget.devicertl.nvptx)
-foreach(sm ${nvptx_sm_list})
-  compileDeviceRTLLibrary(sm_${sm} nvptx nvptx64-nvidia-cuda -fopenmp-targets=nvptx64-nvidia-cuda --cuda-feature=+ptx61)
-endforeach()
-
 add_custom_target(omptarget.devicertl.amdgpu)
-foreach(mcpu ${amdgpu_mcpus})
-  compileDeviceRTLLibrary(${mcpu} amdgpu amdgcn-amd-amdhsa -fopenmp-targets=amdgcn-amd-amdhsa)
+foreach(gpu_arch ${LIBOMPTARGET_DEVICE_ARCHITECTURES})
+  if("${gpu_arch}" IN_LIST all_amdgpu_architectures)
+    compileDeviceRTLLibrary(${gpu_arch} amdgpu amdgcn-amd-amdhsa -fopenmp-targets=amdgcn-amd-amdhsa)
+  elseif("${gpu_arch}" IN_LIST all_nvptx_architectures)
+    compileDeviceRTLLibrary(${gpu_arch} nvptx nvptx64-nvidia-cuda -fopenmp-targets=nvptx64-nvidia-cuda --cuda-feature=+ptx61)
+  else()
+    libomptarget_error_say("Unknown GPU architecture '${gpu_arch}'")
+  endif()
 endforeach()
 
 # Archive all the object files generated above into a static library
 add_library(omptarget.devicertl STATIC)
 set_target_properties(omptarget.devicertl PROPERTIES LINKER_LANGUAGE CXX)
 target_link_libraries(omptarget.devicertl PRIVATE omptarget.devicertl.all_objs)
 
 install(TARGETS omptarget.devicertl ARCHIVE DESTINATION ${OPENMP_INSTALL_LIBDIR})

openmp/libomptarget/cmake/Modules/LibomptargetGetDependencies.cmake

 ################################################################################
 set(LIBOMPTARGET_DEP_CUDA_ARCH "sm_35")
 
 find_program(LIBOMPTARGET_NVPTX_ARCH NAMES nvptx-arch PATHS ${LLVM_BINARY_DIR}/bin)
 if(LIBOMPTARGET_NVPTX_ARCH)
   execute_process(COMMAND ${LIBOMPTARGET_NVPTX_ARCH}
                   OUTPUT_VARIABLE LIBOMPTARGET_NVPTX_ARCH_OUTPUT
                   OUTPUT_STRIP_TRAILING_WHITESPACE)
-  string(FIND "${LIBOMPTARGET_NVPTX_ARCH_OUTPUT}" "\n" first_arch_string)
-  string(SUBSTRING "${LIBOMPTARGET_NVPTX_ARCH_OUTPUT}" 0 ${first_arch_string}
-         arch_string)
-  if(arch_string)
+  string(REPLACE "\n" ";" nvptx_arch_list "${LIBOMPTARGET_NVPTX_ARCH_OUTPUT}")
+  if(nvptx_arch_list)
     set(LIBOMPTARGET_FOUND_NVIDIA_GPU TRUE)
-    set(LIBOMPTARGET_DEP_CUDA_ARCH "${arch_string}")
+    set(LIBOMPTARGET_NVPTX_DETECTED_ARCH_LIST "${nvptx_arch_list}")
+    list(GET nvptx_arch_list 0 LIBOMPTARGET_DEP_CUDA_ARCH)
   endif()
 endif()
 
 
 ################################################################################
 # Looking for AMD GPUs...
 ################################################################################
 
 find_program(LIBOMPTARGET_AMDGPU_ARCH NAMES amdgpu-arch PATHS ${LLVM_BINARY_DIR}/bin)
 if(LIBOMPTARGET_AMDGPU_ARCH)
   execute_process(COMMAND ${LIBOMPTARGET_AMDGPU_ARCH}
                   OUTPUT_VARIABLE LIBOMPTARGET_AMDGPU_ARCH_OUTPUT
                   OUTPUT_STRIP_TRAILING_WHITESPACE)
-  string(FIND "${LIBOMPTARGET_AMDGPU_ARCH_OUTPUT}" "\n" first_arch_string)
-  string(SUBSTRING "${LIBOMPTARGET_AMDGPU_ARCH_OUTPUT}" 0 ${first_arch_string}
-         arch_string)
-  if(arch_string)
+  string(REPLACE "\n" ";" amdgpu_arch_list "${LIBOMPTARGET_AMDGPU_ARCH_OUTPUT}")
+  if(amdgpu_arch_list)
     set(LIBOMPTARGET_FOUND_AMDGPU_GPU TRUE)
-    set(LIBOMPTARGET_DEP_AMDGPU_ARCH "${arch_string}")
+    set(LIBOMPTARGET_AMDGPU_DETECTED_ARCH_LIST "${amdgpu_arch_list}")
   endif()
 endif()
 
 
 ################################################################################
 # Looking for VEO...
 ################################################################################