[flang][runtime] Support in-tree device build of Flang runtime.

[vzakhari]

Summary

I changed the set of files that are built for experimental CUDA/OMP
builds, i.e. the files with enabled device support are built
as such and the rest of the files are built just for the host target.
With this change we can build Flang runtime library that is fully functional
on the host target, so in-tree targets like check-flang become operational.

[PeteSteinfeld]

I tried building using the instructions in GettingStarted.md, but I was not successful. Here's the invocation of cmake I used:

cmake \
  -G Ninja \
  ../llvm \
  -DCMAKE_BUILD_TYPE=Release \
  -DFLANG_EXPERIMENTAL_CUDA_RUNTIME=on \
  -DCMAKE_CUDA_ARCHITECTURES=80 \
  -DLLVM_LIT_ARGS=-v \
  -DFLANG_ENABLE_WERROR=On \
  -DCMAKE_CXX_STANDARD=17 \
  -DCMAKE_C_COMPILER=clang \
  -DCMAKE_CXX_COMPILER=clang++ \
  -DCMAKE_CUDA_COMPILER=clang \
  -DCMAKE_CUDA_HOST_COMPILER=clang++ \
  -DLLVM_ENABLE_PROJECTS="clang;mlir;flang;compiler-rt;openmp" \
  -DLLVM_BUILD_TOOLS=On \
  -DLLVM_INSTALL_UTILS=On \
  -DLLVM_TARGETS_TO_BUILD=host \
  -DLLVM_ENABLE_ASSERTIONS=ON \
  -DCMAKE_CXX_LINK_FLAGS="-Wl,-rpath,$LD_LIBRARY_PATH" \
  -DCMAKE_INSTALL_PREFIX=$INSTALLDIR

But my build failed. Here's an excerpt from the log file of the build:

-- Performing Test CXX_SUPPORTS_NO_CTAD_MAYBE_UNSUPPORTED_FLAG - Success
-- Performing Test FLANG_RUNTIME_HAS_FNO_LTO_FLAG
-- Performing Test FLANG_RUNTIME_HAS_FNO_LTO_FLAG - Success
-- The CUDA compiler identification is unknown
-- Detecting CUDA compiler ABI info
CMake Error: Error required internal CMake variable not set, cmake may not be built correctly.
Missing variable is:
_CMAKE_CUDA_WHOLE_FLAG
CMake Error at /usr/local/share/cmake-3.26/Modules/CMakeDetermineCompilerABI.cmake:57 (try_compile):
  Failed to generate test project build system.
Call Stack (most recent call first):
  /usr/local/share/cmake-3.26/Modules/CMakeTestCUDACompiler.cmake:19 (CMAKE_DETERMINE_COMPILER_ABI)
  /local/home/psteinfeld/main/029/flang/runtime/CMakeLists.txt:163 (enable_language)


-- Configuring incomplete, errors occurred!
Exit with error 0: cmake

[vzakhari]

Hi @PeteSteinfeld, thank you for trying it! I guess the clang compiler that you have in your path does not support NVPTX target. For my local builds I used specially built clang. You may try building with nvcc.

[PeteSteinfeld]

Thanks, @vzakhari.

I tried building with nvcc, and I ran into problems because it doesn't support the -Wall option. I did succeed using gcc version 9.3.0. I also tried building with a version of the clang compiler that I built based on llvm release 16.0. I could not get this to work, even with using nvcc as my CMAKE_CUDA_COMPILER. You might want to mention that the version of clang that you use to configure the build must support the NVPTX target.

[PeteSteinfeld]

@vzakhari, here's more information on what I did.

Here's the script that I used that was successful. Note that the versio of gcc that's on my path is 9.3.0:

cmake \
  -G Ninja \
  ../llvm \
  -DCMAKE_BUILD_TYPE=Release \
  -DFLANG_EXPERIMENTAL_CUDA_RUNTIME=ON \
  -DCMAKE_CUDA_ARCHITECTURES=80 \
  -DCMAKE_C_COMPILER=gcc \
  -DCMAKE_CXX_COMPILER=g++ \
  -DCMAKE_CUDA_COMPILER=nvcc \
  -DCMAKE_CUDA_HOST_COMPILER=g++\
  -DLLVM_LIT_ARGS=-v \
  -DFLANG_ENABLE_WERROR=On \
  -DCMAKE_CXX_STANDARD=17 \
  -DLLVM_ENABLE_PROJECTS="clang;mlir;flang;compiler-rt;openmp" \
  -DLLVM_BUILD_TOOLS=On \
  -DLLVM_INSTALL_UTILS=On \
  -DLLVM_TARGETS_TO_BUILD=host \
  -DLLVM_ENABLE_ASSERTIONS=ON \
  -DCMAKE_CXX_LINK_FLAGS="-Wl,-rpath,$LD_LIBRARY_PATH" \
  -DCMAKE_INSTALL_PREFIX=$INSTALLDIR

ninja

This build succeeded. There's a section of the log file where I can see that checking is done for the CUDA compiler:

  ...
-- Performing Test FLANG_RUNTIME_HAS_FNO_LTO_FLAG
-- Performing Test FLANG_RUNTIME_HAS_FNO_LTO_FLAG - Success
-- The CUDA compiler identification is NVIDIA 12.2.91
-- Detecting CUDA compiler ABI info
-- Detecting CUDA compiler ABI info - done
-- Check for working CUDA compiler: /proj/nv/Linux_x86_64/dev/compilers/bin/nvcc - skipped
-- Detecting CUDA compile features
-- Detecting CUDA compile features - done
-- Could NOT find Doxygen (missing: DOXYGEN_EXECUTABLE) 
  ...

I then changed the mentions of gcc and g++ to point to versions of clang that I personally built from release 16. This resulted in the following script:

cmake \
  -G Ninja \
  ../llvm \
  -DCMAKE_BUILD_TYPE=Release \
  -DFLANG_EXPERIMENTAL_CUDA_RUNTIME=ON \
  -DCMAKE_CUDA_ARCHITECTURES=80 \
  -DCMAKE_C_COMPILER=/local/home/psteinfeld/main/rel16/build/bin/clang \
  -DCMAKE_CXX_COMPILER=/local/home/psteinfeld/main/rel16/build/bin/clang++ \
  -DCMAKE_CUDA_COMPILER=nvcc \
  -DCMAKE_CUDA_HOST_COMPILER=/local/home/psteinfeld/main/rel16/build/bin/clang++
 \
  -DLLVM_LIT_ARGS=-v \
  -DFLANG_ENABLE_WERROR=On \
  -DCMAKE_CXX_STANDARD=17 \
  -DLLVM_ENABLE_PROJECTS="clang;mlir;flang;compiler-rt;openmp" \
  -DLLVM_BUILD_TOOLS=On \
  -DLLVM_INSTALL_UTILS=On \
  -DLLVM_TARGETS_TO_BUILD=host \
  -DLLVM_ENABLE_ASSERTIONS=ON \
  -DCMAKE_CXX_LINK_FLAGS="-Wl,-rpath,$LD_LIBRARY_PATH" \
  -DCMAKE_INSTALL_PREFIX=$INSTALLDIR

ninja

This build failed. Here's an excerpt from the log file:

  ...
-- Performing Test CXX_SUPPORTS_NO_CTAD_MAYBE_UNSUPPORTED_FLAG
-- Performing Test CXX_SUPPORTS_NO_CTAD_MAYBE_UNSUPPORTED_FLAG - Success
-- Performing Test FLANG_RUNTIME_HAS_FNO_LTO_FLAG
-- Performing Test FLANG_RUNTIME_HAS_FNO_LTO_FLAG - Success
CMake Error at /usr/local/share/cmake-3.26/Modules/CMakeDetermineCompilerId.cmake:751 (message):
  Compiling the CUDA compiler identification source file
  "CMakeCUDACompilerId.cu" failed.

  Compiler: /proj/nv/Linux_x86_64/dev/compilers/bin/nvcc

  Build flags:

  Id flags:
  --keep;--keep-dir;tmp;-ccbin=/local/home/psteinfeld/main/rel16/build/bin/clang++
  -v



  The output was:

  1

  #$ _NVVM_BRANCH_=nvvm

  #$ _SPACE_=

  #$ _CUDART_=cudart

  #$ _HERE_=/proj/nv/Linux_x86_64/232943-dev/cuda/12.2/bin

  #$ _THERE_=/proj/nv/Linux_x86_64/232943-dev/cuda/12.2/bin

  #$ _TARGET_SIZE_=

  #$ _TARGET_DIR_=

  #$ _TARGET_DIR_=targets/x86_64-linux

  #$ TOP=/proj/nv/Linux_x86_64/232943-dev/cuda/12.2/bin/..

  #$
  NVVMIR_LIBRARY_DIR=/proj/nv/Linux_x86_64/232943-dev/cuda/12.2/bin/../nvvm/libdevice


  #$
  LD_LIBRARY_PATH=/proj/nv/Linux_x86_64/232943-dev/cuda/12.2/bin/../lib:/home/sw/thirdparty/gcc/gcc-9.3.0/linux86-64/lib64:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/host/linux64/lib:/opt/intel/oneapi/compiler/2023.1.0/linux/lib:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/x64:/opt/intel/oneapi/compiler/2023.1.0/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/tbb/2021.9.0/lib/intel64/gcc4.8:/opt/binutils/latest/lib64:/opt/binutils/latest/lib:/home/sw/envmod/modules/linux86-64/latest/lib


  #$
  PATH=/proj/nv/Linux_x86_64/232943-dev/cuda/12.2/bin/../nvvm/bin:/proj/nv/Linux_x86_64/232943-dev/cuda/12.2/bin:/home/sw/thirdparty/gcc/gcc-9.3.0/linux86-64/bin:/opt/binutils/latest/bin:/opt/intel/oneapi/compiler/2023.1.0/linux/bin/intel64:/opt/intel/oneapi/compiler/2023.1.0/linux/bin:/opt/intel/oneapi/compiler/2023.1.0/linux/lib/oclfpga/bin:/home/sw/envmod/modules/linux86-64/latest/bin:/home/sw/envmod/modules/linux86-64/4.2.5/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:.:./bin:/local/home/psteinfeld/main/rel13/install/bin:/home/psteinfeld/bin:/home/psteinfeld/bin/arcanist/arcanist/bin:/home/sw/thirdparty/llvm/linux86-64/7.1.0-f18/bin:/proj/nv/Linux_x86_64/dev/compilers/bin:/opt/nag-7.0.7048/bin:/proj/ta/bin:/opt/perforce/bin:/home/sw/thirdparty/ccollab-client:/home/psteinfeld/work/pgi/dev/bin:.:./bin:/local/home/psteinfeld/main/rel13/install/bin:/home/psteinfeld/bin:/home/psteinfeld/bin/arcanist/arcanist/bin:/home/sw/thirdparty/llvm/linux86-64/7.1.0-f18/bin:/proj/nv/Linux_x86_64/dev/compilers/bin:/opt/nag-7.0.7048/bin:/proj/ta/bin:/opt/perforce/bin:/home/sw/thirdparty/ccollab-client:/home/psteinfeld/work/pgi/dev/bin:.:./bin:/local/home/psteinfeld/main/rel13/install/bin:/home/psteinfeld/bin:/home/psteinfeld/bin/arcanist/arcanist/bin:/home/sw/thirdparty/llvm/linux86-64/7.1.0-f18/bin:/proj/nv/Linux_x86_64/dev/compilers/bin:/opt/nag-7.0.7048/bin:/proj/ta/bin:/opt/perforce/bin:/home/sw/thirdparty/ccollab-client:/home/psteinfeld/work/pgi/dev/bin:.:./bin:/local/home/psteinfeld/main/rel13/install/bin:/home/psteinfeld/bin:/home/psteinfeld/bin/arcanist/arcanist/bin:/home/sw/thirdparty/llvm/linux86-64/7.1.0-f18/bin:/proj/nv/Linux_x86_64/dev/compilers/bin:/opt/nag-7.0.7048/bin:/proj/ta/bin:/opt/perforce/bin:/home/sw/thirdparty/ccollab-client:/home/psteinfeld/work/pgi/dev/bin


  #$
  INCLUDES="-I/proj/nv/Linux_x86_64/232943-dev/cuda/12.2/bin/../targets/x86_64-linux/include"


  #$ LIBRARIES=
  "-L/proj/nv/Linux_x86_64/232943-dev/cuda/12.2/bin/../targets/x86_64-linux/lib/stubs"
  "-L/proj/nv/Linux_x86_64/232943-dev/cuda/12.2/bin/../targets/x86_64-linux/lib"


  #$ CUDAFE_FLAGS=

  #$ PTXAS_FLAGS=

  #$ rm tmp/a_dlink.reg.c

  #$ "/local/home/psteinfeld/main/rel16/build/bin"/clang++
  -D__CUDA_ARCH_LIST__=520 -E -x c++ -D__CUDACC__ -D__NVCC__
  -I"/proj/nv/Linux_x86_64/232943-dev/cuda/12.2/include"
  -I"/proj/nv/Linux_x86_64/232943-dev/math_libs/12.2/include"
  -I"/proj/nv/Linux_x86_64/232943-dev/comm_libs/12.2/nccl/include"
  -I"/proj/nv/Linux_x86_64/232943-dev/comm_libs/12.2/nvshmem/include"
  "-I/proj/nv/Linux_x86_64/232943-dev/cuda/12.2/bin/../targets/x86_64-linux/include"
  -D__CUDACC_VER_MAJOR__=12 -D__CUDACC_VER_MINOR__=2
  -D__CUDACC_VER_BUILD__=91 -D__CUDA_API_VER_MAJOR__=12
  -D__CUDA_API_VER_MINOR__=2 -D__NVCC_DIAG_PRAGMA_SUPPORT__=1 -include
  "cuda_runtime.h" -m64 "CMakeCUDACompilerId.cu" -o
  "tmp/CMakeCUDACompilerId.cpp4.ii"

  In file included from <built-in>:1:

  In file included from
  /proj/nv/Linux_x86_64/232943-dev/cuda/12.2/include/cuda_runtime.h:82:


  /proj/nv/Linux_x86_64/232943-dev/cuda/12.2/include/crt/host_config.h:144:2:
  error: -- unsupported clang version! clang version must be less than 16 and
  greater than 3.2 .  The nvcc flag '-allow-unsupported-compiler' can be used
  to override this version check; however, using an unsupported host compiler
  may cause compilation failure or incorrect run time execution.  Use at your
  own risk.

  #error -- unsupported clang version! clang version must be less than 16 and
  greater than 3.2 .  The nvcc flag '-allow-unsupported-compiler' can be used
  to override this version check; however, using an unsupported host compiler
  may cause compilation failure or incorrect run time execution.  Use at your
  own risk.

   ^

  1 error generated.

  # --error 0x1 --





Call Stack (most recent call first):
  /usr/local/share/cmake-3.26/Modules/CMakeDetermineCompilerId.cmake:8 (CMAKE_DETERMINE_COMPILER_ID_BUILD)
  /usr/local/share/cmake-3.26/Modules/CMakeDetermineCompilerId.cmake:53 (__determine_compiler_id_test)
  /usr/local/share/cmake-3.26/Modules/CMakeDetermineCUDACompiler.cmake:307 (CMAKE_DETERMINE_COMPILER_ID)
  /local/home/psteinfeld/main/029/flang/runtime/CMakeLists.txt:163 (enable_language)


-- Configuring incomplete, errors occurred!
Exit with error 0: cmake

flang/docs/GettingStarted.md

217

I think this should be rm -rf ...

[vzakhari]

Thank you for the comments, Pete!

I added two notes to highlight potential tool compatibility issues:

• NVCC only supports certain versions of clang as the host compiler.
• Clang only supports certain versions of CUDA toolkit.

flang/docs/GettingStarted.md

 
 #### Building out-of-tree
 
 ##### CUDA build
 Clang with NVPTX backend and NVCC compilers are supported.
 
 ```bash
 cd llvm-project/flang
-mkdir -rf build_flang_runtime
+rm -rf build_flang_runtime
 mkdir build_flang_runtime
 cd build_flang_runtime
 
 cmake \
   -DFLANG_EXPERIMENTAL_CUDA_RUNTIME=ON \
   -DCMAKE_CUDA_ARCHITECTURES=80 \
   -DCMAKE_C_COMPILER=clang \
   -DCMAKE_CXX_COMPILER=clang++ \
   -DCMAKE_CUDA_COMPILER=clang \
+  -DCMAKE_CUDA_HOST_COMPILER=clang++ \
   ../runtime/
 make -j FortranRuntime
 ```
 
+Note that the used version of `clang` must [support](https://releases.llvm.org/16.0.0/tools/clang/docs/ReleaseNotes.html#cuda-support)
+CUDA toolkit version installed on the build machine.  If there are multiple
+CUDA toolkit installations, please use `-DCUDAToolkit_ROOT=/some/path`
+to specify the compatible version.
+
 ```bash
 cd llvm-project/flang
-mkdir -rf build_flang_runtime
+rm -rf build_flang_runtime
 mkdir build_flang_runtime
 cd build_flang_runtime
 
 cmake \
   -DFLANG_EXPERIMENTAL_CUDA_RUNTIME=ON \
   -DCMAKE_CUDA_ARCHITECTURES=80 \
   -DCMAKE_C_COMPILER=clang \
   -DCMAKE_CXX_COMPILER=clang++ \
   -DCMAKE_CUDA_COMPILER=nvcc \
+  -DCMAKE_CUDA_HOST_COMPILER=clang++ \
   ../runtime/
 make -j FortranRuntime
 ```
 
+Note that `nvcc` might limit support to certain
+[versions](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#host-compiler-support-policy) of `CMAKE_CUDA_HOST_COMPILER`,
+so please use compatible versions.
+
 The result of the build is a "fat" library with the host and device
 code.  Note that the packaging of the libraries is different
 between [Clang](https://clang.llvm.org/docs/OffloadingDesign.html#linking-target-device-code) and NVCC, so the library must be linked using
 compatible compiler drivers.
 
+### Bulding in-tree
+One may build Flang runtime library along with building Flang itself
+by providing these additional CMake variables on top of the Flang in-tree
+build config:
+
+For example:
+```bash
+  -DFLANG_EXPERIMENTAL_CUDA_RUNTIME=ON \
+  -DCMAKE_CUDA_ARCHITECTURES=80 \
+  -DCMAKE_C_COMPILER=clang \
+  -DCMAKE_CXX_COMPILER=clang++ \
+  -DCMAKE_CUDA_COMPILER=clang \
+  -DCMAKE_CUDA_HOST_COMPILER=clang++ \
+```
+
+Or:
+```bash
+  -DFLANG_EXPERIMENTAL_CUDA_RUNTIME=ON \
+  -DCMAKE_CUDA_ARCHITECTURES=80 \
+  -DCMAKE_C_COMPILER=gcc \
+  -DCMAKE_CXX_COMPILER=g++ \
+  -DCMAKE_CUDA_COMPILER=nvcc \
+  -DCMAKE_CUDA_HOST_COMPILER=g++ \
+```
+
+Normal `make -j check-flang` will work with such CMake configuration.
+
 ##### OpenMP target offload build
 Only Clang compiler is currently supported.
 
 ```
 cd llvm-project/flang
-mkdir -rf build_flang_runtime
+rm -rf build_flang_runtime
 mkdir build_flang_runtime
 cd build_flang_runtime
 
 cmake \
   -DFLANG_EXPERIMENTAL_OMP_OFFLOAD_BUILD="host_device" \
   -DCMAKE_C_COMPILER=clang \
   -DCMAKE_CXX_COMPILER=clang++ \
   -DFLANG_OMP_DEVICE_ARCHITECTURES="all" \
   ../runtime/
 make -j FortranRuntime
 ```
 
 The result of the build is a "device-only" library, i.e. the host
 part of the library is just a container for the device code.
 The resulting library may be linked to user programs using
 Clang-like device linking pipeline.
 
+The same set of CMake variables works for Flang in-tree build.
+
 ## Supported C++ compilers
 
 Flang is written in C++17.
 
 The code has been compiled and tested with GCC versions from 7.2.0 to 9.3.0.
 
 The code has been compiled and tested with clang version 7.0, 8.0, 9.0 and 10.0
 using either GNU's libstdc++ or LLVM's libc++.

flang/runtime/CMakeLists.txt

option(FLANG_EXPERIMENTAL_CUDA_RUNTIME
   )
 
 # List of files that are buildable for all devices.
 set(supported_files
   transformational.cpp
   )
 
 if (FLANG_EXPERIMENTAL_CUDA_RUNTIME)
-  enable_language(CUDA)
+  if (BUILD_SHARED_LIBS)
+    message(FATAL_ERROR
+      "BUILD_SHARED_LIBS is not supported for CUDA build of Fortran runtime"
+      )
+  endif()
 
-  # Add the unsupported files to LLVM_OPTIONAL_SOURCES.
-  set(todo_files ${sources})
-  list(REMOVE_ITEM todo_files ${supported_files})
-  list(APPEND LLVM_OPTIONAL_SOURCES ${todo_files})
+  enable_language(CUDA)
 
   # TODO: figure out how to make target property CUDA_SEPARABLE_COMPILATION
   # work, and avoid setting CMAKE_CUDA_SEPARABLE_COMPILATION.
   set(CMAKE_CUDA_SEPARABLE_COMPILATION ON)
 
-  # Treat all sources as CUDA files.
-  set(sources ${supported_files})
-  set_source_files_properties(${sources} PROPERTIES LANGUAGE CUDA)
+  # Treat all supported sources as CUDA files.
+  set_source_files_properties(${supported_files} PROPERTIES LANGUAGE CUDA)
+  set(CUDA_COMPILE_OPTIONS)
   if ("${CMAKE_CUDA_COMPILER_ID}" MATCHES "Clang")
     # Allow varargs.
-    add_compile_options(-Xclang -fcuda-allow-variadic-functions)
+    set(CUDA_COMPILE_OPTIONS
+      -Xclang -fcuda-allow-variadic-functions
+      )
   endif()
+  set_source_files_properties(${supported_files} PROPERTIES COMPILE_OPTIONS
+    "${CUDA_COMPILE_OPTIONS}"
+    )
 endif()
 
 set(FLANG_EXPERIMENTAL_OMP_OFFLOAD_BUILD "off" CACHE STRING
   "Compile Fortran runtime as OpenMP target offload sources (experimental). Valid options are 'off', 'host_device', 'nohost'")
 
 set(FLANG_OMP_DEVICE_ARCHITECTURES "all" CACHE STRING
   "List of OpenMP device architectures to be used to compile the Fortran runtime (e.g. 'gfx1103;sm_90')")
 
if (NOT FLANG_EXPERIMENTAL_OMP_OFFLOAD_BUILD STREQUAL "off")
   # In general, the in-tree built Clang should be used for 'nohost' build.
   # Note that 'nohost' build does not produce the host version of Flang
   # runtime library, so there will be two separate distributable objects.
   # 'nohost' build is a TODO.
 
   if (NOT FLANG_EXPERIMENTAL_OMP_OFFLOAD_BUILD STREQUAL "host_device")
     message(FATAL_ERROR "Unsupported OpenMP offload build of Flang runtime")
   endif()
-
-  # Add the unsupported files to LLVM_OPTIONAL_SOURCES.
-  set(todo_files ${sources})
-  list(REMOVE_ITEM todo_files ${supported_files})
-  list(APPEND LLVM_OPTIONAL_SOURCES ${todo_files})
-  set(sources ${supported_files})
+  if (BUILD_SHARED_LIBS)
+    message(FATAL_ERROR
+      "BUILD_SHARED_LIBS is not supported for OpenMP offload build of Fortran runtime"
+      )
+  endif()
 
   if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" AND
       "${CMAKE_C_COMPILER_ID}" MATCHES "Clang")
 
     set(all_amdgpu_architectures
       "gfx700;gfx701;gfx801;gfx803;gfx900;gfx902;gfx906"
       "gfx908;gfx90a;gfx90c;gfx940;gfx1010;gfx1030"
       "gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036"
if (FLANG_OMP_DEVICE_ARCHITECTURES STREQUAL "all")
       set(FLANG_OMP_DEVICE_ARCHITECTURES ${all_gpu_architectures})
     endif()
     list(REMOVE_DUPLICATES FLANG_OMP_DEVICE_ARCHITECTURES)
 
     string(REPLACE ";" "," compile_for_architectures
       "${FLANG_OMP_DEVICE_ARCHITECTURES}"
       )
 
-    add_compile_options(-fopenmp -fvisibility=hidden -fopenmp-cuda-mode)
-    add_compile_options(--offload-arch=${compile_for_architectures})
+    set(OMP_COMPILE_OPTIONS
+      -fopenmp
+      -fvisibility=hidden
+      -fopenmp-cuda-mode
+      --offload-arch=${compile_for_architectures}
       # Force LTO for the device part.
-    add_compile_options(-foffload-lto)
+      -foffload-lto
+      )
+    set_source_files_properties(${supported_files} PROPERTIES COMPILE_OPTIONS
+      "${OMP_COMPILE_OPTIONS}"
+      )
+
+    # Enable "declare target" in the source code.
+    set_source_files_properties(${supported_files}
+      PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
+      )
   else()
     message(FATAL_ERROR
       "Flang runtime build is not supported for these compilers:\n"
       "CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}\n"
       "CMAKE_C_COMPILER_ID: ${CMAKE_C_COMPILER_ID}")
   endif()
-
-  # Enable "declare target" in the source code.
-  add_compile_definitions(OMP_OFFLOAD_BUILD)
 endif()
 
 add_flang_library(FortranRuntime
   ${sources}
   LINK_LIBS
   FortranDecimal
 
   INSTALL_WITH_TOOLCHAIN
 )

flang/unittests/CMakeLists.txt

+if (FLANG_EXPERIMENTAL_CUDA_RUNTIME)
+  # If Fortran runtime is built as CUDA library, the linking
+  # of targets that link FortranRuntime must be done
+  # with CUDA_RESOLVE_DEVICE_SYMBOLS.
+  # CUDA language must be enabled for CUDA_RESOLVE_DEVICE_SYMBOLS
+  # to take effect.
+  enable_language(CUDA)
+endif()
+
 add_custom_target(FlangUnitTests)
 set_target_properties(FlangUnitTests PROPERTIES FOLDER "Flang Unit Tests")
 
+function(add_flang_unittest_offload_properties target)
+  # Set CUDA_RESOLVE_DEVICE_SYMBOLS.
+  if (FLANG_EXPERIMENTAL_CUDA_RUNTIME)
+    set_target_properties(${target}
+      PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON
+      )
+  endif()
+  # Enable OpenMP offload during linking. We may need to replace
+  # LINK_OPTIONS with COMPILE_OPTIONS when there are OpenMP offload
+  # unittests.
+  #
+  # FIXME: replace 'native' in --offload-arch option with the list
+  #        of targets that Fortran Runtime was built for.
+  #        Common code must be moved from flang/runtime/CMakeLists.txt.
+  if (NOT FLANG_EXPERIMENTAL_OMP_OFFLOAD_BUILD STREQUAL "off")
+    set_target_properties(${target}
+      PROPERTIES LINK_OPTIONS
+      "-fopenmp;--offload-arch=native"
+      )
+  endif()
+endfunction()
+
 function(add_flang_unittest test_dirname)
   add_unittest(FlangUnitTests ${test_dirname} ${ARGN})
+  add_flang_unittest_offload_properties(${test_dirname})
 endfunction()
 
 if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
   add_compile_options("-Wno-suggest-override")
 endif()
 
 function(add_flang_nongtest_unittest test_name)
   cmake_parse_arguments(ARG
function(add_flang_nongtest_unittest test_name)
   else()
     llvm_map_components_to_libnames(llvm_libs Support)
   endif()
   target_link_libraries(${test_name}${suffix} ${llvm_libs} ${ARG_UNPARSED_ARGUMENTS})
 
   if(NOT ARG_SLOW_TEST)
     add_dependencies(FlangUnitTests ${test_name}${suffix})
   endif()
+
+  add_flang_unittest_offload_properties(${test_name}${suffix})
 endfunction()
 
 add_subdirectory(Optimizer)
 add_subdirectory(Common)
 add_subdirectory(Decimal)
 add_subdirectory(Evaluate)
 add_subdirectory(Runtime)
 add_subdirectory(Frontend)