[libc] Add startup code implementation for GPU targets

[jhuber6]

Summary

This patch introduces startup code for executing main on a device
compiled for the GPU. We will primarily use this to run standalone
integration tests on the GPU. The actual execution of this routine will
need to be provided by a loader utility to bootstrap execution on the
GPU.

[jhuber6]

Remove unrelated change.

I had a hard time testing these changes. For reasons unknown to me, this does
not seem to actually build the sources. Thus, the install does not have a file
to install. I copied the source mostly from the existing support for linux but
it seems like it doesn't create a standard object library that is linked in.
Rather we extract the object manually. This is also a bit problematic as we will
need to pass --target= to the link job to tell it to use the appropriate
toolchain for the linking job.

[sivachandra]

Reading the patch doesn't show me any obvious problems. Can you tell me how to reproduce what you are seeing?

[jhuber6]

In D143212#4104005, @sivachandra wrote:

Reading the patch doesn't show me any obvious problems. Can you tell me how to reproduce what you are seeing?

The only issue right now is that I don't see the crt1.o being built anywhere. The install line then fails to find it at install time. I wasn't sure how this dependency is supposed to be handled. It might be difficult for you to reproduce right now because I'm assuming you don't have a GPU on your system. I'll need to add those lines that let you force a build of one GPU architecture for testing.

[sivachandra]

libc/startup/gpu/CMakeLists.txt

93

The problem is here: replace ${LIBC_COMPONENT} with libc.

[jhuber6]

Updating the component. This works now with ninja install-libc. it would be nice if we could add a public link interface to use --target= so that anything linking against the startup code will use the correct device linker.

libc/startup/gpu/CMakeLists.txt

+function(add_startup_object name)
+  cmake_parse_arguments(
+    "ADD_STARTUP_OBJECT"
+    "ALIAS" # Option argument
+    "SRC"   # Single value arguments
+    "DEPENDS;COMPILE_OPTIONS" # Multi value arguments
+    ${ARGN}
+  )
+
+  get_fq_target_name(${name} fq_target_name)
+  get_fq_deps_list(fq_deps_list ${ADD_STARTUP_OBJECT_DEPENDS})
+  if(ADD_STARTUP_OBJECT_ALIAS)
+    list(LENGTH ADD_STARTUP_OBJECT_DEPENDS deps_size)
+    if(NOT (${deps_size} EQUAL "1"))
+      message(FATAL_ERROR "A startup object alias should have exactly one dependency.")
+    endif()
+    list(GET ADD_STARTUP_OBJECT_DEPENDS 0 dep)
+    get_fq_dep_name(fq_dep_name ${dep})
+
+    add_custom_target(${fq_target_name})
+    add_dependencies(${fq_target_name} ${fq_dep_name})
+    get_target_property(startup_object ${fq_dep_name} STARTUP_OBJECT)
+    set_target_properties(
+      ${fq_target_name}
+      PROPERTIES
+        "TARGET_TYPE" "${OBJECT_LIBRARY_TARGET_TYPE}"
+        "STARTUP_OBJECT" "${startup_object}"
+        "OBJECT_FILES" ""
+        "DEPS" "${fq_dep_name}"
+    )
+    return()
+  endif()
+
+  add_object_library(
+    ${name}.__objects__
+    SRCS ${ADD_STARTUP_OBJECT_SRC}
+    DEPENDS ${ADD_STARTUP_OBJECT_DEPENDS}
+    COMPILE_OPTIONS ${ADD_STARTUP_OBJECT_COMPILE_OPTIONS}
+  )
+  set(objfile ${LIBC_BUILD_DIR}/lib/${name}.o)
+  add_custom_command(
+    OUTPUT ${objfile}
+    COMMAND cp $<TARGET_OBJECTS:${fq_target_name}.__objects__> ${objfile}
+    DEPENDS $<TARGET_OBJECTS:${fq_target_name}.__objects__>
+  )
+  add_custom_target(
+    ${fq_target_name}
+    DEPENDS ${objfile}
+  )
+  set_target_properties(
+    ${fq_target_name}
+    PROPERTIES
+      "TARGET_TYPE" "${OBJECT_LIBRARY_TARGET_TYPE}"
+      "STARTUP_OBJECT" "${objfile}"
+      "OBJECT_FILES" ""
+      "DEPS" "${fq_target_name}.__objects__"
+  )
+endfunction()
+
+if(LIBC_GPU_TARGET_ARCHITECTURE_IS_AMDGPU)
+  add_subdirectory(amdgpu)
+
+  add_startup_object(
+    crt1
+    ALIAS
+    DEPENDS
+      amdgpu.crt1
+  )
+elseif(LIBC_GPU_TARGET_ARCHITECTURE_IS_NVPTX)
+  add_subdirectory(nvptx)
+
+  add_startup_object(
+    crt1
+    ALIAS
+    DEPENDS
+      .nvptx.crt1
+  )
+else()
+  # Skip building the startup code if there are no supported GPUs.
+  message(STATUS "Skipping startup for gpu target, no GPUs were detected")
+  return()
+endif()
+
+add_custom_target(libc-startup)
+set(startup_components crt1)
+foreach(target IN LISTS startup_components)
+  set(fq_target_name libc.startup.gpu.${target})
+  add_dependencies(libc-startup ${fq_target_name})
+  get_target_property(startup_object ${fq_target_name} STARTUP_OBJECT)
+  install(FILES ${startup_object}
+          DESTINATION ${CMAKE_INSTALL_LIBDIR}
+          COMPONENT libc)
+endforeach()

libc/startup/gpu/amdgpu/CMakeLists.txt

+add_startup_object(
+  crt1
+  SRC
+    start.cpp
+  COMPILE_OPTIONS
+    -ffreestanding # To avoid compiler warnings about calling the main function.
+    -fno-builtin
+    -nogpulib # Do not include any GPU vendor libraries.
+    -nostdinc
+    -mcpu=${LIBC_GPU_TARGET_ARCHITECTURE}
+    -emit-llvm # AMDGPU's intermediate object file format is bitcode.
+    --target=${LIBC_GPU_TARGET_TRIPLE}
+)

libc/startup/gpu/amdgpu/start.cpp

+//===-- Implementation of crt for amdgpu ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+extern "C" int main(int argc, char **argv);
+
+extern "C" [[gnu::visibility("protected"), clang::amdgpu_kernel]] void
+_start(int argc, char **argv, int *ret) {
+  __atomic_fetch_or(ret, main(argc, argv), __ATOMIC_RELAXED);
+}

libc/startup/gpu/nvptx/CMakeLists.txt

+add_startup_object(
+  crt1
+  SRC
+    start.cpp
+  COMPILE_OPTIONS
+    -ffreestanding # To avoid compiler warnings about calling the main function.
+    -fno-builtin
+    -nogpulib # Do not include any GPU vendor libraries.
+    -nostdinc
+    -x cuda # Use the CUDA toolchain to emit the `_start` kernel.
+    --offload-device-only
+    --offload-arch=${LIBC_GPU_TARGET_ARCHITECTURE}
+)

libc/startup/gpu/nvptx/start.cpp

+//===-- Implementation of crt for amdgpu ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+extern "C" __attribute__((device)) int main(int argc, char **argv);
+
+// TODO: We shouldn't need to use the CUDA language to emit a kernel for NVPTX.
+extern "C" [[gnu::visibility("protected")]] __attribute__((global)) void
+_start(int argc, char **argv, int *ret) {
+  __atomic_fetch_or(ret, main(argc, argv), __ATOMIC_RELAXED);
+}