[libc] Extend the linux thread implementation for riscv64.

[sivachandra]

Summary

Also, added riscv64 startup code for static linking which is used
by the integration tests. Functions from the C standard threads
library have been enabled.

[mikhail.ramalho]

LGTM.

Unrelated topic: I started to get some rounding mode errors this morning, are you seeing them too @sivachandra or is it just on my side?

[ RUN      ] LlvmLibcRoundingModeTest.SetAndGet
/home/mgadelha/llvm-project/libc/test/src/fenv/rounding_mode_test.cpp:30: FAILURE
      Expected: s
      Which is: -1
To be equal to: 0
      Which is: 0
/home/mgadelha/llvm-project/libc/test/src/fenv/rounding_mode_test.cpp:32: FAILURE
      Expected: rm
      Which is: 2
To be equal to: FE_UPWARD
      Which is: 8
/home/mgadelha/llvm-project/libc/test/src/fenv/rounding_mode_test.cpp:40: FAILURE
      Expected: s
      Which is: -1
To be equal to: 0
      Which is: 0
/home/mgadelha/llvm-project/libc/test/src/fenv/rounding_mode_test.cpp:42: FAILURE
      Expected: rm
      Which is: 1
To be equal to: FE_TOWARDZERO
      Which is: 4
[  FAILED  ] LlvmLibcRoundingModeTest.SetAndGet
Ran 1 tests.  PASS: 0  FAIL: 1

/home/mgadelha/llvm-project/libc/test/src/fenv/getenv_and_setenv_test.cpp:71: FAILURE
      Expected: rm
      Which is: 0
To be equal to: FE_TONEAREST
      Which is: 2
[  FAILED  ] LlvmLibcFenvTest.Set_FE_DFL_ENV
Ran 2 tests.  PASS: 1  FAIL: 1

[sivachandra]

In D145670#4181239, @mikhail.ramalho wrote:

Unrelated topic: I started to get some rounding mode errors this morning, are you seeing them too @sivachandra or is it just on my side?

I am seeing a clean run in the full build as well as overlay build. Also, I will review your setjmp patch today. Thanks for your reviews and contributions. If everything goes well, I should be able to fire up a riscv builder connected to the LLVM buildbot by end of day today.

libc/config/linux/app.h

struct TLSImage {
   // PT_TLS header.
   uintptr_t init_size;
 
   // The alignment of the TLS layout. It assumed that the alignment
   // value is a power of 2.
   uintptr_t align;
 };
 
-#if defined(LIBC_TARGET_ARCH_IS_X86_64) || defined(LIBC_TARGET_ARCH_IS_AARCH64)
+#if defined(LIBC_TARGET_ARCH_IS_X86_64) ||                                     \
+    defined(LIBC_TARGET_ARCH_IS_AARCH64) ||                                    \
+    defined(LIBC_TARGET_ARCH_IS_RISCV64)
 // At the language level, argc is an int. But we use uint64_t as the x86_64
 // ABI specifies it as an 8 byte value. Likewise, in the ARM64 ABI, arguments
 // are usually passed in registers.  x0 is a doubleword register, so this is
 // 64 bit for aarch64 as well.
 typedef uint64_t ArgcType;
 
 // At the language level, argv is a char** value. However, we use uint64_t as
 // ABIs specify the argv vector be an |argc| long array of 8-byte values.

libc/config/linux/riscv64/entrypoints.txt

list(APPEND TARGET_LIBC_ENTRYPOINTS
     libc.src.stdio.printf
     libc.src.stdio.putc
     libc.src.stdio.putchar
     libc.src.stdio.puts
     libc.src.stdio.stderr
     libc.src.stdio.stdin
     libc.src.stdio.stdout
 
+    # stdlib.h entrypoints
+    libc.src.stdlib._Exit
+    libc.src.stdlib.atexit
+    libc.src.stdlib.exit
+    libc.src.stdlib.getenv
+
+    # threads.h entrypoints
+    libc.src.threads.call_once
+    libc.src.threads.cnd_broadcast
+    libc.src.threads.cnd_destroy
+    libc.src.threads.cnd_init
+    libc.src.threads.cnd_signal
+    libc.src.threads.cnd_wait
+    libc.src.threads.mtx_destroy
+    libc.src.threads.mtx_init
+    libc.src.threads.mtx_lock
+    libc.src.threads.mtx_unlock
+    libc.src.threads.thrd_create
+    libc.src.threads.thrd_current
+    libc.src.threads.thrd_detach
+    libc.src.threads.thrd_equal
+    libc.src.threads.thrd_exit
+    libc.src.threads.thrd_join
+    libc.src.threads.tss_create
+    libc.src.threads.tss_delete
+    libc.src.threads.tss_get
+    libc.src.threads.tss_set
+
     # time.h entrypoints
     libc.src.time.asctime
     libc.src.time.asctime_r
     libc.src.time.clock_gettime
     libc.src.time.clock
     libc.src.time.difftime
     libc.src.time.gettimeofday
     libc.src.time.gmtime
     libc.src.time.gmtime_r
     libc.src.time.mktime
     libc.src.time.nanosleep
     libc.src.time.time
+
+    # unistd.h entrypoints
+    libc.src.unistd.environ
   )
 endif()
 
 set(TARGET_LLVMLIBC_ENTRYPOINTS
   ${TARGET_LIBC_ENTRYPOINTS}
   ${TARGET_LIBM_ENTRYPOINTS}
 )

libc/src/__support/threads/linux/thread.cpp

static constexpr unsigned CLONE_SYSCALL_FLAGS =
     | CLONE_THREAD  // Same thread group as the parent.
     | CLONE_SYSVSEM // Share a single list of System V semaphore adjustment
                     // values
     | CLONE_PARENT_SETTID  // Set child thread ID in |ptid| of the parent.
     | CLONE_CHILD_CLEARTID // Let the kernel clear the tid address
                            // wake the joining thread.
     | CLONE_SETTLS;        // Setup the thread pointer of the new thread.
 
+#ifdef LIBC_TARGET_ARCH_IS_AARCH64
+#define CLONE_RESULT_REGISTER "x0"
+#elif defined(LIBC_TARGET_ARCH_IS_RISCV64)
+#define CLONE_RESULT_REGISTER "t0"
+#elif defined(LIBC_TARGET_ARCH_IS_X86_64)
+#define CLONE_RESULT_REGISTER "rax"
+#else
+#error "CLONE_RESULT_REGISTER not defined for your target architecture"
+#endif
+
 LIBC_INLINE ErrorOr<void *> alloc_stack(size_t size) {
   long mmap_result =
       __llvm_libc::syscall_impl(MMAP_SYSCALL_NUMBER,
                                 0, // No special address
                                 size,
                                 PROT_READ | PROT_WRITE, // Read and write stack
                                 MAP_ANONYMOUS | MAP_PRIVATE, // Process private
                                 -1, // Not backed by any file

 // should be compiled with -fno-omit-frame-pointer.
 #ifdef LIBC_TARGET_ARCH_IS_X86_64
   return reinterpret_cast<uintptr_t>(__builtin_frame_address(0))
          // The x86_64 call instruction pushes resume address on to the stack.
          // Next, The x86_64 SysV ABI requires that the frame pointer be pushed
          // on to the stack. So, we have to step past two 64-bit values to get
          // to the start args.
          + sizeof(uintptr_t) * 2;
-#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
+#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) ||                                  \
+    defined(LIBC_TARGET_ARCH_IS_RISCV64)
   // The frame pointer after cloning the new thread in the Thread::run method
   // is set to the stack pointer where start args are stored. So, we fetch
   // from there.
   return reinterpret_cast<uintptr_t>(__builtin_frame_address(1));
 #endif
 }
 
 __attribute__((noinline)) static void start_thread() {
auto clear_tid = reinterpret_cast<cpp::Atomic<FutexWordType> *>(
       adjusted_stack + sizeof(StartArgs) + sizeof(ThreadAttributes));
   clear_tid->val = CLEAR_TID_VALUE;
   attrib->platform_data = clear_tid;
 
   // The clone syscall takes arguments in an architecture specific order.
   // Also, we want the result of the syscall to be in a register as the child
   // thread gets a completely different stack after it is created. The stack
   // variables from this function will not be availalbe to the child thread.
-#ifdef LIBC_TARGET_ARCH_IS_X86_64
-  long register clone_result asm("rax");
+#if defined(LIBC_TARGET_ARCH_IS_X86_64)
+  long register clone_result asm(CLONE_RESULT_REGISTER);
   clone_result = __llvm_libc::syscall_impl(
       SYS_clone, CLONE_SYSCALL_FLAGS, adjusted_stack,
       &attrib->tid,    // The address where the child tid is written
       &clear_tid->val, // The futex where the child thread status is signalled
       tls.tp           // The thread pointer value for the new thread.
   );
-#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
-  long register clone_result asm("x0");
+#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) ||                                  \
+    defined(LIBC_TARGET_ARCH_IS_RISCV64)
+  long register clone_result asm(CLONE_RESULT_REGISTER);
   clone_result = __llvm_libc::syscall_impl(
       SYS_clone, CLONE_SYSCALL_FLAGS, adjusted_stack,
       &attrib->tid,   // The address where the child tid is written
       tls.tp,         // The thread pointer value for the new thread.
       &clear_tid->val // The futex where the child thread status is signalled
   );
 #else
 #error "Unsupported architecture for the clone syscall."
 #endif
 
   if (clone_result == 0) {
 #ifdef LIBC_TARGET_ARCH_IS_AARCH64
     // We set the frame pointer to be the same as the "sp" so that start args
     // can be sniffed out from start_thread.
 #ifdef __clang__
     // GCC does not currently implement __arm_wsr64/__arm_rsr64.
     __arm_wsr64("x29", __arm_rsr64("sp"));
 #else
     asm volatile("mov x29, sp");
 #endif
+#elif defined(LIBC_TARGET_ARCH_IS_RISCV64)
+    asm volatile("mv fp, sp");
 #endif
     start_thread();
   } else if (clone_result < 0) {
     cleanup_thread_resources(attrib);
     return -clone_result;
   }
 
   return 0;

libc/src/__support/threads/thread.h

union ThreadReturnValue {
   void *posix_retval;
   int stdc_retval;
   constexpr ThreadReturnValue() : posix_retval(nullptr) {}
   constexpr ThreadReturnValue(int r) : stdc_retval(r) {}
   constexpr ThreadReturnValue(void *r) : posix_retval(r) {}
 };
 
 #if (defined(LIBC_TARGET_ARCH_IS_AARCH64) ||                                   \
-     defined(LIBC_TARGET_ARCH_IS_X86_64))
+     defined(LIBC_TARGET_ARCH_IS_X86_64) ||                                    \
+     defined(LIBC_TARGET_ARCH_IS_RISCV64))
 constexpr unsigned int STACK_ALIGNMENT = 16;
 #endif
 // TODO: Provide stack alignment requirements for other architectures.
 
 enum class DetachState : uint32_t {
   JOINABLE = 0x11,
   EXITING = 0x22,
   DETACHED = 0x33

libc/startup/linux/riscv64/CMakeLists.txt

+add_startup_object(
+  crt1
+  SRC
+    start.cpp
+  DEPENDS
+    libc.config.linux.app_h
+    libc.include.sys_mman
+    libc.include.sys_syscall
+    libc.src.__support.OSUtil.osutil
+    libc.src.stdlib.exit
+    libc.src.stdlib.atexit
+    libc.src.string.memory_utils.memcpy_implementation
+  COMPILE_OPTIONS
+    -fno-omit-frame-pointer
+    -ffreestanding # To avoid compiler warnings about calling the main function.
+)

libc/startup/linux/riscv64/start.cpp

+//===-- Implementation of crt for riscv64 ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "config/linux/app.h"
+#include "src/__support/OSUtil/syscall.h"
+#include "src/__support/threads/thread.h"
+#include "src/stdlib/atexit.h"
+#include "src/stdlib/exit.h"
+#include "src/string/memory_utils/memcpy_implementations.h"
+
+#include <linux/auxvec.h>
+#include <linux/elf.h>
+#include <stdint.h>
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+extern "C" int main(int, char **, char **);
+// The BFD linker requires a reference to __dso_handle to trigger creating
+// a symbol for it when -nostdlib is used..
+extern "C" void *__dso_handle = nullptr;
+
+namespace __llvm_libc {
+
+#ifdef SYS_mmap2
+static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2;
+#elif SYS_mmap
+static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap;
+#else
+#error "Target platform does not have SYS_mmap or SYS_mmap2 defined"
+#endif
+
+AppProperties app;
+
+static ThreadAttributes main_thread_attrib;
+
+void init_tls(TLSDescriptor &tls_descriptor) {
+  if (app.tls.size == 0) {
+    tls_descriptor.size = 0;
+    tls_descriptor.tp = 0;
+    return;
+  }
+
+  // riscv64 follows the variant 1 TLS layout:
+  const uintptr_t size_of_pointers = 2 * sizeof(uintptr_t);
+  uintptr_t padding = 0;
+  const uintptr_t ALIGNMENT_MASK = app.tls.align - 1;
+  uintptr_t diff = size_of_pointers & ALIGNMENT_MASK;
+  if (diff != 0)
+    padding += (ALIGNMENT_MASK - diff) + 1;
+
+  uintptr_t alloc_size = size_of_pointers + padding + app.tls.size;
+
+  // We cannot call the mmap function here as the functions set errno on
+  // failure. Since errno is implemented via a thread local variable, we cannot
+  // use errno before TLS is setup.
+  long mmap_ret_val = __llvm_libc::syscall_impl(
+      MMAP_SYSCALL_NUMBER, nullptr, alloc_size, PROT_READ | PROT_WRITE,
+      MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  // We cannot check the return value with MAP_FAILED as that is the return
+  // of the mmap function and not the mmap syscall.
+  if (mmap_ret_val < 0 && static_cast<uintptr_t>(mmap_ret_val) > -app.pageSize)
+    __llvm_libc::syscall_impl(SYS_exit, 1);
+  uintptr_t thread_ptr = uintptr_t(reinterpret_cast<uintptr_t *>(mmap_ret_val));
+  uintptr_t tls_addr = thread_ptr + size_of_pointers + padding;
+  __llvm_libc::inline_memcpy(reinterpret_cast<char *>(tls_addr),
+                             reinterpret_cast<const char *>(app.tls.address),
+                             app.tls.init_size);
+  tls_descriptor.size = alloc_size;
+  tls_descriptor.addr = thread_ptr;
+  tls_descriptor.tp = tls_addr;
+}
+
+void cleanup_tls(uintptr_t addr, uintptr_t size) {
+  if (size == 0)
+    return;
+  __llvm_libc::syscall_impl(SYS_munmap, addr, size);
+}
+
+static void set_thread_ptr(uintptr_t val) {
+  LIBC_INLINE_ASM("ld tp, %0\n\t" : : "m"(val));
+}
+
+using InitCallback = void(int, char **, char **);
+using FiniCallback = void(void);
+
+extern "C" {
+// These arrays are present in the .init_array and .fini_array sections.
+// The symbols are inserted by linker when it sees references to them.
+extern uintptr_t __preinit_array_start[];
+extern uintptr_t __preinit_array_end[];
+extern uintptr_t __init_array_start[];
+extern uintptr_t __init_array_end[];
+extern uintptr_t __fini_array_start[];
+extern uintptr_t __fini_array_end[];
+}
+
+static void call_init_array_callbacks(int argc, char **argv, char **env) {
+  size_t preinit_array_size = __preinit_array_end - __preinit_array_start;
+  for (size_t i = 0; i < preinit_array_size; ++i)
+    reinterpret_cast<InitCallback *>(__preinit_array_start[i])(argc, argv, env);
+  size_t init_array_size = __init_array_end - __init_array_start;
+  for (size_t i = 0; i < init_array_size; ++i)
+    reinterpret_cast<InitCallback *>(__init_array_start[i])(argc, argv, env);
+}
+
+static void call_fini_array_callbacks() {
+  size_t fini_array_size = __fini_array_end - __fini_array_start;
+  for (size_t i = fini_array_size; i > 0; --i)
+    reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])();
+}
+
+} // namespace __llvm_libc
+
+using __llvm_libc::app;
+
+// TODO: Would be nice to use the aux entry structure from elf.h when available.
+struct AuxEntry {
+  uint64_t type;
+  uint64_t value;
+};
+
+__attribute__((noinline)) static void do_start() {
+  auto tid = __llvm_libc::syscall_impl(SYS_gettid);
+  if (tid <= 0)
+    __llvm_libc::syscall_impl(SYS_exit, 1);
+  __llvm_libc::main_thread_attrib.tid = tid;
+
+  // After the argv array, is a 8-byte long NULL value before the array of env
+  // values. The end of the env values is marked by another 8-byte long NULL
+  // value. We step over it (the "+ 1" below) to get to the env values.
+  uint64_t *env_ptr = app.args->argv + app.args->argc + 1;
+  uint64_t *env_end_marker = env_ptr;
+  app.envPtr = env_ptr;
+  while (*env_end_marker)
+    ++env_end_marker;
+
+  // Initialize the POSIX global declared in unistd.h
+  environ = reinterpret_cast<char **>(env_ptr);
+
+  // After the env array, is the aux-vector. The end of the aux-vector is
+  // denoted by an AT_NULL entry.
+  Elf64_Phdr *programHdrTable = nullptr;
+  uintptr_t programHdrCount;
+  for (AuxEntry *aux_entry = reinterpret_cast<AuxEntry *>(env_end_marker + 1);
+       aux_entry->type != AT_NULL; ++aux_entry) {
+    switch (aux_entry->type) {
+    case AT_PHDR:
+      programHdrTable = reinterpret_cast<Elf64_Phdr *>(aux_entry->value);
+      break;
+    case AT_PHNUM:
+      programHdrCount = aux_entry->value;
+      break;
+    case AT_PAGESZ:
+      app.pageSize = aux_entry->value;
+      break;
+    default:
+      break; // TODO: Read other useful entries from the aux vector.
+    }
+  }
+
+  app.tls.size = 0;
+  for (uintptr_t i = 0; i < programHdrCount; ++i) {
+    Elf64_Phdr *phdr = programHdrTable + i;
+    if (phdr->p_type != PT_TLS)
+      continue;
+    // TODO: p_vaddr value has to be adjusted for static-pie executables.
+    app.tls.address = phdr->p_vaddr;
+    app.tls.size = phdr->p_memsz;
+    app.tls.init_size = phdr->p_filesz;
+    app.tls.align = phdr->p_align;
+  }
+
+  __llvm_libc::TLSDescriptor tls;
+  __llvm_libc::init_tls(tls);
+  if (tls.size != 0)
+    __llvm_libc::set_thread_ptr(tls.tp);
+
+  __llvm_libc::self.attrib = &__llvm_libc::main_thread_attrib;
+  __llvm_libc::main_thread_attrib.atexit_callback_mgr =
+      __llvm_libc::internal::get_thread_atexit_callback_mgr();
+
+  // We want the fini array callbacks to be run after other atexit
+  // callbacks are run. So, we register them before running the init
+  // array callbacks as they can potentially register their own atexit
+  // callbacks.
+  __llvm_libc::atexit(&__llvm_libc::call_fini_array_callbacks);
+
+  __llvm_libc::call_init_array_callbacks(
+      app.args->argc, reinterpret_cast<char **>(app.args->argv),
+      reinterpret_cast<char **>(env_ptr));
+
+  int retval = main(app.args->argc, reinterpret_cast<char **>(app.args->argv),
+                    reinterpret_cast<char **>(env_ptr));
+
+  // TODO: TLS cleanup should be done after all other atexit callbacks
+  // are run. So, register a cleanup callback for it with atexit before
+  // everything else.
+  __llvm_libc::cleanup_tls(tls.addr, tls.size);
+  __llvm_libc::exit(retval);
+}
+
+extern "C" void _start() {
+  // Fetch the args using the frame pointer.
+  app.args = reinterpret_cast<__llvm_libc::Args *>(
+      reinterpret_cast<uintptr_t *>(__builtin_frame_address(0)));
+  do_start();
+}