[libc] Add TLS image initialization to aarch64 startup code.

[sivachandra]

Summary

The TLS loader test has been enabled for aarch64.
Handling of PT_TLS' filesz and memsz for x86_64 has also been fixed.

[jeffbailey]

libc/test/integration/loader/linux/CMakeLists.txt

41–43

I wonder if we should have a variable of "LIBC_ARCH_SUPPORTS_LOADERS" or something like this so that we're not flapping this in and out for every arch that gets added.

libc/config/linux/app.h

 
 namespace __llvm_libc {
 
 // Data structure to capture properties of the linux/ELF TLS.
 struct TLS {
   // The load address of the TLS.
   uintptr_t address;
 
-  // The bytes size of the TLS.
+  // The byte size of the TLS image consisting of both initialized and
+  // uninitialized memory. In ELF executables, it is size of .tdata + size of
+  // .tbss. Put in another way, it is the memsz field of the PT_TLS header.
   uintptr_t size;
 
+  // The byte size of initialized memory in the TLS image. In ELF exectubles,
+  // this is the size of .tdata. Put in another way, it is the filesz of the
+  // PT_TLS header.
+  uintptr_t init_size;
+
   // The alignment of the TLS layout. It assumed that the alignment
   // value is a power of 2.
   uintptr_t align;
 };
 
 #if defined(LLVM_LIBC_ARCH_X86_64) || defined(LLVM_LIBC_ARCH_AARCH64)
 // At the language level, argc is an int. But we use uint64_t as the x86_64
 // ABI specifies it as an 8 byte value. Likewise, in the ARM64 ABI, arguments

libc/loader/linux/aarch64/CMakeLists.txt

 add_loader_object(
   crt1
   SRC
     start.cpp
   DEPENDS
     libc.config.linux.app_h
+    libc.include.sys_mman
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
+    libc.src.string.memory_utils.memcpy_implementation
   COMPILE_OPTIONS
     -fno-omit-frame-pointer
     -ffreestanding # To avoid compiler warnings about calling the main function.
 )

libc/loader/linux/aarch64/start.cpp

 //===-- Implementation of crt for aarch64 ---------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "config/linux/app.h"
 #include "src/__support/OSUtil/syscall.h"
+#include "src/string/memory_utils/memcpy_implementations.h"
+
+#include <arm_acle.h>
 
 #include <linux/auxvec.h>
 #include <linux/elf.h>
+#include <stddef.h>
 #include <stdint.h>
+#include <sys/mman.h>
 #include <sys/syscall.h>
 
 extern "C" int main(int, char **, char **);
 
 // Source documentation:
 // https://github.com/ARM-software/abi-aa/tree/main/sysvabi64
 
 namespace __llvm_libc {
 
+#ifdef SYS_mmap2
+static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap2;
+#elif SYS_mmap
+static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap;
+#else
+#error "Target platform does not have SYS_mmap or SYS_mmap2 defined"
+#endif
+
 AppProperties app;
 
+void initTLS() {
+  if (app.tls.size == 0)
+    return;
+
+  // aarch64 follows the variant 1 TLS layout:
+  //
+  // 1. First entry is the dynamic thread vector pointer
+  // 2. Second entry is a 8-byte reserved word.
+  // 3. Padding for alignment.
+  // 4. The TLS data from the ELF image.
+  //
+  // The thread pointer points to the first entry.
+
+  const size_t size_of_pointers = 2 * sizeof(uintptr_t);
+  size_t padding = 0;
+  const size_t ALIGNMENT_MASK = app.tls.align - 1;
+  size_t diff = size_of_pointers & ALIGNMENT_MASK;
+  if (diff != 0)
+    padding += (ALIGNMENT_MASK - diff) + 1;
+
+  size_t alloc_size = size_of_pointers + padding + app.tls.size;
+
+  // We cannot call the mmap function here as the functions set errno on
+  // failure. Since errno is implemented via a thread local variable, we cannot
+  // use errno before TLS is setup.
+  long mmap_ret_val = __llvm_libc::syscall(MMAP_SYSCALL_NUMBER, nullptr,
+                                           alloc_size, PROT_READ | PROT_WRITE,
+                                           MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  // We cannot check the return value with MAP_FAILED as that is the return
+  // of the mmap function and not the mmap syscall.
+  if (mmap_ret_val < 0 && static_cast<uintptr_t>(mmap_ret_val) > -app.pageSize)
+    __llvm_libc::syscall(SYS_exit, 1);
+  uintptr_t thread_ptr = uintptr_t(reinterpret_cast<uintptr_t *>(mmap_ret_val));
+  uintptr_t tls_addr = thread_ptr + size_of_pointers + padding;
+  __llvm_libc::inline_memcpy(reinterpret_cast<char *>(tls_addr),
+                             reinterpret_cast<const char *>(app.tls.address),
+                             app.tls.init_size);
+  __arm_wsr64("tpidr_el0", thread_ptr);
+}
+
 } // namespace __llvm_libc
 
 using __llvm_libc::app;
 
 // TODO: Would be nice to use the aux entry structure from elf.h when available.
 struct AuxEntry {
   uint64_t type;
   uint64_t value;
extern "C" void _start() {
   // value. We step over it (the "+ 1" below) to get to the env values.
   uint64_t *env_ptr = app.args->argv + app.args->argc + 1;
   uint64_t *env_end_marker = env_ptr;
   while (*env_end_marker)
     ++env_end_marker;
 
   // After the env array, is the aux-vector. The end of the aux-vector is
   // denoted by an AT_NULL entry.
+  Elf64_Phdr *programHdrTable = nullptr;
+  uintptr_t programHdrCount;
   for (AuxEntry *aux_entry = reinterpret_cast<AuxEntry *>(env_end_marker + 1);
        aux_entry->type != AT_NULL; ++aux_entry) {
     switch (aux_entry->type) {
+    case AT_PHDR:
+      programHdrTable = reinterpret_cast<Elf64_Phdr *>(aux_entry->value);
+      break;
+    case AT_PHNUM:
+      programHdrCount = aux_entry->value;
+      break;
     case AT_PAGESZ:
       app.pageSize = aux_entry->value;
       break;
     default:
       break; // TODO: Read other useful entries from the aux vector.
     }
   }
 
-  // TODO: Init TLS
+  app.tls.size = 0;
+  for (uintptr_t i = 0; i < programHdrCount; ++i) {
+    Elf64_Phdr *phdr = programHdrTable + i;
+    if (phdr->p_type != PT_TLS)
+      continue;
+    // TODO: p_vaddr value has to be adjusted for static-pie executables.
+    app.tls.address = phdr->p_vaddr;
+    app.tls.size = phdr->p_memsz;
+    app.tls.init_size = phdr->p_filesz;
+    app.tls.align = phdr->p_align;
+  }
+
+  __llvm_libc::initTLS();
 
   __llvm_libc::syscall(SYS_exit, main(app.args->argc,
                                       reinterpret_cast<char **>(app.args->argv),
                                       reinterpret_cast<char **>(env_ptr)));
 }

libc/loader/linux/x86_64/start.cpp

void initTLS() {
 
   // x86_64 TLS faces down from the thread pointer with the first entry
   // pointing to the address of the first real TLS byte.
   uintptr_t endPtr = reinterpret_cast<uintptr_t>(tlsAddr) + tlsSize;
   *reinterpret_cast<uintptr_t *>(endPtr) = endPtr;
 
   __llvm_libc::inline_memcpy(reinterpret_cast<char *>(tlsAddr),
                              reinterpret_cast<const char *>(app.tls.address),
-                             app.tls.size);
+                             app.tls.init_size);
   if (__llvm_libc::syscall(SYS_arch_prctl, ARCH_SET_FS, endPtr) == -1)
     __llvm_libc::syscall(SYS_exit, 1);
 }
 
 } // namespace __llvm_libc
 
 using __llvm_libc::app;
 
for (AuxEntry *aux_entry = reinterpret_cast<AuxEntry *>(env_end_marker + 1);
     case AT_PAGESZ:
       app.pageSize = aux_entry->value;
       break;
     default:
       break; // TODO: Read other useful entries from the aux vector.
     }
   }
 
+  app.tls.size = 0;
   for (uintptr_t i = 0; i < programHdrCount; ++i) {
     Elf64_Phdr *phdr = programHdrTable + i;
     if (phdr->p_type != PT_TLS)
       continue;
     // TODO: p_vaddr value has to be adjusted for static-pie executables.
     app.tls.address = phdr->p_vaddr;
     app.tls.size = phdr->p_memsz;
+    app.tls.init_size = phdr->p_filesz;
     app.tls.align = phdr->p_align;
   }
 
   __llvm_libc::initTLS();
 
   __llvm_libc::syscall(SYS_exit, main(app.args->argc,
                                       reinterpret_cast<char **>(app.args->argv),
                                       reinterpret_cast<char **>(env_ptr)));
 }

libc/test/integration/loader/linux/CMakeLists.txt

add_integration_test(
   loader_no_args_test
   SUITE libc-loader-tests
   LOADER
     libc.loader.linux.crt1
   SRCS
     main_without_args.cpp
 )
 
-if(NOT (${LIBC_TARGET_ARCHITECTURE} STREQUAL "x86_64"))
-  return()
-endif()
-
 add_integration_test(
   loader_tls_test
   SUITE libc-loader-tests
   LOADER
     libc.loader.linux.crt1
   SRCS
     tls_test.cpp
   DEPENDS
     libc.include.errno
     libc.include.sys_mman
     libc.src.errno.errno
     libc.src.sys.mman.mmap
 )