[libc] Setup TLS in x86_64 loader.

[sivachandra]

Summary

The new code added is still very x86_64 specific. AArch64 support will
be added very soon and refactoring of the loader code will be done as
part of the patches adding it.

[sivachandra]

Add TODO to adjust TLS address for static-pie executables.

[MaskRay]

libc/loader/linux/x86_64/start.cpp

38

This should be implemented directly in a generic place, instead of adding a TODO and moving all content immediately when implementing aarch64.

46

This formula can be written as: tlsSize = (app.tls.size + app.tls.align) & -app.tls.align
However, there is a larger issue. See below.

51

The size should be 2 words plus sizeof(pthread) plus p_memsz(PT_TLS) plus (headroom left for padding).

Initially dtv[0] = 1 (one module), dtv[1] = DTP offset. The pthread structure is placed at the end of dtv.

60

This can call an internal crash function instead of SYS_exit.

mmapRetVal < 0 is sufficient.

[sivachandra]

libc/loader/linux/x86_64/start.cpp

51

Size of pthread can/will be added once we develop or need an equivalent of that in LLVM-libc. So, without that, and without support for dynamic loading, can you give an example wherein the code as it stands in this patch would fail?

[I will add a note in the commit message and also a comment that we don't yet have dynamic loading support.]

[asteinhauser]

libc/config/linux/app.h

16

It would be good to add source-code comments about the semantics of those two structs and their fields.

libc/loader/linux/x86_64/start.cpp

25

There should probably be SYS_mmap2 instead of SYS_mmaps2.

46

The existing comment about tlsSize = (app.tls.size + app.tls.align) & -app.tls.align seems relevant to me. It assumes that app.tls.align is a power of two which is a valid assumption, but might be mentioned in libc/config/linux/app.h comments.

59

Why "if (mmapRetVal == MAP_FAILED)" is not a sufficient check?

[sivachandra]

• Rebase
• Address comments.
• Along the way, fixed some loader test depenency handling.

[sivachandra]

libc/loader/linux/x86_64/start.cpp

25

Good catch! Fixed.

59

Added a comment to explain this.

libc/config/linux/CMakeLists.txt

 add_gen_header(
   linux_syscall_h
   DEF_FILE syscall.h.def
   GEN_HDR syscall.h
   PARAMS
     inline_syscalls=${LIBC_TARGET_MACHINE}/syscall.h.inc
   DATA_FILES
     ${LIBC_TARGET_MACHINE}/syscall.h.inc
   DEPENDS
     libc.src.__support.common
 )
+
+add_header(
+  app_h
+  HDR
+    app.h
+)

libc/config/linux/app.h

+//===-- Classes to capture properites of linux applications -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_CONFIG_LINUX_APP_H
+#define LLVM_LIBC_CONFIG_LINUX_APP_H
+
+#include <stdint.h>
+
+namespace __llvm_libc {
+
+struct TLS {
+  uintptr_t address;
+  uintptr_t size;
+  uintptr_t align;
+};
+
+struct AppProperties {
+  uintptr_t pageSize;
+  TLS tls;
+};
+
+// Creates and initializes the TLS area for the current thread. Should not
+// be called before app.tls has been initialized.
+void initTLS();
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_CONFIG_LINUX_APP_H

libc/loader/linux/x86_64/CMakeLists.txt

 add_loader_object(
   crt1
   SRC
     start.cpp
   DEPENDS
     libc.config.linux.linux_syscall_h
+    libc.config.linux.app_h
     libc.include.sys_syscall
+    libc.src.string.memcpy
+    libc.src.sys.mman.mmap
   COMPILE_OPTIONS
     -fno-omit-frame-pointer
     -ffreestanding # To avoid compiler warnings about calling the main function.
 )

libc/loader/linux/x86_64/start.cpp

 //===-- Implementation of crt for x86_64 ----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
+#include "config/linux/app.h"
 #include "config/linux/syscall.h"
+#include "include/sys/mman.h"
 #include "include/sys/syscall.h"
+#include "src/string/memcpy.h"
+#include "src/sys/mman/mmap.h"
 
+#include <asm/prctl.h>
 #include <linux/auxvec.h>
+#include <linux/elf.h>
 #include <stdint.h>
 
 extern "C" int main(int, char **, char **);
 
+namespace __llvm_libc {
+
+#ifdef SYS_mmaps2
+static constexpr long mmapSyscallNumber = SYS_mmap2;
+#elif SYS_mmap
+static constexpr long mmapSyscallNumber = SYS_mmap;
+#else
+#error "Target platform does not have SYS_mmap or SYS_mmap2 defined"
+#endif
+
+// TODO: Declare var an extern var in config/linux/app.h so that other
+// libc functions can make use of the application wide information. For
+// example, mmap can pick up the page size from here.
+AppProperties app;
+
+// TODO: The function is x86_64 specific. Move it to config/linux/app.h
+// and generalize it.
+void initTLS() {
+  if (app.tls.size == 0)
+    return;
+
+  uintptr_t tlsSize = app.tls.size;
+  uintptr_t misAlign = tlsSize % app.tls.align;
+  tlsSize += (misAlign ? app.tls.align - misAlign : 0);
+
+  // Per the x86_64 TLS ABI, the entry pointed to by the thread pointer is the
+  // address of the TLS block. So, we add more size to accomodate this address
+  // entry.
+  size_t tlsSizeWithAddr = tlsSize + sizeof(uintptr_t);
+
+  // We cannot call the mmap function here as the functions set errno on
+  // failure. Since errno is implemented via a thread local variable, we cannot
+  // use errno before TLS is setup.
+  long mmapRetVal = __llvm_libc::syscall(
+      mmapSyscallNumber, nullptr, tlsSizeWithAddr, PROT_READ | PROT_WRITE,
+      MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  if (mmapRetVal < 0 && static_cast<uintptr_t>(mmapRetVal) > -app.pageSize)
+    __llvm_libc::syscall(SYS_exit, 1);
+  uintptr_t *tlsAddr = reinterpret_cast<uintptr_t *>(mmapRetVal);
+
+  // x86_64 TLS faces down from the thread pointer with the first entry
+  // pointing to the address of the first real TLS byte.
+  uintptr_t endPtr = reinterpret_cast<uintptr_t>(tlsAddr) + tlsSize;
+  *reinterpret_cast<uintptr_t *>(endPtr) =
+      reinterpret_cast<uintptr_t>(tlsAddr) + tlsSize;
+
+  __llvm_libc::memcpy(tlsAddr, reinterpret_cast<const void *>(app.tls.address),
+                      app.tls.size);
+  if (__llvm_libc::syscall(SYS_arch_prctl, ARCH_SET_FS, endPtr) == -1)
+    __llvm_libc::syscall(SYS_exit, 1);
+}
+
+} // namespace __llvm_libc
+
+using __llvm_libc::app;
+
 struct Args {
   // At the language level, argc is an int. But we use uint64_t as the x86_64
   // ABI specifies it as an 8 byte value.
   uint64_t argc;
 
   // At the language level, argv is a char** value. However, we use uint64_t as
   // the x86_64 ABI specifies the argv vector be an |argc| long array of 8-byte
   // values. Even though a flexible length array would be more suitable here, we
extern "C" void _start() {
   // value. We step over it (the "+ 1" below) to get to the env values.
   uint64_t *env_ptr = args->argv + args->argc + 1;
   uint64_t *env_end_marker = env_ptr;
   while (*env_end_marker)
     ++env_end_marker;
 
   // After the env array, is the aux-vector. The end of the aux-vector is
   // denoted by an AT_NULL entry.
+  Elf64_Phdr *programHdrTable = nullptr;
+  uintptr_t programHdrCount;
   for (AuxEntry *aux_entry = reinterpret_cast<AuxEntry *>(env_end_marker + 1);
        aux_entry->type != AT_NULL; ++aux_entry) {
-    // TODO: Read the aux vector and store necessary information in a libc wide
-    // data structure.
+    switch (aux_entry->type) {
+    case AT_PHDR:
+      programHdrTable = reinterpret_cast<Elf64_Phdr *>(aux_entry->value);
+      break;
+    case AT_PHNUM:
+      programHdrCount = aux_entry->value;
+      break;
+    case AT_PAGESZ:
+      app.pageSize = aux_entry->value;
+      break;
+    default:
+      break; // TODO: Read other useful entries from the aux vector.
     }
+  }
+
+  for (uintptr_t i = 0; i < programHdrCount; ++i) {
+    Elf64_Phdr *phdr = programHdrTable + i;
+    if (phdr->p_type != PT_TLS)
+      continue;
+    // TODO: p_vaddr value has to be adjusted for static-pie executables.
+    app.tls.address = phdr->p_vaddr;
+    app.tls.size = phdr->p_memsz;
+    app.tls.align = phdr->p_align;
+  }
+
+  __llvm_libc::initTLS();
 
   __llvm_libc::syscall(SYS_exit,
                        main(args->argc, reinterpret_cast<char **>(args->argv),
                             reinterpret_cast<char **>(env_ptr)));
 }

libc/test/loader/linux/CMakeLists.txt

 
 add_loader_test(
   loader_no_args_test
   SRC
     main_without_args.cpp
   DEPENDS
     libc.loader.linux.crt1
 )
+
+add_loader_test(
+  loader_tls_test
+  SRC
+    tls_test.cpp
+  DEPENDS
+    libc.config.linux.app_h
+    libc.include.errno
+    libc.include.sys_mman
+    libc.loader.linux.crt1
+    libc.src.assert.__assert_fail
+    libc.src.errno.__errno_location
+    libc.src.sys.mman.mmap
+)

libc/test/loader/linux/tls_test.cpp

+//===-- Loader test to check if tls size is read correctly ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/errno.h"
+#include "include/sys/mman.h"
+
+#undef NDEBUG
+#include "src/assert/assert.h"
+
+#include "src/errno/llvmlibc_errno.h"
+#include "src/sys/mman/mmap.h"
+
+constexpr int threadLocalDataSize = 101;
+_Thread_local int a[threadLocalDataSize] = {123};
+
+int main(int argc, char **argv, char **envp) {
+  assert(a[0] == 123);
+
+  for (int i = 1; i < threadLocalDataSize; ++i)
+    a[i] = i;
+  for (int i = 1; i < threadLocalDataSize; ++i)
+    assert(a[i] == i);
+
+  // Call mmap with bad params so that an error value is
+  // set in errno. Since errno is implemented using a thread
+  // local var, this helps us test setting of errno and
+  // reading it back.
+  assert(llvmlibc_errno == 0);
+  void *addr = __llvm_libc::mmap(nullptr, 0, PROT_READ,
+                                 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  assert(addr == MAP_FAILED);
+  assert(llvmlibc_errno == EINVAL);
+
+  return 0;
+}