[libc] Linux threads - Setup TLS area of a new thread and cleanup at exit.

[sivachandra]

Fix typos in aarch64 loader.

[lntue]

libc/config/linux/app.h

83

Why is this uintptr_t but not size_t?

[sivachandra]

libc/config/linux/app.h

83

This size value actually comes from the ELF TLS image description. In ELF64 it is a 64-bit value, and in ELF32 it is a 32-bit value. So, capturing it as a uintptr_t value will capture it without any potential mix ups.

[sivachandra]

Rebase

[lntue]

libc/config/linux/app.h

83

In the implementation below, all the size computations were done with size_t, and then get converted to uintptr_t of TLSDescriptor. Shall we add a static assertion to make sure that it won't silently overflow, i.e. sizeof(size_t) <= sizeof(uintptr_t)? At least they are the same for most of our platforms of interest anyway.

[michaelrj]

overall LGTM, with a few stylish nits

libc/config/linux/app.h

18

Nit: If you're adjusting the comment below to talk about a TLS image instead of just a TLS, you should do it here too.

libc/src/__support/threads/linux/thread.cpp

112–113

Nit: delete this line.

[sivachandra]

Address comments.

libc/config/linux/app.h

83

Good catch. I fixed it now to use uintptr_t everywhere.

libc/config/linux/app.h

 #define LLVM_LIBC_CONFIG_LINUX_APP_H
 
 #include "src/__support/architectures.h"
 
 #include <stdint.h>
 
 namespace __llvm_libc {
 
 // Data structure to capture properties of the linux/ELF TLS.
-struct TLS {
+struct TLSImage {
   // The load address of the TLS.
   uintptr_t address;
 
   // The byte size of the TLS image consisting of both initialized and
   // uninitialized memory. In ELF executables, it is size of .tdata + size of
   // .tbss. Put in another way, it is the memsz field of the PT_TLS header.
   uintptr_t size;
 

 
 // Data structure which captures properties of a linux application.
 struct AppProperties {
   // Page size used for the application.
   uintptr_t pageSize;
 
   Args *args;
 
-  // The properties of an application's TLS.
-  TLS tls;
+  // The properties of an application's TLS image.
+  TLSImage tls;
 
   // Environment data.
   uint64_t *envPtr;
 };
 
 extern AppProperties app;
 
-// Creates and initializes the TLS area for the current thread. Should not
+// The descriptor of a thread's TLS area.
+struct TLSDescriptor {
+  // The size of the TLS area.
+  uintptr_t size = 0;
+
+  // The address of the TLS area. This address can be passed to cleanup
+  // functions like munmap.
+  uintptr_t addr = 0;
+
+  // The value the thread pointer register should be initialized to.
+  // Note that, dependending the target architecture ABI, it can be the
+  // same as |addr| or something else.
+  uintptr_t tp = 0;
+
+  constexpr TLSDescriptor() = default;
+};
+
+// Create and initialize the TLS area for the current thread. Should not
 // be called before app.tls has been initialized.
-void initTLS();
+void init_tls(TLSDescriptor &tls);
+
+// Cleanup the TLS area as described in |tls_descriptor|.
+void cleanup_tls(uintptr_t tls_addr, uintptr_t tls_size);
 
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_CONFIG_LINUX_APP_H

libc/loader/linux/aarch64/start.cpp

 #elif SYS_mmap
 static constexpr long MMAP_SYSCALL_NUMBER = SYS_mmap;
 #else
 #error "Target platform does not have SYS_mmap or SYS_mmap2 defined"
 #endif
 
 AppProperties app;
 
-void initTLS() {
-  if (app.tls.size == 0)
+void init_tls(TLSDescriptor &tls) {
+  if (app.tls.size == 0) {
+    tls_descriptor.size = 0;
+    tls_descriptor.tp = 0;
     return;
+  }
 
   // aarch64 follows the variant 1 TLS layout:
   //
   // 1. First entry is the dynamic thread vector pointer
   // 2. Second entry is a 8-byte reserved word.
   // 3. Padding for alignment.
   // 4. The TLS data from the ELF image.
   //
void init_tls(TLSDescriptor &tls) {
   // of the mmap function and not the mmap syscall.
   if (mmap_ret_val < 0 && static_cast<uintptr_t>(mmap_ret_val) > -app.pageSize)
     __llvm_libc::syscall(SYS_exit, 1);
   uintptr_t thread_ptr = uintptr_t(reinterpret_cast<uintptr_t *>(mmap_ret_val));
   uintptr_t tls_addr = thread_ptr + size_of_pointers + padding;
   __llvm_libc::inline_memcpy(reinterpret_cast<char *>(tls_addr),
                              reinterpret_cast<const char *>(app.tls.address),
                              app.tls.init_size);
-  __arm_wsr64("tpidr_el0", thread_ptr);
+  tls.size = alooc_size;
+  tls.addr = thread_ptr;
+  tls.tp = thread_ptr;
+}
+
+void cleanup_tls(uintptr_t addr, uintptr_t size) {
+  if (size == 0)
+    return;
+  __llvm_libc::syscall(SYS_munmap, addr, size);
 }
 
+static void set_thread_ptr(uintptr_t val) { __arm_wsr64("tpidr_el0", val); }
+
 } // namespace __llvm_libc
 
 using __llvm_libc::app;
 
 // TODO: Would be nice to use the aux entry structure from elf.h when available.
 struct AuxEntry {
   uint64_t type;
   uint64_t value;
if (phdr->p_type != PT_TLS)
       continue;
     // TODO: p_vaddr value has to be adjusted for static-pie executables.
     app.tls.address = phdr->p_vaddr;
     app.tls.size = phdr->p_memsz;
     app.tls.init_size = phdr->p_filesz;
     app.tls.align = phdr->p_align;
   }
 
-  __llvm_libc::initTLS();
-
-  __llvm_libc::syscall(SYS_exit, main(app.args->argc,
-                                      reinterpret_cast<char **>(app.args->argv),
-                                      reinterpret_cast<char **>(env_ptr)));
+  __llvm_libc::TLSDescriptor tls;
+  __llvm_libc::init_tls(tls);
+  if (tls.size != 0)
+    __llvm_libc::set_thread_ptr(tls.tp);
+
+  int retval = main(app.args->argc, reinterpret_cast<char **>(app.args->argv),
+                    reinterpret_cast<char **>(env_ptr));
+  __llvm_libc::cleanup_tls(tls.addr, tls.size);
+  __llvm_libc::syscall(SYS_exit, retval);
 }

libc/loader/linux/x86_64/CMakeLists.txt

 add_loader_object(
   crt1
   SRC
     start.cpp
   DEPENDS
     libc.config.linux.app_h
     libc.include.sys_mman
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
     libc.src.string.memory_utils.memcpy_implementation
   COMPILE_OPTIONS
     -fno-omit-frame-pointer
     -ffreestanding # To avoid compiler warnings about calling the main function.
+    -fno-builtin
 )

libc/loader/linux/x86_64/start.cpp

 #else
 #error "Target platform does not have SYS_mmap or SYS_mmap2 defined"
 #endif
 
 AppProperties app;
 
 // TODO: The function is x86_64 specific. Move it to config/linux/app.h
 // and generalize it. Also, dynamic loading is not handled currently.
-void initTLS() {
-  if (app.tls.size == 0)
+void init_tls(TLSDescriptor &tls_descriptor) {
+  if (app.tls.size == 0) {
+    tls_descriptor.size = 0;
+    tls_descriptor.tp = 0;
     return;
+  }
 
   // We will assume the alignment is always a power of two.
   uintptr_t tlsSize = app.tls.size & -app.tls.align;
   if (tlsSize != app.tls.size)
     tlsSize += app.tls.align;
 
   // Per the x86_64 TLS ABI, the entry pointed to by the thread pointer is the
   // address of the TLS block. So, we add more size to accomodate this address
void init_tls(TLSDescriptor &tls_descriptor) {
   // x86_64 TLS faces down from the thread pointer with the first entry
   // pointing to the address of the first real TLS byte.
   uintptr_t endPtr = reinterpret_cast<uintptr_t>(tlsAddr) + tlsSize;
   *reinterpret_cast<uintptr_t *>(endPtr) = endPtr;
 
   __llvm_libc::inline_memcpy(reinterpret_cast<char *>(tlsAddr),
                              reinterpret_cast<const char *>(app.tls.address),
                              app.tls.init_size);
-  if (__llvm_libc::syscall(SYS_arch_prctl, ARCH_SET_FS, endPtr) == -1)
-    __llvm_libc::syscall(SYS_exit, 1);
+
+  tls_descriptor = {tlsSizeWithAddr, uintptr_t(tlsAddr), endPtr};
+  return;
+}
+
+void cleanup_tls(uintptr_t addr, uintptr_t size) {
+  if (size == 0)
+    return;
+  __llvm_libc::syscall(SYS_munmap, addr, size);
+}
+
+// Sets the thread pointer to |val|. Returns true on success, false on failure.
+static bool set_thread_ptr(uintptr_t val) {
+  return __llvm_libc::syscall(SYS_arch_prctl, ARCH_SET_FS, val) == -1 ? false
+                                                                      : true;
 }
 
 } // namespace __llvm_libc
 
 using __llvm_libc::app;
 
 // TODO: Would be nice to use the aux entry structure from elf.h when available.
 struct AuxEntry {
if (phdr->p_type != PT_TLS)
       continue;
     // TODO: p_vaddr value has to be adjusted for static-pie executables.
     app.tls.address = phdr->p_vaddr;
     app.tls.size = phdr->p_memsz;
     app.tls.init_size = phdr->p_filesz;
     app.tls.align = phdr->p_align;
   }
 
-  __llvm_libc::initTLS();
+  __llvm_libc::TLSDescriptor tls;
+  __llvm_libc::init_tls(tls);
+  if (tls.size != 0 && !__llvm_libc::set_thread_ptr(tls.tp))
+    __llvm_libc::syscall(SYS_exit, 1);
 
-  __llvm_libc::syscall(SYS_exit, main(app.args->argc,
-                                      reinterpret_cast<char **>(app.args->argv),
-                                      reinterpret_cast<char **>(env_ptr)));
+  int retval = main(app.args->argc, reinterpret_cast<char **>(app.args->argv),
+                    reinterpret_cast<char **>(env_ptr));
+  __llvm_libc::cleanup_tls(tls.addr, tls.size);
+  __llvm_libc::syscall(SYS_exit, retval);
 }

libc/src/__support/threads/linux/CMakeLists.txt

 )
 
 add_object_library(
   thread
   SRCS
     thread.cpp
   DEPENDS
     .futex_word_type
+    libc.config.linux.app_h
     libc.include.sys_syscall
     libc.src.__support.CPP.atomic
     libc.src.__support.CPP.error
     libc.src.__support.threads.thread_common
   COMPILE_OPTIONS
     -O3
     -fno-omit-frame-pointer # This allows us to sniff out the thread args from
                             # the new thread's stack reliably.
 )

libc/src/__support/threads/linux/thread.cpp

 //===--- Implementation of a Linux thread class -----------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/threads/thread.h"
+#include "config/linux/app.h"
 #include "src/__support/CPP/atomic.h"
 #include "src/__support/CPP/error.h"
 #include "src/__support/OSUtil/syscall.h"           // For syscall functions.
 #include "src/__support/threads/linux/futex_word.h" // For FutexWordType
 
 #ifdef LLVM_LIBC_ARCH_AARCH64
 #include <arm_acle.h>
 #endif

 static constexpr unsigned CLONE_SYSCALL_FLAGS =
     CLONE_VM        // Share the memory space with the parent.
     | CLONE_FS      // Share the file system with the parent.
     | CLONE_FILES   // Share the files with the parent.
     | CLONE_SIGHAND // Share the signal handlers with the parent.
     | CLONE_THREAD  // Same thread group as the parent.
     | CLONE_SYSVSEM // Share a single list of System V semaphore adjustment
                     // values
-    | CLONE_PARENT_SETTID   // Set child thread ID in |ptid| of the parent.
-    | CLONE_CHILD_CLEARTID; // Let the kernel clear the tid address
-                            // wake the joining thread.
-// TODO: Add the CLONE_SETTLS flag and setup the TLS area correctly
-// when making the clone syscall.
+    | CLONE_PARENT_SETTID  // Set child thread ID in |ptid| of the parent.
+    | CLONE_CHILD_CLEARTID // Let the kernel clear the tid address
+    | CLONE_SETTLS;        // Setup the thread pointer of the new thread.
 
 static inline cpp::ErrorOr<void *> alloc_stack(size_t size) {
   long mmap_result =
       __llvm_libc::syscall(MMAP_SYSCALL_NUMBER,
                            0, // No special address
                            size,
                            PROT_READ | PROT_WRITE,      // Read and write stack
                            MAP_ANONYMOUS | MAP_PRIVATE, // Process private

 // If different architecture in future requires higher alignment, then we
 // can add a platform specific alignment spec.
 struct alignas(STACK_ALIGNMENT) StartArgs {
   ThreadAttributes *thread_attrib;
   ThreadRunner runner;
   void *arg;
 };
 
+static void cleanup_thread_resources(ThreadAttributes *attrib) {
+  // Cleanup the TLS before the stack as the TLS information is stored on
+  // the stack.
+  cleanup_tls(attrib->tls, attrib->tls_size);
+  if (attrib->owned_stack)
+    free_stack(attrib->stack, attrib->stack_size);
+}
+
 __attribute__((always_inline)) inline uintptr_t get_start_args_addr() {
 // NOTE: For __builtin_frame_address to work reliably across compilers,
 // architectures and various optimization levels, the TU including this file
 // should be compiled with -fno-omit-frame-pointer.
 #ifdef LLVM_LIBC_ARCH_X86_64
   return reinterpret_cast<uintptr_t>(__builtin_frame_address(0))
          // The x86_64 call instruction pushes resume address on to the stack.
          // Next, The x86_64 SysV ABI requires that the frame pointer be pushed
          // on to the stack. So, we have to step past two 64-bit values to get
          // to the start args.
          + sizeof(uintptr_t) * 2;
 #elif defined(LLVM_LIBC_ARCH_AARCH64)
   // The frame pointer after cloning the new thread in the Thread::run method
   // is set to the stack pointer where start args are stored. So, we fetch
   // from there.
   return reinterpret_cast<uintptr_t>(__builtin_frame_address(1));
 #endif
 }
 
 static void start_thread() __attribute__((noinline)) {
   auto *start_args = reinterpret_cast<StartArgs *>(get_start_args_addr());
   auto *attrib = start_args->thread_attrib;
+
   long retval;
   if (attrib->style == ThreadStyle::POSIX) {
     attrib->retval.posix_retval =
         start_args->runner.posix_runner(start_args->arg);
     retval = long(attrib->retval.posix_retval);
   } else {
     attrib->retval.stdc_retval =
         start_args->runner.stdc_runner(start_args->arg);
     retval = long(attrib->retval.stdc_retval);
   }
 
   uint32_t joinable_state = uint32_t(DetachState::JOINABLE);
   if (!attrib->detach_state.compare_exchange_strong(
           joinable_state, uint32_t(DetachState::EXITING))) {
     // Thread is detached so cleanup the resources.
-    if (attrib->owned_stack)
-      free_stack(attrib->stack, attrib->stack_size);
+    cleanup_thread_resources(attrib);
 
     // Set the CLEAR_TID address to nullptr to prevent the kernel
     // from signalling at a non-existent futex location.
     __llvm_libc::syscall(SYS_set_tid_address, 0);
   }
 
   __llvm_libc::syscall(SYS_exit, retval);
 }
 
 int Thread::run(ThreadStyle style, ThreadRunner runner, void *arg, void *stack,
                 size_t size, bool detached) {
   bool owned_stack = false;
   if (stack == nullptr) {
     if (size == 0)
       size = DEFAULT_STACK_SIZE;
     auto alloc = alloc_stack(size);
     if (!alloc)
       return alloc.error_code();
     else
       stack = alloc.value();
     owned_stack = true;
   }
 
+  TLSDescriptor tls;
+  init_tls(tls);
+
   // When the new thread is spawned by the kernel, the new thread gets the
   // stack we pass to the clone syscall. However, this stack is empty and does
   // not have any local vars present in this function. Hence, one cannot
   // pass arguments to the thread start function, or use any local vars from
   // here. So, we pack them into the new stack from where the thread can sniff
   // them out.
   //
   // Likewise, the actual thread state information is also stored on the
   // stack memory.
   uintptr_t adjusted_stack = reinterpret_cast<uintptr_t>(stack) + size -
                              sizeof(StartArgs) - sizeof(ThreadAttributes) -
                              sizeof(cpp::Atomic<FutexWordType>);
   adjusted_stack &= ~(uintptr_t(STACK_ALIGNMENT) - 1);
 
   auto *start_args = reinterpret_cast<StartArgs *>(adjusted_stack);
 
   attrib =
       reinterpret_cast<ThreadAttributes *>(adjusted_stack + sizeof(StartArgs));
   attrib->style = style;
   attrib->detach_state =
       uint32_t(detached ? DetachState::DETACHED : DetachState::JOINABLE);
   attrib->stack = stack;
   attrib->stack_size = size;
   attrib->owned_stack = owned_stack;
+  attrib->tls = tls.addr;
+  attrib->tls_size = tls.size;
 
   start_args->thread_attrib = attrib;
   start_args->runner = runner;
   start_args->arg = arg;
 
   auto clear_tid = reinterpret_cast<cpp::Atomic<FutexWordType> *>(
       adjusted_stack + sizeof(StartArgs) + sizeof(ThreadAttributes));
   clear_tid->val = CLEAR_TID_VALUE;
   platform_data = clear_tid;
 
   // The clone syscall takes arguments in an architecture specific order.
   // Also, we want the result of the syscall to be in a register as the child
   // thread gets a completely different stack after it is created. The stack
   // variables from this function will not be availalbe to the child thread.
 #ifdef LLVM_LIBC_ARCH_X86_64
   long register clone_result asm("rax");
   clone_result = __llvm_libc::syscall(
       SYS_clone, CLONE_SYSCALL_FLAGS, adjusted_stack,
       &attrib->tid,    // The address where the child tid is written
       &clear_tid->val, // The futex where the child thread status is signalled
-      0                // Set TLS to null for now.
+      tls.tp           // The thread pointer value for the new thread.
   );
 #elif defined(LLVM_LIBC_ARCH_AARCH64)
   long register clone_result asm("x0");
   clone_result = __llvm_libc::syscall(
       SYS_clone, CLONE_SYSCALL_FLAGS, adjusted_stack,
       &attrib->tid,   // The address where the child tid is written
-      0,              // Set TLS to null for now.
+      tls.tp,         // The thread pointer value for the new thread.
       &clear_tid->val // The futex where the child thread status is signalled
   );
 #else
 #error "Unsupported architecture for the clone syscall."
 #endif
 
   if (clone_result == 0) {
 #ifdef LLVM_LIBC_ARCH_AARCH64
     // We set the frame pointer to be the same as the "sp" so that start args
     // can be sniffed out from start_thread.
     __arm_wsr64("x29", __arm_rsr64("sp"));
 #endif
     start_thread();
   } else if (clone_result < 0) {
-    if (attrib->owned_stack)
-      free_stack(attrib->stack, attrib->stack_size);
+    cleanup_thread_resources(attrib);
     return -clone_result;
   }
 
   return 0;
 }
 
 int Thread::join(ThreadReturnValue &retval) {
   wait();
 
   if (attrib->style == ThreadStyle::POSIX)
     retval.posix_retval = attrib->retval.posix_retval;
   else
     retval.stdc_retval = attrib->retval.stdc_retval;
 
-  if (attrib->owned_stack)
-    free_stack(attrib->stack, attrib->stack_size);
+  cleanup_thread_resources(attrib);
 
   return 0;
 }
 
 int Thread::detach() {
   uint32_t joinable_state = uint32_t(DetachState::JOINABLE);
   if (attrib->detach_state.compare_exchange_strong(
           joinable_state, uint32_t(DetachState::DETACHED))) {
     return int(DetachType::SIMPLE);
   }
 
   // If the thread was already detached, then the detach method should not
   // be called at all. If the thread is exiting, then we wait for it to exit
   // and free up resources.
   wait();
 
-  if (attrib->owned_stack)
-    free_stack(attrib->stack, attrib->stack_size);
+  cleanup_thread_resources(attrib);
+
   return int(DetachType::CLEANUP);
 }
 
 void Thread::wait() {
   // The kernel should set the value at the clear tid address to zero.
   // If not, it is a spurious wake and we should continue to wait on
   // the futex.
   auto *clear_tid =

libc/src/__support/threads/thread.h

struct alignas(STACK_ALIGNMENT) ThreadAttributes {
   //      (a) If the detach method sees the state as JOINABLE, then it will
   //          compare exchange to a state of DETACHED. The thread will clean
   //          itself up after it finishes.
   //      (b) If the detach method does not see JOINABLE in (a), then it will
   //          conclude that the thread is EXITING and will wait until the thread
   //          exits. It will clean up the thread resources once the thread
   //          exits.
   cpp::Atomic<uint32_t> detach_state;
-  void *stack; // Pointer to the thread stack
-  void *tls;
+  void *stack;                   // Pointer to the thread stack
   unsigned long long stack_size; // Size of the stack
+  uintptr_t tls;                 // Address to the thread TLS memory
+  uintptr_t tls_size;            // The size of area pointed to by |tls|.
   unsigned char owned_stack; // Indicates if the thread owns this stack memory
   int tid;
   ThreadStyle style;
   ThreadReturnValue retval;
 };
 
 struct Thread {
   ThreadAttributes *attrib;

libc/test/integration/src/__support/threads/CMakeLists.txt

add_integration_test(
   SRCS
     thread_detach_test.cpp
   LOADER
     libc.loader.linux.crt1
   DEPENDS
     libc.src.__support.threads.mutex
     libc.src.__support.threads.thread
 )
+
+add_integration_test(
+  thread_tls_test
+  SUITE
+    libc-support-threads-integration-tests
+  SRCS
+    thread_tls_test.cpp
+  LOADER
+    libc.loader.linux.crt1
+  DEPENDS
+    libc.src.__support.threads.thread
+)

libc/test/integration/src/__support/threads/thread_tls_test.cpp

+//===-- Test handling of thread local data --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/threads/thread.h"
+#include "utils/IntegrationTest/test.h"
+
+static constexpr int INIT_VAL = 100;
+static constexpr int UPDATE_VAL = 123;
+
+static thread_local int tlval = INIT_VAL;
+
+// This function updates the tlval and returns its old value.
+int func(void *) {
+  int old_tlval = tlval;
+  tlval = UPDATE_VAL;
+  return old_tlval;
+}
+
+void thread_local_test() {
+  int retval;
+
+  __llvm_libc::Thread th1;
+  th1.run(func, nullptr, nullptr, 0);
+  th1.join(&retval);
+  ASSERT_EQ(retval, INIT_VAL);
+
+  __llvm_libc::Thread th2;
+  th2.run(func, nullptr, nullptr, 0);
+  th2.join(&retval);
+  ASSERT_EQ(retval, INIT_VAL);
+}
+
+int main() {
+  // From the main thread, we will update the main thread's tlval.
+  // This should not affect the child thread's tlval;
+  ASSERT_EQ(tlval, INIT_VAL);
+  tlval = UPDATE_VAL;
+  ASSERT_EQ(tlval, UPDATE_VAL);
+  thread_local_test();
+  return 0;
+}