diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td --- a/libc/config/linux/api.td +++ b/libc/config/linux/api.td @@ -216,7 +216,7 @@ def ThreadsAPI : PublicAPI<"threads.h"> { let Macros = [ - SimpleMacroDef<"ONCE_FLAG_INIT", "0">, + SimpleMacroDef<"ONCE_FLAG_INIT", "{0}">, ]; let Types = [ diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt --- a/libc/include/llvm-libc-types/CMakeLists.txt +++ b/libc/include/llvm-libc-types/CMakeLists.txt @@ -1,5 +1,6 @@ add_header(__bsearchcompare_t HDR __bsearchcompare_t.h) add_header(__call_once_func_t HDR __call_once_func_t.h) +add_header(__futex_word HDR __futex_word.h) add_header(__qsortcompare_t HDR __qsortcompare_t.h) add_header(__sighandler_t HDR __sighandler_t.h) add_header(cnd_t HDR cnd_t.h) @@ -13,9 +14,9 @@ add_header(float_t HDR float_t.h) add_header(imaxdiv_t HDR imaxdiv_t.h) add_header(mode_t HDR mode_t.h) -add_header(mtx_t HDR mtx_t.h) +add_header(mtx_t HDR mtx_t.h DEPENDS .__futex_word) add_header(off_t HDR off_t.h) -add_header(once_flag HDR once_flag.h) +add_header(once_flag HDR once_flag.h DEPENDS .__futex_word) add_header(size_t HDR size_t.h) add_header(ssize_t HDR ssize_t.h) add_header(struct_sigaction HDR struct_sigaction.h) diff --git a/libc/include/llvm-libc-types/__futex_word.h b/libc/include/llvm-libc-types/__futex_word.h new file mode 100644 --- /dev/null +++ b/libc/include/llvm-libc-types/__futex_word.h @@ -0,0 +1,24 @@ +//===-- Definition of type which can represent a futex word ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __LLVM_LIBC_TYPES_FUTEX_WORD_H__ +#define __LLVM_LIBC_TYPES_FUTEX_WORD_H__ + +typedef struct { +#if defined(__unix__) && (defined(__x86_64__) || defined(__aarch64__)) + // Futex word should be aligned appropriately to allow target atomic + // instructions. This declaration mimics the internal setup. + _Alignas(sizeof(unsigned int) > _Alignof(unsigned int) + ? sizeof(unsigned int) + : _Alignof(unsigned int)) unsigned int __word; +#else +#error "A type to represent a futex word is not available for the target arch." +#endif +} __futex_word; + +#endif // __LLVM_LIBC_TYPES_FUTEX_WORD_H__ diff --git a/libc/include/llvm-libc-types/mtx_t.h b/libc/include/llvm-libc-types/mtx_t.h --- a/libc/include/llvm-libc-types/mtx_t.h +++ b/libc/include/llvm-libc-types/mtx_t.h @@ -9,17 +9,13 @@ #ifndef __LLVM_LIBC_TYPES_MTX_T_H__ #define __LLVM_LIBC_TYPES_MTX_T_H__ +#include + typedef struct { -#if defined(__x86_64__) || defined(__aarch64__) - // Futex word should be aligned appropriately to allow target atomic - // instructions. This declaration mimics the internal setup. - struct { - _Alignas(sizeof(unsigned int) > _Alignof(unsigned int) - ? sizeof(unsigned int) - : _Alignof(unsigned int)) unsigned int __word; - } __futex_word; +#ifdef __unix__ + __futex_word __ftxw; #else -#error "Mutex type mtx_t is not available for the target architecture." +#error "mtx_t type not defined for the target platform." #endif int __mtx_type; } mtx_t; diff --git a/libc/include/llvm-libc-types/once_flag.h b/libc/include/llvm-libc-types/once_flag.h --- a/libc/include/llvm-libc-types/once_flag.h +++ b/libc/include/llvm-libc-types/once_flag.h @@ -9,6 +9,12 @@ #ifndef __LLVM_LIBC_TYPES_ONCE_FLAG_H__ #define __LLVM_LIBC_TYPES_ONCE_FLAG_H__ -typedef unsigned int once_flag; +#include -#endif // __LLVM_LIBC_TYPES_ONCE_FLAg_H__ +#ifdef __unix__ +typedef __futex_word once_flag; +#else +#error "Once flag type not defined for the target platform." +#endif + +#endif // __LLVM_LIBC_TYPES_ONCE_FLAG_H__ diff --git a/libc/include/llvm-libc-types/thrd_t.h b/libc/include/llvm-libc-types/thrd_t.h --- a/libc/include/llvm-libc-types/thrd_t.h +++ b/libc/include/llvm-libc-types/thrd_t.h @@ -9,8 +9,10 @@ #ifndef __LLVM_LIBC_TYPES_THRD_T_H__ #define __LLVM_LIBC_TYPES_THRD_T_H__ +#include + typedef struct { - unsigned char __clear_tid[4]; + __futex_word __clear_tid; int __tid; void *__stack; int __stack_size; diff --git a/libc/src/__support/CPP/atomic.h b/libc/src/__support/CPP/atomic.h --- a/libc/src/__support/CPP/atomic.h +++ b/libc/src/__support/CPP/atomic.h @@ -43,6 +43,8 @@ // operations should be performed using the atomic methods however. alignas(ALIGNMENT) value_type val; + constexpr Atomic() = default; + // Intializes the value without using atomic operations. constexpr Atomic(value_type v) : val(v) {} @@ -73,6 +75,18 @@ int(mem_ord), int(mem_ord)); } + T exchange(T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST) { + return __atomic_exchange_n(&val, desired, int(mem_ord)); + } + + T fetch_add(T increment, MemoryOrder mem_ord = MemoryOrder::SEQ_CST) { + return __atomic_fetch_add(&val, increment, int(mem_ord)); + } + + T fetch_sub(T decrement, MemoryOrder mem_ord = MemoryOrder::SEQ_CST) { + return __atomic_fetch_sub(&val, decrement, int(mem_ord)); + } + // Set the value without using an atomic operation. This is useful // in initializing atomic values without a constructor. void set(T rhs) { val = rhs; } diff --git a/libc/src/threads/linux/CMakeLists.txt b/libc/src/threads/linux/CMakeLists.txt --- a/libc/src/threads/linux/CMakeLists.txt +++ b/libc/src/threads/linux/CMakeLists.txt @@ -18,6 +18,7 @@ .threads_utils libc.include.sys_syscall libc.include.threads + libc.src.__support.CPP.atomic libc.src.__support.OSUtil.osutil ) @@ -67,6 +68,7 @@ .threads_utils libc.include.sys_syscall libc.include.threads + libc.src.__support.CPP.atomic libc.src.__support.common libc.src.__support.OSUtil.osutil libc.src.sys.mman.munmap diff --git a/libc/src/threads/linux/Futex.h b/libc/src/threads/linux/Futex.h --- a/libc/src/threads/linux/Futex.h +++ b/libc/src/threads/linux/Futex.h @@ -9,19 +9,22 @@ #ifndef LLVM_LIBC_SRC_THREADS_LINUX_FUTEX_H #define LLVM_LIBC_SRC_THREADS_LINUX_FUTEX_H -#include +#include "src/__support/architectures.h" // Architecture macros namespace __llvm_libc { +#if (defined(LLVM_LIBC_ARCH_AARCH64) || defined(LLVM_LIBC_ARCH_X86_64)) // The futex data has to be exactly 4 bytes long. However, we use a uint type -// here as we do not want to use `_Atomic uint32_t` as the _Atomic keyword which -// is C only. The header stdatomic.h does not define an atomic type -// corresponding to `uint32_t` or to something which is exactly 4 bytes wide. -using FutexWord = atomic_uint; -static_assert(sizeof(atomic_uint) == 4, - "Size of the `atomic_uint` type is not 4 bytes on your platform. " - "The implementation of the standard threads library for linux " - "requires that size of `atomic_uint` be 4 bytes."); +// here as we do not want to use `uint32_t` type to match the public definitions +// of types which include a field for a futex word. With public definitions, we +// cannot include so we stick to the `unsigned int` type for x86_64 +// and aarch64 +using FutexWordType = unsigned int; +static_assert(sizeof(FutexWordType) == 4, + "Unexpected size of unsigned int type."); +#else +#error "Futex word base type not defined for the target architecture." +#endif } // namespace __llvm_libc diff --git a/libc/src/threads/linux/Mutex.h b/libc/src/threads/linux/Mutex.h --- a/libc/src/threads/linux/Mutex.h +++ b/libc/src/threads/linux/Mutex.h @@ -9,6 +9,8 @@ #ifndef LLVM_LIBC_SRC_THREADS_LINUX_MUTEX_H #define LLVM_LIBC_SRC_THREADS_LINUX_MUTEX_H +#include "Futex.h" + #include "include/sys/syscall.h" // For syscall numbers. #include "include/threads.h" // For values like thrd_success etc. #include "src/__support/CPP/atomic.h" // For atomics support @@ -18,14 +20,6 @@ namespace __llvm_libc { -#if (defined(LLVM_LIBC_ARCH_AARCH64) || defined(LLVM_LIBC_ARCH_X86_64)) -static_assert(sizeof(unsigned int) == 4, - "Unexpected size of unsigned int type."); -typedef unsigned int FutexWordType; -#else -#error "Futex word base type not defined for the target architecture." -#endif - struct Mutex { enum Status : FutexWordType { MS_Free, diff --git a/libc/src/threads/linux/Thread.h b/libc/src/threads/linux/Thread.h --- a/libc/src/threads/linux/Thread.h +++ b/libc/src/threads/linux/Thread.h @@ -11,7 +11,6 @@ #include "thread_start_args.h" -#include #include namespace __llvm_libc { diff --git a/libc/src/threads/linux/call_once.cpp b/libc/src/threads/linux/call_once.cpp --- a/libc/src/threads/linux/call_once.cpp +++ b/libc/src/threads/linux/call_once.cpp @@ -6,27 +6,32 @@ // //===----------------------------------------------------------------------===// -#include "src/threads/call_once.h" +#include "Futex.h" + #include "include/sys/syscall.h" // For syscall numbers. #include "include/threads.h" // For call_once related type definition. +#include "src/__support/CPP/atomic.h" #include "src/__support/OSUtil/syscall.h" // For syscall functions. #include "src/__support/common.h" +#include "src/threads/call_once.h" #include "src/threads/linux/Futex.h" #include #include -#include namespace __llvm_libc { -static constexpr unsigned START = 0x11; -static constexpr unsigned WAITING = 0x22; -static constexpr unsigned FINISH = 0x33; +static constexpr FutexWordType START = 0x11; +static constexpr FutexWordType WAITING = 0x22; +static constexpr FutexWordType FINISH = 0x33; +static constexpr once_flag ONCE_FLAG_INIT_VAL = ONCE_FLAG_INIT; LLVM_LIBC_FUNCTION(void, call_once, (once_flag * flag, __call_once_func_t func)) { - FutexWord *futex_word = reinterpret_cast(flag); - unsigned int not_called = ONCE_FLAG_INIT; + auto *futex_word = reinterpret_cast *>(flag); + static_assert(sizeof(*futex_word) == sizeof(once_flag)); + + FutexWordType not_called = ONCE_FLAG_INIT_VAL.__word; // The C standard wording says: // @@ -37,21 +42,21 @@ // What this means is that, the call_once call can return only after // the called function |func| returns. So, we use futexes to synchronize // calls with the same flag value. - if (::atomic_compare_exchange_strong(futex_word, ¬_called, START)) { + if (futex_word->compare_exchange_strong(not_called, START)) { func(); - auto status = ::atomic_exchange(futex_word, FINISH); + auto status = futex_word->exchange(FINISH); if (status == WAITING) { - __llvm_libc::syscall(SYS_futex, futex_word, FUTEX_WAKE_PRIVATE, + __llvm_libc::syscall(SYS_futex, &futex_word->val, FUTEX_WAKE_PRIVATE, INT_MAX, // Wake all waiters. 0, 0, 0); } return; } - unsigned int status = START; - if (::atomic_compare_exchange_strong(futex_word, &status, WAITING) || + FutexWordType status = START; + if (futex_word->compare_exchange_strong(status, WAITING) || status == WAITING) { - __llvm_libc::syscall(SYS_futex, futex_word, FUTEX_WAIT_PRIVATE, + __llvm_libc::syscall(SYS_futex, &futex_word->val, FUTEX_WAIT_PRIVATE, WAITING, // Block only if status is still |WAITING|. 0, 0, 0); } diff --git a/libc/src/threads/linux/thrd_create.cpp b/libc/src/threads/linux/thrd_create.cpp --- a/libc/src/threads/linux/thrd_create.cpp +++ b/libc/src/threads/linux/thrd_create.cpp @@ -6,7 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "src/threads/thrd_create.h" +#include "Futex.h" + #include "include/errno.h" // For E* error values. #include "include/sys/mman.h" // For PROT_* and MAP_* definitions. #include "include/sys/syscall.h" // For syscall numbers. @@ -16,8 +17,8 @@ #include "src/errno/llvmlibc_errno.h" #include "src/sys/mman/mmap.h" #include "src/sys/mman/munmap.h" -#include "src/threads/linux/Futex.h" #include "src/threads/linux/Thread.h" +#include "src/threads/thrd_create.h" #include // For CLONE_* flags. #include @@ -61,8 +62,7 @@ thread->__stack = stack; thread->__stack_size = ThreadParams::DEFAULT_STACK_SIZE; thread->__retval = -1; - FutexWord *clear_tid_address = - reinterpret_cast(thread->__clear_tid); + FutexWordType *clear_tid_address = &thread->__clear_tid.__word; *clear_tid_address = ThreadParams::CLEAR_TID_VALUE; // When the new thread is spawned by the kernel, the new thread gets the diff --git a/libc/src/threads/linux/thrd_join.cpp b/libc/src/threads/linux/thrd_join.cpp --- a/libc/src/threads/linux/thrd_join.cpp +++ b/libc/src/threads/linux/thrd_join.cpp @@ -6,31 +6,32 @@ // //===----------------------------------------------------------------------===// -#include "src/threads/thrd_join.h" -#include "include/sys/syscall.h" // For syscall numbers. -#include "include/threads.h" // For thrd_* type definitions. +#include "Futex.h" + +#include "include/sys/syscall.h" // For syscall numbers. +#include "include/threads.h" // For thrd_* type definitions. +#include "src/__support/CPP/atomic.h" #include "src/__support/OSUtil/syscall.h" // For syscall function. #include "src/__support/common.h" #include "src/sys/mman/munmap.h" -#include "src/threads/linux/Futex.h" #include "src/threads/linux/Thread.h" +#include "src/threads/thrd_join.h" #include // For futex operations. -#include // For atomic_load. namespace __llvm_libc { LLVM_LIBC_FUNCTION(int, thrd_join, (thrd_t * thread, int *retval)) { - FutexWord *clear_tid_address = - reinterpret_cast(thread->__clear_tid); + auto *clear_tid_address = + reinterpret_cast *>(&thread->__clear_tid); // The kernel should set the value at the clear tid address to zero. // If not, it is a spurious wake and we should continue to wait on // the futex. - while (atomic_load(clear_tid_address) != 0) { + while (clear_tid_address->load() != 0) { // We cannot do a FUTEX_WAIT_PRIVATE here as the kernel does a // FUTEX_WAKE and not a FUTEX_WAKE_PRIVATE. - __llvm_libc::syscall(SYS_futex, clear_tid_address, FUTEX_WAIT, + __llvm_libc::syscall(SYS_futex, &clear_tid_address->val, FUTEX_WAIT, ThreadParams::CLEAR_TID_VALUE, nullptr); } diff --git a/libc/test/src/threads/CMakeLists.txt b/libc/test/src/threads/CMakeLists.txt --- a/libc/test/src/threads/CMakeLists.txt +++ b/libc/test/src/threads/CMakeLists.txt @@ -15,6 +15,7 @@ libc.src.threads.mtx_unlock libc.src.threads.thrd_create libc.src.threads.thrd_join + libc.src.__support.CPP.atomic ) add_libc_unittest( diff --git a/libc/test/src/threads/call_once_test.cpp b/libc/test/src/threads/call_once_test.cpp --- a/libc/test/src/threads/call_once_test.cpp +++ b/libc/test/src/threads/call_once_test.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "include/threads.h" +#include "src/__support/CPP/atomic.h" #include "src/threads/call_once.h" #include "src/threads/mtx_destroy.h" #include "src/threads/mtx_init.h" @@ -16,10 +17,8 @@ #include "src/threads/thrd_join.h" #include "utils/UnitTest/Test.h" -#include - static constexpr unsigned int NUM_THREADS = 5; -static atomic_uint thread_count; +static __llvm_libc::cpp::Atomic thread_count; static unsigned int call_count; static void call_once_func() { ++call_count; } @@ -28,7 +27,7 @@ static once_flag flag = ONCE_FLAG_INIT; __llvm_libc::call_once(&flag, call_once_func); - ++thread_count; // This is a an atomic update. + thread_count.fetch_add(1); return 0; } @@ -51,7 +50,7 @@ ASSERT_EQ(retval, 0); } - EXPECT_EQ(static_cast(thread_count), 5U); + EXPECT_EQ(thread_count.val, 5U); EXPECT_EQ(call_count, 1U); } @@ -61,13 +60,13 @@ __llvm_libc::mtx_unlock(&once_func_blocker); } -static atomic_uint start_count; -static atomic_uint done_count; +static __llvm_libc::cpp::Atomic start_count; +static __llvm_libc::cpp::Atomic done_count; static int once_func_caller(void *) { static once_flag flag; - ++start_count; + start_count.fetch_add(1); __llvm_libc::call_once(&flag, blocking_once_func); - ++done_count; + done_count.fetch_add(1); return 0; } @@ -90,11 +89,11 @@ ASSERT_EQ(__llvm_libc::thrd_create(&t2, once_func_caller, nullptr), static_cast(thrd_success)); - while (start_count != 2) + while (start_count.load() != 2) ; // Spin until both threads start. // Since the once func is blocked, the threads should not be done yet. - EXPECT_EQ(static_cast(done_count), 0U); + EXPECT_EQ(done_count.val, 0U); // Unlock the blocking mutex so that the once func blocks. ASSERT_EQ(__llvm_libc::mtx_unlock(&once_func_blocker), @@ -108,7 +107,7 @@ static_cast(thrd_success)); ASSERT_EQ(retval, 0); - ASSERT_EQ(static_cast(done_count), 2U); + ASSERT_EQ(done_count.val, 2U); __llvm_libc::mtx_destroy(&once_func_blocker); } diff --git a/libc/test/src/threads/cnd_test.cpp b/libc/test/src/threads/cnd_test.cpp --- a/libc/test/src/threads/cnd_test.cpp +++ b/libc/test/src/threads/cnd_test.cpp @@ -6,9 +6,8 @@ // //===----------------------------------------------------------------------===// -#include - #include "include/threads.h" +#include "src/__support/CPP/atomic.h" #include "src/threads/cnd_broadcast.h" #include "src/threads/cnd_destroy.h" #include "src/threads/cnd_init.h" @@ -34,15 +33,15 @@ // |broadcast_count| by 1 before they start waiting on |broadcast_cnd|, and // decrement it by 1 after getting signalled on |broadcast_cnd|. -constexpr int THRD_COUNT = 10000; +constexpr unsigned int THRD_COUNT = 10000; -static atomic_uint broadcast_count = 0; +static __llvm_libc::cpp::Atomic broadcast_count(0); static cnd_t broadcast_cnd, threads_ready_cnd; static mtx_t broadcast_mtx, threads_ready_mtx; int broadcast_thread_func(void *) { __llvm_libc::mtx_lock(&broadcast_mtx); - int oldval = atomic_fetch_add(&broadcast_count, 1); + int oldval = broadcast_count.fetch_add(1); if (oldval == THRD_COUNT - 1) { __llvm_libc::mtx_lock(&threads_ready_mtx); __llvm_libc::cnd_signal(&threads_ready_cnd); @@ -51,7 +50,7 @@ __llvm_libc::cnd_wait(&broadcast_cnd, &broadcast_mtx); __llvm_libc::mtx_unlock(&broadcast_mtx); - atomic_fetch_sub(&broadcast_count, 1); + broadcast_count.fetch_sub(1); return 0; } @@ -70,7 +69,7 @@ __llvm_libc::mtx_unlock(&threads_ready_mtx); __llvm_libc::mtx_lock(&broadcast_mtx); - ASSERT_EQ(int(broadcast_count), THRD_COUNT); + ASSERT_EQ(broadcast_count.val, THRD_COUNT); __llvm_libc::cnd_broadcast(&broadcast_cnd); __llvm_libc::mtx_unlock(&broadcast_mtx); @@ -80,7 +79,7 @@ ASSERT_EQ(retval, 0); } - ASSERT_EQ(int(broadcast_count), 0); + ASSERT_EQ(broadcast_count.val, 0U); __llvm_libc::cnd_destroy(&broadcast_cnd); __llvm_libc::cnd_destroy(&threads_ready_cnd);