diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td --- a/libc/config/linux/api.td +++ b/libc/config/linux/api.td @@ -234,6 +234,7 @@ def PThreadAPI : PublicAPI<"pthread.h"> { let Types = [ + "__pthread_once_func_t", "__pthread_start_t", "__pthread_tss_dtor_t", "pthread_attr_t", @@ -241,6 +242,7 @@ "pthread_mutexattr_t", "pthread_t", "pthread_key_t", + "pthread_once_t", ]; } diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -321,6 +321,7 @@ libc.src.pthread.pthread_mutexattr_setpshared libc.src.pthread.pthread_mutexattr_setrobust libc.src.pthread.pthread_mutexattr_settype + libc.src.pthread.pthread_once libc.src.pthread.pthread_setspecific # stdio.h entrypoints diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt --- a/libc/include/CMakeLists.txt +++ b/libc/include/CMakeLists.txt @@ -188,6 +188,8 @@ .llvm-libc-types.pthread_key_t .llvm-libc-types.pthread_mutex_t .llvm-libc-types.pthread_mutexattr_t + .llvm-libc-types.pthread_once_t + .llvm-libc-types.__pthread_once_func_t .llvm-libc-types.pthread_t ) diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt --- a/libc/include/llvm-libc-types/CMakeLists.txt +++ b/libc/include/llvm-libc-types/CMakeLists.txt @@ -4,6 +4,7 @@ add_header(__call_once_func_t HDR __call_once_func_t.h) add_header(__futex_word HDR __futex_word.h) add_header(__mutex_type HDR __mutex_type.h DEPENDS .__futex_word) +add_header(__pthread_once_func_t HDR __pthread_once_func_t.h) add_header(__pthread_start_t HDR __pthread_start_t.h) add_header(__pthread_tss_dtor_t HDR __pthread_tss_dtor_t.h) add_header(__qsortcompare_t HDR __qsortcompare_t.h) @@ -38,6 +39,7 @@ add_header(pthread_mutex_t HDR pthread_mutex_t.h DEPENDS .__futex_word .__mutex_type) add_header(pthread_t HDR pthread_t.h DEPENDS .__thread_type) add_header(pthread_mutexattr_t HDR pthread_mutexattr_t.h) +add_header(pthread_once_t HDR pthread_once_t.h DEPENDS .__futex_word) add_header(rlim_t HDR rlim_t.h) add_header(struct_rlimit HDR struct_rlimit.h DEPENDS .rlim_t) add_header(ssize_t HDR ssize_t.h) diff --git a/libc/include/llvm-libc-types/__pthread_once_func_t.h b/libc/include/llvm-libc-types/__pthread_once_func_t.h new file mode 100644 --- /dev/null +++ b/libc/include/llvm-libc-types/__pthread_once_func_t.h @@ -0,0 +1,14 @@ +//===-- Definition of __pthread_once_func_t type --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __LLVM_LIBC_TYPES_PTHREAD_ONCE_FUNC_T_H__ +#define __LLVM_LIBC_TYPES_PTHREAD_ONCE_FUNC_T_H__ + +typedef void (*__pthread_once_func_t)(void); + +#endif // __LLVM_LIBC_TYPES_PTHREAD_ONCE_FUNC_T_H__ diff --git a/libc/include/llvm-libc-types/pthread_once_t.h b/libc/include/llvm-libc-types/pthread_once_t.h new file mode 100644 --- /dev/null +++ b/libc/include/llvm-libc-types/pthread_once_t.h @@ -0,0 +1,20 @@ +//===-- Definition of pthread_once_t type ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __LLVM_LIBC_TYPES_PTHREAD_ONCE_T_H__ +#define __LLVM_LIBC_TYPES_PTHREAD_ONCE_T_H__ + +#include + +#ifdef __unix__ +typedef __futex_word pthread_once_t; +#else +#error "Once flag type not defined for the target platform." +#endif + +#endif // __LLVM_LIBC_TYPES_PTHREAD_ONCE_T_H__ diff --git a/libc/include/pthread.h.def b/libc/include/pthread.h.def --- a/libc/include/pthread.h.def +++ b/libc/include/pthread.h.def @@ -13,6 +13,8 @@ #define PTHREAD_STACK_MIN (1 << 14) // 16KB +#define PTHREAD_ONCE_INIT {0} + enum { PTHREAD_CREATE_JOINABLE = 0x0, PTHREAD_CREATE_DETACHED = 0x1, diff --git a/libc/spec/posix.td b/libc/spec/posix.td --- a/libc/spec/posix.td +++ b/libc/spec/posix.td @@ -14,6 +14,9 @@ def PThreadTSSDtorT : NamedType<"__pthread_tss_dtor_t">; def PThreadKeyT : NamedType<"pthread_key_t">; def PThreadKeyTPtr : PtrType; +def PThreadOnceT : NamedType<"pthread_once_t">; +def PThreadOnceTPtr : PtrType; +def PThreadOnceCallback : NamedType<"__pthread_once_func_t">; def InoT : NamedType<"ino_t">; def UidT : NamedType<"uid_t">; @@ -675,6 +678,8 @@ PThreadKeyT, PThreadMutexAttrTType, PThreadMutexTType, + PThreadOnceCallback, + PThreadOnceT, PThreadStartT, PThreadTSSDtorT, PThreadTType, @@ -861,6 +866,11 @@ RetValSpec, [ArgSpec, ArgSpec] >, + FunctionSpec< + "pthread_once", + RetValSpec, + [ArgSpec, ArgSpec] + >, ] >; diff --git a/libc/src/__support/threads/CMakeLists.txt b/libc/src/__support/threads/CMakeLists.txt --- a/libc/src/__support/threads/CMakeLists.txt +++ b/libc/src/__support/threads/CMakeLists.txt @@ -43,3 +43,17 @@ libc.src.__support.CPP.optional ) endif() + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS}/callonce.cpp) + add_object_library( + callonce + SRCS + ${LIBC_TARGET_OS}/callonce.cpp + HDRS + callonce.h + DEPENDS + libc.include.sys_syscall + libc.src.__support.CPP.atomic + libc.src.__support.OSUtil.osutil + ) +endif() diff --git a/libc/src/threads/call_once.h b/libc/src/__support/threads/callonce.h copy from libc/src/threads/call_once.h copy to libc/src/__support/threads/callonce.h --- a/libc/src/threads/call_once.h +++ b/libc/src/__support/threads/callonce.h @@ -1,4 +1,4 @@ -//===-- Implementation header for call_once function ------------*- C++ -*-===// +//===-- Types related to the callonce function ----------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,15 +6,11 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_THREADS_CALL_ONCE_H -#define LLVM_LIBC_SRC_THREADS_CALL_ONCE_H - -#include "include/threads.h" - namespace __llvm_libc { -void call_once(once_flag *flag, __call_once_func_t func); +struct CallOnceFlag; +using CallOnceCallback = void(void); -} // namespace __llvm_libc +int callonce(CallOnceFlag *flag, CallOnceCallback *callback); -#endif // LLVM_LIBC_SRC_THREADS_CALL_ONCE_H +} // namespace __llvm_libc diff --git a/libc/src/__support/threads/linux/callonce.cpp b/libc/src/__support/threads/linux/callonce.cpp new file mode 100644 --- /dev/null +++ b/libc/src/__support/threads/linux/callonce.cpp @@ -0,0 +1,55 @@ +//===-- Linux implementation of the callonce function ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "futex_word.h" + +#include "include/sys/syscall.h" // For syscall numbers. +#include "src/__support/CPP/atomic.h" +#include "src/__support/OSUtil/syscall.h" // For syscall functions. +#include "src/__support/threads/callonce.h" + +#include +#include + +namespace __llvm_libc { + +static constexpr FutexWordType NOT_CALLED = 0x0; +static constexpr FutexWordType START = 0x11; +static constexpr FutexWordType WAITING = 0x22; +static constexpr FutexWordType FINISH = 0x33; + +int callonce(CallOnceFlag *flag, CallOnceCallback *func) { + auto *futex_word = reinterpret_cast *>(flag); + + FutexWordType not_called = NOT_CALLED; + + // The call_once call can return only after the called function |func| + // returns. So, we use futexes to synchronize calls with the same flag value. + if (futex_word->compare_exchange_strong(not_called, START)) { + func(); + auto status = futex_word->exchange(FINISH); + if (status == WAITING) { + __llvm_libc::syscall(SYS_futex, &futex_word->val, FUTEX_WAKE_PRIVATE, + INT_MAX, // Wake all waiters. + 0, 0, 0); + } + return 0; + } + + FutexWordType status = START; + if (futex_word->compare_exchange_strong(status, WAITING) || + status == WAITING) { + __llvm_libc::syscall(SYS_futex, &futex_word->val, FUTEX_WAIT_PRIVATE, + WAITING, // Block only if status is still |WAITING|. + 0, 0, 0); + } + + return 0; +} + +} // namespace __llvm_libc diff --git a/libc/src/pthread/CMakeLists.txt b/libc/src/pthread/CMakeLists.txt --- a/libc/src/pthread/CMakeLists.txt +++ b/libc/src/pthread/CMakeLists.txt @@ -386,3 +386,14 @@ libc.include.pthread libc.src.__support.threads.thread ) + +add_entrypoint_object( + pthread_once + SRCS + pthread_once.cpp + HDRS + pthread_once.h + DEPENDS + libc.include.pthread + libc.src.__support.threads.callonce +) diff --git a/libc/src/threads/call_once.h b/libc/src/pthread/pthread_once.h copy from libc/src/threads/call_once.h copy to libc/src/pthread/pthread_once.h --- a/libc/src/threads/call_once.h +++ b/libc/src/pthread/pthread_once.h @@ -1,4 +1,4 @@ -//===-- Implementation header for call_once function ------------*- C++ -*-===// +//===-- Implementation header for pthread_once function ---------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,15 +6,15 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_SRC_THREADS_CALL_ONCE_H -#define LLVM_LIBC_SRC_THREADS_CALL_ONCE_H +#ifndef LLVM_LIBC_SRC_THREADS_PTHREAD_ONCE_H +#define LLVM_LIBC_SRC_THREADS_PTHREAD_ONCE_H -#include "include/threads.h" +#include namespace __llvm_libc { -void call_once(once_flag *flag, __call_once_func_t func); +int pthread_once(pthread_once_t *flag, __pthread_once_func_t func); } // namespace __llvm_libc -#endif // LLVM_LIBC_SRC_THREADS_CALL_ONCE_H +#endif // LLVM_LIBC_SRC_THREADS_PTHREAD_ONCE_H diff --git a/libc/src/pthread/pthread_once.cpp b/libc/src/pthread/pthread_once.cpp new file mode 100644 --- /dev/null +++ b/libc/src/pthread/pthread_once.cpp @@ -0,0 +1,23 @@ +//===-- Linux implementation of the pthread_once function -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "pthread_once.h" +#include "src/__support/common.h" +#include "src/__support/threads/callonce.h" + +#include // For pthread_once_t and __pthread_once_func_t definitions. + +namespace __llvm_libc { + +LLVM_LIBC_FUNCTION(int, pthread_once, + (pthread_once_t * flag, __pthread_once_func_t func)) { + return callonce(reinterpret_cast(flag), + reinterpret_cast(func)); +} + +} // namespace __llvm_libc diff --git a/libc/src/threads/CMakeLists.txt b/libc/src/threads/CMakeLists.txt --- a/libc/src/threads/CMakeLists.txt +++ b/libc/src/threads/CMakeLists.txt @@ -4,9 +4,13 @@ add_entrypoint_object( call_once - ALIAS + SRCS + call_once.cpp + HDRS + call_once.h DEPENDS - .${LIBC_TARGET_OS}.call_once + libc.include.threads + libc.src.__support.threads.callonce ) add_entrypoint_object( diff --git a/libc/src/threads/call_once.h b/libc/src/threads/call_once.h --- a/libc/src/threads/call_once.h +++ b/libc/src/threads/call_once.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SRC_THREADS_CALL_ONCE_H #define LLVM_LIBC_SRC_THREADS_CALL_ONCE_H -#include "include/threads.h" +#include namespace __llvm_libc { diff --git a/libc/src/threads/call_once.cpp b/libc/src/threads/call_once.cpp new file mode 100644 --- /dev/null +++ b/libc/src/threads/call_once.cpp @@ -0,0 +1,23 @@ +//===-- Linux implementation of the call_once function --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/threads/call_once.h" +#include "src/__support/common.h" +#include "src/__support/threads/callonce.h" + +#include // For once_flag and __call_once_func_t definitions. + +namespace __llvm_libc { + +LLVM_LIBC_FUNCTION(void, call_once, + (once_flag * flag, __call_once_func_t func)) { + callonce(reinterpret_cast(flag), + reinterpret_cast(func)); +} + +} // namespace __llvm_libc diff --git a/libc/src/threads/linux/CMakeLists.txt b/libc/src/threads/linux/CMakeLists.txt --- a/libc/src/threads/linux/CMakeLists.txt +++ b/libc/src/threads/linux/CMakeLists.txt @@ -1,17 +1,3 @@ -add_entrypoint_object( - call_once - SRCS - call_once.cpp - HDRS - ../call_once.h - DEPENDS - .threads_utils - libc.include.sys_syscall - libc.include.threads - libc.src.__support.CPP.atomic - libc.src.__support.OSUtil.osutil -) - add_header_library( threads_utils HDRS diff --git a/libc/test/integration/src/pthread/CMakeLists.txt b/libc/test/integration/src/pthread/CMakeLists.txt --- a/libc/test/integration/src/pthread/CMakeLists.txt +++ b/libc/test/integration/src/pthread/CMakeLists.txt @@ -110,3 +110,23 @@ libc.src.pthread.pthread_getspecific libc.src.pthread.pthread_setspecific ) + +add_integration_test( + pthread_once_test + SUITE + libc-pthread-integration-tests + SRCS + pthread_once_test.cpp + LOADER + libc.loader.linux.crt1 + DEPENDS + libc.include.pthread + libc.src.pthread.pthread_once + libc.src.pthread.pthread_mutex_destroy + libc.src.pthread.pthread_mutex_init + libc.src.pthread.pthread_mutex_lock + libc.src.pthread.pthread_mutex_unlock + libc.src.pthread.pthread_create + libc.src.pthread.pthread_join + libc.src.__support.CPP.atomic +) diff --git a/libc/test/integration/src/pthread/pthread_once_test.cpp b/libc/test/integration/src/pthread/pthread_once_test.cpp new file mode 100644 --- /dev/null +++ b/libc/test/integration/src/pthread/pthread_once_test.cpp @@ -0,0 +1,115 @@ +//===-- Tests for pthread_once --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/atomic.h" +#include "src/pthread/pthread_create.h" +#include "src/pthread/pthread_join.h" +#include "src/pthread/pthread_mutex_destroy.h" +#include "src/pthread/pthread_mutex_init.h" +#include "src/pthread/pthread_mutex_lock.h" +#include "src/pthread/pthread_mutex_unlock.h" +#include "src/pthread/pthread_once.h" + +#include "utils/IntegrationTest/test.h" + +#include + +static constexpr unsigned int NUM_THREADS = 5; +static __llvm_libc::cpp::Atomic thread_count; + +static unsigned int call_count; +static void pthread_once_func() { ++call_count; } + +static void *func(void *) { + static pthread_once_t flag = PTHREAD_ONCE_INIT; + __llvm_libc::pthread_once(&flag, pthread_once_func); + + thread_count.fetch_add(1); + + return nullptr; +} + +void call_from_5_threads() { + // Ensure the call count and thread count are 0 to begin with. + call_count = 0; + thread_count = 0; + + pthread_t threads[NUM_THREADS]; + for (unsigned int i = 0; i < NUM_THREADS; ++i) { + ASSERT_EQ(__llvm_libc::pthread_create(threads + i, nullptr, func, nullptr), + 0); + } + + for (unsigned int i = 0; i < NUM_THREADS; ++i) { + void *retval; + ASSERT_EQ(__llvm_libc::pthread_join(threads[i], &retval), 0); + ASSERT_EQ(uintptr_t(retval), uintptr_t(0)); + } + + EXPECT_EQ(thread_count.val, 5U); + EXPECT_EQ(call_count, 1U); +} + +static pthread_mutex_t once_func_blocker; +static void blocking_once_func() { + __llvm_libc::pthread_mutex_lock(&once_func_blocker); + __llvm_libc::pthread_mutex_unlock(&once_func_blocker); +} + +static __llvm_libc::cpp::Atomic start_count; +static __llvm_libc::cpp::Atomic done_count; +static void *once_func_caller(void *) { + static pthread_once_t flag; + start_count.fetch_add(1); + __llvm_libc::pthread_once(&flag, blocking_once_func); + done_count.fetch_add(1); + return nullptr; +} + +// Test the synchronization aspect of the pthread_once function. +// This is not a fool proof test, but something which might be +// useful when we add a flakiness detection scheme to UnitTest. +void test_synchronization() { + start_count = 0; + done_count = 0; + + ASSERT_EQ(__llvm_libc::pthread_mutex_init(&once_func_blocker, nullptr), 0); + // Lock the blocking mutex so that the once func blocks. + ASSERT_EQ(__llvm_libc::pthread_mutex_lock(&once_func_blocker), 0); + + pthread_t t1, t2; + ASSERT_EQ( + __llvm_libc::pthread_create(&t1, nullptr, once_func_caller, nullptr), 0); + ASSERT_EQ( + __llvm_libc::pthread_create(&t2, nullptr, once_func_caller, nullptr), 0); + + while (start_count.load() != 2) + ; // Spin until both threads start. + + // Since the once func is blocked, the threads should not be done yet. + EXPECT_EQ(done_count.val, 0U); + + // Unlock the blocking mutex so that the once func blocks. + ASSERT_EQ(__llvm_libc::pthread_mutex_unlock(&once_func_blocker), 0); + + void *retval; + ASSERT_EQ(__llvm_libc::pthread_join(t1, &retval), uintptr_t(0)); + ASSERT_EQ(uintptr_t(retval), 0); + ASSERT_EQ(__llvm_libc::pthread_join(t2, &retval), uintptr_t(0)); + ASSERT_EQ(uintptr_t(retval), 0); + + ASSERT_EQ(done_count.val, 2U); + + __llvm_libc::pthread_mutex_destroy(&once_func_blocker); +} + +TEST_MAIN() { + call_from_5_threads(); + test_synchronization(); + return 0; +}