diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt --- a/libc/include/llvm-libc-types/CMakeLists.txt +++ b/libc/include/llvm-libc-types/CMakeLists.txt @@ -1,6 +1,7 @@ add_header(__bsearchcompare_t HDR __bsearchcompare_t.h) add_header(__call_once_func_t HDR __call_once_func_t.h) add_header(__futex_word HDR __futex_word.h) +add_header(__mutex_type HDR __mutex_type.h) add_header(__qsortcompare_t HDR __qsortcompare_t.h) add_header(__sighandler_t HDR __sighandler_t.h) add_header(cnd_t HDR cnd_t.h) @@ -14,7 +15,7 @@ add_header(float_t HDR float_t.h) add_header(imaxdiv_t HDR imaxdiv_t.h) add_header(mode_t HDR mode_t.h) -add_header(mtx_t HDR mtx_t.h DEPENDS .__futex_word) +add_header(mtx_t HDR mtx_t.h DEPENDS .__futex_word .__mutex_type) add_header(off_t HDR off_t.h) add_header(once_flag HDR once_flag.h DEPENDS .__futex_word) add_header(size_t HDR size_t.h) diff --git a/libc/include/llvm-libc-types/mtx_t.h b/libc/include/llvm-libc-types/__mutex_type.h copy from libc/include/llvm-libc-types/mtx_t.h copy to libc/include/llvm-libc-types/__mutex_type.h --- a/libc/include/llvm-libc-types/mtx_t.h +++ b/libc/include/llvm-libc-types/__mutex_type.h @@ -1,4 +1,4 @@ -//===-- Definition of mtx_t type ------------------------------------------===// +//===-- Definition of a common mutex type ---------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,18 +6,19 @@ // //===----------------------------------------------------------------------===// -#ifndef __LLVM_LIBC_TYPES_MTX_T_H__ -#define __LLVM_LIBC_TYPES_MTX_T_H__ - #include typedef struct { + unsigned char __timed; + unsigned char __recursive; + unsigned char __robust; + + void *__owner; + unsigned long long __lock_count; + #ifdef __unix__ __futex_word __ftxw; #else -#error "mtx_t type not defined for the target platform." +#error "Mutex type not defined for the target platform." #endif - int __mtx_type; -} mtx_t; - -#endif // __LLVM_LIBC_TYPES_MTX_T_H__ +} __mutex_type; diff --git a/libc/include/llvm-libc-types/mtx_t.h b/libc/include/llvm-libc-types/mtx_t.h --- a/libc/include/llvm-libc-types/mtx_t.h +++ b/libc/include/llvm-libc-types/mtx_t.h @@ -9,15 +9,8 @@ #ifndef __LLVM_LIBC_TYPES_MTX_T_H__ #define __LLVM_LIBC_TYPES_MTX_T_H__ -#include +#include -typedef struct { -#ifdef __unix__ - __futex_word __ftxw; -#else -#error "mtx_t type not defined for the target platform." -#endif - int __mtx_type; -} mtx_t; +typedef __mutex_type mtx_t; #endif // __LLVM_LIBC_TYPES_MTX_T_H__ diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt --- a/libc/src/__support/CMakeLists.txt +++ b/libc/src/__support/CMakeLists.txt @@ -57,3 +57,4 @@ add_subdirectory(File) add_subdirectory(FPUtil) add_subdirectory(OSUtil) +add_subdirectory(threads) diff --git a/libc/src/__support/threads/CMakeLists.txt b/libc/src/__support/threads/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/libc/src/__support/threads/CMakeLists.txt @@ -0,0 +1,11 @@ +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS}) + add_subdirectory(${LIBC_TARGET_OS}) +endif() + +add_header_library( + thread + HDRS + mutex.h + DEPS + .${LIBC_TARGET_OS}.thread +) diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/libc/src/__support/threads/linux/CMakeLists.txt @@ -0,0 +1,9 @@ +add_header_library( + thread + HDRS + mutex.h + DEPENDS + libc.include.sys_syscall + libc.src.__support.CPP.atomic + libc.src.__support.OSUtil.osutil +) diff --git a/libc/src/__support/threads/linux/mutex.h b/libc/src/__support/threads/linux/mutex.h new file mode 100644 --- /dev/null +++ b/libc/src/__support/threads/linux/mutex.h @@ -0,0 +1,137 @@ +#ifndef LLVM_LIBC_SRC_SUPPORT_THREAD_LINUX_MUTEX_H +#define LLVM_LIBC_SRC_SUPPORT_THREAD_LINUX_MUTEX_H + +#include "include/sys/syscall.h" // For syscall numbers. +#include "src/__support/CPP/atomic.h" +#include "src/__support/OSUtil/syscall.h" // For syscall functions. +#include "src/__support/threads/mutex.h" + +#include +#include +#include + +namespace __llvm_libc { + +struct Mutex { + unsigned char timed; + unsigned char recursive; + unsigned char robust; + + void *owner; + unsigned long long lock_count; + + using FutexWordType = unsigned int; + + cpp::Atomic futex_word; + + enum class LockState : FutexWordType { + Free, + Locked, + Waiting, + }; + +public: + constexpr Mutex(bool istimed, bool isrecursive, bool isrobust) + : timed(istimed), recursive(isrecursive), robust(isrobust), + owner(nullptr), lock_count(0), + futex_word(FutexWordType(LockState::Free)) {} + + static MutexError init(Mutex *mutex, bool istimed, bool isrecur, + bool isrobust) { + mutex->timed = istimed; + mutex->recursive = isrecur; + mutex->robust = isrobust; + mutex->owner = nullptr; + mutex->lock_count = 0; + mutex->futex_word.set(FutexWordType(LockState::Free)); + return MutexError::NONE; + } + + static MutexError init(mtx_t *m, bool istimed, bool isrecur, bool isrobust) { + auto *mutex = reinterpret_cast(m); + return init(mutex, istimed, isrecur, isrobust); + } + + static MutexError destroy(mtx_t *) { return MutexError::NONE; } + + MutexError reset(); + + MutexError lock() { + bool was_waiting = false; + while (true) { + FutexWordType mutex_status = FutexWordType(LockState::Free); + FutexWordType locked_status = FutexWordType(LockState::Locked); + + if (futex_word.compare_exchange_strong( + mutex_status, FutexWordType(LockState::Locked))) { + if (was_waiting) + futex_word = FutexWordType(LockState::Waiting); + return MutexError::NONE; + } + + switch (LockState(mutex_status)) { + case LockState::Waiting: + // If other threads are waiting already, then join them. Note that the + // futex syscall will block if the futex data is still + // `LockState::Waiting` (the 4th argument to the syscall function + // below.) + __llvm_libc::syscall(SYS_futex, &futex_word.val, FUTEX_WAIT_PRIVATE, + FutexWordType(LockState::Waiting), 0, 0, 0); + was_waiting = true; + // Once woken up/unblocked, try everything all over. + continue; + case LockState::Locked: + // Mutex has been locked by another thread so set the status to + // LockState::Waiting. + if (futex_word.compare_exchange_strong( + locked_status, FutexWordType(LockState::Waiting))) { + // If we are able to set the futex data to `LockState::Waiting`, then + // we will wait for the futex to be woken up. Note again that the + // following syscall will block only if the futex data is still + // `LockState::Waiting`. + __llvm_libc::syscall(SYS_futex, &futex_word, FUTEX_WAIT_PRIVATE, + FutexWordType(LockState::Waiting), 0, 0, 0); + was_waiting = true; + } + continue; + case LockState::Free: + // If it was LockState::Free, we shouldn't be here at all. + [[clang::fallthrough]]; + default: + // Mutex status cannot be anything else. So control should not reach + // here at all. + return MutexError::BAD_LOCK_STATE; + } + } + } + + MutexError unlock() { + while (true) { + FutexWordType mutex_status = FutexWordType(LockState::Waiting); + if (futex_word.compare_exchange_strong(mutex_status, + FutexWordType(LockState::Free))) { + // If any thread is waiting to be woken up, then do it. + __llvm_libc::syscall(SYS_futex, &futex_word, FUTEX_WAKE_PRIVATE, 1, 0, + 0, 0); + return MutexError::NONE; + } + + if (mutex_status == FutexWordType(LockState::Locked)) { + // If nobody was waiting at this point, just free it. + if (futex_word.compare_exchange_strong(mutex_status, + FutexWordType(LockState::Free))) + return MutexError::NONE; + } else { + // This can happen, for example if some thread tries to unlock an + // already free mutex. + return MutexError::UNLOCK_WITHOUT_LOCK; + } + } + } + + MutexError trylock(); +}; + +} // namespace __llvm_libc + +#endif // LLVM_LIBC_SRC_SUPPORT_THREAD_LINUX_MUTEX_H diff --git a/libc/src/__support/threads/mutex.h b/libc/src/__support/threads/mutex.h new file mode 100644 --- /dev/null +++ b/libc/src/__support/threads/mutex.h @@ -0,0 +1,64 @@ +#ifndef LLVM_LIBC_SRC_SUPPORT_THREAD_MUTEX_H +#define LLVM_LIBC_SRC_SUPPORT_THREAD_MUTEX_H + +namespace __llvm_libc { + +enum class MutexError : int { + NONE, + BUSY, + TIMEOUT, + UNLOCK_WITHOUT_LOCK, + BAD_LOCK_STATE, +}; + +} // namespace __llvm_libc + +// Platform independent code will include this header file which pulls +// the platfrom specific specializations using platform macros. +// +// The platform specific specializations should define a class by name +// Mutex with non-static methods having the following signature: +// +// MutexError lock(); +// MutexError trylock(); +// MutexError timedlock(...); +// MutexError unlock(); +// MutexError reset(); // Used to reset inconsistent robust mutexes. +// +// Apart from the above non-static methods, the specializations should +// also provide few static methods with the following signature: +// +// static MutexError init(mtx_t *); +// static MutexError destroy(mtx_t *); +// +// All of the static and non-static methods should ideally be implemented +// as inline functions so that implementations of public functions can +// call them without a function call overhead. +// +// Another point to keep in mind that is that the libc internally needs a +// few global locks. So, to avoid static initialization order fiasco, we +// want the constructors of the Mutex classes to be constexprs. + +#ifdef __unix__ +#include "linux/mutex.h" +#endif // __unix__ + +namespace __llvm_libc { + +static_assert(sizeof(Mutex) <= sizeof(mtx_t), + "The public mtx_t type cannot accommodate the internal mutex " + "type."); + +// An RAII class for easy locking and unlocking of mutexes. +class MutexLock { + Mutex *mutex; + +public: + explicit MutexLock(Mutex *m) : mutex(m) { mutex->lock(); } + + ~MutexLock() { mutex->unlock(); } +}; + +} // namespace __llvm_libc + +#endif // LLVM_LIBC_SRC_SUPPORT_THREAD_MUTEX_H diff --git a/libc/src/threads/CMakeLists.txt b/libc/src/threads/CMakeLists.txt --- a/libc/src/threads/CMakeLists.txt +++ b/libc/src/threads/CMakeLists.txt @@ -25,30 +25,46 @@ add_entrypoint_object( mtx_init - ALIAS + SRCS + mtx_init.cpp + HDRS + mtx_init.h DEPENDS - .${LIBC_TARGET_OS}.mtx_init + libc.include.threads + libc.src.__support.threads.thread ) add_entrypoint_object( mtx_destroy - ALIAS + SRCS + mtx_destroy.cpp + HDRS + mtx_destroy.h DEPENDS - .${LIBC_TARGET_OS}.mtx_destroy + libc.include.threads + libc.src.__support.threads.thread ) add_entrypoint_object( mtx_lock - ALIAS + SRCS + mtx_lock.cpp + HDRS + mtx_lock.h DEPENDS - .${LIBC_TARGET_OS}.mtx_lock + libc.include.threads + libc.src.__support.threads.thread ) add_entrypoint_object( mtx_unlock - ALIAS + SRCS + mtx_unlock.cpp + HDRS + mtx_unlock.h DEPENDS - .${LIBC_TARGET_OS}.mtx_unlock + libc.include.threads + libc.src.__support.threads.thread ) add_entrypoint_object( diff --git a/libc/src/threads/linux/CMakeLists.txt b/libc/src/threads/linux/CMakeLists.txt --- a/libc/src/threads/linux/CMakeLists.txt +++ b/libc/src/threads/linux/CMakeLists.txt @@ -27,7 +27,6 @@ HDRS CndVar.h Futex.h - Mutex.h Thread.h DEPENDS .thread_start_args_h @@ -35,6 +34,7 @@ libc.include.threads libc.src.__support.CPP.atomic libc.src.__support.OSUtil.osutil + libc.src.__support.threads.thread ) add_entrypoint_object( @@ -74,50 +74,6 @@ libc.src.sys.mman.munmap ) -add_entrypoint_object( - mtx_init - SRCS - mtx_init.cpp - HDRS - ../mtx_init.h - DEPENDS - .threads_utils - libc.include.threads -) - -add_entrypoint_object( - mtx_destroy - SRCS - mtx_destroy.cpp - HDRS - ../mtx_destroy.h - DEPENDS - .threads_utils - libc.include.threads -) - -add_entrypoint_object( - mtx_lock - SRCS - mtx_lock.cpp - HDRS - ../mtx_lock.h - DEPENDS - .threads_utils - libc.include.threads -) - -add_entrypoint_object( - mtx_unlock - SRCS - mtx_unlock.cpp - HDRS - ../mtx_unlock.h - DEPENDS - .threads_utils - libc.include.threads -) - add_entrypoint_object( cnd_init SRCS @@ -149,6 +105,7 @@ DEPENDS .threads_utils libc.include.threads + libc.src.__support.threads.thread ) add_entrypoint_object( diff --git a/libc/src/threads/linux/CndVar.h b/libc/src/threads/linux/CndVar.h --- a/libc/src/threads/linux/CndVar.h +++ b/libc/src/threads/linux/CndVar.h @@ -9,12 +9,11 @@ #ifndef LLVM_LIBC_SRC_THREADS_LINUX_CNDVAR_H #define LLVM_LIBC_SRC_THREADS_LINUX_CNDVAR_H -#include "Mutex.h" - #include "include/sys/syscall.h" // For syscall numbers. #include "include/threads.h" // For values like thrd_success etc. #include "src/__support/CPP/atomic.h" #include "src/__support/OSUtil/syscall.h" // For syscall functions. +#include "src/__support/threads/mutex.h" #include // For futex operations. #include @@ -38,7 +37,8 @@ static int init(CndVar *cv) { cv->waitq_front = cv->waitq_back = nullptr; - return Mutex::init(&cv->qmtx, mtx_plain); + auto err = Mutex::init(&cv->qmtx, false, false, false); + return err == MutexError::NONE ? thrd_success : thrd_error; } static void destroy(CndVar *cv) { @@ -67,7 +67,7 @@ waitq_back = &waiter; } - if (m->unlock() != thrd_success) { + if (m->unlock() != MutexError::NONE) { // If we do not remove the queued up waiter before returning, // then another thread can potentially signal a non-existing // waiter. Note also that we do this with |qmtx| locked. This @@ -88,7 +88,8 @@ // At this point, if locking |m| fails, we can simply return as the // queued up waiter would have been removed from the queue. - return m->lock(); + auto err = m->lock(); + return err == MutexError::NONE ? thrd_success : thrd_error; } int notify_one() { @@ -105,7 +106,7 @@ if (waitq_front == nullptr) waitq_back = nullptr; - qmtx.futex_word = Mutex::MS_Free; + qmtx.futex_word = Mutex::FutexWordType(Mutex::LockState::Free); __llvm_libc::syscall( SYS_futex, &qmtx.futex_word.val, FUTEX_WAKE_OP, 1, 1, diff --git a/libc/src/threads/linux/Mutex.h b/libc/src/threads/linux/Mutex.h deleted file mode 100644 --- a/libc/src/threads/linux/Mutex.h +++ /dev/null @@ -1,122 +0,0 @@ -//===-- Utility mutex classes -----------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIBC_SRC_THREADS_LINUX_MUTEX_H -#define LLVM_LIBC_SRC_THREADS_LINUX_MUTEX_H - -#include "Futex.h" - -#include "include/sys/syscall.h" // For syscall numbers. -#include "include/threads.h" // For values like thrd_success etc. -#include "src/__support/CPP/atomic.h" // For atomics support -#include "src/__support/OSUtil/syscall.h" // For syscall functions. - -#include // For futex operations. - -namespace __llvm_libc { - -struct Mutex { - enum Status : FutexWordType { - MS_Free, - MS_Locked, - MS_Waiting, - }; - - cpp::Atomic futex_word; - int type; - - static int init(Mutex *mutex, int type) { - mutex->futex_word = MS_Free; - mutex->type = type; - return thrd_success; - } - - int lock() { - bool was_waiting = false; - while (true) { - FutexWordType mutex_status = MS_Free; - FutexWordType locked_status = MS_Locked; - - if (futex_word.compare_exchange_strong(mutex_status, MS_Locked)) { - if (was_waiting) - futex_word = MS_Waiting; - return thrd_success; - } - - switch (mutex_status) { - case MS_Waiting: - // If other threads are waiting already, then join them. Note that the - // futex syscall will block if the futex data is still `MS_Waiting` (the - // 4th argument to the syscall function below.) - __llvm_libc::syscall(SYS_futex, &futex_word.val, FUTEX_WAIT_PRIVATE, - MS_Waiting, 0, 0, 0); - was_waiting = true; - // Once woken up/unblocked, try everything all over. - continue; - case MS_Locked: - // Mutex has been locked by another thread so set the status to - // MS_Waiting. - if (futex_word.compare_exchange_strong(locked_status, MS_Waiting)) { - // If we are able to set the futex data to `MS_Waiting`, then we will - // wait for the futex to be woken up. Note again that the following - // syscall will block only if the futex data is still `MS_Waiting`. - __llvm_libc::syscall(SYS_futex, &futex_word.val, FUTEX_WAIT_PRIVATE, - MS_Waiting, 0, 0, 0); - was_waiting = true; - } - continue; - case MS_Free: - // If it was MS_Free, we shouldn't be here at all. - [[clang::fallthrough]]; - default: - // Mutex status cannot be anything else. So control should not reach - // here at all. - return thrd_error; - } - } - } - - int unlock() { - while (true) { - FutexWordType mutex_status = MS_Waiting; - if (futex_word.compare_exchange_strong(mutex_status, MS_Free)) { - // If any thread is waiting to be woken up, then do it. - __llvm_libc::syscall(SYS_futex, &futex_word.val, FUTEX_WAKE_PRIVATE, 1, - 0, 0, 0); - return thrd_success; - } - - if (mutex_status == MS_Locked) { - // If nobody was waiting at this point, just free it. - if (futex_word.compare_exchange_strong(mutex_status, MS_Free)) - return thrd_success; - } else { - // This can happen, for example if some thread tries to unlock an - // already free mutex. - return thrd_error; - } - } - } -}; - -static_assert(sizeof(Mutex) == sizeof(mtx_t), - "Sizes of internal representation of mutex and the public mtx_t " - "do not match."); - -class MutexLock { - Mutex *mutex; - -public: - explicit MutexLock(Mutex *m) : mutex(m) { mutex->lock(); } - - ~MutexLock() { mutex->unlock(); } -}; - -} // namespace __llvm_libc - -#endif // LLVM_LIBC_SRC_THREADS_LINUX_MUTEX_H diff --git a/libc/src/threads/linux/cnd_wait.cpp b/libc/src/threads/linux/cnd_wait.cpp --- a/libc/src/threads/linux/cnd_wait.cpp +++ b/libc/src/threads/linux/cnd_wait.cpp @@ -7,10 +7,10 @@ //===----------------------------------------------------------------------===// #include "CndVar.h" -#include "Mutex.h" -#include "src/threads/cnd_wait.h" #include "src/__support/common.h" +#include "src/__support/threads/mutex.h" +#include "src/threads/cnd_wait.h" namespace __llvm_libc { diff --git a/libc/src/threads/linux/mtx_destroy.cpp b/libc/src/threads/mtx_destroy.cpp rename from libc/src/threads/linux/mtx_destroy.cpp rename to libc/src/threads/mtx_destroy.cpp --- a/libc/src/threads/linux/mtx_destroy.cpp +++ b/libc/src/threads/mtx_destroy.cpp @@ -9,7 +9,7 @@ #include "src/threads/mtx_destroy.h" #include "include/threads.h" // For mtx_t definition. #include "src/__support/common.h" -#include "src/threads/linux/Mutex.h" +#include "src/__support/threads/mutex.h" namespace __llvm_libc { diff --git a/libc/src/threads/linux/mtx_init.cpp b/libc/src/threads/mtx_init.cpp rename from libc/src/threads/linux/mtx_init.cpp rename to libc/src/threads/mtx_init.cpp --- a/libc/src/threads/linux/mtx_init.cpp +++ b/libc/src/threads/mtx_init.cpp @@ -9,13 +9,13 @@ #include "src/threads/mtx_init.h" #include "include/threads.h" // For mtx_t definition. #include "src/__support/common.h" -#include "src/threads/linux/Mutex.h" +#include "src/__support/threads/mutex.h" namespace __llvm_libc { -LLVM_LIBC_FUNCTION(int, mtx_init, (mtx_t * mutex, int type)) { - auto *m = reinterpret_cast(mutex); - return Mutex::init(m, type); +LLVM_LIBC_FUNCTION(int, mtx_init, (mtx_t * m, int type)) { + auto err = Mutex::init(m, type | mtx_timed, type | mtx_recursive, 0); + return err == MutexError::NONE ? thrd_success : thrd_error; } } // namespace __llvm_libc diff --git a/libc/src/threads/linux/mtx_lock.cpp b/libc/src/threads/mtx_lock.cpp rename from libc/src/threads/linux/mtx_lock.cpp rename to libc/src/threads/mtx_lock.cpp --- a/libc/src/threads/linux/mtx_lock.cpp +++ b/libc/src/threads/mtx_lock.cpp @@ -7,16 +7,17 @@ //===----------------------------------------------------------------------===// #include "src/threads/mtx_lock.h" -#include "include/threads.h" // For mtx_t definition. +#include "include/threads.h" // For mtx_t definition. #include "src/__support/common.h" -#include "src/threads/linux/Mutex.h" +#include "src/__support/threads/mutex.h" namespace __llvm_libc { // The implementation currently handles only plain mutexes. LLVM_LIBC_FUNCTION(int, mtx_lock, (mtx_t * mutex)) { auto *m = reinterpret_cast(mutex); - return m->lock(); + auto err = m->lock(); + return err == MutexError::NONE ? thrd_success : thrd_error; } } // namespace __llvm_libc diff --git a/libc/src/threads/linux/mtx_unlock.cpp b/libc/src/threads/mtx_unlock.cpp rename from libc/src/threads/linux/mtx_unlock.cpp rename to libc/src/threads/mtx_unlock.cpp --- a/libc/src/threads/linux/mtx_unlock.cpp +++ b/libc/src/threads/mtx_unlock.cpp @@ -7,16 +7,17 @@ //===----------------------------------------------------------------------===// #include "src/threads/mtx_unlock.h" -#include "include/threads.h" // For mtx_t definition. +#include "include/threads.h" // For mtx_t definition. #include "src/__support/common.h" -#include "src/threads/linux/Mutex.h" +#include "src/__support/threads/mutex.h" namespace __llvm_libc { // The implementation currently handles only plain mutexes. LLVM_LIBC_FUNCTION(int, mtx_unlock, (mtx_t * mutex)) { auto *m = reinterpret_cast(mutex); - return m->unlock(); + auto err = m->unlock(); + return err == MutexError::NONE ? thrd_success : thrd_error; } } // namespace __llvm_libc