[libc] Add a class "Atomic" as a simple equivalent of std::atomic.

[sivachandra]

Summary

Only the methods currently required by the libc have been added.
Most of the existing uses of atomic operations have been switched over
to this new class. A future change will clean up the rest of uses.

This change now allows building mutex and condition variable code with a
C++ compiler which does not have stdatomic.h, for example g++.

[sivachandra]

Add the unit test files which were missed in the previous upload.

[lntue]

libc/include/llvm-libc-types/mtx_t.h

13–23

__aarch64__

[sivachandra]

Fix typo.

libc/include/__llvm-libc-common.h

 #define __BEGIN_C_DECLS extern "C" {
 
 #undef __END_C_DECLS
 #define __END_C_DECLS }
 
 #undef _Noreturn
 #define _Noreturn [[noreturn]]
 
+#undef _Alignas
+#define _Alignas alignas
+
+#undef _Alignof
+#define _Alignof alignof
+
 #else // not __cplusplus
 
 #undef __BEGIN_C_DECLS
 #define __BEGIN_C_DECLS
 
 #undef __END_C_DECLS
 #define __END_C_DECLS
 
 #undef __restrict
 #define __restrict restrict // C99 and above support the restrict keyword.
 
 #endif // __cplusplus
 
 #endif // LLVM_LIBC___COMMON_H

libc/include/llvm-libc-types/cnd_t.h

 //===-- Definition of cnd_t type ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef __LLVM_LIBC_TYPES_CND_T_H__
 #define __LLVM_LIBC_TYPES_CND_T_H__
 
+#include "mtx_t.h"
+
 typedef struct {
   void *__qfront;
   void *__qback;
-  struct {
-    unsigned char __w[4];
-    int __t;
-  } __qmtx;
+  mtx_t __qmtx;
 } cnd_t;
 
 #endif // __LLVM_LIBC_TYPES_CND_T_H__

libc/include/llvm-libc-types/mtx_t.h

 //===-- Definition of mtx_t type ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef __LLVM_LIBC_TYPES_MTX_T_H__
 #define __LLVM_LIBC_TYPES_MTX_T_H__
 
 typedef struct {
-  unsigned char __internal_data[4];
+#if defined(__x86_64__) || defined(__aarch64__)
+  // Futex word should be aligned appropriately to allow target atomic
+  // instructions. This declaration mimics the internal setup.
+  struct {
+    _Alignas(sizeof(unsigned int) > _Alignof(unsigned int)
+                 ? sizeof(unsigned int)
+                 : _Alignof(unsigned int)) unsigned int __word;
+  } __futex_word;
+#else
+#error "Mutex type mtx_t is not available for the target architecture."
+#endif
   int __mtx_type;
 } mtx_t;
 
 #endif // __LLVM_LIBC_TYPES_MTX_T_H__

libc/src/__support/CPP/CMakeLists.txt

 
 add_header_library(
   vector
   HDRS
     vector.h
   DEPENDS
     libc.include.stdlib
 )
+
+add_header_library(
+  atomic
+  HDRS
+    atomic.h
+)

libc/src/__support/CPP/atomic.h

+//===-- A simple equivalent of std::atomic ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SUPPORT_CPP_ATOMIC_H
+#define LLVM_LIBC_SRC_SUPPORT_CPP_ATOMIC_H
+
+#include "TypeTraits.h"
+
+namespace __llvm_libc {
+namespace cpp {
+
+enum class MemoryOrder : int {
+  RELAXED = __ATOMIC_RELAXED,
+  CONSUME = __ATOMIC_CONSUME,
+  ACQUIRE = __ATOMIC_ACQUIRE,
+  RELEASE = __ATOMIC_RELEASE,
+  ACQ_REL = __ATOMIC_ACQ_REL,
+  SEQ_CST = __ATOMIC_SEQ_CST
+};
+
+template <typename T> struct Atomic {
+  // For now, we will restrict to only arithmetic types.
+  static_assert(IsArithmetic<T>::Value, "Only arithmetic types can be atomic.");
+
+private:
+  // The value stored should be appropriately aligned so that
+  // hardware instructions used to perform atomic operations work
+  // correctly.
+  static constexpr int ALIGNMENT = sizeof(T) > alignof(T) ? sizeof(T)
+                                                          : alignof(T);
+
+public:
+  using value_type = T;
+
+  // We keep the internal value public so that it can be addressable.
+  // This is useful in places like the Linux futex operations where
+  // we need pointers to the memory of the atomic values. Load and store
+  // operations should be performed using the atomic methods however.
+  alignas(ALIGNMENT) value_type val;
+
+  // Intializes the value without using atomic operations.
+  constexpr Atomic(value_type v) : val(v) {}
+
+  Atomic(const Atomic &) = delete;
+  Atomic &operator=(const Atomic &) = delete;
+
+  // Atomic load
+  operator T() { return __atomic_load_n(&val, int(MemoryOrder::SEQ_CST)); }
+
+  T load(MemoryOrder mem_ord = MemoryOrder::SEQ_CST) {
+    return __atomic_load_n(&val, int(mem_ord));
+  }
+
+  // Atomic store
+  T operator=(T rhs) {
+    __atomic_store_n(&val, rhs, int(MemoryOrder::SEQ_CST));
+    return rhs;
+  }
+
+  void store(T rhs, MemoryOrder mem_ord = MemoryOrder::SEQ_CST) {
+    __atomic_store_n(&val, rhs, int(mem_ord));
+  }
+
+  // Atomic compare exchange
+  bool compare_exchange_strong(T &expected, T desired,
+                               MemoryOrder mem_ord = MemoryOrder::SEQ_CST) {
+    return __atomic_compare_exchange_n(&val, &expected, desired, false,
+                                       int(mem_ord), int(mem_ord));
+  }
+
+  // Set the value without using an atomic operation. This is useful
+  // in initializing atomic values without a constructor.
+  void set(T rhs) { val = rhs; }
+};
+
+} // namespace cpp
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_SUPPORT_CPP_ATOMIC_H

libc/src/threads/linux/CMakeLists.txt

HDRS
     CndVar.h
     Futex.h
     Mutex.h
     Thread.h
   DEPENDS
     .thread_start_args_h
     libc.include.sys_syscall
     libc.include.threads
+    libc.src.__support.CPP.atomic
     libc.src.__support.OSUtil.osutil
 )
 
 add_entrypoint_object(
   thrd_create
   SRCS
     thrd_create.cpp
   HDRS

libc/src/threads/linux/CndVar.h

 //===-- Utility condition variable class ------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_THREADS_LINUX_CNDVAR_H
 #define LLVM_LIBC_SRC_THREADS_LINUX_CNDVAR_H
 
-#include "Futex.h"
 #include "Mutex.h"
 
-#include "include/sys/syscall.h"          // For syscall numbers.
-#include "include/threads.h"              // For values like thrd_success etc.
+#include "include/sys/syscall.h" // For syscall numbers.
+#include "include/threads.h"     // For values like thrd_success etc.
+#include "src/__support/CPP/atomic.h"
 #include "src/__support/OSUtil/syscall.h" // For syscall functions.
 
 #include <linux/futex.h> // For futex operations.
-#include <stdatomic.h>   // For atomic operations
+#include <stdint.h>
 
 namespace __llvm_libc {
 
 struct CndVar {
   enum CndWaiterStatus : uint32_t {
     WS_Waiting = 0xE,
     WS_Signalled = 0x5,
   };
 
   struct CndWaiter {
-    FutexWord futex_word = WS_Waiting;
+    cpp::Atomic<uint32_t> futex_word = WS_Waiting;
     CndWaiter *next = nullptr;
   };
 
   CndWaiter *waitq_front;
   CndWaiter *waitq_back;
   Mutex qmtx;
 
   static int init(CndVar *cv) {
CndWaiter waiter;
           waitq_front = nullptr;
         else
           waitq_back->next = nullptr;
 
         return thrd_error;
       }
     }
 
-    __llvm_libc::syscall(SYS_futex, &waiter.futex_word, FUTEX_WAIT, WS_Waiting,
-                         0, 0, 0);
+    __llvm_libc::syscall(SYS_futex, &waiter.futex_word.val, FUTEX_WAIT,
+                         WS_Waiting, 0, 0, 0);
 
     // At this point, if locking |m| fails, we can simply return as the
     // queued up waiter would have been removed from the queue.
     return m->lock();
   }
 
   int notify_one() {
     // We don't use an RAII locker in this method as we want to unlock
     // |qmtx| and signal the waiter using a single FUTEX_WAKE_OP signal.
     qmtx.lock();
     if (waitq_front == nullptr) {
       qmtx.unlock();
       return thrd_success;
     }
 
     CndWaiter *first = waitq_front;
     waitq_front = waitq_front->next;
     if (waitq_front == nullptr)
       waitq_back = nullptr;
 
-    atomic_store(&qmtx.futex_word, Mutex::MS_Free);
+    qmtx.futex_word = Mutex::MS_Free;
 
     __llvm_libc::syscall(
-        SYS_futex, &qmtx.futex_word, FUTEX_WAKE_OP, 1, 1, &first->futex_word,
+        SYS_futex, &qmtx.futex_word.val, FUTEX_WAKE_OP, 1, 1,
+        &first->futex_word.val,
         FUTEX_OP(FUTEX_OP_SET, WS_Signalled, FUTEX_OP_CMP_EQ, WS_Waiting));
     return thrd_success;
   }
 
   int broadcast() {
     MutexLock ml(&qmtx);
-    FutexWord dummy_futex_word;
+    uint32_t dummy_futex_word;
     CndWaiter *waiter = waitq_front;
     waitq_front = waitq_back = nullptr;
     while (waiter != nullptr) {
       // FUTEX_WAKE_OP is used instead of just FUTEX_WAKE as it allows us to
       // atomically update the waiter status to WS_Signalled before waking
       // up the waiter. A dummy location is used for the other futex of
       // FUTEX_WAKE_OP.
       __llvm_libc::syscall(
           SYS_futex, &dummy_futex_word, FUTEX_WAKE_OP, 1, 1,
-          &waiter->futex_word,
+          &waiter->futex_word.val,
           FUTEX_OP(FUTEX_OP_SET, WS_Signalled, FUTEX_OP_CMP_EQ, WS_Waiting));
       waiter = waiter->next;
     }
     return thrd_success;
   }
 };
 
 static_assert(sizeof(CndVar) == sizeof(cnd_t),
               "Mismatch in the size of the "
               "internal representation of condition variable and the public "
               "cnd_t type.");
 
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_THREADS_LINUX_CNDVAR_H

libc/src/threads/linux/Mutex.h

 //===-- Utility mutex classes -----------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_THREADS_LINUX_MUTEX_H
 #define LLVM_LIBC_SRC_THREADS_LINUX_MUTEX_H
 
-#include "Futex.h"
-
 #include "include/sys/syscall.h"          // For syscall numbers.
 #include "include/threads.h"              // For values like thrd_success etc.
+#include "src/__support/CPP/atomic.h"     // For atomics support
 #include "src/__support/OSUtil/syscall.h" // For syscall functions.
 
 #include <linux/futex.h> // For futex operations.
-#include <stdatomic.h>
-#include <stdint.h>
 
 namespace __llvm_libc {
 
+#if (defined(LLVM_LIBC_ARCH_AARCH64) || defined(LLVM_LIBC_ARCH_X86_64))
+static_assert(sizeof(unsigned int) == 4,
+              "Unexpected size of unsigned int type.");
+typedef unsigned int FutexWordType;
+#else
+#error "Futex word base type not defined for the target architecture."
+#endif
+
 struct Mutex {
-  enum Status : uint32_t {
+  enum Status : FutexWordType {
     MS_Free,
     MS_Locked,
     MS_Waiting,
   };
 
-  FutexWord futex_word;
+  cpp::Atomic<FutexWordType> futex_word;
   int type;
 
   static int init(Mutex *mutex, int type) {
     mutex->futex_word = MS_Free;
     mutex->type = type;
     return thrd_success;
   }
 
   int lock() {
     bool was_waiting = false;
     while (true) {
-      uint32_t mutex_status = MS_Free;
-      uint32_t locked_status = MS_Locked;
+      FutexWordType mutex_status = MS_Free;
+      FutexWordType locked_status = MS_Locked;
 
-      if (atomic_compare_exchange_strong(&futex_word, &mutex_status,
-                                         MS_Locked)) {
+      if (futex_word.compare_exchange_strong(mutex_status, MS_Locked)) {
         if (was_waiting)
-          atomic_store(&futex_word, MS_Waiting);
+          futex_word = MS_Waiting;
         return thrd_success;
       }
 
       switch (mutex_status) {
       case MS_Waiting:
         // If other threads are waiting already, then join them. Note that the
         // futex syscall will block if the futex data is still `MS_Waiting` (the
         // 4th argument to the syscall function below.)
-        __llvm_libc::syscall(SYS_futex, &futex_word, FUTEX_WAIT_PRIVATE,
+        __llvm_libc::syscall(SYS_futex, &futex_word.val, FUTEX_WAIT_PRIVATE,
                              MS_Waiting, 0, 0, 0);
         was_waiting = true;
         // Once woken up/unblocked, try everything all over.
         continue;
       case MS_Locked:
         // Mutex has been locked by another thread so set the status to
         // MS_Waiting.
-        if (atomic_compare_exchange_strong(&futex_word, &locked_status,
-                                           MS_Waiting)) {
+        if (futex_word.compare_exchange_strong(locked_status, MS_Waiting)) {
           // If we are able to set the futex data to `MS_Waiting`, then we will
           // wait for the futex to be woken up. Note again that the following
           // syscall will block only if the futex data is still `MS_Waiting`.
-          __llvm_libc::syscall(SYS_futex, &futex_word, FUTEX_WAIT_PRIVATE,
+          __llvm_libc::syscall(SYS_futex, &futex_word.val, FUTEX_WAIT_PRIVATE,
                                MS_Waiting, 0, 0, 0);
           was_waiting = true;
         }
         continue;
       case MS_Free:
         // If it was MS_Free, we shouldn't be here at all.
         [[clang::fallthrough]];
       default:
         // Mutex status cannot be anything else. So control should not reach
         // here at all.
         return thrd_error;
       }
     }
   }
 
   int unlock() {
     while (true) {
-      uint32_t mutex_status = MS_Waiting;
-      if (atomic_compare_exchange_strong(&futex_word, &mutex_status, MS_Free)) {
+      FutexWordType mutex_status = MS_Waiting;
+      if (futex_word.compare_exchange_strong(mutex_status, MS_Free)) {
         // If any thread is waiting to be woken up, then do it.
-        __llvm_libc::syscall(SYS_futex, &futex_word, FUTEX_WAKE_PRIVATE, 1, 0,
-                             0, 0);
+        __llvm_libc::syscall(SYS_futex, &futex_word.val, FUTEX_WAKE_PRIVATE, 1,
+                             0, 0, 0);
         return thrd_success;
       }
 
       if (mutex_status == MS_Locked) {
         // If nobody was waiting at this point, just free it.
-        if (atomic_compare_exchange_strong(&futex_word, &mutex_status, MS_Free))
+        if (futex_word.compare_exchange_strong(mutex_status, MS_Free))
           return thrd_success;
       } else {
         // This can happen, for example if some thread tries to unlock an
         // already free mutex.
         return thrd_error;
       }
     }
   }

libc/test/src/__support/CPP/CMakeLists.txt

add_libc_unittest(
   int_seq_test
   SUITE
     libc_cpp_utils_unittests
   SRCS
     integer_sequence_test.cpp
   DEPENDS
     libc.src.__support.CPP.standalone_cpp
 )
+
+add_libc_unittest(
+  atomic_test
+  SUITE
+    libc_cpp_utils_unittests
+  SRCS
+    atomic_test.cpp
+  DEPENDS
+    libc.src.__support.CPP.atomic
+)

libc/test/src/__support/CPP/atomic_test.cpp

+//===-- Unittests for Atomic ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/CPP/atomic.h"
+#include "utils/UnitTest/Test.h"
+
+// Tests in this file do not test atomicity as it would require using
+// threads, at which point it becomes a chicken and egg problem.
+
+TEST(LlvmLibcAtomicTest, LoadStore) {
+  __llvm_libc::cpp::Atomic<int> aint(123);
+  ASSERT_EQ(aint.load(), 123);
+
+  aint.store(100);
+  ASSERT_EQ(aint.load(), 100);
+
+  aint = 1234; // Equivalent of store
+  ASSERT_EQ(aint.load(), 1234);
+}
+
+TEST(LlvmLibcAtomicTest, CompareExchangeStrong) {
+  int desired = 123;
+  __llvm_libc::cpp::Atomic<int> aint(desired);
+  ASSERT_TRUE(aint.compare_exchange_strong(desired, 100));
+  ASSERT_EQ(aint.load(), 100);
+
+  ASSERT_FALSE(aint.compare_exchange_strong(desired, 100));
+  ASSERT_EQ(aint.load(), 100);
+}