diff --git a/compiler-rt/lib/builtins/atomic.c b/compiler-rt/lib/builtins/atomic.c --- a/compiler-rt/lib/builtins/atomic.c +++ b/compiler-rt/lib/builtins/atomic.c @@ -27,6 +27,15 @@ #include #include +#if defined(__x86_64__) || defined(__i386__) +#include +#endif + +#if defined(__aarch64__) +#include +#include +#endif + #include "assembly.h" // Clang objects if you redefine a builtin. This little hack allows us to @@ -120,14 +129,53 @@ return locks + (hash & SPINLOCK_MASK); } -/// Macros for determining whether a size is lock free. Clang can not yet -/// codegen __atomic_is_lock_free(16), so for now we assume 16-byte values are -/// not lock free. +#if defined(__x86_64__) || defined(__i386__) + +#ifdef __x86_64__ +#define FEAT_REG ecx +#define MASK bit_CMPXCHG16B +#define WS 16 +#else +#define FEAT_REG edx +#define MASK bit_CMPXCHG8B +#define WS 8 +#endif + +static inline unsigned int check_x86_atomic_cas(void) { + unsigned int eax, ebx, ecx = 0, edx = 0; + __get_cpuid(1, &eax, &ebx, &ecx, &edx); + return FEAT_REG & MASK; +} + +static unsigned int __has_atomic_cas = check_x86_atomic_cas(); + +#endif // defined(__x86_64__) || defined(__i386__) + +#ifdef __aarch64__ +static unsigned int __has_atomic_cap = getauxval(AT_HWCAP) & HWCAP_ATOMICS; +#endif + +bool __atomic_is_lock_free(size_t size, void *ptr) { +#if defined(__x86_64__) || defined(__i386__) + if (size != WS) + return false; + return __atomic_has_cas; +#elif defined(__arm__) + // Even GCC doesn't seem to support it. + return false; +#elseif defined(__aarch64__) + return __has_atomic_cap && size == 16; +#endif + + return false; +} + +/// Macros for determining whether a size is lock free. #define IS_LOCK_FREE_1 __c11_atomic_is_lock_free(1) #define IS_LOCK_FREE_2 __c11_atomic_is_lock_free(2) #define IS_LOCK_FREE_4 __c11_atomic_is_lock_free(4) #define IS_LOCK_FREE_8 __c11_atomic_is_lock_free(8) -#define IS_LOCK_FREE_16 0 +#define IS_LOCK_FREE_16 __c11_atomic_is_lock_free(16) /// Macro that calls the compiler-generated lock-free versions of functions /// when they exist. @@ -156,8 +204,7 @@ break; \ case 16: \ if (IS_LOCK_FREE_16) { \ - /* FIXME: __uint128_t isn't available on 32 bit platforms. \ - LOCK_FREE_ACTION(__uint128_t);*/ \ + LOCK_FREE_ACTION(__uint128_t); \ } \ break; \ } \