diff --git a/compiler-rt/lib/builtins/atomic.c b/compiler-rt/lib/builtins/atomic.c --- a/compiler-rt/lib/builtins/atomic.c +++ b/compiler-rt/lib/builtins/atomic.c @@ -120,45 +120,39 @@ return locks + (hash & SPINLOCK_MASK); } -/// Macros for determining whether a size is lock free. Clang can not yet -/// codegen __atomic_is_lock_free(16), so for now we assume 16-byte values are -/// not lock free. -#define IS_LOCK_FREE_1 __c11_atomic_is_lock_free(1) -#define IS_LOCK_FREE_2 __c11_atomic_is_lock_free(2) -#define IS_LOCK_FREE_4 __c11_atomic_is_lock_free(4) -#define IS_LOCK_FREE_8 __c11_atomic_is_lock_free(8) -#define IS_LOCK_FREE_16 0 +/// Macros for determining whether a size is lock free. +#define ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(size, p) \ + (__atomic_always_lock_free(size, p) || \ + (__atomic_always_lock_free(size, NULL) && ((uintptr_t)p & size) == 0)) +#define IS_LOCK_FREE_1(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(1, p) +#define IS_LOCK_FREE_2(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(2, p) +#define IS_LOCK_FREE_4(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(4, p) +#define IS_LOCK_FREE_8(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(8, p) +#define IS_LOCK_FREE_16(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(16, p) /// Macro that calls the compiler-generated lock-free versions of functions /// when they exist. -#define LOCK_FREE_CASES() \ +#define TRY_LOCK_FREE_CASE(n, type, ptr) \ + case n: \ + if (IS_LOCK_FREE_##n(ptr)) { \ + LOCK_FREE_ACTION(type); \ + } \ + break; +#ifdef __SIZEOF_INT128__ +#define TRY_LOCK_FREE_CASE_16(p) TRY_LOCK_FREE_CASE(16, __uint128_t, p) +#else +#define TRY_LOCK_FREE_CASE_16(p) /* __uint128_t not available */ +#endif + +#define LOCK_FREE_CASES(ptr) \ do { \ switch (size) { \ - case 1: \ - if (IS_LOCK_FREE_1) { \ - LOCK_FREE_ACTION(uint8_t); \ - } \ - break; \ - case 2: \ - if (IS_LOCK_FREE_2) { \ - LOCK_FREE_ACTION(uint16_t); \ - } \ - break; \ - case 4: \ - if (IS_LOCK_FREE_4) { \ - LOCK_FREE_ACTION(uint32_t); \ - } \ - break; \ - case 8: \ - if (IS_LOCK_FREE_8) { \ - LOCK_FREE_ACTION(uint64_t); \ - } \ - break; \ - case 16: \ - if (IS_LOCK_FREE_16) { \ - /* FIXME: __uint128_t isn't available on 32 bit platforms. \ - LOCK_FREE_ACTION(__uint128_t);*/ \ - } \ + TRY_LOCK_FREE_CASE(1, uint8_t, ptr) \ + TRY_LOCK_FREE_CASE(2, uint16_t, ptr) \ + TRY_LOCK_FREE_CASE(4, uint32_t, ptr) \ + TRY_LOCK_FREE_CASE(8, uint64_t, ptr) \ + TRY_LOCK_FREE_CASE_16(ptr) /* __uint128_t may not be supported */ \ + default: \ break; \ } \ } while (0) @@ -169,7 +163,7 @@ #define LOCK_FREE_ACTION(type) \ *((type *)dest) = __c11_atomic_load((_Atomic(type) *)src, model); \ return; - LOCK_FREE_CASES(); + LOCK_FREE_CASES(src); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(src); lock(l); @@ -183,7 +177,7 @@ #define LOCK_FREE_ACTION(type) \ __c11_atomic_store((_Atomic(type) *)dest, *(type *)src, model); \ return; - LOCK_FREE_CASES(); + LOCK_FREE_CASES(dest); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(dest); lock(l); @@ -202,7 +196,7 @@ return __c11_atomic_compare_exchange_strong( \ (_Atomic(type) *)ptr, (type *)expected, *(type *)desired, success, \ failure) - LOCK_FREE_CASES(); + LOCK_FREE_CASES(ptr); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(ptr); lock(l); @@ -223,7 +217,7 @@ *(type *)old = \ __c11_atomic_exchange((_Atomic(type) *)ptr, *(type *)val, model); \ return; - LOCK_FREE_CASES(); + LOCK_FREE_CASES(ptr); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(ptr); lock(l); @@ -253,7 +247,7 @@ #define OPTIMISED_CASE(n, lockfree, type) \ type __atomic_load_##n(type *src, int model) { \ - if (lockfree) \ + if (lockfree(src)) \ return __c11_atomic_load((_Atomic(type) *)src, model); \ Lock *l = lock_for_pointer(src); \ lock(l); \ @@ -266,7 +260,7 @@ #define OPTIMISED_CASE(n, lockfree, type) \ void __atomic_store_##n(type *dest, type val, int model) { \ - if (lockfree) { \ + if (lockfree(dest)) { \ __c11_atomic_store((_Atomic(type) *)dest, val, model); \ return; \ } \ @@ -281,7 +275,7 @@ #define OPTIMISED_CASE(n, lockfree, type) \ type __atomic_exchange_##n(type *dest, type val, int model) { \ - if (lockfree) \ + if (lockfree(dest)) \ return __c11_atomic_exchange((_Atomic(type) *)dest, val, model); \ Lock *l = lock_for_pointer(dest); \ lock(l); \ @@ -296,7 +290,7 @@ #define OPTIMISED_CASE(n, lockfree, type) \ bool __atomic_compare_exchange_##n(type *ptr, type *expected, type desired, \ int success, int failure) { \ - if (lockfree) \ + if (lockfree(ptr)) \ return __c11_atomic_compare_exchange_strong( \ (_Atomic(type) *)ptr, expected, desired, success, failure); \ Lock *l = lock_for_pointer(ptr); \ @@ -318,7 +312,7 @@ //////////////////////////////////////////////////////////////////////////////// #define ATOMIC_RMW(n, lockfree, type, opname, op) \ type __atomic_fetch_##opname##_##n(type *ptr, type val, int model) { \ - if (lockfree) \ + if (lockfree(ptr)) \ return __c11_atomic_fetch_##opname((_Atomic(type) *)ptr, val, model); \ Lock *l = lock_for_pointer(ptr); \ lock(l); \