asan: fix atomic operations on ARM

[dvyukov]

Summary

implement correct atomic load/store for ARM
add test for atomic load/store

[kcc]

lib/sanitizer_common/sanitizer_atomic_clang.h

70

I would split this into multiple files (one with ifdef arm, etc)

[eugenis]

CMakeLists.txt

185

Is this still needed if we use _sync_val_compare_and_swap and not hand-written assembly?

lib/sanitizer_common/sanitizer_atomic.h

56

Why not put the attribute on struct definition? AFAIK, this way 'atomic_uint64_t' is aligned, but "struct atomic_uint64_t" is not?

[dvyukov]

add 2 separate files for x86 and other platforms
instead of ifdef spaghetti

[dvyukov]

do not do
typedef X X;
instead do
typedef X Y;

[eugenis]

LGTM

lib/sanitizer_common/sanitizer_atomic.h

40

It would be nice to rename the rest of these to _s + typedef, but that's optional.

[dvyukov]

move ALIGNED to a different location

[kcc]

LGTM

[Eugene.Zelenko]

Committed in r199802.

CMakeLists.txt

 # destructors.
 check_cxx_compiler_flag(-Wno-non-virtual-dtor SUPPORTS_NO_NON_VIRTUAL_DTOR_FLAG)
 if (SUPPORTS_NO_NON_VIRTUAL_DTOR_FLAG)
   list(APPEND SANITIZER_COMMON_CFLAGS -Wno-non-virtual-dtor)
 endif()
 check_cxx_compiler_flag(-Wglobal-constructors SUPPORTS_GLOBAL_CONSTRUCTORS_FLAG)
 # Not all sanitizers forbid global constructors.
 
+# 64-bit atomic operations require armv7.
+if(ANDROID)
+  list(APPEND SANITIZER_COMMON_CFLAGS -march=armv7)
+endif()
+
 if(APPLE)
   # Obtain the iOS Simulator SDK path from xcodebuild.
   execute_process(
     COMMAND xcodebuild -version -sdk iphonesimulator Path
     OUTPUT_VARIABLE IOSSIM_SDK_DIR
     OUTPUT_STRIP_TRAILING_WHITESPACE
   )
   set(SANITIZER_COMMON_SUPPORTED_DARWIN_OS osx)

lib/asan/asan_allocator.h

struct AsanThreadLocalMallocStorage {
   uptr quarantine_cache[16];
   uptr allocator2_cache[96 * (512 * 8 + 16)];  // Opaque.
   void CommitBack();
  private:
   // These objects are allocated via mmap() and are zero-initialized.
   AsanThreadLocalMallocStorage() {}
 };
 
+typedef ALIGNED(8) AsanThreadLocalMallocStorage AsanThreadLocalMallocStorage;
+
 void *asan_memalign(uptr alignment, uptr size, StackTrace *stack,
                     AllocType alloc_type);
 void asan_free(void *ptr, StackTrace *stack, AllocType alloc_type);
 
 void *asan_malloc(uptr size, StackTrace *stack);
 void *asan_calloc(uptr nmemb, uptr size, StackTrace *stack);
 void *asan_realloc(void *p, uptr size, StackTrace *stack);
 void *asan_valloc(uptr size, StackTrace *stack);

lib/sanitizer_common/sanitizer_atomic.h

struct atomic_uint8_t {
   volatile Type val_dont_use;
 };
 
 struct atomic_uint16_t {
   typedef u16 Type;
   volatile Type val_dont_use;
 };
 
 struct atomic_uint32_t {
   typedef u32 Type;
   volatile Type val_dont_use;
 };
 
 struct atomic_uint64_t {
   typedef u64 Type;
   volatile Type val_dont_use;
 };
 
 struct atomic_uintptr_t {
   typedef uptr Type;
   volatile Type val_dont_use;
 };
 
+// On 32-bit platforms u64 is not necessary aligned on 8 bytes.
+typedef ALIGNED(8) atomic_uint64_t atomic_uint64_t;
+
 }  // namespace __sanitizer
 
 #if defined(__GNUC__)
 # include "sanitizer_atomic_clang.h"
 #elif defined(_MSC_VER)
 # include "sanitizer_atomic_msvc.h"
 #else
 # error "Unsupported compiler"
 #endif
 
 #endif  // SANITIZER_ATOMIC_H

lib/sanitizer_common/sanitizer_atomic_clang.h

INLINE void proc_yield(int cnt) {
   __asm__ __volatile__("" ::: "memory");
 #if defined(__i386__) || defined(__x86_64__)
   for (int i = 0; i < cnt; i++)
     __asm__ __volatile__("pause");
 #endif
   __asm__ __volatile__("" ::: "memory");
 }
 
+// We would like to just use compiler builtin atomic operations
+// for loads and stores, but they are mostly broken in clang:
+// - they lead to vastly inefficient code generation
+// (http://llvm.org/bugs/show_bug.cgi?id=17281)
+// - 64-bit atomic operations are not implemented on x86_32
+// (http://llvm.org/bugs/show_bug.cgi?id=15034)
+// - they are not implemented on ARM
+// error: undefined reference to '__atomic_load_4'
+
+// See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
+// for mappings of the memory model to different processors.
+
 template<typename T>
 INLINE typename T::Type atomic_load(
     const volatile T *a, memory_order mo) {
   DCHECK(mo & (memory_order_relaxed | memory_order_consume
       | memory_order_acquire | memory_order_seq_cst));
   DCHECK(!((uptr)a % sizeof(*a)));
   typename T::Type v;
-  // FIXME:
-  // 64-bit atomic operations are not atomic on 32-bit platforms.
-  // The implementation lacks necessary memory fences on ARM/PPC.
-  // We would like to use compiler builtin atomic operations,
-  // but they are mostly broken:
-  // - they lead to vastly inefficient code generation
-  // (http://llvm.org/bugs/show_bug.cgi?id=17281)
-  // - 64-bit atomic operations are not implemented on x86_32
-  // (http://llvm.org/bugs/show_bug.cgi?id=15034)
-  // - they are not implemented on ARM
-  // error: undefined reference to '__atomic_load_4'
+
+  if (sizeof(*a) < 8 || sizeof(void*) == 8) {
+    // Assume that aligned loads are atomic.
     if (mo == memory_order_relaxed) {
       v = a->val_dont_use;
-  } else {
-    atomic_signal_fence(memory_order_seq_cst);
+    } else if (mo == memory_order_consume) {
+      // Assume that processor respects data dependencies
+      // (and that compiler won't break them).
+      __asm__ __volatile__("" ::: "memory");
+      v = a->val_dont_use;
+      __asm__ __volatile__("" ::: "memory");
+    } else if (mo == memory_order_acquire) {
+      __asm__ __volatile__("" ::: "memory");
+      v = a->val_dont_use;
+#if defined(__i386__) || defined(__x86_64__)
+      // On x86 loads are implicitly acquire.
+      __asm__ __volatile__("" ::: "memory");
+#else
+      __sync_synchronize();
+#endif
+    } else {  // seq_cst
+#if defined(__i386__) || defined(__x86_64__)
+      // On x86 plain MOV is enough for seq_cst store.
+      __asm__ __volatile__("" ::: "memory");
+#else
+      // E.g. on POWER we need a hw fence even before the store.
+      __sync_synchronize();
+#endif
       v = a->val_dont_use;
-    atomic_signal_fence(memory_order_seq_cst);
+#if defined(__i386__) || defined(__x86_64__)
+      __asm__ __volatile__("" ::: "memory");
+#else
+      __sync_synchronize();
+#endif
+    }
+  } else {
+    // 64-bit load on 32-bit platform.
+    // FIXME(dvyukov): not implemented
+#if defined(__i386__)
+    __asm__ __volatile__(
+        "movq %1, %%mm0;"  // Use mmx reg for 64-bit atomic moves
+        "movq %%mm0, %0;"  // (ptr could be read-only)
+        "emms;"            // Empty mmx state/Reset FP regs
+        : "=m" (v)
+        : "m" (a->val_dont_use)
+        : // mark the FP stack and mmx registers as clobbered
+          "st", "st(1)", "st(2)", "st(3)", "st(4)", "st(5)", "st(6)", "st(7)",
+#ifdef __MMX__
+          "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7",
+#endif  // #ifdef __MMX__
+          "memory");
+#else  // #if defined(__i386__)
+    // Gross, but simple and reliable.
+    // Assume that it is not in read-only memory.
+    v = __sync_fetch_and_add((typename T::Type volatile*)&a->val_dont_use, 0);
+#endif
   }
   return v;
 }
 
 template<typename T>
 INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
   DCHECK(mo & (memory_order_relaxed | memory_order_release
       | memory_order_seq_cst));
   DCHECK(!((uptr)a % sizeof(*a)));
+
+  if (sizeof(*a) < 8 || sizeof(void*) == 8) {
+    // Assume that aligned loads are atomic.
     if (mo == memory_order_relaxed) {
       a->val_dont_use = v;
-  } else {
-    atomic_signal_fence(memory_order_seq_cst);
+    } else if (mo == memory_order_release) {
+#if defined(__i386__) || defined(__x86_64__)
+      // On x86 stores are implicitly release.
+      __asm__ __volatile__("" ::: "memory");
+#else
+      __sync_synchronize();
+#endif
+      a->val_dont_use = v;
+      __asm__ __volatile__("" ::: "memory");
+    } else {  // seq_cst
+#if defined(__i386__) || defined(__x86_64__)
+      // On x86 stores are implicitly release.
+      __asm__ __volatile__("" ::: "memory");
+#else
+      __sync_synchronize();
+#endif
       a->val_dont_use = v;
-    atomic_signal_fence(memory_order_seq_cst);
+      __sync_synchronize();
     }
+  } else {
+    // 64-bit store on 32-bit platform.
+#if defined(__i386__)
+    __asm__ __volatile__(
+        "movq %1, %%mm0;"  // Use mmx reg for 64-bit atomic moves
+        "movq %%mm0, %0;"
+        "emms;"            // Empty mmx state/Reset FP regs
+        : "=m" (a->val_dont_use)
+        : "m" (v)
+        : // mark the FP stack and mmx registers as clobbered
+          "st", "st(1)", "st(2)", "st(3)", "st(4)", "st(5)", "st(6)", "st(7)",
+#ifdef __MMX__
+          "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7",
+#endif  // #ifdef __MMX__
+          "memory");
     if (mo == memory_order_seq_cst)
-    atomic_thread_fence(memory_order_seq_cst);
+      __sync_synchronize();
+#else  // #if defined(__i386__)
+    // Gross, but simple and reliable.
+    typename T::Type cmp = a->val_dont_use;
+    typename T::Type cur;
+    for (;;) {
+      cur = __sync_val_compare_and_swap(&a->val_dont_use, cmp, v);
+      if (cmp == v)
+        break;
+      cmp = cur;
+    }
+#endif
+  }
 }
 
 template<typename T>
 INLINE typename T::Type atomic_fetch_add(volatile T *a,
     typename T::Type v, memory_order mo) {
   (void)mo;
   DCHECK(!((uptr)a % sizeof(*a)));
   return __sync_fetch_and_add(&a->val_dont_use, v);

lib/sanitizer_common/tests/sanitizer_atomic_test.cc

 // This file is a part of ThreadSanitizer/AddressSanitizer runtime.
 //
 //===----------------------------------------------------------------------===//
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "gtest/gtest.h"
 
 namespace __sanitizer {
 
+template<typename T>
+struct ValAndMagic {
+  typename T::Type magic0;
+  T a;
+  typename T::Type magic1;
+
+  static ValAndMagic<T> *sink;
+};
+
+template<typename T>
+ValAndMagic<T> *ValAndMagic<T>::sink;
+
+template<typename T, memory_order load_mo, memory_order store_mo>
+void CheckStoreLoad() {
+  typedef typename T::Type Type;
+  ValAndMagic<T> val;
+  // Prevent the compiler from scalarizing the struct.
+  ValAndMagic<T>::sink = &val;
+  // Ensure that surrounding memory is not overwritten.
+  val.magic0 = val.magic1 = (Type)-3;
+  for (u64 i = 0; i < 100; i++) {
+    // Generate a value that occupies all bytes of the variable.
+    u64 v = i;
+    v |= v << 8;
+    v |= v << 16;
+    v |= v << 32;
+    val.a.val_dont_use = (Type)v;
+    EXPECT_EQ(atomic_load(&val.a, load_mo), (Type)v);
+    val.a.val_dont_use = (Type)-1;
+    atomic_store(&val.a, (Type)v, store_mo);
+    EXPECT_EQ(val.a.val_dont_use, (Type)v);
+  }
+  EXPECT_EQ(val.magic0, (Type)-3);
+  EXPECT_EQ(val.magic1, (Type)-3);
+}
+
+TEST(SanitizerCommon, AtomicStoreLoad) {
+  CheckStoreLoad<atomic_uint8_t, memory_order_relaxed, memory_order_relaxed>();
+  CheckStoreLoad<atomic_uint8_t, memory_order_consume, memory_order_relaxed>();
+  CheckStoreLoad<atomic_uint8_t, memory_order_acquire, memory_order_relaxed>();
+  CheckStoreLoad<atomic_uint8_t, memory_order_relaxed, memory_order_release>();
+  CheckStoreLoad<atomic_uint8_t, memory_order_seq_cst, memory_order_seq_cst>();
+
+  CheckStoreLoad<atomic_uint16_t, memory_order_relaxed, memory_order_relaxed>();
+  CheckStoreLoad<atomic_uint16_t, memory_order_consume, memory_order_relaxed>();
+  CheckStoreLoad<atomic_uint16_t, memory_order_acquire, memory_order_relaxed>();
+  CheckStoreLoad<atomic_uint16_t, memory_order_relaxed, memory_order_release>();
+  CheckStoreLoad<atomic_uint16_t, memory_order_seq_cst, memory_order_seq_cst>();
+
+  CheckStoreLoad<atomic_uint32_t, memory_order_relaxed, memory_order_relaxed>();
+  CheckStoreLoad<atomic_uint32_t, memory_order_consume, memory_order_relaxed>();
+  CheckStoreLoad<atomic_uint32_t, memory_order_acquire, memory_order_relaxed>();
+  CheckStoreLoad<atomic_uint32_t, memory_order_relaxed, memory_order_release>();
+  CheckStoreLoad<atomic_uint32_t, memory_order_seq_cst, memory_order_seq_cst>();
+
+  CheckStoreLoad<atomic_uint64_t, memory_order_relaxed, memory_order_relaxed>();
+  CheckStoreLoad<atomic_uint64_t, memory_order_consume, memory_order_relaxed>();
+  CheckStoreLoad<atomic_uint64_t, memory_order_acquire, memory_order_relaxed>();
+  CheckStoreLoad<atomic_uint64_t, memory_order_relaxed, memory_order_release>();
+  CheckStoreLoad<atomic_uint64_t, memory_order_seq_cst, memory_order_seq_cst>();
+
+  CheckStoreLoad<atomic_uintptr_t, memory_order_relaxed, memory_order_relaxed>
+      ();
+  CheckStoreLoad<atomic_uintptr_t, memory_order_consume, memory_order_relaxed>
+      ();
+  CheckStoreLoad<atomic_uintptr_t, memory_order_acquire, memory_order_relaxed>
+      ();
+  CheckStoreLoad<atomic_uintptr_t, memory_order_relaxed, memory_order_release>
+      ();
+  CheckStoreLoad<atomic_uintptr_t, memory_order_seq_cst, memory_order_seq_cst>
+      ();
+}
+
 // Clang crashes while compiling this test for Android:
 // http://llvm.org/bugs/show_bug.cgi?id=15587
 #if !SANITIZER_ANDROID
 template<typename T>
 void CheckAtomicCompareExchange() {
   typedef typename T::Type Type;
   {
     Type old_val = 42;