[libcxx] Implement <experimental/simd> aligned load and store when compiled with Clang.

[timshen]

Summary

Simply use attribute((align_value(...))).

libcxx/include/experimental/simd

 struct simd_size<_Tp, __simd_abi<__kind, _Np>>
     : std::integral_constant<size_t, _Np> {
   static_assert(
       std::is_arithmetic<_Tp>::value &&
           !std::is_same<typename std::remove_const<_Tp>::type, bool>::value,
       "Element type should be vectorizable");
 };
 
-template <class _SimdType, class _ValueType = typename _SimdType::value_type>
-struct memory_alignment;
+template <class _ValueType, class _Up, class _Flags>
+struct __memory_alignment_impl : std::integral_constant<size_t, alignof(_Up)> {
+};
 
-// TODO: May extend this after implementing vector_aligned.
 template <class _Tp, class _Abi, class _Up>
-struct memory_alignment<simd<_Tp, _Abi>, _Up>
+struct __memory_alignment_impl<simd<_Tp, _Abi>, _Up, vector_aligned_tag>
     : std::integral_constant<size_t, alignof(simd<_Tp, _Abi>)> {};
 
-template <class _Tp, class _Abi>
-struct memory_alignment<simd_mask<_Tp, _Abi>, bool>
-    : std::integral_constant<size_t, alignof(simd_mask<_Tp, _Abi>)> {};
+// TODO: Figure out a useful alignment based on simd_mask load and store
+// implementation. Currently, make sure that the buffer is suitable for aligned
+// SIMD load.
+template <class _Tp, class _Abi, class _Up>
+struct __memory_alignment_impl<simd_mask<_Tp, _Abi>, _Up, vector_aligned_tag>
+    : std::integral_constant<size_t, alignof(simd<uint8_t, _Abi>)> {};
+
+template <class _ValueType, class _Up, size_t __alignment>
+struct __memory_alignment_impl<_ValueType, _Up, overaligned_tag<__alignment>>
+    : std::integral_constant<size_t, __alignment> {};
+
+template <class _SimdType, class _Up = typename _SimdType::value_type>
+struct memory_alignment
+    : __memory_alignment_impl<_SimdType, _Up, vector_aligned_tag> {};
 
 #if _LIBCPP_STD_VER >= 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES)
 template <class _Tp, class _Abi = simd_abi::compatible<_Tp>>
 _LIBCPP_INLINE_VAR constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
 
-template <class _Tp, class _Up = typename _Tp::value_type>
+template <class _SimdType, class _Up = typename _SimdType::value_type>
 _LIBCPP_INLINE_VAR constexpr size_t memory_alignment_v =
-    memory_alignment<_Tp, _Up>::value;
+    memory_alignment<_SimdType, _Up>::value;
 #endif
 
 // class template simd [simd.class]
 template <class _Tp>
 using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
 template <class _Tp, int _Np>
 using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
 
private:
   template <class _Generator, size_t... __indices>
   void __generator_init(_Generator&& __g, std::index_sequence<__indices...>) {
     int __unused[] = {
         ((*this)[__indices] = __g(std::integral_constant<size_t, __indices>()),
          0)...};
     (void)__unused;
   }
 
+  template <size_t __alignment, class _Up>
+  void __copy_from_impl(const _Up* __buffer
+                        __attribute__((align_value(__alignment)))) {
+    for (size_t __i = 0; __i < size(); __i++) {
+      (*this)[__i] = static_cast<_Tp>(__buffer[__i]);
+    }
+  }
+
+  template <size_t __alignment, class _Up>
+  void __copy_to_impl(_Up* __buffer
+                      __attribute__((align_value(__alignment)))) const {
+    for (size_t __i = 0; __i < size(); __i++) {
+      __buffer[__i] = static_cast<_Up>((*this)[__i]);
+    }
+  }
+
 public:
   // implicit type conversion constructor
   template <class _Up,
             class = typename std::enable_if<
                 std::is_same<_Abi, simd_abi::fixed_size<size()>>::value &&
                 __is_non_narrowing_arithmetic_convertible<_Up, _Tp>()>::type>
   simd(const simd<_Up, simd_abi::fixed_size<size()>>& __v) {
     for (size_t __i = 0; __i < size(); __i++) {
public:
   }
 
   // load constructor
   template <
       class _Up, class _Flags,
       class = typename std::enable_if<__vectorizable<_Up>()>::type,
       class = typename std::enable_if<is_simd_flag_type<_Flags>::value>::type>
   simd(const _Up* __buffer, _Flags) {
-    // TODO: optimize for overaligned flags
-    for (size_t __i = 0; __i < size(); __i++) {
-      (*this)[__i] = static_cast<_Tp>(__buffer[__i]);
-    }
+    __copy_from_impl<__memory_alignment_impl<simd, _Up, _Flags>::value>(
+        __buffer);
   }
 
   // loads [simd.load]
   template <class _Up, class _Flags>
   typename std::enable_if<__vectorizable<_Up>() &&
                           is_simd_flag_type<_Flags>::value>::type
   copy_from(const _Up* __buffer, _Flags) {
     *this = simd(__buffer, _Flags());
   }
 
   // stores [simd.store]
   template <class _Up, class _Flags>
   typename std::enable_if<__vectorizable<_Up>() &&
                           is_simd_flag_type<_Flags>::value>::type
   copy_to(_Up* __buffer, _Flags) const {
-    // TODO: optimize for overaligned flags
-    for (size_t __i = 0; __i < size(); __i++) {
-      __buffer[__i] = static_cast<_Up>((*this)[__i]);
-    }
+    __copy_to_impl<__memory_alignment_impl<simd, _Up, _Flags>::value>(__buffer);
   }
 
   // scalar access [simd.subscr]
   reference operator[](size_t __i) { return reference(&__s_, __i); }
 
   value_type operator[](size_t __i) const { return __s_.__get(__i); }
 
   // unary operators [simd.unary]
class simd_mask {
   using __element_type = typename __unsigned_traits<sizeof(_Tp)>::type;
 
   simd<__element_type, _Abi> __s_;
 
   simd_mask(const simd<__element_type, _Abi>& __mask) : __s_(__mask) {}
 
   friend struct __simd_mask_friend;
 
+  // Use a non-member function, only because Clang 3.8 crashes with a member function.
+  template <size_t __alignment>
+  static void __copy_from_impl(simd_mask* __mask, const bool* __buffer
+                               __attribute__((align_value(__alignment)))) {
+    for (size_t __i = 0; __i < size(); __i++) {
+      (*__mask)[__i] = __buffer[__i];
+    }
+  }
+
+  // Use a non-member function, only because Clang 3.8 crashes with a member function.
+  template <size_t __alignment>
+  static void __copy_to_impl(const simd_mask* __mask, bool* __buffer
+                             __attribute__((align_value(__alignment)))) {
+    for (size_t __i = 0; __i < size(); __i++) {
+      __buffer[__i] = (*__mask)[__i];
+    }
+  }
+
 public:
   using value_type = bool;
   using reference = __simd_reference<bool, __element_type, _Abi>;
   using simd_type = simd<_Tp, _Abi>;
   using abi_type = _Abi;
 
   static constexpr size_t size() noexcept {
     return simd<__element_type, _Abi>::size();
for (size_t __i = 0; __i < size(); __i++) {
       (*this)[__i] = __v[__i];
     }
   }
 
   // load constructor
   template <class _Flags, class = typename std::enable_if<
                               is_simd_flag_type<_Flags>::value>::type>
   simd_mask(const value_type* __buffer, _Flags) {
-    // TODO: optimize for overaligned flags
-    for (size_t __i = 0; __i < size(); __i++) {
-      (*this)[__i] = __buffer[__i];
-    }
+    __copy_from_impl<__memory_alignment_impl<simd_mask, bool, _Flags>::value>(
+        this, __buffer);
   }
 
   template <class _Up = _Tp>
   simd_mask(typename __native_type_traits<__element_type,
                                           sizeof(_Up) * size()>::type __r)
       : __s_(__r) {}
 
   template <class _Up = _Tp>
public:
   typename std::enable_if<is_simd_flag_type<_Flags>::value>::type
   copy_from(const value_type* __buffer, _Flags) {
     *this = simd_mask(__buffer, _Flags());
   }
 
   template <class _Flags>
   typename std::enable_if<is_simd_flag_type<_Flags>::value>::type
   copy_to(value_type* __buffer, _Flags) const {
-    // TODO: optimize for overaligned flags
-    for (size_t __i = 0; __i < size(); __i++) {
-      __buffer[__i] = (*this)[__i];
-    }
+    __copy_to_impl<__memory_alignment_impl<simd_mask, bool, _Flags>::value>(
+        this, __buffer);
   }
 
   // scalar access [simd.mask.subscr]
   reference operator[](size_t __i) { return reference(&__s_.__s_, __i); }
 
   value_type operator[](size_t __i) const { return __s_[__i]; }
 
   // unary operators [simd.mask.unary]
public:
   }
 
   friend simd_mask operator!=(const simd_mask& __a,
                               const simd_mask& __b) noexcept {
     return __a ^ __b;
   }
 };
 
-template <class _Tp, class _Abi, class _Up, class _Flags>
+template <size_t __alignment, class _Tp, class _Abi, class _Up>
 void __mask_copy_to(const simd<_Tp, _Abi>& __v, const simd_mask<_Tp, _Abi>& __m,
-                    _Up* __buffer, _Flags) {
-  // TODO: optimize for overaligned flags
+                    _Up* __buffer __attribute__((align_value(__alignment)))) {
+  // TODO: optimize based on bool's bit pattern.
   for (size_t __i = 0; __i < __v.size(); __i++) {
     if (__m[__i]) {
       __buffer[__i] = static_cast<_Up>(__v[__i]);
     }
   }
 }
 
-template <class _Tp, class _Abi, class _Up, class _Flags>
+template <size_t __alignment, class _Tp, class _Abi, class _Up>
 void __mask_copy_to(const simd_mask<_Tp, _Abi>& __v,
-                    const simd_mask<_Tp, _Abi>& __m, _Up* __buffer, _Flags) {
+                    const simd_mask<_Tp, _Abi>& __m,
+                    _Up* __buffer __attribute__((align_value(__alignment)))) {
   // TODO: optimize based on bool's bit pattern.
   for (size_t __i = 0; __i < __v.size(); __i++) {
     if (__m[__i]) {
       __buffer[__i] = static_cast<_Up>(__v[__i]);
     }
   }
 }
 
-template <class _Tp, class _Up, class _Flags>
-void __mask_copy_to(_Tp __val, bool __m, _Up* __buffer, _Flags) {
+template <size_t __alignment, class _Tp, class _Up>
+void __mask_copy_to(_Tp __val, bool __m,
+                    _Up* __buffer __attribute__((align_value(__alignment)))) {
   if (__m) {
     *__buffer = static_cast<_Up>(__val);
   }
 }
 
-template <class _Tp, class _Abi, class _Up, class _Flags>
+template <size_t __alignment, class _Tp, class _Abi, class _Up>
 void __mask_copy_from(simd<_Tp, _Abi>& __v, const simd_mask<_Tp, _Abi>& __m,
-                      const _Up* __buffer, _Flags) {
-  // TODO: optimize for overaligned flags
+                      const _Up* __buffer
+                      __attribute__((align_value(__alignment)))) {
+  // TODO: optimize based on bool's bit pattern.
   for (size_t __i = 0; __i < __v.size(); __i++) {
     if (__m[__i]) {
       __v[__i] = static_cast<_Tp>(__buffer[__i]);
     }
   }
 }
 
-template <class _Tp, class _Abi, class _Up, class _Flags>
+template <size_t __alignment, class _Tp, class _Abi, class _Up>
 void __mask_copy_from(simd_mask<_Tp, _Abi>& __v,
-                      const simd_mask<_Tp, _Abi>& __m, const _Up* __buffer,
-                      _Flags) {
+                      const simd_mask<_Tp, _Abi>& __m,
+                      const _Up* __buffer
+                      __attribute__((align_value(__alignment)))) {
   // TODO: optimize based on bool's bit pattern.
   for (size_t __i = 0; __i < __v.size(); __i++) {
     if (__m[__i]) {
       __v[__i] = static_cast<bool>(__buffer[__i]);
     }
   }
 }
 
-template <class _Tp, class _Up, class _Flags>
-void __mask_copy_from(_Tp& __val, bool __m, const _Up* __buffer, _Flags) {
+template <size_t __alignment, class _Tp, class _Up>
+void __mask_copy_from(_Tp& __val, bool __m,
+                      const _Up* __buffer
+                      __attribute__((align_value(__alignment)))) {
   if (__m) {
     __val = static_cast<_Tp>(*__buffer);
   }
 }
 
 template <class _ValueType>
 struct __simd_value_type_traits {
   static_assert(std::is_arithmetic<_ValueType>::value, "");
typename std::remove_const<_ValueType>::type operator-() const&& {
     return __simd_mask_friend::__simd_select(__v_, _ValueType(0), __m_) -
            __simd_mask_friend::__simd_select(_ValueType(0), __v_, __m_);
   }
 
   template <class _Up, class _Flags>
   typename std::enable_if<std::is_same<_Tp, _Up>::value ||
                           !std::is_same<_Tp, bool>::value>::type
   copy_to(_Up* __buffer, _Flags) const&& {
-    __mask_copy_to(__v_, __m_, __buffer, _Flags());
+    __mask_copy_to<__memory_alignment_impl<_ValueType, _Up, _Flags>::value>(
+        __v_, __m_, __buffer);
   }
 };
 
 template <class _MaskType, class _ValueType>
 class where_expression : public const_where_expression<_MaskType, _ValueType> {
   using _Tp = typename __simd_value_type_traits<
       typename remove_const<_ValueType>::type>::type;
 
public:
   void operator--() { *this -= _ValueType(1); }
 
   void operator--(int) { --*this; }
 
   template <class _Up, class _Flags>
   typename std::enable_if<std::is_same<_Tp, _Up>::value ||
                           !std::is_same<_Tp, bool>::value>::type
   copy_from(const _Up* __buffer, _Flags) {
-    __mask_copy_from(this->__v_, this->__m_, __buffer, _Flags());
+    __mask_copy_from<__memory_alignment_impl<_ValueType, _Up, _Flags>::value>(
+        this->__v_, this->__m_, __buffer);
   }
 };
 
 template <class _Tp, class _Abi>
 where_expression<simd_mask<_Tp, _Abi>, simd<_Tp, _Abi>>
 where(const typename simd<_Tp, _Abi>::mask_type& __m,
       simd<_Tp, _Abi>& __v) noexcept {
   return where_expression<simd_mask<_Tp, _Abi>, simd<_Tp, _Abi>>(__m, __v);