diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -912,6 +912,8 @@ experimental/__simd/abi_tag.h experimental/__simd/aligned_tag.h experimental/__simd/declaration.h + experimental/__simd/internal_declaration.h + experimental/__simd/reference.h experimental/__simd/scalar.h experimental/__simd/simd.h experimental/__simd/simd_mask.h diff --git a/libcxx/include/experimental/__simd/scalar.h b/libcxx/include/experimental/__simd/internal_declaration.h copy from libcxx/include/experimental/__simd/scalar.h copy to libcxx/include/experimental/__simd/internal_declaration.h --- a/libcxx/include/experimental/__simd/scalar.h +++ b/libcxx/include/experimental/__simd/internal_declaration.h @@ -7,23 +7,28 @@ // //===----------------------------------------------------------------------===// -#ifndef _LIBCPP_EXPERIMENTAL___SIMD_SCALAR_H -#define _LIBCPP_EXPERIMENTAL___SIMD_SCALAR_H - -#include +#ifndef _LIBCPP_EXPERIMENTAL___SIMD_INTERNAL_DECLARATION_H +#define _LIBCPP_EXPERIMENTAL___SIMD_INTERNAL_DECLARATION_H #if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) _LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL inline namespace parallelism_v2 { -namespace simd_abi { -struct __scalar { - static constexpr size_t __simd_size = 1; -}; -} // namespace simd_abi +template +struct __simd_storage; + +template +struct __mask_storage; + +template +struct __simd_operations; + +template +struct __mask_operations; + } // namespace parallelism_v2 _LIBCPP_END_NAMESPACE_EXPERIMENTAL #endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) -#endif // _LIBCPP_EXPERIMENTAL___SIMD_SCALAR_H +#endif // _LIBCPP_EXPERIMENTAL___SIMD_INTERNAL_DECLARATION_H diff --git a/libcxx/include/experimental/__simd/reference.h b/libcxx/include/experimental/__simd/reference.h new file mode 100644 --- /dev/null +++ b/libcxx/include/experimental/__simd/reference.h @@ -0,0 +1,53 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP_EXPERIMENTAL___SIMD_REFERENCE_H +#define _LIBCPP_EXPERIMENTAL___SIMD_REFERENCE_H + +#include + +#if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) + +_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL +inline namespace parallelism_v2 { +template +class __simd_reference { + template + friend class simd; + template + friend class simd_mask; + + _Storage& __s_; + size_t __idx_; + + _LIBCPP_HIDE_FROM_ABI __simd_reference(_Storage& __s, size_t __idx) : __s_(__s), __idx_(__idx) {} + + _LIBCPP_HIDE_FROM_ABI _Vp __get() const { return __s_.__get(__idx_); } + + _LIBCPP_HIDE_FROM_ABI void __set(_Vp __v) { + if constexpr (is_same_v<_Vp, bool>) + __s_.__set(__idx_, std::experimental::parallelism_v2::__set_all_bits<_Tp>(__v)); + else + __s_.__set(__idx_, __v); + } + +public: + using value_type = _Vp; + + __simd_reference() = delete; + __simd_reference(const __simd_reference&) = delete; + + _LIBCPP_HIDE_FROM_ABI operator value_type() const noexcept { return __get(); } +}; + +} // namespace parallelism_v2 +_LIBCPP_END_NAMESPACE_EXPERIMENTAL + +#endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) +#endif // _LIBCPP_EXPERIMENTAL___SIMD_REFERENCE_H diff --git a/libcxx/include/experimental/__simd/scalar.h b/libcxx/include/experimental/__simd/scalar.h --- a/libcxx/include/experimental/__simd/scalar.h +++ b/libcxx/include/experimental/__simd/scalar.h @@ -11,6 +11,7 @@ #define _LIBCPP_EXPERIMENTAL___SIMD_SCALAR_H #include +#include #if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) @@ -20,8 +21,36 @@ struct __scalar { static constexpr size_t __simd_size = 1; }; - } // namespace simd_abi + +template +struct __simd_storage<_Tp, simd_abi::__scalar> { + _Tp __data; + + _Tp __get([[maybe_unused]] size_t __idx) const noexcept { + _LIBCPP_ASSERT_UNCATEGORIZED(__idx == 0, "Index is out of bounds"); + return __data; + } + void __set([[maybe_unused]] size_t __idx, _Tp __v) noexcept { + _LIBCPP_ASSERT_UNCATEGORIZED(__idx == 0, "Index is out of bounds"); + __data = __v; + } +}; + +template +struct __mask_storage<_Tp, simd_abi::__scalar> : __simd_storage {}; + +template +struct __simd_operations<_Tp, simd_abi::__scalar> { + using _SimdStorage = __simd_storage<_Tp, simd_abi::__scalar>; + using _MaskStorage = __mask_storage<_Tp, simd_abi::__scalar>; +}; + +template +struct __mask_operations<_Tp, simd_abi::__scalar> { + using _MaskStorage = __mask_storage<_Tp, simd_abi::__scalar>; +}; + } // namespace parallelism_v2 _LIBCPP_END_NAMESPACE_EXPERIMENTAL diff --git a/libcxx/include/experimental/__simd/simd.h b/libcxx/include/experimental/__simd/simd.h --- a/libcxx/include/experimental/__simd/simd.h +++ b/libcxx/include/experimental/__simd/simd.h @@ -12,7 +12,10 @@ #include #include +#include +#include #include +#include #if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) @@ -23,8 +26,14 @@ // TODO: implement simd class template class simd { + using _Impl = __simd_operations<_Tp, _Abi>; + using _Storage = typename _Impl::_SimdStorage; + + _Storage __s_; + public: using value_type = _Tp; + using reference = __simd_reference<_Tp, _Storage, value_type>; using mask_type = simd_mask<_Tp, _Abi>; using abi_type = _Abi; diff --git a/libcxx/include/experimental/__simd/simd_mask.h b/libcxx/include/experimental/__simd/simd_mask.h --- a/libcxx/include/experimental/__simd/simd_mask.h +++ b/libcxx/include/experimental/__simd/simd_mask.h @@ -12,6 +12,9 @@ #include #include +#include +#include +#include #if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) @@ -22,8 +25,14 @@ // TODO: implement simd_mask class template class simd_mask { + using _Impl = __mask_operations<_Tp, _Abi>; + using _Storage = typename _Impl::_MaskStorage; + + _Storage __s_; + public: using value_type = bool; + using reference = __simd_reference<_Tp, _Storage, value_type>; using simd_type = simd<_Tp, _Abi>; using abi_type = _Abi; diff --git a/libcxx/include/experimental/__simd/utility.h b/libcxx/include/experimental/__simd/utility.h --- a/libcxx/include/experimental/__simd/utility.h +++ b/libcxx/include/experimental/__simd/utility.h @@ -15,6 +15,8 @@ #include <__type_traits/is_constant_evaluated.h> #include <__type_traits/is_same.h> #include <__type_traits/is_volatile.h> +#include +#include #if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) @@ -23,6 +25,32 @@ template constexpr bool __is_vectorizable_v = is_arithmetic_v<_Tp> && !is_const_v<_Tp> && !is_volatile_v<_Tp> && !is_same_v<_Tp, bool>; + +template +_LIBCPP_HIDE_FROM_ABI auto __choose_mask_type() { + if constexpr (sizeof(_Tp) == 1) { + return uint8_t{}; + } else if constexpr (sizeof(_Tp) == 2) { + return uint16_t{}; + } else if constexpr (sizeof(_Tp) == 4) { + return uint32_t{}; + } else if constexpr (sizeof(_Tp) == 8) { + return uint64_t{}; + } +# ifndef _LIBCPP_HAS_NO_INT128 + else if constexpr (sizeof(_Tp) == 16) { + return __uint128_t{}; + } +# endif + else + static_assert(sizeof(_Tp) == 0, "Unexpected size"); +} + +template +_LIBCPP_HIDE_FROM_ABI auto constexpr __set_all_bits(bool __v) { + return __v ? std::numeric_limits())>::max() : 0; +} + } // namespace parallelism_v2 _LIBCPP_END_NAMESPACE_EXPERIMENTAL diff --git a/libcxx/include/experimental/__simd/vec_ext.h b/libcxx/include/experimental/__simd/vec_ext.h --- a/libcxx/include/experimental/__simd/vec_ext.h +++ b/libcxx/include/experimental/__simd/vec_ext.h @@ -10,7 +10,10 @@ #ifndef _LIBCPP_EXPERIMENTAL___SIMD_VEC_EXT_H #define _LIBCPP_EXPERIMENTAL___SIMD_VEC_EXT_H +#include <__bit/bit_ceil.h> #include +#include +#include #if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) @@ -21,8 +24,37 @@ struct __vec_ext { static constexpr size_t __simd_size = _Np; }; - } // namespace simd_abi + +template +struct __simd_storage<_Tp, simd_abi::__vec_ext<_Np>> { + _Tp __data __attribute__((vector_size(std::__bit_ceil((sizeof(_Tp) * _Np))))); + + _Tp __get(size_t __idx) const noexcept { + _LIBCPP_ASSERT_UNCATEGORIZED(__idx > 0 && __idx <= _Np, "Index is out of bounds"); + return __data[__idx]; + } + void __set(size_t __idx, _Tp __v) noexcept { + _LIBCPP_ASSERT_UNCATEGORIZED(__idx > 0 && __idx <= _Np, "Index is out of bounds"); + __data[__idx] = __v; + } +}; + +template +struct __mask_storage<_Tp, simd_abi::__vec_ext<_Np>> + : __simd_storage()), simd_abi::__vec_ext<_Np>> {}; + +template +struct __simd_operations<_Tp, simd_abi::__vec_ext<_Np>> { + using _SimdStorage = __simd_storage<_Tp, simd_abi::__vec_ext<_Np>>; + using _MaskStorage = __mask_storage<_Tp, simd_abi::__vec_ext<_Np>>; +}; + +template +struct __mask_operations<_Tp, simd_abi::__vec_ext<_Np>> { + using _MaskStorage = __mask_storage<_Tp, simd_abi::__vec_ext<_Np>>; +}; + } // namespace parallelism_v2 _LIBCPP_END_NAMESPACE_EXPERIMENTAL