diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -886,6 +886,7 @@ experimental/__config experimental/__memory experimental/__simd/abi_tag.h + experimental/__simd/aligned_tag.h experimental/__simd/declaration.h experimental/__simd/scalar.h experimental/__simd/simd.h diff --git a/libcxx/include/experimental/__simd/aligned_tag.h b/libcxx/include/experimental/__simd/aligned_tag.h new file mode 100644 --- /dev/null +++ b/libcxx/include/experimental/__simd/aligned_tag.h @@ -0,0 +1,62 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H +#define _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H + +#include + +#if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) + +_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL +inline namespace parallelism_v2 { +// memory alignment +struct element_aligned_tag { + template + static constexpr size_t __alignment = alignof(_Up); + + template + static _LIBCPP_HIDE_FROM_ABI constexpr _Up* __apply(_Up* __ptr) { + return __ptr; + } +}; + +struct vector_aligned_tag { + template + static constexpr size_t __alignment = std::experimental::parallelism_v2::__next_pow_of_2(sizeof(_Up) * _Tp::size()); + + template + static _LIBCPP_HIDE_FROM_ABI constexpr _Up* __apply(_Up* __ptr) { + return static_cast<_Up*>(__builtin_assume_aligned(__ptr, __alignment<_Tp, _Up>)); + } +}; + +template +struct overaligned_tag { + template + static constexpr size_t __alignment = _Np; + + template + static _LIBCPP_HIDE_FROM_ABI constexpr _Up* __apply(_Up* __ptr) { + return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _Np)); + } +}; + +inline constexpr element_aligned_tag element_aligned{}; + +inline constexpr vector_aligned_tag vector_aligned{}; + +template +inline constexpr overaligned_tag<_Np> overaligned{}; + +} // namespace parallelism_v2 +_LIBCPP_END_NAMESPACE_EXPERIMENTAL + +#endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) +#endif // _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H diff --git a/libcxx/include/experimental/__simd/traits.h b/libcxx/include/experimental/__simd/traits.h --- a/libcxx/include/experimental/__simd/traits.h +++ b/libcxx/include/experimental/__simd/traits.h @@ -11,6 +11,7 @@ #define _LIBCPP_EXPERIMENTAL___SIMD_TRAITS_H #include +#include #include #include @@ -50,6 +51,21 @@ template struct is_simd_mask : bool_constant> {}; +template +inline constexpr bool is_simd_flag_type_v = false; + +template <> +inline constexpr bool is_simd_flag_type_v = true; + +template <> +inline constexpr bool is_simd_flag_type_v = true; + +template +inline constexpr bool is_simd_flag_type_v> = true; + +template +struct is_simd_flag_type : bool_constant> {}; + template , class = enable_if_t<__is_vectorizable<_Tp>() && is_abi_tag_v<_Abi>>> @@ -58,6 +74,15 @@ template > struct simd_size : integral_constant> {}; +template < + class _Tp, + class _Up = typename _Tp::value_type, + class = enable_if_t<(is_simd_v<_Tp> && __is_vectorizable<_Up>()) || (is_simd_mask_v<_Tp> && is_same_v<_Up, bool>)>> +inline constexpr size_t memory_alignment_v = vector_aligned_tag::__alignment<_Tp, _Up>; + +template +struct memory_alignment : integral_constant> {}; + } // namespace parallelism_v2 _LIBCPP_END_NAMESPACE_EXPERIMENTAL diff --git a/libcxx/include/experimental/__simd/utility.h b/libcxx/include/experimental/__simd/utility.h --- a/libcxx/include/experimental/__simd/utility.h +++ b/libcxx/include/experimental/__simd/utility.h @@ -20,6 +20,14 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __is_vectorizable() { return is_arithmetic_v<_Tp> && !is_const_v<_Tp> && !is_volatile_v<_Tp> && !is_same_v<_Tp, bool>; } + +// TODO: replace it by std::bit_ceil when bump to C++20 +_LIBCPP_HIDE_FROM_ABI constexpr size_t __next_pow_of_2(size_t __val) { + size_t __pow = 1; + while (__pow < __val) + __pow <<= 1; + return __pow; +} } // namespace parallelism_v2 _LIBCPP_END_NAMESPACE_EXPERIMENTAL diff --git a/libcxx/include/experimental/simd b/libcxx/include/experimental/simd --- a/libcxx/include/experimental/simd +++ b/libcxx/include/experimental/simd @@ -37,6 +37,14 @@ template using native_simd_mask = simd_mask>; template using fixed_size_simd_mask = simd_mask>; + // memory alignment + struct element_aligned_tag {}; + struct vector_aligned_tag {}; + template struct overaligned_tag {}; + inline constexpr element_aligned_tag element_aligned{}; + inline constexpr vector_aligned_tag vector_aligned{}; + template inline constexpr overaligned_tag overaligned{}; + // traits [simd.traits] template struct is_abi_tag; template inline constexpr bool is_abi_tag_v = is_abi_tag::value; @@ -47,10 +55,17 @@ template struct is_simd_mask; template inline constexpr bool is_simd_mask_v = is_simd_mask::value; + template struct is_simd_flag_type; + template inline constexpr bool is_simd_flag_type_v = is_simd_flag_type::value; + template> struct simd_size; template> inline constexpr size_t simd_size_v = simd_size::value; + template struct memory_alignment; + template + inline constexpr size_t memory_alignment_v = memory_alignment::value; + } // namespace parallelism_v2 } // namespace std::experimental @@ -64,6 +79,7 @@ #include #include +#include #include #include #include diff --git a/libcxx/test/std/experimental/simd/simd.traits/is_simd_flag_type.pass.cpp b/libcxx/test/std/experimental/simd/simd.traits/is_simd_flag_type.pass.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/std/experimental/simd/simd.traits/is_simd_flag_type.pass.cpp @@ -0,0 +1,50 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14 + +// +// +// [simd.traits] +// template struct is_simd_flag_type; +// template inline constexpr bool ex::is_simd_flag_type_v = ex::is_simd_flag_type::value; + +#include "../test_utils.h" +#include + +namespace ex = std::experimental::parallelism_v2; + +class EmptyEntry {}; + +template +void test_simd_abi() {} +template +void test_simd_abi() { + static_assert(ex::is_simd_flag_type::value); + static_assert(ex::is_simd_flag_type::value); + static_assert(ex::is_simd_flag_type>::value); + + static_assert(!ex::is_simd_flag_type<_Tp>::value); + static_assert(!ex::is_simd_flag_type>::value); + static_assert(!ex::is_simd_flag_type>::value); + + static_assert(ex::is_simd_flag_type_v); + static_assert(ex::is_simd_flag_type_v); + static_assert(ex::is_simd_flag_type_v>); + + static_assert(!ex::is_simd_flag_type_v<_Tp>); + static_assert(!ex::is_simd_flag_type_v>); + static_assert(!ex::is_simd_flag_type_v>); + + test_simd_abi(); +} + +int main(int, char**) { + test_all_simd_abi(); + return 0; +} diff --git a/libcxx/test/std/experimental/simd/simd.traits/memory_alignment.pass.cpp b/libcxx/test/std/experimental/simd/simd.traits/memory_alignment.pass.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/std/experimental/simd/simd.traits/memory_alignment.pass.cpp @@ -0,0 +1,98 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14 + +// +// +// [simd.traits] +// template struct memory_alignment; +// template +// inline constexpr size_t memory_alignment_v = memory_alignment::value; + +#include "../test_utils.h" +#include + +namespace ex = std::experimental::parallelism_v2; + +struct CheckMemoryAlignmentNative { + template + void operator()() { + static_assert(ex::memory_alignment>{}.value == _LIBCPP_NATIVE_SIMD_WIDTH_IN_BYTES); + static_assert(ex::memory_alignment_v> == _LIBCPP_NATIVE_SIMD_WIDTH_IN_BYTES); + } +}; + +struct CheckMemoryAlignmentCompatible { + template + void operator()() { + static_assert(ex::memory_alignment>{}.value == 16); + static_assert(ex::memory_alignment_v> == 16); + } +}; + +struct CheckMemoryAlignmentScalar { + template + void operator()() { + static_assert(ex::memory_alignment>{}.value == sizeof(_Tp)); + static_assert(ex::memory_alignment_v> == sizeof(_Tp)); + } +}; + +struct CheckMemoryAlignmentFixedSize { + template + void operator()() { + static_assert(ex::memory_alignment>{}.value == sizeof(_Tp) * next_pow2(_Np)); + static_assert(ex::memory_alignment_v> == sizeof(_Tp) * next_pow2(_Np)); + } +}; + +struct CheckMemoryAlignmentDeduce { + template + void operator()() { + static_assert(ex::memory_alignment>{}.value == sizeof(_Tp) * next_pow2(_Np)); + static_assert(ex::memory_alignment_v> == sizeof(_Tp) * next_pow2(_Np)); + } +}; + +struct CheckMemoryAlignmentMask { + template + void operator()() { + static_assert(ex::memory_alignment>{}.value == + next_pow2(sizeof(bool) * ex::simd_size_v<_Tp, SimdAbi>)); + static_assert( + ex::memory_alignment_v> == next_pow2(sizeof(bool) * ex::simd_size_v<_Tp, SimdAbi>)); + } +}; + +template +void test_simd_abi() {} +template +void test_simd_abi() { + if constexpr (std::is_same_v) { + F{}.template operator()<_Tp, ex::simd_abi::native<_Tp>, _Np>(); + } else if constexpr (std::is_same_v) { + F{}.template operator()<_Tp, ex::simd_abi::compatible<_Tp>, _Np>(); + } else if constexpr (std::is_same_v) { + F{}.template operator()<_Tp, ex::simd_abi::scalar, _Np>(); + } else if constexpr (std::is_same_v) { + F{}.template operator()<_Tp, ex::simd_abi::fixed_size<_Np>, _Np>(); + } else { + F{}.template operator()<_Tp, ex::simd_abi::deduce_t<_Tp, _Np>, _Np>(); + } + test_simd_abi(); +} + +int main(int, char**) { + test_all_simd_abi(); + test_all_simd_abi(); + test_all_simd_abi(); + test_all_simd_abi(); + test_all_simd_abi(); + return 0; +} diff --git a/libcxx/test/std/experimental/simd/test_utils.h b/libcxx/test/std/experimental/simd/test_utils.h --- a/libcxx/test/std/experimental/simd/test_utils.h +++ b/libcxx/test/std/experimental/simd/test_utils.h @@ -19,6 +19,14 @@ using arithmetic_no_bool_types = types::concatenate_t; +// TODO: replace it by std::bit_ceil when bump to C++20 +constexpr size_t next_pow2(size_t val) { + size_t pow = 1; + while (pow < val) + pow <<= 1; + return pow; +} + template void test_simd_abi();