diff --git a/libcxx/docs/Status/ParallelismProjects.csv b/libcxx/docs/Status/ParallelismProjects.csv --- a/libcxx/docs/Status/ParallelismProjects.csv +++ b/libcxx/docs/Status/ParallelismProjects.csv @@ -4,12 +4,13 @@ | `[parallel.alg] `_, "Parallel algorithms", None, unassigned, |Not Started| | `[parallel.taskblock] `_, "Task Block", None, unassigned, |Not Started| | `[parallel.simd.abi] `_, "`simd ABI tags `_", [parallel.simd.class] [parallel.simd.mask.class] declarations and alias, Yin Zhang, |Complete| +| `[parallel.simd.aligned] `_, "`simd aligned tags `_", [parallel.simd.class] [parallel.simd.mask.class] declarations, Yin Zhang, |Complete| | `[parallel.simd.traits] `_, "`simd type traits is_abi_tag[_v] `_", [parallel.simd.abi], Yin Zhang, |Complete| | `[parallel.simd.traits] `_, "`simd type traits is_simd[_v] `_", [parallel.simd.class] declaration and alias, Yin Zhang, |Complete| | `[parallel.simd.traits] `_, "`simd type traits is_simd_mask[_v] `_", [parallel.simd.mask.class] declaration and alias, Yin Zhang, |Complete| | `[parallel.simd.traits] `_, "simd type traits is_simd_flag_type", None, Yin Zhang, |In Progress| | `[parallel.simd.traits] `_, "`simd type traits simd_size[_v] `_", [parallel.simd.traits] is_abi_tag[_v], Yin Zhang, |Complete| -| `[parallel.simd.traits] `_, "simd type traits memory_alignment", None, Yin Zhang, |In Progress| +| `[parallel.simd.traits] `_, "`simd type traits memory_alignment `_", [parallel.simd.aligned] simd aligned tags, Yin Zhang, |Complete| | `[parallel.simd.traits] `_, "simd type traits rebind_simd", None, Yin Zhang, |In Progress| | `[parallel.simd.traits] `_, "simd type traits resize_simd", None, Yin Zhang, |In Progress| | `[parallel.simd.whereexpr] `_, "Where expression class templates", None, Yin Zhang, |In Progress| diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -886,6 +886,7 @@ experimental/__config experimental/__memory experimental/__simd/abi_tag.h + experimental/__simd/aligned_tag.h experimental/__simd/declaration.h experimental/__simd/scalar.h experimental/__simd/simd.h diff --git a/libcxx/include/experimental/__simd/aligned_tag.h b/libcxx/include/experimental/__simd/aligned_tag.h new file mode 100644 --- /dev/null +++ b/libcxx/include/experimental/__simd/aligned_tag.h @@ -0,0 +1,62 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H +#define _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H + +#include + +#if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) + +_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL +inline namespace parallelism_v2 { +// memory alignment +struct element_aligned_tag { + template + static constexpr size_t __alignment = alignof(_Up); + + template + static _LIBCPP_HIDE_FROM_ABI constexpr _Up* __apply(_Up* __ptr) { + return __ptr; + } +}; + +struct vector_aligned_tag { + template + static constexpr size_t __alignment = std::experimental::parallelism_v2::__bit_ceil(sizeof(_Up) * _Tp::size()); + + template + static _LIBCPP_HIDE_FROM_ABI constexpr _Up* __apply(_Up* __ptr) { + return __assume_aligned<__alignment<_Tp, _Up>, _Up>(__ptr); + } +}; + +template +struct overaligned_tag { + template + static constexpr size_t __alignment = _Np; + + template + static _LIBCPP_HIDE_FROM_ABI constexpr _Up* __apply(_Up* __ptr) { + return __assume_aligned<__alignment<_Tp, _Up>, _Up>(__ptr); + } +}; + +inline constexpr element_aligned_tag element_aligned{}; + +inline constexpr vector_aligned_tag vector_aligned{}; + +template +inline constexpr overaligned_tag<_Np> overaligned{}; + +} // namespace parallelism_v2 +_LIBCPP_END_NAMESPACE_EXPERIMENTAL + +#endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) +#endif // _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H diff --git a/libcxx/include/experimental/__simd/traits.h b/libcxx/include/experimental/__simd/traits.h --- a/libcxx/include/experimental/__simd/traits.h +++ b/libcxx/include/experimental/__simd/traits.h @@ -11,6 +11,7 @@ #define _LIBCPP_EXPERIMENTAL___SIMD_TRAITS_H #include +#include #include #include @@ -50,6 +51,21 @@ template struct is_simd_mask : bool_constant> {}; +template +inline constexpr bool is_simd_flag_type_v = false; + +template <> +inline constexpr bool is_simd_flag_type_v = true; + +template <> +inline constexpr bool is_simd_flag_type_v = true; + +template +inline constexpr bool is_simd_flag_type_v> = true; + +template +struct is_simd_flag_type : bool_constant> {}; + template , class = enable_if_t<__is_vectorizable_v<_Tp> && is_abi_tag_v<_Abi>>> @@ -58,6 +74,15 @@ template > struct simd_size : integral_constant> {}; +template < + class _Tp, + class _Up = typename _Tp::value_type, + class = enable_if_t<(is_simd_v<_Tp> && __is_vectorizable_v<_Up>) || (is_simd_mask_v<_Tp> && is_same_v<_Up, bool>)>> +inline constexpr size_t memory_alignment_v = vector_aligned_tag::__alignment<_Tp, _Up>; + +template +struct memory_alignment : integral_constant> {}; + } // namespace parallelism_v2 _LIBCPP_END_NAMESPACE_EXPERIMENTAL diff --git a/libcxx/include/experimental/__simd/utility.h b/libcxx/include/experimental/__simd/utility.h --- a/libcxx/include/experimental/__simd/utility.h +++ b/libcxx/include/experimental/__simd/utility.h @@ -12,6 +12,7 @@ #include <__type_traits/is_arithmetic.h> #include <__type_traits/is_const.h> +#include <__type_traits/is_constant_evaluated.h> #include <__type_traits/is_same.h> #include <__type_traits/is_volatile.h> @@ -22,6 +23,25 @@ template constexpr bool __is_vectorizable_v = is_arithmetic_v<_Tp> && !is_const_v<_Tp> && !is_volatile_v<_Tp> && !is_same_v<_Tp, bool>; + +_LIBCPP_HIDE_FROM_ABI constexpr size_t __bit_ceil(size_t __val) { + size_t __pow = 1; + while (__pow < __val) + __pow <<= 1; + return __pow; +} + +template +[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp* __assume_aligned(_Tp* __ptr) { + static_assert(_Np != 0 && (_Np & (_Np - 1)) == 0, "std::assume_aligned(p) requires N to be a power of two"); + + if (__libcpp_is_constant_evaluated()) { + return __ptr; + } else { + _LIBCPP_ASSERT(reinterpret_cast(__ptr) % _Np == 0, "Alignment assumption is violated"); + return static_cast<_Tp*>(__builtin_assume_aligned(__ptr, _Np)); + } +} } // namespace parallelism_v2 _LIBCPP_END_NAMESPACE_EXPERIMENTAL diff --git a/libcxx/include/experimental/simd b/libcxx/include/experimental/simd --- a/libcxx/include/experimental/simd +++ b/libcxx/include/experimental/simd @@ -37,6 +37,14 @@ template using native_simd_mask = simd_mask>; template using fixed_size_simd_mask = simd_mask>; + // memory alignment + struct element_aligned_tag {}; + struct vector_aligned_tag {}; + template struct overaligned_tag {}; + inline constexpr element_aligned_tag element_aligned{}; + inline constexpr vector_aligned_tag vector_aligned{}; + template inline constexpr overaligned_tag overaligned{}; + // traits [simd.traits] template struct is_abi_tag; template inline constexpr bool is_abi_tag_v = is_abi_tag::value; @@ -47,10 +55,17 @@ template struct is_simd_mask; template inline constexpr bool is_simd_mask_v = is_simd_mask::value; + template struct is_simd_flag_type; + template inline constexpr bool is_simd_flag_type_v = is_simd_flag_type::value; + template> struct simd_size; template> inline constexpr size_t simd_size_v = simd_size::value; + template struct memory_alignment; + template + inline constexpr size_t memory_alignment_v = memory_alignment::value; + } // namespace parallelism_v2 } // namespace std::experimental @@ -64,6 +79,7 @@ #include #include +#include #include #include #include diff --git a/libcxx/test/std/experimental/simd/simd.traits/is_simd_flag_type.pass.cpp b/libcxx/test/std/experimental/simd/simd.traits/is_simd_flag_type.pass.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/std/experimental/simd/simd.traits/is_simd_flag_type.pass.cpp @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14 + +// +// +// [simd.traits] +// template struct is_simd_flag_type; +// template inline constexpr bool ex::is_simd_flag_type_v = ex::is_simd_flag_type::value; + +#include "../test_utils.h" + +namespace ex = std::experimental::parallelism_v2; + +template +struct CheckIsSimdFlagType { + template + void operator()() { + static_assert(ex::is_simd_flag_type::value); + static_assert(ex::is_simd_flag_type::value); + static_assert(ex::is_simd_flag_type>::value); + + static_assert(!ex::is_simd_flag_type::value); + static_assert(!ex::is_simd_flag_type>::value); + static_assert(!ex::is_simd_flag_type>::value); + + static_assert(ex::is_simd_flag_type_v); + static_assert(ex::is_simd_flag_type_v); + static_assert(ex::is_simd_flag_type_v>); + + static_assert(!ex::is_simd_flag_type_v); + static_assert(!ex::is_simd_flag_type_v>); + static_assert(!ex::is_simd_flag_type_v>); + } +}; + +int main(int, char**) { + test_all_simd_abi(); + return 0; +} diff --git a/libcxx/test/std/experimental/simd/simd.traits/memory_alignment.pass.cpp b/libcxx/test/std/experimental/simd/simd.traits/memory_alignment.pass.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/std/experimental/simd/simd.traits/memory_alignment.pass.cpp @@ -0,0 +1,98 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14 + +// +// +// [simd.traits] +// template struct memory_alignment; +// template +// inline constexpr std::size_t memory_alignment_v = memory_alignment::value; + +#include "../test_utils.h" + +namespace ex = std::experimental::parallelism_v2; + +template +struct CheckMemoryAlignmentMask { + template + void operator()() { + static_assert(ex::memory_alignment>{}.value == + bit_ceil(sizeof(bool) * ex::simd_size_v)); + static_assert( + ex::memory_alignment_v> == bit_ceil(sizeof(bool) * ex::simd_size_v)); + } +}; + +template +struct CheckMemoryAlignmentLongDouble { + template + void operator()() { + if constexpr (std::is_same_v) { + // The size of long double is platform-specific. For instance, on i686-w64-mingw32-clang++, + // the size of long double is 12 bytes. Hence, disambiguation is needed. + static_assert( + ex::memory_alignment>{}.value == bit_ceil(sizeof(T) * ex::simd_size_v)); + static_assert(ex::memory_alignment_v> == bit_ceil(sizeof(T) * ex::simd_size_v)); + } + } +}; + +struct CheckMemAlignmentFixedDeduce { + template + void check() { + if constexpr (!std::is_same_v) { + static_assert(ex::memory_alignment_v>> == sizeof(T) * bit_ceil(N), + "Memory Alignment mismatch with abi fixed_size"); + static_assert(ex::memory_alignment>>{}.value == sizeof(T) * bit_ceil(N), + "Memory Alignment mismatch with abi fixed_size"); + + static_assert(ex::memory_alignment_v>> == sizeof(T) * bit_ceil(N), + "Memory Alignment mismatch with abi deduce"); + static_assert(ex::memory_alignment>>{}.value == sizeof(T) * bit_ceil(N), + "Memory Alignment mismatch with abi deduce"); + } + } + + template + void performChecks(std::index_sequence) { + (check(), ...); + } + + template + void operator()() { + performChecks(std::make_index_sequence{}); + } +}; + +struct CheckMemAlignmentScalarNativeCompatible { + template + void operator()() { + if constexpr (!std::is_same_v) { + static_assert(ex::memory_alignment>{}.value == sizeof(T)); + static_assert(ex::memory_alignment_v> == sizeof(T)); + + LIBCPP_STATIC_ASSERT(ex::memory_alignment>>{}.value == 16); + LIBCPP_STATIC_ASSERT(ex::memory_alignment_v>> == 16); + + LIBCPP_STATIC_ASSERT( + ex::memory_alignment>>{}.value == _LIBCPP_NATIVE_SIMD_WIDTH_IN_BYTES); + LIBCPP_STATIC_ASSERT( + ex::memory_alignment_v>> == _LIBCPP_NATIVE_SIMD_WIDTH_IN_BYTES); + } + } +}; + +int main(int, char**) { + types::for_each(arithmetic_no_bool_types(), CheckMemAlignmentFixedDeduce()); + types::for_each(arithmetic_no_bool_types(), CheckMemAlignmentScalarNativeCompatible()); + test_all_simd_abi(); + test_all_simd_abi(); + return 0; +} diff --git a/libcxx/test/std/experimental/simd/test_utils.h b/libcxx/test/std/experimental/simd/test_utils.h --- a/libcxx/test/std/experimental/simd/test_utils.h +++ b/libcxx/test/std/experimental/simd/test_utils.h @@ -43,4 +43,11 @@ types::for_each(arithmetic_no_bool_types(), TestAllSimdAbiFunctor()); } +constexpr size_t bit_ceil(size_t val) { + size_t pow = 1; + while (pow < val) + pow <<= 1; + return pow; +} + #endif // TEST_UTIL_H \ No newline at end of file