diff --git a/libcxx/docs/Status/ParallelismProjects.csv b/libcxx/docs/Status/ParallelismProjects.csv --- a/libcxx/docs/Status/ParallelismProjects.csv +++ b/libcxx/docs/Status/ParallelismProjects.csv @@ -4,12 +4,13 @@ | `[parallel.alg] `_, "Parallel algorithms", None, unassigned, |Not Started| | `[parallel.taskblock] `_, "Task Block", None, unassigned, |Not Started| | `[parallel.simd.abi] `_, "`simd ABI tags `_", [parallel.simd.class] [parallel.simd.mask.class] declarations and alias, Yin Zhang, |Complete| +| `[parallel.simd.aligned] `_, "`simd aligned tags `_", [parallel.simd.class] [parallel.simd.mask.class] declarations, Yin Zhang, |Complete| | `[parallel.simd.traits] `_, "`simd type traits is_abi_tag[_v] `_", [parallel.simd.abi], Yin Zhang, |Complete| | `[parallel.simd.traits] `_, "`simd type traits is_simd[_v] `_", [parallel.simd.class] declaration and alias, Yin Zhang, |Complete| | `[parallel.simd.traits] `_, "`simd type traits is_simd_mask[_v] `_", [parallel.simd.mask.class] declaration and alias, Yin Zhang, |Complete| | `[parallel.simd.traits] `_, "simd type traits is_simd_flag_type", None, Yin Zhang, |In Progress| | `[parallel.simd.traits] `_, "`simd type traits simd_size[_v] `_", [parallel.simd.traits] is_abi_tag[_v], Yin Zhang, |Complete| -| `[parallel.simd.traits] `_, "simd type traits memory_alignment", None, Yin Zhang, |In Progress| +| `[parallel.simd.traits] `_, "`simd type traits memory_alignment `_", [parallel.simd.aligned] simd aligned tags, Yin Zhang, |Complete| | `[parallel.simd.traits] `_, "simd type traits rebind_simd", None, Yin Zhang, |In Progress| | `[parallel.simd.traits] `_, "simd type traits resize_simd", None, Yin Zhang, |In Progress| | `[parallel.simd.whereexpr] `_, "Where expression class templates", None, Yin Zhang, |In Progress| diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -886,6 +886,7 @@ experimental/__config experimental/__memory experimental/__simd/abi_tag.h + experimental/__simd/aligned_tag.h experimental/__simd/declaration.h experimental/__simd/scalar.h experimental/__simd/simd.h diff --git a/libcxx/include/__bit/bit_ceil.h b/libcxx/include/__bit/bit_ceil.h --- a/libcxx/include/__bit/bit_ceil.h +++ b/libcxx/include/__bit/bit_ceil.h @@ -21,13 +21,11 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if _LIBCPP_STD_VER >= 20 - -template <__libcpp_unsigned_integer _Tp> -_LIBCPP_HIDE_FROM_ABI constexpr _Tp bit_ceil(_Tp __t) noexcept { +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp __bit_ceil(_Tp __t) _NOEXCEPT { if (__t < 2) return 1; - const unsigned __n = numeric_limits<_Tp>::digits - std::countl_zero((_Tp)(__t - 1u)); + const unsigned __n = numeric_limits<_Tp>::digits - std::__countl_zero((_Tp)(__t - 1u)); _LIBCPP_ASSERT(__n != numeric_limits<_Tp>::digits, "Bad input to bit_ceil"); if constexpr (sizeof(_Tp) >= sizeof(unsigned)) @@ -39,6 +37,13 @@ } } +#if _LIBCPP_STD_VER >= 20 + +template <__libcpp_unsigned_integer _Tp> +_LIBCPP_HIDE_FROM_ABI constexpr _Tp bit_ceil(_Tp __t) noexcept { + return std::__bit_ceil(__t); +} + #endif // _LIBCPP_STD_VER >= 20 _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__memory/assume_aligned.h b/libcxx/include/__memory/assume_aligned.h --- a/libcxx/include/__memory/assume_aligned.h +++ b/libcxx/include/__memory/assume_aligned.h @@ -22,16 +22,12 @@ _LIBCPP_BEGIN_NAMESPACE_STD -#if _LIBCPP_STD_VER >= 20 - template -[[nodiscard]] -_LIBCPP_HIDE_FROM_ABI -constexpr _Tp* assume_aligned(_Tp* __ptr) { +[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp* __assume_aligned(_Tp* __ptr) { static_assert(_Np != 0 && (_Np & (_Np - 1)) == 0, "std::assume_aligned(p) requires N to be a power of two"); - if (is_constant_evaluated()) { + if (__libcpp_is_constant_evaluated()) { return __ptr; } else { _LIBCPP_ASSERT(reinterpret_cast(__ptr) % _Np == 0, "Alignment assumption is violated"); @@ -39,6 +35,13 @@ } } +#if _LIBCPP_STD_VER >= 20 + +template +[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp* assume_aligned(_Tp* __ptr) { + return std::__assume_aligned<_Np>(__ptr); +} + #endif // _LIBCPP_STD_VER >= 20 _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/experimental/__simd/aligned_tag.h b/libcxx/include/experimental/__simd/aligned_tag.h new file mode 100644 --- /dev/null +++ b/libcxx/include/experimental/__simd/aligned_tag.h @@ -0,0 +1,64 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H +#define _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H + +#include <__bit/bit_ceil.h> +#include <__memory/assume_aligned.h> +#include + +#if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) + +_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL +inline namespace parallelism_v2 { +// memory alignment +struct element_aligned_tag { + template + static constexpr size_t __alignment = alignof(_Up); + + template + static _LIBCPP_HIDE_FROM_ABI constexpr _Up* __apply(_Up* __ptr) { + return __ptr; + } +}; + +struct vector_aligned_tag { + template + static constexpr size_t __alignment = __bit_ceil(sizeof(_Up) * _Tp::size()); + + template + static _LIBCPP_HIDE_FROM_ABI constexpr _Up* __apply(_Up* __ptr) { + return __assume_aligned<__alignment<_Tp, _Up>, _Up>(__ptr); + } +}; + +template +struct overaligned_tag { + template + static constexpr size_t __alignment = _Np; + + template + static _LIBCPP_HIDE_FROM_ABI constexpr _Up* __apply(_Up* __ptr) { + return __assume_aligned<__alignment<_Tp, _Up>, _Up>(__ptr); + } +}; + +inline constexpr element_aligned_tag element_aligned{}; + +inline constexpr vector_aligned_tag vector_aligned{}; + +template +inline constexpr overaligned_tag<_Np> overaligned{}; + +} // namespace parallelism_v2 +_LIBCPP_END_NAMESPACE_EXPERIMENTAL + +#endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) +#endif // _LIBCPP_EXPERIMENTAL___SIMD_ALIGNED_TAG_H diff --git a/libcxx/include/experimental/__simd/traits.h b/libcxx/include/experimental/__simd/traits.h --- a/libcxx/include/experimental/__simd/traits.h +++ b/libcxx/include/experimental/__simd/traits.h @@ -11,6 +11,7 @@ #define _LIBCPP_EXPERIMENTAL___SIMD_TRAITS_H #include +#include #include #include @@ -50,6 +51,21 @@ template struct is_simd_mask : bool_constant> {}; +template +inline constexpr bool is_simd_flag_type_v = false; + +template <> +inline constexpr bool is_simd_flag_type_v = true; + +template <> +inline constexpr bool is_simd_flag_type_v = true; + +template +inline constexpr bool is_simd_flag_type_v> = true; + +template +struct is_simd_flag_type : bool_constant> {}; + template , class = enable_if_t<__is_vectorizable_v<_Tp> && is_abi_tag_v<_Abi>>> @@ -58,6 +74,15 @@ template > struct simd_size : integral_constant> {}; +template < + class _Tp, + class _Up = typename _Tp::value_type, + class = enable_if_t<(is_simd_v<_Tp> && __is_vectorizable_v<_Up>) || (is_simd_mask_v<_Tp> && is_same_v<_Up, bool>)>> +inline constexpr size_t memory_alignment_v = vector_aligned_tag::__alignment<_Tp, _Up>; + +template +struct memory_alignment : integral_constant> {}; + } // namespace parallelism_v2 _LIBCPP_END_NAMESPACE_EXPERIMENTAL diff --git a/libcxx/include/experimental/__simd/utility.h b/libcxx/include/experimental/__simd/utility.h --- a/libcxx/include/experimental/__simd/utility.h +++ b/libcxx/include/experimental/__simd/utility.h @@ -12,6 +12,7 @@ #include <__type_traits/is_arithmetic.h> #include <__type_traits/is_const.h> +#include <__type_traits/is_constant_evaluated.h> #include <__type_traits/is_same.h> #include <__type_traits/is_volatile.h> diff --git a/libcxx/include/experimental/simd b/libcxx/include/experimental/simd --- a/libcxx/include/experimental/simd +++ b/libcxx/include/experimental/simd @@ -37,6 +37,14 @@ template using native_simd_mask = simd_mask>; template using fixed_size_simd_mask = simd_mask>; + // memory alignment + struct element_aligned_tag {}; + struct vector_aligned_tag {}; + template struct overaligned_tag {}; + inline constexpr element_aligned_tag element_aligned{}; + inline constexpr vector_aligned_tag vector_aligned{}; + template inline constexpr overaligned_tag overaligned{}; + // traits [simd.traits] template struct is_abi_tag; template inline constexpr bool is_abi_tag_v = is_abi_tag::value; @@ -47,10 +55,17 @@ template struct is_simd_mask; template inline constexpr bool is_simd_mask_v = is_simd_mask::value; + template struct is_simd_flag_type; + template inline constexpr bool is_simd_flag_type_v = is_simd_flag_type::value; + template> struct simd_size; template> inline constexpr size_t simd_size_v = simd_size::value; + template struct memory_alignment; + template + inline constexpr size_t memory_alignment_v = memory_alignment::value; + } // namespace parallelism_v2 } // namespace std::experimental @@ -64,6 +79,8 @@ #include #include +#include +#include #include #include #include diff --git a/libcxx/test/libcxx/transitive_includes/cxx03.csv b/libcxx/test/libcxx/transitive_includes/cxx03.csv --- a/libcxx/test/libcxx/transitive_includes/cxx03.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx03.csv @@ -262,6 +262,7 @@ experimental/set experimental/memory_resource experimental/set set experimental/simd cstddef +experimental/simd cstdint experimental/string experimental/memory_resource experimental/string string experimental/unordered_map algorithm diff --git a/libcxx/test/libcxx/transitive_includes/cxx11.csv b/libcxx/test/libcxx/transitive_includes/cxx11.csv --- a/libcxx/test/libcxx/transitive_includes/cxx11.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx11.csv @@ -262,6 +262,7 @@ experimental/set experimental/memory_resource experimental/set set experimental/simd cstddef +experimental/simd cstdint experimental/string experimental/memory_resource experimental/string string experimental/unordered_map algorithm diff --git a/libcxx/test/libcxx/transitive_includes/cxx14.csv b/libcxx/test/libcxx/transitive_includes/cxx14.csv --- a/libcxx/test/libcxx/transitive_includes/cxx14.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx14.csv @@ -262,6 +262,7 @@ experimental/set experimental/memory_resource experimental/set set experimental/simd cstddef +experimental/simd cstdint experimental/string experimental/memory_resource experimental/string string experimental/type_traits initializer_list diff --git a/libcxx/test/libcxx/transitive_includes/cxx17.csv b/libcxx/test/libcxx/transitive_includes/cxx17.csv --- a/libcxx/test/libcxx/transitive_includes/cxx17.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx17.csv @@ -262,6 +262,7 @@ experimental/set experimental/memory_resource experimental/set set experimental/simd cstddef +experimental/simd cstdint experimental/string experimental/memory_resource experimental/string string experimental/type_traits initializer_list diff --git a/libcxx/test/libcxx/transitive_includes/cxx20.csv b/libcxx/test/libcxx/transitive_includes/cxx20.csv --- a/libcxx/test/libcxx/transitive_includes/cxx20.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx20.csv @@ -269,6 +269,7 @@ experimental/set experimental/memory_resource experimental/set set experimental/simd cstddef +experimental/simd cstdint experimental/string experimental/memory_resource experimental/string string experimental/type_traits initializer_list diff --git a/libcxx/test/libcxx/transitive_includes/cxx23.csv b/libcxx/test/libcxx/transitive_includes/cxx23.csv --- a/libcxx/test/libcxx/transitive_includes/cxx23.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx23.csv @@ -175,6 +175,7 @@ experimental/set experimental/memory_resource experimental/set set experimental/simd cstddef +experimental/simd cstdint experimental/string experimental/memory_resource experimental/string string experimental/type_traits initializer_list diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv --- a/libcxx/test/libcxx/transitive_includes/cxx26.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv @@ -175,6 +175,7 @@ experimental/set experimental/memory_resource experimental/set set experimental/simd cstddef +experimental/simd cstdint experimental/string experimental/memory_resource experimental/string string experimental/type_traits initializer_list diff --git a/libcxx/test/std/experimental/simd/simd.traits/is_simd_flag_type.pass.cpp b/libcxx/test/std/experimental/simd/simd.traits/is_simd_flag_type.pass.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/std/experimental/simd/simd.traits/is_simd_flag_type.pass.cpp @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14 + +// +// +// [simd.traits] +// template struct is_simd_flag_type; +// template inline constexpr bool ex::is_simd_flag_type_v = ex::is_simd_flag_type::value; + +#include "../test_utils.h" + +namespace ex = std::experimental::parallelism_v2; + +template +struct CheckIsSimdFlagType { + template + void operator()() { + static_assert(ex::is_simd_flag_type::value); + static_assert(ex::is_simd_flag_type::value); + static_assert(ex::is_simd_flag_type>::value); + + static_assert(!ex::is_simd_flag_type::value); + static_assert(!ex::is_simd_flag_type>::value); + static_assert(!ex::is_simd_flag_type>::value); + + static_assert(ex::is_simd_flag_type_v); + static_assert(ex::is_simd_flag_type_v); + static_assert(ex::is_simd_flag_type_v>); + + static_assert(!ex::is_simd_flag_type_v); + static_assert(!ex::is_simd_flag_type_v>); + static_assert(!ex::is_simd_flag_type_v>); + } +}; + +int main(int, char**) { + test_all_simd_abi(); + return 0; +} diff --git a/libcxx/test/std/experimental/simd/simd.traits/memory_alignment.pass.cpp b/libcxx/test/std/experimental/simd/simd.traits/memory_alignment.pass.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/std/experimental/simd/simd.traits/memory_alignment.pass.cpp @@ -0,0 +1,119 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14 + +// +// +// [simd.traits] +// template struct memory_alignment; +// template +// inline constexpr std::size_t memory_alignment_v = memory_alignment::value; + +#include "../test_utils.h" + +namespace ex = std::experimental::parallelism_v2; + +template +struct CheckMemoryAlignmentMask { + template + void operator()() { + static_assert( + ex::memory_alignment>::value == bit_ceil(sizeof(bool) * ex::simd_size_v)); + static_assert( + ex::memory_alignment_v> == bit_ceil(sizeof(bool) * ex::simd_size_v)); + } +}; + +template +struct CheckMemoryAlignmentLongDouble { + template + void operator()() { + if constexpr (std::is_same_v) { + // on i686-w64-mingw32-clang++, the size of long double is 12 bytes. Disambiguation is needed. + static_assert( + ex::memory_alignment>::value == bit_ceil(sizeof(T) * ex::simd_size_v)); + static_assert(ex::memory_alignment_v> == bit_ceil(sizeof(T) * ex::simd_size_v)); + } + } +}; + +struct CheckMemAlignmentFixedDeduce { + template + void check() { + if constexpr (!std::is_same_v) { + static_assert(ex::memory_alignment_v>> == sizeof(T) * bit_ceil(N), + "Memory Alignment mismatch with abi fixed_size"); + static_assert(ex::memory_alignment>>::value == sizeof(T) * bit_ceil(N), + "Memory Alignment mismatch with abi fixed_size"); + + static_assert(ex::memory_alignment_v>> == sizeof(T) * bit_ceil(N), + "Memory Alignment mismatch with abi deduce"); + static_assert(ex::memory_alignment>>::value == sizeof(T) * bit_ceil(N), + "Memory Alignment mismatch with abi deduce"); + } + } + + template + void performChecks(std::index_sequence) { + (check(), ...); + } + + template + void operator()() { + performChecks(std::make_index_sequence{}); + } +}; + +struct CheckMemAlignmentScalarNativeCompatible { + template + void operator()() { + if constexpr (!std::is_same_v) { + static_assert(ex::memory_alignment>::value == sizeof(T)); + static_assert(ex::memory_alignment_v> == sizeof(T)); + + LIBCPP_STATIC_ASSERT(ex::memory_alignment>>::value == 16); + LIBCPP_STATIC_ASSERT(ex::memory_alignment_v>> == 16); + + LIBCPP_STATIC_ASSERT( + ex::memory_alignment>>::value == _LIBCPP_NATIVE_SIMD_WIDTH_IN_BYTES); + LIBCPP_STATIC_ASSERT( + ex::memory_alignment_v>> == _LIBCPP_NATIVE_SIMD_WIDTH_IN_BYTES); + } + } +}; + +template +struct has_memory_alignment : std::false_type {}; + +template +struct has_memory_alignment)>> : std::true_type {}; + +struct CheckMemoryAlignmentTraits { + template + void operator()() { + static_assert(has_memory_alignment>::value); + static_assert(has_memory_alignment>::value); + static_assert(has_memory_alignment, T>::value); + static_assert(has_memory_alignment, bool>::value); + + static_assert(!has_memory_alignment::value); + static_assert(!has_memory_alignment::value); + static_assert(!has_memory_alignment, bool>::value); + static_assert(!has_memory_alignment, T>::value); + } +}; + +int main(int, char**) { + types::for_each(arithmetic_no_bool_types(), CheckMemAlignmentFixedDeduce()); + types::for_each(arithmetic_no_bool_types(), CheckMemAlignmentScalarNativeCompatible()); + test_all_simd_abi(); + test_all_simd_abi(); + types::for_each(arithmetic_no_bool_types(), CheckMemoryAlignmentTraits()); + return 0; +} diff --git a/libcxx/test/std/experimental/simd/test_utils.h b/libcxx/test/std/experimental/simd/test_utils.h --- a/libcxx/test/std/experimental/simd/test_utils.h +++ b/libcxx/test/std/experimental/simd/test_utils.h @@ -43,4 +43,11 @@ types::for_each(arithmetic_no_bool_types(), TestAllSimdAbiFunctor()); } +constexpr size_t bit_ceil(size_t val) { + size_t pow = 1; + while (pow < val) + pow <<= 1; + return pow; +} + #endif // TEST_UTIL_H