diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -70,6 +70,10 @@ __algorithm/pop_heap.h __algorithm/prev_permutation.h __algorithm/pstl_any_all_none_of.h + __algorithm/pstl_backend.h + __algorithm/pstl_backends/cpu_backend.h + __algorithm/pstl_backends/cpu_backends/serial.h + __algorithm/pstl_backends/cpu_backends/thread.h __algorithm/pstl_fill.h __algorithm/pstl_find.h __algorithm/pstl_for_each.h diff --git a/libcxx/include/__algorithm/pstl_backend.h b/libcxx/include/__algorithm/pstl_backend.h new file mode 100644 --- /dev/null +++ b/libcxx/include/__algorithm/pstl_backend.h @@ -0,0 +1,96 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKEND_H +#define _LIBCPP___ALGORITHM_PSTL_BACKEND_H + +#include <__algorithm/pstl_backends/cpu_backend.h> +#include <__config> +#include + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD + +/* +TODO: Documentation of how backends work + +A PSTL parallel backend is a tag type to which the following functions are associated, at minimum: + + template + void __for_each(_Backend, _ExecutionPolicy&&, _Iterator __first, _Iterator __last, _Func __f); + + template + _Tp __reduce(_Backend, _ExecutionPolicy&&, _Iterator __first, _Iterator __last, _Tp const& __value, _BinOp __op); + + etc... + +The following functions are optional but can be provided. If provided, they are used by the corresponding +algorithms, otherwise they are implemented in terms of the basis operations mentioned above: + + template + void __for_each_n(_Backend, _ExecutionPolicy&&, _Iterator __first, _Size __n, _Func __f); + + etc... + +*/ + +void __pstl_for_each(); +void __pstl_for_each_n(); + +template +struct __select_backend; + +template <> +struct __select_backend { + using type = __cpu_backend_tag; +}; + +template <> +struct __select_backend { + using type = __cpu_backend_tag; +}; + +// TODO: Make this a proper customization point +#define _PSTL_PAR_BACKEND_SERIAL + +# if defined(_PSTL_PAR_BACKEND_STD_THREAD) +# include <__algorithm/pstl_backends/thread.h> +template <> struct __select_backend { using type = __std_thread_backend; }; +template <> struct __select_backend { using type = __std_thread_backend; }; + +# elif defined(_PSTL_PAR_BACKEND_GCD) +# include <__algorithm/pstl_backends/gcd.h> +template <> struct __select_backend { using type = __gcd_backend; }; +template <> struct __select_backend { using type = __gcd_backend; }; + +# elif defined(_PSTL_PAR_BACKEND_TBB) +# include <__algorithm/pstl_backends/tbb.h> +template <> struct __select_backend { using type = __tbb_backend; }; +template <> struct __select_backend { using type = __tbb_backend; }; + +# elif defined(_PSTL_PAR_BACKEND_SERIAL) +template <> struct __select_backend { using type = __cpu_backend_tag; }; +template <> struct __select_backend { using type = __cpu_backend_tag; }; + +# else + +// ...New vendors can add parallel backends here... + +# error "Invalid choice of a PSTL parallel backend" +# endif + +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKEND_H diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backend.h b/libcxx/include/__algorithm/pstl_backends/cpu_backend.h new file mode 100644 --- /dev/null +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backend.h @@ -0,0 +1,63 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H + +#include <__config> + +/* + TODO: Document the parallel backend +*/ + +#ifdef _LIBCPP_HAS_NO_THREADS +# include <__algorithm/pstl_backends/cpu_backends/thread.h> +#else +# include <__algorithm/pstl_backends/cpu_backends/serial.h> +#endif + +#include <__algorithm/for_each.h> +#include <__iterator/iterator_traits.h> +#include <__type_traits/is_execution_policy.h> +#include <__utility/terminate_on_exception.h> + +_LIBCPP_BEGIN_NAMESPACE_STD + +struct __cpu_backend_tag {}; + +template +_LIBCPP_HIDE_FROM_ABI _Iterator __simd_walk_1(_Iterator __first, _DifferenceType __n, _Function __f) noexcept { + // _PSTL_PRAGMA_SIMD + for (_DifferenceType __i = 0; __i < __n; ++__i) + __f(__first[__i]); + + return __first + __n; +} + +template +void __pstl_for_each(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Functor __func) { + if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && + __is_cpp17_random_access_iterator<_ForwardIterator>::value) { + std::__terminate_on_exception([&] { + std::__par_backend::__parallel_for( + __first, __last, [__func](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { + std::__pstl_for_each<__remove_parallel_policy_t<_ExecutionPolicy>>( + __cpu_backend_tag{}, __brick_first, __brick_last, __func); + }); + }); + } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && + __is_cpp17_random_access_iterator<_ForwardIterator>::value) { + std::__simd_walk_1(__first, __last - __first, __func); + } else { + std::for_each(__first, __last, __func); + } +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h new file mode 100644 --- /dev/null +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h @@ -0,0 +1,57 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H + +#include <__algorithm/for_each.h> +#include <__algorithm/for_each_n.h> +#include <__config> +#include <__numeric/reduce.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD + +namespace __par_backend::inline __serial_cpu_backend { + +// +// Mandatory customization points +// +template +_LIBCPP_HIDE_FROM_ABI void __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fp __f) { + __f(__first, __last); +} + +template +_LIBCPP_HIDE_FROM_ABI _Tp +__parallel_reduce(_RandomAccessIterator __first, _RandomAccessIterator __last, _Tp const& __value, _BinOp __op) { + return std::reduce(__first, __last, __value, __op); +} + +// etc... + +// +// Optional customization points +// +template +_LIBCPP_HIDE_FROM_ABI void __pstl_for_each_n(_Iterator __first, _Size __n, _Fp __f) { + std::for_each_n(__first, __n, __f); +} +} // namespace __par_backend::__serial_cpu_backend + +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && && _LIBCPP_STD_VER >= 17 + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h new file mode 100644 --- /dev/null +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h @@ -0,0 +1,47 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_THREAD_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_THREAD_H + +#include <__algorithm/for_each.h> +#include <__config> +#include <__numeric/reduce.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +_LIBCPP_BEGIN_NAMESPACE_STD + +namespace __par_backend::inline __thread_cpu_backend { + +template +_LIBCPP_HIDE_FROM_ABI void +__parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func) { + // TODO: implement +} + +template +_LIBCPP_HIDE_FROM_ABI _Tp +__parallel_reduce(_RandomAccessIterator __first, _RandomAccessIterator __last, _Tp const& __value, _BinOp __op) { + // TODO: Implement +} + +// etc... + +} // namespace __par_backend::__thread_cpu_backend + +_LIBCPP_END_NAMESPACE_STD + +#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_THREAD_H diff --git a/libcxx/include/__algorithm/pstl_backends/some_gpu_backend.h b/libcxx/include/__algorithm/pstl_backends/some_gpu_backend.h new file mode 100644 --- /dev/null +++ b/libcxx/include/__algorithm/pstl_backends/some_gpu_backend.h @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_SOME_GPU_BACKEND_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_SOME_GPU_BACKEND_H + +#include <__config> + +_LIBCPP_BEGIN_NAMESPACE_STD + +struct __some_gpu_backend_tag {}; + +// Mandatory algorithms +template +void __pstl_for_each(__some_gpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Functor __func) { + // Implement +} + +// ... + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_SOME_GPU_BACKEND_H diff --git a/libcxx/include/__algorithm/pstl_for_each.h b/libcxx/include/__algorithm/pstl_for_each.h --- a/libcxx/include/__algorithm/pstl_for_each.h +++ b/libcxx/include/__algorithm/pstl_for_each.h @@ -11,12 +11,14 @@ #include <__algorithm/for_each.h> #include <__algorithm/for_each_n.h> +#include <__algorithm/pstl_backend.h> #include <__config> #include <__iterator/iterator_traits.h> #include <__pstl/internal/parallel_backend.h> #include <__pstl/internal/unseq_backend_simd.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> +#include <__type_traits/void_t.h> #include <__utility/terminate_on_exception.h> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) @@ -30,40 +32,33 @@ template >, int> = 0> + class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>, + enable_if_t, int> = 0> _LIBCPP_HIDE_FROM_ABI void for_each(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Function __func) { - if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> && - __is_cpp17_random_access_iterator<_ForwardIterator>::value) { - std::__terminate_on_exception([&] { - __pstl::__par_backend::__parallel_for( - {}, - __policy, - __first, - __last, - [&__policy, __func](_ForwardIterator __brick_first, _ForwardIterator __brick_last) { - std::for_each(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __func); - }); - }); - } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> && - __is_cpp17_random_access_iterator<_ForwardIterator>::value) { - __pstl::__unseq_backend::__simd_walk_1(__first, __last - __first, __func); - } else { - std::for_each(__first, __last, __func); - } + using _Backend = typename __select_backend<_RawPolicy>::type; + std::__pstl_for_each<_RawPolicy>(_Backend{}, std::move(__first), std::move(__last), std::move(__func)); } template >, int> = 0> + class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>, + enable_if_t, int> = 0> _LIBCPP_HIDE_FROM_ABI void for_each_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _Size __size, _Function __func) { - if constexpr (__is_cpp17_random_access_iterator<_ForwardIterator>::value) { - std::for_each(__policy, __first, __first + __size, __func); + using _Backend = typename __select_backend<_RawPolicy>::type; + if constexpr (requires {std::__pstl_for_each_n(_Backend{}, __first, __size, __func); }) { + __pstl_for_each_n<_RawPolicy>(_Backend{}, std::move(__first), __size, std::move(__func)); } else { - std::for_each_n(__first, __size, __func); + // Default implementation + if constexpr (__is_cpp17_random_access_iterator<_ForwardIterator>::value && + __is_parallel_execution_policy_v<_RawPolicy>) { + std::for_each(__policy, __first, __first + __size, __func); + } else { + std::for_each_n(__first, __size, __func); + } } } diff --git a/libcxx/include/__type_traits/is_execution_policy.h b/libcxx/include/__type_traits/is_execution_policy.h --- a/libcxx/include/__type_traits/is_execution_policy.h +++ b/libcxx/include/__type_traits/is_execution_policy.h @@ -36,10 +36,13 @@ template inline constexpr bool __is_parallel_execution_policy_v = __is_parallel_execution_policy_impl<__remove_cvref_t<_Tp>>; +template +_LIBCPP_HIDE_FROM_ABI auto __remove_parallel_policy(); + // Removes the "parallel" part of an execution policy. // For example, turns par_unseq into unseq, and par into seq. template -_LIBCPP_HIDE_FROM_ABI const auto& __remove_parallel_policy(_ExecutionPolicy&&); +using __remove_parallel_policy_t = decltype(std::__remove_parallel_policy<_ExecutionPolicy>()); _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/execution b/libcxx/include/execution --- a/libcxx/include/execution +++ b/libcxx/include/execution @@ -135,12 +135,11 @@ struct is_execution_policy : bool_constant> {}; template -_LIBCPP_HIDE_FROM_ABI const auto& __remove_parallel_policy(_ExecutionPolicy&&) { - using _ExecPol = __remove_cvref_t<_ExecutionPolicy>; - if constexpr (is_same_v<_ExecPol, execution::parallel_policy>) { - return execution::seq; - } else if constexpr (is_same_v<_ExecPol, execution::parallel_unsequenced_policy>) { - return execution::__unseq; +_LIBCPP_HIDE_FROM_ABI auto __remove_parallel_policy() { + if constexpr (is_same_v<_ExecutionPolicy, execution::parallel_policy>) { + return execution::sequenced_policy(execution::__disable_user_instantiations_tag{}); + } else if constexpr (is_same_v<_ExecutionPolicy, execution::parallel_unsequenced_policy>) { + return execution::__unsequenced_policy{execution::__disable_user_instantiations_tag{}}; } }