diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt --- a/libcxx/CMakeLists.txt +++ b/libcxx/CMakeLists.txt @@ -806,9 +806,11 @@ config_define(1 _LIBCPP_PSTL_CPU_BACKEND_SERIAL) elseif(LIBCXX_PSTL_CPU_BACKEND STREQUAL "std_thread") config_define(1 _LIBCPP_PSTL_CPU_BACKEND_THREAD) +elseif(LIBCXX_PSTL_CPU_BACKEND STREQUAL "libdispatch") + config_define(1 _LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) else() message(FATAL_ERROR "LIBCXX_PSTL_CPU_BACKEND is set to ${LIBCXX_PSTL_CPU_BACKEND}, which is not a valid backend. - Valid backends are: serial, std_thread") + Valid backends are: serial, std_thread and libdispatch") endif() if (LIBCXX_ABI_DEFINES) diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -78,6 +78,7 @@ __algorithm/pstl_backends/cpu_backends/fill.h __algorithm/pstl_backends/cpu_backends/find_if.h __algorithm/pstl_backends/cpu_backends/for_each.h + __algorithm/pstl_backends/cpu_backends/libdispatch.h __algorithm/pstl_backends/cpu_backends/merge.h __algorithm/pstl_backends/cpu_backends/serial.h __algorithm/pstl_backends/cpu_backends/stable_sort.h diff --git a/libcxx/include/__algorithm/pstl_backend.h b/libcxx/include/__algorithm/pstl_backend.h --- a/libcxx/include/__algorithm/pstl_backend.h +++ b/libcxx/include/__algorithm/pstl_backend.h @@ -169,7 +169,8 @@ }; # endif -# if defined(_LIBCPP_PSTL_CPU_BACKEND_SERIAL) || defined(_LIBCPP_PSTL_CPU_BACKEND_THREAD) +# if defined(_LIBCPP_PSTL_CPU_BACKEND_SERIAL) || defined(_LIBCPP_PSTL_CPU_BACKEND_THREAD) || \ + defined(_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) template <> struct __select_backend { using type = __cpu_backend_tag; diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h @@ -16,6 +16,8 @@ # include <__algorithm/pstl_backends/cpu_backends/serial.h> #elif defined(_LIBCPP_PSTL_CPU_BACKEND_THREAD) # include <__algorithm/pstl_backends/cpu_backends/thread.h> +#elif defined(_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) +# include <__algorithm/pstl_backends/cpu_backends/libdispatch.h> #else # error "Invalid CPU backend choice" #endif diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h new file mode 100644 --- /dev/null +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h @@ -0,0 +1,196 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H + +#include <__algorithm/lower_bound.h> +#include <__algorithm/upper_bound.h> +#include <__atomic/atomic.h> +#include <__config> +#include <__exception/terminate.h> +#include <__iterator/iterator_traits.h> +#include <__memory/construct_at.h> +#include <__memory/uninitialized_buffer.h> +#include <__memory/unique_ptr.h> +#include <__numeric/transform_reduce.h> +#include <__utility/exception_guard.h> +#include <__utility/move.h> +#include <__utility/terminate_on_exception.h> +#include +#include + +_LIBCPP_BEGIN_NAMESPACE_STD + +namespace __par_backend { +inline namespace __libdispatch { + +// ::dispatch_apply is marked as __attribute__((nothrow)) because it doesn't let exceptions propagate, and neither do +// we. +[[clang::callback(__func, __context, __)]] _LIBCPP_EXPORTED_FROM_ABI void +__dispatch_apply(size_t __chunk_count, void* __context, void (*__func)(void* __context, size_t __chunk)) noexcept; + +template +_LIBCPP_HIDE_FROM_ABI void __dispatch_apply(size_t __chunk_count, _Func __func) noexcept { + __libdispatch::__dispatch_apply(__chunk_count, &__func, [](void* __context, size_t __chunk) { + (*static_cast<_Func*>(__context))(__chunk); + }); +} + +// Preliminary size of each chunk: requires further discussion +constexpr std::size_t __default_chunk_size = 2048; + +struct __chunk_partitions { + size_t __chunk_count_; // includes the first chunk + size_t __chunk_size_; + size_t __first_chunk_size_; +}; + +template +_LIBCPP_HIDE_FROM_ABI __chunk_partitions +__partition_chunks(_RandomAccessIterator __first, _RandomAccessIterator __last) { + const auto __requested_chunk_size = __default_chunk_size; + const auto __element_count = __last - __first; + + if (__element_count < __requested_chunk_size) + return {/*__chunk_count_ =*/1, /*__chunk_size_ =*/__element_count, /*__first_chunk_size_ =*/__element_count}; + + using _Size = __iter_diff_t<_RandomAccessIterator>; + + __chunk_partitions __partitions; + __partitions.__chunk_count_ = (__element_count / __requested_chunk_size) + 1; + __partitions.__chunk_size_ = __element_count / __partitions.__chunk_count_; + __partitions.__first_chunk_size_ = __partitions.__chunk_size_; + + const _Size __leftover_item_count = __element_count - (__partitions.__chunk_count_ * __partitions.__chunk_size_); + + if (__leftover_item_count == 0) + return __partitions; + + if (__leftover_item_count == __partitions.__chunk_size_) { + __partitions.__chunk_count_ += 1; + return __partitions; + } + + const _Size __n_extra_items_per_chunk = __leftover_item_count / __partitions.__chunk_count_; + const _Size __n_final_leftover_items = + __leftover_item_count - (__n_extra_items_per_chunk * __partitions.__chunk_count_); + + __partitions.__chunk_size_ += __n_extra_items_per_chunk; + __partitions.__first_chunk_size_ = __partitions.__chunk_size_ + __n_final_leftover_items; + return __partitions; +} + +template +_LIBCPP_HIDE_FROM_ABI void __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __f) { + auto __partitions = __libdispatch::__partition_chunks(__first, __last); + + // Perform the chunked execution. + __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) { + auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_; + auto __index = + __chunk == 0 + ? 0 + : (__chunk * __partitions.__chunk_size_) + (__partitions.__first_chunk_size_ - __partitions.__chunk_size_); + __f(__first + __index, __first + __index + __this_chunk_size); + }); +} + +template +_LIBCPP_HIDE_FROM_ABI void __libdispatch_invoke_parallel(_Func1&& __f1, _Func2&& __f2) { + __libdispatch::__dispatch_apply(2, [&](size_t __index) { + if (__index == 0) + __f1(); + else + __f2(); + }); +} + +template +_LIBCPP_HIDE_FROM_ABI void __parallel_merge( + _RandomAccessIterator1 __first1, + _RandomAccessIterator1 __last1, + _RandomAccessIterator2 __first2, + _RandomAccessIterator2 __last2, + _RandomAccessIterator3 __result, + _Compare __comp, + _LeafMerge __leaf_merge) { + std::size_t __size_x = __last1 - __first1; + std::size_t __size_y = __last2 - __first2; + + if (__size_x + __size_y <= __default_chunk_size) { + __leaf_merge(__first1, __last1, __first2, __last2, __result, __comp); + return; + } + + _RandomAccessIterator1 __middle1; + _RandomAccessIterator2 __middle2; + + if (__size_x < __size_y) { + __middle2 = __first2 + (__size_y / 2); + __middle1 = std::upper_bound(__first1, __last1, *__middle2, __comp); + } else { + __middle1 = __first1 + (__size_x / 2); + __middle2 = std::lower_bound(__first2, __last2, *__middle1, __comp); + } + + auto __zm = __result + (__middle1 - __first1) + (__middle2 - __first2); + + __libdispatch::__libdispatch_invoke_parallel( + [__first1, __middle1, __first2, __middle2, __result, __comp, &__leaf_merge]() { + __libdispatch::__parallel_merge(__first1, __middle1, __first2, __middle2, __result, __comp, __leaf_merge); + }, + [__middle1, __last1, __middle2, __last2, __zm, __comp, &__leaf_merge]() { + __libdispatch::__parallel_merge(__middle1, __last1, __middle2, __last2, __zm, __comp, __leaf_merge); + }); +} + +template +_LIBCPP_HIDE_FROM_ABI _Value __parallel_transform_reduce( + _RandomAccessIterator __first, + _RandomAccessIterator __last, + _Transform __transform, + _Value __init, + _Combiner __combiner, + _Reduction __reduction) { + auto __partitions = __libdispatch::__partition_chunks(__first, __last); + auto __values = std::__make_uninitialized_buffer<_Value[]>( + nothrow, __partitions.__chunk_count_, [](_Value* __ptr, size_t __count) { std::destroy_n(__ptr, __count); }); + + if (__values == nullptr) + std::__throw_pstl_bad_alloc(); + + // __dispatch_apply is noexcept + __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) { + auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_; + auto __index = + __chunk == 0 + ? 0 + : (__chunk * __partitions.__chunk_size_) + (__partitions.__first_chunk_size_ - __partitions.__chunk_size_); + std::__construct_at( + __values + __index, + __reduction(__first + __index + 2, + __first + __index + __this_chunk_size, + __combiner(__transform((__first + __index)[0], (__first + __index)[1])))); + }); + + return std::transform_reduce(__values, __values + __partitions.__chunk_count_, __init, __transform, __reduction); +} + +_LIBCPP_HIDE_FROM_ABI inline void __cancel_execution() {} + +} // namespace __libdispatch +} // namespace __par_backend + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h @@ -163,8 +163,8 @@ std::move(__last), [__transform](_ForwardIterator __iter) { return __transform(*__iter); }, std::move(__init), - std::move(__reduce), - [=](_ForwardIterator __brick_first, _ForwardIterator __brick_last, _Tp __brick_init) { + __reduce, + [=](auto __brick_first, auto __brick_last, _Tp __brick_init) { return std::__pstl_transform_reduce<__remove_parallel_policy_t<_ExecutionPolicy>>( __cpu_backend_tag{}, std::move(__brick_first), diff --git a/libcxx/include/__config_site.in b/libcxx/include/__config_site.in --- a/libcxx/include/__config_site.in +++ b/libcxx/include/__config_site.in @@ -34,6 +34,7 @@ // PSTL backends #cmakedefine _LIBCPP_PSTL_CPU_BACKEND_SERIAL #cmakedefine _LIBCPP_PSTL_CPU_BACKEND_THREAD +#cmakedefine _LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH // __USE_MINGW_ANSI_STDIO gets redefined on MinGW #ifdef __clang__ diff --git a/libcxx/include/__utility/terminate_on_exception.h b/libcxx/include/__utility/terminate_on_exception.h --- a/libcxx/include/__utility/terminate_on_exception.h +++ b/libcxx/include/__utility/terminate_on_exception.h @@ -11,6 +11,7 @@ #include <__config> #include <__exception/terminate.h> +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header @@ -26,6 +27,8 @@ _LIBCPP_HIDE_FROM_ABI auto __terminate_on_exception(_Func __func) { try { return __func(); + } catch (const __pstl_bad_alloc&) { + throw; } catch (...) { std::terminate(); } diff --git a/libcxx/include/new b/libcxx/include/new --- a/libcxx/include/new +++ b/libcxx/include/new @@ -172,6 +172,20 @@ _LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void __throw_bad_alloc(); // not in C++ spec +#if _LIBCPP_STD_VER >= 17 +struct _LIBCPP_EXPORTED_FROM_ABI __pstl_bad_alloc : bad_alloc { + const char* what() const noexcept override; +}; + +[[noreturn]] _LIBCPP_HIDE_FROM_ABI inline void __throw_pstl_bad_alloc() { +#ifndef _LIBCPP_HAS_NO_EXCEPTIONS + throw __pstl_bad_alloc{}; +#else + _LIBCPP_VERBOSE_ABORT("PSTL allocation failed in -fno-exceptions mode"); +#endif +} +#endif // _LIBCPP_STD_VER >= 17 + _LIBCPP_NORETURN inline _LIBCPP_INLINE_VISIBILITY void __throw_bad_array_new_length() { diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt --- a/libcxx/src/CMakeLists.txt +++ b/libcxx/src/CMakeLists.txt @@ -313,6 +313,11 @@ experimental/memory_resource.cpp ) +if (LIBCXX_PSTL_CPU_BACKEND STREQUAL "libdispatch") + set(LIBCXX_EXPERIMENTAL_SOURCES ${LIBCXX_EXPERIMENTAL_SOURCES} + pstl/libdispatch.cpp) +endif() + add_library(cxx_experimental STATIC ${LIBCXX_EXPERIMENTAL_SOURCES}) target_link_libraries(cxx_experimental PUBLIC cxx-headers) if (LIBCXX_ENABLE_SHARED) diff --git a/libcxx/src/new.cpp b/libcxx/src/new.cpp --- a/libcxx/src/new.cpp +++ b/libcxx/src/new.cpp @@ -31,6 +31,10 @@ #endif // !LIBSTDCXX +const char* __pstl_bad_alloc::what() const noexcept { + return "PSTL memory allocation failed"; +} + } // std #if !defined(__GLIBCXX__) && \ diff --git a/libcxx/src/pstl/libdispatch.cpp b/libcxx/src/pstl/libdispatch.cpp new file mode 100644 --- /dev/null +++ b/libcxx/src/pstl/libdispatch.cpp @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <__algorithm/pstl_backends/cpu_backends/libdispatch.h> +#include <__config> +#include + +_LIBCPP_BEGIN_NAMESPACE_STD + +namespace __par_backend::inline __libdispatch { + +void __dispatch_apply(size_t chunk_count, void* context, void (*func)(void* context, size_t chunk)) noexcept { + ::dispatch_apply_f(chunk_count, DISPATCH_APPLY_AUTO, context, func); +} + +} // namespace __par_backend::inline __libdispatch + +_LIBCPP_END_NAMESPACE_STD