diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt --- a/libcxx/CMakeLists.txt +++ b/libcxx/CMakeLists.txt @@ -806,9 +806,11 @@ config_define(1 _LIBCPP_PSTL_CPU_BACKEND_SERIAL) elseif(LIBCXX_PSTL_CPU_BACKEND STREQUAL "std_thread") config_define(1 _LIBCPP_PSTL_CPU_BACKEND_THREAD) +elseif(LIBCXX_PSTL_CPU_BACKEND STREQUAL "libdispatch") + config_define(1 _LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) else() message(FATAL_ERROR "LIBCXX_PSTL_CPU_BACKEND is set to ${LIBCXX_PSTL_CPU_BACKEND}, which is not a valid backend. - Valid backends are: serial, std_thread") + Valid backends are: serial, std_thread and libdispatch") endif() if (LIBCXX_ABI_DEFINES) diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -78,6 +78,7 @@ __algorithm/pstl_backends/cpu_backends/fill.h __algorithm/pstl_backends/cpu_backends/find_if.h __algorithm/pstl_backends/cpu_backends/for_each.h + __algorithm/pstl_backends/cpu_backends/libdispatch.h __algorithm/pstl_backends/cpu_backends/merge.h __algorithm/pstl_backends/cpu_backends/serial.h __algorithm/pstl_backends/cpu_backends/stable_sort.h diff --git a/libcxx/include/__algorithm/pstl_backend.h b/libcxx/include/__algorithm/pstl_backend.h --- a/libcxx/include/__algorithm/pstl_backend.h +++ b/libcxx/include/__algorithm/pstl_backend.h @@ -169,7 +169,8 @@ }; # endif -# if defined(_LIBCPP_PSTL_CPU_BACKEND_SERIAL) || defined(_LIBCPP_PSTL_CPU_BACKEND_THREAD) +# if defined(_LIBCPP_PSTL_CPU_BACKEND_SERIAL) || defined(_LIBCPP_PSTL_CPU_BACKEND_THREAD) || \ + defined(_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) template <> struct __select_backend { using type = __cpu_backend_tag; diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h @@ -16,6 +16,8 @@ # include <__algorithm/pstl_backends/cpu_backends/serial.h> #elif defined(_LIBCPP_PSTL_CPU_BACKEND_THREAD) # include <__algorithm/pstl_backends/cpu_backends/thread.h> +#elif defined(_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) +# include <__algorithm/pstl_backends/cpu_backends/libdispatch.h> #else # error "Invalid CPU backend choice" #endif diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h new file mode 100644 --- /dev/null +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h @@ -0,0 +1,214 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H +#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H + +#include <__algorithm/lower_bound.h> +#include <__algorithm/upper_bound.h> +#include <__atomic/atomic.h> +#include <__config> +#include <__exception/terminate.h> +#include <__iterator/iterator_traits.h> +#include <__memory/construct_at.h> +#include <__memory/uninitialized_buffer.h> +#include <__memory/unique_ptr.h> +#include <__numeric/transform_reduce.h> +#include <__utility/exception_guard.h> +#include <__utility/move.h> +#include <__utility/terminate_on_exception.h> +#include +#include +#include + +_LIBCPP_BEGIN_NAMESPACE_STD + +namespace __par_backend { +inline namespace __libdispatch { + +// ::dispatch_apply is marked as __attribute__((nothrow)) because it doesn't let exceptions propagate, and neither do +// we. +[[_Clang::__callback__(__func, __context, __)]] _LIBCPP_EXPORTED_FROM_ABI void +__dispatch_apply(size_t __chunk_count, void* __context, void (*__func)(void* __context, size_t __chunk)) noexcept; + +template +_LIBCPP_HIDE_FROM_ABI void __dispatch_apply(size_t __chunk_count, _Func __func) noexcept { + __libdispatch::__dispatch_apply(__chunk_count, &__func, [](void* __context, size_t __chunk) { + (*static_cast<_Func*>(__context))(__chunk); + }); +} + +// Preliminary size of each chunk: requires further discussion +constexpr ptrdiff_t __default_chunk_size = 2048; + +struct __chunk_partitions { + ptrdiff_t __chunk_count_; // includes the first chunk + ptrdiff_t __chunk_size_; + ptrdiff_t __first_chunk_size_; +}; + +[[__gnu__::__const__]] _LIBCPP_EXPORTED_FROM_ABI pmr::memory_resource* __get_memory_resource(); +[[__gnu__::__const__]] _LIBCPP_EXPORTED_FROM_ABI __chunk_partitions __partition_chunks(ptrdiff_t __size); + +template +_LIBCPP_HIDE_FROM_ABI void __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __f) { + auto __partitions = __libdispatch::__partition_chunks(__first, __last); + + // Perform the chunked execution. + __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) { + auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_; + auto __index = + __chunk == 0 + ? 0 + : (__chunk * __partitions.__chunk_size_) + (__partitions.__first_chunk_size_ - __partitions.__chunk_size_); + __f(__first + __index, __first + __index + __this_chunk_size); + }); +} + +template +_LIBCPP_HIDE_FROM_ABI void __libdispatch_invoke_parallel(_Func1&& __f1, _Func2&& __f2) { + __libdispatch::__dispatch_apply(2, [&](size_t __index) { + if (__index == 0) + __f1(); + else + __f2(); + }); +} + +template +struct __merge_range { + _RandomAccessIterator1 __first1; + _RandomAccessIterator1 __last1; + _RandomAccessIterator2 __first2; + _RandomAccessIterator2 __last2; + _RandomAccessIteratorOut __result; +}; + +template +_LIBCPP_HIDE_FROM_ABI void __calculate_merge_ranges( + _RandomAccessIterator1 __first1, + _RandomAccessIterator1 __last1, + _RandomAccessIterator2 __first2, + _RandomAccessIterator2 __last2, + _RandomAccessIteratorOut __result, + _Comp __comp, + pmr::vector<__merge_range<_RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIteratorOut>>& __ranges) { + std::size_t __size_x = __last1 - __first1; + std::size_t __size_y = __last2 - __first2; + + if (__size_x + __size_y <= __default_chunk_size) { + __ranges.emplace_back( + std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), std::move(__result)); + return; + } + + _RandomAccessIterator1 __middle1; + _RandomAccessIterator2 __middle2; + + if (__size_x < __size_y) { + __middle2 = __first2 + (__size_y / 2); + __middle1 = std::upper_bound(__first1, __last1, *__middle2, __comp); + } else { + __middle1 = __first1 + (__size_x / 2); + __middle2 = std::lower_bound(__first2, __last2, *__middle1, __comp); + } + + auto __zm = __result + (__middle1 - __first1) + (__middle2 - __first2); + + __libdispatch::__calculate_merge_ranges(__first1, __middle1, __first2, __middle2, __result, __comp); + __libdispatch::__calculate_merge_ranges(__middle1, __last1, __middle2, __last2, __zm, __comp); +} + +template +_LIBCPP_HIDE_FROM_ABI + pmr::vector<__merge_range<_RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIteratorOut>> + __calculate_merge_ranges( + _RandomAccessIterator1 __first1, + _RandomAccessIterator1 __last1, + _RandomAccessIterator2 __first2, + _RandomAccessIterator2 __last2, + _RandomAccessIteratorOut __result, + _Comp __comp) { + pmr::vector<__merge_range<_RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIteratorOut>> __ranges( + __libdispatch::__get_memory_resource()); + + __libdispatch::__calculate_merge_ranges(__first1, __last1, __first2, __last2, __result, __comp); + + return __ranges; +} + +template +_LIBCPP_HIDE_FROM_ABI void __parallel_merge( + _RandomAccessIterator1 __first1, + _RandomAccessIterator1 __last1, + _RandomAccessIterator2 __first2, + _RandomAccessIterator2 __last2, + _RandomAccessIterator3 __result, + _Compare __comp, + _LeafMerge __leaf_merge) { +#ifndef _LIBCPP_HAS_NO_EXCEPTIONS + try { +#endif + auto __ranges = __libdispatch::__calculate_merge_ranges(__first1, __last1, __first2, __last2, __result, __comp); + + // __dispatch_apply is noexcept, so we don't convert any user exceptions + __libdispatch::__dispatch_apply(__ranges.size(), [&](size_t __index) { + auto __range = __ranges[__index]; + __leaf_merge(__range.__first1, __range.__last1, __range.__first2, __range.__last2, __range.__result); + }); +#ifndef _LIBCPP_HAS_NO_EXCEPTIONS + } catch (const bad_alloc&) { + throw __pstl_bad_alloc(); + } +#endif +} + +template +_LIBCPP_HIDE_FROM_ABI _Value __parallel_transform_reduce( + _RandomAccessIterator __first, + _RandomAccessIterator __last, + _Transform __transform, + _Value __init, + _Combiner __combiner, + _Reduction __reduction) { + auto __partitions = __libdispatch::__partition_chunks(__first, __last); + auto __values = std::__make_uninitialized_buffer<_Value[]>( + nothrow, __partitions.__chunk_count_, [](_Value* __ptr, size_t __count) { std::destroy_n(__ptr, __count); }); + + if (__values == nullptr) + std::__throw_pstl_bad_alloc(); + + // __dispatch_apply is noexcept + __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) { + auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_; + auto __index = + __chunk == 0 + ? 0 + : (__chunk * __partitions.__chunk_size_) + (__partitions.__first_chunk_size_ - __partitions.__chunk_size_); + std::__construct_at( + __values + __index, + __reduction(__first + __index + 2, + __first + __index + __this_chunk_size, + __combiner(__transform((__first + __index)[0], (__first + __index)[1])))); + }); + + return std::transform_reduce(__values, __values + __partitions.__chunk_count_, __init, __transform, __reduction); +} + +_LIBCPP_HIDE_FROM_ABI inline void __cancel_execution() {} + +} // namespace __libdispatch +} // namespace __par_backend + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h @@ -163,8 +163,8 @@ std::move(__last), [__transform](_ForwardIterator __iter) { return __transform(*__iter); }, std::move(__init), - std::move(__reduce), - [=](_ForwardIterator __brick_first, _ForwardIterator __brick_last, _Tp __brick_init) { + __reduce, + [__transform, __reduce](auto __brick_first, auto __brick_last, _Tp __brick_init) { return std::__pstl_transform_reduce<__remove_parallel_policy_t<_ExecutionPolicy>>( __cpu_backend_tag{}, std::move(__brick_first), diff --git a/libcxx/include/__config_site.in b/libcxx/include/__config_site.in --- a/libcxx/include/__config_site.in +++ b/libcxx/include/__config_site.in @@ -34,6 +34,7 @@ // PSTL backends #cmakedefine _LIBCPP_PSTL_CPU_BACKEND_SERIAL #cmakedefine _LIBCPP_PSTL_CPU_BACKEND_THREAD +#cmakedefine _LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH // __USE_MINGW_ANSI_STDIO gets redefined on MinGW #ifdef __clang__ diff --git a/libcxx/include/__utility/terminate_on_exception.h b/libcxx/include/__utility/terminate_on_exception.h --- a/libcxx/include/__utility/terminate_on_exception.h +++ b/libcxx/include/__utility/terminate_on_exception.h @@ -11,6 +11,7 @@ #include <__config> #include <__exception/terminate.h> +#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header @@ -26,6 +27,8 @@ _LIBCPP_HIDE_FROM_ABI auto __terminate_on_exception(_Func __func) { try { return __func(); + } catch (const __pstl_bad_alloc&) { + throw; } catch (...) { std::terminate(); } diff --git a/libcxx/include/new b/libcxx/include/new --- a/libcxx/include/new +++ b/libcxx/include/new @@ -172,6 +172,20 @@ _LIBCPP_NORETURN _LIBCPP_EXPORTED_FROM_ABI void __throw_bad_alloc(); // not in C++ spec +#if _LIBCPP_STD_VER >= 17 +struct _LIBCPP_EXPORTED_FROM_ABI __pstl_bad_alloc : bad_alloc { + const char* what() const noexcept override; +}; + +[[noreturn]] _LIBCPP_HIDE_FROM_ABI inline void __throw_pstl_bad_alloc() { +#ifndef _LIBCPP_HAS_NO_EXCEPTIONS + throw __pstl_bad_alloc{}; +#else + _LIBCPP_VERBOSE_ABORT("PSTL allocation failed in -fno-exceptions mode"); +#endif +} +#endif // _LIBCPP_STD_VER >= 17 + _LIBCPP_NORETURN inline _LIBCPP_INLINE_VISIBILITY void __throw_bad_array_new_length() { diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt --- a/libcxx/src/CMakeLists.txt +++ b/libcxx/src/CMakeLists.txt @@ -311,8 +311,13 @@ set(LIBCXX_EXPERIMENTAL_SOURCES experimental/memory_resource.cpp + pstl/pstl_bad_alloc.cpp ) +if (LIBCXX_PSTL_CPU_BACKEND STREQUAL "libdispatch") + list(APPEND LIBCXX_EXPERIMENTAL_SOURCES pstl/libdispatch.cpp) +endif() + add_library(cxx_experimental STATIC ${LIBCXX_EXPERIMENTAL_SOURCES}) target_link_libraries(cxx_experimental PUBLIC cxx-headers) if (LIBCXX_ENABLE_SHARED) diff --git a/libcxx/src/pstl/libdispatch.cpp b/libcxx/src/pstl/libdispatch.cpp new file mode 100644 --- /dev/null +++ b/libcxx/src/pstl/libdispatch.cpp @@ -0,0 +1,60 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <__algorithm/pstl_backends/cpu_backends/libdispatch.h> +#include <__config> +#include +#include + +#include + +_LIBCPP_BEGIN_NAMESPACE_STD + +pmr::memory_resource* __get_memory_resource() { + static std::pmr::synchronized_pool_resource pool{pmr::new_delete_resource()}; + return &pool; +} + +namespace __par_backend::inline __libdispatch { + +void __dispatch_apply(size_t chunk_count, void* context, void (*func)(void* context, size_t chunk)) noexcept { + ::dispatch_apply_f(chunk_count, DISPATCH_APPLY_AUTO, context, func); +} + +__chunk_partitions __partition_chunks(ptrdiff_t element_count) { + const auto requested_chunk_size = __default_chunk_size / thread::hardware_concurrency(); + + if (element_count < requested_chunk_size) + return {/*__chunk_count_ =*/1, /*__chunk_size_ =*/element_count, /*__first_chunk_size_ =*/element_count}; + + __chunk_partitions partitions; + partitions.__chunk_count_ = (element_count / requested_chunk_size) + 1; + partitions.__chunk_size_ = element_count / partitions.__chunk_count_; + partitions.__first_chunk_size_ = partitions.__chunk_size_; + + const ptrdiff_t leftover_item_count = element_count - (partitions.__chunk_count_ * partitions.__chunk_size_); + + if (leftover_item_count == 0) + return partitions; + + if (leftover_item_count == partitions.__chunk_size_) { + partitions.__chunk_count_ += 1; + return partitions; + } + + const ptrdiff_t n_extra_items_per_chunk = leftover_item_count / partitions.__chunk_count_; + const ptrdiff_t n_final_leftover_items = leftover_item_count - (n_extra_items_per_chunk * partitions.__chunk_count_); + + partitions.__chunk_size_ += n_extra_items_per_chunk; + partitions.__first_chunk_size_ = partitions.__chunk_size_ + n_final_leftover_items; + return partitions; +} + +} // namespace __par_backend::inline __libdispatch + +_LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/src/pstl/pstl_bad_alloc.cpp b/libcxx/src/pstl/pstl_bad_alloc.cpp new file mode 100644 --- /dev/null +++ b/libcxx/src/pstl/pstl_bad_alloc.cpp @@ -0,0 +1,16 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <__config> +#include + +namespace std { // purposefully not versioned + +const char* __pstl_bad_alloc::what() const noexcept { return "PSTL memory allocation failed"; } + +} // namespace std