Index: pstl/trunk/include/pstl/internal/algorithm_impl.h =================================================================== --- pstl/trunk/include/pstl/internal/algorithm_impl.h +++ pstl/trunk/include/pstl/internal/algorithm_impl.h @@ -56,7 +56,7 @@ __pattern_any_of(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept { - return __brick_any_of(__first, __last, __pred, __is_vector); + return __internal::__brick_any_of(__first, __last, __pred, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -65,11 +65,11 @@ __pattern_any_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred, _IsVector __is_vector, /*parallel=*/std::true_type) { - return __except_handler([&]() { - return __parallel_or(std::forward<_ExecutionPolicy>(__exec), __first, __last, - [__pred, __is_vector](_ForwardIterator __i, _ForwardIterator __j) { - return __brick_any_of(__i, __j, __pred, __is_vector); - }); + return __internal::__except_handler([&]() { + return __internal::__parallel_or(std::forward<_ExecutionPolicy>(__exec), __first, __last, + [__pred, __is_vector](_ForwardIterator __i, _ForwardIterator __j) { + return __internal::__brick_any_of(__i, __j, __pred, __is_vector); + }); }); } #endif @@ -112,7 +112,7 @@ _IsVector __is_vector, /*parallel=*/std::false_type) noexcept { - __brick_walk1(__first, __last, __f, __is_vector); + __internal::__brick_walk1(__first, __last, __f, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -122,10 +122,10 @@ _IsVector __is_vector, /*parallel=*/std::true_type) { - __except_handler([&]() { + __internal::__except_handler([&]() { __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last, [__f, __is_vector](_ForwardIterator __i, _ForwardIterator __j) { - __brick_walk1(__i, __j, __f, __is_vector); + __internal::__brick_walk1(__i, __j, __f, __is_vector); }); }); } @@ -145,7 +145,7 @@ __pattern_walk_brick(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Brick __brick, /*parallel=*/std::true_type) { - __except_handler([&]() { + __internal::__except_handler([&]() { __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last, [__brick](_ForwardIterator __i, _ForwardIterator __j) { __brick(__i, __j); }); }); @@ -159,8 +159,8 @@ _ForwardIterator __brick_walk1_n(_ForwardIterator __first, _Size __n, _Function __f, /*_IsVectorTag=*/std::false_type) { - return __for_each_n_it_serial(__first, __n, - [&__f](_ForwardIterator __it) { __f(*__it); }); // calling serial version + return __internal::__for_each_n_it_serial(__first, __n, + [&__f](_ForwardIterator __it) { __f(*__it); }); // calling serial version } template @@ -176,7 +176,7 @@ __pattern_walk1_n(_ExecutionPolicy&&, _ForwardIterator __first, _Size __n, _Function __f, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_walk1_n(__first, __n, __f, __is_vector); + return __internal::__brick_walk1_n(__first, __n, __f, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -186,7 +186,8 @@ _IsVector __is_vector, /*is_parallel=*/std::true_type) { - __pattern_walk1(std::forward<_ExecutionPolicy>(__exec), __first, __first + __n, __f, __is_vector, std::true_type()); + __internal::__pattern_walk1(std::forward<_ExecutionPolicy>(__exec), __first, __first + __n, __f, __is_vector, + std::true_type()); return __first + __n; } #endif @@ -205,7 +206,7 @@ __pattern_walk_brick_n(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _Size __n, _Brick __brick, /*is_parallel=*/std::true_type) { - return __except_handler([&]() { + return __internal::__except_handler([&]() { __par_backend::__parallel_for( std::forward<_ExecutionPolicy>(__exec), __first, __first + __n, [__brick](_RandomAccessIterator __i, _RandomAccessIterator __j) { __brick(__i, __j - __i); }); @@ -260,7 +261,7 @@ __pattern_walk2(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Function __f, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept { - return __brick_walk2(__first1, __last1, __first2, __f, __is_vector); + return __internal::__brick_walk2(__first1, __last1, __first2, __f, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -269,11 +270,11 @@ __pattern_walk2(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Function __f, _IsVector __is_vector, /*parallel=*/std::true_type) { - return __except_handler([&]() { + return __internal::__except_handler([&]() { __par_backend::__parallel_for( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, [__f, __first1, __first2, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) { - __brick_walk2(__i, __j, __first2 + (__i - __first1), __f, __is_vector); + __internal::__brick_walk2(__i, __j, __first2 + (__i - __first1), __f, __is_vector); }); return __first2 + (__last1 - __first1); }); @@ -286,7 +287,7 @@ __pattern_walk2_n(_ExecutionPolicy&&, _ForwardIterator1 __first1, _Size n, _ForwardIterator2 __first2, _Function f, _IsVector is_vector, /*parallel=*/std::false_type) noexcept { - return __brick_walk2_n(__first1, n, __first2, f, is_vector); + return __internal::__brick_walk2_n(__first1, n, __first2, f, is_vector); } template (__exec), __first1, __first1 + n, __first2, f, is_vector, - std::true_type()); + return __internal::__pattern_walk2(std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + n, __first2, f, + is_vector, std::true_type()); } template @@ -313,7 +314,7 @@ __pattern_walk2_brick(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _Brick __brick, /*parallel=*/std::true_type) { - return __except_handler([&]() { + return __internal::__except_handler([&]() { __par_backend::__parallel_for( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, [__first1, __first2, __brick](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) { @@ -330,7 +331,7 @@ __pattern_walk2_brick_n(_ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1, _Size __n, _RandomAccessIterator2 __first2, _Brick __brick, /*parallel=*/std::true_type) { - return __except_handler([&]() { + return __internal::__except_handler([&]() { __par_backend::__parallel_for( std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n, [__first1, __first2, __brick](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) { @@ -378,7 +379,7 @@ __pattern_walk3(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator3 __first3, _Function __f, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept { - return __brick_walk3(__first1, __last1, __first2, __first3, __f, __is_vector); + return __internal::__brick_walk3(__first1, __last1, __first2, __first3, __f, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -389,11 +390,12 @@ _RandomAccessIterator2 __first2, _RandomAccessIterator3 __first3, _Function __f, _IsVector __is_vector, /*parallel=*/std::true_type) { - return __except_handler([&]() { + return __internal::__except_handler([&]() { __par_backend::__parallel_for( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, [__f, __first1, __first2, __first3, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) { - __brick_walk3(__i, __j, __first2 + (__i - __first1), __first3 + (__i - __first1), __f, __is_vector); + __internal::__brick_walk3(__i, __j, __first2 + (__i - __first1), __first3 + (__i - __first1), __f, + __is_vector); }); return __first3 + (__last1 - __first1); }); @@ -427,7 +429,7 @@ __pattern_equal(_ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _BinaryPredicate __p, _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept { - return __brick_equal(__first1, __last1, __first2, __p, __is_vector); + return __internal::__brick_equal(__first1, __last1, __first2, __p, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -438,11 +440,11 @@ _RandomAccessIterator2 __first2, _BinaryPredicate __p, _IsVector __is_vector, /*is_parallel=*/std::true_type) { - return __except_handler([&]() { - return !__parallel_or( + return __internal::__except_handler([&]() { + return !__internal::__parallel_or( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, [__first1, __first2, __p, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) { - return !__brick_equal(__i, __j, __first2 + (__i - __first1), __p, __is_vector); + return !__internal::__brick_equal(__i, __j, __first2 + (__i - __first1), __p, __is_vector); }); }); } @@ -476,7 +478,7 @@ _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_find_if(__first, __last, __pred, __is_vector); + return __internal::__brick_find_if(__first, __last, __pred, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -486,11 +488,11 @@ _IsVector __is_vector, /*is_parallel=*/std::true_type) { - return __except_handler([&]() { - return __parallel_find( + return __internal::__except_handler([&]() { + return __internal::__parallel_find( std::forward<_ExecutionPolicy>(__exec), __first, __last, [__pred, __is_vector](_ForwardIterator __i, _ForwardIterator __j) { - return __brick_find_if(__i, __j, __pred, __is_vector); + return __internal::__brick_find_if(__i, __j, __pred, __is_vector); }, std::less::difference_type>(), /*is_first=*/true); @@ -528,14 +530,14 @@ while (__first != __last && (__global_last - __first >= __n2)) { // find position of *s_first in [first, last) (it can be start of subsequence) - __first = __brick_find_if(__first, __last, - __equal_value_by_pred<_ValueType, _BinaryPredicate>(*__s_first, __pred), __is_vector); + __first = __internal::__brick_find_if( + __first, __last, __equal_value_by_pred<_ValueType, _BinaryPredicate>(*__s_first, __pred), __is_vector); // if position that was found previously is the start of subsequence // then we can exit the loop (b_first == true) or keep the position // (b_first == false) if (__first != __last && (__global_last - __first >= __n2) && - __brick_equal(__s_first + 1, __s_last, __first + 1, __pred, __is_vector)) + __internal::__brick_equal(__s_first + 1, __s_last, __first + 1, __pred, __is_vector)) { if (__b_first) { @@ -575,12 +577,12 @@ auto __unary_pred = __equal_value_by_pred<_Tp, _BinaryPredicate>(__value, __pred); while (__first != __last && (__global_last - __first >= __count)) { - __first = __brick_find_if(__first, __last, __unary_pred, __is_vector); + __first = __internal::__brick_find_if(__first, __last, __unary_pred, __is_vector); // check that all of elements in [first+1, first+count) equal to value if (__first != __last && (__global_last - __first >= __count) && - !__brick_any_of(__first + 1, __first + __count, __not_pred(__unary_pred), - __is_vector)) + !__internal::__brick_any_of(__first + 1, __first + __count, + __not_pred(__unary_pred), __is_vector)) { return __first; } @@ -619,7 +621,7 @@ _ForwardIterator2 __s_last, _BinaryPredicate __pred, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_find_end(__first, __last, __s_first, __s_last, __pred, __is_vector); + return __internal::__brick_find_end(__first, __last, __s_first, __s_last, __pred, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -632,17 +634,18 @@ { if (__last - __first == __s_last - __s_first) { - const bool __res = __pattern_equal(std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __pred, - __is_vector, std::true_type()); + const bool __res = __internal::__pattern_equal(std::forward<_ExecutionPolicy>(__exec), __first, __last, + __s_first, __pred, __is_vector, std::true_type()); return __res ? __first : __last; } else { - return __except_handler([&]() { - return __parallel_find( + return __internal::__except_handler([&]() { + return __internal::__parallel_find( std::forward<_ExecutionPolicy>(__exec), __first, __last, [__last, __s_first, __s_last, __pred, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) { - return __find_subrange(__i, __j, __last, __s_first, __s_last, __pred, false, __is_vector); + return __internal::__find_subrange(__i, __j, __last, __s_first, __s_last, __pred, false, + __is_vector); }, std::greater::difference_type>(), /*is_first=*/false); }); @@ -676,7 +679,7 @@ _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_find_first_of(__first, __last, __s_first, __s_last, __pred, __is_vector); + return __internal::__brick_find_first_of(__first, __last, __s_first, __s_last, __pred, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -687,11 +690,11 @@ _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred, _IsVector __is_vector, /*is_parallel=*/std::true_type) noexcept { - return __except_handler([&]() { - return __parallel_find( + return __internal::__except_handler([&]() { + return __internal::__parallel_find( std::forward<_ExecutionPolicy>(__exec), __first, __last, [__s_first, __s_last, __pred, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) { - return __brick_find_first_of(__i, __j, __s_first, __s_last, __pred, __is_vector); + return __internal::__brick_find_first_of(__i, __j, __s_first, __s_last, __pred, __is_vector); }, std::less::difference_type>(), /*is_first=*/true); }); @@ -714,7 +717,7 @@ __brick_search(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept { - return __find_subrange(__first, __last, __last, __s_first, __s_last, __pred, true, std::true_type()); + return __internal::__find_subrange(__first, __last, __last, __s_first, __s_last, __pred, true, std::true_type()); } template (__exec), __first, __last, __s_first, __pred, - __is_vector, std::true_type()); + const bool __res = __internal::__pattern_equal(std::forward<_ExecutionPolicy>(__exec), __first, __last, + __s_first, __pred, __is_vector, std::true_type()); return __res ? __first : __last; } else { - return __except_handler([&]() { - return __parallel_find( + return __internal::__except_handler([&]() { + return __internal::__parallel_find( std::forward<_ExecutionPolicy>(__exec), __first, __last, [__last, __s_first, __s_last, __pred, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) { - return __find_subrange(__i, __j, __last, __s_first, __s_last, __pred, true, __is_vector); + return __internal::__find_subrange(__i, __j, __last, __s_first, __s_last, __pred, true, + __is_vector); }, std::less::difference_type>(), /*is_first=*/true); }); @@ -772,7 +776,7 @@ __brick_search_n(_ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value, _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept { - return __find_subrange(__first, __last, __last, __count, __value, __pred, std::true_type()); + return __internal::__find_subrange(__first, __last, __last, __count, __value, __pred, std::true_type()); } template (__exec), __first, __last, [&__value, &__pred](const _Tp& __val) { return !__pred(__val, __value); }, __is_vector, /*is_parallel*/ std::true_type()); @@ -803,11 +807,11 @@ } else { - return __except_handler([&__exec, __first, __last, __count, &__value, __pred, __is_vector]() { - return __parallel_find( + return __internal::__except_handler([&__exec, __first, __last, __count, &__value, __pred, __is_vector]() { + return __internal::__parallel_find( std::forward<_ExecutionPolicy>(__exec), __first, __last, [__last, __count, &__value, __pred, __is_vector](_RandomAccessIterator __i, _RandomAccessIterator __j) { - return __find_subrange(__i, __j, __last, __count, __value, __pred, __is_vector); + return __internal::__find_subrange(__i, __j, __last, __count, __value, __pred, __is_vector); }, std::less::difference_type>(), /*is_first=*/true); }); @@ -975,7 +979,7 @@ #if (__PSTL_MONOTONIC_PRESENT) __unseq_backend::__simd_copy_by_mask(__first, __last - __first, __result, __mask, __assigner); #else - __brick_copy_by_mask(__first, __last, __result, __mask, __assigner, std::false_type()); + __internal::__brick_copy_by_mask(__first, __last, __result, __mask, __assigner, std::false_type()); #endif } @@ -1007,7 +1011,7 @@ #if (__PSTL_MONOTONIC_PRESENT) __unseq_backend::__simd_partition_by_mask(__first, __last - __first, __out_true, __out_false, __mask); #else - __brick_partition_by_mask(__first, __last, __out_true, __out_false, __mask, std::false_type()); + __internal::__brick_partition_by_mask(__first, __last, __out_true, __out_false, __mask, std::false_type()); #endif } @@ -1016,7 +1020,7 @@ __pattern_copy_if(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, _UnaryPredicate __pred, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept { - return __brick_copy_if(__first, __last, __result, __pred, __is_vector); + return __internal::__brick_copy_if(__first, __last, __result, __pred, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -1031,19 +1035,19 @@ if (_DifferenceType(1) < __n) { __par_backend::__buffer __mask_buf(__n); - return __except_handler([&__exec, __n, __first, __result, __is_vector, __pred, &__mask_buf]() { + return __internal::__except_handler([&__exec, __n, __first, __result, __is_vector, __pred, &__mask_buf]() { bool* __mask = __mask_buf.get(); _DifferenceType __m{}; __par_backend::parallel_strict_scan( std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0), [=](_DifferenceType __i, _DifferenceType __len) { // Reduce - return __brick_calc_mask_1<_DifferenceType>(__first + __i, __first + (__i + __len), __mask + __i, - __pred, __is_vector) + return __internal::__brick_calc_mask_1<_DifferenceType>(__first + __i, __first + (__i + __len), + __mask + __i, __pred, __is_vector) .first; }, std::plus<_DifferenceType>(), // Combine [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { // Scan - __brick_copy_by_mask( + __internal::__brick_copy_by_mask( __first + __i, __first + (__i + __len), __result + __initial, __mask + __i, [](_RandomAccessIterator __x, _OutputIterator __z) { *__z = *__x; }, __is_vector); }, @@ -1052,7 +1056,7 @@ }); } // trivial sequence - use serial algorithm - return __brick_copy_if(__first, __last, __result, __pred, __is_vector); + return __internal::__brick_copy_if(__first, __last, __result, __pred, __is_vector); } #endif @@ -1080,7 +1084,7 @@ __pattern_count(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred, /* is_parallel */ std::false_type, _IsVector __is_vector) noexcept { - return __brick_count(__first, __last, __pred, __is_vector); + return __internal::__brick_count(__first, __last, __pred, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -1090,11 +1094,11 @@ /* is_parallel */ std::true_type, _IsVector __is_vector) { typedef typename std::iterator_traits<_ForwardIterator>::difference_type _SizeType; - return __except_handler([&]() { + return __internal::__except_handler([&]() { return __par_backend::__parallel_reduce( std::forward<_ExecutionPolicy>(__exec), __first, __last, _SizeType(0), [__pred, __is_vector](_ForwardIterator __begin, _ForwardIterator __end, _SizeType __value) -> _SizeType { - return __value + __brick_count(__begin, __end, __pred, __is_vector); + return __value + __internal::__brick_count(__begin, __end, __pred, __is_vector); }, std::plus<_SizeType>()); }); @@ -1127,7 +1131,7 @@ __pattern_unique(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_unique(__first, __last, __pred, __is_vector); + return __internal::__brick_unique(__first, __last, __pred, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -1135,15 +1139,15 @@ // So, a caller passes _CalcMask brick into remove_elements. template _ForwardIterator -remove_elements(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _CalcMask __calc_mask, - _IsVector __is_vector) +__remove_elements(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _CalcMask __calc_mask, + _IsVector __is_vector) { typedef typename std::iterator_traits<_ForwardIterator>::difference_type _DifferenceType; typedef typename std::iterator_traits<_ForwardIterator>::value_type _Tp; _DifferenceType __n = __last - __first; __par_backend::__buffer __mask_buf(__n); // 1. find a first iterator that should be removed - return __except_handler([&]() { + return __internal::__except_handler([&]() { bool* __mask = __mask_buf.get(); _DifferenceType __min = __par_backend::__parallel_reduce( std::forward<_ExecutionPolicy>(__exec), _DifferenceType(0), __n, __n, @@ -1158,8 +1162,8 @@ return __local_min; } // find first iterator that should be removed - bool* __result = __brick_find_if( - __mask + __i, __mask + __j, [](bool __val) { return !__val; }, __is_vector); + bool* __result = __internal::__brick_find_if(__mask + __i, __mask + __j, + [](bool __val) { return !__val; }, __is_vector); if (__result - __mask == __j) { return __local_min; @@ -1186,17 +1190,16 @@ __par_backend::parallel_strict_scan( std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0), [__mask, __is_vector](_DifferenceType __i, _DifferenceType __len) { - return __brick_count( - __mask + __i, __mask + __i + __len, [](bool __val) { return __val; }, __is_vector); + return __internal::__brick_count(__mask + __i, __mask + __i + __len, [](bool __val) { return __val; }, + __is_vector); }, std::plus<_DifferenceType>(), [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { - __brick_copy_by_mask( + __internal::__brick_copy_by_mask( __first + __i, __first + __i + __len, __result + __initial, __mask + __i, [](_ForwardIterator __x, _Tp* __z) { - __invoke_if_else( - std::is_trivial<_Tp>(), [&]() { *__z = std::move(*__x); }, - [&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); }); + __internal::__invoke_if_else(std::is_trivial<_Tp>(), [&]() { *__z = std::move(*__x); }, + [&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); }); }, __is_vector); }, @@ -1205,7 +1208,7 @@ // 3. Elements from result are moved to [first, last) __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __result, __result + __m, [__result, __first, __is_vector](_Tp* __i, _Tp* __j) { - __brick_move(__i, __j, __first + (__i - __result), __is_vector); + __internal::__brick_move(__i, __j, __first + (__i - __result), __is_vector); }); return __first + __m; }); @@ -1227,12 +1230,12 @@ if (__first + 1 == __last || __first + 2 == __last) { // Trivial sequence - use serial algorithm - return __brick_unique(__first, __last, __pred, __is_vector); + return __internal::__brick_unique(__first, __last, __pred, __is_vector); } - return remove_elements( + return __internal::__remove_elements( std::forward<_ExecutionPolicy>(__exec), ++__first, __last, [&__pred, __is_vector](bool* __b, bool* __e, _ForwardIterator __it) { - __brick_walk3( + __internal::__brick_walk3( __b, __e, __it - 1, __it, [&__pred](bool& __x, _ReferenceType __y, _ReferenceType __z) { __x = !__pred(__y, __z); }, __is_vector); }, @@ -1270,7 +1273,7 @@ __pattern_unique_copy(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, _BinaryPredicate __pred, _IsVector __is_vector, /*parallel=*/std::false_type) noexcept { - return __brick_unique_copy(__first, __last, __result, __pred, __is_vector); + return __internal::__brick_unique_copy(__first, __last, __result, __pred, __is_vector); } template @@ -1310,7 +1313,7 @@ __par_backend::__buffer __mask_buf(__n); if (_DifferenceType(2) < __n) { - return __except_handler([&__exec, __n, __first, __result, __pred, __is_vector, &__mask_buf]() { + return __internal::__except_handler([&__exec, __n, __first, __result, __pred, __is_vector, &__mask_buf]() { bool* __mask = __mask_buf.get(); _DifferenceType __m{}; __par_backend::parallel_strict_scan( @@ -1326,14 +1329,14 @@ ++__i; ++__extra; } - return __brick_calc_mask_2<_DifferenceType>(__first + __i, __first + (__i + __len), - __mask + __i, __pred, __is_vector) + + return __internal::__brick_calc_mask_2<_DifferenceType>(__first + __i, __first + (__i + __len), + __mask + __i, __pred, __is_vector) + __extra; }, std::plus<_DifferenceType>(), // Combine [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { // Scan // Phase 2 is same as for __pattern_copy_if - __brick_copy_by_mask( + __internal::__brick_copy_by_mask( __first + __i, __first + (__i + __len), __result + __initial, __mask + __i, [](_RandomAccessIterator __x, _OutputIterator __z) { *__z = *__x; }, __is_vector); }, @@ -1343,7 +1346,7 @@ } } // trivial sequence - use serial algorithm - return __brick_unique_copy(__first, __last, __result, __pred, __is_vector); + return __internal::__brick_unique_copy(__first, __last, __result, __pred, __is_vector); } #endif @@ -1405,7 +1408,7 @@ _IsVector _is_vector, /*is_parallel=*/std::false_type) noexcept { - __brick_reverse(__first, __last, _is_vector); + __internal::__brick_reverse(__first, __last, _is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -1417,7 +1420,7 @@ __par_backend::__parallel_for( std::forward<_ExecutionPolicy>(__exec), __first, __first + (__last - __first) / 2, [__is_vector, __first, __last](_BidirectionalIterator __inner_first, _BidirectionalIterator __inner_last) { - __brick_reverse(__inner_first, __inner_last, __last - (__inner_first - __first), __is_vector); + __internal::__brick_reverse(__inner_first, __inner_last, __last - (__inner_first - __first), __is_vector); }); } #endif @@ -1451,7 +1454,7 @@ __pattern_reverse_copy(_ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __d_first, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_reverse_copy(__first, __last, __d_first, __is_vector); + return __internal::__brick_reverse_copy(__first, __last, __d_first, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -1464,8 +1467,9 @@ __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last, [__is_vector, __first, __len, __d_first](_BidirectionalIterator __inner_first, _BidirectionalIterator __inner_last) { - __brick_reverse_copy(__inner_first, __inner_last, - __d_first + (__len - (__inner_last - __first)), __is_vector); + __internal::__brick_reverse_copy(__inner_first, __inner_last, + __d_first + (__len - (__inner_last - __first)), + __is_vector); }); return __d_first + __len; } @@ -1533,7 +1537,7 @@ __pattern_rotate(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_rotate(__first, __middle, __last, __is_vector); + return __internal::__brick_rotate(__first, __middle, __last, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -1548,22 +1552,24 @@ if (__m <= __n / 2) { __par_backend::__buffer<_Tp> __buf(__n - __m); - return __except_handler([&__exec, __n, __m, __first, __middle, __last, __is_vector, &__buf]() { + return __internal::__except_handler([&__exec, __n, __m, __first, __middle, __last, __is_vector, &__buf]() { _Tp* __result = __buf.get(); __par_backend::__parallel_for( std::forward<_ExecutionPolicy>(__exec), __middle, __last, [__middle, __result, __is_vector](_ForwardIterator __b, _ForwardIterator __e) { - __brick_uninitialized_move(__b, __e, __result + (__b - __middle), __is_vector); + __internal::__brick_uninitialized_move(__b, __e, __result + (__b - __middle), __is_vector); }); __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __middle, [__last, __middle, __is_vector](_ForwardIterator __b, _ForwardIterator __e) { - __brick_move(__b, __e, __b + (__last - __middle), __is_vector); + __internal::__brick_move(__b, __e, __b + (__last - __middle), + __is_vector); }); __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __result, __result + (__n - __m), [__first, __result, __is_vector](_Tp* __b, _Tp* __e) { - __brick_move(__b, __e, __first + (__b - __result), __is_vector); + __internal::__brick_move(__b, __e, __first + (__b - __result), + __is_vector); }); return __first + (__last - __middle); @@ -1572,23 +1578,24 @@ else { __par_backend::__buffer<_Tp> __buf(__m); - return __except_handler([&__exec, __n, __m, __first, __middle, __last, __is_vector, &__buf]() { + return __internal::__except_handler([&__exec, __n, __m, __first, __middle, __last, __is_vector, &__buf]() { _Tp* __result = __buf.get(); __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __middle, [__first, __result, __is_vector](_ForwardIterator __b, _ForwardIterator __e) { - __brick_uninitialized_move(__b, __e, __result + (__b - __first), - __is_vector); + __internal::__brick_uninitialized_move( + __b, __e, __result + (__b - __first), __is_vector); }); __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __middle, __last, [__first, __middle, __is_vector](_ForwardIterator __b, _ForwardIterator __e) { - __brick_move(__b, __e, __first + (__b - __middle), __is_vector); + __internal::__brick_move(__b, __e, __first + (__b - __middle), + __is_vector); }); __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __result, __result + __m, [__n, __m, __first, __result, __is_vector](_Tp* __b, _Tp* __e) { - __brick_move(__b, __e, __first + ((__n - __m) + (__b - __result)), - __is_vector); + __internal::__brick_move( + __b, __e, __first + ((__n - __m) + (__b - __result)), __is_vector); }); return __first + (__last - __middle); @@ -1614,8 +1621,8 @@ __brick_rotate_copy(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last, _OutputIterator __result, /*__is_vector=*/std::true_type) noexcept { - _OutputIterator __res = __brick_copy(__middle, __last, __result, std::true_type()); - return __brick_copy(__first, __middle, __res, std::true_type()); + _OutputIterator __res = __internal::__brick_copy(__middle, __last, __result, std::true_type()); + return __internal::__brick_copy(__first, __middle, __res, std::true_type()); } template @@ -1623,7 +1630,7 @@ __pattern_rotate_copy(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last, _OutputIterator __result, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_rotate_copy(__first, __middle, __last, __result, __is_vector); + return __internal::__brick_rotate_copy(__first, __middle, __last, __result, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -1638,19 +1645,19 @@ [__first, __last, __middle, __result, __is_vector](_ForwardIterator __b, _ForwardIterator __e) { if (__b > __middle) { - __brick_copy(__b, __e, __result + (__b - __middle), __is_vector); + __internal::__brick_copy(__b, __e, __result + (__b - __middle), __is_vector); } else { _OutputIterator __new_result = __result + ((__last - __middle) + (__b - __first)); if (__e < __middle) { - __brick_copy(__b, __e, __new_result, __is_vector); + __internal::__brick_copy(__b, __e, __new_result, __is_vector); } else { - __brick_copy(__b, __middle, __new_result, __is_vector); - __brick_copy(__middle, __e, __result, __is_vector); + __internal::__brick_copy(__b, __middle, __new_result, __is_vector); + __internal::__brick_copy(__middle, __e, __result, __is_vector); } } }); @@ -1702,7 +1709,7 @@ __pattern_is_partitioned(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_is_partitioned(__first, __last, __pred, __is_vector); + return __internal::__brick_is_partitioned(__first, __last, __pred, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -1717,7 +1724,7 @@ } else { - return __except_handler([&]() { + return __internal::__except_handler([&]() { // State of current range: // broken - current range is not partitioned by pred // all_true - all elements in current range satisfy pred @@ -1753,11 +1760,11 @@ { // find first element that don't satisfy pred _ForwardIterator __x = - __brick_find_if(__i + 1, __j, __not_pred<_UnaryPredicate>(__pred), __is_vector); + __internal::__brick_find_if(__i + 1, __j, __not_pred<_UnaryPredicate>(__pred), __is_vector); if (__x != __j) { // find first element after "x" that satisfy pred - _ForwardIterator __y = __brick_find_if(__x + 1, __j, __pred, __is_vector); + _ForwardIterator __y = __internal::__brick_find_if(__x + 1, __j, __pred, __is_vector); // if it was found then range isn't partitioned by pred if (__y != __j) { @@ -1777,7 +1784,7 @@ { // if first element doesn't satisfy pred // then we should find the first element that satisfy pred. // If we found it then range isn't partitioned by pred - if (__brick_find_if(__i + 1, __j, __pred, __is_vector) != __j) + if (__internal::__brick_find_if(__i + 1, __j, __pred, __is_vector) != __j) { return __broken; } @@ -1830,7 +1837,7 @@ __pattern_partition(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_partition(__first, __last, __pred, __is_vector); + return __internal::__brick_partition(__first, __last, __pred, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -1849,7 +1856,7 @@ _ForwardIterator __end; }; - return __except_handler([&]() { + return __internal::__except_handler([&]() { _PartitionRange __init{__last, __last, __last}; // lambda for merging two partitioned ranges to one partitioned range @@ -1870,7 +1877,8 @@ __par_backend::__parallel_for( std::forward<_ExecutionPolicy>(__exec), __val1.__pivot, __val1.__pivot + __size1, [__val1, __val2, __size1, __is_vector](_ForwardIterator __i, _ForwardIterator __j) { - __brick_swap_ranges(__i, __j, (__val2.__pivot - __size1) + (__i - __val1.__pivot), __is_vector); + __internal::__brick_swap_ranges(__i, __j, (__val2.__pivot - __size1) + (__i - __val1.__pivot), + __is_vector); }); return {__new_begin, __val2.__pivot - __size1, __val2.__end}; } @@ -1880,7 +1888,7 @@ __par_backend::__parallel_for( std::forward<_ExecutionPolicy>(__exec), __val1.__pivot, __val1.__pivot + __size2, [__val1, __val2, __is_vector](_ForwardIterator __i, _ForwardIterator __j) { - __brick_swap_ranges(__i, __j, __val2.__begin + (__i - __val1.__pivot), __is_vector); + __internal::__brick_swap_ranges(__i, __j, __val2.__begin + (__i - __val1.__pivot), __is_vector); }); return {__new_begin, __val1.__pivot + __size2, __val2.__end}; } @@ -1891,7 +1899,7 @@ [__pred, __is_vector, __reductor](_ForwardIterator __i, _ForwardIterator __j, _PartitionRange __value) -> _PartitionRange { //1. serial partition - _ForwardIterator __pivot = __brick_partition(__i, __j, __pred, __is_vector); + _ForwardIterator __pivot = __internal::__brick_partition(__i, __j, __pred, __is_vector); // 2. merging of two ranges (left and right respectively) return __reductor(__value, {__i, __pivot, __j}); @@ -1929,7 +1937,7 @@ _UnaryPredicate __pred, _IsVector __is_vector, /*is_parallelization=*/std::false_type) noexcept { - return __brick_stable_partition(__first, __last, __pred, __is_vector); + return __internal::__brick_stable_partition(__first, __last, __pred, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -1948,7 +1956,7 @@ _BidirectionalIterator __end; }; - return __except_handler([&]() { + return __internal::__except_handler([&]() { _PartitionRange __init{__last, __last, __last}; // lambda for merging two partitioned ranges to one partitioned range @@ -1965,7 +1973,7 @@ // then we should swap the false part of left range and last part of true part of right range else { - __brick_rotate(__val1.__pivot, __val2.__begin, __val2.__pivot, __is_vector); + __internal::__brick_rotate(__val1.__pivot, __val2.__begin, __val2.__pivot, __is_vector); return {__new_begin, __val2.__pivot - __size1, __val2.__end}; } }; @@ -1975,7 +1983,7 @@ [&__pred, __is_vector, __reductor](_BidirectionalIterator __i, _BidirectionalIterator __j, _PartitionRange __value) -> _PartitionRange { //1. serial stable_partition - _BidirectionalIterator __pivot = __brick_stable_partition(__i, __j, __pred, __is_vector); + _BidirectionalIterator __pivot = __internal::__brick_stable_partition(__i, __j, __pred, __is_vector); // 2. merging of two ranges (left and right respectively) return __reductor(__value, {__i, __pivot, __j}); @@ -2017,7 +2025,7 @@ _OutputIterator1 __out_true, _OutputIterator2 __out_false, _UnaryPredicate __pred, _IsVector __is_vector, /*is_parallelization=*/std::false_type) noexcept { - return __brick_partition_copy(__first, __last, __out_true, __out_false, __pred, __is_vector); + return __internal::__brick_partition_copy(__first, __last, __out_true, __out_false, __pred, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -2034,28 +2042,30 @@ if (_DifferenceType(1) < __n) { __par_backend::__buffer __mask_buf(__n); - return __except_handler([&__exec, __n, __first, __out_true, __out_false, __is_vector, __pred, &__mask_buf]() { + return __internal::__except_handler([&__exec, __n, __first, __out_true, __out_false, __is_vector, __pred, + &__mask_buf]() { bool* __mask = __mask_buf.get(); _ReturnType __m{}; __par_backend::parallel_strict_scan( std::forward<_ExecutionPolicy>(__exec), __n, std::make_pair(_DifferenceType(0), _DifferenceType(0)), [=](_DifferenceType __i, _DifferenceType __len) { // Reduce - return __brick_calc_mask_1<_DifferenceType>(__first + __i, __first + (__i + __len), __mask + __i, - __pred, __is_vector); + return __internal::__brick_calc_mask_1<_DifferenceType>(__first + __i, __first + (__i + __len), + __mask + __i, __pred, __is_vector); }, [](const _ReturnType& __x, const _ReturnType& __y) -> _ReturnType { return std::make_pair(__x.first + __y.first, __x.second + __y.second); }, // Combine [=](_DifferenceType __i, _DifferenceType __len, _ReturnType __initial) { // Scan - __brick_partition_by_mask(__first + __i, __first + (__i + __len), __out_true + __initial.first, - __out_false + __initial.second, __mask + __i, __is_vector); + __internal::__brick_partition_by_mask(__first + __i, __first + (__i + __len), + __out_true + __initial.first, __out_false + __initial.second, + __mask + __i, __is_vector); }, [&__m](_ReturnType __total) { __m = __total; }); return std::make_pair(__out_true + __m.first, __out_false + __m.second); }); } // trivial sequence - use serial algorithm - return __brick_partition_copy(__first, __last, __out_true, __out_false, __pred, __is_vector); + return __internal::__brick_partition_copy(__first, __last, __out_true, __out_false, __pred, __is_vector); } #endif @@ -2078,13 +2088,11 @@ __pattern_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _IsVector /*is_vector*/, /*is_parallel=*/std::true_type, /*is_move_constructible=*/std::true_type) { - __except_handler([&]() { - __par_backend::__parallel_stable_sort( - std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp, - [](_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { - std::sort(__first, __last, __comp); - }, - __last - __first); + __internal::__except_handler([&]() { + __par_backend::__parallel_stable_sort(std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp, + [](_RandomAccessIterator __first, _RandomAccessIterator __last, + _Compare __comp) { std::sort(__first, __last, __comp); }, + __last - __first); }); } #endif @@ -2107,7 +2115,7 @@ __pattern_stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _IsVector /*is_vector*/, /*is_parallel=*/std::true_type) { - __except_handler([&]() { + __internal::__except_handler([&]() { __par_backend::__parallel_stable_sort(std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp, [](_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { std::stable_sort(__first, __last, __comp); }); @@ -2135,7 +2143,7 @@ _RandomAccessIterator __last, _Compare __comp, _IsVector, /*is_parallel=*/std::true_type) { const auto __n = __middle - __first; - __except_handler([&]() { + __internal::__except_handler([&]() { __par_backend::__parallel_stable_sort( std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp, [__n](_RandomAccessIterator __begin, _RandomAccessIterator __end, _Compare __comp) { @@ -2175,7 +2183,7 @@ } auto __n1 = __last - __first; auto __n2 = __d_last - __d_first; - return __except_handler([&]() { + return __internal::__except_handler([&]() { if (__n2 >= __n1) { __par_backend::__parallel_stable_sort( @@ -2187,7 +2195,7 @@ // 1. Copy elements from input to output #if !__PSTL_ICC_18_OMP_SIMD_BROKEN - __brick_copy(__i1, __j1, __i, __is_vector); + __internal::__brick_copy(__i1, __j1, __i, __is_vector); #else std::copy(__i1, __j1, __i); #endif @@ -2204,29 +2212,28 @@ __par_backend::__buffer<_T1> __buf(__n1); _T1* __r = __buf.get(); - __par_backend::__parallel_stable_sort( - std::forward<_ExecutionPolicy>(__exec), __r, __r + __n1, __comp, - [__n2, __first, __r](_T1* __i, _T1* __j, _Compare __comp) { - _ForwardIterator __it = __first + (__i - __r); - - // 1. Copy elements from input to raw memory - for (_T1* __k = __i; __k != __j; ++__k, ++__it) - { - ::new (__k) _T2(*__it); - } - - // 2. Sort elements in temporary __buffer - if (__n2 < __j - __i) - std::partial_sort(__i, __i + __n2, __j, __comp); - else - std::sort(__i, __j, __comp); - }, - __n2); + __par_backend::__parallel_stable_sort(std::forward<_ExecutionPolicy>(__exec), __r, __r + __n1, __comp, + [__n2, __first, __r](_T1* __i, _T1* __j, _Compare __comp) { + _ForwardIterator __it = __first + (__i - __r); + + // 1. Copy elements from input to raw memory + for (_T1* __k = __i; __k != __j; ++__k, ++__it) + { + ::new (__k) _T2(*__it); + } + + // 2. Sort elements in temporary __buffer + if (__n2 < __j - __i) + std::partial_sort(__i, __i + __n2, __j, __comp); + else + std::sort(__i, __j, __comp); + }, + __n2); // 3. Move elements from temporary __buffer to output __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __r, __r + __n2, [__r, __d_first, __is_vector](_T1* __i, _T1* __j) { - __brick_move(__i, __j, __d_first + (__i - __r), __is_vector); + __internal::__brick_move(__i, __j, __d_first + (__i - __r), __is_vector); }); return __d_first + __n2; } @@ -2258,7 +2265,7 @@ __pattern_adjacent_find(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred, /* is_parallel */ std::false_type, _IsVector __is_vector, bool __or_semantic) noexcept { - return __brick_adjacent_find(__first, __last, __pred, __is_vector, __or_semantic); + return __internal::__brick_adjacent_find(__first, __last, __pred, __is_vector, __or_semantic); } #if __PSTL_USE_PAR_POLICIES @@ -2271,7 +2278,7 @@ if (__last - __first < 2) return __last; - return __except_handler([&]() { + return __internal::__except_handler([&]() { return __par_backend::__parallel_reduce( std::forward<_ExecutionPolicy>(__exec), __first, __last, __last, [__last, __pred, __is_vector, __or_semantic](_RandomAccessIterator __begin, _RandomAccessIterator __end, @@ -2295,7 +2302,7 @@ //correct the global result iterator if the "brick" returns a local "__last" const _RandomAccessIterator __res = - __brick_adjacent_find(__begin, __end, __pred, __is_vector, __or_semantic); + __internal::__brick_adjacent_find(__begin, __end, __pred, __is_vector, __or_semantic); if (__res < __end) __value = __res; } @@ -2339,10 +2346,10 @@ _RandomAccessIterator __x; do { - __x = __pattern_partition( - std::forward<_ExecutionPolicy>(__exec), __first + 1, __last, - [&__comp, __first](const _Tp& __x) { return __comp(__x, *__first); }, __is_vector, - /*is_parallel=*/std::true_type()); + __x = __internal::__pattern_partition(std::forward<_ExecutionPolicy>(__exec), __first + 1, __last, + [&__comp, __first](const _Tp& __x) { return __comp(__x, *__first); }, + __is_vector, + /*is_parallel=*/std::true_type()); --__x; if (__x != __first) { @@ -2395,7 +2402,7 @@ __pattern_fill(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, /*is_parallel=*/std::false_type, _IsVector __is_vector) noexcept { - __brick_fill(__first, __last, __value, __is_vector); + __internal::__brick_fill(__first, __last, __value, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -2404,10 +2411,10 @@ __pattern_fill(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, /*is_parallel=*/std::true_type, _IsVector __is_vector) { - return __except_handler([&__exec, __first, __last, &__value, __is_vector]() { + return __internal::__except_handler([&__exec, __first, __last, &__value, __is_vector]() { __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last, [&__value, __is_vector](_ForwardIterator __begin, _ForwardIterator __end) { - __brick_fill(__begin, __end, __value, __is_vector); + __internal::__brick_fill(__begin, __end, __value, __is_vector); }); return __last; }); @@ -2433,7 +2440,7 @@ __pattern_fill_n(_ExecutionPolicy&&, _OutputIterator __first, _Size __count, const _Tp& __value, /*is_parallel=*/std::false_type, _IsVector __is_vector) noexcept { - return __brick_fill_n(__first, __count, __value, __is_vector); + return __internal::__brick_fill_n(__first, __count, __value, __is_vector); } template @@ -2441,8 +2448,8 @@ __pattern_fill_n(_ExecutionPolicy&& __exec, _OutputIterator __first, _Size __count, const _Tp& __value, /*is_parallel=*/std::true_type, _IsVector __is_vector) { - return __pattern_fill(std::forward<_ExecutionPolicy>(__exec), __first, __first + __count, __value, std::true_type(), - __is_vector); + return __internal::__pattern_fill(std::forward<_ExecutionPolicy>(__exec), __first, __first + __count, __value, + std::true_type(), __is_vector); } //------------------------------------------------------------------------ @@ -2469,7 +2476,7 @@ __pattern_generate(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Generator __g, /*is_parallel=*/std::false_type, _IsVector __is_vector) noexcept { - __brick_generate(__first, __last, __g, __is_vector); + __internal::__brick_generate(__first, __last, __g, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -2478,10 +2485,10 @@ __pattern_generate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Generator __g, /*is_parallel=*/std::true_type, _IsVector __is_vector) { - return __except_handler([&]() { + return __internal::__except_handler([&]() { __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __first, __last, [__g, __is_vector](_ForwardIterator __begin, _ForwardIterator __end) { - __brick_generate(__begin, __end, __g, __is_vector); + __internal::__brick_generate(__begin, __end, __g, __is_vector); }); return __last; }); @@ -2507,7 +2514,7 @@ __pattern_generate_n(_ExecutionPolicy&&, _OutputIterator __first, _Size __count, _Generator __g, /*is_parallel=*/std::false_type, _IsVector __is_vector) noexcept { - return __brick_generate_n(__first, __count, __g, __is_vector); + return __internal::__brick_generate_n(__first, __count, __g, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -2518,8 +2525,8 @@ { static_assert(__is_random_access_iterator<_OutputIterator>::value, "Pattern-brick error. Should be a random access iterator."); - return __pattern_generate(std::forward<_ExecutionPolicy>(__exec), __first, __first + __count, __g, std::true_type(), - __is_vector); + return __internal::__pattern_generate(std::forward<_ExecutionPolicy>(__exec), __first, __first + __count, __g, + std::true_type(), __is_vector); } #endif @@ -2552,7 +2559,7 @@ __pattern_remove_if(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred, _IsVector __is_vector, /*is_parallel*/ std::false_type) noexcept { - return __brick_remove_if(__first, __last, __pred, __is_vector); + return __internal::__brick_remove_if(__first, __last, __pred, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -2566,14 +2573,14 @@ if (__first == __last || __first + 1 == __last) { // Trivial sequence - use serial algorithm - return __brick_remove_if(__first, __last, __pred, __is_vector); + return __internal::__brick_remove_if(__first, __last, __pred, __is_vector); } - return remove_elements( + return __internal::__remove_elements( std::forward<_ExecutionPolicy>(__exec), __first, __last, [&__pred, __is_vector](bool* __b, bool* __e, _ForwardIterator __it) { - __brick_walk2( - __b, __e, __it, [&__pred](bool& __x, _ReferenceType __y) { __x = !__pred(__y); }, __is_vector); + __internal::__brick_walk2(__b, __e, __it, [&__pred](bool& __x, _ReferenceType __y) { __x = !__pred(__y); }, + __is_vector); }, __is_vector); } @@ -2609,7 +2616,7 @@ _ForwardIterator2 __last2, _OutputIterator __d_first, _Compare __comp, _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept { - return __brick_merge(__first1, __last1, __first2, __last2, __d_first, __comp, __is_vector); + return __internal::__brick_merge(__first1, __last1, __first2, __last2, __d_first, __comp, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -2623,8 +2630,9 @@ __par_backend::__parallel_merge( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __d_first, __comp, [__is_vector](_RandomAccessIterator1 __f1, _RandomAccessIterator1 __l1, _RandomAccessIterator2 __f2, - _RandomAccessIterator2 __l2, _OutputIterator __f3, - _Compare __comp) { return __brick_merge(__f1, __l1, __f2, __l2, __f3, __comp, __is_vector); }); + _RandomAccessIterator2 __l2, _OutputIterator __f3, _Compare __comp) { + return __internal::__brick_merge(__f1, __l1, __f2, __l2, __f3, __comp, __is_vector); + }); return __d_first + (__last1 - __first1) + (__last2 - __first2); } #endif @@ -2655,7 +2663,7 @@ _BidirectionalIterator __last, _Compare __comp, _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept { - __brick_inplace_merge(__first, __middle, __last, __comp, __is_vector); + __internal::__brick_inplace_merge(__first, __middle, __last, __comp, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -2673,15 +2681,14 @@ auto __n = __last - __first; __par_backend::__buffer<_Tp> __buf(__n); _Tp* __r = __buf.get(); - __except_handler([&]() { + __internal::__except_handler([&]() { auto __move_values = [](_BidirectionalIterator __x, _Tp* __z) { - __invoke_if_else( - std::is_trivial<_Tp>(), [&]() { *__z = std::move(*__x); }, - [&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); }); + __internal::__invoke_if_else(std::is_trivial<_Tp>(), [&]() { *__z = std::move(*__x); }, + [&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); }); }; auto __move_sequences = [](_BidirectionalIterator __first1, _BidirectionalIterator __last1, _Tp* __first2) { - return __brick_uninitialized_move(__first1, __last1, __first2, _IsVector()); + return __internal::__brick_uninitialized_move(__first1, __last1, __first2, _IsVector()); }; __par_backend::__parallel_merge( @@ -2697,7 +2704,7 @@ __par_backend::__parallel_for(std::forward<_ExecutionPolicy>(__exec), __r, __r + __n, [__r, __first, __is_vector](_Tp* __i, _Tp* __j) { - __brick_move(__i, __j, __first + (__i - __r), __is_vector); + __internal::__brick_move(__i, __j, __first + (__i - __r), __is_vector); }); }); } @@ -2736,8 +2743,8 @@ if (__last2 - __first2 == 1) return !__comp(*__first1, *__first2) && !__comp(*__first2, *__first1); - return __except_handler([&]() { - return !__parallel_or( + return __internal::__except_handler([&]() { + return !__internal::__parallel_or( std::forward<_ExecutionPolicy>(__exec), __first2, __last2, [__first1, __last1, __first2, __last2, &__comp](_ForwardIterator2 __i, _ForwardIterator2 __j) { assert(__j > __i); @@ -2801,14 +2808,14 @@ __par_backend::__buffer<_T> __buf(__size_func(__n1, __n2)); - return __except_handler([&__exec, __n1, __first1, __last1, __first2, __last2, __result, __is_vector, __comp, - __size_func, __set_op, &__buf]() { + return __internal::__except_handler([&__exec, __n1, __first1, __last1, __first2, __last2, __result, __is_vector, + __comp, __size_func, __set_op, &__buf]() { auto __buffer = __buf.get(); _DifferenceType __m{}; auto __scan = [=](_DifferenceType, _DifferenceType, const _SetRange& __s) { // Scan if (!__s.empty()) - __brick_move(__buffer + __s.__buf_pos, __buffer + (__s.__buf_pos + __s.__len), __result + __s.__pos, - __is_vector); + __internal::__brick_move(__buffer + __s.__buf_pos, __buffer + (__s.__buf_pos + __s.__len), + __result + __s.__pos, __is_vector); }; __par_backend::parallel_strict_scan( std::forward<_ExecutionPolicy>(__exec), __n1, _SetRange{0, 0, 0}, //-1, 0}, @@ -2881,24 +2888,24 @@ const auto __n2 = __last2 - __first2; auto copy_range1 = [__is_vector](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res) { - return __brick_copy(__begin, __end, __res, __is_vector); + return __internal::__brick_copy(__begin, __end, __res, __is_vector); }; auto copy_range2 = [__is_vector](_ForwardIterator2 __begin, _ForwardIterator2 __end, _OutputIterator __res) { - return __brick_copy(__begin, __end, __res, __is_vector); + return __internal::__brick_copy(__begin, __end, __res, __is_vector); }; // {1} {}: parallel copying just first sequence if (__n2 == 0) - return __pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result, copy_range1, - std::true_type()); + return __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result, + copy_range1, std::true_type()); // {} {2}: parallel copying justmake second sequence if (__n1 == 0) - return __pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __last2, __result, copy_range2, - std::true_type()); + return __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __last2, __result, + copy_range2, std::true_type()); // testing whether the sequences are intersected - _ForwardIterator1 __left_bound_seq_1 = lower_bound(__first1, __last1, *__first2, __comp); + _ForwardIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp); if (__left_bound_seq_1 == __last1) { @@ -2906,18 +2913,18 @@ __par_backend::__parallel_invoke( std::forward<_ExecutionPolicy>(__exec), [=] { - __pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result, copy_range1, - std::true_type()); + __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result, + copy_range1, std::true_type()); }, [=] { - __pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __last2, __result + __n1, - copy_range2, std::true_type()); + __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __last2, + __result + __n1, copy_range2, std::true_type()); }); return __result + __n1 + __n2; } // testing whether the sequences are intersected - _ForwardIterator2 __left_bound_seq_2 = lower_bound(__first2, __last2, *__first1, __comp); + _ForwardIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp); if (__left_bound_seq_2 == __last2) { @@ -2925,12 +2932,12 @@ __par_backend::__parallel_invoke( std::forward<_ExecutionPolicy>(__exec), [=] { - __pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __last2, __result, copy_range2, - std::true_type()); + __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __last2, __result, + copy_range2, std::true_type()); }, [=] { - __pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result + __n2, - copy_range1, std::true_type()); + __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, + __result + __n2, copy_range1, std::true_type()); }); return __result + __n1 + __n2; } @@ -2944,11 +2951,11 @@ std::forward<_ExecutionPolicy>(__exec), //do parallel copying of [first1; left_bound_seq_1) [=] { - __pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __left_bound_seq_1, __res_or, - copy_range1, std::true_type()); + __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first1, __left_bound_seq_1, + __res_or, copy_range1, std::true_type()); }, [=, &__result] { - __result = __parallel_set_op( + __result = __internal::__parallel_set_op( std::forward<_ExecutionPolicy>(__exec), __left_bound_seq_1, __last1, __first2, __last2, __result, __comp, [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; }, __set_union_op, __is_vector); @@ -2966,11 +2973,11 @@ std::forward<_ExecutionPolicy>(__exec), //do parallel copying of [first2; left_bound_seq_2) [=] { - __pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __left_bound_seq_2, __res_or, - copy_range2, std::true_type()); + __internal::__pattern_walk2_brick(std::forward<_ExecutionPolicy>(__exec), __first2, __left_bound_seq_2, + __res_or, copy_range2, std::true_type()); }, [=, &__result] { - __result = __parallel_set_op( + __result = __internal::__parallel_set_op( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __left_bound_seq_2, __last2, __result, __comp, [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; }, __set_union_op, __is_vector); @@ -2978,7 +2985,7 @@ return __result; } - return __parallel_set_op( + return __internal::__parallel_set_op( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp, [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; }, __set_union_op, __is_vector); } @@ -3015,7 +3022,7 @@ _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_set_union(__first1, __last1, __first2, __last2, __result, __comp, __is_vector); + return __internal::__brick_set_union(__first1, __last1, __first2, __last2, __result, __comp, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -3035,7 +3042,7 @@ return std::set_union(__first1, __last1, __first2, __last2, __result, __comp); typedef typename std::iterator_traits<_OutputIterator>::value_type _T; - return __parallel_set_union_op( + return __internal::__parallel_set_union_op( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp, [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2, _T* __result, @@ -3074,7 +3081,7 @@ _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_set_intersection(__first1, __last1, __first2, __last2, __result, __comp, __is_vector); + return __internal::__brick_set_intersection(__first1, __last1, __first2, __last2, __result, __comp, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -3096,13 +3103,13 @@ return __result; // testing whether the sequences are intersected - _ForwardIterator1 __left_bound_seq_1 = lower_bound(__first1, __last1, *__first2, __comp); + _ForwardIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp); //{1} < {2}: seq 2 is wholly greater than seq 1, so, the intersection is empty if (__left_bound_seq_1 == __last1) return __result; // testing whether the sequences are intersected - _ForwardIterator2 __left_bound_seq_2 = lower_bound(__first2, __last2, *__first1, __comp); + _ForwardIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp); //{2} < {1}: seq 1 is wholly greater than seq 2, so, the intersection is empty if (__left_bound_seq_2 == __last2) return __result; @@ -3111,7 +3118,7 @@ if (__m1 > __set_algo_cut_off) { //we know proper offset due to [first1; left_bound_seq_1) < [first2; last2) - return __parallel_set_op( + return __internal::__parallel_set_op( std::forward<_ExecutionPolicy>(__exec), __left_bound_seq_1, __last1, __first2, __last2, __result, __comp, [](_DifferenceType __n, _DifferenceType __m) { return std::min(__n, __m); }, [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, @@ -3125,7 +3132,7 @@ if (__m2 > __set_algo_cut_off) { //we know proper offset due to [first2; left_bound_seq_2) < [first1; last1) - __result = __parallel_set_op( + __result = __internal::__parallel_set_op( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __left_bound_seq_2, __last2, __result, __comp, [](_DifferenceType __n, _DifferenceType __m) { return std::min(__n, __m); }, [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, @@ -3171,7 +3178,7 @@ _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_set_difference(__first1, __last1, __first2, __last2, __result, __comp, __is_vector); + return __internal::__brick_set_difference(__first1, __last1, __first2, __last2, __result, __comp, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -3194,37 +3201,37 @@ // {1} \ {}: parallel copying just first sequence if (__n2 == 0) - return __pattern_walk2_brick( + return __internal::__pattern_walk2_brick( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result, [__is_vector](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res) { - return __brick_copy(__begin, __end, __res, __is_vector); + return __internal::__brick_copy(__begin, __end, __res, __is_vector); }, std::true_type()); // testing whether the sequences are intersected - _ForwardIterator1 __left_bound_seq_1 = lower_bound(__first1, __last1, *__first2, __comp); + _ForwardIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp); //{1} < {2}: seq 2 is wholly greater than seq 1, so, parallel copying just first sequence if (__left_bound_seq_1 == __last1) - return __pattern_walk2_brick( + return __internal::__pattern_walk2_brick( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result, [__is_vector](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res) { - return __brick_copy(__begin, __end, __res, __is_vector); + return __internal::__brick_copy(__begin, __end, __res, __is_vector); }, std::true_type()); // testing whether the sequences are intersected - _ForwardIterator2 __left_bound_seq_2 = lower_bound(__first2, __last2, *__first1, __comp); + _ForwardIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp); //{2} < {1}: seq 1 is wholly greater than seq 2, so, parallel copying just first sequence if (__left_bound_seq_2 == __last2) - return __pattern_walk2_brick( + return __internal::__pattern_walk2_brick( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result, [__is_vector](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res) { - return __brick_copy(__begin, __end, __res, __is_vector); + return __internal::__brick_copy(__begin, __end, __res, __is_vector); }, std::true_type()); if (__n1 + __n2 > __set_algo_cut_off) - return __parallel_set_op( + return __internal::__parallel_set_op( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp, [](_DifferenceType __n, _DifferenceType __m) { return __n; }, [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, @@ -3267,7 +3274,8 @@ _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp, __is_vector); + return __internal::__brick_set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp, + __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -3287,7 +3295,7 @@ return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp); typedef typename std::iterator_traits<_OutputIterator>::value_type _T; - return __parallel_set_union_op( + return __internal::__parallel_set_union_op( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp, [](_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2, _T* __result, _Compare __comp) { @@ -3327,7 +3335,7 @@ __pattern_is_heap_until(_ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept { - return __brick_is_heap_until(__first, __last, __comp, __is_vector); + return __internal::__brick_is_heap_until(__first, __last, __comp, __is_vector); } template @@ -3365,11 +3373,11 @@ if (__last - __first < 2) return __last; - return __except_handler([&]() { + return __internal::__except_handler([&]() { return __parallel_find( std::forward<_ExecutionPolicy>(__exec), __first, __last, [__first, __comp, __is_vector](_RandomAccessIterator __i, _RandomAccessIterator __j) { - return __is_heap_until_local(__first, __i - __first, __j - __first, __comp, __is_vector); + return __internal::__is_heap_until_local(__first, __i - __first, __j - __first, __comp, __is_vector); }, std::less::difference_type>(), /*is_first=*/true); }); @@ -3405,7 +3413,7 @@ __pattern_min_element(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp, _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept { - return __brick_min_element(__first, __last, __comp, __is_vector); + return __internal::__brick_min_element(__first, __last, __comp, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -3417,16 +3425,17 @@ if (__first == __last) return __last; - return __except_handler([&]() { + return __internal::__except_handler([&]() { return __par_backend::__parallel_reduce( std::forward<_ExecutionPolicy>(__exec), __first + 1, __last, __first, [=](_RandomAccessIterator __begin, _RandomAccessIterator __end, _RandomAccessIterator __init) -> _RandomAccessIterator { - const _RandomAccessIterator subresult = __brick_min_element(__begin, __end, __comp, __is_vector); - return __cmp_iterators_by_values(__init, subresult, __comp); + const _RandomAccessIterator subresult = + __internal::__brick_min_element(__begin, __end, __comp, __is_vector); + return __internal::__cmp_iterators_by_values(__init, subresult, __comp); }, [=](_RandomAccessIterator __it1, _RandomAccessIterator __it2) -> _RandomAccessIterator { - return __cmp_iterators_by_values(__it1, __it2, __comp); + return __internal::__cmp_iterators_by_values(__it1, __it2, __comp); }); }); } @@ -3461,7 +3470,7 @@ __pattern_minmax_element(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp, _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept { - return __brick_minmax_element(__first, __last, __comp, __is_vector); + return __internal::__brick_minmax_element(__first, __last, __comp, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -3473,21 +3482,21 @@ if (__first == __last) return std::make_pair(__first, __first); - return __except_handler([&]() { + return __internal::__except_handler([&]() { typedef std::pair<_ForwardIterator, _ForwardIterator> _Result; return __par_backend::__parallel_reduce( std::forward<_ExecutionPolicy>(__exec), __first + 1, __last, std::make_pair(__first, __first), [=](_ForwardIterator __begin, _ForwardIterator __end, _Result __init) -> _Result { - const _Result __subresult = __brick_minmax_element(__begin, __end, __comp, __is_vector); - return std::make_pair( - __cmp_iterators_by_values(__subresult.first, __init.first, __comp), - __cmp_iterators_by_values(__init.second, __subresult.second, __not_pred<_Compare>(__comp))); + const _Result __subresult = __internal::__brick_minmax_element(__begin, __end, __comp, __is_vector); + return std::make_pair(__internal::__cmp_iterators_by_values(__subresult.first, __init.first, __comp), + __internal::__cmp_iterators_by_values(__init.second, __subresult.second, + __not_pred<_Compare>(__comp))); }, [=](_Result __p1, _Result __p2) -> _Result { return std::make_pair( - __cmp_iterators_by_values(__p1.first, __p2.first, __comp), - __cmp_iterators_by_values(__p2.second, __p1.second, __not_pred<_Compare>(__comp))); + __internal::__cmp_iterators_by_values(__p1.first, __p2.first, __comp), + __internal::__cmp_iterators_by_values(__p2.second, __p1.second, __not_pred<_Compare>(__comp))); }); }); } @@ -3534,7 +3543,7 @@ _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Predicate __pred, _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept { - return __brick_mismatch(__first1, __last1, __first2, __last2, __pred, __is_vector); + return __internal::__brick_mismatch(__first1, __last1, __first2, __last2, __pred, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -3545,13 +3554,13 @@ _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2, _Predicate __pred, _IsVector __is_vector, /* is_parallel = */ std::true_type) noexcept { - return __except_handler([&]() { + return __internal::__except_handler([&]() { auto __n = std::min(__last1 - __first1, __last2 - __first2); - auto __result = __parallel_find( + auto __result = __internal::__parallel_find( std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n, [__first1, __first2, __pred, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) { - return __brick_mismatch(__i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1), __pred, - __is_vector) + return __internal::__brick_mismatch(__i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1), + __pred, __is_vector) .first; }, std::less::difference_type>(), /*is_first=*/true); @@ -3615,7 +3624,7 @@ _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp, _IsVector __is_vector, /* is_parallel = */ std::false_type) noexcept { - return __brick_lexicographical_compare(__first1, __last1, __first2, __last2, __comp, __is_vector); + return __internal::__brick_lexicographical_compare(__first1, __last1, __first2, __last2, __comp, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -3640,15 +3649,14 @@ --__last1; --__last2; auto __n = std::min(__last1 - __first1, __last2 - __first2); - auto __result = __parallel_find( + auto __result = __internal::__parallel_find( std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n, [__first1, __first2, &__comp, __is_vector](_ForwardIterator1 __i, _ForwardIterator1 __j) { - return __brick_mismatch( - __i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1), - [&__comp](const _RefType1 __x, const _RefType2 __y) { - return !__comp(__x, __y) && !__comp(__y, __x); - }, - __is_vector) + return __internal::__brick_mismatch(__i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1), + [&__comp](const _RefType1 __x, const _RefType2 __y) { + return !__comp(__x, __y) && !__comp(__y, __x); + }, + __is_vector) .first; }, std::less::difference_type>(), /*is_first=*/true); Index: pstl/trunk/include/pstl/internal/execution_defs.h =================================================================== --- pstl/trunk/include/pstl/internal/execution_defs.h +++ pstl/trunk/include/pstl/internal/execution_defs.h @@ -123,27 +123,27 @@ }; template <> -struct is_execution_policy : std::true_type +struct is_execution_policy<__pstl::execution::sequenced_policy> : std::true_type { }; #if __PSTL_USE_PAR_POLICIES template <> -struct is_execution_policy : std::true_type +struct is_execution_policy<__pstl::execution::parallel_policy> : std::true_type { }; template <> -struct is_execution_policy : std::true_type +struct is_execution_policy<__pstl::execution::parallel_unsequenced_policy> : std::true_type { }; #endif template <> -struct is_execution_policy : std::true_type +struct is_execution_policy<__pstl::execution::unsequenced_policy> : std::true_type { }; #if __PSTL_CPP14_VARIABLE_TEMPLATES_PRESENT template -constexpr bool is_execution_policy_v = is_execution_policy::value; +constexpr bool is_execution_policy_v = __pstl::execution::is_execution_policy::value; #endif } // namespace v1 Index: pstl/trunk/include/pstl/internal/execution_impl.h =================================================================== --- pstl/trunk/include/pstl/internal/execution_impl.h +++ pstl/trunk/include/pstl/internal/execution_impl.h @@ -54,8 +54,8 @@ template struct __is_random_access_iterator { - static constexpr bool value = - __is_random_access_iterator<_IteratorType>::value && __is_random_access_iterator<_OtherIteratorTypes...>::value; + static constexpr bool value = __internal::__is_random_access_iterator<_IteratorType>::value && + __internal::__is_random_access_iterator<_OtherIteratorTypes...>::value; typedef std::integral_constant type; }; @@ -106,46 +106,54 @@ #endif template -using __collector_t = typename __policy_traits::type>::__collector_type; +using __collector_t = + typename __internal::__policy_traits::type>::__collector_type; template -using __allow_vector = typename __policy_traits::type>::__allow_vector; +using __allow_vector = + typename __internal::__policy_traits::type>::__allow_vector; template -using __allow_unsequenced = typename __policy_traits::type>::__allow_unsequenced; +using __allow_unsequenced = + typename __internal::__policy_traits::type>::__allow_unsequenced; template -using __allow_parallel = typename __policy_traits::type>::__allow_parallel; +using __allow_parallel = + typename __internal::__policy_traits::type>::__allow_parallel; template auto __is_vectorization_preferred(_ExecutionPolicy&& __exec) - -> decltype(__lazy_and(__exec.__allow_vector(), typename __is_random_access_iterator<_IteratorTypes...>::type())) + -> decltype(__internal::__lazy_and(__exec.__allow_vector(), + typename __internal::__is_random_access_iterator<_IteratorTypes...>::type())) { - return __lazy_and(__exec.__allow_vector(), typename __is_random_access_iterator<_IteratorTypes...>::type()); + return __internal::__lazy_and(__exec.__allow_vector(), + typename __internal::__is_random_access_iterator<_IteratorTypes...>::type()); } template auto __is_parallelization_preferred(_ExecutionPolicy&& __exec) - -> decltype(__lazy_and(__exec.__allow_parallel(), typename __is_random_access_iterator<_IteratorTypes...>::type())) + -> decltype(__internal::__lazy_and(__exec.__allow_parallel(), + typename __internal::__is_random_access_iterator<_IteratorTypes...>::type())) { - return __lazy_and(__exec.__allow_parallel(), typename __is_random_access_iterator<_IteratorTypes...>::type()); + return __internal::__lazy_and(__exec.__allow_parallel(), + typename __internal::__is_random_access_iterator<_IteratorTypes...>::type()); } template struct __prefer_unsequenced_tag { - static constexpr bool value = - __allow_unsequenced::value && __is_random_access_iterator<_IteratorTypes...>::value; + static constexpr bool value = __internal::__allow_unsequenced::value && + __internal::__is_random_access_iterator<_IteratorTypes...>::value; typedef std::integral_constant type; }; template struct __prefer_parallel_tag { - static constexpr bool value = - __allow_parallel::value && __is_random_access_iterator<_IteratorTypes...>::value; + static constexpr bool value = __internal::__allow_parallel::value && + __internal::__is_random_access_iterator<_IteratorTypes...>::value; typedef std::integral_constant type; }; Index: pstl/trunk/include/pstl/internal/glue_algorithm_impl.h =================================================================== --- pstl/trunk/include/pstl/internal/glue_algorithm_impl.h +++ pstl/trunk/include/pstl/internal/glue_algorithm_impl.h @@ -358,16 +358,15 @@ { using namespace __pstl; typedef typename iterator_traits<_ForwardIterator>::reference _ElementType; - __internal::__pattern_walk1( - std::forward<_ExecutionPolicy>(__exec), __first, __last, - [&__pred, &__new_value](_ElementType __elem) { - if (__pred(__elem)) - { - __elem = __new_value; - } - }, - __internal::__is_vectorization_preferred<_ExecutionPolicy, _ForwardIterator>(__exec), - __internal::__is_parallelization_preferred<_ExecutionPolicy, _ForwardIterator>(__exec)); + __internal::__pattern_walk1(std::forward<_ExecutionPolicy>(__exec), __first, __last, + [&__pred, &__new_value](_ElementType __elem) { + if (__pred(__elem)) + { + __elem = __new_value; + } + }, + __internal::__is_vectorization_preferred<_ExecutionPolicy, _ForwardIterator>(__exec), + __internal::__is_parallelization_preferred<_ExecutionPolicy, _ForwardIterator>(__exec)); } template Index: pstl/trunk/include/pstl/internal/glue_memory_impl.h =================================================================== --- pstl/trunk/include/pstl/internal/glue_memory_impl.h +++ pstl/trunk/include/pstl/internal/glue_memory_impl.h @@ -44,12 +44,11 @@ __is_parallel); }, [&]() { - return __internal::__pattern_walk2( - std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, - [](_ReferenceType1 __val1, _ReferenceType2 __val2) { - ::new (std::addressof(__val2)) _ValueType2(__val1); - }, - __is_vector, __is_parallel); + return __internal::__pattern_walk2(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, + [](_ReferenceType1 __val1, _ReferenceType2 __val2) { + ::new (std::addressof(__val2)) _ValueType2(__val1); + }, + __is_vector, __is_parallel); }); } @@ -79,12 +78,11 @@ __is_parallel); }, [&]() { - return __internal::__pattern_walk2_n( - std::forward<_ExecutionPolicy>(__exec), __first, __n, __result, - [](_ReferenceType1 __val1, _ReferenceType2 __val2) { - ::new (std::addressof(__val2)) _ValueType2(__val1); - }, - __is_vector, __is_parallel); + return __internal::__pattern_walk2_n(std::forward<_ExecutionPolicy>(__exec), __first, __n, __result, + [](_ReferenceType1 __val1, _ReferenceType2 __val2) { + ::new (std::addressof(__val2)) _ValueType2(__val1); + }, + __is_vector, __is_parallel); }); } @@ -116,12 +114,11 @@ __is_parallel); }, [&]() { - return __internal::__pattern_walk2( - std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, - [](_ReferenceType1 __val1, _ReferenceType2 __val2) { - ::new (std::addressof(__val2)) _ValueType2(std::move(__val1)); - }, - __is_vector, __is_parallel); + return __internal::__pattern_walk2(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, + [](_ReferenceType1 __val1, _ReferenceType2 __val2) { + ::new (std::addressof(__val2)) _ValueType2(std::move(__val1)); + }, + __is_vector, __is_parallel); }); } @@ -151,12 +148,11 @@ __is_parallel); }, [&]() { - return __internal::__pattern_walk2_n( - std::forward<_ExecutionPolicy>(__exec), __first, __n, __result, - [](_ReferenceType1 __val1, _ReferenceType2 __val2) { - ::new (std::addressof(__val2)) _ValueType2(std::move(__val1)); - }, - __is_vector, __is_parallel); + return __internal::__pattern_walk2_n(std::forward<_ExecutionPolicy>(__exec), __first, __n, __result, + [](_ReferenceType1 __val1, _ReferenceType2 __val2) { + ::new (std::addressof(__val2)) _ValueType2(std::move(__val1)); + }, + __is_vector, __is_parallel); }); } @@ -173,22 +169,23 @@ const auto __is_parallel = __internal::__is_parallelization_preferred<_ExecutionPolicy, _ForwardIterator>(__exec); const auto __is_vector = __internal::__is_vectorization_preferred<_ExecutionPolicy, _ForwardIterator>(__exec); - __internal::__invoke_if_else( - std::is_arithmetic<_ValueType>(), - [&]() { - __internal::__pattern_walk_brick( - std::forward<_ExecutionPolicy>(__exec), __first, __last, - [&__value, &__is_vector](_ForwardIterator __begin, _ForwardIterator __end) { - __internal::__brick_fill(__begin, __end, _ValueType(__value), __is_vector); - }, - __is_parallel); - }, - [&]() { - __internal::__pattern_walk1( - std::forward<_ExecutionPolicy>(__exec), __first, __last, - [&__value](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType(__value); }, __is_vector, - __is_parallel); - }); + __internal::__invoke_if_else(std::is_arithmetic<_ValueType>(), + [&]() { + __internal::__pattern_walk_brick( + std::forward<_ExecutionPolicy>(__exec), __first, __last, + [&__value, &__is_vector](_ForwardIterator __begin, _ForwardIterator __end) { + __internal::__brick_fill(__begin, __end, _ValueType(__value), __is_vector); + }, + __is_parallel); + }, + [&]() { + __internal::__pattern_walk1(std::forward<_ExecutionPolicy>(__exec), __first, + __last, + [&__value](_ReferenceType __val) { + ::new (std::addressof(__val)) _ValueType(__value); + }, + __is_vector, __is_parallel); + }); } template @@ -234,9 +231,8 @@ const auto __is_vector = __internal::__is_vectorization_preferred<_ExecutionPolicy, _ForwardIterator>(__exec); __internal::__invoke_if_not(std::is_trivially_destructible<_ValueType>(), [&]() { - __internal::__pattern_walk1( - std::forward<_ExecutionPolicy>(__exec), __first, __last, [](_ReferenceType __val) { __val.~_ValueType(); }, - __is_vector, __is_parallel); + __internal::__pattern_walk1(std::forward<_ExecutionPolicy>(__exec), __first, __last, + [](_ReferenceType __val) { __val.~_ValueType(); }, __is_vector, __is_parallel); }); } @@ -254,9 +250,9 @@ return __internal::__invoke_if_else( std::is_trivially_destructible<_ValueType>(), [&]() { return std::next(__first, __n); }, [&]() { - return __internal::__pattern_walk1_n( - std::forward<_ExecutionPolicy>(__exec), __first, __n, [](_ReferenceType __val) { __val.~_ValueType(); }, - __is_vector, __is_parallel); + return __internal::__pattern_walk1_n(std::forward<_ExecutionPolicy>(__exec), __first, __n, + [](_ReferenceType __val) { __val.~_ValueType(); }, __is_vector, + __is_parallel); }); } @@ -274,9 +270,9 @@ const auto __is_vector = __internal::__is_vectorization_preferred<_ExecutionPolicy, _ForwardIterator>(__exec); __internal::__invoke_if_not(std::is_trivial<_ValueType>(), [&]() { - __internal::__pattern_walk1( - std::forward<_ExecutionPolicy>(__exec), __first, __last, - [](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType; }, __is_vector, __is_parallel); + __internal::__pattern_walk1(std::forward<_ExecutionPolicy>(__exec), __first, __last, + [](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType; }, __is_vector, + __is_parallel); }); } @@ -291,13 +287,13 @@ const auto __is_parallel = __internal::__is_parallelization_preferred<_ExecutionPolicy, _ForwardIterator>(__exec); const auto __is_vector = __internal::__is_vectorization_preferred<_ExecutionPolicy, _ForwardIterator>(__exec); - return __internal::__invoke_if_else( - std::is_trivial<_ValueType>(), [&]() { return std::next(__first, __n); }, - [&]() { - return __internal::__pattern_walk1_n( - std::forward<_ExecutionPolicy>(__exec), __first, __n, - [](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType; }, __is_vector, __is_parallel); - }); + return __internal::__invoke_if_else(std::is_trivial<_ValueType>(), [&]() { return std::next(__first, __n); }, + [&]() { + return __internal::__pattern_walk1_n( + std::forward<_ExecutionPolicy>(__exec), __first, __n, + [](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType; }, + __is_vector, __is_parallel); + }); } // [uninitialized.construct.value] @@ -316,17 +312,16 @@ __internal::__invoke_if_else( std::is_trivial<_ValueType>(), [&]() { - __internal::__pattern_walk_brick( - std::forward<_ExecutionPolicy>(__exec), __first, __last, - [__is_vector](_ForwardIterator __begin, _ForwardIterator __end) { - __internal::__brick_fill(__begin, __end, _ValueType(), __is_vector); - }, - __is_parallel); + __internal::__pattern_walk_brick(std::forward<_ExecutionPolicy>(__exec), __first, __last, + [__is_vector](_ForwardIterator __begin, _ForwardIterator __end) { + __internal::__brick_fill(__begin, __end, _ValueType(), __is_vector); + }, + __is_parallel); }, [&]() { - __internal::__pattern_walk1( - std::forward<_ExecutionPolicy>(__exec), __first, __last, - [](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType(); }, __is_vector, __is_parallel); + __internal::__pattern_walk1(std::forward<_ExecutionPolicy>(__exec), __first, __last, + [](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType(); }, + __is_vector, __is_parallel); }); } @@ -344,12 +339,12 @@ return __internal::__invoke_if_else( std::is_trivial<_ValueType>(), [&]() { - return __internal::__pattern_walk_brick_n( - std::forward<_ExecutionPolicy>(__exec), __first, __n, - [__is_vector](_ForwardIterator __begin, _Size __count) { - return __internal::__brick_fill_n(__begin, __count, _ValueType(), __is_vector); - }, - __is_parallel); + return __internal::__pattern_walk_brick_n(std::forward<_ExecutionPolicy>(__exec), __first, __n, + [__is_vector](_ForwardIterator __begin, _Size __count) { + return __internal::__brick_fill_n(__begin, __count, + _ValueType(), __is_vector); + }, + __is_parallel); }, [&]() { return __internal::__pattern_walk1_n( Index: pstl/trunk/include/pstl/internal/numeric_impl.h =================================================================== --- pstl/trunk/include/pstl/internal/numeric_impl.h +++ pstl/trunk/include/pstl/internal/numeric_impl.h @@ -71,7 +71,7 @@ _RandomAccessIterator2 __first2, _Tp __init, _BinaryOperation1 __binary_op1, _BinaryOperation2 __binary_op2, _IsVector __is_vector, /*is_parallel=*/std::true_type) { - return __except_handler([&]() { + return __internal::__except_handler([&]() { return __par_backend::__parallel_transform_reduce( std::forward<_ExecutionPolicy>(__exec), __first1, __last1, [__first1, __first2, __binary_op2](_RandomAccessIterator1 __i) mutable { @@ -81,8 +81,8 @@ __binary_op1, // Combine [__first1, __first2, __binary_op1, __binary_op2, __is_vector](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j, _Tp __init) -> _Tp { - return __brick_transform_reduce(__i, __j, __first2 + (__i - __first1), __init, __binary_op1, - __binary_op2, __is_vector); + return __internal::__brick_transform_reduce(__i, __j, __first2 + (__i - __first1), __init, __binary_op1, + __binary_op2, __is_vector); }); }); } @@ -122,7 +122,7 @@ _BinaryOperation __binary_op, _UnaryOperation __unary_op, _IsVector __is_vector, /*is_parallel=*/std::false_type) noexcept { - return __brick_transform_reduce(__first, __last, __init, __binary_op, __unary_op, __is_vector); + return __internal::__brick_transform_reduce(__first, __last, __init, __binary_op, __unary_op, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -133,12 +133,12 @@ _BinaryOperation __binary_op, _UnaryOperation __unary_op, _IsVector __is_vector, /*is_parallel=*/std::true_type) { - return __except_handler([&]() { + return __internal::__except_handler([&]() { return __par_backend::__parallel_transform_reduce( std::forward<_ExecutionPolicy>(__exec), __first, __last, [__unary_op](_ForwardIterator __i) mutable { return __unary_op(*__i); }, __init, __binary_op, [__unary_op, __binary_op, __is_vector](_ForwardIterator __i, _ForwardIterator __j, _Tp __init) { - return __brick_transform_reduce(__i, __j, __init, __binary_op, __unary_op, __is_vector); + return __internal::__brick_transform_reduce(__i, __j, __init, __binary_op, __unary_op, __is_vector); }); }); } @@ -201,8 +201,8 @@ _Inclusive()); #else // We need to call serial brick here to call function for inclusive and exclusive scan that depends on _Inclusive() value - return __brick_transform_scan(__first, __last, __result, __unary_op, __init, __binary_op, _Inclusive(), - /*is_vector=*/std::false_type()); + return __internal::__brick_transform_scan(__first, __last, __result, __unary_op, __init, __binary_op, _Inclusive(), + /*is_vector=*/std::false_type()); #endif } @@ -213,8 +213,8 @@ _UnaryOperation __unary_op, _Tp __init, _BinaryOperation __binary_op, _Inclusive, /*is_vector=*/std::true_type) noexcept { - return __brick_transform_scan(__first, __last, __result, __unary_op, __init, __binary_op, _Inclusive(), - /*is_vector=*/std::false_type()); + return __internal::__brick_transform_scan(__first, __last, __result, __unary_op, __init, __binary_op, _Inclusive(), + /*is_vector=*/std::false_type()); } template ::difference_type _DifferenceType; - return __except_handler([&]() { + return __internal::__except_handler([&]() { __par_backend::__parallel_transform_scan( std::forward<_ExecutionPolicy>(__exec), __last - __first, [__first, __unary_op](_DifferenceType __i) mutable { return __unary_op(__first[__i]); }, __init, __binary_op, [__first, __unary_op, __binary_op](_DifferenceType __i, _DifferenceType __j, _Tp __init) { // Execute serial __brick_transform_reduce, due to the explicit SIMD vectorization (reduction) requires a commutative operation for the guarantee of correct scan. - return __brick_transform_reduce(__first + __i, __first + __j, __init, __binary_op, __unary_op, - /*__is_vector*/ std::false_type()); + return __internal::__brick_transform_reduce(__first + __i, __first + __j, __init, __binary_op, + __unary_op, + /*__is_vector*/ std::false_type()); }, [__first, __unary_op, __binary_op, __result, __is_vector](_DifferenceType __i, _DifferenceType __j, _Tp __init) { - return __brick_transform_scan(__first + __i, __first + __j, __result + __i, __unary_op, __init, - __binary_op, _Inclusive(), __is_vector) + return __internal::__brick_transform_scan(__first + __i, __first + __j, __result + __i, __unary_op, + __init, __binary_op, _Inclusive(), __is_vector) .second; }); return __result + (__last - __first); @@ -274,12 +276,12 @@ { return __result; } - return __except_handler([&]() { + return __internal::__except_handler([&]() { __par_backend::parallel_strict_scan( std::forward<_ExecutionPolicy>(__exec), __n, __init, [__first, __unary_op, __binary_op, __result, __is_vector](_DifferenceType __i, _DifferenceType __len) { - return __brick_transform_scan(__first + __i, __first + (__i + __len), __result + __i, __unary_op, _Tp{}, - __binary_op, _Inclusive(), __is_vector) + return __internal::__brick_transform_scan(__first + __i, __first + (__i + __len), __result + __i, + __unary_op, _Tp{}, __binary_op, _Inclusive(), __is_vector) .second; }, __binary_op, @@ -333,7 +335,7 @@ _OutputIterator __d_first, _BinaryOperation __op, _IsVector __is_vector, /*is_parallel*/ std::false_type) noexcept { - return __brick_adjacent_difference(__first, __last, __d_first, __op, __is_vector); + return __internal::__brick_adjacent_difference(__first, __last, __d_first, __op, __is_vector); } #if __PSTL_USE_PAR_POLICIES @@ -353,7 +355,7 @@ std::forward<_ExecutionPolicy>(__exec), __first, __last - 1, [&__op, __is_vector, __d_first, __first](_ForwardIterator1 __b, _ForwardIterator1 __e) { _ForwardIterator2 __d_b = __d_first + (__b - __first); - __brick_walk3( + __internal::__brick_walk3( __b, __e, __b + 1, __d_b + 1, [&__op](_ReferenceType1 __x, _ReferenceType1 __y, _ReferenceType2 __z) { __z = __op(__y, __x); }, __is_vector); Index: pstl/trunk/include/pstl/internal/parallel_backend_tbb.h =================================================================== --- pstl/trunk/include/pstl/internal/parallel_backend_tbb.h +++ pstl/trunk/include/pstl/internal/parallel_backend_tbb.h @@ -190,7 +190,7 @@ __parallel_transform_reduce(_ExecutionPolicy&&, _Index __first, _Index __last, _Up __u, _Tp __init, _Cp __combine, _Rp __brick_reduce) { - __par_trans_red_body<_Index, _Up, _Tp, _Cp, _Rp> __body(__u, __init, __combine, __brick_reduce); + __par_backend::__par_trans_red_body<_Index, _Up, _Tp, _Cp, _Rp> __body(__u, __init, __combine, __brick_reduce); // The grain size of 3 is used in order to provide mininum 2 elements for each body tbb::this_task_arena::isolate( [__first, __last, &__body]() { tbb::parallel_reduce(tbb::blocked_range<_Index>(__first, __last, 3), __body); }); @@ -304,8 +304,10 @@ { _Index __k = __split(__m); tbb::parallel_invoke( - [=] { __upsweep(__i, __k, __tilesize, __r, __tilesize, __reduce, __combine); }, - [=] { __upsweep(__i + __k, __m - __k, __tilesize, __r + __k, __lastsize, __reduce, __combine); }); + [=] { __par_backend::__upsweep(__i, __k, __tilesize, __r, __tilesize, __reduce, __combine); }, + [=] { + __par_backend::__upsweep(__i + __k, __m - __k, __tilesize, __r + __k, __lastsize, __reduce, __combine); + }); if (__m == 2 * __k) __r[__m - 1] = __combine(__r[__k - 1], __r[__m - 1]); } @@ -321,13 +323,14 @@ else { const _Index __k = __split(__m); - tbb::parallel_invoke([=] { __downsweep(__i, __k, __tilesize, __r, __tilesize, __initial, __combine, __scan); }, - // Assumes that __combine never throws. - //TODO: Consider adding a requirement for user functors to be constant. - [=, &__combine] { - __downsweep(__i + __k, __m - __k, __tilesize, __r + __k, __lastsize, - __combine(__initial, __r[__k - 1]), __combine, __scan); - }); + tbb::parallel_invoke( + [=] { __par_backend::__downsweep(__i, __k, __tilesize, __r, __tilesize, __initial, __combine, __scan); }, + // Assumes that __combine never throws. + //TODO: Consider adding a requirement for user functors to be constant. + [=, &__combine] { + __par_backend::__downsweep(__i + __k, __m - __k, __tilesize, __r + __k, __lastsize, + __combine(__initial, __r[__k - 1]), __combine, __scan); + }); } } @@ -358,7 +361,8 @@ _Index __m = (__n - 1) / __tilesize; __buffer<_Tp> __buf(__m + 1); _Tp* __r = __buf.get(); - __upsweep(_Index(0), _Index(__m + 1), __tilesize, __r, __n - __m * __tilesize, __reduce, __combine); + __par_backend::__upsweep(_Index(0), _Index(__m + 1), __tilesize, __r, __n - __m * __tilesize, __reduce, + __combine); // When __apex is a no-op and __combine has no side effects, a good optimizer // should be able to eliminate all code between here and __apex. @@ -369,8 +373,8 @@ while ((__k &= __k - 1)) __t = __combine(__r[__k - 1], __t); __apex(__combine(__initial, __t)); - __downsweep(_Index(0), _Index(__m + 1), __tilesize, __r, __n - __m * __tilesize, __initial, __combine, - __scan); + __par_backend::__downsweep(_Index(0), _Index(__m + 1), __tilesize, __r, __n - __m * __tilesize, __initial, + __combine, __scan); return; } // Fewer than 2 elements in sequence, or out of memory. Handle has single block. @@ -523,7 +527,7 @@ { _M_leaf_sort(_M_xs, _M_xe, _M_comp); if (_M_inplace != 2) - __init_buf(_M_xs, _M_xe, _M_zs, _M_inplace == 0); + __par_backend::__init_buf(_M_xs, _M_xe, _M_zs, _M_inplace == 0); return NULL; } else @@ -536,20 +540,21 @@ auto __move_sequences = [](_RandomAccessIterator2 __first1, _RandomAccessIterator2 __last1, _RandomAccessIterator1 __first2) { return std::move(__first1, __last1, __first2); }; if (_M_inplace == 2) - __m = new (allocate_continuation()) + __m = new (tbb::task::allocate_continuation()) __merge_task<_RandomAccessIterator2, _RandomAccessIterator2, _RandomAccessIterator1, _Compare, __serial_destroy, - __serial_move_merge>( + __par_backend::__serial_move_merge>( _M_zs, __zm, __zm, __ze, _M_xs, _M_comp, __serial_destroy(), - __serial_move_merge(__nmerge, __move_values, - __move_sequences)); + __par_backend::__serial_move_merge( + __nmerge, __move_values, __move_sequences)); else if (_M_inplace) - __m = new (allocate_continuation()) + __m = new (tbb::task::allocate_continuation()) __merge_task<_RandomAccessIterator2, _RandomAccessIterator2, _RandomAccessIterator1, _Compare, - __binary_no_op, __serial_move_merge>( - _M_zs, __zm, __zm, __ze, _M_xs, _M_comp, __binary_no_op(), - __serial_move_merge(__nmerge, __move_values, - __move_sequences)); + __par_backend::__binary_no_op, + __par_backend::__serial_move_merge>( + _M_zs, __zm, __zm, __ze, _M_xs, _M_comp, __par_backend::__binary_no_op(), + __par_backend::__serial_move_merge( + __nmerge, __move_values, __move_sequences)); else { auto __move_values = [](_RandomAccessIterator1 __x, _RandomAccessIterator2 __z) { *__z = std::move(*__x); }; @@ -557,18 +562,19 @@ _RandomAccessIterator2 __first2) { return std::move(__first1, __last1, __first2); }; - __m = new (allocate_continuation()) + __m = new (tbb::task::allocate_continuation()) __merge_task<_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2, _Compare, - __binary_no_op, __serial_move_merge>( - _M_xs, __xm, __xm, _M_xe, _M_zs, _M_comp, __binary_no_op(), - __serial_move_merge(__nmerge, __move_values, - __move_sequences)); + __par_backend::__binary_no_op, + __par_backend::__serial_move_merge>( + _M_xs, __xm, __xm, _M_xe, _M_zs, _M_comp, __par_backend::__binary_no_op(), + __par_backend::__serial_move_merge( + __nmerge, __move_values, __move_sequences)); } __m->set_ref_count(2); task* __right = new (__m->allocate_child()) __stable_sort_task(__xm, _M_xe, __zm, !_M_inplace, _M_comp, _M_leaf_sort, __nmerge); - spawn(*__right); - recycle_as_child_of(*__m); + tbb::task::spawn(*__right); + tbb::task::recycle_as_child_of(*__m); _M_xe = __xm; _M_inplace = !_M_inplace; } @@ -629,10 +635,10 @@ { tbb::this_task_arena::isolate([=]() { typedef __merge_task<_RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator3, _Compare, - __binary_no_op, _LeafMerge> + __par_backend::__binary_no_op, _LeafMerge> _TaskType; tbb::task::spawn_root_and_wait(*new (tbb::task::allocate_root()) _TaskType( - __xs, __xe, __ys, __ye, __zs, __comp, __binary_no_op(), __leaf_merge)); + __xs, __xe, __ys, __ye, __zs, __comp, __par_backend::__binary_no_op(), __leaf_merge)); }); } } Index: pstl/trunk/include/pstl/internal/unseq_backend_simd.h =================================================================== --- pstl/trunk/include/pstl/internal/unseq_backend_simd.h +++ pstl/trunk/include/pstl/internal/unseq_backend_simd.h @@ -796,8 +796,9 @@ { for (; __first != __last; ++__first) { - if (__simd_or(__s_first, __n2, - __internal::__equal_value_by_pred(*__first, __pred))) + if (__unseq_backend::__simd_or( + __s_first, __n2, + __internal::__equal_value_by_pred(*__first, __pred))) { return __first; } @@ -807,10 +808,10 @@ { for (; __s_first != __s_last; ++__s_first) { - const auto __result = __simd_first(__first, _DifferencType(0), __n1, - [__s_first, &__pred](_ForwardIterator1 __it, _DifferencType __i) { - return __pred(__it[__i], *__s_first); - }); + const auto __result = __unseq_backend::__simd_first( + __first, _DifferencType(0), __n1, [__s_first, &__pred](_ForwardIterator1 __it, _DifferencType __i) { + return __pred(__it[__i], *__s_first); + }); if (__result != __last) { return __result; @@ -825,9 +826,9 @@ __simd_remove_if(_RandomAccessIterator __first, _DifferenceType __n, _UnaryPredicate __pred) noexcept { // find first element we need to remove - auto __current = - __simd_first(__first, _DifferenceType(0), __n, - [&__pred](_RandomAccessIterator __it, _DifferenceType __i) { return __pred(__it[__i]); }); + auto __current = __unseq_backend::__simd_first( + __first, _DifferenceType(0), __n, + [&__pred](_RandomAccessIterator __it, _DifferenceType __i) { return __pred(__it[__i]); }); __n -= __current - __first; // if we have in sequence only one element that pred(__current[1]) != false we can exit the function