diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -59,6 +59,7 @@ __algorithm/pop_heap.h __algorithm/prev_permutation.h __algorithm/push_heap.h + __algorithm/radix_sort.h __algorithm/remove_copy_if.h __algorithm/remove_copy.h __algorithm/remove_if.h diff --git a/libcxx/include/__algorithm/radix_sort.h b/libcxx/include/__algorithm/radix_sort.h new file mode 100644 --- /dev/null +++ b/libcxx/include/__algorithm/radix_sort.h @@ -0,0 +1,279 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_RADIX_SORT_H +#define _LIBCPP___ALGORITHM_RADIX_SORT_H + +#include <__algorithm/copy.h> +#include <__algorithm/for_each.h> +#include <__config> +#include <__iterator/iterator_traits.h> +#include <__iterator/move_iterator.h> +#include <__utility/forward.h> +#include +#include +#include + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if _LIBCPP_STD_VER > 14 + +inline void __variadic_expansion_dummy(initializer_list) {} + +#define EXPAND_VARIADIC(expression) __variadic_expansion_dummy({(expression, 0)...}) + +template +constexpr auto __move_assign_please(_Iterator i) + -> enable_if_t::value_type>::value, + move_iterator<_Iterator> > { + return make_move_iterator(move(i)); +} + +template +constexpr auto __move_assign_please(_Iterator i) + -> enable_if_t::value_type>::value, _Iterator> { + return i; +} + +template +constexpr _Integer __intlog2_impl(_Integer integer) { + auto degree = _Integer{0}; + + while ((integer >>= 1) > 0) { + ++degree; + } + + return degree; +} + +template +constexpr _Integer __intlog2(_Integer integer) { + static_assert(is_integral<_Integer>::value, "Must be an integral type"); + + return integer > 0 ? __intlog2_impl(integer) + : throw domain_error("The binary logarithm is not defined on non-positive numbers"); +} + +template +pair<_OutputIterator, typename iterator_traits<_InputIterator>::value_type> +__partial_sum_max(_InputIterator first, _InputIterator last, _OutputIterator result) { + if (first == last) + return {result, 0}; + + auto max = *first; + typename iterator_traits<_InputIterator>::value_type sum = *first; + *result = sum; + + while (++first != last) { + if (max < *first) { + max = *first; + } + sum = move(sum) + *first; + *++result = sum; + } + return {++result, max}; +} + +template +struct __radix_sort_traits { + using image_type = decay_t >; + static_assert(is_integral::value, ""); + static_assert(is_unsigned::value, ""); + + using radix_type = decay_t >; + static_assert(is_integral::value, ""); + + constexpr static auto radix_value_range = numeric_limits::max() + 1; + constexpr static auto radix_size = __intlog2(radix_value_range); + constexpr static auto radix_count = sizeof(image_type) * CHAR_BIT / radix_size; +}; + +template +struct __counting_sort_traits { + using image_type = decay_t >; + static_assert(is_integral::value, ""); + static_assert(is_unsigned::value, ""); + + constexpr static const auto value_range = numeric_limits::max() + 1; + constexpr static auto radix_size = __intlog2(value_range); +}; + +template +auto __nth_radix(size_t radix_number, _Radix radix) { + return [radix_number, radix = move(radix)](auto n) { + using value_type = decltype(n); + static_assert(is_integral::value, ""); + static_assert(is_unsigned::value, ""); + using traits = __counting_sort_traits; + + return radix(static_cast(n >> traits::radix_size * radix_number)); + }; +} + +template +bool __collect_impl(_ForwardIterator first, _ForwardIterator last, _Map map, _Radix radix, + _RandomAccessIterator1 counters, _RandomAccessIterator2 maximums, index_sequence<_Radices...>) { + using value_type = typename iterator_traits<_ForwardIterator>::value_type; + constexpr auto radix_value_range = __radix_sort_traits::radix_value_range; + + auto previous = numeric_limits >::min(); + auto is_sorted = true; + for_each(first, last, [&counters, &map, &radix, &previous, &is_sorted](const auto& value) { + auto current = map(value); + is_sorted &= (current >= previous); + previous = current; + + EXPAND_VARIADIC(++counters[_Radices][__nth_radix(_Radices, radix)(current)]); + }); + + EXPAND_VARIADIC( + maximums[_Radices] = + __partial_sum_max(counters[_Radices], counters[_Radices] + radix_value_range, counters[_Radices]).second); + + return is_sorted; +} + +template +bool __collect(_ForwardIterator first, _ForwardIterator last, _Map map, _Radix radix, _RandomAccessIterator1 counters, + _RandomAccessIterator2 maximums) { + using value_type = typename iterator_traits<_ForwardIterator>::value_type; + constexpr auto radix_count = __radix_sort_traits::radix_count; + return __collect_impl(first, last, map, radix, counters, maximums, make_index_sequence()); +} + +template +void __dispose_backward(BidirectionalIterator first, BidirectionalIterator last, _RandomAccessIterator1 result, + _Map map, _RandomAccessIterator2 counters) { + for_each(make_reverse_iterator(last), make_reverse_iterator(first), [&result, &counters, &map](auto&& preimage) { + auto index = --counters[map(preimage)]; + result[index] = forward(preimage); + }); +} + +template +typename enable_if< + __radix_sort_traits::value_type, _Map, _Radix>::radix_count % 2 == + 0, + void>::type +__radix_sort_impl(_RandomAccessIterator1 first, _RandomAccessIterator1 last, _RandomAccessIterator2 buffer_begin, + _Map map, _Radix radix) { + using value_type = typename iterator_traits<_RandomAccessIterator1>::value_type; + using traits = __radix_sort_traits; + + using difference_type = typename iterator_traits<_RandomAccessIterator1>::difference_type; + difference_type counters[traits::radix_count][traits::radix_value_range] = {{0}}; + difference_type maximums[traits::radix_count] = {0}; + const auto is_sorted = __collect(first, last, map, radix, counters, maximums); + if (not is_sorted) { + const auto range_size = distance(first, last); + auto buffer_end = buffer_begin + range_size; + for (size_t radix_number = 0; radix_number < traits::radix_count; radix_number += 2) { + const auto n0th_is_single = maximums[radix_number] == range_size; + const auto n1th_is_single = maximums[radix_number + 1] == range_size; + + if (n0th_is_single && n1th_is_single) { + continue; + } + + if (n0th_is_single) { + copy(__move_assign_please(first), __move_assign_please(last), buffer_begin); + } else { + auto n0th = [radix_number, &map, &radix](const auto& v) { return __nth_radix(radix_number, radix)(map(v)); }; + __dispose_backward(__move_assign_please(first), __move_assign_please(last), buffer_begin, n0th, + counters[radix_number]); + } + + if (n1th_is_single) { + copy(__move_assign_please(buffer_begin), __move_assign_please(buffer_end), first); + } else { + auto n1th = [radix_number, &map, &radix](const auto& v) { + return __nth_radix(radix_number + 1, radix)(map(v)); + }; + __dispose_backward(__move_assign_please(buffer_begin), __move_assign_please(buffer_end), first, n1th, + counters[radix_number + 1]); + } + } + } +} + +constexpr auto __to_unsigned(bool b) { return b; } + +template +constexpr auto __to_unsigned(_Ip n) { + constexpr const auto min_value = numeric_limits<_Ip>::min(); + return static_cast >(n ^ min_value); +} + +struct __identity_fn { + template + constexpr decltype(auto) operator()(_Tp&& value) const { + return forward<_Tp>(value); + } +}; + +struct __low_byte_fn { + template + constexpr uint8_t operator()(_Ip integer) const { + static_assert(is_integral<_Ip>::value, ""); + static_assert(is_unsigned<_Ip>::value, ""); + + return static_cast(integer & 0xff); + } +}; + +template +void __radix_sort(_RandomAccessIterator1 first, _RandomAccessIterator1 last, _RandomAccessIterator2 buffer, _Map map, + _Radix radix) { + auto map_to_unsigned = [map = move(map)](const auto& x) { return __to_unsigned(map(x)); }; + __radix_sort_impl(first, last, buffer, map_to_unsigned, radix); +} + +template +void __radix_sort(_RandomAccessIterator1 first, _RandomAccessIterator1 last, _RandomAccessIterator2 buffer) { + __radix_sort(first, last, buffer, __identity_fn{}, __low_byte_fn{}); +} + +template +bool __radix_sort(_RandomAccessIterator1 first, _RandomAccessIterator1 last, _RandomAccessIterator2 buffer, + _BoolConstant) { + __radix_sort(first, last, buffer, __identity_fn{}, __low_byte_fn{}); + return true; +} + +template +bool __radix_sort(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2, _BoolConstant) { + return false; +} + +#undef EXPAND_VARIADIC + +#else // _LIBCPP_STD_VER > 14 + +template +bool __radix_sort(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2, _BoolConstant<_B>) { + return false; +} + +#endif // _LIBCPP_STD_VER > 14 + +_LIBCPP_END_NAMESPACE_STD + +_LIBCPP_POP_MACROS + +#endif // _LIBCPP___ALGORITHM_RADIX_SORT_H diff --git a/libcxx/include/__algorithm/stable_sort.h b/libcxx/include/__algorithm/stable_sort.h --- a/libcxx/include/__algorithm/stable_sort.h +++ b/libcxx/include/__algorithm/stable_sort.h @@ -13,6 +13,7 @@ #include <__algorithm/comp.h> #include <__algorithm/comp_ref_type.h> #include <__algorithm/inplace_merge.h> +#include <__algorithm/radix_sort.h> #include <__algorithm/sort.h> #include <__iterator/iterator_traits.h> #include <__utility/swap.h> @@ -94,56 +95,50 @@ *__result = _VSTD::move(*__first2); } -template -void -__stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, - typename iterator_traits<_RandomAccessIterator>::difference_type __len, - typename iterator_traits<_RandomAccessIterator>::value_type* __buff, ptrdiff_t __buff_size); - -template -void -__stable_sort_move(_RandomAccessIterator __first1, _RandomAccessIterator __last1, _Compare __comp, +template +void __stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, typename iterator_traits<_RandomAccessIterator>::difference_type __len, - typename iterator_traits<_RandomAccessIterator>::value_type* __first2) -{ - typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; - switch (__len) - { - case 0: - return; - case 1: - ::new ((void*)__first2) value_type(_VSTD::move(*__first1)); - return; - case 2: - __destruct_n __d(0); - unique_ptr __h2(__first2, __d); - if (__comp(*--__last1, *__first1)) - { - ::new ((void*)__first2) value_type(_VSTD::move(*__last1)); - __d.template __incr(); - ++__first2; - ::new ((void*)__first2) value_type(_VSTD::move(*__first1)); - } - else - { - ::new ((void*)__first2) value_type(_VSTD::move(*__first1)); - __d.template __incr(); - ++__first2; - ::new ((void*)__first2) value_type(_VSTD::move(*__last1)); - } - __h2.release(); - return; - } - if (__len <= 8) - { - _VSTD::__insertion_sort_move<_Compare>(__first1, __last1, __first2, __comp); - return; + typename iterator_traits<_RandomAccessIterator>::value_type* __buff, ptrdiff_t __buff_size, + _BoolConstant<__EnableRadixSort>); + +template +void __stable_sort_move(_RandomAccessIterator __first1, _RandomAccessIterator __last1, _Compare __comp, + typename iterator_traits<_RandomAccessIterator>::difference_type __len, + typename iterator_traits<_RandomAccessIterator>::value_type* __first2, + _BoolConstant<__EnableRadixSort> __rs) { + typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; + switch (__len) { + case 0: + return; + case 1: + ::new ((void*)__first2) value_type(_VSTD::move(*__first1)); + return; + case 2: + __destruct_n __d(0); + unique_ptr __h2(__first2, __d); + if (__comp(*--__last1, *__first1)) { + ::new ((void*)__first2) value_type(_VSTD::move(*__last1)); + __d.template __incr(); + ++__first2; + ::new ((void*)__first2) value_type(_VSTD::move(*__first1)); + } else { + ::new ((void*)__first2) value_type(_VSTD::move(*__first1)); + __d.template __incr(); + ++__first2; + ::new ((void*)__first2) value_type(_VSTD::move(*__last1)); } - typename iterator_traits<_RandomAccessIterator>::difference_type __l2 = __len / 2; - _RandomAccessIterator __m = __first1 + __l2; - _VSTD::__stable_sort<_Compare>(__first1, __m, __comp, __l2, __first2, __l2); - _VSTD::__stable_sort<_Compare>(__m, __last1, __comp, __len - __l2, __first2 + __l2, __len - __l2); - _VSTD::__merge_move_construct<_Compare>(__first1, __m, __m, __last1, __first2, __comp); + __h2.release(); + return; + } + if (__len <= 8) { + _VSTD::__insertion_sort_move<_Compare>(__first1, __last1, __first2, __comp); + return; + } + typename iterator_traits<_RandomAccessIterator>::difference_type __l2 = __len / 2; + _RandomAccessIterator __m = __first1 + __l2; + _VSTD::__stable_sort<_Compare>(__first1, __m, __comp, __l2, __first2, __l2, __rs); + _VSTD::__stable_sort<_Compare>(__m, __last1, __comp, __len - __l2, __first2 + __l2, __len - __l2, __rs); + _VSTD::__merge_move_construct<_Compare>(__first1, __m, __m, __last1, __first2, __comp); } template @@ -152,69 +147,84 @@ static const unsigned value = 128*is_trivially_copy_assignable<_Tp>::value; }; -template -void -__stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, - typename iterator_traits<_RandomAccessIterator>::difference_type __len, - typename iterator_traits<_RandomAccessIterator>::value_type* __buff, ptrdiff_t __buff_size) -{ - typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; - typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; - switch (__len) - { - case 0: - case 1: - return; - case 2: - if (__comp(*--__last, *__first)) - swap(*__first, *__last); - return; - } - if (__len <= static_cast(__stable_sort_switch::value)) - { - _VSTD::__insertion_sort<_Compare>(__first, __last, __comp); - return; - } - typename iterator_traits<_RandomAccessIterator>::difference_type __l2 = __len / 2; - _RandomAccessIterator __m = __first + __l2; - if (__len <= __buff_size) - { - __destruct_n __d(0); - unique_ptr __h2(__buff, __d); - _VSTD::__stable_sort_move<_Compare>(__first, __m, __comp, __l2, __buff); - __d.__set(__l2, (value_type*)nullptr); - _VSTD::__stable_sort_move<_Compare>(__m, __last, __comp, __len - __l2, __buff + __l2); - __d.__set(__len, (value_type*)nullptr); - _VSTD::__merge_move_assign<_Compare>(__buff, __buff + __l2, __buff + __l2, __buff + __len, __first, __comp); -// _VSTD::__merge<_Compare>(move_iterator(__buff), -// move_iterator(__buff + __l2), -// move_iterator<_RandomAccessIterator>(__buff + __l2), -// move_iterator<_RandomAccessIterator>(__buff + __len), -// __first, __comp); - return; +template +struct __radix_sort_min_switch { + static const unsigned value = (1 << 6) * sizeof(_Tp); +}; + +template +struct __radix_sort_max_switch { + static const unsigned value = (1 << 15) * sizeof(_Tp); +}; + +template +void __stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, + typename iterator_traits<_RandomAccessIterator>::difference_type __len, + typename iterator_traits<_RandomAccessIterator>::value_type* __buff, ptrdiff_t __buff_size, + _BoolConstant<__EnableRadixSort> __rs) { + typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; + typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; + switch (__len) { + case 0: + case 1: + return; + case 2: + if (__comp(*--__last, *__first)) + swap(*__first, *__last); + return; + } + if (__len <= static_cast(__stable_sort_switch::value)) { + _VSTD::__insertion_sort<_Compare>(__first, __last, __comp); + return; + } + if (__len <= __buff_size && __len >= __radix_sort_min_switch::value && + __len <= __radix_sort_max_switch::value) { + if (_VSTD::__radix_sort(__first, __last, __buff, __rs)) { + return; } - _VSTD::__stable_sort<_Compare>(__first, __m, __comp, __l2, __buff, __buff_size); - _VSTD::__stable_sort<_Compare>(__m, __last, __comp, __len - __l2, __buff, __buff_size); - _VSTD::__inplace_merge<_Compare>(__first, __m, __last, __comp, __l2, __len - __l2, __buff, __buff_size); + } + typename iterator_traits<_RandomAccessIterator>::difference_type __l2 = __len / 2; + _RandomAccessIterator __m = __first + __l2; + if (__len <= __buff_size) { + __destruct_n __d(0); + unique_ptr __h2(__buff, __d); + _VSTD::__stable_sort_move<_Compare>(__first, __m, __comp, __l2, __buff, __rs); + __d.__set(__l2, (value_type*)nullptr); + _VSTD::__stable_sort_move<_Compare>(__m, __last, __comp, __len - __l2, __buff + __l2, __rs); + __d.__set(__len, (value_type*)nullptr); + _VSTD::__merge_move_assign<_Compare>(__buff, __buff + __l2, __buff + __l2, __buff + __len, __first, __comp); + // _VSTD::__merge<_Compare>(move_iterator(__buff), + // move_iterator(__buff + __l2), + // move_iterator<_RandomAccessIterator>(__buff + __l2), + // move_iterator<_RandomAccessIterator>(__buff + __len), + // __first, __comp); + return; + } + _VSTD::__stable_sort<_Compare>(__first, __m, __comp, __l2, __buff, __buff_size, __rs); + _VSTD::__stable_sort<_Compare>(__m, __last, __comp, __len - __l2, __buff, __buff_size, __rs); + _VSTD::__inplace_merge<_Compare>(__first, __m, __last, __comp, __l2, __len - __l2, __buff, __buff_size); +} + +template +inline _LIBCPP_INLINE_VISIBILITY void __stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, + _Compare __comp, _BoolConstant<__EnableRadixSort> __rs) { + typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; + typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; + difference_type __len = __last - __first; + pair __buf(0, 0); + unique_ptr __h; + if (__len > static_cast(__stable_sort_switch::value)) { + __buf = _VSTD::get_temporary_buffer(__len); + __h.reset(__buf.first); + } + typedef typename __comp_ref_type<_Compare>::type _Comp_ref; + _VSTD::__stable_sort<_Comp_ref>(__first, __last, __comp, __len, __buf.first, __buf.second, __rs); } template -inline _LIBCPP_INLINE_VISIBILITY -void -stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) -{ - typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; - typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; - difference_type __len = __last - __first; - pair __buf(0, 0); - unique_ptr __h; - if (__len > static_cast(__stable_sort_switch::value)) - { - __buf = _VSTD::get_temporary_buffer(__len); - __h.reset(__buf.first); - } - typedef typename __comp_ref_type<_Compare>::type _Comp_ref; - _VSTD::__stable_sort<_Comp_ref>(__first, __last, __comp, __len, __buf.first, __buf.second); +inline _LIBCPP_INLINE_VISIBILITY void stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, + _Compare __comp) { + _VSTD::__stable_sort(__first, __last, __comp, _BoolConstant()); } template @@ -222,7 +232,9 @@ void stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last) { - _VSTD::stable_sort(__first, __last, __less::value_type>()); + typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; + _VSTD::__stable_sort(__first, __last, __less::value_type>(), + _BoolConstant::value>()); } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/test/libcxx/diagnostics/detail.headers/algorithm/radix_sort.module.verify.cpp b/libcxx/test/libcxx/diagnostics/detail.headers/algorithm/radix_sort.module.verify.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/libcxx/diagnostics/detail.headers/algorithm/radix_sort.module.verify.cpp @@ -0,0 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// REQUIRES: modules-build + +// WARNING: This test was generated by 'generate_private_header_tests.py' +// and should not be edited manually. + +// expected-error@*:* {{use of private header from outside its module: '__algorithm/radix_sort.h'}} +#include <__algorithm/radix_sort.h>