diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -9,6 +9,7 @@ __algorithm/copy.h __algorithm/copy_backward.h __algorithm/copy_if.h + __algorithm/copy_move_common.h __algorithm/copy_n.h __algorithm/count.h __algorithm/count_if.h @@ -587,6 +588,7 @@ __type_traits/is_abstract.h __type_traits/is_aggregate.h __type_traits/is_allocator.h + __type_traits/is_always_bitcastable.h __type_traits/is_arithmetic.h __type_traits/is_array.h __type_traits/is_assignable.h diff --git a/libcxx/include/__algorithm/copy.h b/libcxx/include/__algorithm/copy.h --- a/libcxx/include/__algorithm/copy.h +++ b/libcxx/include/__algorithm/copy.h @@ -9,20 +9,11 @@ #ifndef _LIBCPP___ALGORITHM_COPY_H #define _LIBCPP___ALGORITHM_COPY_H -#include <__algorithm/unwrap_iter.h> -#include <__algorithm/unwrap_range.h> +#include <__algorithm/copy_move_common.h> +#include <__algorithm/iterator_operations.h> #include <__config> -#include <__iterator/iterator_traits.h> -#include <__iterator/reverse_iterator.h> -#include <__type_traits/enable_if.h> -#include <__type_traits/is_copy_constructible.h> -#include <__type_traits/is_same.h> -#include <__type_traits/is_trivially_copy_assignable.h> -#include <__type_traits/is_trivially_copyable.h> -#include <__type_traits/remove_const.h> #include <__utility/move.h> #include <__utility/pair.h> -#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header @@ -30,82 +21,43 @@ _LIBCPP_BEGIN_NAMESPACE_STD -// copy +struct __copy_loop { + template + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter> + operator()(_InIter __first, _Sent __last, _OutIter __result) const { + while (__first != __last) { + *__result = *__first; + ++__first; + ++__result; + } -template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_InIter, _OutIter> __copy_impl(_InIter __first, _Sent __last, _OutIter __result) { - while (__first != __last) { - *__result = *__first; - ++__first; - ++__result; + return std::make_pair(std::move(__first), std::move(__result)); } - return pair<_InIter, _OutIter>(std::move(__first), std::move(__result)); -} +}; -template , _OutValueT>::value - && is_trivially_copy_assignable<_OutValueT>::value> > -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_InValueT*, _OutValueT*> __copy_impl(_InValueT* __first, _InValueT* __last, _OutValueT* __result) { - if (__libcpp_is_constant_evaluated() -// TODO: Remove this once GCC supports __builtin_memmove during constant evaluation -#ifndef _LIBCPP_COMPILER_GCC - && !is_trivially_copyable<_InValueT>::value -#endif - ) - return std::__copy_impl<_InValueT*, _InValueT*, _OutValueT*>(__first, __last, __result); - const size_t __n = static_cast(__last - __first); - if (__n > 0) - ::__builtin_memmove(__result, __first, __n * sizeof(_OutValueT)); - return std::make_pair(__first + __n, __result + __n); -} +struct __copy_trivial { + // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer. + template ::value, int> = 0> + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*> + operator()(_In* __first, _In* __last, _Out* __result) const { + return std::__copy_trivial_impl(__first, __last, __result); + } +}; -template >, __iter_value_type<_OutIter> >::value - && __is_cpp17_contiguous_iterator::value - && __is_cpp17_contiguous_iterator::value - && is_trivially_copy_assignable<__iter_value_type<_OutIter> >::value - && __is_reverse_iterator<_InIter>::value - && __is_reverse_iterator<_OutIter>::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 +template pair<_InIter, _OutIter> -__copy_impl(_InIter __first, _InIter __last, _OutIter __result) { - auto __first_base = std::__unwrap_iter(__first.base()); - auto __last_base = std::__unwrap_iter(__last.base()); - auto __result_base = std::__unwrap_iter(__result.base()); - auto __result_first = __result_base - (__first_base - __last_base); - std::__copy_impl(__last_base, __first_base, __result_first); - return std::make_pair(__last, _OutIter(std::__rewrap_iter(__result.base(), __result_first))); -} - -template ::value - && is_copy_constructible<_Sent>::value - && is_copy_constructible<_OutIter>::value), int> = 0 > -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_InIter, _OutIter> __copy(_InIter __first, _Sent __last, _OutIter __result) { - return std::__copy_impl(std::move(__first), std::move(__last), std::move(__result)); -} - -template ::value - && is_copy_constructible<_Sent>::value - && is_copy_constructible<_OutIter>::value, int> = 0> inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_InIter, _OutIter> __copy(_InIter __first, _Sent __last, _OutIter __result) { - auto __range = std::__unwrap_range(__first, __last); - auto __ret = std::__copy_impl(std::move(__range.first), std::move(__range.second), std::__unwrap_iter(__result)); - return std::make_pair( - std::__rewrap_range<_Sent>(__first, __ret.first), std::__rewrap_iter(__result, __ret.second)); +__copy(_InIter __first, _Sent __last, _OutIter __result) { + return std::__dispatch_copy_or_move<_AlgPolicy, __copy_loop, __copy_trivial>( + std::move(__first), std::move(__last), std::move(__result)); } template inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result) { - return std::__copy(__first, __last, __result).second; + return std::__copy<_ClassicAlgPolicy>(__first, __last, __result).second; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/copy_backward.h b/libcxx/include/__algorithm/copy_backward.h --- a/libcxx/include/__algorithm/copy_backward.h +++ b/libcxx/include/__algorithm/copy_backward.h @@ -9,19 +9,11 @@ #ifndef _LIBCPP___ALGORITHM_COPY_BACKWARD_H #define _LIBCPP___ALGORITHM_COPY_BACKWARD_H -#include <__algorithm/copy.h> +#include <__algorithm/copy_move_common.h> #include <__algorithm/iterator_operations.h> -#include <__algorithm/ranges_copy.h> -#include <__algorithm/unwrap_iter.h> -#include <__concepts/same_as.h> #include <__config> -#include <__iterator/iterator_traits.h> -#include <__iterator/reverse_iterator.h> -#include <__ranges/subrange.h> #include <__utility/move.h> #include <__utility/pair.h> -#include -#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header @@ -29,32 +21,51 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template ::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InputIterator, _OutputIterator> -__copy_backward(_InputIterator __first, _InputIterator __last, _OutputIterator __result) { - auto __ret = std::__copy( - __unconstrained_reverse_iterator<_InputIterator>(__last), - __unconstrained_reverse_iterator<_InputIterator>(__first), - __unconstrained_reverse_iterator<_OutputIterator>(__result)); - return pair<_InputIterator, _OutputIterator>(__ret.first.base(), __ret.second.base()); -} +template +struct __copy_backward_loop { + template + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter> + operator()(_InIter __first, _Sent __last, _OutIter __result) const { + auto __last_iter = _IterOps<_AlgPolicy>::next(__first, __last); + auto __original_last_iter = __last_iter; + + while (__first != __last_iter) { + *--__result = *--__last_iter; + } + + return std::make_pair(std::move(__original_last_iter), std::move(__result)); + } +}; -#if _LIBCPP_STD_VER > 17 -template ::value, int> = 0> -_LIBCPP_HIDE_FROM_ABI constexpr pair<_Iter1, _Iter2> __copy_backward(_Iter1 __first, _Sent1 __last, _Iter2 __result) { - auto __last_iter = _IterOps<_AlgPolicy>::next(__first, std::move(__last)); - auto __reverse_range = std::__reverse_range(std::ranges::subrange(std::move(__first), __last_iter)); - auto __ret = ranges::copy(std::move(__reverse_range), std::make_reverse_iterator(__result)); - return std::make_pair(__last_iter, __ret.out.base()); +struct __copy_backward_trivial { + // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer. + template ::value, int> = 0> + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*> + operator()(_In* __first, _In* __last, _Out* __result) const { + return std::__copy_backward_trivial_impl(__first, __last, __result); + } +}; + +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 +pair<_BidirectionalIterator1, _BidirectionalIterator2> +__copy_backward(_BidirectionalIterator1 __first, _Sentinel __last, _BidirectionalIterator2 __result) { + return std::__dispatch_copy_or_move<_AlgPolicy, __copy_backward_loop<_AlgPolicy>, __copy_backward_trivial>( + std::move(__first), std::move(__last), std::move(__result)); } -#endif // _LIBCPP_STD_VER > 17 template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _BidirectionalIterator2 -copy_backward(_BidirectionalIterator1 __first, _BidirectionalIterator1 __last, _BidirectionalIterator2 __result) { - return std::__copy_backward<_ClassicAlgPolicy>(__first, __last, __result).second; +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 +_BidirectionalIterator2 +copy_backward(_BidirectionalIterator1 __first, _BidirectionalIterator1 __last, + _BidirectionalIterator2 __result) +{ + static_assert(std::is_copy_constructible<_BidirectionalIterator1>::value && + std::is_copy_constructible<_BidirectionalIterator1>::value, "Iterators must be copy constructible."); + + return std::__copy_backward<_ClassicAlgPolicy>( + std::move(__first), std::move(__last), std::move(__result)).second; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/copy_move_common.h b/libcxx/include/__algorithm/copy_move_common.h new file mode 100644 --- /dev/null +++ b/libcxx/include/__algorithm/copy_move_common.h @@ -0,0 +1,162 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ALGORITHM_COPY_MOVE_COMMON_H +#define _LIBCPP___ALGORITHM_COPY_MOVE_COMMON_H + +#include <__algorithm/iterator_operations.h> +#include <__algorithm/unwrap_iter.h> +#include <__algorithm/unwrap_range.h> +#include <__config> +#include <__iterator/iterator_traits.h> +#include <__memory/pointer_traits.h> +#include <__type_traits/enable_if.h> +#include <__type_traits/is_always_bitcastable.h> +#include <__type_traits/is_constant_evaluated.h> +#include <__type_traits/is_copy_constructible.h> +#include <__type_traits/is_trivially_assignable.h> +#include <__type_traits/is_trivially_copyable.h> +#include <__type_traits/is_volatile.h> +#include <__utility/move.h> +#include <__utility/pair.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +// Type traits. + +template +struct __can_lower_copy_assignment_to_memmove { + static const bool value = + // If the types are always bitcastable, it's valid to do a bitwise copy between them. + __is_always_bitcastable<_From, _To>::value && + // Reject conversions that wouldn't be performed by the regular built-in assignment (e.g. between arrays). + is_trivially_assignable<_To&, const _From&>::value && + // `memmove` doesn't accept `volatile` pointers, make sure the optimization SFINAEs away in that case. + !is_volatile<_From>::value && + !is_volatile<_To>::value; +}; + +template +struct __can_lower_move_assignment_to_memmove { + static const bool value = + __is_always_bitcastable<_From, _To>::value && + is_trivially_assignable<_To&, _From&&>::value && + !is_volatile<_From>::value && + !is_volatile<_To>::value; +}; + +// `memmove` algorithms implementation. + +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*> +__copy_trivial_impl(_In* __first, _In* __last, _Out* __result) { + const size_t __n = static_cast(__last - __first); + ::__builtin_memmove(__result, __first, __n * sizeof(_Out)); + + return std::make_pair(__last, __result + __n); +} + +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*> +__copy_backward_trivial_impl(_In* __first, _In* __last, _Out* __result) { + const size_t __n = static_cast(__last - __first); + __result -= __n; + + ::__builtin_memmove(__result, __first, __n * sizeof(_Out)); + + return std::make_pair(__last, __result); +} + +// Iterator unwrapping and dispatching to the correct overload. + +template +struct __overload : _F1, _F2 { + using _F1::operator(); + using _F2::operator(); +}; + +template +struct __can_rewrap : false_type {}; + +template +struct __can_rewrap<_InIter, + _Sent, + _OutIter, + // Note that sentinels are always copy-constructible. + __enable_if_t< is_copy_constructible<_InIter>::value && + is_copy_constructible<_OutIter>::value > > : true_type {}; + +template ::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 pair<_InIter, _OutIter> +__unwrap_and_dispatch(_InIter __first, _Sent __last, _OutIter __out_first) { + auto __range = std::__unwrap_range(__first, std::move(__last)); + auto __result = _Algorithm()(std::move(__range.first), std::move(__range.second), std::__unwrap_iter(__out_first)); + return std::make_pair(std::__rewrap_range<_Sent>(std::move(__first), std::move(__result.first)), + std::__rewrap_iter(std::move(__out_first), std::move(__result.second))); +} + +template ::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 pair<_InIter, _OutIter> +__unwrap_and_dispatch(_InIter __first, _Sent __last, _OutIter __out_first) { + return _Algorithm()(std::move(__first), std::move(__last), std::move(__out_first)); +} + +template +struct __can_copy_without_conversion : false_type {}; + +template +struct __can_copy_without_conversion< + _IterOps, + _InValue, + _OutIter, + __enable_if_t >::value> > : true_type {}; + +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 pair<_InIter, _OutIter> +__dispatch_copy_or_move(_InIter __first, _Sent __last, _OutIter __out_first) { +#ifdef _LIBCPP_COMPILER_GCC + // GCC doesn't support `__builtin_memmove` during constant evaluation. + if (__libcpp_is_constant_evaluated()) { + return std::__unwrap_and_dispatch<_NaiveAlgorithm>(std::move(__first), std::move(__last), std::move(__out_first)); + } +#else + // In Clang, `__builtin_memmove` only supports fully trivially copyable types (just having trivial copy assignment is + // insufficient). Also, conversions are not supported. + if (__libcpp_is_constant_evaluated()) { + using _InValue = typename _IterOps<_AlgPolicy>::template __value_type<_InIter>; + if (!is_trivially_copyable<_InValue>::value || + !__can_copy_without_conversion<_IterOps<_AlgPolicy>, _InValue, _OutIter>::value) { + return std::__unwrap_and_dispatch<_NaiveAlgorithm>(std::move(__first), std::move(__last), std::move(__out_first)); + } + } +#endif // _LIBCPP_COMPILER_GCC + + using _Algorithm = __overload<_NaiveAlgorithm, _OptimizedAlgorithm>; + return std::__unwrap_and_dispatch<_Algorithm>(std::move(__first), std::move(__last), std::move(__out_first)); +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ALGORITHM_COPY_MOVE_COMMON_H diff --git a/libcxx/include/__algorithm/move.h b/libcxx/include/__algorithm/move.h --- a/libcxx/include/__algorithm/move.h +++ b/libcxx/include/__algorithm/move.h @@ -9,19 +9,11 @@ #ifndef _LIBCPP___ALGORITHM_MOVE_H #define _LIBCPP___ALGORITHM_MOVE_H +#include <__algorithm/copy_move_common.h> #include <__algorithm/iterator_operations.h> -#include <__algorithm/unwrap_iter.h> #include <__config> -#include <__iterator/iterator_traits.h> -#include <__iterator/reverse_iterator.h> -#include <__type_traits/is_constant_evaluated.h> -#include <__type_traits/is_copy_constructible.h> -#include <__type_traits/is_trivially_copyable.h> -#include <__type_traits/is_trivially_move_assignable.h> -#include <__type_traits/remove_const.h> #include <__utility/move.h> #include <__utility/pair.h> -#include #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header @@ -29,93 +21,46 @@ _LIBCPP_BEGIN_NAMESPACE_STD -// move - -template -inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX17 -pair<_InIter, _OutIter> __move_impl(_InIter __first, _Sent __last, _OutIter __result) { - while (__first != __last) { - *__result = _IterOps<_AlgPolicy>::__iter_move(__first); - ++__first; - ++__result; +template +struct __move_loop { + template + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter> + operator()(_InIter __first, _Sent __last, _OutIter __result) const { + while (__first != __last) { + *__result = _IterOps<_AlgPolicy>::__iter_move(__first); + ++__first; + ++__result; + } + return std::make_pair(std::move(__first), std::move(__result)); } - return std::make_pair(std::move(__first), std::move(__result)); -} - -template , _OutType>::value - && is_trivially_move_assignable<_OutType>::value> > -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -pair<_InType*, _OutType*> __move_impl(_InType* __first, _InType* __last, _OutType* __result) { - if (__libcpp_is_constant_evaluated() -// TODO: Remove this once GCC supports __builtin_memmove during constant evaluation -#ifndef _LIBCPP_COMPILER_GCC - && !is_trivially_copyable<_InType>::value -#endif - ) - return std::__move_impl<_AlgPolicy, _InType*, _InType*, _OutType*>(__first, __last, __result); - const size_t __n = static_cast(__last - __first); - ::__builtin_memmove(__result, __first, __n * sizeof(_OutType)); - return std::make_pair(__first + __n, __result + __n); -} - -template -struct __is_trivially_move_assignable_unwrapped_impl : false_type {}; - -template -struct __is_trivially_move_assignable_unwrapped_impl<_Type*> : is_trivially_move_assignable<_Type> {}; - -template -struct __is_trivially_move_assignable_unwrapped - : __is_trivially_move_assignable_unwrapped_impl(std::declval<_Iter>()))> {}; - -template ::value_type>, - typename iterator_traits<_OutIter>::value_type>::value - && __is_cpp17_contiguous_iterator<_InIter>::value - && __is_cpp17_contiguous_iterator<_OutIter>::value - && is_trivially_move_assignable<__iter_value_type<_OutIter> >::value, int> = 0> -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 -pair, reverse_iterator<_OutIter> > -__move_impl(reverse_iterator<_InIter> __first, - reverse_iterator<_InIter> __last, - reverse_iterator<_OutIter> __result) { - auto __first_base = std::__unwrap_iter(__first.base()); - auto __last_base = std::__unwrap_iter(__last.base()); - auto __result_base = std::__unwrap_iter(__result.base()); - auto __result_first = __result_base - (__first_base - __last_base); - std::__move_impl<_AlgPolicy>(__last_base, __first_base, __result_first); - return std::make_pair(__last, reverse_iterator<_OutIter>(std::__rewrap_iter(__result.base(), __result_first))); -} - -template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -__enable_if_t::value - && is_copy_constructible<_Sent>::value - && is_copy_constructible<_OutIter>::value, pair<_InIter, _OutIter> > -__move(_InIter __first, _Sent __last, _OutIter __result) { - auto __ret = std::__move_impl<_AlgPolicy>( - std::__unwrap_iter(__first), std::__unwrap_iter(__last), std::__unwrap_iter(__result)); - return std::make_pair(std::__rewrap_iter(__first, __ret.first), std::__rewrap_iter(__result, __ret.second)); -} +}; + +struct __move_trivial { + // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer. + template ::value, int> = 0> + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*> + operator()(_In* __first, _In* __last, _Out* __result) const { + return std::__copy_trivial_impl(__first, __last, __result); + } +}; template inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 -__enable_if_t::value - || !is_copy_constructible<_Sent>::value - || !is_copy_constructible<_OutIter>::value, pair<_InIter, _OutIter> > +pair<_InIter, _OutIter> __move(_InIter __first, _Sent __last, _OutIter __result) { - return std::__move_impl<_AlgPolicy>(std::move(__first), std::move(__last), std::move(__result)); + return std::__dispatch_copy_or_move<_AlgPolicy, __move_loop<_AlgPolicy>, __move_trivial>( + std::move(__first), std::move(__last), std::move(__result)); } template inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator move(_InputIterator __first, _InputIterator __last, _OutputIterator __result) { - return std::__move<_ClassicAlgPolicy>(__first, __last, __result).second; + static_assert(is_copy_constructible<_InputIterator>::value, "Iterators has to be copy constructible."); + static_assert(is_copy_constructible<_OutputIterator>::value, "The output iterator has to be copy constructible."); + + return std::__move<_ClassicAlgPolicy>( + std::move(__first), std::move(__last), std::move(__result)).second; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/move_backward.h b/libcxx/include/__algorithm/move_backward.h --- a/libcxx/include/__algorithm/move_backward.h +++ b/libcxx/include/__algorithm/move_backward.h @@ -9,12 +9,11 @@ #ifndef _LIBCPP___ALGORITHM_MOVE_BACKWARD_H #define _LIBCPP___ALGORITHM_MOVE_BACKWARD_H +#include <__algorithm/copy_move_common.h> #include <__algorithm/iterator_operations.h> -#include <__algorithm/unwrap_iter.h> #include <__config> #include <__utility/move.h> -#include -#include +#include <__utility/pair.h> #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) # pragma GCC system_header @@ -22,57 +21,41 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template -inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX17 -_OutputIterator -__move_backward_constexpr(_InputIterator __first, _InputIterator __last, _OutputIterator __result) -{ - while (__first != __last) - *--__result = _IterOps<_AlgPolicy>::__iter_move(--__last); - return __result; -} +template +struct __move_backward_loop { + template + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter> + operator()(_InIter __first, _Sent __last, _OutIter __result) const { + auto __last_iter = _IterOps<_AlgPolicy>::next(__first, __last); + auto __original_last_iter = __last_iter; -template -inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX17 -_OutputIterator -__move_backward_impl(_InputIterator __first, _InputIterator __last, _OutputIterator __result) -{ - return _VSTD::__move_backward_constexpr<_AlgPolicy>(__first, __last, __result); -} - -template -inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX17 -typename enable_if -< - is_same<__remove_const_t<_Tp>, _Up>::value && - is_trivially_move_assignable<_Up>::value, - _Up* ->::type -__move_backward_impl(_Tp* __first, _Tp* __last, _Up* __result) -{ - const size_t __n = static_cast(__last - __first); - if (__n > 0) - { - __result -= __n; - _VSTD::memmove(__result, __first, __n * sizeof(_Up)); + while (__first != __last_iter) { + *--__result = _IterOps<_AlgPolicy>::__iter_move(--__last_iter); } - return __result; -} -template -inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20 -_BidirectionalIterator2 -__move_backward(_BidirectionalIterator1 __first, _BidirectionalIterator1 __last, - _BidirectionalIterator2 __result) -{ - if (__libcpp_is_constant_evaluated()) { - return _VSTD::__move_backward_constexpr<_AlgPolicy>(__first, __last, __result); - } else { - return _VSTD::__rewrap_iter(__result, - _VSTD::__move_backward_impl<_AlgPolicy>(_VSTD::__unwrap_iter(__first), - _VSTD::__unwrap_iter(__last), - _VSTD::__unwrap_iter(__result))); - } + return std::make_pair(std::move(__original_last_iter), std::move(__result)); + } +}; + +struct __move_backward_trivial { + // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer. + template ::value, int> = 0> + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*> + operator()(_In* __first, _In* __last, _Out* __result) const { + return std::__copy_backward_trivial_impl(__first, __last, __result); + } +}; + +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 +pair<_BidirectionalIterator1, _BidirectionalIterator2> +__move_backward(_BidirectionalIterator1 __first, _Sentinel __last, _BidirectionalIterator2 __result) { + static_assert(std::is_copy_constructible<_BidirectionalIterator1>::value && + std::is_copy_constructible<_BidirectionalIterator1>::value, "Iterators must be copy constructible."); + + return std::__dispatch_copy_or_move<_AlgPolicy, __move_backward_loop<_AlgPolicy>, __move_backward_trivial>( + std::move(__first), std::move(__last), std::move(__result)); } template @@ -81,7 +64,8 @@ move_backward(_BidirectionalIterator1 __first, _BidirectionalIterator1 __last, _BidirectionalIterator2 __result) { - return std::__move_backward<_ClassicAlgPolicy>(std::move(__first), std::move(__last), std::move(__result)); + return std::__move_backward<_ClassicAlgPolicy>( + std::move(__first), std::move(__last), std::move(__result)).second; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__algorithm/ranges_copy.h b/libcxx/include/__algorithm/ranges_copy.h --- a/libcxx/include/__algorithm/ranges_copy.h +++ b/libcxx/include/__algorithm/ranges_copy.h @@ -11,6 +11,7 @@ #include <__algorithm/copy.h> #include <__algorithm/in_out_result.h> +#include <__algorithm/iterator_operations.h> #include <__config> #include <__functional/identity.h> #include <__iterator/concepts.h> @@ -40,7 +41,7 @@ requires indirectly_copyable<_InIter, _OutIter> _LIBCPP_HIDE_FROM_ABI constexpr copy_result<_InIter, _OutIter> operator()(_InIter __first, _Sent __last, _OutIter __result) const { - auto __ret = std::__copy(std::move(__first), std::move(__last), std::move(__result)); + auto __ret = std::__copy<_RangeAlgPolicy>(std::move(__first), std::move(__last), std::move(__result)); return {std::move(__ret.first), std::move(__ret.second)}; } @@ -48,7 +49,7 @@ requires indirectly_copyable, _OutIter> _LIBCPP_HIDE_FROM_ABI constexpr copy_result, _OutIter> operator()(_Range&& __r, _OutIter __result) const { - auto __ret = std::__copy(ranges::begin(__r), ranges::end(__r), std::move(__result)); + auto __ret = std::__copy<_RangeAlgPolicy>(ranges::begin(__r), ranges::end(__r), std::move(__result)); return {std::move(__ret.first), std::move(__ret.second)}; } }; diff --git a/libcxx/include/__algorithm/ranges_copy_backward.h b/libcxx/include/__algorithm/ranges_copy_backward.h --- a/libcxx/include/__algorithm/ranges_copy_backward.h +++ b/libcxx/include/__algorithm/ranges_copy_backward.h @@ -14,7 +14,6 @@ #include <__algorithm/iterator_operations.h> #include <__config> #include <__iterator/concepts.h> -#include <__iterator/reverse_iterator.h> #include <__ranges/access.h> #include <__ranges/concepts.h> #include <__ranges/dangling.h> diff --git a/libcxx/include/__algorithm/ranges_copy_n.h b/libcxx/include/__algorithm/ranges_copy_n.h --- a/libcxx/include/__algorithm/ranges_copy_n.h +++ b/libcxx/include/__algorithm/ranges_copy_n.h @@ -11,6 +11,7 @@ #include <__algorithm/copy.h> #include <__algorithm/in_out_result.h> +#include <__algorithm/iterator_operations.h> #include <__algorithm/ranges_copy.h> #include <__config> #include <__functional/identity.h> @@ -51,7 +52,7 @@ template _LIBCPP_HIDE_FROM_ABI constexpr static copy_n_result<_InIter, _OutIter> __go(_InIter __first, _DiffType __n, _OutIter __result) { - auto __ret = std::__copy(__first, __first + __n, __result); + auto __ret = std::__copy<_RangeAlgPolicy>(__first, __first + __n, __result); return {__ret.first, __ret.second}; } diff --git a/libcxx/include/__algorithm/ranges_move.h b/libcxx/include/__algorithm/ranges_move.h --- a/libcxx/include/__algorithm/ranges_move.h +++ b/libcxx/include/__algorithm/ranges_move.h @@ -14,7 +14,6 @@ #include <__algorithm/move.h> #include <__config> #include <__iterator/concepts.h> -#include <__iterator/iter_move.h> #include <__ranges/access.h> #include <__ranges/concepts.h> #include <__ranges/dangling.h> diff --git a/libcxx/include/__algorithm/ranges_move_backward.h b/libcxx/include/__algorithm/ranges_move_backward.h --- a/libcxx/include/__algorithm/ranges_move_backward.h +++ b/libcxx/include/__algorithm/ranges_move_backward.h @@ -10,12 +10,12 @@ #define _LIBCPP___ALGORITHM_RANGES_MOVE_BACKWARD_H #include <__algorithm/in_out_result.h> -#include <__algorithm/ranges_move.h> +#include <__algorithm/iterator_operations.h> +#include <__algorithm/move_backward.h> #include <__config> #include <__iterator/concepts.h> #include <__iterator/iter_move.h> #include <__iterator/next.h> -#include <__iterator/reverse_iterator.h> #include <__ranges/access.h> #include <__ranges/concepts.h> #include <__ranges/dangling.h> @@ -40,11 +40,8 @@ template _LIBCPP_HIDE_FROM_ABI constexpr static move_backward_result<_InIter, _OutIter> __move_backward_impl(_InIter __first, _Sent __last, _OutIter __result) { - auto __last_iter = ranges::next(__first, std::move(__last)); - auto __ret = ranges::move(std::make_reverse_iterator(__last_iter), - std::make_reverse_iterator(__first), - std::make_reverse_iterator(__result)); - return {std::move(__last_iter), std::move(__ret.out.base())}; + auto __ret = std::__move_backward<_RangeAlgPolicy>(std::move(__first), std::move(__last), std::move(__result)); + return {std::move(__ret.first), std::move(__ret.second)}; } template _Sent, bidirectional_iterator _OutIter> diff --git a/libcxx/include/__algorithm/ranges_set_difference.h b/libcxx/include/__algorithm/ranges_set_difference.h --- a/libcxx/include/__algorithm/ranges_set_difference.h +++ b/libcxx/include/__algorithm/ranges_set_difference.h @@ -10,6 +10,7 @@ #define _LIBCPP___ALGORITHM_RANGES_SET_DIFFERENCE_H #include <__algorithm/in_out_result.h> +#include <__algorithm/iterator_operations.h> #include <__algorithm/make_projected.h> #include <__algorithm/set_difference.h> #include <__config> @@ -60,7 +61,7 @@ _Comp __comp = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const { - auto __ret = std::__set_difference( + auto __ret = std::__set_difference<_RangeAlgPolicy>( __first1, __last1, __first2, __last2, __result, ranges::__make_projected_comp(__comp, __proj1, __proj2)); return {std::move(__ret.first), std::move(__ret.second)}; } @@ -81,7 +82,7 @@ _Comp __comp = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const { - auto __ret = std::__set_difference( + auto __ret = std::__set_difference<_RangeAlgPolicy>( ranges::begin(__range1), ranges::end(__range1), ranges::begin(__range2), diff --git a/libcxx/include/__algorithm/ranges_set_symmetric_difference.h b/libcxx/include/__algorithm/ranges_set_symmetric_difference.h --- a/libcxx/include/__algorithm/ranges_set_symmetric_difference.h +++ b/libcxx/include/__algorithm/ranges_set_symmetric_difference.h @@ -10,6 +10,7 @@ #define _LIBCPP___ALGORITHM_RANGES_SET_SYMMETRIC_DIFFERENCE_H #include <__algorithm/in_in_out_result.h> +#include <__algorithm/iterator_operations.h> #include <__algorithm/make_projected.h> #include <__algorithm/set_symmetric_difference.h> #include <__config> @@ -58,7 +59,7 @@ _Comp __comp = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const { - auto __ret = std::__set_symmetric_difference( + auto __ret = std::__set_symmetric_difference<_RangeAlgPolicy>( std::move(__first1), std::move(__last1), std::move(__first2), @@ -92,7 +93,7 @@ _Comp __comp = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const { - auto __ret = std::__set_symmetric_difference( + auto __ret = std::__set_symmetric_difference<_RangeAlgPolicy>( ranges::begin(__range1), ranges::end(__range1), ranges::begin(__range2), diff --git a/libcxx/include/__algorithm/ranges_set_union.h b/libcxx/include/__algorithm/ranges_set_union.h --- a/libcxx/include/__algorithm/ranges_set_union.h +++ b/libcxx/include/__algorithm/ranges_set_union.h @@ -10,6 +10,7 @@ #define _LIBCPP___ALGORITHM_RANGES_SET_UNION_H #include <__algorithm/in_in_out_result.h> +#include <__algorithm/iterator_operations.h> #include <__algorithm/make_projected.h> #include <__algorithm/set_union.h> #include <__config> @@ -61,7 +62,7 @@ _Comp __comp = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const { - auto __ret = std::__set_union( + auto __ret = std::__set_union<_RangeAlgPolicy>( std::move(__first1), std::move(__last1), std::move(__first2), @@ -95,7 +96,7 @@ _Comp __comp = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const { - auto __ret = std::__set_union( + auto __ret = std::__set_union<_RangeAlgPolicy>( ranges::begin(__range1), ranges::end(__range1), ranges::begin(__range2), diff --git a/libcxx/include/__algorithm/rotate.h b/libcxx/include/__algorithm/rotate.h --- a/libcxx/include/__algorithm/rotate.h +++ b/libcxx/include/__algorithm/rotate.h @@ -48,7 +48,7 @@ _BidirectionalIterator __lm1 = _Ops::prev(__last); value_type __tmp = _Ops::__iter_move(__lm1); - _BidirectionalIterator __fp1 = std::__move_backward<_AlgPolicy>(__first, __lm1, std::move(__last)); + _BidirectionalIterator __fp1 = std::__move_backward<_AlgPolicy>(__first, __lm1, std::move(__last)).second; *__first = _VSTD::move(__tmp); return __fp1; } diff --git a/libcxx/include/__algorithm/set_difference.h b/libcxx/include/__algorithm/set_difference.h --- a/libcxx/include/__algorithm/set_difference.h +++ b/libcxx/include/__algorithm/set_difference.h @@ -12,6 +12,7 @@ #include <__algorithm/comp.h> #include <__algorithm/comp_ref_type.h> #include <__algorithm/copy.h> +#include <__algorithm/iterator_operations.h> #include <__config> #include <__functional/identity.h> #include <__functional/invoke.h> @@ -26,7 +27,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template < class _Comp, class _InIter1, class _Sent1, class _InIter2, class _Sent2, class _OutIter> +template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<__remove_cvref_t<_InIter1>, __remove_cvref_t<_OutIter> > __set_difference( _InIter1&& __first1, _Sent1&& __last1, _InIter2&& __first2, _Sent2&& __last2, _OutIter&& __result, _Comp&& __comp) { @@ -42,7 +43,7 @@ ++__first2; } } - return std::__copy(std::move(__first1), std::move(__last1), std::move(__result)); + return std::__copy<_AlgPolicy>(std::move(__first1), std::move(__last1), std::move(__result)); } template @@ -53,7 +54,8 @@ _InputIterator2 __last2, _OutputIterator __result, _Compare __comp) { - return std::__set_difference<__comp_ref_type<_Compare> >(__first1, __last1, __first2, __last2, __result, __comp) + return std::__set_difference<_ClassicAlgPolicy, __comp_ref_type<_Compare> >( + __first1, __last1, __first2, __last2, __result, __comp) .second; } @@ -64,7 +66,7 @@ _InputIterator2 __first2, _InputIterator2 __last2, _OutputIterator __result) { - return std::__set_difference( + return std::__set_difference<_ClassicAlgPolicy>( __first1, __last1, __first2, diff --git a/libcxx/include/__algorithm/set_symmetric_difference.h b/libcxx/include/__algorithm/set_symmetric_difference.h --- a/libcxx/include/__algorithm/set_symmetric_difference.h +++ b/libcxx/include/__algorithm/set_symmetric_difference.h @@ -12,6 +12,7 @@ #include <__algorithm/comp.h> #include <__algorithm/comp_ref_type.h> #include <__algorithm/copy.h> +#include <__algorithm/iterator_operations.h> #include <__config> #include <__iterator/iterator_traits.h> #include <__utility/move.h> @@ -35,13 +36,13 @@ : __in1_(std::move(__in_iter1)), __in2_(std::move(__in_iter2)), __out_(std::move(__out_iter)) {} }; -template +template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __set_symmetric_difference_result<_InIter1, _InIter2, _OutIter> __set_symmetric_difference( _InIter1 __first1, _Sent1 __last1, _InIter2 __first2, _Sent2 __last2, _OutIter __result, _Compare&& __comp) { while (__first1 != __last1) { if (__first2 == __last2) { - auto __ret1 = std::__copy_impl(std::move(__first1), std::move(__last1), std::move(__result)); + auto __ret1 = std::__copy<_AlgPolicy>(std::move(__first1), std::move(__last1), std::move(__result)); return __set_symmetric_difference_result<_InIter1, _InIter2, _OutIter>( std::move(__ret1.first), std::move(__first2), std::move((__ret1.second))); } @@ -59,7 +60,7 @@ ++__first2; } } - auto __ret2 = std::__copy_impl(std::move(__first2), std::move(__last2), std::move(__result)); + auto __ret2 = std::__copy<_AlgPolicy>(std::move(__first2), std::move(__last2), std::move(__result)); return __set_symmetric_difference_result<_InIter1, _InIter2, _OutIter>( std::move(__first1), std::move(__ret2.first), std::move((__ret2.second))); } @@ -72,7 +73,7 @@ _InputIterator2 __last2, _OutputIterator __result, _Compare __comp) { - return std::__set_symmetric_difference<__comp_ref_type<_Compare> >( + return std::__set_symmetric_difference<_ClassicAlgPolicy, __comp_ref_type<_Compare> >( std::move(__first1), std::move(__last1), std::move(__first2), diff --git a/libcxx/include/__algorithm/set_union.h b/libcxx/include/__algorithm/set_union.h --- a/libcxx/include/__algorithm/set_union.h +++ b/libcxx/include/__algorithm/set_union.h @@ -12,6 +12,7 @@ #include <__algorithm/comp.h> #include <__algorithm/comp_ref_type.h> #include <__algorithm/copy.h> +#include <__algorithm/iterator_operations.h> #include <__config> #include <__iterator/iterator_traits.h> #include <__utility/move.h> @@ -35,12 +36,12 @@ : __in1_(std::move(__in_iter1)), __in2_(std::move(__in_iter2)), __out_(std::move(__out_iter)) {} }; -template +template _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __set_union_result<_InIter1, _InIter2, _OutIter> __set_union( _InIter1 __first1, _Sent1 __last1, _InIter2 __first2, _Sent2 __last2, _OutIter __result, _Compare&& __comp) { for (; __first1 != __last1; ++__result) { if (__first2 == __last2) { - auto __ret1 = std::__copy_impl(std::move(__first1), std::move(__last1), std::move(__result)); + auto __ret1 = std::__copy<_AlgPolicy>(std::move(__first1), std::move(__last1), std::move(__result)); return __set_union_result<_InIter1, _InIter2, _OutIter>( std::move(__ret1.first), std::move(__first2), std::move((__ret1.second))); } @@ -55,7 +56,7 @@ ++__first1; } } - auto __ret2 = std::__copy_impl(std::move(__first2), std::move(__last2), std::move(__result)); + auto __ret2 = std::__copy<_AlgPolicy>(std::move(__first2), std::move(__last2), std::move(__result)); return __set_union_result<_InIter1, _InIter2, _OutIter>( std::move(__first1), std::move(__ret2.first), std::move((__ret2.second))); } @@ -68,7 +69,7 @@ _InputIterator2 __last2, _OutputIterator __result, _Compare __comp) { - return std::__set_union<__comp_ref_type<_Compare> >( + return std::__set_union<_ClassicAlgPolicy, __comp_ref_type<_Compare> >( std::move(__first1), std::move(__last1), std::move(__first2), diff --git a/libcxx/include/__iterator/reverse_iterator.h b/libcxx/include/__iterator/reverse_iterator.h --- a/libcxx/include/__iterator/reverse_iterator.h +++ b/libcxx/include/__iterator/reverse_iterator.h @@ -202,12 +202,6 @@ #endif // _LIBCPP_STD_VER > 17 }; -template -struct __is_reverse_iterator : false_type {}; - -template -struct __is_reverse_iterator > : true_type {}; - template inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX17 bool @@ -485,9 +479,6 @@ } }; -template -struct __is_reverse_iterator<__unconstrained_reverse_iterator<_Iter>> : true_type {}; - #endif // _LIBCPP_STD_VER <= 17 template