diff --git a/libcxx/include/algorithm b/libcxx/include/algorithm --- a/libcxx/include/algorithm +++ b/libcxx/include/algorithm @@ -1629,68 +1629,6 @@ __value_, __equal_to<__v, _Tp>()); } -// copy -template -inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 -_Iter -__unwrap_iter(_Iter __i) -{ - return __i; -} - -template -inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 -typename enable_if -< - is_trivially_copy_assignable<_Tp>::value, - _Tp* ->::type -__unwrap_iter(move_iterator<_Tp*> __i) -{ - return __i.base(); -} - -#if _LIBCPP_DEBUG_LEVEL < 2 - -template -inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_IF_NODEBUG -typename enable_if -< - is_trivially_copy_assignable<_Tp>::value, - _Tp* ->::type -__unwrap_iter(__wrap_iter<_Tp*> __i) -{ - return __i.base(); -} - -template -inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_IF_NODEBUG -typename enable_if -< - is_trivially_copy_assignable<_Tp>::value, - const _Tp* ->::type -__unwrap_iter(__wrap_iter __i) -{ - return __i.base(); -} - -#else - -template -inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_IF_NODEBUG -typename enable_if -< - is_trivially_copy_assignable<_Tp>::value, - __wrap_iter<_Tp*> ->::type -__unwrap_iter(__wrap_iter<_Tp*> __i) -{ - return __i; -} - -#endif // _LIBCPP_DEBUG_LEVEL < 2 template inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17 @@ -1705,38 +1643,45 @@ template inline _LIBCPP_INLINE_VISIBILITY _OutputIterator -__copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result) +__copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result, false_type) { - return __copy_constexpr(__first, __last, __result); + return std::__copy_constexpr(__first, __last, __result); } template -inline _LIBCPP_INLINE_VISIBILITY -typename enable_if -< - is_same::type, _Up>::value && - is_trivially_copy_assignable<_Up>::value, - _Up* ->::type -__copy(_Tp* __first, _Tp* __last, _Up* __result) +_Up +__copy(_Tp __first, _Tp __last, _Up __result, true_type) { + const size_t __n = static_cast(__last - __first); if (__n > 0) - _VSTD::memmove(__result, __first, __n * sizeof(_Up)); + _VSTD::memmove(std::__to_address(__result), std::__to_address(__first), __n * sizeof(typename iterator_traits<_Tp>::value_type)); return __result + __n; } +template +using __can_optimize_copy_on_pointers = _And< + __is_unwrappable_contiguous_iterator<_InputIterator>, + __is_unwrappable_contiguous_iterator<_OutputIterator>, + is_lvalue_reference::reference>, + _IsSame< + typename remove_const::value_type>::type, + typename iterator_traits<_OutputIterator>::value_type + >, + is_trivially_copy_assignable::value_type> + >; + + template inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17_WITH_IS_CONSTANT_EVALUATED _OutputIterator copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result) { + if (__libcpp_is_constant_evaluated()) { - return _VSTD::__copy_constexpr( - __unwrap_iter(__first), __unwrap_iter(__last), __unwrap_iter(__result)); + return _VSTD::__copy_constexpr(__first, __last, __result); } else { - return _VSTD::__copy( - __unwrap_iter(__first), __unwrap_iter(__last), __unwrap_iter(__result)); + return _VSTD::__copy(__first, __last, __result, __can_optimize_copy_on_pointers<_InputIterator, _OutputIterator>() ); } } @@ -1755,26 +1700,24 @@ template inline _LIBCPP_INLINE_VISIBILITY _OutputIterator -__copy_backward(_BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __result) +__copy_backward(_BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __result, false_type) { - return __copy_backward_constexpr(__first, __last, __result); + return std::__copy_backward_constexpr(__first, __last, __result); } template inline _LIBCPP_INLINE_VISIBILITY -typename enable_if -< - is_same::type, _Up>::value && - is_trivially_copy_assignable<_Up>::value, - _Up* ->::type -__copy_backward(_Tp* __first, _Tp* __last, _Up* __result) +_Up +__copy_backward(_Tp __first, _Tp __last, _Up __result, true_type) { + const size_t __n = static_cast(__last - __first); if (__n > 0) { __result -= __n; - _VSTD::memmove(__result, __first, __n * sizeof(_Up)); + _VSTD::memmove(std::__unwrap_contiguous(__result), + std::__unwrap_contiguous(__first), + __n * sizeof(typename iterator_traits<_Tp>::value_type)); } return __result; } @@ -1785,14 +1728,12 @@ copy_backward(_BidirectionalIterator1 __first, _BidirectionalIterator1 __last, _BidirectionalIterator2 __result) { + if (__libcpp_is_constant_evaluated()) { - return _VSTD::__copy_backward_constexpr(__unwrap_iter(__first), - __unwrap_iter(__last), - __unwrap_iter(__result)); + return _VSTD::__copy_backward_constexpr(__first, __last, __result); } else { - return _VSTD::__copy_backward(__unwrap_iter(__first), - __unwrap_iter(__last), - __unwrap_iter(__result)); + return _VSTD::__copy_backward(__first, __last, __result, + __can_optimize_copy_on_pointers<_BidirectionalIterator1, _BidirectionalIterator2>()); } } @@ -1862,7 +1803,7 @@ template inline _LIBCPP_INLINE_VISIBILITY _OutputIterator -__move(_InputIterator __first, _InputIterator __last, _OutputIterator __result) +__move(_InputIterator __first, _InputIterator __last, _OutputIterator __result, false_type) { for (; __first != __last; ++__first, (void) ++__result) *__result = _VSTD::move(*__first); @@ -1871,17 +1812,12 @@ template inline _LIBCPP_INLINE_VISIBILITY -typename enable_if -< - is_same::type, _Up>::value && - is_trivially_copy_assignable<_Up>::value, - _Up* ->::type -__move(_Tp* __first, _Tp* __last, _Up* __result) +_Up +__move(_Tp __first, _Tp __last, _Up __result, true_type) { const size_t __n = static_cast(__last - __first); if (__n > 0) - _VSTD::memmove(__result, __first, __n * sizeof(_Up)); + _VSTD::memmove(std::__unwrap_contiguous(__result), std::__unwrap_contiguous(__first), __n * sizeof(typename iterator_traits<_Tp>::value_type)); return __result + __n; } @@ -1890,7 +1826,7 @@ _OutputIterator move(_InputIterator __first, _InputIterator __last, _OutputIterator __result) { - return _VSTD::__move(__unwrap_iter(__first), __unwrap_iter(__last), __unwrap_iter(__result)); + return _VSTD::__move(__first, __last, __result, __can_optimize_copy_on_pointers<_InputIterator, _OutputIterator>() ); } // move_backward @@ -1898,7 +1834,7 @@ template inline _LIBCPP_INLINE_VISIBILITY _OutputIterator -__move_backward(_InputIterator __first, _InputIterator __last, _OutputIterator __result) +__move_backward(_InputIterator __first, _InputIterator __last, _OutputIterator __result, false_type) { while (__first != __last) *--__result = _VSTD::move(*--__last); @@ -1907,19 +1843,14 @@ template inline _LIBCPP_INLINE_VISIBILITY -typename enable_if -< - is_same::type, _Up>::value && - is_trivially_copy_assignable<_Up>::value, - _Up* ->::type -__move_backward(_Tp* __first, _Tp* __last, _Up* __result) +_Up +__move_backward(_Tp __first, _Tp __last, _Up __result, true_type) { const size_t __n = static_cast(__last - __first); if (__n > 0) { __result -= __n; - _VSTD::memmove(__result, __first, __n * sizeof(_Up)); + _VSTD::memmove(std::__unwrap_contiguous(__result), std::__unwrap_contiguous(__first), __n * sizeof(typename iterator_traits<_Up>::value_type)); } return __result; } @@ -1930,7 +1861,7 @@ move_backward(_BidirectionalIterator1 __first, _BidirectionalIterator1 __last, _BidirectionalIterator2 __result) { - return _VSTD::__move_backward(__unwrap_iter(__first), __unwrap_iter(__last), __unwrap_iter(__result)); + return _VSTD::__move_backward(__first, __last, __result, __can_optimize_copy_on_pointers<_BidirectionalIterator1, _BidirectionalIterator2>()); } // iter_swap diff --git a/libcxx/include/iterator b/libcxx/include/iterator --- a/libcxx/include/iterator +++ b/libcxx/include/iterator @@ -579,6 +579,16 @@ template struct __has_iterator_category_convertible_to<_Tp, _Up, false> : public false_type {}; + +template ::value > +struct __has_iterator_concept_convertible_to + : public integral_constant, _Up>::value> +{}; + +template +struct __has_iterator_concept_convertible_to<_Tp, _Up, false> : public false_type {}; + + template struct __is_cpp17_input_iterator : public __has_iterator_category_convertible_to<_Tp, input_iterator_tag> {}; @@ -594,9 +604,13 @@ #if _LIBCPP_STD_VER > 17 template struct __is_cpp17_contiguous_iterator : public __has_iterator_category_convertible_to<_Tp, contiguous_iterator_tag> {}; +template +struct __has_contiguous_iterator_concept : public __has_iterator_concept_convertible_to<_Tp, contiguous_iterator_tag> {}; #else template struct __is_cpp17_contiguous_iterator : public false_type {}; +template +struct __has_contiguous_iterator_concept : public false_type {}; #endif @@ -1377,32 +1391,22 @@ template _Op _LIBCPP_INLINE_VISIBILITY move(_Ip, _Ip, _Op); template _B2 _LIBCPP_INLINE_VISIBILITY move_backward(_B1, _B1, _B2); -#if _LIBCPP_DEBUG_LEVEL < 2 - -template -_LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_IF_NODEBUG -typename enable_if -< - is_trivially_copy_assignable<_Tp>::value, - _Tp* ->::type -__unwrap_iter(__wrap_iter<_Tp*>); - -#else - -template -inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_IF_NODEBUG -typename enable_if -< - is_trivially_copy_assignable<_Tp>::value, - __wrap_iter<_Tp*> ->::type -__unwrap_iter(__wrap_iter<_Tp*> __i); - +#if _LIBCPP_STD_VER > 17 +struct __empty_base {}; +struct __contiguous_iterator_concept_base { + using iterator_concept = contiguous_iterator_tag; +}; #endif template class __wrap_iter +#if _LIBCPP_STD_VER > 17 +: public _If< + __has_iterator_concept_convertible_to<_Iter, contiguous_iterator_tag>::value, + __contiguous_iterator_concept_base, + __empty_base + > +#endif { public: typedef _Iter iterator_type; @@ -1411,6 +1415,7 @@ typedef typename iterator_traits::difference_type difference_type; typedef typename iterator_traits::pointer pointer; typedef typename iterator_traits::reference reference; + private: iterator_type __i; public: @@ -1588,25 +1593,6 @@ template friend _Op move(_Ip, _Ip, _Op); template friend _B2 move_backward(_B1, _B1, _B2); -#if _LIBCPP_DEBUG_LEVEL < 2 - template - _LIBCPP_CONSTEXPR_IF_NODEBUG friend - typename enable_if - < - is_trivially_copy_assignable<_Tp>::value, - _Tp* - >::type - __unwrap_iter(__wrap_iter<_Tp*>); -#else - template - inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_IF_NODEBUG - typename enable_if - < - is_trivially_copy_assignable<_Tp>::value, - __wrap_iter<_Tp*> - >::type - __unwrap_iter(__wrap_iter<_Tp*> __i); -#endif }; template diff --git a/libcxx/include/memory b/libcxx/include/memory --- a/libcxx/include/memory +++ b/libcxx/include/memory @@ -1123,7 +1123,8 @@ }; template -using __choose_to_address = __to_address_helper<_IsValidExpansion<__to_address_helper<_Dummy>::template __return_type, _Pointer>::value>; +using __choose_to_address = __to_address_helper< + _IsValidExpansion<__to_address_helper<_Dummy>::template __return_type, _Pointer>::value>; template @@ -1172,6 +1173,29 @@ } #endif + +template +struct __to_address_test : _IsSame< + decltype(std::__to_address(std::declval<_Tp&>())), + _Desired +> {}; + +template +using __is_unwrappable_contiguous_iterator = _And< + __is_cpp17_random_access_iterator<_Iter>, + __has_contiguous_iterator_concept<_Iter>, + __to_address_test<_Iter, + typename remove_reference::pointer>::type + > +>; + +template ::value> > +typename iterator_traits<_Iter>::pointer +__unwrap_contiguous(_Iter __it) { + static_assert(std::is_pointer::pointer>::value, ""); + return std::__to_address(__it); +} + template struct __has_size_type : false_type {};