diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -339,6 +339,7 @@ __memory/ranges_uninitialized_algorithms.h __memory/raw_storage_iterator.h __memory/shared_ptr.h + __memory/swap_allocator.h __memory/temporary_buffer.h __memory/uninitialized_algorithms.h __memory/unique_ptr.h diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table --- a/libcxx/include/__hash_table +++ b/libcxx/include/__hash_table @@ -18,6 +18,7 @@ #include <__debug> #include <__functional/hash.h> #include <__iterator/iterator_traits.h> +#include <__memory/swap_allocator.h> #include <__utility/swap.h> #include #include diff --git a/libcxx/include/__memory/swap_allocator.h b/libcxx/include/__memory/swap_allocator.h new file mode 100644 --- /dev/null +++ b/libcxx/include/__memory/swap_allocator.h @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___MEMORY_SWAP_ALLOCATOR_H +#define _LIBCPP___MEMORY_SWAP_ALLOCATOR_H + +#include <__config> +#include <__memory/allocator_traits.h> +#include <__type_traits/integral_constant.h> +#include <__utility/swap.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11 void __swap_allocator(_Alloc& __a1, _Alloc& __a2, true_type) +#if _LIBCPP_STD_VER > 11 + _NOEXCEPT +#else + _NOEXCEPT_(__is_nothrow_swappable<_Alloc>::value) +#endif +{ + using _VSTD::swap; + swap(__a1, __a2); +} + +template +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11 void +__swap_allocator(_Alloc&, _Alloc&, false_type) _NOEXCEPT {} + +template +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11 void __swap_allocator(_Alloc& __a1, _Alloc& __a2) +#if _LIBCPP_STD_VER > 11 + _NOEXCEPT +#else + _NOEXCEPT_(__is_nothrow_swappable<_Alloc>::value) +#endif +{ + _VSTD::__swap_allocator( + __a1, __a2, integral_constant::propagate_on_container_swap::value>()); +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___MEMORY_SWAP_ALLOCATOR_H diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h --- a/libcxx/include/__memory/uninitialized_algorithms.h +++ b/libcxx/include/__memory/uninitialized_algorithms.h @@ -10,11 +10,14 @@ #ifndef _LIBCPP___MEMORY_UNINITIALIZED_ALGORITHMS_H #define _LIBCPP___MEMORY_UNINITIALIZED_ALGORITHMS_H +#include <__algorithm/min.h> #include <__config> #include <__iterator/iterator_traits.h> +#include <__iterator/reverse_iterator.h> #include <__memory/addressof.h> #include <__memory/allocator_traits.h> #include <__memory/construct_at.h> +#include <__memory/pointer_traits.h> #include <__memory/voidify.h> #include <__utility/move.h> #include <__utility/pair.h> @@ -25,6 +28,9 @@ # pragma GCC system_header #endif +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + _LIBCPP_BEGIN_NAMESPACE_STD // This is a simplified version of C++20 `unreachable_sentinel` that doesn't use concepts and thus can be used in any @@ -492,6 +498,94 @@ #endif // _LIBCPP_STD_VER > 14 +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 _Iter2 +__uninitialized_allocator_copy(_Alloc& __alloc, _Iter1 __first1, _Sent1 __last1, _Iter2 __first2) { + auto __destruct_first = __first2; +#ifndef _LIBCPP_NO_EXCEPTIONS + try { +#endif + while (__first1 != __last1) { + allocator_traits<_Alloc>::construct(__alloc, std::__to_address(__first2), *__first1); + ++__first1; + ++__first2; + } +#ifndef _LIBCPP_NO_EXCEPTIONS + } catch (...) { + std::__allocator_destroy_multidimensional(__alloc, __destruct_first, __first2); + throw; + } +#endif + return __first2; +} + +template ::type, + class = __enable_if_t::value && + is_same<_RawType1, typename remove_const<_Type2>::type>::value && + (__is_default_allocator<_Alloc>::value || + !__has_construct<_Alloc, _Type2*, _Type1&>::value)> > +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 _Type2* +__uninitialized_allocator_copy(_Alloc&, _Type1* __first1, _Type1* __last1, _Type2* __first2) { + auto __diff = __last1 - __first1; + ::__builtin_memcpy(const_cast<_RawType1*>(__first2), __first1, __diff * sizeof(_Type1)); + return __first2 + __diff; +} + +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 _Iter2 __uninitialized_allocator_move_if_noexcept( + _Alloc& __alloc, _Iter1 __first1, _Sent1 __last1, _Iter2 __first2) { + static_assert(__is_cpp17_move_insertable<_Alloc>::value, + "The specified type does not meet the requirements of Cpp17MoveInsertable"); +#ifndef _LIBCPP_NO_EXCEPTIONS + auto __destruct_first = __first2; + try { +#endif + while (__first1 != __last1) { + allocator_traits<_Alloc>::construct(__alloc, std::__to_address(__first2), std::move_if_noexcept(*__first1)); + ++__first1; + ++__first2; + } +#ifndef _LIBCPP_NO_EXCEPTIONS + } catch (...) { + std::__allocator_destroy_multidimensional(__alloc, __destruct_first, __first2); + throw; + } +#endif + return __first2; +} + +template < + class _Alloc, + class _Type, + class = __enable_if_t<(__is_default_allocator<_Alloc>::value || !__has_construct<_Alloc, _Type*, _Type>::value) && + is_trivially_move_constructible<_Type>::value> > +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 _Type* +__uninitialized_allocator_move_if_noexcept(_Alloc&, _Type* __first1, _Type* __last1, _Type* __first2) { + auto __diff = __last1 - __first1; + ::__builtin_memcpy(__first2, __first1, __diff * sizeof(_Type)); + return __first2 + __diff; +} + +template < + class _Alloc, + class _Type, + class = __enable_if_t<(__is_default_allocator<_Alloc>::value || !__has_construct<_Alloc, _Type*, _Type>::value) && + is_trivially_move_constructible<_Type>::value> > +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 reverse_iterator<_Type*> +__uninitialized_allocator_move_if_noexcept(_Alloc&, + reverse_iterator<_Type*> __first1, + reverse_iterator<_Type*> __last1, + reverse_iterator<_Type*> __first2) { + auto __diff = __last1 - __first1; + ::__builtin_memcpy(__first2.base() - __diff, __last1.base(), __diff * sizeof(_Type)); + return __first2 + __diff; +} + _LIBCPP_END_NAMESPACE_STD +_LIBCPP_POP_MACROS + #endif // _LIBCPP___MEMORY_UNINITIALIZED_ALGORITHMS_H diff --git a/libcxx/include/__split_buffer b/libcxx/include/__split_buffer --- a/libcxx/include/__split_buffer +++ b/libcxx/include/__split_buffer @@ -19,6 +19,7 @@ #include <__iterator/move_iterator.h> #include <__memory/allocator.h> #include <__memory/compressed_pair.h> +#include <__memory/swap_allocator.h> #include <__utility/forward.h> #include #include diff --git a/libcxx/include/__tree b/libcxx/include/__tree --- a/libcxx/include/__tree +++ b/libcxx/include/__tree @@ -17,6 +17,7 @@ #include <__iterator/distance.h> #include <__iterator/iterator_traits.h> #include <__iterator/next.h> +#include <__memory/swap_allocator.h> #include <__utility/forward.h> #include <__utility/swap.h> #include diff --git a/libcxx/include/__utility/move.h b/libcxx/include/__utility/move.h --- a/libcxx/include/__utility/move.h +++ b/libcxx/include/__utility/move.h @@ -26,10 +26,15 @@ return static_cast<_Up&&>(__t); } +#ifdef _LIBCPP_NO_EXCEPTIONS +template +using __move_if_noexcept_result_t = _Tp&&; +#else template using __move_if_noexcept_result_t = typename conditional::value && is_copy_constructible<_Tp>::value, const _Tp&, _Tp&&>::type; +#endif template _LIBCPP_NODISCARD_EXT inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11 __move_if_noexcept_result_t<_Tp> diff --git a/libcxx/include/forward_list b/libcxx/include/forward_list --- a/libcxx/include/forward_list +++ b/libcxx/include/forward_list @@ -188,6 +188,7 @@ #include <__iterator/iterator_traits.h> #include <__iterator/move_iterator.h> #include <__iterator/next.h> +#include <__memory/swap_allocator.h> #include <__utility/forward.h> #include #include diff --git a/libcxx/include/list b/libcxx/include/list --- a/libcxx/include/list +++ b/libcxx/include/list @@ -194,6 +194,7 @@ #include <__iterator/next.h> #include <__iterator/prev.h> #include <__iterator/reverse_iterator.h> +#include <__memory/swap_allocator.h> #include <__utility/forward.h> #include <__utility/move.h> #include <__utility/swap.h> diff --git a/libcxx/include/memory b/libcxx/include/memory --- a/libcxx/include/memory +++ b/libcxx/include/memory @@ -880,93 +880,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template -_LIBCPP_INLINE_VISIBILITY -void __construct_forward_with_exception_guarantees(_Alloc& __a, _Ptr __begin1, _Ptr __end1, _Ptr& __begin2) { - static_assert(__is_cpp17_move_insertable<_Alloc>::value, - "The specified type does not meet the requirements of Cpp17MoveInsertable"); - typedef allocator_traits<_Alloc> _Traits; - for (; __begin1 != __end1; ++__begin1, (void)++__begin2) { - _Traits::construct(__a, _VSTD::__to_address(__begin2), -#ifdef _LIBCPP_NO_EXCEPTIONS - _VSTD::move(*__begin1) -#else - _VSTD::move_if_noexcept(*__begin1) -#endif - ); - } -} - -template ::value || !__has_construct<_Alloc, _Tp*, _Tp>::value) && - is_trivially_move_constructible<_Tp>::value ->::type> -_LIBCPP_INLINE_VISIBILITY -void __construct_forward_with_exception_guarantees(_Alloc&, _Tp* __begin1, _Tp* __end1, _Tp*& __begin2) { - ptrdiff_t _Np = __end1 - __begin1; - if (_Np > 0) { - _VSTD::memcpy(__begin2, __begin1, _Np * sizeof(_Tp)); - __begin2 += _Np; - } -} - -template -_LIBCPP_INLINE_VISIBILITY -void __construct_range_forward(_Alloc& __a, _Iter __begin1, _Iter __end1, _Ptr& __begin2) { - typedef allocator_traits<_Alloc> _Traits; - for (; __begin1 != __end1; ++__begin1, (void) ++__begin2) { - _Traits::construct(__a, _VSTD::__to_address(__begin2), *__begin1); - } -} - -template ::type, - class _RawDest = typename remove_const<_Dest>::type, - class = - typename enable_if< - is_trivially_copy_constructible<_Dest>::value && - is_same<_RawSource, _RawDest>::value && - (__is_default_allocator<_Alloc>::value || !__has_construct<_Alloc, _Dest*, _Source&>::value) - >::type> -_LIBCPP_INLINE_VISIBILITY -void __construct_range_forward(_Alloc&, _Source* __begin1, _Source* __end1, _Dest*& __begin2) { - ptrdiff_t _Np = __end1 - __begin1; - if (_Np > 0) { - _VSTD::memcpy(const_cast<_RawDest*>(__begin2), __begin1, _Np * sizeof(_Dest)); - __begin2 += _Np; - } -} - -template -_LIBCPP_INLINE_VISIBILITY -void __construct_backward_with_exception_guarantees(_Alloc& __a, _Ptr __begin1, _Ptr __end1, _Ptr& __end2) { - static_assert(__is_cpp17_move_insertable<_Alloc>::value, - "The specified type does not meet the requirements of Cpp17MoveInsertable"); - typedef allocator_traits<_Alloc> _Traits; - while (__end1 != __begin1) { - _Traits::construct(__a, _VSTD::__to_address(__end2 - 1), -#ifdef _LIBCPP_NO_EXCEPTIONS - _VSTD::move(*--__end1) -#else - _VSTD::move_if_noexcept(*--__end1) -#endif - ); - --__end2; - } -} - -template ::value || !__has_construct<_Alloc, _Tp*, _Tp>::value) && - is_trivially_move_constructible<_Tp>::value ->::type> -_LIBCPP_INLINE_VISIBILITY -void __construct_backward_with_exception_guarantees(_Alloc&, _Tp* __begin1, _Tp* __end1, _Tp*& __end2) { - ptrdiff_t _Np = __end1 - __begin1; - __end2 -= _Np; - if (_Np > 0) - _VSTD::memcpy(static_cast(__end2), static_cast(__begin1), _Np * sizeof(_Tp)); -} - struct __destruct_n { private: @@ -1008,37 +921,6 @@ _LIBCPP_FUNC_VIS void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space); -// --- Helper for container swap -- -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11 -void __swap_allocator(_Alloc & __a1, _Alloc & __a2, true_type) -#if _LIBCPP_STD_VER > 11 - _NOEXCEPT -#else - _NOEXCEPT_(__is_nothrow_swappable<_Alloc>::value) -#endif -{ - using _VSTD::swap; - swap(__a1, __a2); -} - -template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11 -void __swap_allocator(_Alloc &, _Alloc &, false_type) _NOEXCEPT {} - -template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11 -void __swap_allocator(_Alloc & __a1, _Alloc & __a2) -#if _LIBCPP_STD_VER > 11 - _NOEXCEPT -#else - _NOEXCEPT_(__is_nothrow_swappable<_Alloc>::value) -#endif -{ - _VSTD::__swap_allocator(__a1, __a2, - integral_constant::propagate_on_container_swap::value>()); -} - template > struct __noexcept_move_assign_container : public integral_constant #include <__iterator/wrap_iter.h> #include <__memory/allocate_at_least.h> +#include <__memory/swap_allocator.h> #include <__string/char_traits.h> #include <__string/extern_template_lists.h> #include <__utility/auto_cast.h> diff --git a/libcxx/include/vector b/libcxx/include/vector --- a/libcxx/include/vector +++ b/libcxx/include/vector @@ -291,6 +291,8 @@ #include <__iterator/reverse_iterator.h> #include <__iterator/wrap_iter.h> #include <__memory/allocate_at_least.h> +#include <__memory/pointer_traits.h> +#include <__memory/swap_allocator.h> #include <__split_buffer> #include <__utility/forward.h> #include <__utility/move.h> @@ -897,9 +899,14 @@ void vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer& __v) { - __annotate_delete(); - _VSTD::__construct_backward_with_exception_guarantees(this->__alloc(), this->__begin_, this->__end_, __v.__begin_); + using _RevIter = std::reverse_iterator; + __v.__begin_ = + std::__uninitialized_allocator_move_if_noexcept(__alloc(), + _RevIter(__end_), + _RevIter(__begin_), + _RevIter(__v.__begin_)) + .base(); _VSTD::swap(this->__begin_, __v.__begin_); _VSTD::swap(this->__end_, __v.__end_); _VSTD::swap(this->__end_cap(), __v.__end_cap()); @@ -914,8 +921,14 @@ { __annotate_delete(); pointer __r = __v.__begin_; - _VSTD::__construct_backward_with_exception_guarantees(this->__alloc(), this->__begin_, __p, __v.__begin_); - _VSTD::__construct_forward_with_exception_guarantees(this->__alloc(), __p, this->__end_, __v.__end_); + using _RevIter = std::reverse_iterator; + __v.__begin_ = + std::__uninitialized_allocator_move_if_noexcept(__alloc(), + _RevIter(__p), + _RevIter(__begin_), + _RevIter(__v.__begin_)) + .base(); + __v.__end_ = std::__uninitialized_allocator_move_if_noexcept(__alloc(), __p, __end_, __v.__end_); _VSTD::swap(this->__begin_, __v.__begin_); _VSTD::swap(this->__end_, __v.__end_); _VSTD::swap(this->__end_cap(), __v.__end_cap()); @@ -1003,8 +1016,8 @@ >::type vector<_Tp, _Allocator>::__construct_at_end(_ForwardIterator __first, _ForwardIterator __last, size_type __n) { - _ConstructTransaction __tx(*this, __n); - _VSTD::__construct_range_forward(this->__alloc(), __first, __last, __tx.__pos_); + _ConstructTransaction __tx(*this, __n); + __tx.__pos_ = std::__uninitialized_allocator_copy(__alloc(), __first, __last, __tx.__pos_); } // Default constructs __n objects starting at __end_ diff --git a/libcxx/test/libcxx/containers/sequences/vector/asan_throw.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/asan_throw.pass.cpp --- a/libcxx/test/libcxx/containers/sequences/vector/asan_throw.pass.cpp +++ b/libcxx/test/libcxx/containers/sequences/vector/asan_throw.pass.cpp @@ -99,9 +99,9 @@ v.insert(v.end(), a, a + 2); assert(0); } catch (int e) { - assert(v.size() == 3); + assert(v.size() == 2); } - assert(v.size() == 3); + assert(v.size() == 2); assert(is_contiguous_container_asan_correct(v)); } diff --git a/libcxx/test/libcxx/private_headers.verify.cpp b/libcxx/test/libcxx/private_headers.verify.cpp --- a/libcxx/test/libcxx/private_headers.verify.cpp +++ b/libcxx/test/libcxx/private_headers.verify.cpp @@ -370,6 +370,7 @@ #include <__memory/ranges_uninitialized_algorithms.h> // expected-error@*:* {{use of private header from outside its module: '__memory/ranges_uninitialized_algorithms.h'}} #include <__memory/raw_storage_iterator.h> // expected-error@*:* {{use of private header from outside its module: '__memory/raw_storage_iterator.h'}} #include <__memory/shared_ptr.h> // expected-error@*:* {{use of private header from outside its module: '__memory/shared_ptr.h'}} +#include <__memory/swap_allocator.h> // expected-error@*:* {{use of private header from outside its module: '__memory/swap_allocator.h'}} #include <__memory/temporary_buffer.h> // expected-error@*:* {{use of private header from outside its module: '__memory/temporary_buffer.h'}} #include <__memory/uninitialized_algorithms.h> // expected-error@*:* {{use of private header from outside its module: '__memory/uninitialized_algorithms.h'}} #include <__memory/unique_ptr.h> // expected-error@*:* {{use of private header from outside its module: '__memory/unique_ptr.h'}}