diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -341,6 +341,7 @@ __memory/ranges_uninitialized_algorithms.h __memory/raw_storage_iterator.h __memory/shared_ptr.h + __memory/swap_allocator.h __memory/temporary_buffer.h __memory/uninitialized_algorithms.h __memory/unique_ptr.h diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table --- a/libcxx/include/__hash_table +++ b/libcxx/include/__hash_table @@ -18,6 +18,7 @@ #include <__debug> #include <__functional/hash.h> #include <__iterator/iterator_traits.h> +#include <__memory/swap_allocator.h> #include <__utility/swap.h> #include #include diff --git a/libcxx/include/__memory/swap_allocator.h b/libcxx/include/__memory/swap_allocator.h new file mode 100644 --- /dev/null +++ b/libcxx/include/__memory/swap_allocator.h @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___MEMORY_SWAP_ALLOCATOR_H +#define _LIBCPP___MEMORY_SWAP_ALLOCATOR_H + +#include <__config> +#include <__memory/allocator_traits.h> +#include <__type_traits/integral_constant.h> +#include <__utility/swap.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11 void __swap_allocator(_Alloc& __a1, _Alloc& __a2, true_type) +#if _LIBCPP_STD_VER > 11 + _NOEXCEPT +#else + _NOEXCEPT_(__is_nothrow_swappable<_Alloc>::value) +#endif +{ + using _VSTD::swap; + swap(__a1, __a2); +} + +template +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11 void +__swap_allocator(_Alloc&, _Alloc&, false_type) _NOEXCEPT {} + +template +inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11 void __swap_allocator(_Alloc& __a1, _Alloc& __a2) +#if _LIBCPP_STD_VER > 11 + _NOEXCEPT +#else + _NOEXCEPT_(__is_nothrow_swappable<_Alloc>::value) +#endif +{ + _VSTD::__swap_allocator( + __a1, __a2, integral_constant::propagate_on_container_swap::value>()); +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___MEMORY_SWAP_ALLOCATOR_H diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h --- a/libcxx/include/__memory/uninitialized_algorithms.h +++ b/libcxx/include/__memory/uninitialized_algorithms.h @@ -10,11 +10,14 @@ #ifndef _LIBCPP___MEMORY_UNINITIALIZED_ALGORITHMS_H #define _LIBCPP___MEMORY_UNINITIALIZED_ALGORITHMS_H +#include <__algorithm/move.h> #include <__config> #include <__iterator/iterator_traits.h> +#include <__iterator/reverse_iterator.h> #include <__memory/addressof.h> #include <__memory/allocator_traits.h> #include <__memory/construct_at.h> +#include <__memory/pointer_traits.h> #include <__memory/voidify.h> #include <__utility/move.h> #include <__utility/pair.h> @@ -25,6 +28,9 @@ # pragma GCC system_header #endif +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + _LIBCPP_BEGIN_NAMESPACE_STD // This is a simplified version of C++20 `unreachable_sentinel` that doesn't use concepts and thus can be used in any @@ -492,6 +498,91 @@ #endif // _LIBCPP_STD_VER > 14 +// Copy-construct [__first1, __last1) in [__first2, __first2 + distance(__first1, __last1)). The caller has to ensure +// that __first2 can hold at least distance(__first1, __last1) uninitialized elements. If an exception is thrown the +// already copied elements are destroyed again. +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 _Iter2 +__uninitialized_allocator_copy(_Alloc& __alloc, _Iter1 __first1, _Sent1 __last1, _Iter2 __first2) { + auto __destruct_first = __first2; +#ifndef _LIBCPP_NO_EXCEPTIONS + try { +#endif + while (__first1 != __last1) { + allocator_traits<_Alloc>::construct(__alloc, std::__to_address(__first2), *__first1); + ++__first1; + ++__first2; + } +#ifndef _LIBCPP_NO_EXCEPTIONS + } catch (...) { + std::__allocator_destroy_multidimensional(__alloc, __destruct_first, __first2); + throw; + } +#endif + return __first2; +} + +template ::type, + class _RawType2 = typename remove_const<_Type2>::type, + class = __enable_if_t::value && + is_same<_RawType1, _RawType2>::value && + (__is_default_allocator<_Alloc>::value || + !__has_construct<_Alloc, _Type2*, _Type1&>::value)> > +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 _Type2* +__uninitialized_allocator_copy(_Alloc&, _Type1* __first1, _Type1* __last1, _Type2* __first2) { + auto __diff = __last1 - __first1; + ::__builtin_memcpy(const_cast<_RawType1*>(__first2), __first1, __diff * sizeof(_Type1)); + return __first2 + __diff; +} + +// Move-construct the elements [__first1, __last1) into [__first2, __first2 + distance(__first1, __last1)) if the +// move constructor is noexcept. Otherwise try to copy all elements. If an exception is thrown the already copied +// elements are destroyed again. +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 _Iter2 __uninitialized_allocator_move_if_noexcept( + _Alloc& __alloc, _Iter1 __first1, _Sent1 __last1, _Iter2 __first2) { + static_assert(__is_cpp17_move_insertable<_Alloc>::value, + "The specified type does not meet the requirements of Cpp17MoveInsertable"); +#ifndef _LIBCPP_NO_EXCEPTIONS + auto __destruct_first = __first2; + try { +#endif + while (__first1 != __last1) { +#ifndef _LIBCPP_NO_EXCEPTIONS + allocator_traits<_Alloc>::construct(__alloc, std::__to_address(__first2), std::move_if_noexcept(*__first1)); +#else + allocator_traits<_Alloc>::construct(__alloc, std::__to_address(__first2), std::move(*__first1)); +#endif + ++__first1; + ++__first2; + } +#ifndef _LIBCPP_NO_EXCEPTIONS + } catch (...) { + std::__allocator_destroy_multidimensional(__alloc, __destruct_first, __first2); + throw; + } +#endif + return __first2; +} + +template < + class _Alloc, + class _Iter1, + class _Iter2, + class _Type1 = typename iterator_traits<_Iter1>::value_type, + class _Type2 = typename iterator_traits<_Iter2>::value_type, + class = __enable_if_t<(__is_default_allocator<_Alloc>::value || !__has_construct<_Alloc, _Type1*, _Type1>::value) && + is_trivially_move_constructible<_Type1>::value && is_same<_Type1, _Type2>::value> > +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX17 _Iter1 +__uninitialized_allocator_move_if_noexcept(_Alloc&, _Iter1 __first1, _Iter1 __last1, _Iter2 __first2) { + return std::move(__first1, __last1, __first2); +} + _LIBCPP_END_NAMESPACE_STD +_LIBCPP_POP_MACROS + #endif // _LIBCPP___MEMORY_UNINITIALIZED_ALGORITHMS_H diff --git a/libcxx/include/__split_buffer b/libcxx/include/__split_buffer --- a/libcxx/include/__split_buffer +++ b/libcxx/include/__split_buffer @@ -19,6 +19,7 @@ #include <__iterator/move_iterator.h> #include <__memory/allocator.h> #include <__memory/compressed_pair.h> +#include <__memory/swap_allocator.h> #include <__utility/forward.h> #include #include diff --git a/libcxx/include/__tree b/libcxx/include/__tree --- a/libcxx/include/__tree +++ b/libcxx/include/__tree @@ -17,6 +17,7 @@ #include <__iterator/distance.h> #include <__iterator/iterator_traits.h> #include <__iterator/next.h> +#include <__memory/swap_allocator.h> #include <__utility/forward.h> #include <__utility/swap.h> #include diff --git a/libcxx/include/forward_list b/libcxx/include/forward_list --- a/libcxx/include/forward_list +++ b/libcxx/include/forward_list @@ -188,6 +188,7 @@ #include <__iterator/iterator_traits.h> #include <__iterator/move_iterator.h> #include <__iterator/next.h> +#include <__memory/swap_allocator.h> #include <__utility/forward.h> #include #include diff --git a/libcxx/include/list b/libcxx/include/list --- a/libcxx/include/list +++ b/libcxx/include/list @@ -194,6 +194,7 @@ #include <__iterator/next.h> #include <__iterator/prev.h> #include <__iterator/reverse_iterator.h> +#include <__memory/swap_allocator.h> #include <__utility/forward.h> #include <__utility/move.h> #include <__utility/swap.h> diff --git a/libcxx/include/memory b/libcxx/include/memory --- a/libcxx/include/memory +++ b/libcxx/include/memory @@ -880,93 +880,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD -template -_LIBCPP_INLINE_VISIBILITY -void __construct_forward_with_exception_guarantees(_Alloc& __a, _Ptr __begin1, _Ptr __end1, _Ptr& __begin2) { - static_assert(__is_cpp17_move_insertable<_Alloc>::value, - "The specified type does not meet the requirements of Cpp17MoveInsertable"); - typedef allocator_traits<_Alloc> _Traits; - for (; __begin1 != __end1; ++__begin1, (void)++__begin2) { - _Traits::construct(__a, _VSTD::__to_address(__begin2), -#ifdef _LIBCPP_NO_EXCEPTIONS - _VSTD::move(*__begin1) -#else - _VSTD::move_if_noexcept(*__begin1) -#endif - ); - } -} - -template ::value || !__has_construct<_Alloc, _Tp*, _Tp>::value) && - is_trivially_move_constructible<_Tp>::value ->::type> -_LIBCPP_INLINE_VISIBILITY -void __construct_forward_with_exception_guarantees(_Alloc&, _Tp* __begin1, _Tp* __end1, _Tp*& __begin2) { - ptrdiff_t _Np = __end1 - __begin1; - if (_Np > 0) { - _VSTD::memcpy(__begin2, __begin1, _Np * sizeof(_Tp)); - __begin2 += _Np; - } -} - -template -_LIBCPP_INLINE_VISIBILITY -void __construct_range_forward(_Alloc& __a, _Iter __begin1, _Iter __end1, _Ptr& __begin2) { - typedef allocator_traits<_Alloc> _Traits; - for (; __begin1 != __end1; ++__begin1, (void) ++__begin2) { - _Traits::construct(__a, _VSTD::__to_address(__begin2), *__begin1); - } -} - -template ::type, - class _RawDest = typename remove_const<_Dest>::type, - class = - typename enable_if< - is_trivially_copy_constructible<_Dest>::value && - is_same<_RawSource, _RawDest>::value && - (__is_default_allocator<_Alloc>::value || !__has_construct<_Alloc, _Dest*, _Source&>::value) - >::type> -_LIBCPP_INLINE_VISIBILITY -void __construct_range_forward(_Alloc&, _Source* __begin1, _Source* __end1, _Dest*& __begin2) { - ptrdiff_t _Np = __end1 - __begin1; - if (_Np > 0) { - _VSTD::memcpy(const_cast<_RawDest*>(__begin2), __begin1, _Np * sizeof(_Dest)); - __begin2 += _Np; - } -} - -template -_LIBCPP_INLINE_VISIBILITY -void __construct_backward_with_exception_guarantees(_Alloc& __a, _Ptr __begin1, _Ptr __end1, _Ptr& __end2) { - static_assert(__is_cpp17_move_insertable<_Alloc>::value, - "The specified type does not meet the requirements of Cpp17MoveInsertable"); - typedef allocator_traits<_Alloc> _Traits; - while (__end1 != __begin1) { - _Traits::construct(__a, _VSTD::__to_address(__end2 - 1), -#ifdef _LIBCPP_NO_EXCEPTIONS - _VSTD::move(*--__end1) -#else - _VSTD::move_if_noexcept(*--__end1) -#endif - ); - --__end2; - } -} - -template ::value || !__has_construct<_Alloc, _Tp*, _Tp>::value) && - is_trivially_move_constructible<_Tp>::value ->::type> -_LIBCPP_INLINE_VISIBILITY -void __construct_backward_with_exception_guarantees(_Alloc&, _Tp* __begin1, _Tp* __end1, _Tp*& __end2) { - ptrdiff_t _Np = __end1 - __begin1; - __end2 -= _Np; - if (_Np > 0) - _VSTD::memcpy(static_cast(__end2), static_cast(__begin1), _Np * sizeof(_Tp)); -} - struct __destruct_n { private: @@ -1008,37 +921,6 @@ _LIBCPP_FUNC_VIS void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space); -// --- Helper for container swap -- -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11 -void __swap_allocator(_Alloc & __a1, _Alloc & __a2, true_type) -#if _LIBCPP_STD_VER > 11 - _NOEXCEPT -#else - _NOEXCEPT_(__is_nothrow_swappable<_Alloc>::value) -#endif -{ - using _VSTD::swap; - swap(__a1, __a2); -} - -template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11 -void __swap_allocator(_Alloc &, _Alloc &, false_type) _NOEXCEPT {} - -template -inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_AFTER_CXX11 -void __swap_allocator(_Alloc & __a1, _Alloc & __a2) -#if _LIBCPP_STD_VER > 11 - _NOEXCEPT -#else - _NOEXCEPT_(__is_nothrow_swappable<_Alloc>::value) -#endif -{ - _VSTD::__swap_allocator(__a1, __a2, - integral_constant::propagate_on_container_swap::value>()); -} - template > struct __noexcept_move_assign_container : public integral_constant #include <__iterator/wrap_iter.h> #include <__memory/allocate_at_least.h> +#include <__memory/swap_allocator.h> #include <__string/char_traits.h> #include <__string/extern_template_lists.h> #include <__utility/auto_cast.h> diff --git a/libcxx/include/vector b/libcxx/include/vector --- a/libcxx/include/vector +++ b/libcxx/include/vector @@ -291,6 +291,8 @@ #include <__iterator/reverse_iterator.h> #include <__iterator/wrap_iter.h> #include <__memory/allocate_at_least.h> +#include <__memory/pointer_traits.h> +#include <__memory/swap_allocator.h> #include <__split_buffer> #include <__utility/forward.h> #include <__utility/move.h> @@ -897,9 +899,14 @@ void vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer& __v) { - __annotate_delete(); - _VSTD::__construct_backward_with_exception_guarantees(this->__alloc(), this->__begin_, this->__end_, __v.__begin_); + using _RevIter = std::reverse_iterator; + __v.__begin_ = + std::__uninitialized_allocator_move_if_noexcept(__alloc(), + _RevIter(__end_), + _RevIter(__begin_), + _RevIter(__v.__begin_)) + .base(); _VSTD::swap(this->__begin_, __v.__begin_); _VSTD::swap(this->__end_, __v.__end_); _VSTD::swap(this->__end_cap(), __v.__end_cap()); @@ -914,8 +921,14 @@ { __annotate_delete(); pointer __r = __v.__begin_; - _VSTD::__construct_backward_with_exception_guarantees(this->__alloc(), this->__begin_, __p, __v.__begin_); - _VSTD::__construct_forward_with_exception_guarantees(this->__alloc(), __p, this->__end_, __v.__end_); + using _RevIter = std::reverse_iterator; + __v.__begin_ = + std::__uninitialized_allocator_move_if_noexcept(__alloc(), + _RevIter(__p), + _RevIter(__begin_), + _RevIter(__v.__begin_)) + .base(); + __v.__end_ = std::__uninitialized_allocator_move_if_noexcept(__alloc(), __p, __end_, __v.__end_); _VSTD::swap(this->__begin_, __v.__begin_); _VSTD::swap(this->__end_, __v.__end_); _VSTD::swap(this->__end_cap(), __v.__end_cap()); @@ -1003,8 +1016,8 @@ >::type vector<_Tp, _Allocator>::__construct_at_end(_ForwardIterator __first, _ForwardIterator __last, size_type __n) { - _ConstructTransaction __tx(*this, __n); - _VSTD::__construct_range_forward(this->__alloc(), __first, __last, __tx.__pos_); + _ConstructTransaction __tx(*this, __n); + __tx.__pos_ = std::__uninitialized_allocator_copy(__alloc(), __first, __last, __tx.__pos_); } // Default constructs __n objects starting at __end_ diff --git a/libcxx/test/libcxx/containers/sequences/vector/asan_throw.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/asan_throw.pass.cpp --- a/libcxx/test/libcxx/containers/sequences/vector/asan_throw.pass.cpp +++ b/libcxx/test/libcxx/containers/sequences/vector/asan_throw.pass.cpp @@ -99,9 +99,9 @@ v.insert(v.end(), a, a + 2); assert(0); } catch (int e) { - assert(v.size() == 3); + assert(v.size() == 2); } - assert(v.size() == 3); + assert(v.size() == 2); assert(is_contiguous_container_asan_correct(v)); } diff --git a/libcxx/test/libcxx/private_headers.verify.cpp b/libcxx/test/libcxx/private_headers.verify.cpp --- a/libcxx/test/libcxx/private_headers.verify.cpp +++ b/libcxx/test/libcxx/private_headers.verify.cpp @@ -372,6 +372,7 @@ #include <__memory/ranges_uninitialized_algorithms.h> // expected-error@*:* {{use of private header from outside its module: '__memory/ranges_uninitialized_algorithms.h'}} #include <__memory/raw_storage_iterator.h> // expected-error@*:* {{use of private header from outside its module: '__memory/raw_storage_iterator.h'}} #include <__memory/shared_ptr.h> // expected-error@*:* {{use of private header from outside its module: '__memory/shared_ptr.h'}} +#include <__memory/swap_allocator.h> // expected-error@*:* {{use of private header from outside its module: '__memory/swap_allocator.h'}} #include <__memory/temporary_buffer.h> // expected-error@*:* {{use of private header from outside its module: '__memory/temporary_buffer.h'}} #include <__memory/uninitialized_algorithms.h> // expected-error@*:* {{use of private header from outside its module: '__memory/uninitialized_algorithms.h'}} #include <__memory/unique_ptr.h> // expected-error@*:* {{use of private header from outside its module: '__memory/unique_ptr.h'}} diff --git a/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_initializer_list.pass.cpp --- a/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_initializer_list.pass.cpp +++ b/libcxx/test/std/containers/sequences/vector/vector.modifiers/insert_iter_initializer_list.pass.cpp @@ -19,9 +19,42 @@ #include "min_allocator.h" #include "asan_testing.h" -int main(int, char**) -{ - { +int throw_if_zero = 2; +int constructed_count = 0; + +struct ThrowSometimes { + ThrowSometimes() { ++constructed_count; } + ThrowSometimes(const ThrowSometimes&) { + if (--throw_if_zero == 0) + throw 1; + ++constructed_count; + } + ThrowSometimes& operator=(const ThrowSometimes&) { + if (--throw_if_zero == 0) + throw 1; + ++constructed_count; + return *this; + } + ~ThrowSometimes() { --constructed_count; } +}; + +void test_throwing() { + std::vector v; + v.reserve(4); + v.emplace_back(); + v.emplace_back(); + try { + v.insert(v.end(), {ThrowSometimes{}, ThrowSometimes{}}); + assert(false); + } catch (int) { + assert(v.size() == 2); + assert(constructed_count == 2); + } +} + +int main(int, char**) { + test_throwing(); + { std::vector d(10, 1); std::vector::iterator i = d.insert(d.cbegin() + 2, {3, 4, 5, 6}); assert(d.size() == 14); @@ -41,8 +74,8 @@ assert(d[11] == 1); assert(d[12] == 1); assert(d[13] == 1); - } - { + } + { std::vector> d(10, 1); std::vector>::iterator i = d.insert(d.cbegin() + 2, {3, 4, 5, 6}); assert(d.size() == 14); @@ -62,7 +95,7 @@ assert(d[11] == 1); assert(d[12] == 1); assert(d[13] == 1); - } + } return 0; }