Index: libcxx/include/__memory/shared_ptr.h =================================================================== --- libcxx/include/__memory/shared_ptr.h +++ libcxx/include/__memory/shared_ptr.h @@ -1059,7 +1059,7 @@ explicit __unbounded_array_control_block(_Alloc const& __alloc, size_t __count, _Tp const& __arg) : __alloc_(__alloc), __count_(__count) { - std::__uninitialized_allocator_fill_n_multidimensional(__alloc_, std::begin(__data_), __count_, __arg); + std::__uninitialized_allocator_construct_n_multidimensional(__alloc_, __data_, __count_, __arg); } _LIBCPP_HIDE_FROM_ABI @@ -1072,10 +1072,10 @@ // There's currently no way of expressing default initialization in an allocator-aware manner anyway. std::uninitialized_default_construct_n(std::begin(__data_), __count_); } else { - std::__uninitialized_allocator_value_construct_n_multidimensional(__alloc_, std::begin(__data_), __count_); + std::__uninitialized_allocator_construct_n_multidimensional(__alloc_, __data_, __count_); } #else - std::__uninitialized_allocator_value_construct_n_multidimensional(__alloc_, std::begin(__data_), __count_); + std::__uninitialized_allocator_construct_n_multidimensional(__alloc_, __data_, __count_); #endif } @@ -1105,11 +1105,11 @@ std::__reverse_destroy(__data_, __data_ + __count_); } else { __allocator_traits_rebind_t<_Alloc, _Tp> __value_alloc(__alloc_); - std::__allocator_destroy_multidimensional(__value_alloc, __data_, __data_ + __count_); + std::__allocator_destroy_n_multidimensional(__value_alloc, __data_, __count_); } #else __allocator_traits_rebind_t<_Alloc, _Tp> __value_alloc(__alloc_); - std::__allocator_destroy_multidimensional(__value_alloc, __data_, __data_ + __count_); + std::__allocator_destroy_n_multidimensional(__value_alloc, __data_, __count_); #endif } @@ -1163,7 +1163,7 @@ _LIBCPP_HIDE_FROM_ABI explicit __bounded_array_control_block(_Alloc const& __alloc, _Tp const& __arg) : __alloc_(__alloc) { - std::__uninitialized_allocator_fill_n_multidimensional(__alloc_, std::addressof(__data_[0]), _Count, __arg); + std::__uninitialized_allocator_construct_n_multidimensional(__alloc_, __data_, _Count, __arg); } _LIBCPP_HIDE_FROM_ABI @@ -1174,10 +1174,10 @@ // There's currently no way of expressing default initialization in an allocator-aware manner anyway. std::uninitialized_default_construct_n(std::addressof(__data_[0]), _Count); } else { - std::__uninitialized_allocator_value_construct_n_multidimensional(__alloc_, std::addressof(__data_[0]), _Count); + std::__uninitialized_allocator_construct_n_multidimensional(__alloc_, __data_, _Count); } #else - std::__uninitialized_allocator_value_construct_n_multidimensional(__alloc_, std::addressof(__data_[0]), _Count); + std::__uninitialized_allocator_construct_n_multidimensional(__alloc_, __data_, _Count); #endif } @@ -1191,11 +1191,11 @@ std::__reverse_destroy(__data_, __data_ + _Count); } else { __allocator_traits_rebind_t<_Alloc, _Tp> __value_alloc(__alloc_); - std::__allocator_destroy_multidimensional(__value_alloc, __data_, __data_ + _Count); + std::__allocator_destroy_n_multidimensional(__value_alloc, __data_, _Count); } #else __allocator_traits_rebind_t<_Alloc, _Tp> __value_alloc(__alloc_); - std::__allocator_destroy_multidimensional(__value_alloc, __data_, __data_ + _Count); + std::__allocator_destroy_n_multidimensional(__value_alloc, __data_, _Count); #endif } Index: libcxx/include/__memory/uninitialized_algorithms.h =================================================================== --- libcxx/include/__memory/uninitialized_algorithms.h +++ libcxx/include/__memory/uninitialized_algorithms.h @@ -363,6 +363,37 @@ __unreachable_sentinel(), __iter_move); } +template +struct _IsBoundedArray { static constexpr bool value = false; }; +template +struct _IsBoundedArray<_Tp[_Num]> { static constexpr bool value = true; }; + +// For a type T[N][M] calculate T and N*M +template +struct __flatten +{ + using element_type = _Tp; + static constexpr int element_count = 1; +}; + +template +struct __flatten<_Tp[_Num]> +{ + using element_type = typename __flatten<_Tp>::element_type; + static constexpr int element_count = __flatten<_Tp>::element_count * _Num; +}; + +// Get the address of the first element of a possibly nested c-style array +template +_LIBCPP_HIDE_FROM_ABI constexpr +auto* __flat_addressof(_Tp& __t) +{ + if constexpr (_IsBoundedArray<_Tp>::value) + return std::__flat_addressof(__t[0]); + else + return std::addressof(__t); +} + // TODO: Rewrite this to iterate left to right and use reverse_iterators when calling // Destroys every element in the range [first, last) FROM RIGHT TO LEFT using allocator // destruction. If elements are themselves C-style arrays, they are recursively destroyed @@ -370,107 +401,23 @@ // // This function assumes that destructors do not throw, and that the allocator is bound to // the correct type. -template::value ->> -_LIBCPP_HIDE_FROM_ABI -constexpr void __allocator_destroy_multidimensional(_Alloc& __alloc, _BidirIter __first, _BidirIter __last) noexcept { - using _ValueType = typename iterator_traits<_BidirIter>::value_type; - static_assert(is_same_v::value_type, _ValueType>, - "The allocator should already be rebound to the correct type"); - - if (__first == __last) +template::difference_type> +_LIBCPP_HIDE_FROM_ABI constexpr void +__allocator_destroy_n_multidimensional(const _Alloc& __alloc, _Tp* __first, _Size __n) noexcept { + if (__n == 0) return; - if constexpr (is_array_v<_ValueType>) { - static_assert(!__libcpp_is_unbounded_array<_ValueType>::value, - "arrays of unbounded arrays don't exist, but if they did we would mess up here"); - - using _Element = remove_extent_t<_ValueType>; - __allocator_traits_rebind_t<_Alloc, _Element> __elem_alloc(__alloc); - do { - --__last; - decltype(auto) __array = *__last; - std::__allocator_destroy_multidimensional(__elem_alloc, __array, __array + extent_v<_ValueType>); - } while (__last != __first); - } else { - do { - --__last; - allocator_traits<_Alloc>::destroy(__alloc, std::addressof(*__last)); - } while (__last != __first); - } -} + using _FlatTp = typename __flatten<_Tp>::element_type; + __n *= __flatten<_Tp>::element_count; -// Constructs the object at the given location using the allocator's construct method. -// -// If the object being constructed is an array, each element of the array is allocator-constructed, -// recursively. If an exception is thrown during the construction of an array, the initialized -// elements are destroyed in reverse order of initialization using allocator destruction. -// -// This function assumes that the allocator is bound to the correct type. -template -_LIBCPP_HIDE_FROM_ABI -constexpr void __allocator_construct_at_multidimensional(_Alloc& __alloc, _Tp* __loc) { - static_assert(is_same_v::value_type, _Tp>, - "The allocator should already be rebound to the correct type"); - - if constexpr (is_array_v<_Tp>) { - using _Element = remove_extent_t<_Tp>; - __allocator_traits_rebind_t<_Alloc, _Element> __elem_alloc(__alloc); - size_t __i = 0; - _Tp& __array = *__loc; - - // If an exception is thrown, destroy what we have constructed so far in reverse order. - auto __guard = std::__make_exception_guard([&]() { - std::__allocator_destroy_multidimensional(__elem_alloc, __array, __array + __i); - }); - - for (; __i != extent_v<_Tp>; ++__i) { - std::__allocator_construct_at_multidimensional(__elem_alloc, std::addressof(__array[__i])); - } - __guard.__complete(); - } else { - allocator_traits<_Alloc>::construct(__alloc, __loc); - } -} + using _FlatTpAlloc = __allocator_traits_rebind_t<_Alloc, _FlatTp>; + _FlatTpAlloc __ftp_alloc(__alloc); -// Constructs the object at the given location using the allocator's construct method, passing along -// the provided argument. -// -// If the object being constructed is an array, the argument is also assumed to be an array. Each -// each element of the array being constructed is allocator-constructed from the corresponding -// element of the argument array. If an exception is thrown during the construction of an array, -// the initialized elements are destroyed in reverse order of initialization using allocator -// destruction. -// -// This function assumes that the allocator is bound to the correct type. -template -_LIBCPP_HIDE_FROM_ABI -constexpr void __allocator_construct_at_multidimensional(_Alloc& __alloc, _Tp* __loc, _Arg const& __arg) { - static_assert(is_same_v::value_type, _Tp>, - "The allocator should already be rebound to the correct type"); - - if constexpr (is_array_v<_Tp>) { - static_assert(is_array_v<_Arg>, - "Provided non-array initialization argument to __allocator_construct_at_multidimensional when " - "trying to construct an array."); - - using _Element = remove_extent_t<_Tp>; - __allocator_traits_rebind_t<_Alloc, _Element> __elem_alloc(__alloc); - size_t __i = 0; - _Tp& __array = *__loc; - - // If an exception is thrown, destroy what we have constructed so far in reverse order. - auto __guard = std::__make_exception_guard([&]() { - std::__allocator_destroy_multidimensional(__elem_alloc, __array, __array + __i); - }); - for (; __i != extent_v<_Tp>; ++__i) { - std::__allocator_construct_at_multidimensional(__elem_alloc, std::addressof(__array[__i]), __arg[__i]); - } - __guard.__complete(); - } else { - allocator_traits<_Alloc>::construct(__alloc, __loc, __arg); - } + auto* __flat_first = std::__flat_addressof(*__first); + auto* __flat_last = __flat_first + __n; + + while (__flat_first != __flat_last) + allocator_traits<_FlatTpAlloc>::destroy(__ftp_alloc, --__flat_last); } // Given a range starting at it and containing n elements, initializes each element in the @@ -478,37 +425,43 @@ // correct type). // // If an exception is thrown, the initialized elements are destroyed in reverse order of -// initialization using allocator_traits destruction. If the elements in the range are C-style -// arrays, they are initialized element-wise using allocator construction, and recursively so. -template::difference_type> +// initialization using allocator_traits destruction. +// +// If the elements in the range are C-style arrays, they are initialized element-wise +// using allocator construction and treating them as single larger flat array +template::difference_type> _LIBCPP_HIDE_FROM_ABI constexpr void -__uninitialized_allocator_fill_n_multidimensional(_Alloc& __alloc, _BidirIter __it, _Size __n, _Tp const& __value) { - using _ValueType = typename iterator_traits<_BidirIter>::value_type; - __allocator_traits_rebind_t<_Alloc, _ValueType> __value_alloc(__alloc); - _BidirIter __begin = __it; - - // If an exception is thrown, destroy what we have constructed so far in reverse order. - auto __guard = std::__make_exception_guard([&]() { std::__allocator_destroy_multidimensional(__value_alloc, __begin, __it); }); - for (; __n != 0; --__n, ++__it) { - std::__allocator_construct_at_multidimensional(__value_alloc, std::addressof(*__it), __value); - } - __guard.__complete(); -} +__uninitialized_allocator_construct_n_multidimensional(const _Alloc& __alloc, _Tp* __it, _Size __n, _OptArg const&... __opt_arg) { -// Same as __uninitialized_allocator_fill_n_multidimensional, but doesn't pass any initialization argument -// to the allocator's construct method, which results in value initialization. -template ::difference_type> -_LIBCPP_HIDE_FROM_ABI constexpr void -__uninitialized_allocator_value_construct_n_multidimensional(_Alloc& __alloc, _BidirIter __it, _Size __n) { - using _ValueType = typename iterator_traits<_BidirIter>::value_type; - __allocator_traits_rebind_t<_Alloc, _ValueType> __value_alloc(__alloc); - _BidirIter __begin = __it; - - // If an exception is thrown, destroy what we have constructed so far in reverse order. - auto __guard = std::__make_exception_guard([&]() { std::__allocator_destroy_multidimensional(__value_alloc, __begin, __it); }); - for (; __n != 0; --__n, ++__it) { - std::__allocator_construct_at_multidimensional(__value_alloc, std::addressof(*__it)); + // Either default initialization or value initialization + static_assert(sizeof...(_OptArg) <= 1); + + using _FlatTp = typename __flatten<_Tp>::element_type; + __n *= __flatten<_Tp>::element_count; + + using _FlatTpAlloc = __allocator_traits_rebind_t<_Alloc, _FlatTp>; + _FlatTpAlloc __ftp_alloc(__alloc); + + auto* __flat_it = std::__flat_addressof(*__it); + _Size __idx = 0; + + auto __guard = std::__make_exception_guard([&]() { + for (; __idx != 0; --__idx) + allocator_traits<_FlatTpAlloc>::destroy(__ftp_alloc, --__flat_it); + }); + + if constexpr (sizeof...(_OptArg) == 1) { + auto* __fill_ptr = std::__flat_addressof(__opt_arg...); + constexpr int __fill_element_count = __flatten<_OptArg...>::element_count; + + for (; __idx < __n; ++__flat_it, ++__idx) + allocator_traits<_FlatTpAlloc>::construct(__ftp_alloc, __flat_it, __fill_ptr[__idx % __fill_element_count]); + + } else { + for (; __idx < __n; ++__flat_it, ++__idx) + allocator_traits<_FlatTpAlloc>::construct(__ftp_alloc, __flat_it); } + __guard.__complete(); }