diff --git a/libcxx/include/__config b/libcxx/include/__config --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -118,6 +118,12 @@ # endif #endif +#if defined(_LIBCPP_ABI_UNSTABLE) +# if __has_cpp_attribute(no_unique_address) +# define _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS 1 +# endif +#endif + #ifdef _LIBCPP_TRIVIAL_PAIR_COPY_CTOR #error "_LIBCPP_TRIVIAL_PAIR_COPY_CTOR" is no longer supported. \ use _LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR instead diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table --- a/libcxx/include/__hash_table +++ b/libcxx/include/__hash_table @@ -769,39 +769,82 @@ typedef allocator_traits __alloc_traits; typedef typename __alloc_traits::size_type size_type; +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + size_type __size_; + [[no_unique_address]] allocator_type __alloc_; +#else __compressed_pair __data_; +#endif public: typedef typename __alloc_traits::pointer pointer; _LIBCPP_INLINE_VISIBILITY __bucket_list_deallocator() _NOEXCEPT_(is_nothrow_default_constructible::value) - : __data_(0) {} +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __size_(0), __alloc_() +#else + : __data_(0) +#endif + {} _LIBCPP_INLINE_VISIBILITY __bucket_list_deallocator(const allocator_type& __a, size_type __size) _NOEXCEPT_(is_nothrow_copy_constructible::value) - : __data_(__size, __a) {} +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __size_(__size), __alloc_(__a) +#else + : __data_(__size, __a) +#endif + {} #ifndef _LIBCPP_CXX03_LANG _LIBCPP_INLINE_VISIBILITY __bucket_list_deallocator(__bucket_list_deallocator&& __x) _NOEXCEPT_(is_nothrow_move_constructible::value) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __size_(__x.__size_), __alloc_(_VSTD::move(__x.__alloc_)) +#else : __data_(_VSTD::move(__x.__data_)) +#endif { __x.size() = 0; } #endif _LIBCPP_INLINE_VISIBILITY - size_type& size() _NOEXCEPT {return __data_.first();} + size_type& size() _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + return __size_; +#else + return __data_.first(); +#endif + } _LIBCPP_INLINE_VISIBILITY - size_type size() const _NOEXCEPT {return __data_.first();} + size_type size() const _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + return __size_; +#else + return __data_.first(); +#endif + } _LIBCPP_INLINE_VISIBILITY - allocator_type& __alloc() _NOEXCEPT {return __data_.second();} + allocator_type& __alloc() _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + return __alloc_; +#else + return __data_.second(); +#endif + } _LIBCPP_INLINE_VISIBILITY - const allocator_type& __alloc() const _NOEXCEPT {return __data_.second();} + const allocator_type& __alloc() const _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + return __alloc_; +#else + return __data_.second(); +#endif + } _LIBCPP_INLINE_VISIBILITY void operator()(pointer __p) _NOEXCEPT @@ -952,11 +995,58 @@ // --- Member data begin --- __bucket_list __bucket_list_; +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __first_node __first_node_; + size_type __size_; + float __max_load_factor_; + [[no_unique_address]] __node_allocator __alloc_; + [[no_unique_address]] hasher __hash_fn_; + [[no_unique_address]] key_equal __eq_; + + // --- Member data end --- + + _LIBCPP_INLINE_VISIBILITY + __first_node& __first() _NOEXCEPT { return __first_node_; } + _LIBCPP_INLINE_VISIBILITY + const __first_node& __first() const _NOEXCEPT { return __first_node_; } + + _LIBCPP_INLINE_VISIBILITY + size_type& size() _NOEXCEPT {return __size_;} +public: + _LIBCPP_INLINE_VISIBILITY + size_type size() const _NOEXCEPT {return __size_;} + + _LIBCPP_INLINE_VISIBILITY + hasher& hash_function() _NOEXCEPT {return __hash_fn_;} + _LIBCPP_INLINE_VISIBILITY + const hasher& hash_function() const _NOEXCEPT {return __hash_fn_;} + + _LIBCPP_INLINE_VISIBILITY + float& max_load_factor() _NOEXCEPT {return __max_load_factor_;} + _LIBCPP_INLINE_VISIBILITY + float max_load_factor() const _NOEXCEPT {return __max_load_factor_;} + + _LIBCPP_INLINE_VISIBILITY + key_equal& key_eq() _NOEXCEPT {return __eq_;} + _LIBCPP_INLINE_VISIBILITY + const key_equal& key_eq() const _NOEXCEPT {return __eq_;} + + _LIBCPP_INLINE_VISIBILITY + __node_allocator& __node_alloc() _NOEXCEPT {return __alloc_;} + _LIBCPP_INLINE_VISIBILITY + const __node_allocator& __node_alloc() const _NOEXCEPT {return __alloc_;} +#else __compressed_pair<__first_node, __node_allocator> __p1_; __compressed_pair __p2_; __compressed_pair __p3_; + // --- Member data end --- + _LIBCPP_INLINE_VISIBILITY + __first_node& __first() _NOEXCEPT { return __p1_.first(); } + _LIBCPP_INLINE_VISIBILITY + const __first_node& __first() const _NOEXCEPT { return __p1_.first(); } + _LIBCPP_INLINE_VISIBILITY size_type& size() _NOEXCEPT {return __p2_.first();} public: @@ -983,6 +1073,7 @@ _LIBCPP_INLINE_VISIBILITY const __node_allocator& __node_alloc() const _NOEXCEPT {return __p1_.second();} +#endif public: typedef __hash_iterator<__node_pointer> iterator; @@ -1416,19 +1507,34 @@ is_nothrow_default_constructible<__node_allocator>::value && is_nothrow_default_constructible::value && is_nothrow_default_constructible::value) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __size_(0), + __max_load_factor_(1.0f), + __alloc_(), + __hash_fn_(), + __eq_() +#else : __p2_(0), __p3_(1.0f) +#endif { } template -inline -__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const hasher& __hf, - const key_equal& __eql) +inline __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table( + const hasher &__hf, const key_equal &__eql) : __bucket_list_(nullptr, __bucket_list_deleter()), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __size_(0), + __max_load_factor_(1.0f), + __alloc_(), + __hash_fn_(__hf), + __eq_(__eql) +#else __p1_(), __p2_(0, __hf), __p3_(1.0f, __eql) +#endif { } @@ -1437,18 +1543,34 @@ const key_equal& __eql, const allocator_type& __a) : __bucket_list_(nullptr, __bucket_list_deleter(__pointer_allocator(__a), 0)), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __size_(0), + __max_load_factor_(1.0f), + __alloc_(__a), + __hash_fn_(__hf), + __eq_(__eql) +#else __p1_(__second_tag(), __node_allocator(__a)), __p2_(0, __hf), __p3_(1.0f, __eql) +#endif { } template __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const allocator_type& __a) : __bucket_list_(nullptr, __bucket_list_deleter(__pointer_allocator(__a), 0)), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __size_(0), + __max_load_factor_(1.0f), + __alloc_(__a), + __hash_fn_(), + __eq_() +#else __p1_(__second_tag(), __node_allocator(__a)), __p2_(0), __p3_(1.0f) +#endif { } @@ -1458,10 +1580,19 @@ __bucket_list_deleter(allocator_traits<__pointer_allocator>:: select_on_container_copy_construction( __u.__bucket_list_.get_deleter().__alloc()), 0)), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __size_(0), + __max_load_factor_(__u.__max_load_factor_), + __alloc_(allocator_traits<__node_allocator>:: + select_on_container_copy_construction(__u.__node_alloc())), + __hash_fn_(__u.__hash_fn_), + __eq_(__u.__eq_) +#else __p1_(__second_tag(), allocator_traits<__node_allocator>:: select_on_container_copy_construction(__u.__node_alloc())), __p2_(0, __u.hash_function()), __p3_(__u.__p3_) +#endif { } @@ -1469,9 +1600,17 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __u, const allocator_type& __a) : __bucket_list_(nullptr, __bucket_list_deleter(__pointer_allocator(__a), 0)), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __size_(0), + __max_load_factor_(__u.__max_load_factor_), + __alloc_(__a), + __hash_fn_(__u.__hash_fn_), + __eq_(__u.__eq_) +#else __p1_(__second_tag(), __node_allocator(__a)), __p2_(0, __u.hash_function()), __p3_(__u.__p3_) +#endif { } @@ -1486,15 +1625,24 @@ is_nothrow_move_constructible::value && is_nothrow_move_constructible::value) : __bucket_list_(_VSTD::move(__u.__bucket_list_)), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __first_node_(_VSTD::move(__u.__first_node_)), + __size_(__u.__size_), + __max_load_factor_(__u.__max_load_factor_), + __alloc_(_VSTD::move(__u.__alloc_)), + __hash_fn_(_VSTD::move(__u.__hash_fn_)), + __eq_(_VSTD::move(__u.__eq_)) +#else __p1_(_VSTD::move(__u.__p1_)), __p2_(_VSTD::move(__u.__p2_)), __p3_(_VSTD::move(__u.__p3_)) +#endif { if (size() > 0) { - __bucket_list_[__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = - __p1_.first().__ptr(); - __u.__p1_.first().__next_ = nullptr; + __bucket_list_[__constrain_hash(__first().__next_->__hash(), bucket_count())] = + __first().__ptr(); + __u.__first().__next_ = nullptr; __u.size() = 0; } } @@ -1503,9 +1651,17 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(__hash_table&& __u, const allocator_type& __a) : __bucket_list_(nullptr, __bucket_list_deleter(__pointer_allocator(__a), 0)), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __size_(0), + __max_load_factor_(__u.__max_load_factor_), + __alloc_(__a), + __hash_fn_(_VSTD::move(__u.__hash_fn_)), + __eq_(_VSTD::move(__u.__eq_)) +#else __p1_(__second_tag(), __node_allocator(__a)), __p2_(0, _VSTD::move(__u.hash_function())), __p3_(_VSTD::move(__u.__p3_)) +#endif { if (__a == allocator_type(__u.__node_alloc())) { @@ -1514,10 +1670,10 @@ __u.__bucket_list_.get_deleter().size() = 0; if (__u.size() > 0) { - __p1_.first().__next_ = __u.__p1_.first().__next_; - __u.__p1_.first().__next_ = nullptr; - __bucket_list_[__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = - __p1_.first().__ptr(); + __first().__next_ = __u.__first().__next_; + __u.__first().__next_ = nullptr; + __bucket_list_[__constrain_hash(__first().__next_->__hash(), bucket_count())] = + __first().__ptr(); size() = __u.size(); __u.size() = 0; } @@ -1536,7 +1692,7 @@ "Hasher must be copy-constructible."); #endif - __deallocate_node(__p1_.first().__next_); + __deallocate_node(__first().__next_); #if _LIBCPP_DEBUG_LEVEL >= 2 __get_db()->__erase_c(this); #endif @@ -1611,8 +1767,8 @@ for (size_type __i = 0; __i < __bc; ++__i) __bucket_list_[__i] = nullptr; size() = 0; - __next_pointer __cache = __p1_.first().__next_; - __p1_.first().__next_ = nullptr; + __next_pointer __cache = __first().__next_; + __first().__next_ = nullptr; return __cache; } @@ -1636,12 +1792,12 @@ hash_function() = _VSTD::move(__u.hash_function()); max_load_factor() = __u.max_load_factor(); key_eq() = _VSTD::move(__u.key_eq()); - __p1_.first().__next_ = __u.__p1_.first().__next_; + __first().__next_ = __u.__first().__next_; if (size() > 0) { - __bucket_list_[__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = - __p1_.first().__ptr(); - __u.__p1_.first().__next_ = nullptr; + __bucket_list_[__constrain_hash(__first().__next_->__hash(), bucket_count())] = + __first().__ptr(); + __u.__first().__next_ = nullptr; __u.size() = 0; } #if _LIBCPP_DEBUG_LEVEL >= 2 @@ -1799,9 +1955,9 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::begin() _NOEXCEPT { #if _LIBCPP_DEBUG_LEVEL >= 2 - return iterator(__p1_.first().__next_, this); + return iterator(__first().__next_, this); #else - return iterator(__p1_.first().__next_); + return iterator(__first().__next_); #endif } @@ -1823,9 +1979,9 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::begin() const _NOEXCEPT { #if _LIBCPP_DEBUG_LEVEL >= 2 - return const_iterator(__p1_.first().__next_, this); + return const_iterator(__first().__next_, this); #else - return const_iterator(__p1_.first().__next_); + return const_iterator(__first().__next_); #endif } @@ -1847,8 +2003,8 @@ { if (size() > 0) { - __deallocate_node(__p1_.first().__next_); - __p1_.first().__next_ = nullptr; + __deallocate_node(__first().__next_); + __first().__next_ = nullptr; size_type __bc = bucket_count(); for (size_type __i = 0; __i < __bc; ++__i) __bucket_list_[__i] = nullptr; @@ -1911,7 +2067,7 @@ __next_pointer __pn = __bucket_list_[__chash]; if (__pn == nullptr) { - __pn =__p1_.first().__ptr(); + __pn =__first().__ptr(); __nd->__next_ = __pn->__next_; __pn->__next_ = __nd->__ptr(); // fix up __bucket_list_ @@ -2009,7 +2165,7 @@ size_t __chash = __constrain_hash(__cp->__hash_, __bc); if (__pn == nullptr) { - __pn =__p1_.first().__ptr(); + __pn =__first().__ptr(); __cp->__next_ = __pn->__next_; __pn->__next_ = __cp->__ptr(); // fix up __bucket_list_ @@ -2137,7 +2293,7 @@ __next_pointer __pn = __bucket_list_[__chash]; if (__pn == nullptr) { - __pn = __p1_.first().__ptr(); + __pn = __first().__ptr(); __h->__next_ = __pn->__next_; __pn->__next_ = __h.get()->__ptr(); // fix up __bucket_list_ @@ -2408,7 +2564,7 @@ { for (size_type __i = 0; __i < __nbc; ++__i) __bucket_list_[__i] = nullptr; - __next_pointer __pp = __p1_.first().__ptr(); + __next_pointer __pp = __first().__ptr(); __next_pointer __cp = __pp->__next_; if (__cp != nullptr) { @@ -2671,7 +2827,7 @@ // Fix up __bucket_list_ // if __pn is not in same bucket (before begin is not in same bucket) && // if __cn->__next_ is not in same bucket (nullptr is not in same bucket) - if (__pn == __p1_.first().__ptr() + if (__pn == __first().__ptr() || __constrain_hash(__pn->__hash(), __bc) != __chash) { if (__cn->__next_ == nullptr @@ -2827,19 +2983,22 @@ __bucket_list_.reset(__u.__bucket_list_.release()); __u.__bucket_list_.reset(__npp); } - _VSTD::swap(__bucket_list_.get_deleter().size(), __u.__bucket_list_.get_deleter().size()); + using _VSTD::swap; + swap(__bucket_list_.get_deleter().size(), __u.__bucket_list_.get_deleter().size()); __swap_allocator(__bucket_list_.get_deleter().__alloc(), __u.__bucket_list_.get_deleter().__alloc()); __swap_allocator(__node_alloc(), __u.__node_alloc()); - _VSTD::swap(__p1_.first().__next_, __u.__p1_.first().__next_); - __p2_.swap(__u.__p2_); - __p3_.swap(__u.__p3_); + swap(__first().__next_, __u.__first().__next_); + swap(size(), __u.size()); + swap(hash_function(), __u.hash_function()); + swap(max_load_factor(), __u.max_load_factor()); + swap(key_eq(), __u.key_eq()); if (size() > 0) - __bucket_list_[__constrain_hash(__p1_.first().__next_->__hash(), bucket_count())] = - __p1_.first().__ptr(); + __bucket_list_[__constrain_hash(__first().__next_->__hash(), bucket_count())] = + __first().__ptr(); if (__u.size() > 0) - __u.__bucket_list_[__constrain_hash(__u.__p1_.first().__next_->__hash(), __u.bucket_count())] = - __u.__p1_.first().__ptr(); + __u.__bucket_list_[__constrain_hash(__u.__first().__next_->__hash(), __u.bucket_count())] = + __u.__first().__ptr(); #if _LIBCPP_DEBUG_LEVEL >= 2 __get_db()->swap(this, &__u); #endif diff --git a/libcxx/include/__split_buffer b/libcxx/include/__split_buffer --- a/libcxx/include/__split_buffer +++ b/libcxx/include/__split_buffer @@ -48,15 +48,27 @@ pointer __first_; pointer __begin_; pointer __end_; +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + pointer __end_cap_; + [[no_unique_address]] allocator_type __alloc_; +#else __compressed_pair __end_cap_; +#endif typedef typename add_lvalue_reference::type __alloc_ref; typedef typename add_lvalue_reference::type __alloc_const_ref; +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + _LIBCPP_INLINE_VISIBILITY __alloc_rr& __alloc() _NOEXCEPT {return __alloc_;} + _LIBCPP_INLINE_VISIBILITY const __alloc_rr& __alloc() const _NOEXCEPT {return __alloc_;} + _LIBCPP_INLINE_VISIBILITY pointer& __end_cap() _NOEXCEPT {return __end_cap_;} + _LIBCPP_INLINE_VISIBILITY const pointer& __end_cap() const _NOEXCEPT {return __end_cap_;} +#else _LIBCPP_INLINE_VISIBILITY __alloc_rr& __alloc() _NOEXCEPT {return __end_cap_.second();} _LIBCPP_INLINE_VISIBILITY const __alloc_rr& __alloc() const _NOEXCEPT {return __end_cap_.second();} _LIBCPP_INLINE_VISIBILITY pointer& __end_cap() _NOEXCEPT {return __end_cap_.first();} _LIBCPP_INLINE_VISIBILITY const pointer& __end_cap() const _NOEXCEPT {return __end_cap_.first();} +#endif _LIBCPP_INLINE_VISIBILITY __split_buffer() @@ -306,7 +318,11 @@ template __split_buffer<_Tp, _Allocator>::__split_buffer(size_type __cap, size_type __start, __alloc_rr& __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __end_cap_(nullptr), __alloc_(__a) +#else : __end_cap_(nullptr, __a) +#endif { __first_ = __cap != 0 ? __alloc_traits::allocate(__alloc(), __cap) : nullptr; __begin_ = __end_ = __first_ + __start; @@ -318,20 +334,33 @@ __split_buffer<_Tp, _Allocator>::__split_buffer() _NOEXCEPT_(is_nothrow_default_constructible::value) : __first_(nullptr), __begin_(nullptr), __end_(nullptr), __end_cap_(nullptr) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + , __alloc_() +#endif { } template inline __split_buffer<_Tp, _Allocator>::__split_buffer(__alloc_rr& __a) - : __first_(nullptr), __begin_(nullptr), __end_(nullptr), __end_cap_(nullptr, __a) + : __first_(nullptr), __begin_(nullptr), __end_(nullptr), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __end_cap_(nullptr), __alloc_(__a) +#else + __end_cap_(nullptr, __a) +#endif { } template inline __split_buffer<_Tp, _Allocator>::__split_buffer(const __alloc_rr& __a) - : __first_(nullptr), __begin_(nullptr), __end_(nullptr), __end_cap_(nullptr, __a) + : __first_(nullptr), __begin_(nullptr), __end_(nullptr), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __end_cap_(nullptr), __alloc_(__a) +#else + __end_cap_(nullptr, __a) +#endif { } @@ -352,6 +381,9 @@ __begin_(_VSTD::move(__c.__begin_)), __end_(_VSTD::move(__c.__end_)), __end_cap_(_VSTD::move(__c.__end_cap_)) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + , __alloc_(_VSTD::move(__c.__alloc_)) +#endif { __c.__first_ = nullptr; __c.__begin_ = nullptr; @@ -361,7 +393,11 @@ template __split_buffer<_Tp, _Allocator>::__split_buffer(__split_buffer&& __c, const __alloc_rr& __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __alloc_(__a) +#else : __end_cap_(__second_tag(), __a) +#endif { if (__a == __c.__alloc()) { diff --git a/libcxx/include/__tree b/libcxx/include/__tree --- a/libcxx/include/__tree +++ b/libcxx/include/__tree @@ -1030,53 +1030,84 @@ private: __iter_pointer __begin_node_; +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __end_node_t __end_node_; + size_type __size_; + [[no_unique_address]] __node_allocator __alloc_; + [[no_unique_address]] value_compare __comp_; + + _LIBCPP_INLINE_VISIBILITY + __end_node_t& __end_node_impl() _NOEXCEPT {return __end_node_;} + _LIBCPP_INLINE_VISIBILITY + const __end_node_t& __end_node_impl() const _NOEXCEPT {return __end_node_;} + _LIBCPP_INLINE_VISIBILITY + size_type& size() _NOEXCEPT {return __size_;} +public: + _LIBCPP_INLINE_VISIBILITY + const size_type& size() const _NOEXCEPT {return __size_;} + _LIBCPP_INLINE_VISIBILITY + value_compare& value_comp() _NOEXCEPT {return __comp_;} + _LIBCPP_INLINE_VISIBILITY + const value_compare& value_comp() const _NOEXCEPT {return __comp_;} + _LIBCPP_INLINE_VISIBILITY + __node_allocator& __node_alloc() _NOEXCEPT {return __alloc_;} +private: + _LIBCPP_INLINE_VISIBILITY + const __node_allocator& __node_alloc() const _NOEXCEPT {return __alloc_;} + +#else __compressed_pair<__end_node_t, __node_allocator> __pair1_; __compressed_pair __pair3_; + _LIBCPP_INLINE_VISIBILITY + __end_node_t& __end_node_impl() _NOEXCEPT {return __pair1_.first();} + _LIBCPP_INLINE_VISIBILITY + const __end_node_t& __end_node_impl() const _NOEXCEPT {return __pair1_.first();} + _LIBCPP_INLINE_VISIBILITY + size_type& size() _NOEXCEPT {return __pair3_.first();} public: _LIBCPP_INLINE_VISIBILITY - __iter_pointer __end_node() _NOEXCEPT - { - return static_cast<__iter_pointer>( - pointer_traits<__end_node_ptr>::pointer_to(__pair1_.first()) - ); - } + const size_type& size() const _NOEXCEPT {return __pair3_.first();} _LIBCPP_INLINE_VISIBILITY - __iter_pointer __end_node() const _NOEXCEPT - { - return static_cast<__iter_pointer>( - pointer_traits<__end_node_ptr>::pointer_to( - const_cast<__end_node_t&>(__pair1_.first()) - ) - ); - } + value_compare& value_comp() _NOEXCEPT {return __pair3_.second();} + _LIBCPP_INLINE_VISIBILITY + const value_compare& value_comp() const _NOEXCEPT {return __pair3_.second();} _LIBCPP_INLINE_VISIBILITY __node_allocator& __node_alloc() _NOEXCEPT {return __pair1_.second();} private: _LIBCPP_INLINE_VISIBILITY - const __node_allocator& __node_alloc() const _NOEXCEPT - {return __pair1_.second();} + const __node_allocator& __node_alloc() const _NOEXCEPT {return __pair1_.second();} +#endif + +private: _LIBCPP_INLINE_VISIBILITY __iter_pointer& __begin_node() _NOEXCEPT {return __begin_node_;} _LIBCPP_INLINE_VISIBILITY const __iter_pointer& __begin_node() const _NOEXCEPT {return __begin_node_;} + public: _LIBCPP_INLINE_VISIBILITY allocator_type __alloc() const _NOEXCEPT {return allocator_type(__node_alloc());} -private: - _LIBCPP_INLINE_VISIBILITY - size_type& size() _NOEXCEPT {return __pair3_.first();} -public: - _LIBCPP_INLINE_VISIBILITY - const size_type& size() const _NOEXCEPT {return __pair3_.first();} + _LIBCPP_INLINE_VISIBILITY - value_compare& value_comp() _NOEXCEPT {return __pair3_.second();} + __iter_pointer __end_node() _NOEXCEPT + { + return static_cast<__iter_pointer>( + pointer_traits<__end_node_ptr>::pointer_to(__end_node_impl()) + ); + } _LIBCPP_INLINE_VISIBILITY - const value_compare& value_comp() const _NOEXCEPT - {return __pair3_.second();} -public: + __iter_pointer __end_node() const _NOEXCEPT + { + return static_cast<__iter_pointer>( + pointer_traits<__end_node_ptr>::pointer_to( + const_cast<__end_node_t&>(__end_node_impl()) + ) + ); + } +public: _LIBCPP_INLINE_VISIBILITY __node_pointer __root() const _NOEXCEPT {return static_cast<__node_pointer>(__end_node()->__left_);} @@ -1527,7 +1558,11 @@ _NOEXCEPT_( is_nothrow_default_constructible<__node_allocator>::value && is_nothrow_copy_constructible::value) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __size_(0), __alloc_(), __comp_(__comp) +#else : __pair3_(0, __comp) +#endif { __begin_node() = __end_node(); } @@ -1535,8 +1570,12 @@ template __tree<_Tp, _Compare, _Allocator>::__tree(const allocator_type& __a) : __begin_node_(__iter_pointer()), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __size_(0), __alloc_(__a), __comp_() +#else __pair1_(__second_tag(), __node_allocator(__a)), __pair3_(0) +#endif { __begin_node() = __end_node(); } @@ -1545,8 +1584,12 @@ __tree<_Tp, _Compare, _Allocator>::__tree(const value_compare& __comp, const allocator_type& __a) : __begin_node_(__iter_pointer()), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __size_(0), __alloc_(__a), __comp_(__comp) +#else __pair1_(__second_tag(), __node_allocator(__a)), __pair3_(0, __comp) +#endif { __begin_node() = __end_node(); } @@ -1702,8 +1745,14 @@ template __tree<_Tp, _Compare, _Allocator>::__tree(const __tree& __t) : __begin_node_(__iter_pointer()), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __size_(0), + __alloc_(__node_traits::select_on_container_copy_construction(__t.__node_alloc())), + __comp_(__t.__comp_) +#else __pair1_(__second_tag(), __node_traits::select_on_container_copy_construction(__t.__node_alloc())), __pair3_(0, __t.value_comp()) +#endif { __begin_node() = __end_node(); } @@ -1716,8 +1765,15 @@ is_nothrow_move_constructible<__node_allocator>::value && is_nothrow_move_constructible::value) : __begin_node_(_VSTD::move(__t.__begin_node_)), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __end_node_(_VSTD::move(__t.__end_node_)), + __size_(__t.__size_), + __alloc_(_VSTD::move(__t.__alloc_)), + __comp_(_VSTD::move(__t.__comp_)) +#else __pair1_(_VSTD::move(__t.__pair1_)), __pair3_(_VSTD::move(__t.__pair3_)) +#endif { if (size() == 0) __begin_node() = __end_node(); @@ -1732,8 +1788,12 @@ template __tree<_Tp, _Compare, _Allocator>::__tree(__tree&& __t, const allocator_type& __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __size_(0), __alloc_(__a), __comp_(_VSTD::move(__t.__comp_)) +#else : __pair1_(__second_tag(), __node_allocator(__a)), __pair3_(0, _VSTD::move(__t.value_comp())) +#endif { if (__a == __t.__alloc()) { @@ -1764,9 +1824,10 @@ { destroy(static_cast<__node_pointer>(__end_node()->__left_)); __begin_node_ = __t.__begin_node_; - __pair1_.first() = __t.__pair1_.first(); + __end_node_impl() = __t.__end_node_impl(); __move_assign_alloc(__t); - __pair3_ = _VSTD::move(__t.__pair3_); + size() = __t.size(); + value_comp() = _VSTD::move(__t.value_comp()); if (size() == 0) __begin_node() = __end_node(); else @@ -1877,9 +1938,10 @@ { using _VSTD::swap; swap(__begin_node_, __t.__begin_node_); - swap(__pair1_.first(), __t.__pair1_.first()); + swap(__end_node_impl(), __t.__end_node_impl()); __swap_allocator(__node_alloc(), __t.__node_alloc()); - __pair3_.swap(__t.__pair3_); + swap(size(), __t.size()); + swap(value_comp(), __t.value_comp()); if (size() == 0) __begin_node() = __end_node(); else diff --git a/libcxx/include/deque b/libcxx/include/deque --- a/libcxx/include/deque +++ b/libcxx/include/deque @@ -959,19 +959,33 @@ protected: __map __map_; size_type __start_; +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + size_type __size_; + [[no_unique_address]] allocator_type __alloc_; +#else __compressed_pair __size_; +#endif iterator begin() _NOEXCEPT; const_iterator begin() const _NOEXCEPT; iterator end() _NOEXCEPT; const_iterator end() const _NOEXCEPT; +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + _LIBCPP_INLINE_VISIBILITY size_type& size() {return __size_;} + _LIBCPP_INLINE_VISIBILITY + const size_type& size() const _NOEXCEPT {return __size_;} + _LIBCPP_INLINE_VISIBILITY allocator_type& __alloc() {return __alloc_;} + _LIBCPP_INLINE_VISIBILITY + const allocator_type& __alloc() const _NOEXCEPT {return __alloc_;} +#else _LIBCPP_INLINE_VISIBILITY size_type& size() {return __size_.first();} _LIBCPP_INLINE_VISIBILITY const size_type& size() const _NOEXCEPT {return __size_.first();} _LIBCPP_INLINE_VISIBILITY allocator_type& __alloc() {return __size_.second();} _LIBCPP_INLINE_VISIBILITY const allocator_type& __alloc() const _NOEXCEPT {return __size_.second();} +#endif _LIBCPP_INLINE_VISIBILITY __deque_base() @@ -1103,12 +1117,22 @@ inline __deque_base<_Tp, _Allocator>::__deque_base() _NOEXCEPT_(is_nothrow_default_constructible::value) - : __start_(0), __size_(0) {} + : __start_(0), __size_(0) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + , __alloc_() +#endif +{} template inline __deque_base<_Tp, _Allocator>::__deque_base(const allocator_type& __a) - : __map_(__pointer_allocator(__a)), __start_(0), __size_(0, __a) {} + : __map_(__pointer_allocator(__a)), __start_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __size_(0), __alloc_(__a) +#else + __size_(0, __a) +#endif +{} template __deque_base<_Tp, _Allocator>::~__deque_base() @@ -1128,6 +1152,9 @@ : __map_(_VSTD::move(__c.__map_)), __start_(_VSTD::move(__c.__start_)), __size_(_VSTD::move(__c.__size_)) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + , __alloc_(_VSTD::move(__c.__alloc_)) +#endif { __c.__start_ = 0; __c.size() = 0; @@ -1137,7 +1164,12 @@ __deque_base<_Tp, _Allocator>::__deque_base(__deque_base&& __c, const allocator_type& __a) : __map_(_VSTD::move(__c.__map_), __pointer_allocator(__a)), __start_(_VSTD::move(__c.__start_)), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __size_(_VSTD::move(__c.size())), + __alloc_(__a) +#else __size_(_VSTD::move(__c.size()), __a) +#endif { if (__a == __c.__alloc()) { diff --git a/libcxx/include/forward_list b/libcxx/include/forward_list --- a/libcxx/include/forward_list +++ b/libcxx/include/forward_list @@ -466,6 +466,22 @@ "internal allocator type must differ from user-specified " "type; otherwise overload resolution breaks"); +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __begin_node __before_begin_; + [[no_unique_address]] __node_allocator __alloc_; + + _LIBCPP_INLINE_VISIBILITY + __begin_node_pointer __before_begin() _NOEXCEPT + {return pointer_traits<__begin_node_pointer>::pointer_to(__before_begin_);} + _LIBCPP_INLINE_VISIBILITY + __begin_node_pointer __before_begin() const _NOEXCEPT + {return pointer_traits<__begin_node_pointer>::pointer_to(const_cast<__begin_node&>(__before_begin_));} + + _LIBCPP_INLINE_VISIBILITY + __node_allocator& __alloc() _NOEXCEPT {return __alloc_;} + _LIBCPP_INLINE_VISIBILITY + const __node_allocator& __alloc() const _NOEXCEPT {return __alloc_;} +#else __compressed_pair<__begin_node, __node_allocator> __before_begin_; _LIBCPP_INLINE_VISIBILITY @@ -481,6 +497,7 @@ _LIBCPP_INLINE_VISIBILITY const __node_allocator& __alloc() const _NOEXCEPT {return __before_begin_.second();} +#endif typedef __forward_list_iterator<__node_pointer> iterator; typedef __forward_list_const_iterator<__node_pointer> const_iterator; @@ -488,13 +505,27 @@ _LIBCPP_INLINE_VISIBILITY __forward_list_base() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value) - : __before_begin_(__begin_node()) {} + : __before_begin_(__begin_node()) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + , __alloc_() +#endif + {} _LIBCPP_INLINE_VISIBILITY explicit __forward_list_base(const allocator_type& __a) - : __before_begin_(__begin_node(), __node_allocator(__a)) {} +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __before_begin_(__begin_node()), __alloc_(__a) +#else + : __before_begin_(__begin_node(), __node_allocator(__a)) +#endif + {} _LIBCPP_INLINE_VISIBILITY explicit __forward_list_base(const __node_allocator& __a) - : __before_begin_(__begin_node(), __a) {} +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __before_begin_(__begin_node()), __alloc_(__a) +#else + : __before_begin_(__begin_node(), __a) +#endif + {} #ifndef _LIBCPP_CXX03_LANG public: _LIBCPP_INLINE_VISIBILITY @@ -563,6 +594,9 @@ __forward_list_base<_Tp, _Alloc>::__forward_list_base(__forward_list_base&& __x) _NOEXCEPT_(is_nothrow_move_constructible<__node_allocator>::value) : __before_begin_(_VSTD::move(__x.__before_begin_)) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + , __alloc_(_VSTD::move(__x.__alloc_)) +#endif { __x.__before_begin()->__next_ = nullptr; } @@ -571,7 +605,11 @@ inline __forward_list_base<_Tp, _Alloc>::__forward_list_base(__forward_list_base&& __x, const allocator_type& __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __before_begin_(__begin_node()), __alloc_(__a) +#else : __before_begin_(__begin_node(), __node_allocator(__a)) +#endif { if (__alloc() == __x.__alloc()) { diff --git a/libcxx/include/functional b/libcxx/include/functional --- a/libcxx/include/functional +++ b/libcxx/include/functional @@ -1482,6 +1482,73 @@ template class __default_alloc_func; +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS +template +class __alloc_func<_Fp, _Ap, _Rp(_ArgTypes...)> +{ + [[no_unique_address]] _Fp __target_; + [[no_unique_address]] _Ap __alloc_; + + public: + typedef _Fp _Target; + typedef _Ap _Alloc; + + _LIBCPP_INLINE_VISIBILITY + const _Target& __target() const { return __target_; } + + // WIN32 APIs may define __allocator, so use __get_allocator instead. + _LIBCPP_INLINE_VISIBILITY + const _Alloc& __get_allocator() const { return __alloc_; } + + _LIBCPP_INLINE_VISIBILITY + explicit __alloc_func(_Target&& __f) + : __target_(_VSTD::move(__f)), __alloc_() + { + } + + _LIBCPP_INLINE_VISIBILITY + explicit __alloc_func(const _Target& __f, const _Alloc& __a) + : __target_(__f), __alloc_(__a) + { + } + + _LIBCPP_INLINE_VISIBILITY + explicit __alloc_func(const _Target& __f, _Alloc&& __a) + : __target_(__f), __alloc_(_VSTD::move(__a)) + { + } + + _LIBCPP_INLINE_VISIBILITY + explicit __alloc_func(_Target&& __f, _Alloc&& __a) + : __target_(_VSTD::move(__f)), __alloc_(_VSTD::move(__a)) + { + } + + _LIBCPP_INLINE_VISIBILITY + _Rp operator()(_ArgTypes&&... __arg) + { + typedef __invoke_void_return_wrapper<_Rp> _Invoker; + return _Invoker::__call(__target_, _VSTD::forward<_ArgTypes>(__arg)...); + } + + _LIBCPP_INLINE_VISIBILITY + __alloc_func* __clone() const + { + typedef allocator_traits<_Alloc> __alloc_traits; + typedef + typename __rebind_alloc_helper<__alloc_traits, __alloc_func>::type + _AA; + _AA __a(__alloc_); + typedef __allocator_destructor<_AA> _Dp; + unique_ptr<__alloc_func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); + ::new ((void*)__hold.get()) __alloc_func(__target_, _Alloc(__a)); + return __hold.release(); + } + + _LIBCPP_INLINE_VISIBILITY + void destroy() _NOEXCEPT { this->~__alloc_func(); } +}; +#else template class __alloc_func<_Fp, _Ap, _Rp(_ArgTypes...)> { @@ -1560,6 +1627,7 @@ __a.deallocate(__f, 1); } }; +#endif template class __default_alloc_func<_Fp, _Rp(_ArgTypes...)> { diff --git a/libcxx/include/future b/libcxx/include/future --- a/libcxx/include/future +++ b/libcxx/include/future @@ -1760,6 +1760,57 @@ template class __packaged_task_func; +#if _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS +template +class _LIBCPP_AVAILABILITY_FUTURE __packaged_task_func<_Fp, _Alloc, _Rp(_ArgTypes...)> + : public __packaged_task_base<_Rp(_ArgTypes...)> +{ + [[no_unique_address]] _Fp __f_; + [[no_unique_address]] _Alloc __alloc_; +public: + _LIBCPP_INLINE_VISIBILITY + explicit __packaged_task_func(const _Fp& __f) : __f_(__f), __alloc_() {} + _LIBCPP_INLINE_VISIBILITY + explicit __packaged_task_func(_Fp&& __f) : __f_(_VSTD::move(__f)), __alloc_() {} + _LIBCPP_INLINE_VISIBILITY + __packaged_task_func(const _Fp& __f, const _Alloc& __a) + : __f_(__f), __alloc_(__a) {} + _LIBCPP_INLINE_VISIBILITY + __packaged_task_func(_Fp&& __f, const _Alloc& __a) + : __f_(_VSTD::move(__f)), __alloc_(__a) {} + virtual void __move_to(__packaged_task_base<_Rp(_ArgTypes...)>*) _NOEXCEPT; + virtual void destroy(); + virtual void destroy_deallocate(); + virtual _Rp operator()(_ArgTypes&& ... __args); +}; + +template +void +__packaged_task_func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__move_to( + __packaged_task_base<_Rp(_ArgTypes...)>* __p) _NOEXCEPT +{ + ::new (__p) __packaged_task_func(_VSTD::move(__f_), _VSTD::move(__alloc_)); +} + +template +void +__packaged_task_func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy_deallocate() +{ + typedef typename __allocator_traits_rebind<_Alloc, __packaged_task_func>::type _Ap; + typedef allocator_traits<_Ap> _ATraits; + typedef pointer_traits _PTraits; + _Ap __a(__alloc_); + this->~__packaged_task_func(); + __a.deallocate(_PTraits::pointer_to(*this), 1); +} + +template +_Rp +__packaged_task_func<_Fp, _Alloc, _Rp(_ArgTypes...)>::operator()(_ArgTypes&& ... __arg) +{ + return __invoke(__f_, _VSTD::forward<_ArgTypes>(__arg)...); +} +#else template class _LIBCPP_AVAILABILITY_FUTURE __packaged_task_func<_Fp, _Alloc, _Rp(_ArgTypes...)> : public __packaged_task_base<_Rp(_ArgTypes...)> @@ -1790,13 +1841,6 @@ ::new (__p) __packaged_task_func(_VSTD::move(__f_.first()), _VSTD::move(__f_.second())); } -template -void -__packaged_task_func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy() -{ - __f_.~__compressed_pair<_Fp, _Alloc>(); -} - template void __packaged_task_func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy_deallocate() @@ -1805,7 +1849,7 @@ typedef allocator_traits<_Ap> _ATraits; typedef pointer_traits _PTraits; _Ap __a(__f_.second()); - __f_.~__compressed_pair<_Fp, _Alloc>(); + this->~__packaged_task_func(); __a.deallocate(_PTraits::pointer_to(*this), 1); } @@ -1815,6 +1859,14 @@ { return __invoke(__f_.first(), _VSTD::forward<_ArgTypes>(__arg)...); } +#endif + +template +void +__packaged_task_func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy() +{ + this->~__packaged_task_func(); +} template class __packaged_task_function; diff --git a/libcxx/include/list b/libcxx/include/list --- a/libcxx/include/list +++ b/libcxx/include/list @@ -267,6 +267,9 @@ struct __list_node : public __list_node_base<_Tp, _VoidPtr> { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + [[no_unique_address]] +#endif _Tp __value_; typedef __list_node_base<_Tp, _VoidPtr> __base; @@ -566,7 +569,12 @@ "type; otherwise overload resolution breaks"); __node_base __end_; +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + size_type __size_; + [[no_unique_address]] __node_allocator __alloc_; +#else __compressed_pair __size_alloc_; +#endif _LIBCPP_INLINE_VISIBILITY __link_pointer __end_as_link() const _NOEXCEPT { @@ -574,6 +582,16 @@ const_cast<__node_base&>(__end_).__self()); } +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + _LIBCPP_INLINE_VISIBILITY + size_type& __sz() _NOEXCEPT {return __size_;} + _LIBCPP_INLINE_VISIBILITY + const size_type& __sz() const _NOEXCEPT {return __size_;} + _LIBCPP_INLINE_VISIBILITY + __node_allocator& __node_alloc() _NOEXCEPT {return __alloc_;} + _LIBCPP_INLINE_VISIBILITY + const __node_allocator& __node_alloc() const _NOEXCEPT {return __alloc_;} +#else _LIBCPP_INLINE_VISIBILITY size_type& __sz() _NOEXCEPT {return __size_alloc_.first();} _LIBCPP_INLINE_VISIBILITY @@ -585,6 +603,7 @@ _LIBCPP_INLINE_VISIBILITY const __node_allocator& __node_alloc() const _NOEXCEPT {return __size_alloc_.second();} +#endif _LIBCPP_INLINE_VISIBILITY size_type __node_alloc_max_size() const _NOEXCEPT { @@ -714,26 +733,46 @@ inline __list_imp<_Tp, _Alloc>::__list_imp() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __size_(0), __alloc_() +#else : __size_alloc_(0) +#endif { } template inline __list_imp<_Tp, _Alloc>::__list_imp(const allocator_type& __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __size_(0), __alloc_(__node_allocator(__a)) +#else : __size_alloc_(0, __node_allocator(__a)) +#endif { } template inline __list_imp<_Tp, _Alloc>::__list_imp(const __node_allocator& __a) - : __size_alloc_(0, __a) {} +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __size_(0), __alloc_(__a) +#else + : __size_alloc_(0, __a) +#endif +{ +} #ifndef _LIBCPP_CXX03_LANG template inline __list_imp<_Tp, _Alloc>::__list_imp(__node_allocator&& __a) _NOEXCEPT - : __size_alloc_(0, std::move(__a)) {} +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __size_(0), __alloc_(std::move(__a)) +#else + : __size_alloc_(0, std::move(__a)) #endif +#endif +{ +} template __list_imp<_Tp, _Alloc>::~__list_imp() { diff --git a/libcxx/include/memory b/libcxx/include/memory --- a/libcxx/include/memory +++ b/libcxx/include/memory @@ -2127,6 +2127,7 @@ }; #endif +#ifndef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS template ::value && !__libcpp_is_final<_Tp>::value> @@ -2312,6 +2313,7 @@ __is_nothrow_swappable<_T2>::value) { __x.swap(__y); } +#endif // default_delete @@ -2403,7 +2405,12 @@ "the specified deleter type cannot be an rvalue reference"); private: +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + pointer __ptr_; + [[no_unique_address]] deleter_type __del_; +#else __compressed_pair __ptr_; +#endif struct __nat { int __for_bool_; }; @@ -2452,29 +2459,50 @@ template > _LIBCPP_INLINE_VISIBILITY - _LIBCPP_CONSTEXPR unique_ptr() _NOEXCEPT : __ptr_(pointer()) {} + _LIBCPP_CONSTEXPR unique_ptr() _NOEXCEPT : __ptr_(pointer()) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + , __del_() +#endif + {} template > _LIBCPP_INLINE_VISIBILITY - _LIBCPP_CONSTEXPR unique_ptr(nullptr_t) _NOEXCEPT : __ptr_(pointer()) {} + _LIBCPP_CONSTEXPR unique_ptr(nullptr_t) _NOEXCEPT : __ptr_(pointer()) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + , __del_() +#endif + {} template > _LIBCPP_INLINE_VISIBILITY - explicit unique_ptr(pointer __p) _NOEXCEPT : __ptr_(__p) {} + explicit unique_ptr(pointer __p) _NOEXCEPT : __ptr_(__p) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + , __del_() +#endif + {} template > > _LIBCPP_INLINE_VISIBILITY unique_ptr(pointer __p, _LValRefType<_Dummy> __d) _NOEXCEPT - : __ptr_(__p, __d) {} +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __ptr_(__p), __del_(__d) +#else + : __ptr_(__p, __d) +#endif + {} template > > _LIBCPP_INLINE_VISIBILITY unique_ptr(pointer __p, _GoodRValRefType<_Dummy> __d) _NOEXCEPT +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __ptr_(__p), __del_(_VSTD::move(__d)) { +#else : __ptr_(__p, _VSTD::move(__d)) { +#endif static_assert(!is_reference::value, "rvalue deleter bound to reference"); } @@ -2486,7 +2514,13 @@ _LIBCPP_INLINE_VISIBILITY unique_ptr(unique_ptr&& __u) _NOEXCEPT - : __ptr_(__u.release(), _VSTD::forward(__u.get_deleter())) { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __ptr_(__u.release()), + __del_(_VSTD::forward(__u.get_deleter())) +#else + : __ptr_(__u.release(), _VSTD::forward(__u.get_deleter())) +#endif + { } template _LIBCPP_INLINE_VISIBILITY unique_ptr(unique_ptr<_Up, _Ep>&& __u) _NOEXCEPT - : __ptr_(__u.release(), _VSTD::forward<_Ep>(__u.get_deleter())) {} +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __ptr_(__u.release()), __del_(_VSTD::forward<_Ep>(__u.get_deleter())) +#else + : __ptr_(__u.release(), _VSTD::forward<_Ep>(__u.get_deleter())) +#endif + { + } #if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR) template @@ -2504,13 +2544,21 @@ typename enable_if::value && is_same<_Dp, default_delete<_Tp> >::value, __nat>::type = __nat()) _NOEXCEPT - : __ptr_(__p.release()) {} + : __ptr_(__p.release()) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + , __del_() +#endif + {} #endif _LIBCPP_INLINE_VISIBILITY unique_ptr& operator=(unique_ptr&& __u) _NOEXCEPT { reset(__u.release()); +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __del_ = _VSTD::forward(__u.__del_); +#else __ptr_.second() = _VSTD::forward(__u.get_deleter()); +#endif return *this; } @@ -2521,7 +2569,11 @@ _LIBCPP_INLINE_VISIBILITY unique_ptr& operator=(unique_ptr<_Up, _Ep>&& __u) _NOEXCEPT { reset(__u.release()); +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __del_ = _VSTD::forward<_Ep>(__u.get_deleter()); +#else __ptr_.second() = _VSTD::forward<_Ep>(__u.get_deleter()); +#endif return *this; } @@ -2555,47 +2607,85 @@ _LIBCPP_INLINE_VISIBILITY typename add_lvalue_reference<_Tp>::type operator*() const { - return *__ptr_.first(); + return *__ptr_ +#ifndef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + .first() +#endif + ; } _LIBCPP_INLINE_VISIBILITY pointer operator->() const _NOEXCEPT { - return __ptr_.first(); + return __ptr_ +#ifndef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + .first() +#endif + ; } _LIBCPP_INLINE_VISIBILITY pointer get() const _NOEXCEPT { - return __ptr_.first(); + return __ptr_ +#ifndef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + .first() +#endif + ; } _LIBCPP_INLINE_VISIBILITY deleter_type& get_deleter() _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + return __del_; +#else return __ptr_.second(); +#endif } _LIBCPP_INLINE_VISIBILITY const deleter_type& get_deleter() const _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + return __del_; +#else return __ptr_.second(); +#endif } _LIBCPP_INLINE_VISIBILITY _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT { - return __ptr_.first() != nullptr; + return get() != nullptr; } _LIBCPP_INLINE_VISIBILITY pointer release() _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + pointer __t = __ptr_; + __ptr_ = pointer(); +#else pointer __t = __ptr_.first(); __ptr_.first() = pointer(); +#endif return __t; } _LIBCPP_INLINE_VISIBILITY void reset(pointer __p = pointer()) _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + pointer __tmp = __ptr_; + __ptr_ = __p; + if (__tmp) + __del_(__tmp); +#else pointer __tmp = __ptr_.first(); __ptr_.first() = __p; if (__tmp) __ptr_.second()(__tmp); +#endif } _LIBCPP_INLINE_VISIBILITY void swap(unique_ptr& __u) _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + using _VSTD::swap; + swap(__ptr_, __u.__ptr_); + swap(__del_, __u.__del_); +#else __ptr_.swap(__u.__ptr_); +#endif } }; @@ -2608,7 +2698,12 @@ typedef typename __pointer_type<_Tp, deleter_type>::type pointer; private: +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + pointer __ptr_; + [[no_unique_address]] deleter_type __del_; +#else __compressed_pair __ptr_; +#endif template struct _CheckArrayPointerConversion : is_same<_From, pointer> {}; @@ -2675,39 +2770,66 @@ template > _LIBCPP_INLINE_VISIBILITY - _LIBCPP_CONSTEXPR unique_ptr() _NOEXCEPT : __ptr_(pointer()) {} + _LIBCPP_CONSTEXPR unique_ptr() _NOEXCEPT : __ptr_(pointer()) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + , __del_() +#endif + {} template > _LIBCPP_INLINE_VISIBILITY - _LIBCPP_CONSTEXPR unique_ptr(nullptr_t) _NOEXCEPT : __ptr_(pointer()) {} + _LIBCPP_CONSTEXPR unique_ptr(nullptr_t) _NOEXCEPT : __ptr_(pointer()) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + , __del_() +#endif + {} template , class = _EnableIfPointerConvertible<_Pp> > _LIBCPP_INLINE_VISIBILITY explicit unique_ptr(_Pp __p) _NOEXCEPT - : __ptr_(__p) {} + : __ptr_(__p) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + , __del_() +#endif + {} template >, class = _EnableIfPointerConvertible<_Pp> > _LIBCPP_INLINE_VISIBILITY unique_ptr(_Pp __p, _LValRefType<_Dummy> __d) _NOEXCEPT - : __ptr_(__p, __d) {} +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __ptr_(__p), __del_(__d) +#else + : __ptr_(__p, __d) +#endif + {} template > > _LIBCPP_INLINE_VISIBILITY unique_ptr(nullptr_t, _LValRefType<_Dummy> __d) _NOEXCEPT - : __ptr_(nullptr, __d) {} +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __ptr_(nullptr), __del_(__d) +#else + : __ptr_(nullptr, __d) +#endif + {} template >, class = _EnableIfPointerConvertible<_Pp> > _LIBCPP_INLINE_VISIBILITY unique_ptr(_Pp __p, _GoodRValRefType<_Dummy> __d) _NOEXCEPT - : __ptr_(__p, _VSTD::move(__d)) { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __ptr_(__p), __del_(_VSTD::move(__d)) +#else + : __ptr_(__p, _VSTD::move(__d)) +#endif + { static_assert(!is_reference::value, "rvalue deleter bound to reference"); } @@ -2716,7 +2838,12 @@ class = _EnableIfDeleterConstructible<_GoodRValRefType<_Dummy> > > _LIBCPP_INLINE_VISIBILITY unique_ptr(nullptr_t, _GoodRValRefType<_Dummy> __d) _NOEXCEPT - : __ptr_(nullptr, _VSTD::move(__d)) { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __ptr_(nullptr), __del_(_VSTD::move(__d)) +#else + : __ptr_(nullptr, _VSTD::move(__d)) +#endif + { static_assert(!is_reference::value, "rvalue deleter bound to reference"); } @@ -2729,13 +2856,22 @@ _LIBCPP_INLINE_VISIBILITY unique_ptr(unique_ptr&& __u) _NOEXCEPT - : __ptr_(__u.release(), _VSTD::forward(__u.get_deleter())) { - } +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __ptr_(__u.release()), + __del_(_VSTD::forward(__u.__del_)) +#else + : __ptr_(__u.release(), _VSTD::forward(__u.get_deleter())) +#endif + {} _LIBCPP_INLINE_VISIBILITY unique_ptr& operator=(unique_ptr&& __u) _NOEXCEPT { reset(__u.release()); +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __del_ = _VSTD::forward(__u.__del_); +#else __ptr_.second() = _VSTD::forward(__u.get_deleter()); +#endif return *this; } @@ -2745,7 +2881,11 @@ > _LIBCPP_INLINE_VISIBILITY unique_ptr(unique_ptr<_Up, _Ep>&& __u) _NOEXCEPT +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __ptr_(__u.release()), __del_(_VSTD::forward<_Ep>(__u.get_deleter())) { +#else : __ptr_(__u.release(), _VSTD::forward<_Ep>(__u.get_deleter())) { +#endif } template && __u) _NOEXCEPT { reset(__u.release()); +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __del_ = _VSTD::forward<_Ep>(__u.get_deleter()); +#else __ptr_.second() = _VSTD::forward<_Ep>(__u.get_deleter()); +#endif return *this; } @@ -2778,31 +2922,52 @@ _LIBCPP_INLINE_VISIBILITY typename add_lvalue_reference<_Tp>::type operator[](size_t __i) const { - return __ptr_.first()[__i]; + return __ptr_ +#ifndef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + .first() +#endif + [__i]; } _LIBCPP_INLINE_VISIBILITY pointer get() const _NOEXCEPT { - return __ptr_.first(); + return __ptr_ +#ifndef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + .first() +#endif + ; } _LIBCPP_INLINE_VISIBILITY deleter_type& get_deleter() _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + return __del_; +#else return __ptr_.second(); +#endif } _LIBCPP_INLINE_VISIBILITY const deleter_type& get_deleter() const _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + return __del_; +#else return __ptr_.second(); +#endif } _LIBCPP_INLINE_VISIBILITY _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT { - return __ptr_.first() != nullptr; + return get() != nullptr; } _LIBCPP_INLINE_VISIBILITY pointer release() _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + pointer __t = __ptr_; + __ptr_ = pointer(); +#else pointer __t = __ptr_.first(); __ptr_.first() = pointer(); +#endif return __t; } @@ -2812,23 +2977,43 @@ _CheckArrayPointerConversion<_Pp>::value >::type reset(_Pp __p) _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + pointer __tmp = __ptr_; + __ptr_ = __p; + if (__tmp) + __del_(__tmp); +#else pointer __tmp = __ptr_.first(); __ptr_.first() = __p; if (__tmp) __ptr_.second()(__tmp); +#endif } _LIBCPP_INLINE_VISIBILITY void reset(nullptr_t = nullptr) _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + pointer __tmp = __ptr_; + __ptr_ = nullptr; + if (__tmp) + __del_(__tmp); +#else pointer __tmp = __ptr_.first(); __ptr_.first() = nullptr; if (__tmp) __ptr_.second()(__tmp); +#endif } _LIBCPP_INLINE_VISIBILITY void swap(unique_ptr& __u) _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + using _VSTD::swap; + swap(__ptr_, __u.__ptr_); + swap(__del_, __u.__del_); +#else __ptr_.swap(__u.__ptr_); +#endif } }; @@ -3478,11 +3663,22 @@ class __shared_ptr_pointer : public __shared_weak_count { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + _Tp __p_; + _Dp __d_; + _Alloc __a_; +#else __compressed_pair<__compressed_pair<_Tp, _Dp>, _Alloc> __data_; +#endif public: _LIBCPP_INLINE_VISIBILITY __shared_ptr_pointer(_Tp __p, _Dp __d, _Alloc __a) - : __data_(__compressed_pair<_Tp, _Dp>(__p, _VSTD::move(__d)), _VSTD::move(__a)) {} +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __p_(__p), __d_(_VSTD::move(__d)), __a_(_VSTD::move(__a)) +#else + : __data_(__compressed_pair<_Tp, _Dp>(__p, _VSTD::move(__d)), _VSTD::move(__a)) +#endif + {} #ifndef _LIBCPP_NO_RTTI virtual const void* __get_deleter(const type_info&) const _NOEXCEPT; @@ -3499,7 +3695,13 @@ const void* __shared_ptr_pointer<_Tp, _Dp, _Alloc>::__get_deleter(const type_info& __t) const _NOEXCEPT { - return __t == typeid(_Dp) ? _VSTD::addressof(__data_.first().second()) : nullptr; + return __t == typeid(_Dp) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + ? _VSTD::addressof(__d_) +#else + ? _VSTD::addressof(__data_.first().second()) +#endif + : nullptr; } #endif // _LIBCPP_NO_RTTI @@ -3508,8 +3710,13 @@ void __shared_ptr_pointer<_Tp, _Dp, _Alloc>::__on_zero_shared() _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __d_(__p_); + __d_.~_Dp(); +#else __data_.first().second()(__data_.first().first()); __data_.first().second().~_Dp(); +#endif } template @@ -3520,8 +3727,13 @@ typedef allocator_traits<_Al> _ATraits; typedef pointer_traits _PTraits; +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + _Al __a(__a_); + __a_.~_Alloc(); +#else _Al __a(__data_.second()); __data_.second().~_Alloc(); +#endif __a.deallocate(_PTraits::pointer_to(*this), 1); } @@ -3529,19 +3741,34 @@ class __shared_ptr_emplace : public __shared_weak_count { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + _Tp __v_; + [[no_unique_address]] _Alloc __a_; +#else __compressed_pair<_Alloc, _Tp> __data_; +#endif public: #ifndef _LIBCPP_HAS_NO_VARIADICS _LIBCPP_INLINE_VISIBILITY __shared_ptr_emplace(_Alloc __a) - : __data_(_VSTD::move(__a)) {} +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __v_(), __a_(_VSTD::move(__a)) +#else + : __data_(_VSTD::move(__a)) +#endif + {} template _LIBCPP_INLINE_VISIBILITY __shared_ptr_emplace(_Alloc __a, _Args&& ...__args) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __v_(_VSTD::forward<_Args>(__args)...), __a_(__a) +#else : __data_(piecewise_construct, _VSTD::forward_as_tuple(__a), - _VSTD::forward_as_tuple(_VSTD::forward<_Args>(__args)...)) {} + _VSTD::forward_as_tuple(_VSTD::forward<_Args>(__args)...)) +#endif + {} #else // _LIBCPP_HAS_NO_VARIADICS @@ -3571,14 +3798,22 @@ virtual void __on_zero_shared_weak() _NOEXCEPT; public: _LIBCPP_INLINE_VISIBILITY +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + _Tp* get() _NOEXCEPT {return _VSTD::addressof(__v_);} +#else _Tp* get() _NOEXCEPT {return _VSTD::addressof(__data_.second());} +#endif }; template void __shared_ptr_emplace<_Tp, _Alloc>::__on_zero_shared() _NOEXCEPT { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __v_.~_Tp(); +#else __data_.second().~_Tp(); +#endif } template @@ -3588,8 +3823,13 @@ typedef typename __allocator_traits_rebind<_Alloc, __shared_ptr_emplace>::type _Al; typedef allocator_traits<_Al> _ATraits; typedef pointer_traits _PTraits; +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + _Al __a(__a_); + __a_.~_Alloc(); +#else _Al __a(__data_.first()); __data_.first().~_Alloc(); +#endif __a.deallocate(_PTraits::pointer_to(*this), 1); } diff --git a/libcxx/include/string b/libcxx/include/string --- a/libcxx/include/string +++ b/libcxx/include/string @@ -781,7 +781,30 @@ }; }; - __compressed_pair<__rep, allocator_type> __r_; +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __rep __r_; + [[no_unique_address]] allocator_type __alloc_; + + _LIBCPP_INLINE_VISIBILITY __rep& __r() _NOEXCEPT {return __r_;} + _LIBCPP_INLINE_VISIBILITY const __rep& __r() const _NOEXCEPT {return __r_;} + _LIBCPP_INLINE_VISIBILITY + allocator_type& __alloc() _NOEXCEPT {return __alloc_;} + _LIBCPP_INLINE_VISIBILITY + const allocator_type& __alloc() const _NOEXCEPT {return __alloc_;} +#else + __compressed_pair<__rep, allocator_type> __r_alloc_; + + _LIBCPP_INLINE_VISIBILITY __rep& __r() _NOEXCEPT + {return __r_alloc_.first();} + _LIBCPP_INLINE_VISIBILITY const __rep& __r() const _NOEXCEPT + {return __r_alloc_.first();} + _LIBCPP_INLINE_VISIBILITY + allocator_type& __alloc() _NOEXCEPT + {return __r_alloc_.second();} + _LIBCPP_INLINE_VISIBILITY + const allocator_type& __alloc() const _NOEXCEPT + {return __r_alloc_.second();} +#endif public: static const size_type npos = -1; @@ -1423,7 +1446,7 @@ _LIBCPP_INLINE_VISIBILITY bool __is_long() const _NOEXCEPT - {return bool(__r_.first().__s.__size_ & __short_mask);} + {return bool(__r().__s.__size_ & __short_mask);} #if _LIBCPP_DEBUG_LEVEL >= 2 @@ -1435,29 +1458,22 @@ #endif // _LIBCPP_DEBUG_LEVEL >= 2 private: - _LIBCPP_INLINE_VISIBILITY - allocator_type& __alloc() _NOEXCEPT - {return __r_.second();} - _LIBCPP_INLINE_VISIBILITY - const allocator_type& __alloc() const _NOEXCEPT - {return __r_.second();} - #ifdef _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT _LIBCPP_INLINE_VISIBILITY void __set_short_size(size_type __s) _NOEXCEPT # ifdef _LIBCPP_BIG_ENDIAN - {__r_.first().__s.__size_ = (unsigned char)(__s << 1);} + {__r().__s.__size_ = (unsigned char)(__s << 1);} # else - {__r_.first().__s.__size_ = (unsigned char)(__s);} + {__r().__s.__size_ = (unsigned char)(__s);} # endif _LIBCPP_INLINE_VISIBILITY size_type __get_short_size() const _NOEXCEPT # ifdef _LIBCPP_BIG_ENDIAN - {return __r_.first().__s.__size_ >> 1;} + {return __r().__s.__size_ >> 1;} # else - {return __r_.first().__s.__size_;} + {return __r().__s.__size_;} # endif #else // _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT @@ -1465,53 +1481,53 @@ _LIBCPP_INLINE_VISIBILITY void __set_short_size(size_type __s) _NOEXCEPT # ifdef _LIBCPP_BIG_ENDIAN - {__r_.first().__s.__size_ = (unsigned char)(__s);} + {__r().__s.__size_ = (unsigned char)(__s);} # else - {__r_.first().__s.__size_ = (unsigned char)(__s << 1);} + {__r().__s.__size_ = (unsigned char)(__s << 1);} # endif _LIBCPP_INLINE_VISIBILITY size_type __get_short_size() const _NOEXCEPT # ifdef _LIBCPP_BIG_ENDIAN - {return __r_.first().__s.__size_;} + {return __r().__s.__size_;} # else - {return __r_.first().__s.__size_ >> 1;} + {return __r().__s.__size_ >> 1;} # endif #endif // _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT _LIBCPP_INLINE_VISIBILITY void __set_long_size(size_type __s) _NOEXCEPT - {__r_.first().__l.__size_ = __s;} + {__r().__l.__size_ = __s;} _LIBCPP_INLINE_VISIBILITY size_type __get_long_size() const _NOEXCEPT - {return __r_.first().__l.__size_;} + {return __r().__l.__size_;} _LIBCPP_INLINE_VISIBILITY void __set_size(size_type __s) _NOEXCEPT {if (__is_long()) __set_long_size(__s); else __set_short_size(__s);} _LIBCPP_INLINE_VISIBILITY void __set_long_cap(size_type __s) _NOEXCEPT - {__r_.first().__l.__cap_ = __long_mask | __s;} + {__r().__l.__cap_ = __long_mask | __s;} _LIBCPP_INLINE_VISIBILITY size_type __get_long_cap() const _NOEXCEPT - {return __r_.first().__l.__cap_ & size_type(~__long_mask);} + {return __r().__l.__cap_ & size_type(~__long_mask);} _LIBCPP_INLINE_VISIBILITY void __set_long_pointer(pointer __p) _NOEXCEPT - {__r_.first().__l.__data_ = __p;} + {__r().__l.__data_ = __p;} _LIBCPP_INLINE_VISIBILITY pointer __get_long_pointer() _NOEXCEPT - {return __r_.first().__l.__data_;} + {return __r().__l.__data_;} _LIBCPP_INLINE_VISIBILITY const_pointer __get_long_pointer() const _NOEXCEPT - {return __r_.first().__l.__data_;} + {return __r().__l.__data_;} _LIBCPP_INLINE_VISIBILITY pointer __get_short_pointer() _NOEXCEPT - {return pointer_traits::pointer_to(__r_.first().__s.__data_[0]);} + {return pointer_traits::pointer_to(__r().__s.__data_[0]);} _LIBCPP_INLINE_VISIBILITY const_pointer __get_short_pointer() const _NOEXCEPT - {return pointer_traits::pointer_to(__r_.first().__s.__data_[0]);} + {return pointer_traits::pointer_to(__r().__s.__data_[0]);} _LIBCPP_INLINE_VISIBILITY pointer __get_pointer() _NOEXCEPT {return __is_long() ? __get_long_pointer() : __get_short_pointer();} @@ -1522,7 +1538,7 @@ _LIBCPP_INLINE_VISIBILITY void __zero() _NOEXCEPT { - size_type (&__a)[__n_words] = __r_.first().__r.__words; + size_type (&__a)[__n_words] = __r().__r.__words; for (unsigned __i = 0; __i < __n_words; ++__i) __a[__i] = 0; } @@ -1742,7 +1758,11 @@ #else _NOEXCEPT #endif -: __r_(__second_tag(), __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __alloc_(__a) +#else + : __r_alloc_(__second_tag(), __a) +#endif { #if _LIBCPP_DEBUG_LEVEL >= 2 __get_db()->__insert_c(this); @@ -1804,7 +1824,11 @@ template #endif basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, const _Allocator& __a) - : __r_(__second_tag(), __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __alloc_(__a) +#else + : __r_alloc_(__second_tag(), __a) +#endif { _LIBCPP_ASSERT(__s != nullptr, "basic_string(const char*, allocator) detected nullptr"); __init(__s, traits_type::length(__s)); @@ -1827,7 +1851,11 @@ template inline basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, size_type __n, const _Allocator& __a) - : __r_(__second_tag(), __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __alloc_(__a) +#else + : __r_alloc_(__second_tag(), __a) +#endif { _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "basic_string(const char*, n, allocator) detected nullptr"); __init(__s, __n); @@ -1838,10 +1866,14 @@ template basic_string<_CharT, _Traits, _Allocator>::basic_string(const basic_string& __str) - : __r_(__second_tag(), __alloc_traits::select_on_container_copy_construction(__str.__alloc())) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __alloc_(__alloc_traits::select_on_container_copy_construction(__str.__alloc())) +#else + : __r_alloc_(__second_tag(), __alloc_traits::select_on_container_copy_construction(__str.__alloc())) +#endif { if (!__str.__is_long()) - __r_.first().__r = __str.__r_.first().__r; + __r().__r = __str.__r().__r; else __init(_VSTD::__to_raw_pointer(__str.__get_long_pointer()), __str.__get_long_size()); #if _LIBCPP_DEBUG_LEVEL >= 2 @@ -1852,10 +1884,14 @@ template basic_string<_CharT, _Traits, _Allocator>::basic_string( const basic_string& __str, const allocator_type& __a) - : __r_(__second_tag(), __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __alloc_(__a) +#else + : __r_alloc_(__second_tag(), __a) +#endif { if (!__str.__is_long()) - __r_.first().__r = __str.__r_.first().__r; + __r().__r = __str.__r().__r; else __init(_VSTD::__to_raw_pointer(__str.__get_long_pointer()), __str.__get_long_size()); #if _LIBCPP_DEBUG_LEVEL >= 2 @@ -1873,7 +1909,11 @@ #else _NOEXCEPT #endif - : __r_(_VSTD::move(__str.__r_)) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __r_(_VSTD::move(__str.__r_)), __alloc_(_VSTD::move(__str.__alloc_)) +#else + : __r_alloc_(_VSTD::move(__str.__r_alloc_)) +#endif { __str.__zero(); #if _LIBCPP_DEBUG_LEVEL >= 2 @@ -1886,13 +1926,17 @@ template inline basic_string<_CharT, _Traits, _Allocator>::basic_string(basic_string&& __str, const allocator_type& __a) - : __r_(__second_tag(), __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __alloc_(__a) +#else + : __r_alloc_(__second_tag(), __a) +#endif { if (__str.__is_long() && __a != __str.__alloc()) // copy, not move __init(_VSTD::__to_raw_pointer(__str.__get_long_pointer()), __str.__get_long_size()); else { - __r_.first().__r = __str.__r_.first().__r; + __r().__r = __str.__r().__r; __str.__zero(); } #if _LIBCPP_DEBUG_LEVEL >= 2 @@ -1943,7 +1987,11 @@ template #endif basic_string<_CharT, _Traits, _Allocator>::basic_string(size_type __n, _CharT __c, const _Allocator& __a) - : __r_(__second_tag(), __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __alloc_(__a) +#else + : __r_alloc_(__second_tag(), __a) +#endif { __init(__n, __c); #if _LIBCPP_DEBUG_LEVEL >= 2 @@ -1955,7 +2003,11 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string(const basic_string& __str, size_type __pos, size_type __n, const _Allocator& __a) - : __r_(__second_tag(), __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __alloc_(__a) +#else + : __r_alloc_(__second_tag(), __a) +#endif { size_type __str_sz = __str.size(); if (__pos > __str_sz) @@ -1970,7 +2022,11 @@ inline basic_string<_CharT, _Traits, _Allocator>::basic_string(const basic_string& __str, size_type __pos, const _Allocator& __a) - : __r_(__second_tag(), __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __alloc_(__a) +#else + : __r_alloc_(__second_tag(), __a) +#endif { size_type __str_sz = __str.size(); if (__pos > __str_sz) @@ -1985,7 +2041,11 @@ template basic_string<_CharT, _Traits, _Allocator>::basic_string( const _Tp& __t, size_type __pos, size_type __n, const allocator_type& __a) - : __r_(__second_tag(), __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __alloc_(__a) +#else + : __r_alloc_(__second_tag(), __a) +#endif { __self_view __sv0 = __t; __self_view __sv = __sv0.substr(__pos, __n); @@ -2009,7 +2069,11 @@ template template basic_string<_CharT, _Traits, _Allocator>::basic_string(const _Tp & __t, const _Allocator& __a) - : __r_(__second_tag(), __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __alloc_(__a) +#else + : __r_alloc_(__second_tag(), __a) +#endif { __self_view __sv = __t; __init(__sv.data(), __sv.size()); @@ -2092,7 +2156,11 @@ inline basic_string<_CharT, _Traits, _Allocator>::basic_string(_InputIterator __first, _InputIterator __last, const allocator_type& __a) - : __r_(__second_tag(), __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __alloc_(__a) +#else + : __r_alloc_(__second_tag(), __a) +#endif { __init(__first, __last); #if _LIBCPP_DEBUG_LEVEL >= 2 @@ -2118,7 +2186,11 @@ basic_string<_CharT, _Traits, _Allocator>::basic_string( initializer_list<_CharT> __il, const _Allocator& __a) - : __r_(__second_tag(), __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + : __alloc_(__a) +#else + : __r_alloc_(__second_tag(), __a) +#endif { __init(__il.begin(), __il.end()); #if _LIBCPP_DEBUG_LEVEL >= 2 @@ -2300,7 +2372,7 @@ #endif { __clear_and_shrink(); - __r_.first() = __str.__r_.first(); + __r() = __str.__r(); __move_assign_alloc(__str); __str.__zero(); } @@ -3312,7 +3384,7 @@ __alloc_traits::propagate_on_container_swap::value || __alloc_traits::is_always_equal::value || __alloc() == __str.__alloc(), "swapping non-equal allocators"); - _VSTD::swap(__r_.first(), __str.__r_.first()); + _VSTD::swap(__r(), __str.__r()); __swap_allocator(__alloc(), __str.__alloc()); } diff --git a/libcxx/include/vector b/libcxx/include/vector --- a/libcxx/include/vector +++ b/libcxx/include/vector @@ -341,20 +341,34 @@ pointer __begin_; pointer __end_; - __compressed_pair __end_cap_; +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + pointer __end_cap_; + [[no_unique_address]] allocator_type __alloc_; + + _LIBCPP_INLINE_VISIBILITY + allocator_type& __alloc() _NOEXCEPT {return __alloc_;} + _LIBCPP_INLINE_VISIBILITY + const allocator_type& __alloc() const _NOEXCEPT {return __alloc_;} + _LIBCPP_INLINE_VISIBILITY + pointer& __end_cap() _NOEXCEPT {return __end_cap_;} + _LIBCPP_INLINE_VISIBILITY + const pointer& __end_cap() const _NOEXCEPT {return __end_cap_;} +#else + __compressed_pair __end_cap_alloc_; _LIBCPP_INLINE_VISIBILITY allocator_type& __alloc() _NOEXCEPT - {return __end_cap_.second();} + {return __end_cap_alloc_.second();} _LIBCPP_INLINE_VISIBILITY const allocator_type& __alloc() const _NOEXCEPT - {return __end_cap_.second();} + {return __end_cap_alloc_.second();} _LIBCPP_INLINE_VISIBILITY pointer& __end_cap() _NOEXCEPT - {return __end_cap_.first();} + {return __end_cap_alloc_.first();} _LIBCPP_INLINE_VISIBILITY const pointer& __end_cap() const _NOEXCEPT - {return __end_cap_.first();} + {return __end_cap_alloc_.first();} +#endif _LIBCPP_INLINE_VISIBILITY __vector_base() @@ -433,7 +447,12 @@ _NOEXCEPT_(is_nothrow_default_constructible::value) : __begin_(nullptr), __end_(nullptr), - __end_cap_(nullptr) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __end_cap_(nullptr), + __alloc_() +#else + __end_cap_alloc_(nullptr) +#endif { } @@ -442,7 +461,12 @@ __vector_base<_Tp, _Allocator>::__vector_base(const allocator_type& __a) : __begin_(nullptr), __end_(nullptr), - __end_cap_(nullptr, __a) +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __end_cap_(nullptr), + __alloc_(__a) +#else + __end_cap_alloc_(nullptr, __a) +#endif { } @@ -452,7 +476,14 @@ __vector_base<_Tp, _Allocator>::__vector_base(allocator_type&& __a) _NOEXCEPT : __begin_(nullptr), __end_(nullptr), - __end_cap_(nullptr, std::move(__a)) {} +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __end_cap_(nullptr), + __alloc_(std::move(__a)) +#else + __end_cap_alloc_(nullptr, std::move(__a)) +#endif +{ +} #endif template @@ -2215,11 +2246,26 @@ __storage_pointer __begin_; size_type __size_; +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + size_type __cap_; + [[no_unique_address]] __storage_allocator __alloc_; +#else __compressed_pair __cap_alloc_; +#endif public: typedef __bit_reference reference; typedef __bit_const_reference const_reference; private: +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + _LIBCPP_INLINE_VISIBILITY + size_type& __cap() _NOEXCEPT {return __cap_;} + _LIBCPP_INLINE_VISIBILITY + const size_type& __cap() const _NOEXCEPT {return __cap_;} + _LIBCPP_INLINE_VISIBILITY + __storage_allocator& __alloc() _NOEXCEPT {return __alloc_;} + _LIBCPP_INLINE_VISIBILITY + const __storage_allocator& __alloc() const _NOEXCEPT {return __alloc_;} +#else _LIBCPP_INLINE_VISIBILITY size_type& __cap() _NOEXCEPT {return __cap_alloc_.first();} @@ -2232,6 +2278,7 @@ _LIBCPP_INLINE_VISIBILITY const __storage_allocator& __alloc() const _NOEXCEPT {return __cap_alloc_.second();} +#endif static const unsigned __bits_per_word = static_cast(sizeof(__storage_type) * CHAR_BIT); @@ -2652,7 +2699,11 @@ _NOEXCEPT_(is_nothrow_default_constructible::value) : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_() +#else __cap_alloc_(0) +#endif { } @@ -2666,7 +2717,11 @@ #endif : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_(__a) +#else __cap_alloc_(0, static_cast<__storage_allocator>(__a)) +#endif { } @@ -2674,7 +2729,11 @@ vector::vector(size_type __n) : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_() +#else __cap_alloc_(0) +#endif { if (__n > 0) { @@ -2688,7 +2747,11 @@ vector::vector(size_type __n, const allocator_type& __a) : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_(__a) +#else __cap_alloc_(0, static_cast<__storage_allocator>(__a)) +#endif { if (__n > 0) { @@ -2702,7 +2765,11 @@ vector::vector(size_type __n, const value_type& __x) : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_() +#else __cap_alloc_(0) +#endif { if (__n > 0) { @@ -2715,7 +2782,11 @@ vector::vector(size_type __n, const value_type& __x, const allocator_type& __a) : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_(__a) +#else __cap_alloc_(0, static_cast<__storage_allocator>(__a)) +#endif { if (__n > 0) { @@ -2731,7 +2802,11 @@ !__is_forward_iterator<_InputIterator>::value>::type*) : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_() +#else __cap_alloc_(0) +#endif { #ifndef _LIBCPP_NO_EXCEPTIONS try @@ -2758,7 +2833,11 @@ !__is_forward_iterator<_InputIterator>::value>::type*) : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_(__a) +#else __cap_alloc_(0, static_cast<__storage_allocator>(__a)) +#endif { #ifndef _LIBCPP_NO_EXCEPTIONS try @@ -2784,7 +2863,11 @@ typename enable_if<__is_forward_iterator<_ForwardIterator>::value>::type*) : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_() +#else __cap_alloc_(0) +#endif { size_type __n = static_cast(_VSTD::distance(__first, __last)); if (__n > 0) @@ -2800,7 +2883,11 @@ typename enable_if<__is_forward_iterator<_ForwardIterator>::value>::type*) : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_(__a) +#else __cap_alloc_(0, static_cast<__storage_allocator>(__a)) +#endif { size_type __n = static_cast(_VSTD::distance(__first, __last)); if (__n > 0) @@ -2816,7 +2903,11 @@ vector::vector(initializer_list __il) : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_() +#else __cap_alloc_(0) +#endif { size_type __n = static_cast(__il.size()); if (__n > 0) @@ -2830,7 +2921,11 @@ vector::vector(initializer_list __il, const allocator_type& __a) : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_(__a) +#else __cap_alloc_(0, static_cast<__storage_allocator>(__a)) +#endif { size_type __n = static_cast(__il.size()); if (__n > 0) @@ -2854,7 +2949,11 @@ vector::vector(const vector& __v) : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_(__storage_traits::select_on_container_copy_construction(__v.__alloc_)) +#else __cap_alloc_(0, __storage_traits::select_on_container_copy_construction(__v.__alloc())) +#endif { if (__v.size() > 0) { @@ -2867,7 +2966,11 @@ vector::vector(const vector& __v, const allocator_type& __a) : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_(__a) +#else __cap_alloc_(0, __a) +#endif { if (__v.size() > 0) { @@ -2908,7 +3011,12 @@ #endif : __begin_(__v.__begin_), __size_(__v.__size_), - __cap_alloc_(std::move(__v.__cap_alloc_)) { +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(__v.__cap_), __alloc_(_VSTD::move(__v.__alloc_)) +#else + __cap_alloc_(_VSTD::move(__v.__cap_alloc_)) +#endif +{ __v.__begin_ = nullptr; __v.__size_ = 0; __v.__cap() = 0; @@ -2918,7 +3026,11 @@ vector::vector(vector&& __v, const allocator_type& __a) : __begin_(nullptr), __size_(0), +#ifdef _LIBCPP_ABI_USE_NO_UNIQUE_ADDRESS + __cap_(0), __alloc_(__a) +#else __cap_alloc_(0, __a) +#endif { if (__a == allocator_type(__v.__alloc())) { diff --git a/libcxx/test/libcxx/containers/unord/unord.set/missing_hash_specialization.fail.cpp b/libcxx/test/libcxx/containers/unord/unord.set/missing_hash_specialization.fail.cpp --- a/libcxx/test/libcxx/containers/unord/unord.set/missing_hash_specialization.fail.cpp +++ b/libcxx/test/libcxx/containers/unord/unord.set/missing_hash_specialization.fail.cpp @@ -52,7 +52,7 @@ // FIXME: It would be great to suppress the below diagnostic all together. // but for now it's sufficient that it appears last. However there is // currently no way to test the order diagnostics are issued. - // expected-error@memory:* {{call to implicitly-deleted default constructor of '__compressed_pair_elem}} + // expected-error@* {{call to implicitly-deleted default constructor}} } { using Set = std::unordered_set; diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared.protected.fail.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared.protected.fail.cpp --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared.protected.fail.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared.protected.fail.cpp @@ -24,7 +24,9 @@ int main(int, char**) { + // We only see the second error here if we don't use EBO. std::shared_ptr p = std::make_shared(); // expected-error-re@memory:* {{static_assert failed{{.*}} "Can't construct object in make_shared"}} + // expected-error@memory:* 0-1{{'S' has protected default constructor}} return 0; }