diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table --- a/libcxx/include/__hash_table +++ b/libcxx/include/__hash_table @@ -21,6 +21,7 @@ #include <__memory/addressof.h> #include <__memory/allocator_traits.h> #include <__memory/compressed_pair.h> +#include <__memory/construct_at.h> #include <__memory/pointer_traits.h> #include <__memory/swap_allocator.h> #include <__memory/unique_ptr.h> @@ -107,6 +108,7 @@ } _LIBCPP_INLINE_VISIBILITY __hash_node_base() _NOEXCEPT : __next_(nullptr) {} + _LIBCPP_HIDE_FROM_ABI explicit __hash_node_base(__next_pointer __next) _NOEXCEPT : __next_(__next) {} }; template @@ -117,9 +119,28 @@ > { typedef _Tp __node_value_type; + using _Base = __hash_node_base<__rebind_pointer_t<_VoidPtr, __hash_node<_Tp, _VoidPtr> > >; + using __next_pointer = typename _Base::__next_pointer; size_t __hash_; + + // Allow starting the lifetime of nodes without initializing the value held by the node. + // Note that we can't do that in C++03 because unions can't contain non-trivially constructible types. +#ifndef _LIBCPP_CXX03_LANG +# define _LIBCPP_HASH_TABLE_PROPER_NODE_LIFETIME +#endif + +#ifdef _LIBCPP_HASH_TABLE_PROPER_NODE_LIFETIME + union { + __node_value_type __value_; + }; +#else __node_value_type __value_; +#endif + + // Don't construct or destroy the value_type -- this is handled by the hash table itself to be allocator-aware + _LIBCPP_HIDE_FROM_ABI explicit __hash_node(__next_pointer __next, size_t __hash) : _Base(__next), __hash_(__hash) {} + _LIBCPP_HIDE_FROM_ABI ~__hash_node() {} }; inline _LIBCPP_INLINE_VISIBILITY @@ -670,8 +691,12 @@ _LIBCPP_INLINE_VISIBILITY void operator()(pointer __p) _NOEXCEPT { - if (__value_constructed) + if (__value_constructed) { __alloc_traits::destroy(__na_, _NodeTypes::__get_ptr(__p->__value_)); +#ifdef _LIBCPP_HASH_TABLE_PROPER_NODE_LIFETIME + std::__destroy_at(std::addressof(*__p)); +#endif + } if (__p) __alloc_traits::deallocate(__na_, __p, 1); } @@ -1366,6 +1391,9 @@ __next_pointer __next = __np->__next_; __node_pointer __real_np = __np->__upcast(); __node_traits::destroy(__na, _NodeTypes::__get_ptr(__real_np->__value_)); +#ifdef _LIBCPP_HASH_TABLE_PROPER_NODE_LIFETIME + std::__destroy_at(std::addressof(*__real_np)); +#endif __node_traits::deallocate(__na, __real_np, 1); __np = __next; } @@ -2193,10 +2221,28 @@ "Construct cannot be called with a hash value type"); __node_allocator& __na = __node_alloc(); __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na)); + + // Begin the lifetime of the node itself. Note that this doesn't begin the lifetime of the value + // held inside the node, since we need to use the allocator's construct() method for that. + // + // We don't use the allocator's construct() method to construct the node itself since the + // Cpp17FooInsertable named requirements don't require the allocator's construct() method + // to work on anything other than the value_type. + // + // Note that we don't begin the lifetime properly in C++03 because that would cause the + // value_type to be default constructed. +#ifdef _LIBCPP_HASH_TABLE_PROPER_NODE_LIFETIME + std::__construct_at(std::addressof(*__h), /* next = */nullptr, /* hash = */0); +#else + __h->__next_ = nullptr; + // __hash_ is set below unconditionally +#endif + + // Now construct the value_type using the allocator's construct() method. __node_traits::construct(__na, _NodeTypes::__get_ptr(__h->__value_), _VSTD::forward<_Args>(__args)...); + __h.get_deleter().__value_constructed = true; __h->__hash_ = hash_function()(__h->__value_); - __h->__next_ = nullptr; return __h; } @@ -2210,12 +2256,18 @@ "Construct cannot be called with a hash value type"); __node_allocator& __na = __node_alloc(); __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na)); + +#ifdef _LIBCPP_HASH_TABLE_PROPER_NODE_LIFETIME + std::__construct_at(std::addressof(*__h), /* next = */nullptr, /* hash = */__hash); +#else + __h->__next_ = nullptr; + __h->__hash_ = __hash; +#endif + __node_traits::construct(__na, _NodeTypes::__get_ptr(__h->__value_), _VSTD::forward<_First>(__f), _VSTD::forward<_Rest>(__rest)...); __h.get_deleter().__value_constructed = true; - __h->__hash_ = __hash; - __h->__next_ = nullptr; return __h; } diff --git a/libcxx/include/forward_list b/libcxx/include/forward_list --- a/libcxx/include/forward_list +++ b/libcxx/include/forward_list @@ -211,6 +211,7 @@ #include <__memory/allocator.h> #include <__memory/allocator_traits.h> #include <__memory/compressed_pair.h> +#include <__memory/construct_at.h> #include <__memory/pointer_traits.h> #include <__memory/swap_allocator.h> #include <__memory_resource/polymorphic_allocator.h> @@ -325,10 +326,23 @@ typedef __begin_node_of<_Tp, _VoidPtr> _Base; typedef typename _Base::pointer _NodePtr; + // Allow starting the lifetime of nodes without initializing the value held by the node. + // Note that we can't do that in C++03 because unions can't contain non-trivially constructible types. +#ifndef _LIBCPP_CXX03_LANG +# define _LIBCPP_FORWARD_LIST_PROPER_NODE_LIFETIME +#endif + +#ifdef _LIBCPP_FORWARD_LIST_PROPER_NODE_LIFETIME + union { + value_type __value_; + }; +#else value_type __value_; +#endif - _LIBCPP_HIDE_FROM_ABI __forward_list_node() = default; - _LIBCPP_HIDE_FROM_ABI __forward_list_node(const value_type& __v, _NodePtr __next) : _Base(__next), __value_(__v) {} + // Don't construct or destroy the value_type -- this is handled by the forward_list itself to be allocator-aware + _LIBCPP_HIDE_FROM_ABI explicit __forward_list_node(_NodePtr __next) : _Base(__next) {} + _LIBCPP_HIDE_FROM_ABI ~__forward_list_node() {} }; @@ -577,15 +591,37 @@ _LIBCPP_HIDE_FROM_ABI __node_pointer __create_node(__node_pointer __next, _Args&& ...__args) { __node_allocator& __a = __alloc(); __allocation_guard<__node_allocator> __guard(__a, 1); + // Begin the lifetime of the node itself. Note that this doesn't begin the lifetime of the value + // held inside the node, since we need to use the allocator's construct() method for that. + // + // We don't use the allocator's construct() method to construct the node itself since the + // Cpp17FooInsertable named requirements don't require the allocator's construct() method + // to work on anything other than the value_type. + // + // Note that we don't begin the lifetime properly in C++03 because that would cause the + // value_type to be default constructed. +#ifdef _LIBCPP_FORWARD_LIST_PROPER_NODE_LIFETIME + std::__construct_at(std::addressof(*__guard.__get()), __next); +#else __guard.__get()->__next_ = __next; +#endif + + // Now construct the value_type using the allocator's construct() method. __node_traits::construct(__a, std::addressof(__guard.__get()->__value_), std::forward<_Args>(__args)...); return __guard.__release_ptr(); } template _LIBCPP_HIDE_FROM_ABI void __delete_node(__node_pointer __node) { + // For the same reason as above, we use the allocator's destroy() method for the value_type, + // but not for the node itself. __node_allocator& __a = __alloc(); __node_traits::destroy(__a, std::addressof(__node->__value_)); + +#ifdef _LIBCPP_FORWARD_LIST_PROPER_NODE_LIFETIME + std::__destroy_at(std::addressof(*__node)); +#endif + __node_traits::deallocate(__a, __node, 1); } diff --git a/libcxx/include/list b/libcxx/include/list --- a/libcxx/include/list +++ b/libcxx/include/list @@ -217,6 +217,7 @@ #include <__memory/allocator.h> #include <__memory/allocator_traits.h> #include <__memory/compressed_pair.h> +#include <__memory/construct_at.h> #include <__memory/pointer_traits.h> #include <__memory/swap_allocator.h> #include <__memory_resource/polymorphic_allocator.h> @@ -308,6 +309,9 @@ __list_node_base() : __prev_(_NodeTraits::__unsafe_link_pointer_cast(__self())), __next_(_NodeTraits::__unsafe_link_pointer_cast(__self())) {} + _LIBCPP_HIDE_FROM_ABI explicit __list_node_base(__link_pointer __prev, __link_pointer __next) + : __prev_(__prev), __next_(__next) {} + _LIBCPP_INLINE_VISIBILITY __base_pointer __self() { return pointer_traits<__base_pointer>::pointer_to(*this); @@ -323,11 +327,27 @@ struct _LIBCPP_STANDALONE_DEBUG __list_node : public __list_node_base<_Tp, _VoidPtr> { + // Allow starting the lifetime of nodes without initializing the value held by the node. + // Note that we can't do that in C++03 because unions can't contain non-trivially constructible types. +#ifndef _LIBCPP_CXX03_LANG +# define _LIBCPP_LIST_PROPER_NODE_LIFETIME +#endif + +#ifdef _LIBCPP_LIST_PROPER_NODE_LIFETIME + union { + _Tp __value_; + }; +#else _Tp __value_; +#endif typedef __list_node_base<_Tp, _VoidPtr> __base; typedef typename __base::__link_pointer __link_pointer; + // Don't construct or destroy the value_type -- this is handled by the list itself to be allocator-aware + _LIBCPP_HIDE_FROM_ABI explicit __list_node(__link_pointer __prev, __link_pointer __next) : __base(__prev, __next) {} + _LIBCPP_HIDE_FROM_ABI ~__list_node() {} + _LIBCPP_INLINE_VISIBILITY __link_pointer __as_link() { return static_cast<__link_pointer>(__base::__self()); @@ -600,16 +620,36 @@ _LIBCPP_HIDE_FROM_ABI __node_pointer __create_node(__link_pointer __prev, __link_pointer __next, _Args&& ...__args) { __node_allocator& __alloc = __node_alloc(); __allocation_guard<__node_allocator> __guard(__alloc, 1); + // Begin the lifetime of the node itself. Note that this doesn't begin the lifetime of the value + // held inside the node, since we need to use the allocator's construct() method for that. + // + // We don't use the allocator's construct() method to construct the node itself since the + // Cpp17FooInsertable named requirements don't require the allocator's construct() method + // to work on anything other than the value_type. + // + // Note that we don't begin the lifetime properly in C++03 because that would cause the + // value_type to be default constructed. +#ifdef _LIBCPP_LIST_PROPER_NODE_LIFETIME + std::__construct_at(std::addressof(*__guard.__get()), __prev, __next); +#else __guard.__get()->__prev_ = __prev; __guard.__get()->__next_ = __next; +#endif + + // Now construct the value_type using the allocator's construct() method. __node_alloc_traits::construct(__alloc, std::addressof(__guard.__get()->__value_), std::forward<_Args>(__args)...); return __guard.__release_ptr(); } template _LIBCPP_HIDE_FROM_ABI void __delete_node(__node_pointer __node) { + // For the same reason as above, we use the allocator's destroy() method for the value_type, + // but not for the node itself. __node_allocator& __alloc = __node_alloc(); __node_alloc_traits::destroy(__alloc, std::addressof(__node->__value_)); +#ifdef _LIBCPP_LIST_PROPER_NODE_LIFETIME + std::__destroy_at(std::addressof(*__node)); +#endif __node_alloc_traits::deallocate(__alloc, __node, 1); }