diff --git a/libcxx/include/memory b/libcxx/include/memory --- a/libcxx/include/memory +++ b/libcxx/include/memory @@ -2570,6 +2570,100 @@ __a.deallocate(_PTraits::pointer_to(*this), 1); } +// This class implements the control block for non-array shared pointers created +// through `std::allocate_shared` and `std::make_shared`. +// +// Prior to C++20 (or if the compiler doesn't support [[no_unique_address]]), +// we use an implementation that performs EBO for both the allocator and the +// object type stored in the `shared_ptr`. This is the original implementation +// that has been used in libc++, and we must be ABI compatible with it. +// +// Starting in C++20, P0674 requires us to use Allocator construction for +// initializing the object type. That requirement rules out the use of the +// EBO for the object type, since using the EBO implies that the base will be +// initialized when the control block is initialized (and hence we can't do it +// through Allocator construction). Hence, supporting P0674 requires changing +// how we store the object type inside the control block, which we do while +// being ABI compatible by using [[no_unique_address]]. +// +// If [[no_unique_address]] is not supported by the compiler, we fall back to +// the old version that doesn't do Allocator construction -- that is better +// than breaking ABI. +// +// !!! THE TWO DEFINITIONS BELOW MUST ALWAYS STAY ABI COMPATIBLE !!! +#if _LIBCPP_STD_VER > 17 && __has_cpp_attribute(no_unique_address) +template +struct __shared_ptr_emplace + : __shared_weak_count +{ + template + _LIBCPP_HIDE_FROM_ABI + explicit __shared_ptr_emplace(_Alloc __a, _Args&& ...__args) + : __storage_{_VSTD::move(__a)} + { + using _TpAlloc = typename __allocator_traits_rebind<_Alloc, _Tp>::type; + _TpAlloc __tmp(*__get_alloc()); + allocator_traits<_TpAlloc>::construct(__tmp, __get_elem(), _VSTD::forward<_Args>(__args)...); + } + + _LIBCPP_HIDE_FROM_ABI + _Alloc* __get_alloc() _NOEXCEPT { return _VSTD::addressof(__storage_.__alloc_); } + + _LIBCPP_HIDE_FROM_ABI + _Tp* __get_elem() _NOEXCEPT { return _VSTD::addressof(__storage_.__data_); } + +private: + virtual void __on_zero_shared() _NOEXCEPT { + using _TpAlloc = typename __allocator_traits_rebind<_Alloc, _Tp>::type; + _TpAlloc __tmp(*__get_alloc()); + allocator_traits<_TpAlloc>::destroy(__tmp, __get_elem()); + } + + virtual void __on_zero_shared_weak() _NOEXCEPT { + using _ControlBlockAlloc = typename __allocator_traits_rebind<_Alloc, __shared_ptr_emplace>::type; + using _ControlBlockPointer = typename allocator_traits<_ControlBlockAlloc>::pointer; + _ControlBlockAlloc __tmp(*__get_alloc()); + __storage_.~_Storage(); + allocator_traits<_ControlBlockAlloc>::deallocate(__tmp, + pointer_traits<_ControlBlockPointer>::pointer_to(*this), 1); + } + + // This implements an ABI compatible emulation of libc++'s compressed pair + // where the data element is stored in a union, and hence it not default + // initialized when the "compressed pair" is initialized. + template ::value, bool = __libcpp_is_final<_TTp>::value> + struct _Storage; + template + struct _Storage<_AAlloc, _TTp, false, false> { + _LIBCPP_HIDE_FROM_ABI explicit _Storage(_Alloc&& __a) : __alloc_(_VSTD::move(__a)) { } + _LIBCPP_HIDE_FROM_ABI ~_Storage() { } + [[no_unique_address]] _Alloc __alloc_; + [[no_unique_address]] union { [[no_unique_address]] _Tp __data_; }; + }; + template + struct _Storage<_AAlloc, _TTp, false, true> { + _LIBCPP_HIDE_FROM_ABI explicit _Storage(_Alloc&& __a) : __alloc_(_VSTD::move(__a)) { } + _LIBCPP_HIDE_FROM_ABI ~_Storage() { } + [[no_unique_address]] _Alloc __alloc_; + union { _Tp __data_; }; + }; + template + struct _Storage<_AAlloc, _TTp, true, false> { + _LIBCPP_HIDE_FROM_ABI explicit _Storage(_Alloc&& __a) : __alloc_(_VSTD::move(__a)) { } + _LIBCPP_HIDE_FROM_ABI ~_Storage() { } + _Alloc __alloc_; + [[no_unique_address]] union { [[no_unique_address]] _Tp __data_; }; + }; + template + struct _Storage<_AAlloc, _TTp, true, true> { + _LIBCPP_HIDE_FROM_ABI explicit _Storage(_Alloc&& __a) : __alloc_(_VSTD::move(__a)) { } + _LIBCPP_HIDE_FROM_ABI ~_Storage() { } + _Alloc __alloc_; + union { _Tp __data_; }; + }; + _Storage<_Alloc, _Tp> __storage_; +}; +#else template struct __shared_ptr_emplace : __shared_weak_count @@ -2612,6 +2706,7 @@ __compressed_pair<_Alloc, _Tp> __data_; }; +#endif // >= C++20 && supports [[no_unique_address]] struct __shared_ptr_dummy_rebind_allocator_type; template <> diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/libcxx.control_block_layout.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/libcxx.control_block_layout.pass.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/libcxx.control_block_layout.pass.cpp @@ -0,0 +1,164 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03 +// REQUIRES: libc++ + +// This test makes sure that the control block implementation used for non-array +// types in std::make_shared and std::allocate_shared is ABI compatbile with the +// original implementation. +// +// This test is relevant because the implementation of that control block is +// different starting in C++20, a change that was required to implement P0674. + +#include +#include +#include +#include +#include + +#include +#include + +#include "test_macros.h" + +// This is the pre-C++20 implementation of the control block used by non-array +// std::allocate_shared and std::make_shared. We keep it here so that we can +// make sure our implementation is backwards compatible with it forever. +// +// Of course, the class and its methods were renamed, but the size and layout +// of the class should remain the same as the original implementation. +template +struct OldEmplaceControlBlock + : std::__shared_weak_count +{ + explicit OldEmplaceControlBlock(Alloc a) : data_(std::move(a), std::__value_init_tag()) { } + T* get_elem() noexcept { return std::addressof(data_.second()); } + Alloc* get_alloc() noexcept { return std::addressof(data_.first()); } + +private: + virtual void __on_zero_shared() noexcept { + // Not implemented + } + + virtual void __on_zero_shared_weak() noexcept { + // Not implemented + } + + std::__compressed_pair data_; +}; + +template class Alloc> +void test() { + using Old = OldEmplaceControlBlock>; + using New = std::__shared_ptr_emplace>; + + static_assert(sizeof(New) == sizeof(Old), ""); + static_assert(alignof(New) == alignof(Old), ""); + + // Also make sure each member is at the same offset + Alloc a; + Old old(a); + New new_(a); + + // 1. Check the stored object + { + char const* old_elem = reinterpret_cast(old.get_elem()); + char const* new_elem = reinterpret_cast(new_.__get_elem()); + std::ptrdiff_t old_offset = old_elem - reinterpret_cast(&old); + std::ptrdiff_t new_offset = new_elem - reinterpret_cast(&new_); + assert(new_offset == old_offset && "offset of stored element changed"); + } + + // 2. Check the allocator + { + char const* old_alloc = reinterpret_cast(old.get_alloc()); + char const* new_alloc = reinterpret_cast(new_.__get_alloc()); + std::ptrdiff_t old_offset = old_alloc - reinterpret_cast(&old); + std::ptrdiff_t new_offset = new_alloc - reinterpret_cast(&new_); + assert(new_offset == old_offset && "offset of allocator changed"); + } + + // Make sure both types have the same triviality (that has ABI impact since + // it determined how objects are passed). Both should be non-trivial. + static_assert(std::is_trivial::value == std::is_trivial::value, ""); +} + +// Object types to store in the control block +struct TrivialEmptyType { }; +struct TrivialNonEmptyType { char c[11]; }; +struct FinalEmptyType final { }; +struct NonTrivialType { + char c[22]; + NonTrivialType() : c{'x'} { } +}; + +// Allocator types +template +struct TrivialEmptyAlloc { + using value_type = T; + TrivialEmptyAlloc() = default; + template TrivialEmptyAlloc(TrivialEmptyAlloc) { } + T* allocate(std::size_t) { return nullptr; } + void deallocate(T*, std::size_t) { } +}; +template +struct TrivialNonEmptyAlloc { + char storage[77]; + using value_type = T; + TrivialNonEmptyAlloc() = default; + template TrivialNonEmptyAlloc(TrivialNonEmptyAlloc) { } + T* allocate(std::size_t) { return nullptr; } + void deallocate(T*, std::size_t) { } +}; +template +struct FinalEmptyAlloc final { + using value_type = T; + FinalEmptyAlloc() = default; + template FinalEmptyAlloc(FinalEmptyAlloc) { } + T* allocate(std::size_t) { return nullptr; } + void deallocate(T*, std::size_t) { } +}; +template +struct NonTrivialAlloc { + char storage[88]; + using value_type = T; + NonTrivialAlloc() { } + template NonTrivialAlloc(NonTrivialAlloc) { } + T* allocate(std::size_t) { return nullptr; } + void deallocate(T*, std::size_t) { } +}; + +int main(int, char**) { + test(); + test(); + test(); + test(); + + test(); + test(); + test(); + test(); + + test(); + test(); + test(); + test(); + + test(); + test(); + test(); + test(); + + // Test a few real world types just to make sure we didn't mess up badly somehow + test(); + test(); + test, std::allocator>(); + + return 0; +} diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared.pass.cpp --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared.pass.cpp @@ -94,6 +94,22 @@ int Three::count = 0; +template +struct AllocNoConstruct : std::allocator +{ + AllocNoConstruct() = default; + + template + AllocNoConstruct(AllocNoConstruct) {} + + template + struct rebind { + typedef AllocNoConstruct other; + }; + + void construct(void*) { assert(false); } +}; + template void test() { @@ -161,5 +177,12 @@ } assert(A::count == 0); + // Test that we don't call construct before C++20. +#if TEST_STD_VER < 20 + { + (void)std::allocate_shared(AllocNoConstruct()); + } +#endif // TEST_STD_VER < 20 + return 0; } diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared_construct.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared_construct.pass.cpp new file mode 100644 --- /dev/null +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared_construct.pass.cpp @@ -0,0 +1,176 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17 + +// + +// shared_ptr + +// template +// shared_ptr allocate_shared(const A& a, Args&&... args); + +// This test checks that allocator_traits::construct is used in allocate_shared +// as requested in C++20 (via P0674R1). + +#include "test_macros.h" + +#include +#include + +static bool construct_called = false; +static bool destroy_called = false; +static unsigned allocator_id = 0; + +template +struct MyAllocator { +public: + typedef T value_type; + typedef T* pointer; + + unsigned id = 0; + + MyAllocator() = default; + MyAllocator(int id) : id(id) {} + + template + MyAllocator(MyAllocator const& other) : id(other.id){}; + + pointer allocate(std::ptrdiff_t n) { + return pointer(static_cast(::operator new(n * sizeof(T)))); + } + + void deallocate(pointer p, std::ptrdiff_t) { return ::operator delete(p); } + + template + void construct(T* p, Args&& ...args) { + construct_called = true; + destroy_called = false; + allocator_id = id; + ::new (p) T(std::forward(args)...); + } + + void destroy(T* p) { + construct_called = false; + destroy_called = true; + allocator_id = id; + p->~T(); + } +}; + +struct Private; + +class Factory { +public: + static std::shared_ptr allocate(); +}; + +template +struct FactoryAllocator; + +struct Private { + int id; + +private: + friend FactoryAllocator; + Private(int id) : id(id) {} + ~Private() {} +}; + +template +struct FactoryAllocator : std::allocator { + FactoryAllocator() = default; + + template + FactoryAllocator(FactoryAllocator) {} + + template + struct rebind { + typedef FactoryAllocator other; + }; + + void construct(void* p, int id) { ::new (p) Private(id); } + void destroy(Private* p) { p->~Private(); } +}; + +std::shared_ptr Factory::allocate() { + FactoryAllocator factory_alloc; + return std::allocate_shared(factory_alloc, 42); +} + +struct mchar { + char c; +}; + +struct Foo { + int val; + + Foo(int val) : val(val) {} + + Foo(Foo a, Foo b) : val(a.val + b.val) {} +}; + +struct Bar { + std::max_align_t y; +}; + +void test_aligned(void* p, size_t align) { + assert(reinterpret_cast(p) % align == 0); +} + +int main(int, char**) { + { + std::shared_ptr p = std::allocate_shared(MyAllocator()); + assert(construct_called); + } + assert(destroy_called); + { + std::shared_ptr p = + std::allocate_shared(MyAllocator(), Foo(42), Foo(100)); + assert(construct_called); + assert(p->val == 142); + } + assert(destroy_called); + { // Make sure allocator is copied. + std::shared_ptr p = std::allocate_shared(MyAllocator(3)); + assert(allocator_id == 3); + + allocator_id = 0; + } + assert(allocator_id == 3); + + { + std::shared_ptr p = std::allocate_shared(MyAllocator(), 42); + assert(construct_called); + assert(*p == 42); + } + assert(destroy_called); + + { // Make sure allocator is properly re-bound. + std::shared_ptr p = + std::allocate_shared(MyAllocator(), 42); + assert(construct_called); + assert(*p == 42); + } + assert(destroy_called); + + { + // Make sure that we call the correct allocator::construct. Private has a private constructor + // so the construct method must be called on its friend Factory's allocator + // (Factory::Allocator). + std::shared_ptr p = Factory().allocate(); + assert(p->id == 42); + } + + { + std::shared_ptr p; + test_aligned(p.get(), alignof(Bar)); + } + + return 0; +}