[libc++] Add __small_buffer

[philnik]

Summary

This is an implementation detail for move_only_function (and potentially other type-erasing classes).

[EricWF]

Why?

Changes need descriptions.

[philnik]

Generate files

[philnik]

Try to fix CI

[ldionne]

libcxx/include/__utility/small_buffer.h

25–28

38–43

This avoids instantiating a lambda and it's pretty easy to do, so we might as well.

45–52

I think the = default ones should be _LIBCPP_HIDE_FROM_ABI, since they *are* generated at the end of the day.

[Mordante]

I mainly glossed over it to see what it does.

libcxx/include/__utility/small_buffer.h

35

Small nit, since this is C++23, this could be a requires instead of a static_assert.

[philnik]

Address comments

[philnik]

• Rebased
• Try to fix CI

[philnik]

Rebased

[philnik]

format

libcxx/include/CMakeLists.txt

set(files
   __utility/forward_like.h
   __utility/in_place.h
   __utility/integer_sequence.h
   __utility/move.h
   __utility/pair.h
   __utility/piecewise_construct.h
   __utility/priority_tag.h
   __utility/rel_ops.h
+  __utility/small_buffer.h
   __utility/swap.h
   __utility/to_underlying.h
   __utility/transaction.h
   __utility/unreachable.h
   __variant/monostate.h
   __verbose_abort
   algorithm
   any

libcxx/include/__utility/small_buffer.h

+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_SMALL_BUFFER_H
+#define _LIBCPP___UTILITY_SMALL_BUFFER_H
+
+#include <__config>
+#include <__memory/construct_at.h>
+#include <__utility/forward.h>
+#include <__utility/transaction.h>
+#include <cstddef>
+#include <new>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+
+// __small_buffer helper for type-erasing classes like move_only_function to store small objects in a local buffer.
+// Only objects that are trivially relocatable can be stored inside it to allow __small_buffer to be trivially
+// relocatable itself. Since the buffer doesn't know what's stored inside it, the user has to manage the object's
+// lifetime, in particular the destruction of the object.
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <size_t _BufferSize, size_t _BufferAlignment>
+class __small_buffer {
+  static_assert(_BufferSize > 0, "The buffer size should not be zero");
+  static_assert(_BufferAlignment > 0, "The buffer alignment should not be zero");
+
+public:
+  template <class _Tp>
+  static constexpr bool __fits_in_buffer = [] {
+    using _DTp = decay_t<_Tp>;
+    return is_trivially_move_constructible_v<_DTp> && is_trivially_destructible_v<_DTp> &&
+           sizeof(_DTp) <= _BufferSize && alignof(_DTp) <= _BufferAlignment;
+  }();
+
+  __small_buffer()                                 = default;
+  __small_buffer(const __small_buffer&)            = delete;
+  __small_buffer& operator=(const __small_buffer&) = delete;
+  ~__small_buffer()                                = default;
+
+  // Relocates the buffer - __delete() should never be called on a moved-from __small_buffer
+  __small_buffer(__small_buffer&&)            = default;
+  __small_buffer& operator=(__small_buffer&&) = default;
+
+  template <class _Stored>
+  _LIBCPP_HIDE_FROM_ABI _Stored* __get() {
+    if constexpr (__fits_in_buffer<_Stored>)
+      return std::launder(reinterpret_cast<_Stored*>(__buffer_));
+    else
+      return *std::launder(reinterpret_cast<_Stored**>(__buffer_));
+  }
+
+  template <class _Stored>
+  _LIBCPP_HIDE_FROM_ABI _Stored* __alloc() {
+    if constexpr (__fits_in_buffer<_Stored>) {
+      return std::launder(reinterpret_cast<_Stored*>(__buffer_));
+    } else {
+      byte* __allocation = static_cast<byte*>(::operator new[](sizeof(_Stored), align_val_t{alignof(_Stored)}));
+      std::construct_at(reinterpret_cast<byte**>(__buffer_), __allocation);
+      return std::launder(reinterpret_cast<_Stored*>(__allocation));
+    }
+  }
+
+  template <class _Stored>
+  _LIBCPP_HIDE_FROM_ABI void __dealloc() noexcept {
+    if constexpr (!__fits_in_buffer<_Stored>)
+      ::operator delete[](*reinterpret_cast<void**>(__buffer_), sizeof(_Stored), align_val_t{alignof(_Stored)});
+  }
+
+  template <class _Stored, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI void __construct(_Args&&... __args) {
+    _Stored* __buffer = __alloc<_Stored>();
+    __transaction __guard([&] { __dealloc<_Stored>(); });
+    std::construct_at(__buffer, std::forward<_Args>(__args)...);
+    __guard.__complete();
+  }
+
+private:
+  alignas(_BufferAlignment) byte __buffer_[_BufferSize];
+};
+
+#  undef _LIBCPP_SMALL_BUFFER_TRIVIAL_ABI
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 23
+
+#endif // _LIBCPP___UTILITY_SMALL_BUFFER_H

libcxx/include/module.modulemap.in

module __utility {
       module in_place            { private header "__utility/in_place.h" }
       module integer_sequence    { private header "__utility/integer_sequence.h" }
       module move                { private header "__utility/move.h" }
       module pair                { private header "__utility/pair.h" }
       module pair_fwd            { private header "__fwd/pair.h" }
       module piecewise_construct { private header "__utility/piecewise_construct.h" }
       module priority_tag        { private header "__utility/priority_tag.h" }
       module rel_ops             { private header "__utility/rel_ops.h" }
+      module small_buffer        { private header "__utility/small_buffer.h" }
       module swap                { private header "__utility/swap.h" }
       module to_underlying       { private header "__utility/to_underlying.h" }
       module transaction         { private header "__utility/transaction.h" }
       module unreachable         { private header "__utility/unreachable.h" }
     }
   }
   module valarray {
     header "valarray"

libcxx/test/libcxx/private_headers.verify.cpp

 #include <__utility/forward_like.h> // expected-error@*:* {{use of private header from outside its module: '__utility/forward_like.h'}}
 #include <__utility/in_place.h> // expected-error@*:* {{use of private header from outside its module: '__utility/in_place.h'}}
 #include <__utility/integer_sequence.h> // expected-error@*:* {{use of private header from outside its module: '__utility/integer_sequence.h'}}
 #include <__utility/move.h> // expected-error@*:* {{use of private header from outside its module: '__utility/move.h'}}
 #include <__utility/pair.h> // expected-error@*:* {{use of private header from outside its module: '__utility/pair.h'}}
 #include <__utility/piecewise_construct.h> // expected-error@*:* {{use of private header from outside its module: '__utility/piecewise_construct.h'}}
 #include <__utility/priority_tag.h> // expected-error@*:* {{use of private header from outside its module: '__utility/priority_tag.h'}}
 #include <__utility/rel_ops.h> // expected-error@*:* {{use of private header from outside its module: '__utility/rel_ops.h'}}
+#include <__utility/small_buffer.h> // expected-error@*:* {{use of private header from outside its module: '__utility/small_buffer.h'}}
 #include <__utility/swap.h> // expected-error@*:* {{use of private header from outside its module: '__utility/swap.h'}}
 #include <__utility/to_underlying.h> // expected-error@*:* {{use of private header from outside its module: '__utility/to_underlying.h'}}
 #include <__utility/transaction.h> // expected-error@*:* {{use of private header from outside its module: '__utility/transaction.h'}}
 #include <__utility/unreachable.h> // expected-error@*:* {{use of private header from outside its module: '__utility/unreachable.h'}}
 #include <__variant/monostate.h> // expected-error@*:* {{use of private header from outside its module: '__variant/monostate.h'}}
 // GENERATED-MARKER

libcxx/test/libcxx/utilities/utility/small_buffer.pass.cpp

+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+#include "test_macros.h"
+
+TEST_DIAGNOSTIC_PUSH
+TEST_CLANG_DIAGNOSTIC_IGNORED("-Wprivate-header")
+#include <__utility/small_buffer.h>
+TEST_DIAGNOSTIC_POP
+
+#include <cassert>
+#include <memory>
+#include <utility>
+
+struct NotTriviallyRelocatable {
+  char c_;
+
+  NotTriviallyRelocatable(char c) : c_(c) {}
+  ~NotTriviallyRelocatable() {}
+};
+
+struct alignas(16) Overaligned {
+  int i;
+};
+
+int main(int, char**) {
+  using BufferT = std::__small_buffer<8, 8>;
+  static_assert(sizeof(BufferT) == 8);
+  static_assert(alignof(BufferT) == 8);
+  static_assert(BufferT::__fits_in_buffer<int>);
+  static_assert(!BufferT::__fits_in_buffer<Overaligned>);
+  static_assert(!BufferT::__fits_in_buffer<NotTriviallyRelocatable>);
+
+  BufferT buf;
+
+  { // construct/destroy in the same place
+    buf.__construct<int>(3);
+    assert(*buf.__get<int>() == 3);
+    std::destroy_at(buf.__get<int>());
+    buf.__dealloc<int>();
+
+    buf.__construct<NotTriviallyRelocatable>(3);
+    assert(buf.__get<NotTriviallyRelocatable>()->c_ == 3);
+    std::destroy_at(buf.__get<NotTriviallyRelocatable>());
+    buf.__dealloc<NotTriviallyRelocatable>();
+  }
+
+  { // Move the buffer around
+    buf.__construct<int>(3);
+    assert(*buf.__get<int>() == 3);
+    auto buf2 = std::move(buf);
+    assert(*buf2.__get<int>() == 3);
+    std::destroy_at(buf2.__get<int>());
+    buf2.__dealloc<int>();
+
+    buf.__construct<NotTriviallyRelocatable>(3);
+    assert(buf.__get<NotTriviallyRelocatable>()->c_ == 3);
+    auto buf3 = std::move(buf);
+    assert(buf3.__get<NotTriviallyRelocatable>()->c_ == 3);
+    std::destroy_at(buf3.__get<NotTriviallyRelocatable>());
+    buf3.__dealloc<NotTriviallyRelocatable>();
+  }
+
+  return 0;
+}