Partially inline copy constructor basic_string(const basic_string&[, allocator]) - ALTERNATIVE to https://reviews.llvm.org/D68617

[mvels]

Summary

Partially inline copy constructor basic_string(const basic_string&[, allocator])

This change optimizes the copy constructor using partial inlining.

• adds default_value_tag() to memory, to support default initialization
• inlines copy contructor: non SSO init delegated to instantiated __init_long() method

Note that this change does not consider existing value initialization ctors, most would likely benefit from default initialization.

Generated code is small, i.e, considerably smaller than other hot inlined functions such as move ctor

given:

void StringCopyCtor(void* mem, const std::string& s) {
    std::string*p = new(mem) std::string{s};
}

asm:

        cmp     byte ptr [rsi + 23], 0
        js      .LBB0_2
        mov     rax, qword ptr [rsi + 16]
        mov     qword ptr [rdi + 16], rax
        movups  xmm0, xmmword ptr [rsi]
        movups  xmmword ptr [rdi], xmm0
        ret
.LBB0_2:
        jmp     std::basic_string::__init_long # TAILCALL

Benchmarks::
BM_StringCopy_Empty                                           5.18ns ± 7%             1.53ns ± 5%  -70.45%        (p=0.000 n=10+10)
BM_StringCopy_Small                                           5.18ns ± 7%             1.54ns ± 5%  -70.21%        (p=0.000 n=10+10)
BM_StringCopy_Large                                           19.0ns ± 1%             19.3ns ± 1%   +1.77%        (p=0.000 n=10+10)
BM_StringCopy_Huge                                             321ns ± 4%              310ns ± 1%     ~            (p=0.408 n=10+8)

[mvels]

• Remove old defines

[mvels]

• Relocate define in __config

[mvels]

• fix ifdef for __init_long

libcxx/include/__config

 #  endif
 // Feature macros for disabling pre ABI v1 features. All of these options
 // are deprecated.
 #  if defined(__FreeBSD__)
 #    define _LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR
 #  endif
 #endif
 
+#ifdef _LIBCPP_ABI_UNSTABLE
+// Optimize the copy constructor to inline SSO copies.
+// This removes the copy constructor from the ABI, and depends on
+// the presence of __init_long() in the dylib / lib.
+#  define _LIBCPP_ABI_OPTIMIZE_COPY_CTOR
+#endif
+
 #ifdef _LIBCPP_TRIVIAL_PAIR_COPY_CTOR
 #error "_LIBCPP_TRIVIAL_PAIR_COPY_CTOR" is no longer supported. \
        use _LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR instead
 #endif
 
 #define _LIBCPP_CONCAT1(_LIBCPP_X,_LIBCPP_Y) _LIBCPP_X##_LIBCPP_Y
 #define _LIBCPP_CONCAT(_LIBCPP_X,_LIBCPP_Y) _LIBCPP_CONCAT1(_LIBCPP_X,_LIBCPP_Y)
 

libcxx/include/memory

 template <>
 class _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX11 auto_ptr<void>
 {
 public:
     typedef void element_type;
 };
 #endif
 
+// Tag used to default initialize one or both of the pair's elements.
+struct __default_value_tag {};
+
 template <class _Tp, int _Idx,
           bool _CanBeEmptyBase =
               is_empty<_Tp>::value && !__libcpp_is_final<_Tp>::value>
 struct __compressed_pair_elem {
   typedef _Tp _ParamT;
   typedef _Tp& reference;
   typedef const _Tp& const_reference;
 
 #ifndef _LIBCPP_CXX03_LANG
   _LIBCPP_INLINE_VISIBILITY constexpr __compressed_pair_elem() : __value_() {}
+  _LIBCPP_INLINE_VISIBILITY constexpr __compressed_pair_elem(__default_value_tag) {}
 
   template <class _Up, class = typename enable_if<
       !is_same<__compressed_pair_elem, typename decay<_Up>::type>::value
   >::type>
   _LIBCPP_INLINE_VISIBILITY
   constexpr explicit
   __compressed_pair_elem(_Up&& __u)
       : __value_(_VSTD::forward<_Up>(__u))

libcxx/include/string

public:
 
     _LIBCPP_INLINE_VISIBILITY explicit basic_string(const allocator_type& __a)
 #if _LIBCPP_STD_VER <= 14
         _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value);
 #else
         _NOEXCEPT;
 #endif
 
+#ifdef _LIBCPP_ABI_OPTIMIZE_COPY_CTOR
+    _LIBCPP_HIDE_FROM_ABI
+#endif
     basic_string(const basic_string& __str);
+#ifdef _LIBCPP_ABI_OPTIMIZE_COPY_CTOR
+    _LIBCPP_HIDE_FROM_ABI
+#endif
     basic_string(const basic_string& __str, const allocator_type& __a);
 
 #ifndef _LIBCPP_CXX03_LANG
     _LIBCPP_INLINE_VISIBILITY
     basic_string(basic_string&& __str)
 #if _LIBCPP_STD_VER <= 14
         _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value);
 #else
#endif  // _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
 
     inline
     void __init(const value_type* __s, size_type __sz, size_type __reserve);
     inline
     void __init(const value_type* __s, size_type __sz);
     inline
     void __init(size_type __n, value_type __c);
 
+#ifndef _LIBCPP_ABI_OPTIMIZE_COPY_CTOR
+    _LIBCPP_HIDE_FROM_ABI
+#endif
+    void __init_long(const basic_string& __str);
+
     template <class _InputIterator>
     inline
     typename enable_if
     <
         __is_exactly_input_iterator<_InputIterator>::value,
         void
     >::type
     __init(_InputIterator __first, _InputIterator __last);
else
         __set_long_cap(__cap+1);
         __set_long_size(__sz);
     }
     traits_type::copy(_VSTD::__to_raw_pointer(__p), __s, __sz);
     traits_type::assign(__p[__sz], value_type());
 }
 
 template <class _CharT, class _Traits, class _Allocator>
+#ifndef _LIBCPP_ABI_OPTIMIZE_COPY_CTOR
+inline
+#endif
+void basic_string<_CharT, _Traits, _Allocator>::__init_long(
+    const basic_string& __str) {
+  const value_type* __s = _VSTD::__to_raw_pointer(__str.__get_long_pointer());
+  size_type __sz = __str.__get_long_size();
+  size_type __cap = __recommend(__sz);
+  pointer __p = __alloc_traits::allocate(__alloc(), __cap + 1);
+  __set_long_pointer(__p);
+  __set_long_cap(__cap + 1);
+  __set_long_size(__sz);
+  traits_type::copy(_VSTD::__to_raw_pointer(__p), __s, __sz);
+  traits_type::assign(__p[__sz], value_type());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
 template <class>
 basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, const _Allocator& __a)
     : __r_(__second_tag(), __a)
 {
     _LIBCPP_ASSERT(__s != nullptr, "basic_string(const char*, allocator) detected nullptr");
     __init(__s, traits_type::length(__s));
 #if _LIBCPP_DEBUG_LEVEL >= 2
     __get_db()->__insert_c(this);
basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, size_type __n, const _Allocator& __a)
     _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "basic_string(const char*, n, allocator) detected nullptr");
     __init(__s, __n);
 #if _LIBCPP_DEBUG_LEVEL >= 2
     __get_db()->__insert_c(this);
 #endif
 }
 
 template <class _CharT, class _Traits, class _Allocator>
+#ifdef _LIBCPP_ABI_OPTIMIZE_COPY_CTOR
+inline
+#endif
 basic_string<_CharT, _Traits, _Allocator>::basic_string(const basic_string& __str)
-    : __r_(__second_tag(), __alloc_traits::select_on_container_copy_construction(__str.__alloc()))
+    : __r_(__default_value_tag(), __alloc_traits::select_on_container_copy_construction(__str.__alloc()))
 {
     if (!__str.__is_long())
         __r_.first().__r = __str.__r_.first().__r;
     else
-        __init(_VSTD::__to_raw_pointer(__str.__get_long_pointer()), __str.__get_long_size());
+        __init_long(__str);
 #if _LIBCPP_DEBUG_LEVEL >= 2
     __get_db()->__insert_c(this);
 #endif
 }
 
 template <class _CharT, class _Traits, class _Allocator>
+#ifdef _LIBCPP_ABI_OPTIMIZE_COPY_CTOR
+inline
+#endif
 basic_string<_CharT, _Traits, _Allocator>::basic_string(
     const basic_string& __str, const allocator_type& __a)
-    : __r_(__second_tag(), __a)
+    : __r_(__default_value_tag(), __a)
 {
     if (!__str.__is_long())
         __r_.first().__r = __str.__r_.first().__r;
     else
-        __init(_VSTD::__to_raw_pointer(__str.__get_long_pointer()), __str.__get_long_size());
+        __init_long(__str);
 #if _LIBCPP_DEBUG_LEVEL >= 2
     __get_db()->__insert_c(this);
 #endif
 }
 
 #ifndef _LIBCPP_CXX03_LANG
 
 template <class _CharT, class _Traits, class _Allocator>