Optimize / partially inline basic_string assignment operator

[mvels]

Summary

This change adds an optimization where assigning a short string value to a short string instance is optimized into a direct memory copy.

'As is', this optimization provides a 2X to 3X+ speed up.

BM_StringAssignStr_Empty_Opaque 5.23ns ± 0% 2.27ns ± 0% -56.62% (p=0.008 n=5+5)
BM_StringAssignStr_Empty_Transparent 5.14ns ± 0% 2.27ns ± 0% -55.89% (p=0.008 n=5+5)
BM_StringAssignStr_Small_Opaque 7.69ns ± 0% 2.27ns ± 0% -70.56% (p=0.008 n=5+5)
BM_StringAssignStr_Small_Transparent 7.65ns ± 0% 2.26ns ± 0% -70.41% (p=0.008 n=5+5)
BM_StringAssignStr_Large_Opaque 24.4ns ± 0% 25.6ns ± 0% +5.11% (p=0.008 n=5+5)
BM_StringAssignStr_Large_Transparent 22.2ns ± 0% 22.9ns ± 0% +3.43% (p=0.008 n=5+5)
BM_StringAssignStr_Huge_Opaque 320ns ± 3% 320ns ± 4% ~ (p=0.841 n=5+5)
BM_StringAssignStr_Huge_Transparent 321ns ± 4% 321ns ± 4% ~ (p=1.000 n=5+5)

Subsequently changing the assignment operator to be inlined (not externally instantiated) provides a speed up of 3X to 5X.

BM_StringAssignStr_Empty_Opaque 5.25ns ± 1% 1.71ns ± 0% -67.33% (p=0.008 n=5+5)
BM_StringAssignStr_Empty_Transparent 5.13ns ± 0% 1.29ns ± 0% -74.89% (p=0.008 n=5+5)
BM_StringAssignStr_Small_Opaque 7.69ns ± 0% 1.71ns ± 0% -77.70% (p=0.008 n=5+5)
BM_StringAssignStr_Small_Transparent 7.64ns ± 0% 1.29ns ± 0% -83.10% (p=0.008 n=5+5)
BM_StringAssignStr_Large_Opaque 24.4ns ± 0% 25.3ns ± 0% +3.97% (p=0.008 n=5+5)
BM_StringAssignStr_Large_Transparent 22.2ns ± 0% 22.6ns ± 0% +2.13% (p=0.008 n=5+5)
BM_StringAssignStr_Huge_Opaque 325ns ± 5% 315ns ± 6% ~ (p=0.222 n=5+5)
BM_StringAssignStr_Huge_Transparent 325ns ± 5% 316ns ± 5% ~ (p=0.310 n=5+5)

libcxx/include/string

public:
 
     _LIBCPP_INLINE_VISIBILITY explicit basic_string(const allocator_type& __a)
 #if _LIBCPP_STD_VER <= 14
         _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value);
 #else
         _NOEXCEPT;
 #endif
 
+    // Optimization opportunity: do not externally instantiate the copy
+    // constructor, which inlines short string initialization. Long string
+    // initialization is delegated to the (external) __init_long()method,
+    // which results in a 3X-4X speed up for SSO initialization.
     basic_string(const basic_string& __str);
     basic_string(const basic_string& __str, const allocator_type& __a);
 
 #ifndef _LIBCPP_CXX03_LANG
     _LIBCPP_INLINE_VISIBILITY
     basic_string(basic_string&& __str)
 #if _LIBCPP_STD_VER <= 14
         _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value);
#ifndef _LIBCPP_CXX03_LANG
     basic_string(initializer_list<_CharT> __il, const _Allocator& __a);
 #endif  // _LIBCPP_CXX03_LANG
 
     inline ~basic_string();
 
     _LIBCPP_INLINE_VISIBILITY
     operator __self_view() const _NOEXCEPT { return __self_view(data(), size()); }
 
+    // Optimization opportunity: do not externally instantiate
     basic_string& operator=(const basic_string& __str);
 
     template <class _Tp, class = typename enable_if<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, void>::type>
     basic_string& operator=(const _Tp& __t)
         {__self_view __sv = __t; return assign(__sv);}
 
 #ifndef _LIBCPP_CXX03_LANG
     _LIBCPP_INLINE_VISIBILITY
#endif  // _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
 
     inline
     void __init(const value_type* __s, size_type __sz, size_type __reserve);
     inline
     void __init(const value_type* __s, size_type __sz);
     inline
     void __init(size_type __n, value_type __c);
 
+    void __init_long(const basic_string& __str);
+
     template <class _InputIterator>
     inline
     typename enable_if
     <
         __is_exactly_cpp17_input_iterator<_InputIterator>::value,
         void
     >::type
     __init(_InputIterator __first, _InputIterator __last);
else
         __set_long_cap(__cap+1);
         __set_long_size(__sz);
     }
     traits_type::copy(_VSTD::__to_address(__p), __s, __sz);
     traits_type::assign(__p[__sz], value_type());
 }
 
 template <class _CharT, class _Traits, class _Allocator>
+void basic_string<_CharT, _Traits, _Allocator>::__init_long(
+    const basic_string& __str) {
+  const value_type* __s = _VSTD::__to_address(__str.__get_long_pointer());
+  size_type __sz = __str.__get_long_size();
+  size_type __cap = __recommend(__sz);
+  pointer __p = __alloc_traits::allocate(__alloc(), __cap + 1);
+  __set_long_pointer(__p);
+  __set_long_cap(__cap + 1);
+  __set_long_size(__sz);
+  traits_type::copy(_VSTD::__to_address(__p), __s, __sz);
+  traits_type::assign(__p[__sz], value_type());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
 template <class>
 basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, const _Allocator& __a)
     : __r_(__default_init_tag(), __a)
 {
     _LIBCPP_ASSERT(__s != nullptr, "basic_string(const char*, allocator) detected nullptr");
     __init(__s, traits_type::length(__s));
 #if _LIBCPP_DEBUG_LEVEL >= 2
     __get_db()->__insert_c(this);

 
 template <class _CharT, class _Traits, class _Allocator>
 basic_string<_CharT, _Traits, _Allocator>::basic_string(const basic_string& __str)
     : __r_(__default_init_tag(), __alloc_traits::select_on_container_copy_construction(__str.__alloc()))
 {
     if (!__str.__is_long())
         __r_.first().__r = __str.__r_.first().__r;
     else
-        __init(_VSTD::__to_address(__str.__get_long_pointer()), __str.__get_long_size());
+        __init_long(__str);
 #if _LIBCPP_DEBUG_LEVEL >= 2
     __get_db()->__insert_c(this);
 #endif
 }
 
 template <class _CharT, class _Traits, class _Allocator>
 basic_string<_CharT, _Traits, _Allocator>::basic_string(
     const basic_string& __str, const allocator_type& __a)
     : __r_(__default_init_tag(), __a)
 {
     if (!__str.__is_long())
         __r_.first().__r = __str.__r_.first().__r;
     else
-        __init(_VSTD::__to_address(__str.__get_long_pointer()), __str.__get_long_size());
+        __init_long(__str);
 #if _LIBCPP_DEBUG_LEVEL >= 2
     __get_db()->__insert_c(this);
 #endif
 }
 
 #ifndef _LIBCPP_CXX03_LANG
 
 template <class _CharT, class _Traits, class _Allocator>

 
 template <class _CharT, class _Traits, class _Allocator>
 basic_string<_CharT, _Traits, _Allocator>&
 basic_string<_CharT, _Traits, _Allocator>::operator=(const basic_string& __str)
 {
     if (this != &__str)
     {
         __copy_assign_alloc(__str);
+        if (__is_long() | __str.__is_long()) {  // LINT: explicit binary or.
           return assign(__str.data(), __str.size());
         }
+        __r_.first().__r = __str.__r_.first().__r;
+    }
     return *this;
 }
 
 #ifndef _LIBCPP_CXX03_LANG
 
 template <class _CharT, class _Traits, class _Allocator>
 inline
 void