Optimize basic_string::assign(s [, n]) for compile time known short strings.

[mvels]

Summary

This change dispatches assignments of strings which length is known at compile time and smaller than min_cap to an inlined assign_in_place() method. This allows the compiler to fully optimize the assignment, removing outline memcpy calls, resulting in up to 6X speed ups for small string assignments.

BM_StringAssignAsciiz_Empty_Transparent 7.30ns ± 0% 1.01ns ± 0% -86.19% (p=0.008 n=5+5)
BM_StringAssignAsciiz_Small_Opaque 10.9ns ± 0% 9.9ns ± 0% -9.06% (p=0.008 n=5+5)
BM_StringAssignAsciiz_Small_Transparent 9.92ns ± 0% 1.33ns ± 0% -86.61% (p=0.008 n=5+5)
BM_StringAssignAsciiz_Large_Opaque 33.7ns ± 0% 30.2ns ± 1% -10.21% (p=0.008 n=5+5)
BM_StringAssignAsciiz_Large_Transparent 30.8ns ± 0% 27.2ns ± 0% -11.53% (p=0.008 n=5+5)
BM_StringAssignAsciiz_Huge_Opaque 388ns ± 4% 391ns ± 3% ~ (p=1.000 n=5+5)
BM_StringAssignAsciiz_Huge_Transparent 385ns ± 4% 388ns ± 3% ~ (p=1.000 n=5+5)
BM_StringAssignAsciizMix_Opaque 15.9ns ± 0% 14.1ns ± 0% -11.62% (p=0.008 n=5+5)
BM_StringAssignAsciizMix_Transparent 15.4ns ± 0% 7.5ns ± 1% -50.97% (p=0.008 n=5+5)

libcxx/include/__config

 #  define _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
 // Use the smallest possible integer type to represent the index of the variant.
 // Previously libc++ used "unsigned int" exclusively.
 #  define _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
 // Unstable attempt to provide a more optimized std::function
 #  define _LIBCPP_ABI_OPTIMIZED_FUNCTION
 // All the regex constants must be distinct and nonzero.
 #  define _LIBCPP_ABI_REGEX_CONSTANTS_NONZERO
+// Inline assign(s, [, n]) code, performing inline assignment for strings
+// which length is known at compile time and smaller than __min_cap, resulting
+// in up to 6X speed ups for (asciiz) string assignments.
+#  define _LIBCPP_ABI_PARTIALLY_INLINE_ASSIGN
 #elif _LIBCPP_ABI_VERSION == 1
 #  if !defined(_LIBCPP_OBJECT_FORMAT_COFF)
 // Enable compiling copies of now inline methods into the dylib to support
 // applications compiled against older libraries. This is unnecessary with
 // COFF dllexport semantics, since dllexport forces a non-inline definition
 // of inline functions to be emitted anyway. Our own non-inline copy would
 // conflict with the dllexport-emitted copy, so we disable it.
 #    define _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS

 #if !defined(_LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK) && __has_attribute(no_sanitize)
 #define _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK __attribute__((__no_sanitize__("unsigned-integer-overflow")))
 #endif
 
 #if __has_builtin(__builtin_launder)
 #define _LIBCPP_COMPILER_HAS_BUILTIN_LAUNDER
 #endif
 
+#if defined(__GNUC__) || __has_builtin(__builtin_constant_p)
+#define _LIBCPP_COMPILER_HAS_BUILTIN_CONSTANT_P
+#endif
+
 #if !__is_identifier(__has_unique_object_representations)
 #define _LIBCPP_HAS_UNIQUE_OBJECT_REPRESENTATIONS
 #endif
 
 #define _LIBCPP_ALWAYS_INLINE __attribute__ ((__always_inline__))
 
 // Literal operators ""d and ""y are supported starting with LLVM Clang 8 and AppleClang 10.0.1
 #if (defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER < 800) || \

libcxx/include/string

public:
 
     _LIBCPP_INLINE_VISIBILITY explicit basic_string(const allocator_type& __a)
 #if _LIBCPP_STD_VER <= 14
         _NOEXCEPT_(is_nothrow_copy_constructible<allocator_type>::value);
 #else
         _NOEXCEPT;
 #endif
 
+    // Optimization opportunity: do not externally instantiate the copy
+    // constructor, which inlines short string initialization. Long string
+    // initialization is delegated to the (external) __init_long()method,
+    // which results in a 3X-4X speed up for SSO initialization.
     basic_string(const basic_string& __str);
     basic_string(const basic_string& __str, const allocator_type& __a);
 
 #ifndef _LIBCPP_CXX03_LANG
     _LIBCPP_INLINE_VISIBILITY
     basic_string(basic_string&& __str)
 #if _LIBCPP_STD_VER <= 14
         _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value);
#ifndef _LIBCPP_CXX03_LANG
     basic_string(initializer_list<_CharT> __il, const _Allocator& __a);
 #endif  // _LIBCPP_CXX03_LANG
 
     inline ~basic_string();
 
     _LIBCPP_INLINE_VISIBILITY
     operator __self_view() const _NOEXCEPT { return __self_view(data(), size()); }
 
+    // Optimization opportunity: do not externally instantiate
     basic_string& operator=(const basic_string& __str);
 
     template <class _Tp, class = typename enable_if<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, void>::type>
     basic_string& operator=(const _Tp& __t)
         {__self_view __sv = __t; return assign(__sv);}
 
 #ifndef _LIBCPP_CXX03_LANG
     _LIBCPP_INLINE_VISIBILITY
#endif
     template <class _Tp>
     _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS
     typename enable_if
         <
             __can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value,
             basic_string&
         >::type
                   assign(const _Tp & __t, size_type __pos, size_type __n=npos);
+
+#ifdef _LIBCPP_ABI_PARTIALLY_INLINE_ASSIGN
+    _LIBCPP_INLINE_VISIBILITY
+    basic_string& assign(const value_type* __s, size_type __n);
+    _LIBCPP_INLINE_VISIBILITY
+    basic_string& assign(const value_type* __s);
+#else
     basic_string& assign(const value_type* __s, size_type __n);
     basic_string& assign(const value_type* __s);
+#endif
+
     basic_string& assign(size_type __n, value_type __c);
     template<class _InputIterator>
     _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS
     typename enable_if
         <
            __is_exactly_cpp17_input_iterator<_InputIterator>::value
                 || !__libcpp_string_gets_noexcept_iterator<_InputIterator>::value,
             basic_string&
#endif  // _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
 
     inline
     void __init(const value_type* __s, size_type __sz, size_type __reserve);
     inline
     void __init(const value_type* __s, size_type __sz);
     inline
     void __init(size_type __n, value_type __c);
 
+    void __init_long(const basic_string& __str);
+
     template <class _InputIterator>
     inline
     typename enable_if
     <
         __is_exactly_cpp17_input_iterator<_InputIterator>::value,
         void
     >::type
     __init(_InputIterator __first, _InputIterator __last);
void __copy_assign_alloc(const basic_string& __str, true_type)
                 }
             }
         }
 
     _LIBCPP_INLINE_VISIBILITY
     void __copy_assign_alloc(const basic_string&, false_type) _NOEXCEPT
         {}
 
+#ifdef _LIBCPP_ABI_PARTIALLY_INLINE_ASSIGN
+    basic_string& __assign(const value_type* __s, size_type __n);
+    basic_string& __assign(const value_type* __s);
+#else
+    _LIBCPP_INLINE_VISIBILITY
+    basic_string& __assign(const value_type* __s, size_type __n);
+    _LIBCPP_INLINE_VISIBILITY
+    basic_string& __assign(const value_type* __s);
+#endif
+
+#ifdef _LIBCPP_COMPILER_HAS_BUILTIN_CONSTANT_P
+    _LIBCPP_INLINE_VISIBILITY
+    basic_string& __assign_in_place(const value_type* __s, size_type __n);
+#endif
+
 #ifndef _LIBCPP_CXX03_LANG
     _LIBCPP_INLINE_VISIBILITY
     void __move_assign(basic_string& __str, false_type)
         _NOEXCEPT_(__alloc_traits::is_always_equal::value);
     _LIBCPP_INLINE_VISIBILITY
     void __move_assign(basic_string& __str, true_type)
 #if _LIBCPP_STD_VER > 14
         _NOEXCEPT;
else
         __set_long_cap(__cap+1);
         __set_long_size(__sz);
     }
     traits_type::copy(_VSTD::__to_address(__p), __s, __sz);
     traits_type::assign(__p[__sz], value_type());
 }
 
 template <class _CharT, class _Traits, class _Allocator>
+void basic_string<_CharT, _Traits, _Allocator>::__init_long(
+    const basic_string& __str) {
+  const value_type* __s = _VSTD::__to_address(__str.__get_long_pointer());
+  size_type __sz = __str.__get_long_size();
+  size_type __cap = __recommend(__sz);
+  pointer __p = __alloc_traits::allocate(__alloc(), __cap + 1);
+  __set_long_pointer(__p);
+  __set_long_cap(__cap + 1);
+  __set_long_size(__sz);
+  traits_type::copy(_VSTD::__to_address(__p), __s, __sz);
+  traits_type::assign(__p[__sz], value_type());
+}
+
+template <class _CharT, class _Traits, class _Allocator>
 template <class>
 basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, const _Allocator& __a)
     : __r_(__default_init_tag(), __a)
 {
     _LIBCPP_ASSERT(__s != nullptr, "basic_string(const char*, allocator) detected nullptr");
     __init(__s, traits_type::length(__s));
 #if _LIBCPP_DEBUG_LEVEL >= 2
     __get_db()->__insert_c(this);

 
 template <class _CharT, class _Traits, class _Allocator>
 basic_string<_CharT, _Traits, _Allocator>::basic_string(const basic_string& __str)
     : __r_(__default_init_tag(), __alloc_traits::select_on_container_copy_construction(__str.__alloc()))
 {
     if (!__str.__is_long())
         __r_.first().__r = __str.__r_.first().__r;
     else
-        __init(_VSTD::__to_address(__str.__get_long_pointer()), __str.__get_long_size());
+        __init_long(__str);
 #if _LIBCPP_DEBUG_LEVEL >= 2
     __get_db()->__insert_c(this);
 #endif
 }
 
 template <class _CharT, class _Traits, class _Allocator>
 basic_string<_CharT, _Traits, _Allocator>::basic_string(
     const basic_string& __str, const allocator_type& __a)
     : __r_(__default_init_tag(), __a)
 {
     if (!__str.__is_long())
         __r_.first().__r = __str.__r_.first().__r;
     else
-        __init(_VSTD::__to_address(__str.__get_long_pointer()), __str.__get_long_size());
+        __init_long(__str);
 #if _LIBCPP_DEBUG_LEVEL >= 2
     __get_db()->__insert_c(this);
 #endif
 }
 
 #ifndef _LIBCPP_CXX03_LANG
 
 template <class _CharT, class _Traits, class _Allocator>
if (__old_cap+1 != __min_cap)
         __alloc_traits::deallocate(__alloc(), __old_p, __old_cap+1);
     __set_long_pointer(__p);
     __set_long_cap(__cap+1);
 }
 
 // assign
 
 template <class _CharT, class _Traits, class _Allocator>
+inline
 basic_string<_CharT, _Traits, _Allocator>&
-basic_string<_CharT, _Traits, _Allocator>::assign(const value_type* __s, size_type __n)
+basic_string<_CharT, _Traits, _Allocator>::__assign_in_place(
+    const value_type* __s, size_type __n) {
+  pointer __p;
+  if (__is_long()) {
+    __p = __get_long_pointer();
+    __set_long_size(__n);
+  } else {
+    __p = __get_short_pointer();
+    __set_short_size(__n);
+  }
+  traits_type::copy(_VSTD::__to_address(__p), __s, __n);
+  traits_type::assign(__p[__n], value_type());
+  __invalidate_iterators_past(__n);
+  return *this;
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::__assign(const value_type* __s, size_type __n)
 {
-    _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string::assign received nullptr");
     size_type __cap = capacity();
     if (__cap >= __n)
     {
         value_type* __p = _VSTD::__to_address(__get_pointer());
         traits_type::move(__p, __s, __n);
         traits_type::assign(__p[__n], value_type());
         __set_size(__n);
         __invalidate_iterators_past(__n);
     }
     else
     {
         size_type __sz = size();
         __grow_by_and_replace(__cap, __n - __cap, __sz, 0, __sz, __n, __s);
     }
     return *this;
 }
 
 template <class _CharT, class _Traits, class _Allocator>
 basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::__assign(const value_type* __s)
+{
+    return __assign(__s, traits_type::length(__s));
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+basic_string<_CharT, _Traits, _Allocator>&
+basic_string<_CharT, _Traits, _Allocator>::assign(const value_type* __s, size_type __n) {
+    _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string::assign received nullptr");
+#ifdef _LIBCPP_COMPILER_HAS_BUILTIN_CONSTANT_P
+    if (__builtin_constant_p(__n) && __n < __min_cap) {
+        return __assign_in_place(__s, __n);
+    }
+#endif
+    return __assign(__s, __n);
+}
+
+template <class _CharT, class _Traits, class _Allocator>
+basic_string<_CharT, _Traits, _Allocator>&
 basic_string<_CharT, _Traits, _Allocator>::assign(size_type __n, value_type __c)
 {
     size_type __cap = capacity();
     if (__cap < __n)
     {
         size_type __sz = size();
         __grow_by(__cap, __n - __cap, __sz, 0, __sz);
     }

 
 template <class _CharT, class _Traits, class _Allocator>
 basic_string<_CharT, _Traits, _Allocator>&
 basic_string<_CharT, _Traits, _Allocator>::operator=(const basic_string& __str)
 {
     if (this != &__str)
     {
         __copy_assign_alloc(__str);
+        if (__is_long() | __str.__is_long()) {  // LINT: explicit binary or.
           return assign(__str.data(), __str.size());
         }
+        __r_.first().__r = __str.__r_.first().__r;
+    }
     return *this;
 }
 
 #ifndef _LIBCPP_CXX03_LANG
 
 template <class _CharT, class _Traits, class _Allocator>
 inline
 void

 }
 
 
 template <class _CharT, class _Traits, class _Allocator>
 basic_string<_CharT, _Traits, _Allocator>&
 basic_string<_CharT, _Traits, _Allocator>::assign(const value_type* __s)
 {
     _LIBCPP_ASSERT(__s != nullptr, "string::assign received nullptr");
-    return assign(__s, traits_type::length(__s));
+#ifdef _LIBCPP_COMPILER_HAS_BUILTIN_CONSTANT_P
+    if (__builtin_constant_p(__s[0]) && traits_type::length(__s) < __min_cap) {
+      return __assign_in_place(__s, traits_type::length(__s));
+    }
+#endif
+    return __assign(__s);
 }
 
 // append
 
 template <class _CharT, class _Traits, class _Allocator>
 basic_string<_CharT, _Traits, _Allocator>&
 basic_string<_CharT, _Traits, _Allocator>::append(const value_type* __s, size_type __n)
 {