diff --git a/libcxx/include/__config b/libcxx/include/__config --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -102,6 +102,10 @@ # define _LIBCPP_ABI_OPTIMIZED_FUNCTION // All the regex constants must be distinct and nonzero. # define _LIBCPP_ABI_REGEX_CONSTANTS_NONZERO +// Inline assign(s, [, n]) code, performing inline assignment for strings +// which length is known at compile time and smaller than __min_cap, resulting +// in up to 6X speed ups for (asciiz) string assignments. +# define _LIBCPP_ABI_PARTIALLY_INLINE_ASSIGN #elif _LIBCPP_ABI_VERSION == 1 # if !defined(_LIBCPP_OBJECT_FORMAT_COFF) // Enable compiling copies of now inline methods into the dylib to support @@ -488,6 +492,10 @@ #define _LIBCPP_COMPILER_HAS_BUILTIN_LAUNDER #endif +#if defined(__GNUC__) || __has_builtin(__builtin_constant_p) +#define _LIBCPP_COMPILER_HAS_BUILTIN_CONSTANT_P +#endif + #if !__is_identifier(__has_unique_object_representations) #define _LIBCPP_HAS_UNIQUE_OBJECT_REPRESENTATIONS #endif diff --git a/libcxx/include/string b/libcxx/include/string --- a/libcxx/include/string +++ b/libcxx/include/string @@ -796,6 +796,10 @@ _NOEXCEPT; #endif + // Optimization opportunity: do not externally instantiate the copy + // constructor, which inlines short string initialization. Long string + // initialization is delegated to the (external) __init_long()method, + // which results in a 3X-4X speed up for SSO initialization. basic_string(const basic_string& __str); basic_string(const basic_string& __str, const allocator_type& __a); @@ -874,6 +878,7 @@ _LIBCPP_INLINE_VISIBILITY operator __self_view() const _NOEXCEPT { return __self_view(data(), size()); } + // Optimization opportunity: do not externally instantiate basic_string& operator=(const basic_string& __str); template ::value, void>::type> @@ -1084,8 +1089,17 @@ basic_string& >::type assign(const _Tp & __t, size_type __pos, size_type __n=npos); + +#ifdef _LIBCPP_ABI_PARTIALLY_INLINE_ASSIGN + _LIBCPP_INLINE_VISIBILITY + basic_string& assign(const value_type* __s, size_type __n); + _LIBCPP_INLINE_VISIBILITY + basic_string& assign(const value_type* __s); +#else basic_string& assign(const value_type* __s, size_type __n); basic_string& assign(const value_type* __s); +#endif + basic_string& assign(size_type __n, value_type __c); template _LIBCPP_METHOD_TEMPLATE_IMPLICIT_INSTANTIATION_VIS @@ -1543,6 +1557,8 @@ inline void __init(size_type __n, value_type __c); + void __init_long(const basic_string& __str); + template inline typename enable_if @@ -1604,6 +1620,21 @@ void __copy_assign_alloc(const basic_string&, false_type) _NOEXCEPT {} +#ifdef _LIBCPP_ABI_PARTIALLY_INLINE_ASSIGN + basic_string& __assign(const value_type* __s, size_type __n); + basic_string& __assign(const value_type* __s); +#else + _LIBCPP_INLINE_VISIBILITY + basic_string& __assign(const value_type* __s, size_type __n); + _LIBCPP_INLINE_VISIBILITY + basic_string& __assign(const value_type* __s); +#endif + +#ifdef _LIBCPP_COMPILER_HAS_BUILTIN_CONSTANT_P + _LIBCPP_INLINE_VISIBILITY + basic_string& __assign_in_place(const value_type* __s, size_type __n); +#endif + #ifndef _LIBCPP_CXX03_LANG _LIBCPP_INLINE_VISIBILITY void __move_assign(basic_string& __str, false_type) @@ -1794,6 +1825,20 @@ traits_type::assign(__p[__sz], value_type()); } +template +void basic_string<_CharT, _Traits, _Allocator>::__init_long( + const basic_string& __str) { + const value_type* __s = _VSTD::__to_address(__str.__get_long_pointer()); + size_type __sz = __str.__get_long_size(); + size_type __cap = __recommend(__sz); + pointer __p = __alloc_traits::allocate(__alloc(), __cap + 1); + __set_long_pointer(__p); + __set_long_cap(__cap + 1); + __set_long_size(__sz); + traits_type::copy(_VSTD::__to_address(__p), __s, __sz); + traits_type::assign(__p[__sz], value_type()); +} + template template basic_string<_CharT, _Traits, _Allocator>::basic_string(const _CharT* __s, const _Allocator& __a) @@ -1837,7 +1882,7 @@ if (!__str.__is_long()) __r_.first().__r = __str.__r_.first().__r; else - __init(_VSTD::__to_address(__str.__get_long_pointer()), __str.__get_long_size()); + __init_long(__str); #if _LIBCPP_DEBUG_LEVEL >= 2 __get_db()->__insert_c(this); #endif @@ -1851,7 +1896,7 @@ if (!__str.__is_long()) __r_.first().__r = __str.__r_.first().__r; else - __init(_VSTD::__to_address(__str.__get_long_pointer()), __str.__get_long_size()); + __init_long(__str); #if _LIBCPP_DEBUG_LEVEL >= 2 __get_db()->__insert_c(this); #endif @@ -2197,11 +2242,29 @@ // assign +template +inline +basic_string<_CharT, _Traits, _Allocator>& +basic_string<_CharT, _Traits, _Allocator>::__assign_in_place( + const value_type* __s, size_type __n) { + pointer __p; + if (__is_long()) { + __p = __get_long_pointer(); + __set_long_size(__n); + } else { + __p = __get_short_pointer(); + __set_short_size(__n); + } + traits_type::copy(_VSTD::__to_address(__p), __s, __n); + traits_type::assign(__p[__n], value_type()); + __invalidate_iterators_past(__n); + return *this; +} + template basic_string<_CharT, _Traits, _Allocator>& -basic_string<_CharT, _Traits, _Allocator>::assign(const value_type* __s, size_type __n) +basic_string<_CharT, _Traits, _Allocator>::__assign(const value_type* __s, size_type __n) { - _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string::assign received nullptr"); size_type __cap = capacity(); if (__cap >= __n) { @@ -2219,6 +2282,25 @@ return *this; } +template +basic_string<_CharT, _Traits, _Allocator>& +basic_string<_CharT, _Traits, _Allocator>::__assign(const value_type* __s) +{ + return __assign(__s, traits_type::length(__s)); +} + +template +basic_string<_CharT, _Traits, _Allocator>& +basic_string<_CharT, _Traits, _Allocator>::assign(const value_type* __s, size_type __n) { + _LIBCPP_ASSERT(__n == 0 || __s != nullptr, "string::assign received nullptr"); +#ifdef _LIBCPP_COMPILER_HAS_BUILTIN_CONSTANT_P + if (__builtin_constant_p(__n) && __n < __min_cap) { + return __assign_in_place(__s, __n); + } +#endif + return __assign(__s, __n); +} + template basic_string<_CharT, _Traits, _Allocator>& basic_string<_CharT, _Traits, _Allocator>::assign(size_type __n, value_type __c) @@ -2266,7 +2348,10 @@ if (this != &__str) { __copy_assign_alloc(__str); - return assign(__str.data(), __str.size()); + if (__is_long() | __str.__is_long()) { // LINT: explicit binary or. + return assign(__str.data(), __str.size()); + } + __r_.first().__r = __str.__r_.first().__r; } return *this; } @@ -2398,7 +2483,12 @@ basic_string<_CharT, _Traits, _Allocator>::assign(const value_type* __s) { _LIBCPP_ASSERT(__s != nullptr, "string::assign received nullptr"); - return assign(__s, traits_type::length(__s)); +#ifdef _LIBCPP_COMPILER_HAS_BUILTIN_CONSTANT_P + if (__builtin_constant_p(__s[0]) && traits_type::length(__s) < __min_cap) { + return __assign_in_place(__s, traits_type::length(__s)); + } +#endif + return __assign(__s); } // append