Index: include/__config =================================================================== --- include/__config +++ include/__config @@ -95,6 +95,8 @@ // Use the smallest possible integer type to represent the index of the variant. // Previously libc++ used "unsigned int" exclusivly. # define _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION +// Unstable attempt to provide a more optimized std::function +# define _LIBCPP_ABI_OPTIMIZED_FUNCTION #elif _LIBCPP_ABI_VERSION == 1 # if !defined(_LIBCPP_OBJECT_FORMAT_COFF) // Enable compiling copies of now inline methods into the dylib to support Index: include/functional =================================================================== --- include/functional +++ include/functional @@ -1859,6 +1859,305 @@ #endif // _LIBCPP_NO_RTTI }; +// Storage for a functor object, to be used with __policy to manage copy and +// destruction. +union __policy_storage +{ + mutable char __small[sizeof(void*) * 2]; + void* __large; +}; + +// True if _Fun can safely be held in __policy_storage.__small. +template +struct __use_small_storage + : public _VSTD::integral_constant< + bool, sizeof(_Fun) <= sizeof(__policy_storage) && + alignof(_Fun) <= alignof(__policy_storage) && + _VSTD::is_trivially_copy_constructible<_Fun>::value && + _VSTD::is_trivially_destructible<_Fun>::value> {}; + +// Policy contains information about how to copy, destroy, and move the +// underlying functor. You can think of it as a vtable of sorts. +struct __policy +{ + // Used to copy or destroy __large values. null for trivial objects. + void* (*const __clone)(const void*); + void (*const __destroy)(void*); + + // True if this is the null policy (no value). + const bool __is_null; + + // The target type. May be null if RTTI is disabled. + const std::type_info* const __type_info; + + // Returns a pointer to a static policy object suitable for the functor + // type. + template + _LIBCPP_INLINE_VISIBILITY static const __policy* __create() + { + return __choose_policy<_Fun>(__use_small_storage<_Fun>()); + } + + _LIBCPP_INLINE_VISIBILITY + static const __policy* __create_empty() + { + static const _LIBCPP_CONSTEXPR __policy __policy_ = {nullptr, nullptr, + true, +#ifndef _LIBCPP_NO_RTTI + &typeid(void) +#else + nullptr +#endif + }; + return &__policy_; + } + + private: + template static void* __large_clone(const void* __s) + { + const _Fun* __f = static_cast(__s); + return __f->__clone(); + } + + template static void __large_destroy(void* __s) + { + typedef allocator_traits __alloc_traits; + typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type + _FunAlloc; + _Fun* __f = static_cast<_Fun*>(__s); + _FunAlloc __a(__f->__allocator()); + __f->destroy(); + __a.deallocate(__f, 1); + } + + template + _LIBCPP_INLINE_VISIBILITY static const __policy* + __choose_policy(/* is_small = */ false_type) + { + static const _LIBCPP_CONSTEXPR __policy __policy_ = { + &__large_clone<_Fun>, &__large_destroy<_Fun>, false, +#ifndef _LIBCPP_NO_RTTI + &typeid(typename _Fun::_Target) +#else + nullptr +#endif + }; + return &__policy_; + } + + template + _LIBCPP_INLINE_VISIBILITY static const __policy* + __choose_policy(/* is_small = */ true_type) + { + static const _LIBCPP_CONSTEXPR __policy __policy_ = { + nullptr, nullptr, false, +#ifndef _LIBCPP_NO_RTTI + &typeid(typename _Fun::_Target) +#else + nullptr +#endif + }; + return &__policy_; + } +}; + +// Used to choose between perfect forwarding or pass-by-value. Pass-by-value is +// faster for types that can be passed in registers. +template +using __fast_forward = + typename _VSTD::conditional<_VSTD::is_scalar<_Tp>::value, _Tp, _Tp&&>::type; + +// __policy_invoker calls an instance of __alloc_func held in __policy_storage. + +template struct __policy_invoker; + +template +struct __policy_invoker<_Rp(_ArgTypes...)> +{ + typedef _Rp (*__Call)(const __policy_storage*, + __fast_forward<_ArgTypes>...); + + __Call __call_; + + // Creates an invoker that throws bad_function_call. + _LIBCPP_INLINE_VISIBILITY + __policy_invoker() : __call_(&__call_empty) {} + + // Creates an invoker that calls the given instance of __func. + template + _LIBCPP_INLINE_VISIBILITY static __policy_invoker __create() + { + return __policy_invoker(&__call_impl<_Fun>); + } + + private: + _LIBCPP_INLINE_VISIBILITY + explicit __policy_invoker(__Call __c) : __call_(__c) {} + + static _Rp __call_empty(const __policy_storage*, + __fast_forward<_ArgTypes>...) + { + __throw_bad_function_call(); + } + + template + static _Rp __call_impl(const __policy_storage* __buf, + __fast_forward<_ArgTypes>... __args) + { + _Fun* __f = reinterpret_cast<_Fun*>(__use_small_storage<_Fun>::value + ? &__buf->__small + : __buf->__large); + return (*__f)(_VSTD::forward<_ArgTypes>(__args)...); + } +}; + +// __policy_func uses a __policy and __policy_invoker to create a type-erased, +// copyable functor. + +template class __policy_func; + +template class __policy_func<_Rp(_ArgTypes...)> +{ + // Inline storage for small objects. + __policy_storage __buf_; + + // Calls the value stored in __buf_. This could technically be part of + // policy, but storing it here eliminates a level of indirection inside + // operator(). + typedef __function::__policy_invoker<_Rp(_ArgTypes...)> __invoker; + __invoker __invoker_; + + // The policy that describes how to move / copy / destroy __buf_. Never + // null, even if the function is empty. + const __policy* __policy_; + + public: + _LIBCPP_INLINE_VISIBILITY + __policy_func() : __policy_(__policy::__create_empty()) {} + + template + _LIBCPP_INLINE_VISIBILITY __policy_func(_Fp&& __f, const _Alloc& __a) + : __policy_(__policy::__create_empty()) + { + typedef __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun; + typedef allocator_traits<_Alloc> __alloc_traits; + typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type + _FunAlloc; + + if (__function::__not_null(__f)) + { + __invoker_ = __invoker::template __create<_Fun>(); + __policy_ = __policy::__create<_Fun>(); + + _FunAlloc __af(__a); + if (__use_small_storage<_Fun>()) + { + ::new ((void*)&__buf_.__small) + _Fun(_VSTD::move(__f), _Alloc(__af)); + } + else + { + typedef __allocator_destructor<_FunAlloc> _Dp; + unique_ptr<_Fun, _Dp> __hold(__af.allocate(1), _Dp(__af, 1)); + ::new ((void*)__hold.get()) + _Fun(_VSTD::move(__f), _Alloc(__af)); + __buf_.__large = __hold.release(); + } + } + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func(const __policy_func& __f) + : __buf_(__f.__buf_), __invoker_(__f.__invoker_), + __policy_(__f.__policy_) + { + if (__policy_->__clone) + __buf_.__large = __policy_->__clone(__f.__buf_.__large); + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func(__policy_func&& __f) + : __buf_(__f.__buf_), __invoker_(__f.__invoker_), + __policy_(__f.__policy_) + { + if (__policy_->__destroy) + { + __f.__policy_ = __policy::__create_empty(); + __f.__invoker_ = __invoker(); + } + } + + _LIBCPP_INLINE_VISIBILITY + ~__policy_func() + { + if (__policy_->__destroy) + __policy_->__destroy(__buf_.__large); + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func& operator=(__policy_func&& __f) + { + *this = nullptr; + __buf_ = __f.__buf_; + __invoker_ = __f.__invoker_; + __policy_ = __f.__policy_; + __f.__policy_ = __policy::__create_empty(); + __f.__invoker_ = __invoker(); + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func& operator=(nullptr_t) + { + const __policy* __p = __policy_; + __policy_ = __policy::__create_empty(); + __invoker_ = __invoker(); + if (__p->__destroy) + __p->__destroy(__buf_.__large); + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + _Rp operator()(_ArgTypes&&... __args) const + { + return __invoker_.__call_(&__buf_, + _VSTD::forward<_ArgTypes>(__args)...); + } + + _LIBCPP_INLINE_VISIBILITY + void swap(__policy_func& __f) + { + _VSTD::swap(__invoker_, __f.__invoker_); + _VSTD::swap(__policy_, __f.__policy_); + _VSTD::swap(__buf_, __f.__buf_); + } + + _LIBCPP_INLINE_VISIBILITY + _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT + { + return !__policy_->__is_null; + } + +#ifndef _LIBCPP_NO_RTTI + _LIBCPP_INLINE_VISIBILITY + const std::type_info& target_type() const _NOEXCEPT + { + return *__policy_->__type_info; + } + + template + _LIBCPP_INLINE_VISIBILITY const _Tp* target() const _NOEXCEPT + { + if (__policy_->__is_null || typeid(_Tp) != *__policy_->__type_info) + return nullptr; + if (__policy_->__clone) // Out of line storage. + return reinterpret_cast(__buf_.__large); + else + return reinterpret_cast(&__buf_.__small); + } +#endif // _LIBCPP_NO_RTTI +}; + } // __function template @@ -1866,7 +2165,11 @@ : public __function::__maybe_derive_from_unary_function<_Rp(_ArgTypes...)>, public __function::__maybe_derive_from_binary_function<_Rp(_ArgTypes...)> { +#ifndef _LIBCPP_ABI_OPTIMIZED_FUNCTION typedef __function::__value_func<_Rp(_ArgTypes...)> __func; +#else + typedef __function::__policy_func<_Rp(_ArgTypes...)> __func; +#endif __func __f_;