Index: include/__config =================================================================== --- include/__config +++ include/__config @@ -95,6 +95,8 @@ // Use the smallest possible integer type to represent the index of the variant. // Previously libc++ used "unsigned int" exclusivly. # define _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION +// Unstable attempt to provide a more optimized std::function +# define _LIBCPP_ABI_OPTIMIZED_FUNCTION #elif _LIBCPP_ABI_VERSION == 1 # if !defined(_LIBCPP_OBJECT_FORMAT_COFF) // Enable compiling copies of now inline methods into the dylib to support Index: include/functional =================================================================== --- include/functional +++ include/functional @@ -1859,6 +1859,493 @@ #endif // _LIBCPP_NO_RTTI }; +// Storage for a functor object, to be used with __policy to manage copy and +// destruction. +union __policy_storage +{ + mutable char __small[sizeof(void*) * 2]; + void* __large; +}; + +// True if _Fun can safely be held in __policy_storage.__small. +template +struct __use_small_storage + : public _VSTD::integral_constant< + bool, sizeof(_Fun) <= sizeof(__policy_storage) && + alignof(_Fun) <= alignof(__policy_storage) && + _VSTD::is_trivially_copy_constructible<_Fun>::value && + _VSTD::is_trivially_destructible<_Fun>::value> {}; + +// Policy contains information about how to copy, destroy, and move the +// underlying functor. You can think of it as a vtable of sorts. +struct __policy +{ + // Used to copy or destroy __large values. null for trivial objects. + void* (*const __clone)(const void*); + void (*const __destroy)(void*); + + // True if this is the null policy (no value). + const bool __is_null; + + // The target type. May be null if RTTI is disabled. + const std::type_info* const __type_info; + + // Returns a pointer to a static policy object suitable for the functor + // type. + template + _LIBCPP_INLINE_VISIBILITY static const __policy* __create() + { + return __choose_policy<_Fun>(__use_small_storage<_Fun>()); + } + + _LIBCPP_INLINE_VISIBILITY + static const __policy* __create_empty() + { + static const _LIBCPP_CONSTEXPR __policy __policy_ = {nullptr, nullptr, + true, +#ifndef _LIBCPP_NO_RTTI + &typeid(void) +#else + nullptr +#endif + }; + return &__policy_; + } + + private: + template static void* __large_clone(const void* __s) + { + const _Fun* __f = static_cast(__s); + return __f->__clone(); + } + + template static void __large_destroy(void* __s) + { + typedef allocator_traits __alloc_traits; + typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type + _FunAlloc; + _Fun* __f = static_cast<_Fun*>(__s); + _FunAlloc __a(__f->__allocator()); + __f->destroy(); + __a.deallocate(__f, 1); + } + + template + _LIBCPP_INLINE_VISIBILITY static const __policy* + __choose_policy(/* is_small = */ false_type) + { + static const _LIBCPP_CONSTEXPR __policy __policy_ = { + &__large_clone<_Fun>, &__large_destroy<_Fun>, false, +#ifndef _LIBCPP_NO_RTTI + &typeid(typename _Fun::_Target) +#else + nullptr +#endif + }; + return &__policy_; + } + + template + _LIBCPP_INLINE_VISIBILITY static const __policy* + __choose_policy(/* is_small = */ true_type) + { + static const _LIBCPP_CONSTEXPR __policy __policy_ = { + nullptr, nullptr, false, +#ifndef _LIBCPP_NO_RTTI + &typeid(typename _Fun::_Target) +#else + nullptr +#endif + }; + return &__policy_; + } +}; + +// Used to choose between perfect forwarding or pass-by-value. Pass-by-value is +// faster for types that can be passed in registers. +template +using __fast_forward = + typename _VSTD::conditional<_VSTD::is_scalar<_Tp>::value, _Tp, _Tp&&>::type; + +// __policy_invoker calls an instance of __alloc_func held in __policy_storage. + +template struct __policy_invoker; + +template +struct __policy_invoker<_Rp(_ArgTypes...)> +{ + typedef _Rp (*__Call)(const __policy_storage*, + __fast_forward<_ArgTypes>...); + + __Call __call_; + + // Creates an invoker that throws bad_function_call. + _LIBCPP_INLINE_VISIBILITY + __policy_invoker() : __call_(&__call_empty) {} + + // Creates an invoker that calls the given instance of __func. + template + _LIBCPP_INLINE_VISIBILITY static __policy_invoker __create() + { + return __policy_invoker(&__call_impl<_Fun>); + } + + private: + _LIBCPP_INLINE_VISIBILITY + explicit __policy_invoker(__Call __c) : __call_(__c) {} + + static _Rp __call_empty(const __policy_storage*, + __fast_forward<_ArgTypes>...) + { + __throw_bad_function_call(); + } + + template + static _Rp __call_impl(const __policy_storage* __buf, + __fast_forward<_ArgTypes>... __args) + { + _Fun* __f = reinterpret_cast<_Fun*>(__use_small_storage<_Fun>::value + ? &__buf->__small + : __buf->__large); + return (*__f)(_VSTD::forward<_ArgTypes>(__args)...); + } +}; + +// __value_func creates a value-type from a __func. + +template class __value_func; + +template class __value_func<_Rp(_ArgTypes...)> +{ + typename aligned_storage<3 * sizeof(void*)>::type __buf_; + + typedef __base<_Rp(_ArgTypes...)> __func; + __func* __f_; + + _LIBCPP_NO_CFI static __func* __as_base(void* p) + { + return reinterpret_cast<__func*>(p); + } + + public: + _LIBCPP_INLINE_VISIBILITY + __value_func() _NOEXCEPT : __f_(0) {} + + template + _LIBCPP_INLINE_VISIBILITY __value_func(_Fp&& __f, const _Alloc __a) + : __f_(0) + { + typedef allocator_traits<_Alloc> __alloc_traits; + typedef __function::__func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun; + typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type + _FunAlloc; + + if (__function::__not_null(__f)) + { + _FunAlloc __af(__a); + if (sizeof(_Fun) <= sizeof(__buf_) && + is_nothrow_copy_constructible<_Fp>::value && + is_nothrow_copy_constructible<_FunAlloc>::value) + { + __f_ = + ::new ((void*)&__buf_) _Fun(_VSTD::move(__f), _Alloc(__af)); + } + else + { + typedef __allocator_destructor<_FunAlloc> _Dp; + unique_ptr<__func, _Dp> __hold(__af.allocate(1), _Dp(__af, 1)); + ::new ((void*)__hold.get()) _Fun(_VSTD::move(__f), _Alloc(__a)); + __f_ = __hold.release(); + } + } + } + + _LIBCPP_INLINE_VISIBILITY + __value_func(const __value_func& __f) + { + if (__f.__f_ == 0) + __f_ = 0; + else if ((void*)__f.__f_ == &__f.__buf_) + { + __f_ = __as_base(&__buf_); + __f.__f_->__clone(__f_); + } + else + __f_ = __f.__f_->__clone(); + } + + _LIBCPP_INLINE_VISIBILITY + __value_func(__value_func&& __f) _NOEXCEPT + { + if (__f.__f_ == 0) + __f_ = 0; + else if ((void*)__f.__f_ == &__f.__buf_) + { + __f_ = __as_base(&__buf_); + __f.__f_->__clone(__f_); + } + else + { + __f_ = __f.__f_; + __f.__f_ = 0; + } + } + + _LIBCPP_INLINE_VISIBILITY + ~__value_func() + { + if ((void*)__f_ == &__buf_) + __f_->destroy(); + else if (__f_) + __f_->destroy_deallocate(); + } + + _LIBCPP_INLINE_VISIBILITY + __value_func& operator=(__value_func&& __f) + { + *this = nullptr; + if (__f.__f_ == 0) + __f_ = 0; + else if ((void*)__f.__f_ == &__f.__buf_) + { + __f_ = __as_base(&__buf_); + __f.__f_->__clone(__f_); + } + else + { + __f_ = __f.__f_; + __f.__f_ = 0; + } + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + __value_func& operator=(nullptr_t) + { + __func* __f = __f_; + __f_ = 0; + if ((void*)__f == &__buf_) + __f->destroy(); + else if (__f) + __f->destroy_deallocate(); + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + _Rp operator()(_ArgTypes&&... __args) const + { + if (__f_ == 0) + __throw_bad_function_call(); + return (*__f_)(_VSTD::forward<_ArgTypes>(__args)...); + } + + _LIBCPP_INLINE_VISIBILITY + void swap(__value_func& __f) _NOEXCEPT + { + if (&__f == this) + return; + if ((void*)__f_ == &__buf_ && (void*)__f.__f_ == &__f.__buf_) + { + typename aligned_storage::type __tempbuf; + __func* __t = __as_base(&__tempbuf); + __f_->__clone(__t); + __f_->destroy(); + __f_ = 0; + __f.__f_->__clone(__as_base(&__buf_)); + __f.__f_->destroy(); + __f.__f_ = 0; + __f_ = __as_base(&__buf_); + __t->__clone(__as_base(&__f.__buf_)); + __t->destroy(); + __f.__f_ = __as_base(&__f.__buf_); + } + else if ((void*)__f_ == &__buf_) + { + __f_->__clone(__as_base(&__f.__buf_)); + __f_->destroy(); + __f_ = __f.__f_; + __f.__f_ = __as_base(&__f.__buf_); + } + else if ((void*)__f.__f_ == &__f.__buf_) + { + __f.__f_->__clone(__as_base(&__buf_)); + __f.__f_->destroy(); + __f.__f_ = __f_; + __f_ = __as_base(&__buf_); + } + else + _VSTD::swap(__f_, __f.__f_); + } + + _LIBCPP_INLINE_VISIBILITY + _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT { return __f_ != 0; } + +#ifndef _LIBCPP_NO_RTTI + _LIBCPP_INLINE_VISIBILITY + const std::type_info& target_type() const _NOEXCEPT + { + if (__f_ == 0) + return typeid(void); + return __f_->target_type(); + } + + template + _LIBCPP_INLINE_VISIBILITY const _Tp* target() const _NOEXCEPT + { + if (__f_ == 0) + return 0; + return (const _Tp*)__f_->target(typeid(_Tp)); + } +#endif // _LIBCPP_NO_RTTI +}; + +// __policy_func uses a __policy and __policy_invoker to create a type-erased, +// copyable functor. + +template class __policy_func; + +template class __policy_func<_Rp(_ArgTypes...)> +{ + // Inline storage for small objects. + __policy_storage __buf_; + + // Calls the value stored in __buf_. This could technically be part of + // policy, but storing it here eliminates a level of indirection inside + // operator(). + typedef __function::__policy_invoker<_Rp(_ArgTypes...)> __invoker; + __invoker __invoker_; + + // The policy that describes how to move / copy / destroy __buf_. Never + // null, even if the function is empty. + const __policy* __policy_; + + public: + _LIBCPP_INLINE_VISIBILITY + __policy_func() : __policy_(__policy::__create_empty()) {} + + template + _LIBCPP_INLINE_VISIBILITY __policy_func(_Fp&& __f, const _Alloc& __a) + : __policy_(__policy::__create_empty()) + { + typedef __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun; + typedef allocator_traits<_Alloc> __alloc_traits; + typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type + _FunAlloc; + + if (__function::__not_null(__f)) + { + __invoker_ = __invoker::template __create<_Fun>(); + __policy_ = __policy::__create<_Fun>(); + + _FunAlloc __af(__a); + if (__use_small_storage<_Fun>()) + { + ::new ((void*)&__buf_.__small) + _Fun(_VSTD::move(__f), _Alloc(__af)); + } + else + { + typedef __allocator_destructor<_FunAlloc> _Dp; + unique_ptr<_Fun, _Dp> __hold(__af.allocate(1), _Dp(__af, 1)); + ::new ((void*)__hold.get()) + _Fun(_VSTD::move(__f), _Alloc(__af)); + __buf_.__large = __hold.release(); + } + } + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func(const __policy_func& __f) + : __buf_(__f.__buf_), __invoker_(__f.__invoker_), + __policy_(__f.__policy_) + { + if (__policy_->__clone) + __buf_.__large = __policy_->__clone(__f.__buf_.__large); + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func(__policy_func&& __f) + : __buf_(__f.__buf_), __invoker_(__f.__invoker_), + __policy_(__f.__policy_) + { + if (__policy_->__destroy) + { + __f.__policy_ = __policy::__create_empty(); + __f.__invoker_ = __invoker(); + } + } + + _LIBCPP_INLINE_VISIBILITY + ~__policy_func() + { + if (__policy_->__destroy) + __policy_->__destroy(__buf_.__large); + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func& operator=(__policy_func&& __f) + { + *this = nullptr; + __buf_ = __f.__buf_; + __invoker_ = __f.__invoker_; + __policy_ = __f.__policy_; + __f.__policy_ = __policy::__create_empty(); + __f.__invoker_ = __invoker(); + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func& operator=(nullptr_t) + { + const __policy* __p = __policy_; + __policy_ = __policy::__create_empty(); + __invoker_ = __invoker(); + if (__p->__destroy) + __p->__destroy(__buf_.__large); + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + _Rp operator()(_ArgTypes&&... __args) const + { + return __invoker_.__call_(&__buf_, + _VSTD::forward<_ArgTypes>(__args)...); + } + + _LIBCPP_INLINE_VISIBILITY + void swap(__policy_func& __f) + { + _VSTD::swap(__invoker_, __f.__invoker_); + _VSTD::swap(__policy_, __f.__policy_); + _VSTD::swap(__buf_, __f.__buf_); + } + + _LIBCPP_INLINE_VISIBILITY + _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT + { + return !__policy_->__is_null; + } + +#ifndef _LIBCPP_NO_RTTI + _LIBCPP_INLINE_VISIBILITY + const std::type_info& target_type() const _NOEXCEPT + { + return *__policy_->__type_info; + } + + template + _LIBCPP_INLINE_VISIBILITY const _Tp* target() const _NOEXCEPT + { + if (__policy_->__is_null || typeid(_Tp) != *__policy_->__type_info) + return nullptr; + if (__policy_->__clone) // Out of line storage. + return reinterpret_cast(__buf_.__large); + else + return reinterpret_cast(&__buf_.__small); + } +#endif // _LIBCPP_NO_RTTI +}; + } // __function template @@ -1866,7 +2353,11 @@ : public __function::__maybe_derive_from_unary_function<_Rp(_ArgTypes...)>, public __function::__maybe_derive_from_binary_function<_Rp(_ArgTypes...)> { +#ifndef _LIBCPP_ABI_OPTIMIZED_FUNCTION typedef __function::__value_func<_Rp(_ArgTypes...)> __func; +#else + typedef __function::__policy_func<_Rp(_ArgTypes...)> __func; +#endif __func __f_; Index: test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_F.pass.cpp =================================================================== --- test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_F.pass.cpp +++ test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_F.pass.cpp @@ -65,7 +65,8 @@ FuncType* target = &FreeFunction; assert(globalMemCounter.checkOutstandingNewEq(0)); std::function f2(std::allocator_arg, alloc, target); - assert(globalMemCounter.checkOutstandingNewEq(0)); + assert(globalMemCounter.checkOutstandingNewEq(0) || + globalMemCounter.checkOutstandingNewEq(1)); assert(f2.template target()); assert(*f2.template target() == target); assert(f2.template target() == 0); @@ -82,7 +83,8 @@ TargetType target = &MemFunClass::foo; assert(globalMemCounter.checkOutstandingNewEq(0)); std::function f2(std::allocator_arg, alloc, target); - assert(globalMemCounter.checkOutstandingNewEq(0)); + assert(globalMemCounter.checkOutstandingNewEq(0) || + globalMemCounter.checkOutstandingNewEq(1)); assert(f2.template target()); assert(*f2.template target() == target); assert(f2.template target() == 0); Index: test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_rfunction.pass.cpp =================================================================== --- test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_rfunction.pass.cpp +++ test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_rfunction.pass.cpp @@ -89,7 +89,6 @@ assert(A::count == 1); assert(f2.target() == nullptr); assert(f2.target()); - assert(f.target()); // f is unchanged because the target is small } { // Test that moving a function constructed from a function pointer @@ -104,6 +103,5 @@ std::move(f)); assert(f2.target() == nullptr); assert(f2.target()); - assert(f.target()); // f is unchanged because the target is small } } Index: test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/copy_move.pass.cpp =================================================================== --- test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/copy_move.pass.cpp +++ test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/copy_move.pass.cpp @@ -141,7 +141,6 @@ assert(A::count == 1); assert(f2.target() == nullptr); assert(f2.target()); - LIBCPP_ASSERT(f.target()); // f is unchanged because the target is small } { // Test that moving a function constructed from a function pointer @@ -159,7 +158,6 @@ std::function f2(std::move(f)); assert(f2.target() == nullptr); assert(f2.target()); - LIBCPP_ASSERT(f.target()); // f is unchanged because the target is small } #endif // TEST_STD_VER >= 11 }