Index: benchmarks/function.bench.cpp =================================================================== --- benchmarks/function.bench.cpp +++ benchmarks/function.bench.cpp @@ -213,6 +213,69 @@ } }; +// Base classes so that we can generate a lot of different functors with +// different traits. +struct TrivialStruct {}; + +struct NonTrivialStruct { + NonTrivialStruct() {} + NonTrivialStruct(const NonTrivialStruct&) {} + ~NonTrivialStruct() {} +}; + +// Is trivial for 1 out of 5 seeds (roughly in-line with a particular real +// application). +template +using ChooseTrivialness = + typename std::conditional<(Seed % 5) == 0, NonTrivialStruct, + TrivialStruct>::type; + +// Size varies based on seed. The sizes should be large enough to exercise code +// paths that allocate and code paths that have small-object-optimization. +template +struct ChooseSize { + // Lambda captures tend to be word-sized (pointers and references). + void* padding[(Seed % 5) + 1]; +}; + +// Every Seed results in a unique type with different code, which should throw +// off CPU branch predictors. +template +struct GeneratedFunctor : public ChooseTrivialness, + public ChooseSize { + int operator()(const S*) const { return Seed; } +}; + +template +void ConstructMoveAndCallGeneratedFunctor() { + S s; + Function f = GeneratedFunctor{}; + benchmark::DoNotOptimize(f); + Function mf = std::move(f); + benchmark::DoNotOptimize(mf); + benchmark::DoNotOptimize(mf(&s)); +} + +// Calls ConstructMoveAndCallGeneratedFunctor() for the given number of generated +// functors. +template +void RunGeneratedFunctors() { + ConstructMoveAndCallGeneratedFunctor(); + RunGeneratedFunctors(); +} + +template <> +void RunGeneratedFunctors<0>() { + ConstructMoveAndCallGeneratedFunctor<0>(); +} + +void BM_MixedFunctorTypes(benchmark::State& state) { + for (auto _ : state) + // Increasing this value can make compilation take a long time. + RunGeneratedFunctors<25>(); +} +BENCHMARK(BM_MixedFunctorTypes); + } // namespace int main(int argc, char** argv) { Index: include/__config =================================================================== --- include/__config +++ include/__config @@ -95,6 +95,8 @@ // Use the smallest possible integer type to represent the index of the variant. // Previously libc++ used "unsigned int" exclusivly. # define _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION +// Unstable attempt to provide a more optimized std::function +# define _LIBCPP_ABI_OPTIMIZED_FUNCTION #elif _LIBCPP_ABI_VERSION == 1 # if !defined(_LIBCPP_OBJECT_FORMAT_COFF) // Enable compiling copies of now inline methods into the dylib to support Index: include/functional =================================================================== --- include/functional +++ include/functional @@ -1468,6 +1468,81 @@ namespace __function { +// __alloc_func holds a functor and an allocator. + +template class __alloc_func; + +template +class __alloc_func<_Fp, _Ap, _Rp(_ArgTypes...)> +{ + __compressed_pair<_Fp, _Ap> __f_; + + public: + typedef _Fp _Target; + typedef _Ap _Alloc; + + _LIBCPP_INLINE_VISIBILITY + const _Target& __target() const { return __f_.first(); } + + _LIBCPP_INLINE_VISIBILITY + const _Alloc& __allocator() const { return __f_.second(); } + + _LIBCPP_INLINE_VISIBILITY + explicit __alloc_func(_Target&& __f) + : __f_(piecewise_construct, _VSTD::forward_as_tuple(_VSTD::move(__f)), + _VSTD::forward_as_tuple()) + { + } + + _LIBCPP_INLINE_VISIBILITY + explicit __alloc_func(const _Target& __f, const _Alloc& __a) + : __f_(piecewise_construct, _VSTD::forward_as_tuple(__f), + _VSTD::forward_as_tuple(__a)) + { + } + + _LIBCPP_INLINE_VISIBILITY + explicit __alloc_func(const _Target& __f, _Alloc&& __a) + : __f_(piecewise_construct, _VSTD::forward_as_tuple(__f), + _VSTD::forward_as_tuple(_VSTD::move(__a))) + { + } + + _LIBCPP_INLINE_VISIBILITY + explicit __alloc_func(_Target&& __f, _Alloc&& __a) + : __f_(piecewise_construct, _VSTD::forward_as_tuple(_VSTD::move(__f)), + _VSTD::forward_as_tuple(_VSTD::move(__a))) + { + } + + _LIBCPP_INLINE_VISIBILITY + _Rp operator()(_ArgTypes&&... __arg) + { + typedef __invoke_void_return_wrapper<_Rp> _Invoker; + return _Invoker::__call(__f_.first(), + _VSTD::forward<_ArgTypes>(__arg)...); + } + + _LIBCPP_INLINE_VISIBILITY + __alloc_func* __clone() const + { + typedef allocator_traits<_Alloc> __alloc_traits; + typedef + typename __rebind_alloc_helper<__alloc_traits, __alloc_func>::type + _AA; + _AA __a(__f_.second()); + typedef __allocator_destructor<_AA> _Dp; + unique_ptr<__alloc_func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); + ::new ((void*)__hold.get()) __alloc_func(__f_.first(), _Alloc(__a)); + return __hold.release(); + } + + _LIBCPP_INLINE_VISIBILITY + void destroy() _NOEXCEPT { __f_.~__compressed_pair<_Target, _Alloc>(); } +}; + +// __base provides an abstract interface for copyable functors. + template class __base; template @@ -1489,37 +1564,37 @@ #endif // _LIBCPP_NO_RTTI }; +// __func implements __base for a given functor type. + template class __func; template class __func<_Fp, _Alloc, _Rp(_ArgTypes...)> : public __base<_Rp(_ArgTypes...)> { - __compressed_pair<_Fp, _Alloc> __f_; + __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> __f_; public: _LIBCPP_INLINE_VISIBILITY explicit __func(_Fp&& __f) - : __f_(piecewise_construct, _VSTD::forward_as_tuple(_VSTD::move(__f)), - _VSTD::forward_as_tuple()) {} + : __f_(_VSTD::move(__f)) {} + _LIBCPP_INLINE_VISIBILITY explicit __func(const _Fp& __f, const _Alloc& __a) - : __f_(piecewise_construct, _VSTD::forward_as_tuple(__f), - _VSTD::forward_as_tuple(__a)) {} + : __f_(__f, __a) {} _LIBCPP_INLINE_VISIBILITY explicit __func(const _Fp& __f, _Alloc&& __a) - : __f_(piecewise_construct, _VSTD::forward_as_tuple(__f), - _VSTD::forward_as_tuple(_VSTD::move(__a))) {} + : __f_(__f, _VSTD::move(__a)) {} _LIBCPP_INLINE_VISIBILITY explicit __func(_Fp&& __f, _Alloc&& __a) - : __f_(piecewise_construct, _VSTD::forward_as_tuple(_VSTD::move(__f)), - _VSTD::forward_as_tuple(_VSTD::move(__a))) {} + : __f_(_VSTD::move(__f), _VSTD::move(__a)) {} + virtual __base<_Rp(_ArgTypes...)>* __clone() const; virtual void __clone(__base<_Rp(_ArgTypes...)>*) const; virtual void destroy() _NOEXCEPT; virtual void destroy_deallocate() _NOEXCEPT; - virtual _Rp operator()(_ArgTypes&& ... __arg); + virtual _Rp operator()(_ArgTypes&&... __arg); #ifndef _LIBCPP_NO_RTTI virtual const void* target(const type_info&) const _NOEXCEPT; virtual const std::type_info& target_type() const _NOEXCEPT; @@ -1532,10 +1607,10 @@ { typedef allocator_traits<_Alloc> __alloc_traits; typedef typename __rebind_alloc_helper<__alloc_traits, __func>::type _Ap; - _Ap __a(__f_.second()); + _Ap __a(__f_.__allocator()); typedef __allocator_destructor<_Ap> _Dp; unique_ptr<__func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); - ::new (__hold.get()) __func(__f_.first(), _Alloc(__a)); + ::new ((void*)__hold.get()) __func(__f_.__target(), _Alloc(__a)); return __hold.release(); } @@ -1543,14 +1618,14 @@ void __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone(__base<_Rp(_ArgTypes...)>* __p) const { - ::new (__p) __func(__f_.first(), __f_.second()); + ::new (__p) __func(__f_.__target(), __f_.__allocator()); } template void __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy() _NOEXCEPT { - __f_.~__compressed_pair<_Fp, _Alloc>(); + __f_.destroy(); } template @@ -1559,8 +1634,8 @@ { typedef allocator_traits<_Alloc> __alloc_traits; typedef typename __rebind_alloc_helper<__alloc_traits, __func>::type _Ap; - _Ap __a(__f_.second()); - __f_.~__compressed_pair<_Fp, _Alloc>(); + _Ap __a(__f_.__allocator()); + __f_.destroy(); __a.deallocate(this, 1); } @@ -1568,8 +1643,7 @@ _Rp __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::operator()(_ArgTypes&& ... __arg) { - typedef __invoke_void_return_wrapper<_Rp> _Invoker; - return _Invoker::__call(__f_.first(), _VSTD::forward<_ArgTypes>(__arg)...); + return __f_(_VSTD::forward<_ArgTypes>(__arg)...); } #ifndef _LIBCPP_NO_RTTI @@ -1579,7 +1653,7 @@ __func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target(const type_info& __ti) const _NOEXCEPT { if (__ti == typeid(_Fp)) - return &__f_.first(); + return &__f_.__target(); return (const void*)0; } @@ -1592,6 +1666,495 @@ #endif // _LIBCPP_NO_RTTI +// Storage for a functor object, to be used with __policy to manage copy and +// destruction. +union __policy_storage +{ + mutable typename aligned_storage<2 * sizeof(void*)>::type __small; + void* __large; +}; + +// True if _Fun can safely be held in __policy_storage.__small. +template +struct __use_small_storage + : public _VSTD::integral_constant< + bool, sizeof(_Fun) <= sizeof(__policy_storage) && + alignof(_Fun) <= alignof(__policy_storage) && + _VSTD::is_trivially_copy_constructible<_Fun>::value && + _VSTD::is_trivially_destructible<_Fun>::value> {}; + +// Policy contains information about how to copy, destroy, and move the +// underlying functor. You can think of it as a vtable of sorts. +struct __policy +{ + // Used to copy or destroy __large values. null for trivial objects. + void* (*const __clone)(const void*); + void (*const __destroy)(void*); + + // True if this is the null policy (no value). + const bool __is_null; + + // The target type. May be null if RTTI is disabled. + const std::type_info* const __type_info; + + // Returns a pointer to a static policy object suitable for the functor + // type. + template + _LIBCPP_INLINE_VISIBILITY static const __policy* __create() + { + return __choose_policy<_Fun>(__use_small_storage<_Fun>()); + } + + _LIBCPP_INLINE_VISIBILITY + static const __policy* __create_empty() + { + static const _LIBCPP_CONSTEXPR __policy __policy_ = {nullptr, nullptr, + true, +#ifndef _LIBCPP_NO_RTTI + &typeid(void) +#else + nullptr +#endif + }; + return &__policy_; + } + + private: + template static void* __large_clone(const void* __s) + { + const _Fun* __f = static_cast(__s); + return __f->__clone(); + } + + template static void __large_destroy(void* __s) + { + typedef allocator_traits __alloc_traits; + typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type + _FunAlloc; + _Fun* __f = static_cast<_Fun*>(__s); + _FunAlloc __a(__f->__allocator()); + __f->destroy(); + __a.deallocate(__f, 1); + } + + template + _LIBCPP_INLINE_VISIBILITY static const __policy* + __choose_policy(/* is_small = */ false_type) + { + static const _LIBCPP_CONSTEXPR __policy __policy_ = { + &__large_clone<_Fun>, &__large_destroy<_Fun>, false, +#ifndef _LIBCPP_NO_RTTI + &typeid(typename _Fun::_Target) +#else + nullptr +#endif + }; + return &__policy_; + } + + template + _LIBCPP_INLINE_VISIBILITY static const __policy* + __choose_policy(/* is_small = */ true_type) + { + static const _LIBCPP_CONSTEXPR __policy __policy_ = { + nullptr, nullptr, false, +#ifndef _LIBCPP_NO_RTTI + &typeid(typename _Fun::_Target) +#else + nullptr +#endif + }; + return &__policy_; + } +}; + +// Used to choose between perfect forwarding or pass-by-value. Pass-by-value is +// faster for types that can be passed in registers. +template +using __fast_forward = + typename _VSTD::conditional<_VSTD::is_scalar<_Tp>::value, _Tp, _Tp&&>::type; + +// __policy_invoker calls an instance of __alloc_func held in __policy_storage. + +template struct __policy_invoker; + +template +struct __policy_invoker<_Rp(_ArgTypes...)> +{ + typedef _Rp (*__Call)(const __policy_storage*, + __fast_forward<_ArgTypes>...); + + __Call __call_; + + // Creates an invoker that throws bad_function_call. + _LIBCPP_INLINE_VISIBILITY + __policy_invoker() : __call_(&__call_empty) {} + + // Creates an invoker that calls the given instance of __func. + template + _LIBCPP_INLINE_VISIBILITY static __policy_invoker __create() + { + return __policy_invoker(&__call_impl<_Fun>); + } + + private: + _LIBCPP_INLINE_VISIBILITY + explicit __policy_invoker(__Call __c) : __call_(__c) {} + + static _Rp __call_empty(const __policy_storage*, + __fast_forward<_ArgTypes>...) + { + __throw_bad_function_call(); + } + + template + static _Rp __call_impl(const __policy_storage* __buf, + __fast_forward<_ArgTypes>... __args) + { + _Fun* __f = reinterpret_cast<_Fun*>(__use_small_storage<_Fun>::value + ? &__buf->__small + : __buf->__large); + return (*__f)(_VSTD::forward<_ArgTypes>(__args)...); + } +}; + +// __value_func creates a value-type from a __func. + +template class __value_func; + +template class __value_func<_Rp(_ArgTypes...)> +{ + typename aligned_storage<3 * sizeof(void*)>::type __buf_; + + typedef __base<_Rp(_ArgTypes...)> __func; + __func* __f_; + + _LIBCPP_NO_CFI static __func* __as_base(void* p) + { + return reinterpret_cast<__func*>(p); + } + + public: + _LIBCPP_INLINE_VISIBILITY + __value_func() _NOEXCEPT : __f_(0) {} + + template + _LIBCPP_INLINE_VISIBILITY __value_func(_Fp&& __f, const _Alloc __a) + : __f_(0) + { + typedef allocator_traits<_Alloc> __alloc_traits; + typedef __function::__func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun; + typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type + _FunAlloc; + + if (__function::__not_null(__f)) + { + _FunAlloc __af(__a); + if (sizeof(_Fun) <= sizeof(__buf_) && + is_nothrow_copy_constructible<_Fp>::value && + is_nothrow_copy_constructible<_FunAlloc>::value) + { + __f_ = + ::new ((void*)&__buf_) _Fun(_VSTD::move(__f), _Alloc(__af)); + } + else + { + typedef __allocator_destructor<_FunAlloc> _Dp; + unique_ptr<__func, _Dp> __hold(__af.allocate(1), _Dp(__af, 1)); + ::new ((void*)__hold.get()) _Fun(_VSTD::move(__f), _Alloc(__a)); + __f_ = __hold.release(); + } + } + } + + _LIBCPP_INLINE_VISIBILITY + __value_func(const __value_func& __f) + { + if (__f.__f_ == 0) + __f_ = 0; + else if ((void*)__f.__f_ == &__f.__buf_) + { + __f_ = __as_base(&__buf_); + __f.__f_->__clone(__f_); + } + else + __f_ = __f.__f_->__clone(); + } + + _LIBCPP_INLINE_VISIBILITY + __value_func(__value_func&& __f) _NOEXCEPT + { + if (__f.__f_ == 0) + __f_ = 0; + else if ((void*)__f.__f_ == &__f.__buf_) + { + __f_ = __as_base(&__buf_); + __f.__f_->__clone(__f_); + } + else + { + __f_ = __f.__f_; + __f.__f_ = 0; + } + } + + _LIBCPP_INLINE_VISIBILITY + ~__value_func() + { + if ((void*)__f_ == &__buf_) + __f_->destroy(); + else if (__f_) + __f_->destroy_deallocate(); + } + + _LIBCPP_INLINE_VISIBILITY + __value_func& operator=(__value_func&& __f) + { + *this = nullptr; + if (__f.__f_ == 0) + __f_ = 0; + else if ((void*)__f.__f_ == &__f.__buf_) + { + __f_ = __as_base(&__buf_); + __f.__f_->__clone(__f_); + } + else + { + __f_ = __f.__f_; + __f.__f_ = 0; + } + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + __value_func& operator=(nullptr_t) + { + __func* __f = __f_; + __f_ = 0; + if ((void*)__f == &__buf_) + __f->destroy(); + else if (__f) + __f->destroy_deallocate(); + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + _Rp operator()(_ArgTypes&&... __args) const + { + if (__f_ == 0) + __throw_bad_function_call(); + return (*__f_)(_VSTD::forward<_ArgTypes>(__args)...); + } + + _LIBCPP_INLINE_VISIBILITY + void swap(__value_func& __f) _NOEXCEPT + { + if (&__f == this) + return; + if ((void*)__f_ == &__buf_ && (void*)__f.__f_ == &__f.__buf_) + { + typename aligned_storage::type __tempbuf; + __func* __t = __as_base(&__tempbuf); + __f_->__clone(__t); + __f_->destroy(); + __f_ = 0; + __f.__f_->__clone(__as_base(&__buf_)); + __f.__f_->destroy(); + __f.__f_ = 0; + __f_ = __as_base(&__buf_); + __t->__clone(__as_base(&__f.__buf_)); + __t->destroy(); + __f.__f_ = __as_base(&__f.__buf_); + } + else if ((void*)__f_ == &__buf_) + { + __f_->__clone(__as_base(&__f.__buf_)); + __f_->destroy(); + __f_ = __f.__f_; + __f.__f_ = __as_base(&__f.__buf_); + } + else if ((void*)__f.__f_ == &__f.__buf_) + { + __f.__f_->__clone(__as_base(&__buf_)); + __f.__f_->destroy(); + __f.__f_ = __f_; + __f_ = __as_base(&__buf_); + } + else + _VSTD::swap(__f_, __f.__f_); + } + + _LIBCPP_INLINE_VISIBILITY + _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT { return __f_ != 0; } + +#ifndef _LIBCPP_NO_RTTI + _LIBCPP_INLINE_VISIBILITY + const std::type_info& target_type() const _NOEXCEPT + { + if (__f_ == 0) + return typeid(void); + return __f_->target_type(); + } + + template + _LIBCPP_INLINE_VISIBILITY const _Tp* target() const _NOEXCEPT + { + if (__f_ == 0) + return 0; + return (const _Tp*)__f_->target(typeid(_Tp)); + } +#endif // _LIBCPP_NO_RTTI +}; + +// __policy_func uses a __policy and __policy_invoker to create a type-erased, +// copyable functor. + +template class __policy_func; + +template class __policy_func<_Rp(_ArgTypes...)> +{ + // Inline storage for small objects. This is first so that alignment + // requirements of inline objects are more likely to be met without making + // this object larger. + __policy_storage __buf_; + + // Calls the value stored in __buf_. This could technically be part of + // policy, but storing it here eliminates a level of indirection inside + // operator(). + typedef __function::__policy_invoker<_Rp(_ArgTypes...)> __invoker; + __invoker __invoker_; + + // The policy that describes how to move / copy / destroy __buf_. Never + // null, even if the function is empty. + const __policy* __policy_; + + public: + _LIBCPP_INLINE_VISIBILITY + __policy_func() : __policy_(__policy::__create_empty()) {} + + template + _LIBCPP_INLINE_VISIBILITY __policy_func(_Fp&& __f, const _Alloc& __a) + : __policy_(__policy::__create_empty()) + { + typedef __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun; + typedef allocator_traits<_Alloc> __alloc_traits; + typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type + _FunAlloc; + + if (__function::__not_null(__f)) + { + __invoker_ = __invoker::template __create<_Fun>(); + __policy_ = __policy::__create<_Fun>(); + + _FunAlloc __af(__a); + if (__use_small_storage<_Fun>()) + { + ::new ((void*)&__buf_.__small) + _Fun(_VSTD::move(__f), _Alloc(__af)); + } + else + { + typedef __allocator_destructor<_FunAlloc> _Dp; + unique_ptr<_Fun, _Dp> __hold(__af.allocate(1), _Dp(__af, 1)); + ::new ((void*)__hold.get()) + _Fun(_VSTD::move(__f), _Alloc(__af)); + __buf_.__large = __hold.release(); + } + } + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func(const __policy_func& __f) + : __buf_(__f.__buf_), __invoker_(__f.__invoker_), + __policy_(__f.__policy_) + { + if (__policy_->__clone) + __buf_.__large = __policy_->__clone(__f.__buf_.__large); + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func(__policy_func&& __f) + : __buf_(__f.__buf_), __invoker_(__f.__invoker_), + __policy_(__f.__policy_) + { + if (__policy_->__destroy) + { + __f.__policy_ = __policy::__create_empty(); + __f.__invoker_ = __invoker(); + } + } + + _LIBCPP_INLINE_VISIBILITY + ~__policy_func() + { + if (__policy_->__destroy) + __policy_->__destroy(__buf_.__large); + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func& operator=(__policy_func&& __f) + { + *this = nullptr; + __buf_ = __f.__buf_; + __invoker_ = __f.__invoker_; + __policy_ = __f.__policy_; + __f.__policy_ = __policy::__create_empty(); + __f.__invoker_ = __invoker(); + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + __policy_func& operator=(nullptr_t) + { + const __policy* __p = __policy_; + __policy_ = __policy::__create_empty(); + __invoker_ = __invoker(); + if (__p->__destroy) + __p->__destroy(__buf_.__large); + return *this; + } + + _LIBCPP_INLINE_VISIBILITY + _Rp operator()(_ArgTypes&&... __args) const + { + return __invoker_.__call_(&__buf_, + _VSTD::forward<_ArgTypes>(__args)...); + } + + _LIBCPP_INLINE_VISIBILITY + void swap(__policy_func& __f) + { + _VSTD::swap(__invoker_, __f.__invoker_); + _VSTD::swap(__policy_, __f.__policy_); + _VSTD::swap(__buf_, __f.__buf_); + } + + _LIBCPP_INLINE_VISIBILITY + _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT + { + return !__policy_->__is_null; + } + +#ifndef _LIBCPP_NO_RTTI + _LIBCPP_INLINE_VISIBILITY + const std::type_info& target_type() const _NOEXCEPT + { + return *__policy_->__type_info; + } + + template + _LIBCPP_INLINE_VISIBILITY const _Tp* target() const _NOEXCEPT + { + if (__policy_->__is_null || typeid(_Tp) != *__policy_->__type_info) + return nullptr; + if (__policy_->__clone) // Out of line storage. + return reinterpret_cast(__buf_.__large); + else + return reinterpret_cast(&__buf_.__small); + } +#endif // _LIBCPP_NO_RTTI +}; + } // __function template @@ -1599,13 +2162,16 @@ : public __function::__maybe_derive_from_unary_function<_Rp(_ArgTypes...)>, public __function::__maybe_derive_from_binary_function<_Rp(_ArgTypes...)> { - typedef __function::__base<_Rp(_ArgTypes...)> __base; - typename aligned_storage<3*sizeof(void*)>::type __buf_; - __base* __f_; + // TODO: REMOVE BEFORE SUBMITTING! +#define _LIBCPP_ABI_OPTIMIZED_FUNCTION - _LIBCPP_NO_CFI static __base *__as_base(void *p) { - return reinterpret_cast<__base*>(p); - } +#ifndef _LIBCPP_ABI_OPTIMIZED_FUNCTION + typedef __function::__value_func<_Rp(_ArgTypes...)> __func; +#else + typedef __function::__policy_func<_Rp(_ArgTypes...)> __func; +#endif + + __func __f_; template , function>::value>, @@ -1632,9 +2198,9 @@ // construct/copy/destroy: _LIBCPP_INLINE_VISIBILITY - function() _NOEXCEPT : __f_(0) {} + function() _NOEXCEPT { } _LIBCPP_INLINE_VISIBILITY - function(nullptr_t) _NOEXCEPT : __f_(0) {} + function(nullptr_t) _NOEXCEPT {} function(const function&); function(function&&) _NOEXCEPT; template> @@ -1643,10 +2209,10 @@ #if _LIBCPP_STD_VER <= 14 template _LIBCPP_INLINE_VISIBILITY - function(allocator_arg_t, const _Alloc&) _NOEXCEPT : __f_(0) {} + function(allocator_arg_t, const _Alloc&) _NOEXCEPT {} template _LIBCPP_INLINE_VISIBILITY - function(allocator_arg_t, const _Alloc&, nullptr_t) _NOEXCEPT : __f_(0) {} + function(allocator_arg_t, const _Alloc&, nullptr_t) _NOEXCEPT {} template function(allocator_arg_t, const _Alloc&, const function&); template @@ -1675,7 +2241,9 @@ // function capacity: _LIBCPP_INLINE_VISIBILITY - _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT {return __f_;} + _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT { + return static_cast(__f_); + } // deleted overloads close possible hole in the type system template @@ -1695,125 +2263,38 @@ }; template -function<_Rp(_ArgTypes...)>::function(const function& __f) -{ - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - __f_ = __f.__f_->__clone(); -} +function<_Rp(_ArgTypes...)>::function(const function& __f) : __f_(__f.__f_) {} #if _LIBCPP_STD_VER <= 14 template template function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc&, - const function& __f) -{ - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - __f_ = __f.__f_->__clone(); -} + const function& __f) : __f_(__f.__f_) {} #endif -template +template function<_Rp(_ArgTypes...)>::function(function&& __f) _NOEXCEPT -{ - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - { - __f_ = __f.__f_; - __f.__f_ = 0; - } -} + : __f_(_VSTD::move(__f.__f_)) {} #if _LIBCPP_STD_VER <= 14 template template function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc&, - function&& __f) -{ - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - { - __f_ = __f.__f_; - __f.__f_ = 0; - } -} + function&& __f) + : __f_(_VSTD::move(__f.__f_)) {} #endif -template +template template function<_Rp(_ArgTypes...)>::function(_Fp __f) - : __f_(0) -{ - if (__function::__not_null(__f)) - { - typedef __function::__func<_Fp, allocator<_Fp>, _Rp(_ArgTypes...)> _FF; - if (sizeof(_FF) <= sizeof(__buf_) && is_nothrow_copy_constructible<_Fp>::value) - { - __f_ = ::new((void*)&__buf_) _FF(_VSTD::move(__f)); - } - else - { - typedef allocator<_FF> _Ap; - _Ap __a; - typedef __allocator_destructor<_Ap> _Dp; - unique_ptr<__base, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); - ::new (__hold.get()) _FF(_VSTD::move(__f), allocator<_Fp>(__a)); - __f_ = __hold.release(); - } - } -} + : __f_(_VSTD::move(__f), allocator<_Fp>()) {} #if _LIBCPP_STD_VER <= 14 -template +template template -function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc& __a0, _Fp __f) - : __f_(0) -{ - typedef allocator_traits<_Alloc> __alloc_traits; - if (__function::__not_null(__f)) - { - typedef __function::__func<_Fp, _Alloc, _Rp(_ArgTypes...)> _FF; - typedef typename __rebind_alloc_helper<__alloc_traits, _FF>::type _Ap; - _Ap __a(__a0); - if (sizeof(_FF) <= sizeof(__buf_) && - is_nothrow_copy_constructible<_Fp>::value && is_nothrow_copy_constructible<_Ap>::value) - { - __f_ = ::new((void*)&__buf_) _FF(_VSTD::move(__f), _Alloc(__a)); - } - else - { - typedef __allocator_destructor<_Ap> _Dp; - unique_ptr<__base, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); - ::new (__hold.get()) _FF(_VSTD::move(__f), _Alloc(__a)); - __f_ = __hold.release(); - } - } -} +function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc& __a, + _Fp __f) + : __f_(_VSTD::move(__f), __a) {} #endif template @@ -1828,19 +2309,7 @@ function<_Rp(_ArgTypes...)>& function<_Rp(_ArgTypes...)>::operator=(function&& __f) _NOEXCEPT { - *this = nullptr; - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - { - __f_ = __f.__f_; - __f.__f_ = 0; - } + __f_ = std::move(__f.__f_); return *this; } @@ -1848,12 +2317,7 @@ function<_Rp(_ArgTypes...)>& function<_Rp(_ArgTypes...)>::operator=(nullptr_t) _NOEXCEPT { - __base* __t = __f_; - __f_ = 0; - if ((void *)__t == &__buf_) - __t->destroy(); - else if (__t) - __t->destroy_deallocate(); + __f_ = nullptr; return *this; } @@ -1867,60 +2331,20 @@ } template -function<_Rp(_ArgTypes...)>::~function() -{ - if ((void *)__f_ == &__buf_) - __f_->destroy(); - else if (__f_) - __f_->destroy_deallocate(); -} +function<_Rp(_ArgTypes...)>::~function() {} template void function<_Rp(_ArgTypes...)>::swap(function& __f) _NOEXCEPT { - if (_VSTD::addressof(__f) == this) - return; - if ((void *)__f_ == &__buf_ && (void *)__f.__f_ == &__f.__buf_) - { - typename aligned_storage::type __tempbuf; - __base* __t = __as_base(&__tempbuf); - __f_->__clone(__t); - __f_->destroy(); - __f_ = 0; - __f.__f_->__clone(__as_base(&__buf_)); - __f.__f_->destroy(); - __f.__f_ = 0; - __f_ = __as_base(&__buf_); - __t->__clone(__as_base(&__f.__buf_)); - __t->destroy(); - __f.__f_ = __as_base(&__f.__buf_); - } - else if ((void *)__f_ == &__buf_) - { - __f_->__clone(__as_base(&__f.__buf_)); - __f_->destroy(); - __f_ = __f.__f_; - __f.__f_ = __as_base(&__f.__buf_); - } - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f.__f_->__clone(__as_base(&__buf_)); - __f.__f_->destroy(); - __f.__f_ = __f_; - __f_ = __as_base(&__buf_); - } - else - _VSTD::swap(__f_, __f.__f_); + __f_.swap(__f.__f_); } template _Rp function<_Rp(_ArgTypes...)>::operator()(_ArgTypes... __arg) const { - if (__f_ == 0) - __throw_bad_function_call(); - return (*__f_)(_VSTD::forward<_ArgTypes>(__arg)...); + return __f_(_VSTD::forward<_ArgTypes>(__arg)...); } #ifndef _LIBCPP_NO_RTTI @@ -1929,9 +2353,7 @@ const std::type_info& function<_Rp(_ArgTypes...)>::target_type() const _NOEXCEPT { - if (__f_ == 0) - return typeid(void); - return __f_->target_type(); + return __f_.target_type(); } template @@ -1939,9 +2361,7 @@ _Tp* function<_Rp(_ArgTypes...)>::target() _NOEXCEPT { - if (__f_ == 0) - return nullptr; - return (_Tp*) const_cast(__f_->target(typeid(_Tp))); + return (_Tp*)(__f_.template target<_Tp>()); } template @@ -1949,9 +2369,7 @@ const _Tp* function<_Rp(_ArgTypes...)>::target() const _NOEXCEPT { - if (__f_ == 0) - return nullptr; - return (const _Tp*)__f_->target(typeid(_Tp)); + return __f_.template target<_Tp>(); } #endif // _LIBCPP_NO_RTTI Index: test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_F.pass.cpp =================================================================== --- test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_F.pass.cpp +++ test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_F.pass.cpp @@ -65,7 +65,8 @@ FuncType* target = &FreeFunction; assert(globalMemCounter.checkOutstandingNewEq(0)); std::function f2(std::allocator_arg, alloc, target); - assert(globalMemCounter.checkOutstandingNewEq(0)); + assert(globalMemCounter.checkOutstandingNewEq(0) || + globalMemCounter.checkOutstandingNewEq(1)); assert(f2.template target()); assert(*f2.template target() == target); assert(f2.template target() == 0); @@ -82,7 +83,8 @@ TargetType target = &MemFunClass::foo; assert(globalMemCounter.checkOutstandingNewEq(0)); std::function f2(std::allocator_arg, alloc, target); - assert(globalMemCounter.checkOutstandingNewEq(0)); + assert(globalMemCounter.checkOutstandingNewEq(0) || + globalMemCounter.checkOutstandingNewEq(1)); assert(f2.template target()); assert(*f2.template target() == target); assert(f2.template target() == 0); Index: test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_rfunction.pass.cpp =================================================================== --- test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_rfunction.pass.cpp +++ test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_rfunction.pass.cpp @@ -89,7 +89,6 @@ assert(A::count == 1); assert(f2.target() == nullptr); assert(f2.target()); - assert(f.target()); // f is unchanged because the target is small } { // Test that moving a function constructed from a function pointer @@ -104,6 +103,5 @@ std::move(f)); assert(f2.target() == nullptr); assert(f2.target()); - assert(f.target()); // f is unchanged because the target is small } } Index: test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/copy_move.pass.cpp =================================================================== --- test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/copy_move.pass.cpp +++ test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/copy_move.pass.cpp @@ -141,7 +141,6 @@ assert(A::count == 1); assert(f2.target() == nullptr); assert(f2.target()); - LIBCPP_ASSERT(f.target()); // f is unchanged because the target is small } { // Test that moving a function constructed from a function pointer @@ -159,7 +158,6 @@ std::function f2(std::move(f)); assert(f2.target() == nullptr); assert(f2.target()); - LIBCPP_ASSERT(f.target()); // f is unchanged because the target is small } #endif // TEST_STD_VER >= 11 }