Index: benchmarks/function.bench.cpp =================================================================== --- benchmarks/function.bench.cpp +++ benchmarks/function.bench.cpp @@ -213,6 +213,62 @@ } }; +struct TrivialStruct {}; + +struct NonTrivialStruct { + NonTrivialStruct() {} + NonTrivialStruct(const NonTrivialStruct&) {} + ~NonTrivialStruct() {} +}; + +template +using ChooseTrivialness = + typename std::conditional<(Seed % 5) == 0, NonTrivialStruct, + TrivialStruct>::type; + +template +struct ChooseSize { + // Lambda captures tend to be word-sized (pointers and references). + void* padding[Seed % 5]; +}; + +template +struct GeneratedFunctor : public ChooseTrivialness, + public ChooseSize { + int operator()(const S*) const { return 0; } +}; + +template +void ConstructMoveAndCallGeneratedFunctor(S* s) { + Function f = GeneratedFunctor{}; + benchmark::DoNotOptimize(f); + Function mf = std::move(f); + benchmark::DoNotOptimize(mf); + benchmark::DoNotOptimize(mf(s)); +} + +template +void RunGeneratedFunctors(S* s) { + ConstructMoveAndCallGeneratedFunctor(s); + RunGeneratedFunctors(s); +} + +template <> +void RunGeneratedFunctors<0>(S* s) { + ConstructMoveAndCallGeneratedFunctor<0>(s); +} + +// Explicitly instantiate some functions to avoid too much template recursion. +template void RunGeneratedFunctors<256>(S* s); +template void RunGeneratedFunctors<512>(S* s); + +void BM_MixedFunctorTypes(benchmark::State& state) { + S s; + for (auto _ : state) + RunGeneratedFunctors<512>(&s); +} +BENCHMARK(BM_MixedFunctorTypes); + } // namespace int main(int argc, char** argv) { Index: include/__config =================================================================== --- include/__config +++ include/__config @@ -95,6 +95,8 @@ // Use the smallest possible integer type to represent the index of the variant. // Previously libc++ used "unsigned int" exclusivly. # define _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION +// Eliminate conditionals and pointer indirection from std::function. +# define _LIBCPP_ABI_OPTIMIZED_FUNCTION #elif _LIBCPP_ABI_VERSION == 1 # if !defined(_LIBCPP_OBJECT_FORMAT_COFF) // Enable compiling copies of now inline methods into the dylib to support Index: include/functional =================================================================== --- include/functional +++ include/functional @@ -1468,18 +1468,79 @@ namespace __function { -template class __base; +// __func holds a functor and an allocator. + +template class __func; + +template +class __func<_Fp, _Ap, _Rp(_ArgTypes...)> { + __compressed_pair<_Fp, _Ap> __f_; + +public: + typedef _Fp _Target; + typedef _Ap _Alloc; + + const _Target& target() const { return __f_.first(); } + const _Alloc& allocator() const { return __f_.second(); } + + _LIBCPP_INLINE_VISIBILITY + explicit __func(_Target&& __f) + : __f_(piecewise_construct, _VSTD::forward_as_tuple(_VSTD::move(__f)), + _VSTD::forward_as_tuple()) {} + + _LIBCPP_INLINE_VISIBILITY + explicit __func(const _Target& __f, const _Alloc& __a) + : __f_(piecewise_construct, _VSTD::forward_as_tuple(__f), + _VSTD::forward_as_tuple(__a)) {} + + _LIBCPP_INLINE_VISIBILITY + explicit __func(const _Target& __f, _Alloc&& __a) + : __f_(piecewise_construct, _VSTD::forward_as_tuple(__f), + _VSTD::forward_as_tuple(_VSTD::move(__a))) {} + + _LIBCPP_INLINE_VISIBILITY + explicit __func(_Target&& __f, _Alloc&& __a) + : __f_(piecewise_construct, _VSTD::forward_as_tuple(_VSTD::move(__f)), + _VSTD::forward_as_tuple(_VSTD::move(__a))) {} + + _LIBCPP_INLINE_VISIBILITY + _Rp operator()(_ArgTypes&& ... __arg) { + typedef __invoke_void_return_wrapper<_Rp> _Invoker; + return _Invoker::__call(__f_.first(), _VSTD::forward<_ArgTypes>(__arg)...); + } + + _LIBCPP_INLINE_VISIBILITY + __func* __clone() const { + typedef allocator_traits<_Alloc> __alloc_traits; + typedef typename __rebind_alloc_helper<__alloc_traits, __func>::type _AA; + _AA __a(__f_.second()); + typedef __allocator_destructor<_AA> _Dp; + unique_ptr<__func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); + ::new ((void*)__hold.get()) __func(__f_.first(), _Alloc(__a)); + return __hold.release(); + } + + _LIBCPP_INLINE_VISIBILITY + void destroy() _NOEXCEPT { + __f_.~__compressed_pair<_Target, _Alloc>(); + } +}; + + +// __vfunc provides an abstract interface for copyable functors. + +template class __vfunc; template -class __base<_Rp(_ArgTypes...)> +class __vfunc<_Rp(_ArgTypes...)> { - __base(const __base&); - __base& operator=(const __base&); + __vfunc(const __vfunc&); + __vfunc& operator=(const __vfunc&); public: - _LIBCPP_INLINE_VISIBILITY __base() {} - _LIBCPP_INLINE_VISIBILITY virtual ~__base() {} - virtual __base* __clone() const = 0; - virtual void __clone(__base*) const = 0; + _LIBCPP_INLINE_VISIBILITY __vfunc() {} + _LIBCPP_INLINE_VISIBILITY virtual ~__vfunc() {} + virtual __vfunc* __clone() const = 0; + virtual void __clone(__vfunc*) const = 0; virtual void destroy() _NOEXCEPT = 0; virtual void destroy_deallocate() _NOEXCEPT = 0; virtual _Rp operator()(_ArgTypes&& ...) = 0; @@ -1489,37 +1550,37 @@ #endif // _LIBCPP_NO_RTTI }; -template class __func; +// __vfunc_impl implements __vfunc for a given functor type. + +template class __vfunc_impl; template -class __func<_Fp, _Alloc, _Rp(_ArgTypes...)> - : public __base<_Rp(_ArgTypes...)> +class __vfunc_impl<_Fp, _Alloc, _Rp(_ArgTypes...)> + : public __vfunc<_Rp(_ArgTypes...)> { - __compressed_pair<_Fp, _Alloc> __f_; + __func<_Fp, _Alloc, _Rp(_ArgTypes...)> __f_; public: _LIBCPP_INLINE_VISIBILITY - explicit __func(_Fp&& __f) - : __f_(piecewise_construct, _VSTD::forward_as_tuple(_VSTD::move(__f)), - _VSTD::forward_as_tuple()) {} + explicit __vfunc_impl(_Fp&& __f) + : __f_(_VSTD::move(__f)) {} + _LIBCPP_INLINE_VISIBILITY - explicit __func(const _Fp& __f, const _Alloc& __a) - : __f_(piecewise_construct, _VSTD::forward_as_tuple(__f), - _VSTD::forward_as_tuple(__a)) {} + explicit __vfunc_impl(const _Fp& __f, const _Alloc& __a) + : __f_(__f, __a) {} _LIBCPP_INLINE_VISIBILITY - explicit __func(const _Fp& __f, _Alloc&& __a) - : __f_(piecewise_construct, _VSTD::forward_as_tuple(__f), - _VSTD::forward_as_tuple(_VSTD::move(__a))) {} + explicit __vfunc_impl(const _Fp& __f, _Alloc&& __a) + : __f_(__f, _VSTD::move(__a)) {} _LIBCPP_INLINE_VISIBILITY - explicit __func(_Fp&& __f, _Alloc&& __a) - : __f_(piecewise_construct, _VSTD::forward_as_tuple(_VSTD::move(__f)), - _VSTD::forward_as_tuple(_VSTD::move(__a))) {} - virtual __base<_Rp(_ArgTypes...)>* __clone() const; - virtual void __clone(__base<_Rp(_ArgTypes...)>*) const; + explicit __vfunc_impl(_Fp&& __f, _Alloc&& __a) + : __f_(_VSTD::move(__f), _VSTD::move(__a)) {} + + virtual __vfunc<_Rp(_ArgTypes...)>* __clone() const; + virtual void __clone(__vfunc<_Rp(_ArgTypes...)>*) const; virtual void destroy() _NOEXCEPT; virtual void destroy_deallocate() _NOEXCEPT; - virtual _Rp operator()(_ArgTypes&& ... __arg); + virtual _Rp operator()(_ArgTypes&&... __arg); #ifndef _LIBCPP_NO_RTTI virtual const void* target(const type_info&) const _NOEXCEPT; virtual const std::type_info& target_type() const _NOEXCEPT; @@ -1527,71 +1588,498 @@ }; template -__base<_Rp(_ArgTypes...)>* -__func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone() const +__vfunc<_Rp(_ArgTypes...)>* +__vfunc_impl<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone() const { typedef allocator_traits<_Alloc> __alloc_traits; - typedef typename __rebind_alloc_helper<__alloc_traits, __func>::type _Ap; - _Ap __a(__f_.second()); + typedef typename __rebind_alloc_helper<__alloc_traits, __vfunc_impl>::type _Ap; + _Ap __a(__f_.allocator()); typedef __allocator_destructor<_Ap> _Dp; - unique_ptr<__func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); - ::new (__hold.get()) __func(__f_.first(), _Alloc(__a)); + unique_ptr<__vfunc_impl, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); + ::new ((void*)__hold.get()) __vfunc_impl(__f_.target(), _Alloc(__a)); return __hold.release(); } template void -__func<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone(__base<_Rp(_ArgTypes...)>* __p) const +__vfunc_impl<_Fp, _Alloc, _Rp(_ArgTypes...)>::__clone(__vfunc<_Rp(_ArgTypes...)>* __p) const { - ::new (__p) __func(__f_.first(), __f_.second()); + ::new (__p) __vfunc_impl(__f_.target(), __f_.allocator()); } template void -__func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy() _NOEXCEPT +__vfunc_impl<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy() _NOEXCEPT { - __f_.~__compressed_pair<_Fp, _Alloc>(); + __f_.destroy(); } template void -__func<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy_deallocate() _NOEXCEPT +__vfunc_impl<_Fp, _Alloc, _Rp(_ArgTypes...)>::destroy_deallocate() _NOEXCEPT { typedef allocator_traits<_Alloc> __alloc_traits; - typedef typename __rebind_alloc_helper<__alloc_traits, __func>::type _Ap; - _Ap __a(__f_.second()); - __f_.~__compressed_pair<_Fp, _Alloc>(); + typedef typename __rebind_alloc_helper<__alloc_traits, __vfunc_impl>::type _Ap; + _Ap __a(__f_.allocator()); + __f_.destroy(); __a.deallocate(this, 1); } template _Rp -__func<_Fp, _Alloc, _Rp(_ArgTypes...)>::operator()(_ArgTypes&& ... __arg) +__vfunc_impl<_Fp, _Alloc, _Rp(_ArgTypes...)>::operator()(_ArgTypes&& ... __arg) { - typedef __invoke_void_return_wrapper<_Rp> _Invoker; - return _Invoker::__call(__f_.first(), _VSTD::forward<_ArgTypes>(__arg)...); + return __f_(_VSTD::forward<_ArgTypes>(__arg)...); } #ifndef _LIBCPP_NO_RTTI template const void* -__func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target(const type_info& __ti) const _NOEXCEPT +__vfunc_impl<_Fp, _Alloc, _Rp(_ArgTypes...)>::target(const type_info& __ti) const _NOEXCEPT { if (__ti == typeid(_Fp)) - return &__f_.first(); + return &__f_.target(); return (const void*)0; } template const std::type_info& -__func<_Fp, _Alloc, _Rp(_ArgTypes...)>::target_type() const _NOEXCEPT +__vfunc_impl<_Fp, _Alloc, _Rp(_ArgTypes...)>::target_type() const _NOEXCEPT { return typeid(_Fp); } #endif // _LIBCPP_NO_RTTI + +union __storage +{ + mutable typename aligned_storage<2*sizeof(void*)>::type __small; + void* __large; +}; + +// True if __func can safely be held in __storage.__small. +template +struct __use_small_storage + : public _VSTD::integral_constant< + bool, sizeof(_Fun) <= sizeof(__storage) && + alignof(_Fun) <= alignof(__storage) && + _VSTD::is_trivially_copy_constructible<_Fun>::value && + _VSTD::is_trivially_destructible<_Fun>::value> {}; + +// Policy contains information about how to copy, destroy, and move the +// undelying functor. You can think of it as a vtable of sorts. +struct __policy +{ + void* (*const __clone)(const void*); + void (*const __destroy)(void*); + + // True if this is the null policy (no value). + bool __is_null; + + // The target type. May be null if RTTI is disabled. + const std::type_info* type_info; + + // Returns a pointer to a static policy object suitable for the functor type. + template + _LIBCPP_INLINE_VISIBILITY + static const __policy* __create() { + return __choose_policy<_Fun>(__use_small_storage<_Fun>()); + } + + _LIBCPP_INLINE_VISIBILITY + static const __policy* __create_empty() { + static _LIBCPP_CONSTEXPR __policy __policy_ = + {nullptr, nullptr, true, +#ifndef _LIBCPP_NO_RTTI + &typeid(void) +#else + nullptr +#endif + }; + return &__policy_; + } + +private: + template + static void* __large_clone(const void* __s) + { + const _Fun* __f = static_cast(__s); + return __f->__clone(); + }; + + template + static void __large_destroy(void* __s) + { + typedef allocator_traits __alloc_traits; + typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type _FunAlloc; + _Fun* __f = static_cast<_Fun*>(__s); + _FunAlloc __a(__f->allocator()); + __f->destroy(); + __a.deallocate(__f, 1); + } + + template + _LIBCPP_INLINE_VISIBILITY + static const __policy* __choose_policy(/* is_small = */ false_type) + { + static _LIBCPP_CONSTEXPR __policy __policy_ = + {&__large_clone<_Fun>, &__large_destroy<_Fun>, false, + #ifndef _LIBCPP_NO_RTTI + &typeid(typename _Fun::_Target) + #else + nullptr + #endif + }; + return &__policy_; + } + + template + _LIBCPP_INLINE_VISIBILITY + static const __policy* __choose_policy(/* is_small = */ true_type) + { + static _LIBCPP_CONSTEXPR __policy __policy_ = + {nullptr, nullptr, false, + #ifndef _LIBCPP_NO_RTTI + &typeid(typename _Fun::_Target) + #else + nullptr + #endif + }; + return &__policy_; + } + +}; + +// Calls an instance of __func held in __storage. + +template struct __invoker; + +template +struct __invoker<_Rp(_ArgTypes...)> { + typedef _Rp (*__Call)(const __storage*, _ArgTypes&&...); + + __Call __call_; + + // Creates an invoker that throws bad_function_call. + _LIBCPP_INLINE_VISIBILITY + __invoker() : __call_(&__call_empty) {} + + // Creates an invoker that calls the given instance of __func. + template + _LIBCPP_INLINE_VISIBILITY + static __invoker __create() { + return __invoker(__choose_call<_Fun>(__use_small_storage<_Fun>())); + } + +private: + _LIBCPP_INLINE_VISIBILITY + explicit __invoker(__Call __c) : __call_(__c) {} + + static _Rp __call_empty(const __storage*, _ArgTypes&&...) { + __throw_bad_function_call(); + } + + template + static _Rp __call_large(const __storage* __buf, _ArgTypes&&... __args) + { + _Fun* __f = reinterpret_cast<_Fun*>(__buf->__large); + return (*__f)(_VSTD::forward<_ArgTypes>(__args)...); + } + + template + static _Rp __call_small(const __storage* __buf, _ArgTypes&&... __args) + { + _Fun* __f = reinterpret_cast<_Fun*>(&__buf->__small); + return (*__f)(_VSTD::forward<_ArgTypes>(__args)...); + } + + template + _LIBCPP_INLINE_VISIBILITY + static __Call __choose_call(/* small == */ false_type) + { + return __call_large<_Fun>; + } + + template + _LIBCPP_INLINE_VISIBILITY + static __Call __choose_call(/* small == */ true_type) + { + return __call_small<_Fun>; + } +}; + +template class __vfunc_value; + +template +class __vfunc_value<_Rp(_ArgTypes...)> { + typename aligned_storage<3*sizeof(void*)>::type __buf_; + + typedef __function::__vfunc<_Rp(_ArgTypes...)> __vfunc; + __vfunc* __f_; + + _LIBCPP_NO_CFI static __vfunc *__as_base(void *p) { + return reinterpret_cast<__vfunc*>(p); + } + +public: + _LIBCPP_INLINE_VISIBILITY + __vfunc_value() _NOEXCEPT : __f_(0) {} + + template + _LIBCPP_INLINE_VISIBILITY + __vfunc_value(_Fp&& __f, const _Alloc __a) + : __f_(0) + { + typedef allocator_traits<_Alloc> __alloc_traits; + typedef __function::__vfunc_impl<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun; + typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type _FunAlloc; + + if (__function::__not_null(__f)) + { + _FunAlloc __af(__a); + if (sizeof(_Fun) <= sizeof(__buf_) && + is_nothrow_copy_constructible<_Fp>::value && + is_nothrow_copy_constructible<_FunAlloc>::value) + { + __f_ = ::new ((void*)&__buf_) _Fun(_VSTD::move(__f), _Alloc(__af)); + } else { + typedef __allocator_destructor<_FunAlloc> _Dp; + unique_ptr<__vfunc, _Dp> __hold(__af.allocate(1), _Dp(__af, 1)); + ::new ((void*)__hold.get()) _Fun(_VSTD::move(__f), _Alloc(__a)); + __f_ = __hold.release(); + } + } + } + + _LIBCPP_INLINE_VISIBILITY + __vfunc_value(const __vfunc_value& __f) { + if (__f.__f_ == 0) + __f_ = 0; + else if ((void*)__f.__f_ == &__f.__buf_) { + __f_ = __as_base(&__buf_); + __f.__f_->__clone(__f_); + } else + __f_ = __f.__f_->__clone(); + } + + _LIBCPP_INLINE_VISIBILITY + __vfunc_value(__vfunc_value&& __f) _NOEXCEPT { + if (__f.__f_ == 0) + __f_ = 0; + else if ((void *)__f.__f_ == &__f.__buf_) + { + __f_ = __as_base(&__buf_); + __f.__f_->__clone(__f_); + } + else + { + __f_ = __f.__f_; + __f.__f_ = 0; + } + } + + _LIBCPP_INLINE_VISIBILITY + ~__vfunc_value() { + if ((void *)__f_ == &__buf_) + __f_->destroy(); + else if (__f_) + __f_->destroy_deallocate(); + } + + _LIBCPP_INLINE_VISIBILITY + void clear() { + __vfunc* __f = __f_; + __f_ = 0; + if ((void *)__f == &__buf_) + __f->destroy(); + else if (__f) + __f->destroy_deallocate(); + } + + _LIBCPP_INLINE_VISIBILITY + _Rp operator()(_ArgTypes&&... __args) const { + if (__f_ == 0) + __throw_bad_function_call(); + return (*__f_)(_VSTD::forward<_ArgTypes>(__args)...); + } + + _LIBCPP_INLINE_VISIBILITY + void swap(__vfunc_value& __f) _NOEXCEPT { + if (&__f == this) + return; + if ((void *)__f_ == &__buf_ && (void *)__f.__f_ == &__f.__buf_) + { + typename aligned_storage::type __tempbuf; + __vfunc* __t = __as_base(&__tempbuf); + __f_->__clone(__t); + __f_->destroy(); + __f_ = 0; + __f.__f_->__clone(__as_base(&__buf_)); + __f.__f_->destroy(); + __f.__f_ = 0; + __f_ = __as_base(&__buf_); + __t->__clone(__as_base(&__f.__buf_)); + __t->destroy(); + __f.__f_ = __as_base(&__f.__buf_); + } + else if ((void *)__f_ == &__buf_) + { + __f_->__clone(__as_base(&__f.__buf_)); + __f_->destroy(); + __f_ = __f.__f_; + __f.__f_ = __as_base(&__f.__buf_); + } + else if ((void *)__f.__f_ == &__f.__buf_) + { + __f.__f_->__clone(__as_base(&__buf_)); + __f.__f_->destroy(); + __f.__f_ = __f_; + __f_ = __as_base(&__buf_); + } + else + _VSTD::swap(__f_, __f.__f_); + } + + _LIBCPP_INLINE_VISIBILITY + _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT { + return __f_ != 0; + } + + _LIBCPP_INLINE_VISIBILITY + const std::type_info& target_type() const _NOEXCEPT { + if (__f_ == 0) + return typeid(void); + return __f_->target_type(); + } + + template + _LIBCPP_INLINE_VISIBILITY + const _Tp* target() const _NOEXCEPT { + if (__f_ == 0) + return 0; + return (const _Tp*)__f_->target(typeid(_Tp)); + } +}; + +template class __pfunc; + +template +class __pfunc<_Rp(_ArgTypes...)> { + __storage __buf_; + + // The invoker that calls the value stored in __buf_. + typedef __function::__invoker<_Rp(_ArgTypes...)> __invoker; + __invoker __invoker_; + + // The policy that describes how to move / copy / destroy __buf_. Never + // null, even if the function is empty. + const __policy* __policy_; + +public: + _LIBCPP_INLINE_VISIBILITY + __pfunc() : __policy_(__policy::__create_empty()) {} + + template + _LIBCPP_INLINE_VISIBILITY + __pfunc(_Fp&& __f, const _Alloc& __a) + : __policy_(__policy::__create_empty()) + { + typedef __function::__func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun; + typedef allocator_traits<_Alloc> __alloc_traits; + typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type _FunAlloc; + + if (__function::__not_null(__f)) + { + __invoker_ = __invoker::template __create<_Fun>(); + __policy_ = __policy::__create<_Fun>(); + + _FunAlloc __af(__a); + if (__use_small_storage<_Fun>()) { + ::new ((void*)&__buf_.__small) _Fun(_VSTD::move(__f), _Alloc(__af)); + } + else + { + typedef __allocator_destructor<_FunAlloc> _Dp; + unique_ptr<_Fun, _Dp> __hold(__af.allocate(1), _Dp(__af, 1)); + ::new ((void*)__hold.get()) _Fun(_VSTD::move(__f), _Alloc(__af)); + __buf_.__large = __hold.release(); + } + } + } + + _LIBCPP_INLINE_VISIBILITY + __pfunc(const __pfunc& __f) + : __buf_(__f.__buf_), + __invoker_(__f.__invoker_), + __policy_(__f.__policy_) { + if (__policy_->__clone) + __buf_.__large = __policy_->__clone(__f.__buf_.__large); + } + + _LIBCPP_INLINE_VISIBILITY + __pfunc(__pfunc&& __f) + : __buf_(__f.__buf_), + __invoker_(__f.__invoker_), + __policy_(__f.__policy_) { + if (__policy_->__destroy) { + __f.__policy_ = __policy::__create_empty(); + __f.__invoker_ = __invoker(); + } + } + + _LIBCPP_INLINE_VISIBILITY + ~__pfunc() { + if (__policy_->__destroy) + __policy_->__destroy(__buf_.__large); + } + + _LIBCPP_INLINE_VISIBILITY + void clear() { + const __policy* __p = __policy_; + __policy_ = __policy::__create_empty(); + __invoker_ = __invoker(); + if (__p->__destroy) + __p->__destroy(__buf_.__large); + } + + _LIBCPP_INLINE_VISIBILITY + _Rp operator()(_ArgTypes&&... __args) const { + return __invoker_.__call_(&__buf_, _VSTD::forward<_ArgTypes>(__args)...); + } + + _LIBCPP_INLINE_VISIBILITY + void swap(__pfunc& __f) { + if (&__f == this) + return; + _VSTD::swap(__invoker_, __f.__invoker_); + _VSTD::swap(__policy_, __f.__policy_); + _VSTD::swap(__buf_, __f.__buf_); + } + + _LIBCPP_INLINE_VISIBILITY + _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT { + return !__policy_->__is_null; + } + + _LIBCPP_INLINE_VISIBILITY + const std::type_info& target_type() const _NOEXCEPT { + return *__policy_->type_info; + } + + template + _LIBCPP_INLINE_VISIBILITY + const _Tp* target() const _NOEXCEPT { + if (__policy_->__is_null || typeid(_Tp) != *__policy_->type_info) + return 0; + if (__policy_->__clone) // Out of line storage. + return (const _Tp*)__buf_.__large; + else + return reinterpret_cast(&__buf_.__small); + } +}; + } // __function template @@ -1599,13 +2087,14 @@ : public __function::__maybe_derive_from_unary_function<_Rp(_ArgTypes...)>, public __function::__maybe_derive_from_binary_function<_Rp(_ArgTypes...)> { - typedef __function::__base<_Rp(_ArgTypes...)> __base; - typename aligned_storage<3*sizeof(void*)>::type __buf_; - __base* __f_; +#define _LIBCPP_ABI_OPTIMIZED_FUNCTION +#ifndef _LIBCPP_ABI_OPTIMIZED_FUNCTION + typedef __function::__vfunc_value<_Rp(_ArgTypes...)> __func; +#else + typedef __function::__pfunc<_Rp(_ArgTypes...)> __func; +#endif - _LIBCPP_NO_CFI static __base *__as_base(void *p) { - return reinterpret_cast<__base*>(p); - } + __func __f_; template , function>::value>, @@ -1632,9 +2121,9 @@ // construct/copy/destroy: _LIBCPP_INLINE_VISIBILITY - function() _NOEXCEPT : __f_(0) {} + function() _NOEXCEPT { } _LIBCPP_INLINE_VISIBILITY - function(nullptr_t) _NOEXCEPT : __f_(0) {} + function(nullptr_t) _NOEXCEPT {} function(const function&); function(function&&) _NOEXCEPT; template> @@ -1643,10 +2132,10 @@ #if _LIBCPP_STD_VER <= 14 template _LIBCPP_INLINE_VISIBILITY - function(allocator_arg_t, const _Alloc&) _NOEXCEPT : __f_(0) {} + function(allocator_arg_t, const _Alloc&) _NOEXCEPT {} template _LIBCPP_INLINE_VISIBILITY - function(allocator_arg_t, const _Alloc&, nullptr_t) _NOEXCEPT : __f_(0) {} + function(allocator_arg_t, const _Alloc&, nullptr_t) _NOEXCEPT {} template function(allocator_arg_t, const _Alloc&, const function&); template @@ -1675,7 +2164,7 @@ // function capacity: _LIBCPP_INLINE_VISIBILITY - _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT {return __f_;} + _LIBCPP_EXPLICIT operator bool() const _NOEXCEPT {return (bool)(__f_);} // deleted overloads close possible hole in the type system template @@ -1695,125 +2184,39 @@ }; template -function<_Rp(_ArgTypes...)>::function(const function& __f) -{ - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - __f_ = __f.__f_->__clone(); +function<_Rp(_ArgTypes...)>::function(const function& __f) : __f_(__f.__f_) { } #if _LIBCPP_STD_VER <= 14 template template function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc&, - const function& __f) -{ - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - __f_ = __f.__f_->__clone(); -} + const function& __f) : __f_(__f.__f_) {} #endif -template +template function<_Rp(_ArgTypes...)>::function(function&& __f) _NOEXCEPT -{ - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - { - __f_ = __f.__f_; - __f.__f_ = 0; - } -} + : __f_(_VSTD::move(__f.__f_)) {} #if _LIBCPP_STD_VER <= 14 template template function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc&, - function&& __f) -{ - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - { - __f_ = __f.__f_; - __f.__f_ = 0; - } -} + function&& __f) + : __f_(_VSTD::move(__f.__f_)) {} #endif -template +template template function<_Rp(_ArgTypes...)>::function(_Fp __f) - : __f_(0) -{ - if (__function::__not_null(__f)) - { - typedef __function::__func<_Fp, allocator<_Fp>, _Rp(_ArgTypes...)> _FF; - if (sizeof(_FF) <= sizeof(__buf_) && is_nothrow_copy_constructible<_Fp>::value) - { - __f_ = ::new((void*)&__buf_) _FF(_VSTD::move(__f)); - } - else - { - typedef allocator<_FF> _Ap; - _Ap __a; - typedef __allocator_destructor<_Ap> _Dp; - unique_ptr<__base, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); - ::new (__hold.get()) _FF(_VSTD::move(__f), allocator<_Fp>(__a)); - __f_ = __hold.release(); - } - } -} + : __f_(_VSTD::move(__f), allocator<_Fp>()) {} #if _LIBCPP_STD_VER <= 14 -template +template template -function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc& __a0, _Fp __f) - : __f_(0) -{ - typedef allocator_traits<_Alloc> __alloc_traits; - if (__function::__not_null(__f)) - { - typedef __function::__func<_Fp, _Alloc, _Rp(_ArgTypes...)> _FF; - typedef typename __rebind_alloc_helper<__alloc_traits, _FF>::type _Ap; - _Ap __a(__a0); - if (sizeof(_FF) <= sizeof(__buf_) && - is_nothrow_copy_constructible<_Fp>::value && is_nothrow_copy_constructible<_Ap>::value) - { - __f_ = ::new((void*)&__buf_) _FF(_VSTD::move(__f), _Alloc(__a)); - } - else - { - typedef __allocator_destructor<_Ap> _Dp; - unique_ptr<__base, _Dp> __hold(__a.allocate(1), _Dp(__a, 1)); - ::new (__hold.get()) _FF(_VSTD::move(__f), _Alloc(__a)); - __f_ = __hold.release(); - } - } -} +function<_Rp(_ArgTypes...)>::function(allocator_arg_t, const _Alloc& __a, + _Fp __f) + : __f_(_VSTD::move(__f), __a) {} #endif template @@ -1828,18 +2231,9 @@ function<_Rp(_ArgTypes...)>& function<_Rp(_ArgTypes...)>::operator=(function&& __f) _NOEXCEPT { - *this = nullptr; - if (__f.__f_ == 0) - __f_ = 0; - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f_ = __as_base(&__buf_); - __f.__f_->__clone(__f_); - } - else - { - __f_ = __f.__f_; - __f.__f_ = 0; + if (&__f != this) { + *this = nullptr; + function(_VSTD::move(__f)).swap(*this); } return *this; } @@ -1848,12 +2242,7 @@ function<_Rp(_ArgTypes...)>& function<_Rp(_ArgTypes...)>::operator=(nullptr_t) _NOEXCEPT { - __base* __t = __f_; - __f_ = 0; - if ((void *)__t == &__buf_) - __t->destroy(); - else if (__t) - __t->destroy_deallocate(); + __f_.clear(); return *this; } @@ -1869,58 +2258,20 @@ template function<_Rp(_ArgTypes...)>::~function() { - if ((void *)__f_ == &__buf_) - __f_->destroy(); - else if (__f_) - __f_->destroy_deallocate(); } template void function<_Rp(_ArgTypes...)>::swap(function& __f) _NOEXCEPT { - if (_VSTD::addressof(__f) == this) - return; - if ((void *)__f_ == &__buf_ && (void *)__f.__f_ == &__f.__buf_) - { - typename aligned_storage::type __tempbuf; - __base* __t = __as_base(&__tempbuf); - __f_->__clone(__t); - __f_->destroy(); - __f_ = 0; - __f.__f_->__clone(__as_base(&__buf_)); - __f.__f_->destroy(); - __f.__f_ = 0; - __f_ = __as_base(&__buf_); - __t->__clone(__as_base(&__f.__buf_)); - __t->destroy(); - __f.__f_ = __as_base(&__f.__buf_); - } - else if ((void *)__f_ == &__buf_) - { - __f_->__clone(__as_base(&__f.__buf_)); - __f_->destroy(); - __f_ = __f.__f_; - __f.__f_ = __as_base(&__f.__buf_); - } - else if ((void *)__f.__f_ == &__f.__buf_) - { - __f.__f_->__clone(__as_base(&__buf_)); - __f.__f_->destroy(); - __f.__f_ = __f_; - __f_ = __as_base(&__buf_); - } - else - _VSTD::swap(__f_, __f.__f_); + __f_.swap(__f.__f_); } template _Rp function<_Rp(_ArgTypes...)>::operator()(_ArgTypes... __arg) const { - if (__f_ == 0) - __throw_bad_function_call(); - return (*__f_)(_VSTD::forward<_ArgTypes>(__arg)...); + return __f_(_VSTD::forward<_ArgTypes>(__arg)...); } #ifndef _LIBCPP_NO_RTTI @@ -1929,9 +2280,7 @@ const std::type_info& function<_Rp(_ArgTypes...)>::target_type() const _NOEXCEPT { - if (__f_ == 0) - return typeid(void); - return __f_->target_type(); + return __f_.target_type(); } template @@ -1939,9 +2288,7 @@ _Tp* function<_Rp(_ArgTypes...)>::target() _NOEXCEPT { - if (__f_ == 0) - return nullptr; - return (_Tp*) const_cast(__f_->target(typeid(_Tp))); + return (_Tp*)(__f_.template target<_Tp>()); } template @@ -1949,9 +2296,7 @@ const _Tp* function<_Rp(_ArgTypes...)>::target() const _NOEXCEPT { - if (__f_ == 0) - return nullptr; - return (const _Tp*)__f_->target(typeid(_Tp)); + return __f_.template target<_Tp>(); } #endif // _LIBCPP_NO_RTTI Index: test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_F.pass.cpp =================================================================== --- test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_F.pass.cpp +++ test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/alloc_F.pass.cpp @@ -65,7 +65,8 @@ FuncType* target = &FreeFunction; assert(globalMemCounter.checkOutstandingNewEq(0)); std::function f2(std::allocator_arg, alloc, target); - assert(globalMemCounter.checkOutstandingNewEq(0)); + assert(globalMemCounter.checkOutstandingNewEq(0) || + globalMemCounter.checkOutstandingNewEq(1)); assert(f2.template target()); assert(*f2.template target() == target); assert(f2.template target() == 0); @@ -82,7 +83,8 @@ TargetType target = &MemFunClass::foo; assert(globalMemCounter.checkOutstandingNewEq(0)); std::function f2(std::allocator_arg, alloc, target); - assert(globalMemCounter.checkOutstandingNewEq(0)); + assert(globalMemCounter.checkOutstandingNewEq(0) || + globalMemCounter.checkOutstandingNewEq(1)); assert(f2.template target()); assert(*f2.template target() == target); assert(f2.template target() == 0); Index: test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/copy_move.pass.cpp =================================================================== --- test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/copy_move.pass.cpp +++ test/std/utilities/function.objects/func.wrap/func.wrap.func/func.wrap.func.con/copy_move.pass.cpp @@ -141,7 +141,6 @@ assert(A::count == 1); assert(f2.target() == nullptr); assert(f2.target()); - LIBCPP_ASSERT(f.target()); // f is unchanged because the target is small } { // Test that moving a function constructed from a function pointer @@ -159,7 +158,6 @@ std::function f2(std::move(f)); assert(f2.target() == nullptr); assert(f2.target()); - LIBCPP_ASSERT(f.target()); // f is unchanged because the target is small } #endif // TEST_STD_VER >= 11 }