[libc++] Optimize ranges::find for vector<bool>

[philnik]

Summary

Benchmark results:

----------------------------------------------------------------
Benchmark                                    old             new
----------------------------------------------------------------
bm_vector_bool_ranges_find/1             5.64 ns         6.08 ns
bm_vector_bool_ranges_find/2             16.5 ns         6.03 ns
bm_vector_bool_ranges_find/3             20.3 ns         6.07 ns
bm_vector_bool_ranges_find/4             22.2 ns         6.08 ns
bm_vector_bool_ranges_find/5             23.5 ns         6.05 ns
bm_vector_bool_ranges_find/6             24.4 ns         6.10 ns
bm_vector_bool_ranges_find/7             26.7 ns         6.10 ns
bm_vector_bool_ranges_find/8             25.0 ns         6.08 ns
bm_vector_bool_ranges_find/16            27.9 ns         6.07 ns
bm_vector_bool_ranges_find/64            44.5 ns         5.35 ns
bm_vector_bool_ranges_find/512            243 ns         25.7 ns
bm_vector_bool_ranges_find/4096          1858 ns         35.6 ns
bm_vector_bool_ranges_find/32768        15461 ns         93.5 ns
bm_vector_bool_ranges_find/262144      126462 ns          571 ns
bm_vector_bool_ranges_find/1048576     497736 ns         2272 ns

[philnik]

Benchmarks:

----------------------------------------------------------------
Benchmark                                    old             new
----------------------------------------------------------------
bm_vector_bool_ranges_find/1             5.64 ns         6.08 ns
bm_vector_bool_ranges_find/2             16.5 ns         6.03 ns
bm_vector_bool_ranges_find/3             20.3 ns         6.07 ns
bm_vector_bool_ranges_find/4             22.2 ns         6.08 ns
bm_vector_bool_ranges_find/5             23.5 ns         6.05 ns
bm_vector_bool_ranges_find/6             24.4 ns         6.10 ns
bm_vector_bool_ranges_find/7             26.7 ns         6.10 ns
bm_vector_bool_ranges_find/8             25.0 ns         6.08 ns
bm_vector_bool_ranges_find/16            27.9 ns         6.07 ns
bm_vector_bool_ranges_find/64            44.5 ns         5.35 ns
bm_vector_bool_ranges_find/512            243 ns         25.7 ns
bm_vector_bool_ranges_find/4096          1858 ns         35.6 ns
bm_vector_bool_ranges_find/32768        15461 ns         93.5 ns
bm_vector_bool_ranges_find/262144      126462 ns          571 ns
bm_vector_bool_ranges_find/1048576     497736 ns         2272 ns

[Mordante]

Does this fix the bug you posted? https://github.com/llvm/llvm-project/issues/64038

The benchmarks look great!

I did not look very close at the patch, I really like to see what the CI thinks first.

[philnik]

In D156039#4525704, @Mordante wrote:

Does this fix the bug you posted? https://github.com/llvm/llvm-project/issues/64038

Partially. There are still lots of optimizations missed. I'll upload more patches to fix other algorithms later.

[var-const]

Great benchmarks!

Nitpick: can you also mention std::find in the commit message? When I first read the title, I thought we had some missed optimization opportunity specifically in the Ranges code.

[Mordante]

I like to look at it when the CI passes or at least can apply the patch. The underlying stack is accepted, please rebase when the underlying patches have landed.

[philnik]

Rebased

[philnik]

Rebased

[Mordante]

LGTM with the comments addressed.

libcxx/benchmarks/algorithms/find.bench.cpp

61

please add the benchmarks to the commit message.

libcxx/include/__bit/invert_if.h

24–28

[philnik]

Try to fix CI

[philnik]

Try to fix CI

libcxx/benchmarks/algorithms/find.bench.cpp

for (auto _ : state) {
     benchmark::DoNotOptimize(std::ranges::find(vec1, T('2')));
     vec1[idx] = '1';
   }
 }
 BENCHMARK(bm_ranges_find<char>)->DenseRange(1, 8)->Range(16, 1 << 20);
 BENCHMARK(bm_ranges_find<short>)->DenseRange(1, 8)->Range(16, 1 << 20);
 BENCHMARK(bm_ranges_find<int>)->DenseRange(1, 8)->Range(16, 1 << 20);
 
+static void bm_vector_bool_find(benchmark::State& state) {
+  std::vector<bool> vec1(state.range(), false);
+  std::mt19937_64 rng(std::random_device{}());
+
+  for (auto _ : state) {
+    auto idx  = rng() % vec1.size();
+    vec1[idx] = true;
+    benchmark::DoNotOptimize(vec1);
+    benchmark::DoNotOptimize(std::find(vec1.begin(), vec1.end(), true));
+    vec1[idx] = false;
+  }
+}
+BENCHMARK(bm_vector_bool_find)->DenseRange(1, 8)->Range(16, 1 << 20);
+
+static void bm_vector_bool_ranges_find(benchmark::State& state) {
+  std::vector<bool> vec1(state.range(), false);
+  std::mt19937_64 rng(std::random_device{}());
+
+  for (auto _ : state) {
+    auto idx  = rng() % vec1.size();
+    vec1[idx] = true;
+    benchmark::DoNotOptimize(vec1);
+    benchmark::DoNotOptimize(std::ranges::find(vec1, true));
+    vec1[idx] = false;
+  }
+}
+BENCHMARK(bm_vector_bool_ranges_find)->DenseRange(1, 8)->Range(16, 1 << 20);
+
 BENCHMARK_MAIN();

libcxx/include/CMakeLists.txt

set(files
   __bit/bit_log2.h
   __bit/bit_width.h
   __bit/blsr.h
   __bit/byteswap.h
   __bit/countl.h
   __bit/countr.h
   __bit/endian.h
   __bit/has_single_bit.h
+  __bit/invert_if.h
   __bit/popcount.h
   __bit/rotate.h
   __bit_reference
   __charconv/chars_format.h
   __charconv/from_chars_integral.h
   __charconv/from_chars_result.h
   __charconv/tables.h
   __charconv/to_chars.h
set(files
   __functional/pointer_to_binary_function.h
   __functional/pointer_to_unary_function.h
   __functional/ranges_operations.h
   __functional/reference_wrapper.h
   __functional/unary_function.h
   __functional/unary_negate.h
   __functional/weak_result_type.h
   __fwd/array.h
+  __fwd/bit_reference.h
   __fwd/fstream.h
   __fwd/get.h
   __fwd/hash.h
   __fwd/ios.h
   __fwd/istream.h
   __fwd/mdspan.h
   __fwd/memory_resource.h
   __fwd/ostream.h

libcxx/include/__algorithm/find.h

 // -*- C++ -*-
 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef _LIBCPP___ALGORITHM_FIND_H
 #define _LIBCPP___ALGORITHM_FIND_H
 
+#include <__algorithm/min.h>
 #include <__algorithm/unwrap_iter.h>
+#include <__bit/countr.h>
+#include <__bit/invert_if.h>
 #include <__config>
 #include <__functional/identity.h>
 #include <__functional/invoke.h>
+#include <__fwd/bit_reference.h>
 #include <__string/constexpr_c_functions.h>
 #include <__type_traits/is_same.h>
 
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
 #  include <cwchar>
 #endif
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+// generic implementation
 template <class _Iter, class _Sent, class _Tp, class _Proj>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter
 __find_impl(_Iter __first, _Sent __last, const _Tp& __value, _Proj& __proj) {
   for (; __first != __last; ++__first)
     if (std::__invoke(__proj, *__first) == __value)
       break;
   return __first;
 }
 
+// trivially equality comparable implementations
 template <class _Tp,
           class _Up,
           class _Proj,
           __enable_if_t<__is_identity<_Proj>::value && __libcpp_is_trivially_equality_comparable<_Tp, _Up>::value &&
                             sizeof(_Tp) == 1,
                         int> = 0>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp*
 __find_impl(_Tp* __first, _Tp* __last, const _Up& __value, _Proj&) {

 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp*
 __find_impl(_Tp* __first, _Tp* __last, const _Up& __value, _Proj&) {
   if (auto __ret = std::__constexpr_wmemchr(__first, __value, __last - __first))
     return __ret;
   return __last;
 }
 #endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
 
+// __bit_iterator implementation
+template <bool _ToFind, class _Cp, bool _IsConst>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, _IsConst>
+__find_bool(__bit_iterator<_Cp, _IsConst> __first, typename _Cp::size_type __n) {
+  using _It            = __bit_iterator<_Cp, _IsConst>;
+  using __storage_type = typename _It::__storage_type;
+
+  const int __bits_per_word = _It::__bits_per_word;
+  // do first partial word
+  if (__first.__ctz_ != 0) {
+    __storage_type __clz_f = static_cast<__storage_type>(__bits_per_word - __first.__ctz_);
+    __storage_type __dn    = std::min(__clz_f, __n);
+    __storage_type __m     = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
+    __storage_type __b     = std::__invert_if<!_ToFind>(*__first.__seg_) & __m;
+    if (__b)
+      return _It(__first.__seg_, static_cast<unsigned>(std::__libcpp_ctz(__b)));
+    if (__n == __dn)
+      return __first + __n;
+    __n -= __dn;
+    ++__first.__seg_;
+  }
+  // do middle whole words
+  for (; __n >= __bits_per_word; ++__first.__seg_, __n -= __bits_per_word) {
+    __storage_type __b = std::__invert_if<!_ToFind>(*__first.__seg_);
+    if (__b)
+      return _It(__first.__seg_, static_cast<unsigned>(std::__libcpp_ctz(__b)));
+  }
+  // do last partial word
+  if (__n > 0) {
+    __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
+    __storage_type __b = std::__invert_if<!_ToFind>(*__first.__seg_) & __m;
+    if (__b)
+      return _It(__first.__seg_, static_cast<unsigned>(std::__libcpp_ctz(__b)));
+  }
+  return _It(__first.__seg_, static_cast<unsigned>(__n));
+}
+
+template <class _Cp, bool _IsConst, class _Tp, class _Proj, __enable_if_t<__is_identity<_Proj>::value, int> = 0>
+inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, _IsConst>
+__find_impl(__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, const _Tp& __value, _Proj&) {
+  if (static_cast<bool>(__value))
+    return std::__find_bool<true>(__first, static_cast<typename _Cp::size_type>(__last - __first));
+  return std::__find_bool<false>(__first, static_cast<typename _Cp::size_type>(__last - __first));
+}
+
 template <class _InputIterator, class _Tp>
 _LIBCPP_NODISCARD_EXT inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
 find(_InputIterator __first, _InputIterator __last, const _Tp& __value) {
   __identity __proj;
   return std::__rewrap_iter(
       __first, std::__find_impl(std::__unwrap_iter(__first), std::__unwrap_iter(__last), __value, __proj));
 }
 
 _LIBCPP_END_NAMESPACE_STD
 
+_LIBCPP_POP_MACROS
+
 #endif // _LIBCPP___ALGORITHM_FIND_H

libcxx/include/__bit/invert_if.h

+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___BIT_INVERT_IF_H
+#define _LIBCPP___BIT_INVERT_IF_H
+
+#include <__concepts/arithmetic.h>
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <bool _Invert, class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp __invert_if(_Tp __v) {
+  if (_Invert)
+    return ~__v;
+  return __v;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___BIT_INVERT_IF_H

libcxx/include/__bit_reference

 // -*- C++ -*-
 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef _LIBCPP___BIT_REFERENCE
 #define _LIBCPP___BIT_REFERENCE
 
 #include <__algorithm/copy_n.h>
 #include <__algorithm/fill_n.h>
 #include <__algorithm/min.h>
 #include <__bit/countr.h>
+#include <__bit/invert_if.h>
 #include <__bit/popcount.h>
 #include <__config>
+#include <__fwd/bit_reference.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/construct_at.h>
 #include <__memory/pointer_traits.h>
 #include <__type_traits/conditional.h>
 #include <__utility/swap.h>
 #include <cstring>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
 _LIBCPP_PUSH_MACROS
 #include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Cp, bool _IsConst, typename _Cp::__storage_type = 0>
-class __bit_iterator;
 template <class _Cp>
 class __bit_const_reference;
 
 template <class _Tp>
 struct __has_storage_type {
   static const bool value = false;
 };
 
private:
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __bit_const_reference(
       __storage_pointer __s, __storage_type __m) _NOEXCEPT
       : __seg_(__s),
         __mask_(__m) {}
 
   __bit_const_reference& operator=(const __bit_const_reference&) = delete;
 };
 
-template <bool _Invert, class _Tp>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp __invert_if(_Tp __v) {
-  if (_Invert)
-    return ~__v;
-  return __v;
-}
-
-// find
-
-template <bool _ToFind, class _Cp, bool _IsConst>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, _IsConst>
-__find_bool(__bit_iterator<_Cp, _IsConst> __first, typename _Cp::size_type __n) {
-  using _It            = __bit_iterator<_Cp, _IsConst>;
-  using __storage_type = typename _It::__storage_type;
-
-  const int __bits_per_word = _It::__bits_per_word;
-  // do first partial word
-  if (__first.__ctz_ != 0) {
-    __storage_type __clz_f = static_cast<__storage_type>(__bits_per_word - __first.__ctz_);
-    __storage_type __dn    = std::min(__clz_f, __n);
-    __storage_type __m     = (~__storage_type(0) << __first.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
-    __storage_type __b     = std::__invert_if<!_ToFind>(*__first.__seg_) & __m;
-    if (__b)
-      return _It(__first.__seg_, static_cast<unsigned>(std::__libcpp_ctz(__b)));
-    if (__n == __dn)
-      return __first + __n;
-    __n -= __dn;
-    ++__first.__seg_;
-  }
-  // do middle whole words
-  for (; __n >= __bits_per_word; ++__first.__seg_, __n -= __bits_per_word) {
-    __storage_type __b = std::__invert_if<!_ToFind>(*__first.__seg_);
-    if (__b)
-      return _It(__first.__seg_, static_cast<unsigned>(std::__libcpp_ctz(__b)));
-  }
-  // do last partial word
-  if (__n > 0) {
-    __storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
-    __storage_type __b = std::__invert_if<!_ToFind>(*__first.__seg_) & __m;
-    if (__b)
-      return _It(__first.__seg_, static_cast<unsigned>(std::__libcpp_ctz(__b)));
-  }
-  return _It(__first.__seg_, static_cast<unsigned>(__n));
-}
-
-template <class _Cp, bool _IsConst, class _Tp>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, _IsConst>
-find(__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, const _Tp& __value) {
-  if (static_cast<bool>(__value))
-    return std::__find_bool<true>(__first, static_cast<typename _Cp::size_type>(__last - __first));
-  return std::__find_bool<false>(__first, static_cast<typename _Cp::size_type>(__last - __first));
-}
-
 // count
 
 template <bool _ToCount, class _Cp, bool _IsConst>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 typename __bit_iterator<_Cp, _IsConst>::difference_type
 __count_bool(__bit_iterator<_Cp, _IsConst> __first, typename _Cp::size_type __n) {
   using _It             = __bit_iterator<_Cp, _IsConst>;
   using __storage_type  = typename _It::__storage_type;
   using difference_type = typename _It::difference_type;

libcxx/include/__fwd/bit_reference.h

+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_BIT_REFERENCE_H
+#define _LIBCPP___FWD_BIT_REFERENCE_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Cp, bool _IsConst, typename _Cp::__storage_type = 0>
+class __bit_iterator;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_BIT_REFERENCE_H

libcxx/include/bitset

 
 template <size_t N> struct hash<std::bitset<N>>;
 
 }  // std
 
 */
 
 #include <__algorithm/fill.h>
+#include <__algorithm/find.h>
 #include <__assert> // all public C++ headers provide the assertion handler
 #include <__bit_reference>
 #include <__config>
 #include <__functional/hash.h>
 #include <__functional/unary_function.h>
 #include <__type_traits/is_char_like_type.h>
 #include <climits>
 #include <cstddef>

libcxx/test/libcxx/transitive_includes/cxx03.csv

 bit type_traits
 bit version
 bitset climits
 bitset concepts
 bitset cstddef
 bitset cstdint
 bitset cstdlib
 bitset cstring
+bitset cwchar
 bitset initializer_list
 bitset iosfwd
 bitset limits
 bitset new
 bitset stdexcept
 bitset string
 bitset string_view
 bitset type_traits

libcxx/test/libcxx/transitive_includes/cxx11.csv

 bit type_traits
 bit version
 bitset climits
 bitset concepts
 bitset cstddef
 bitset cstdint
 bitset cstdlib
 bitset cstring
+bitset cwchar
 bitset initializer_list
 bitset iosfwd
 bitset limits
 bitset new
 bitset stdexcept
 bitset string
 bitset string_view
 bitset type_traits

libcxx/test/libcxx/transitive_includes/cxx14.csv

 bit type_traits
 bit version
 bitset climits
 bitset concepts
 bitset cstddef
 bitset cstdint
 bitset cstdlib
 bitset cstring
+bitset cwchar
 bitset initializer_list
 bitset iosfwd
 bitset limits
 bitset new
 bitset stdexcept
 bitset string
 bitset string_view
 bitset type_traits

libcxx/test/libcxx/transitive_includes/cxx17.csv

 bit type_traits
 bit version
 bitset climits
 bitset concepts
 bitset cstddef
 bitset cstdint
 bitset cstdlib
 bitset cstring
+bitset cwchar
 bitset initializer_list
 bitset iosfwd
 bitset limits
 bitset new
 bitset stdexcept
 bitset string
 bitset string_view
 bitset type_traits

libcxx/test/libcxx/transitive_includes/cxx20.csv

 bit type_traits
 bit version
 bitset climits
 bitset concepts
 bitset cstddef
 bitset cstdint
 bitset cstdlib
 bitset cstring
+bitset cwchar
 bitset initializer_list
 bitset iosfwd
 bitset limits
 bitset new
 bitset stdexcept
 bitset string
 bitset string_view
 bitset type_traits

libcxx/test/libcxx/transitive_includes/cxx23.csv

 barrier version
 bit cstdint
 bit limits
 bit version
 bitset climits
 bitset cstddef
 bitset cstdint
 bitset cstring
+bitset cwchar
 bitset initializer_list
 bitset iosfwd
 bitset limits
 bitset new
 bitset stdexcept
 bitset string
 bitset string_view
 bitset version

libcxx/test/libcxx/transitive_includes/cxx26.csv

 barrier version
 bit cstdint
 bit limits
 bit version
 bitset climits
 bitset cstddef
 bitset cstdint
 bitset cstring
+bitset cwchar
 bitset initializer_list
 bitset iosfwd
 bitset limits
 bitset new
 bitset stdexcept
 bitset string
 bitset string_view
 bitset version