diff --git a/llvm/include/llvm/ADT/bit.h b/llvm/include/llvm/ADT/bit.h --- a/llvm/include/llvm/ADT/bit.h +++ b/llvm/include/llvm/ADT/bit.h @@ -16,6 +16,7 @@ #include #include +#include #include namespace llvm { @@ -40,6 +41,181 @@ return (Value != 0) && ((Value & (Value - 1)) == 0); } +#ifdef _MSC_VER +// Declare these intrinsics manually rather including intrin.h. It's very +// expensive, and MathExtras.h is popular. +// #include +extern "C" { +unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask); +unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask); +unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask); +unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask); +} +#endif + +namespace detail { +template struct TrailingZerosCounter { + static unsigned count(T Val) { + if (!Val) + return std::numeric_limits::digits; + if (Val & 0x1) + return 0; + + // Bisection method. + unsigned ZeroBits = 0; + T Shift = std::numeric_limits::digits >> 1; + T Mask = std::numeric_limits::max() >> Shift; + while (Shift) { + if ((Val & Mask) == 0) { + Val >>= Shift; + ZeroBits |= Shift; + } + Shift >>= 1; + Mask >>= Shift; + } + return ZeroBits; + } +}; + +#if defined(__GNUC__) || defined(_MSC_VER) +template struct TrailingZerosCounter { + static unsigned count(T Val) { + if (Val == 0) + return 32; + +#if __has_builtin(__builtin_ctz) || defined(__GNUC__) + return __builtin_ctz(Val); +#elif defined(_MSC_VER) + unsigned long Index; + _BitScanForward(&Index, Val); + return Index; +#endif + } +}; + +#if !defined(_MSC_VER) || defined(_M_X64) +template struct TrailingZerosCounter { + static unsigned count(T Val) { + if (Val == 0) + return 64; + +#if __has_builtin(__builtin_ctzll) || defined(__GNUC__) + return __builtin_ctzll(Val); +#elif defined(_MSC_VER) + unsigned long Index; + _BitScanForward64(&Index, Val); + return Index; +#endif + } +}; +#endif +#endif +} // namespace detail + +/// Count number of 0's from the least significant bit to the most +/// stopping at the first 1. +/// +/// Only unsigned integral types are allowed. +/// +/// Returns std::numeric_limits::digits on an input of 0. +template int countr_zero(T Val) { + static_assert(std::is_unsigned_v, + "Only unsigned integral types are allowed."); + return llvm::detail::TrailingZerosCounter::count(Val); +} + +namespace detail { +template struct LeadingZerosCounter { + static unsigned count(T Val) { + if (!Val) + return std::numeric_limits::digits; + + // Bisection method. + unsigned ZeroBits = 0; + for (T Shift = std::numeric_limits::digits >> 1; Shift; Shift >>= 1) { + T Tmp = Val >> Shift; + if (Tmp) + Val = Tmp; + else + ZeroBits |= Shift; + } + return ZeroBits; + } +}; + +#if defined(__GNUC__) || defined(_MSC_VER) +template struct LeadingZerosCounter { + static unsigned count(T Val) { + if (Val == 0) + return 32; + +#if __has_builtin(__builtin_clz) || defined(__GNUC__) + return __builtin_clz(Val); +#elif defined(_MSC_VER) + unsigned long Index; + _BitScanReverse(&Index, Val); + return Index ^ 31; +#endif + } +}; + +#if !defined(_MSC_VER) || defined(_M_X64) +template struct LeadingZerosCounter { + static unsigned count(T Val) { + if (Val == 0) + return 64; + +#if __has_builtin(__builtin_clzll) || defined(__GNUC__) + return __builtin_clzll(Val); +#elif defined(_MSC_VER) + unsigned long Index; + _BitScanReverse64(&Index, Val); + return Index ^ 63; +#endif + } +}; +#endif +#endif +} // namespace detail + +/// Count number of 0's from the most significant bit to the least +/// stopping at the first 1. +/// +/// Only unsigned integral types are allowed. +/// +/// Returns std::numeric_limits::digits on an input of 0. +template int countl_zero(T Val) { + static_assert(std::is_unsigned_v, + "Only unsigned integral types are allowed."); + return llvm::detail::LeadingZerosCounter::count(Val); +} + +/// Count the number of ones from the most significant bit to the first +/// zero bit. +/// +/// Ex. countl_one(0xFF0FFF00) == 8. +/// Only unsigned integral types are allowed. +/// +/// Returns std::numeric_limits::digits on an input of all ones. +template int countl_one(T Value) { + static_assert(std::is_unsigned_v, + "Only unsigned integral types are allowed."); + return llvm::countl_zero(~Value); +} + +/// Count the number of ones from the least significant bit to the first +/// zero bit. +/// +/// Ex. countr_one(0x00FF00FF) == 8. +/// Only unsigned integral types are allowed. +/// +/// Returns std::numeric_limits::digits on an input of all ones. +template int countr_one(T Value) { + static_assert(std::is_unsigned_v, + "Only unsigned integral types are allowed."); + return llvm::countr_zero(~Value); +} + namespace detail { template struct PopulationCounter { static int count(T Value) { diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h --- a/llvm/include/llvm/Support/MathExtras.h +++ b/llvm/include/llvm/Support/MathExtras.h @@ -80,65 +80,6 @@ phif = 1.61803399F; // (0x1.9e377aP+0) https://oeis.org/A001622 } // namespace numbers -namespace detail { -template struct TrailingZerosCounter { - static unsigned count(T Val) { - if (!Val) - return std::numeric_limits::digits; - if (Val & 0x1) - return 0; - - // Bisection method. - unsigned ZeroBits = 0; - T Shift = std::numeric_limits::digits >> 1; - T Mask = std::numeric_limits::max() >> Shift; - while (Shift) { - if ((Val & Mask) == 0) { - Val >>= Shift; - ZeroBits |= Shift; - } - Shift >>= 1; - Mask >>= Shift; - } - return ZeroBits; - } -}; - -#if defined(__GNUC__) || defined(_MSC_VER) -template struct TrailingZerosCounter { - static unsigned count(T Val) { - if (Val == 0) - return 32; - -#if __has_builtin(__builtin_ctz) || defined(__GNUC__) - return __builtin_ctz(Val); -#elif defined(_MSC_VER) - unsigned long Index; - _BitScanForward(&Index, Val); - return Index; -#endif - } -}; - -#if !defined(_MSC_VER) || defined(_M_X64) -template struct TrailingZerosCounter { - static unsigned count(T Val) { - if (Val == 0) - return 64; - -#if __has_builtin(__builtin_ctzll) || defined(__GNUC__) - return __builtin_ctzll(Val); -#elif defined(_MSC_VER) - unsigned long Index; - _BitScanForward64(&Index, Val); - return Index; -#endif - } -}; -#endif -#endif -} // namespace detail - /// Count number of 0's from the least significant bit to the most /// stopping at the first 1. /// @@ -148,62 +89,8 @@ template unsigned countTrailingZeros(T Val) { static_assert(std::is_unsigned_v, "Only unsigned integral types are allowed."); - return llvm::detail::TrailingZerosCounter::count(Val); -} - -namespace detail { -template struct LeadingZerosCounter { - static unsigned count(T Val) { - if (!Val) - return std::numeric_limits::digits; - - // Bisection method. - unsigned ZeroBits = 0; - for (T Shift = std::numeric_limits::digits >> 1; Shift; Shift >>= 1) { - T Tmp = Val >> Shift; - if (Tmp) - Val = Tmp; - else - ZeroBits |= Shift; - } - return ZeroBits; - } -}; - -#if defined(__GNUC__) || defined(_MSC_VER) -template struct LeadingZerosCounter { - static unsigned count(T Val) { - if (Val == 0) - return 32; - -#if __has_builtin(__builtin_clz) || defined(__GNUC__) - return __builtin_clz(Val); -#elif defined(_MSC_VER) - unsigned long Index; - _BitScanReverse(&Index, Val); - return Index ^ 31; -#endif - } -}; - -#if !defined(_MSC_VER) || defined(_M_X64) -template struct LeadingZerosCounter { - static unsigned count(T Val) { - if (Val == 0) - return 64; - -#if __has_builtin(__builtin_clzll) || defined(__GNUC__) - return __builtin_clzll(Val); -#elif defined(_MSC_VER) - unsigned long Index; - _BitScanReverse64(&Index, Val); - return Index ^ 63; -#endif - } -}; -#endif -#endif -} // namespace detail + return llvm::countr_zero(Val); +} /// Count number of 0's from the most significant bit to the least /// stopping at the first 1. @@ -214,7 +101,7 @@ template unsigned countLeadingZeros(T Val) { static_assert(std::is_unsigned_v, "Only unsigned integral types are allowed."); - return llvm::detail::LeadingZerosCounter::count(Val); + return llvm::countl_zero(Val); } /// Get the index of the first set bit starting from the least @@ -465,7 +352,7 @@ template unsigned countLeadingOnes(T Value) { static_assert(std::is_unsigned_v, "Only unsigned integral types are allowed."); - return countLeadingZeros(~Value); + return llvm::countl_one(Value); } /// Count the number of ones from the least significant bit to the first @@ -478,7 +365,7 @@ template unsigned countTrailingOnes(T Value) { static_assert(std::is_unsigned_v, "Only unsigned integral types are allowed."); - return countTrailingZeros(~Value); + return llvm::countr_one(Value); } /// Count the number of set bits in a value.