diff --git a/llvm/include/llvm/ADT/bit.h b/llvm/include/llvm/ADT/bit.h --- a/llvm/include/llvm/ADT/bit.h +++ b/llvm/include/llvm/ADT/bit.h @@ -16,6 +16,7 @@ #include #include +#include #include namespace llvm { @@ -40,6 +41,181 @@ return (Value != 0) && ((Value & (Value - 1)) == 0); } +#ifdef _MSC_VER +// Declare these intrinsics manually rather including intrin.h. It's very +// expensive, and bit.h is popular via MathExtras.h. +// #include +extern "C" { +unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask); +unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask); +unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask); +unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask); +} +#endif + +namespace detail { +template struct TrailingZerosCounter { + static unsigned count(T Val) { + if (!Val) + return std::numeric_limits::digits; + if (Val & 0x1) + return 0; + + // Bisection method. + unsigned ZeroBits = 0; + T Shift = std::numeric_limits::digits >> 1; + T Mask = std::numeric_limits::max() >> Shift; + while (Shift) { + if ((Val & Mask) == 0) { + Val >>= Shift; + ZeroBits |= Shift; + } + Shift >>= 1; + Mask >>= Shift; + } + return ZeroBits; + } +}; + +#if defined(__GNUC__) || defined(_MSC_VER) +template struct TrailingZerosCounter { + static unsigned count(T Val) { + if (Val == 0) + return 32; + +#if __has_builtin(__builtin_ctz) || defined(__GNUC__) + return __builtin_ctz(Val); +#elif defined(_MSC_VER) + unsigned long Index; + _BitScanForward(&Index, Val); + return Index; +#endif + } +}; + +#if !defined(_MSC_VER) || defined(_M_X64) +template struct TrailingZerosCounter { + static unsigned count(T Val) { + if (Val == 0) + return 64; + +#if __has_builtin(__builtin_ctzll) || defined(__GNUC__) + return __builtin_ctzll(Val); +#elif defined(_MSC_VER) + unsigned long Index; + _BitScanForward64(&Index, Val); + return Index; +#endif + } +}; +#endif +#endif +} // namespace detail + +/// Count number of 0's from the least significant bit to the most +/// stopping at the first 1. +/// +/// Only unsigned integral types are allowed. +/// +/// Returns std::numeric_limits::digits on an input of 0. +template int countr_zero(T Val) { + static_assert(std::is_unsigned_v, + "Only unsigned integral types are allowed."); + return llvm::detail::TrailingZerosCounter::count(Val); +} + +namespace detail { +template struct LeadingZerosCounter { + static unsigned count(T Val) { + if (!Val) + return std::numeric_limits::digits; + + // Bisection method. + unsigned ZeroBits = 0; + for (T Shift = std::numeric_limits::digits >> 1; Shift; Shift >>= 1) { + T Tmp = Val >> Shift; + if (Tmp) + Val = Tmp; + else + ZeroBits |= Shift; + } + return ZeroBits; + } +}; + +#if defined(__GNUC__) || defined(_MSC_VER) +template struct LeadingZerosCounter { + static unsigned count(T Val) { + if (Val == 0) + return 32; + +#if __has_builtin(__builtin_clz) || defined(__GNUC__) + return __builtin_clz(Val); +#elif defined(_MSC_VER) + unsigned long Index; + _BitScanReverse(&Index, Val); + return Index ^ 31; +#endif + } +}; + +#if !defined(_MSC_VER) || defined(_M_X64) +template struct LeadingZerosCounter { + static unsigned count(T Val) { + if (Val == 0) + return 64; + +#if __has_builtin(__builtin_clzll) || defined(__GNUC__) + return __builtin_clzll(Val); +#elif defined(_MSC_VER) + unsigned long Index; + _BitScanReverse64(&Index, Val); + return Index ^ 63; +#endif + } +}; +#endif +#endif +} // namespace detail + +/// Count number of 0's from the most significant bit to the least +/// stopping at the first 1. +/// +/// Only unsigned integral types are allowed. +/// +/// Returns std::numeric_limits::digits on an input of 0. +template int countl_zero(T Val) { + static_assert(std::is_unsigned_v, + "Only unsigned integral types are allowed."); + return llvm::detail::LeadingZerosCounter::count(Val); +} + +/// Count the number of ones from the most significant bit to the first +/// zero bit. +/// +/// Ex. countl_one(0xFF0FFF00) == 8. +/// Only unsigned integral types are allowed. +/// +/// Returns std::numeric_limits::digits on an input of all ones. +template int countl_one(T Value) { + static_assert(std::is_unsigned_v, + "Only unsigned integral types are allowed."); + return llvm::countl_zero(~Value); +} + +/// Count the number of ones from the least significant bit to the first +/// zero bit. +/// +/// Ex. countr_one(0x00FF00FF) == 8. +/// Only unsigned integral types are allowed. +/// +/// Returns std::numeric_limits::digits on an input of all ones. +template int countr_one(T Value) { + static_assert(std::is_unsigned_v, + "Only unsigned integral types are allowed."); + return llvm::countr_zero(~Value); +} + namespace detail { template struct PopulationCounter { static int count(T Value) { diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h --- a/llvm/include/llvm/Support/MathExtras.h +++ b/llvm/include/llvm/Support/MathExtras.h @@ -14,7 +14,6 @@ #define LLVM_SUPPORT_MATHEXTRAS_H #include "llvm/ADT/bit.h" -#include "llvm/Support/Compiler.h" #include #include #include @@ -22,18 +21,6 @@ #include #include -#ifdef _MSC_VER -// Declare these intrinsics manually rather including intrin.h. It's very -// expensive, and MathExtras.h is popular. -// #include -extern "C" { -unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask); -unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask); -unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask); -unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask); -} -#endif - namespace llvm { /// The behavior an operation has on an input of 0. @@ -80,65 +67,6 @@ phif = 1.61803399F; // (0x1.9e377aP+0) https://oeis.org/A001622 } // namespace numbers -namespace detail { -template struct TrailingZerosCounter { - static unsigned count(T Val) { - if (!Val) - return std::numeric_limits::digits; - if (Val & 0x1) - return 0; - - // Bisection method. - unsigned ZeroBits = 0; - T Shift = std::numeric_limits::digits >> 1; - T Mask = std::numeric_limits::max() >> Shift; - while (Shift) { - if ((Val & Mask) == 0) { - Val >>= Shift; - ZeroBits |= Shift; - } - Shift >>= 1; - Mask >>= Shift; - } - return ZeroBits; - } -}; - -#if defined(__GNUC__) || defined(_MSC_VER) -template struct TrailingZerosCounter { - static unsigned count(T Val) { - if (Val == 0) - return 32; - -#if __has_builtin(__builtin_ctz) || defined(__GNUC__) - return __builtin_ctz(Val); -#elif defined(_MSC_VER) - unsigned long Index; - _BitScanForward(&Index, Val); - return Index; -#endif - } -}; - -#if !defined(_MSC_VER) || defined(_M_X64) -template struct TrailingZerosCounter { - static unsigned count(T Val) { - if (Val == 0) - return 64; - -#if __has_builtin(__builtin_ctzll) || defined(__GNUC__) - return __builtin_ctzll(Val); -#elif defined(_MSC_VER) - unsigned long Index; - _BitScanForward64(&Index, Val); - return Index; -#endif - } -}; -#endif -#endif -} // namespace detail - /// Count number of 0's from the least significant bit to the most /// stopping at the first 1. /// @@ -148,62 +76,8 @@ template unsigned countTrailingZeros(T Val) { static_assert(std::is_unsigned_v, "Only unsigned integral types are allowed."); - return llvm::detail::TrailingZerosCounter::count(Val); -} - -namespace detail { -template struct LeadingZerosCounter { - static unsigned count(T Val) { - if (!Val) - return std::numeric_limits::digits; - - // Bisection method. - unsigned ZeroBits = 0; - for (T Shift = std::numeric_limits::digits >> 1; Shift; Shift >>= 1) { - T Tmp = Val >> Shift; - if (Tmp) - Val = Tmp; - else - ZeroBits |= Shift; - } - return ZeroBits; - } -}; - -#if defined(__GNUC__) || defined(_MSC_VER) -template struct LeadingZerosCounter { - static unsigned count(T Val) { - if (Val == 0) - return 32; - -#if __has_builtin(__builtin_clz) || defined(__GNUC__) - return __builtin_clz(Val); -#elif defined(_MSC_VER) - unsigned long Index; - _BitScanReverse(&Index, Val); - return Index ^ 31; -#endif - } -}; - -#if !defined(_MSC_VER) || defined(_M_X64) -template struct LeadingZerosCounter { - static unsigned count(T Val) { - if (Val == 0) - return 64; - -#if __has_builtin(__builtin_clzll) || defined(__GNUC__) - return __builtin_clzll(Val); -#elif defined(_MSC_VER) - unsigned long Index; - _BitScanReverse64(&Index, Val); - return Index ^ 63; -#endif - } -}; -#endif -#endif -} // namespace detail + return llvm::countr_zero(Val); +} /// Count number of 0's from the most significant bit to the least /// stopping at the first 1. @@ -214,7 +88,7 @@ template unsigned countLeadingZeros(T Val) { static_assert(std::is_unsigned_v, "Only unsigned integral types are allowed."); - return llvm::detail::LeadingZerosCounter::count(Val); + return llvm::countl_zero(Val); } /// Get the index of the first set bit starting from the least @@ -465,7 +339,7 @@ template unsigned countLeadingOnes(T Value) { static_assert(std::is_unsigned_v, "Only unsigned integral types are allowed."); - return countLeadingZeros(~Value); + return llvm::countl_one(Value); } /// Count the number of ones from the least significant bit to the first @@ -478,7 +352,7 @@ template unsigned countTrailingOnes(T Value) { static_assert(std::is_unsigned_v, "Only unsigned integral types are allowed."); - return countTrailingZeros(~Value); + return llvm::countr_one(Value); } /// Count the number of set bits in a value. diff --git a/llvm/unittests/ADT/BitTest.cpp b/llvm/unittests/ADT/BitTest.cpp --- a/llvm/unittests/ADT/BitTest.cpp +++ b/llvm/unittests/ADT/BitTest.cpp @@ -48,6 +48,102 @@ EXPECT_TRUE(llvm::has_single_bit(static_cast(kValueS16))); } +TEST(BitTest, CountlZero) { + uint8_t Z8 = 0; + uint16_t Z16 = 0; + uint32_t Z32 = 0; + uint64_t Z64 = 0; + EXPECT_EQ(8, llvm::countl_zero(Z8)); + EXPECT_EQ(16, llvm::countl_zero(Z16)); + EXPECT_EQ(32, llvm::countl_zero(Z32)); + EXPECT_EQ(64, llvm::countl_zero(Z64)); + + uint8_t NZ8 = 42; + uint16_t NZ16 = 42; + uint32_t NZ32 = 42; + uint64_t NZ64 = 42; + EXPECT_EQ(2, llvm::countl_zero(NZ8)); + EXPECT_EQ(10, llvm::countl_zero(NZ16)); + EXPECT_EQ(26, llvm::countl_zero(NZ32)); + EXPECT_EQ(58, llvm::countl_zero(NZ64)); + + EXPECT_EQ(8, llvm::countl_zero(0x00F000FFu)); + EXPECT_EQ(8, llvm::countl_zero(0x00F12345u)); + for (unsigned i = 0; i <= 30; ++i) { + EXPECT_EQ(int(31 - i), llvm::countl_zero(1u << i)); + } + + EXPECT_EQ(8, llvm::countl_zero(0x00F1234500F12345ULL)); + EXPECT_EQ(1, llvm::countl_zero(1ULL << 62)); + for (unsigned i = 0; i <= 62; ++i) { + EXPECT_EQ(int(63 - i), llvm::countl_zero(1ULL << i)); + } +} + +TEST(BitTest, CountrZero) { + uint8_t Z8 = 0; + uint16_t Z16 = 0; + uint32_t Z32 = 0; + uint64_t Z64 = 0; + EXPECT_EQ(8, llvm::countr_zero(Z8)); + EXPECT_EQ(16, llvm::countr_zero(Z16)); + EXPECT_EQ(32, llvm::countr_zero(Z32)); + EXPECT_EQ(64, llvm::countr_zero(Z64)); + + uint8_t NZ8 = 42; + uint16_t NZ16 = 42; + uint32_t NZ32 = 42; + uint64_t NZ64 = 42; + EXPECT_EQ(1, llvm::countr_zero(NZ8)); + EXPECT_EQ(1, llvm::countr_zero(NZ16)); + EXPECT_EQ(1, llvm::countr_zero(NZ32)); + EXPECT_EQ(1, llvm::countr_zero(NZ64)); +} + +TEST(BitTest, CountlOne) { + for (int i = 30; i >= 0; --i) { + // Start with all ones and unset some bit. + EXPECT_EQ(31 - i, llvm::countl_one(0xFFFFFFFF ^ (1 << i))); + } + for (int i = 62; i >= 0; --i) { + // Start with all ones and unset some bit. + EXPECT_EQ(63 - i, llvm::countl_one(0xFFFFFFFFFFFFFFFFULL ^ (1LL << i))); + } + for (int i = 30; i >= 0; --i) { + // Start with all ones and unset some bit. + EXPECT_EQ(31 - i, llvm::countl_one(0xFFFFFFFF ^ (1 << i))); + } +} + +TEST(BitTest, CountrOne) { + uint8_t AllOnes8 = ~(uint8_t)0; + uint16_t AllOnes16 = ~(uint16_t)0; + uint32_t AllOnes32 = ~(uint32_t)0; + uint64_t AllOnes64 = ~(uint64_t)0; + EXPECT_EQ(8, llvm::countr_one(AllOnes8)); + EXPECT_EQ(16, llvm::countr_one(AllOnes16)); + EXPECT_EQ(32, llvm::countr_one(AllOnes32)); + EXPECT_EQ(64, llvm::countr_one(AllOnes64)); + + uint8_t X8 = 6; + uint16_t X16 = 6; + uint32_t X32 = 6; + uint64_t X64 = 6; + EXPECT_EQ(0, llvm::countr_one(X8)); + EXPECT_EQ(0, llvm::countr_one(X16)); + EXPECT_EQ(0, llvm::countr_one(X32)); + EXPECT_EQ(0, llvm::countr_one(X64)); + + uint8_t Y8 = 23; + uint16_t Y16 = 23; + uint32_t Y32 = 23; + uint64_t Y64 = 23; + EXPECT_EQ(3, llvm::countr_one(Y8)); + EXPECT_EQ(3, llvm::countr_one(Y16)); + EXPECT_EQ(3, llvm::countr_one(Y32)); + EXPECT_EQ(3, llvm::countr_one(Y64)); +} + TEST(BitTest, PopCount) { EXPECT_EQ(0, llvm::popcount(0U)); EXPECT_EQ(0, llvm::popcount(0ULL));