Index: llvm/include/llvm/Support/MathExtras.h =================================================================== --- llvm/include/llvm/Support/MathExtras.h +++ llvm/include/llvm/Support/MathExtras.h @@ -27,15 +27,14 @@ #endif #ifdef _MSC_VER -// Declare these intrinsics manually rather including intrin.h. It's very -// expensive, and MathExtras.h is popular. -// #include -extern "C" { -unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask); -unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask); -unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask); -unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask); -} +#include +#pragma intrinsic(_BitScanForward) +#pragma intrinsic(_BitScanReverse) + +#ifdef _M_X64 +#pragma intrinsic(_BitScanForward64) +#pragma intrinsic(_BitScanReverse64) +#endif #endif namespace llvm { @@ -88,7 +87,7 @@ namespace detail { template struct TrailingZerosCounter { - static unsigned count(T Val, ZeroBehavior) { + static constexpr unsigned count(T Val, ZeroBehavior) { if (!Val) return std::numeric_limits::digits; if (Val & 0x1) @@ -110,31 +109,43 @@ } }; -#if defined(__GNUC__) || defined(_MSC_VER) +#if defined(_MSC_VER) || __has_builtin(__builtin_ctz) || defined(__GNUC__) +#if defined(_MSC_VER) template struct TrailingZerosCounter { static unsigned count(T Val, ZeroBehavior ZB) { +#else +template struct TrailingZerosCounter { + static constexpr unsigned count(T Val, ZeroBehavior ZB) { +#endif if (ZB != ZB_Undefined && Val == 0) return 32; #if __has_builtin(__builtin_ctz) || defined(__GNUC__) return __builtin_ctz(Val); -#elif defined(_MSC_VER) +#else // defined(_MSC_VER) unsigned long Index; _BitScanForward(&Index, Val); return Index; #endif } }; +#endif -#if !defined(_MSC_VER) || defined(_M_X64) +#if (defined(_MSC_VER) && defined(_M_X64)) || __has_builtin(__builtin_ctz) || \ + defined(__GNUC__) +#if defined(_MSC_VER) && defined(_M_X64) template struct TrailingZerosCounter { static unsigned count(T Val, ZeroBehavior ZB) { +#else +template struct TrailingZerosCounter { + static constexpr unsigned count(T Val, ZeroBehavior ZB) { +#endif if (ZB != ZB_Undefined && Val == 0) return 64; #if __has_builtin(__builtin_ctzll) || defined(__GNUC__) return __builtin_ctzll(Val); -#elif defined(_MSC_VER) +#else // defined(_MSC_VER) && defined(_M_X64) unsigned long Index; _BitScanForward64(&Index, Val); return Index; @@ -142,7 +153,6 @@ } }; #endif -#endif } // namespace detail /// Count number of 0's from the least significant bit to the most @@ -152,17 +162,25 @@ /// /// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are /// valid arguments. + +#if defined(_MSC_VER) || __has_builtin(__builtin_ctz) || defined(__GNUC__) +#if defined(_MSC_VER) template unsigned countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) { +#else +template +constexpr unsigned countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) { +#endif static_assert(std::numeric_limits::is_integer && !std::numeric_limits::is_signed, "Only unsigned integral types are allowed."); return llvm::detail::TrailingZerosCounter::count(Val, ZB); } +#endif namespace detail { template struct LeadingZerosCounter { - static unsigned count(T Val, ZeroBehavior) { + static constexpr unsigned count(T Val, ZeroBehavior) { if (!Val) return std::numeric_limits::digits; @@ -179,31 +197,43 @@ } }; -#if defined(__GNUC__) || defined(_MSC_VER) +#if defined(_MSC_VER) || __has_builtin(__builtin_clz) || defined(__GNUC__) +#if defined(_MSC_VER) template struct LeadingZerosCounter { static unsigned count(T Val, ZeroBehavior ZB) { +#else +template struct LeadingZerosCounter { + static constexpr unsigned count(T Val, ZeroBehavior ZB) { +#endif if (ZB != ZB_Undefined && Val == 0) return 32; #if __has_builtin(__builtin_clz) || defined(__GNUC__) return __builtin_clz(Val); -#elif defined(_MSC_VER) +#else // defined(_MSC_VER) unsigned long Index; _BitScanReverse(&Index, Val); return Index ^ 31; #endif } }; +#endif -#if !defined(_MSC_VER) || defined(_M_X64) +#if (defined(_MSC_VER) && defined(_M_X64)) || \ + __has_builtin(__builtin_clzll) || defined(__GNUC__) +#if defined(_MSC_VER) && defined(_M_X64) template struct LeadingZerosCounter { static unsigned count(T Val, ZeroBehavior ZB) { +#else +template struct LeadingZerosCounter { + static constexpr unsigned count(T Val, ZeroBehavior ZB) { +#endif if (ZB != ZB_Undefined && Val == 0) return 64; #if __has_builtin(__builtin_clzll) || defined(__GNUC__) return __builtin_clzll(Val); -#elif defined(_MSC_VER) +#else // defined(_MSC_VER) && defined(_M_X64) unsigned long Index; _BitScanReverse64(&Index, Val); return Index ^ 63; @@ -211,7 +241,6 @@ } }; #endif -#endif } // namespace detail /// Count number of 0's from the most significant bit to the least @@ -221,13 +250,21 @@ /// /// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are /// valid arguments. + +#if defined(_MSC_VER) || __has_builtin(__builtin_clz) || defined(__GNUC__) +#if defined(_MSC_VER) template unsigned countLeadingZeros(T Val, ZeroBehavior ZB = ZB_Width) { +#else +template +constexpr unsigned countLeadingZeros(T Val, ZeroBehavior ZB = ZB_Width) { +#endif static_assert(std::numeric_limits::is_integer && !std::numeric_limits::is_signed, "Only unsigned integral types are allowed."); return llvm::detail::LeadingZerosCounter::count(Val, ZB); } +#endif /// Get the index of the first set bit starting from the least /// significant bit. @@ -236,12 +273,21 @@ /// /// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are /// valid arguments. + +#if defined(_MSC_VER) || __has_builtin(__builtin_ctz) || defined(__GNUC__) +#if defined(_MSC_VER) template T findFirstSet(T Val, ZeroBehavior ZB = ZB_Max) { +#else +template +constexpr T findFirstSet(T Val, ZeroBehavior ZB = ZB_Max) { +#endif + if (ZB == ZB_Max && Val == 0) return std::numeric_limits::max(); return countTrailingZeros(Val, ZB_Undefined); } +#endif /// Create a bitmask with the N right-most bits set to 1, and all other /// bits set to 0. Only unsigned types are allowed. @@ -277,7 +323,13 @@ /// /// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are /// valid arguments. + +#if defined(_MSC_VER) || __has_builtin(__builtin_clz) || defined(__GNUC__) +#if defined(_MSC_VER) template T findLastSet(T Val, ZeroBehavior ZB = ZB_Max) { +#else +template constexpr T findLastSet(T Val, ZeroBehavior ZB = ZB_Max) { +#endif if (ZB == ZB_Max && Val == 0) return std::numeric_limits::max(); @@ -286,6 +338,7 @@ return countLeadingZeros(Val, ZB_Undefined) ^ (std::numeric_limits::digits - 1); } +#endif /// Macro compressed bit reversal table for 256 bits. /// @@ -294,15 +347,14 @@ #define R2(n) n, n + 2 * 64, n + 1 * 64, n + 3 * 64 #define R4(n) R2(n), R2(n + 2 * 16), R2(n + 1 * 16), R2(n + 3 * 16) #define R6(n) R4(n), R4(n + 2 * 4), R4(n + 1 * 4), R4(n + 3 * 4) - R6(0), R6(2), R6(1), R6(3) + R6(0), R6(2), R6(1), R6(3) #undef R2 #undef R4 #undef R6 }; /// Reverse the bits in \p Val. -template -T reverseBits(T Val) { +template T reverseBits(T Val) { unsigned char in[sizeof(Val)]; unsigned char out[sizeof(Val)]; std::memcpy(in, &Val, sizeof(Val)); @@ -313,29 +365,25 @@ } #if __has_builtin(__builtin_bitreverse8) -template<> -inline uint8_t reverseBits(uint8_t Val) { +template <> inline uint8_t reverseBits(uint8_t Val) { return __builtin_bitreverse8(Val); } #endif #if __has_builtin(__builtin_bitreverse16) -template<> -inline uint16_t reverseBits(uint16_t Val) { +template <> inline uint16_t reverseBits(uint16_t Val) { return __builtin_bitreverse16(Val); } #endif #if __has_builtin(__builtin_bitreverse32) -template<> -inline uint32_t reverseBits(uint32_t Val) { +template <> inline uint32_t reverseBits(uint32_t Val) { return __builtin_bitreverse32(Val); } #endif #if __has_builtin(__builtin_bitreverse64) -template<> -inline uint64_t reverseBits(uint64_t Val) { +template <> inline uint64_t reverseBits(uint64_t Val) { return __builtin_bitreverse64(Val); } #endif @@ -356,12 +404,13 @@ /// Make a 64-bit integer from a high / low pair of 32-bit integers. constexpr inline uint64_t Make_64(uint32_t High, uint32_t Low) { - return ((uint64_t)High << 32) | (uint64_t)Low; + return (static_cast(High) << 32) | static_cast(Low); } /// Checks if an integer fits into the given bit width. template constexpr inline bool isInt(int64_t x) { - return N >= 64 || (-(INT64_C(1)<<(N-1)) <= x && x < (INT64_C(1)<<(N-1))); + return N >= 64 || + (-(INT64_C(1) << (N - 1)) <= x && x < (INT64_C(1) << (N - 1))); } // Template specializations to get better code for common cases. template <> constexpr inline bool isInt<8>(int64_t x) { @@ -505,13 +554,20 @@ /// /// \param ZB the behavior on an input of all ones. Only ZB_Width and /// ZB_Undefined are valid arguments. +#if defined(_MSC_VER) || __has_builtin(__builtin_clz) || defined(__GNUC__) +#if defined(_MSC_VER) template unsigned countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) { +#else +template +constexpr unsigned countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) { +#endif static_assert(std::numeric_limits::is_integer && !std::numeric_limits::is_signed, "Only unsigned integral types are allowed."); return countLeadingZeros(~Value, ZB); } +#endif /// Count the number of ones from the least significant bit to the first /// zero bit. @@ -521,20 +577,29 @@ /// /// \param ZB the behavior on an input of all ones. Only ZB_Width and /// ZB_Undefined are valid arguments. + +#if defined(_MSC_VER) || __has_builtin(__builtin_ctz) || defined(__GNUC__) +#if defined(_MSC_VER) template unsigned countTrailingOnes(T Value, ZeroBehavior ZB = ZB_Width) { +#else +template +constexpr unsigned countTrailingOnes(T Value, ZeroBehavior ZB = ZB_Width) { +#endif + static_assert(std::numeric_limits::is_integer && !std::numeric_limits::is_signed, "Only unsigned integral types are allowed."); return countTrailingZeros(~Value, ZB); } +#endif namespace detail { template struct PopulationCounter { - static unsigned count(T Value) { + static constexpr unsigned count(T Value) { // Generic version, forward to 32 bits. static_assert(SizeOfT <= 4, "Not implemented!"); -#if defined(__GNUC__) +#if __has_builtin(__builtin_popcount) || defined(__GNUC__) return __builtin_popcount(Value); #else uint32_t v = Value; @@ -546,15 +611,15 @@ }; template struct PopulationCounter { - static unsigned count(T Value) { -#if defined(__GNUC__) + static constexpr unsigned count(T Value) { +#if __has_builtin(__builtin_popcountll) || defined(__GNUC__) return __builtin_popcountll(Value); #else uint64_t v = Value; v = v - ((v >> 1) & 0x5555555555555555ULL); v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; - return unsigned((uint64_t)(v * 0x0101010101010101ULL) >> 56); + return unsigned(static_cast(v * 0x0101010101010101ULL) >> 56); #endif } }; @@ -563,8 +628,7 @@ /// Count the number of set bits in a value. /// Ex. countPopulation(0xF000F000) = 8 /// Returns 0 if the word is zero. -template -inline unsigned countPopulation(T Value) { +template constexpr inline unsigned countPopulation(T Value) { static_assert(std::numeric_limits::is_integer && !std::numeric_limits::is_signed, "Only unsigned integral types are allowed."); @@ -593,32 +657,59 @@ /// Return the floor log base 2 of the specified value, -1 if the value is zero. /// (32 bit edition.) /// Ex. Log2_32(32) == 5, Log2_32(1) == 0, Log2_32(0) == -1, Log2_32(6) == 2 + +#if defined(_MSC_VER) || __has_builtin(__builtin_clz) || defined(__GNUC__) +#if defined(_MSC_VER) inline unsigned Log2_32(uint32_t Value) { +#else +constexpr inline unsigned Log2_32(uint32_t Value) { +#endif return 31 - countLeadingZeros(Value); } +#endif /// Return the floor log base 2 of the specified value, -1 if the value is zero. /// (64 bit edition.) + +#if (defined(_MSC_VER) && defined(_M_X64)) || \ + __has_builtin(__builtin_clzll) || defined(__GNUC__) +#if defined(_MSC_VER) && defined(_M_X64) inline unsigned Log2_64(uint64_t Value) { +#else +constexpr inline unsigned Log2_64(uint64_t Value) { +#endif return 63 - countLeadingZeros(Value); } +#endif /// Return the ceil log base 2 of the specified value, 32 if the value is zero. /// (32 bit edition). /// Ex. Log2_32_Ceil(32) == 5, Log2_32_Ceil(1) == 0, Log2_32_Ceil(6) == 3 +#if defined(_MSC_VER) || __has_builtin(__builtin_clz) || defined(__GNUC__) +#if defined(_MSC_VER) inline unsigned Log2_32_Ceil(uint32_t Value) { +#else +constexpr inline unsigned Log2_32_Ceil(uint32_t Value) { +#endif return 32 - countLeadingZeros(Value - 1); } +#endif /// Return the ceil log base 2 of the specified value, 64 if the value is zero. /// (64 bit edition.) +#if (defined(_MSC_VER) && defined(_M_X64)) || \ + __has_builtin(__builtin_clzll) || defined(__GNUC__) +#if defined(_MSC_VER) && defined(_M_X64) inline unsigned Log2_64_Ceil(uint64_t Value) { +#else +constexpr inline unsigned Log2_64_Ceil(uint64_t Value) { +#endif return 64 - countLeadingZeros(Value - 1); } +#endif /// Return the greatest common divisor of the values using Euclid's algorithm. -template -inline T greatestCommonDivisor(T A, T B) { +template constexpr inline T greatestCommonDivisor(T A, T B) { while (B) { T Tmp = B; B = A % B; @@ -627,7 +718,7 @@ return A; } -inline uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B) { +constexpr inline uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B) { return greatestCommonDivisor(A, B); } @@ -680,7 +771,7 @@ /// Returns the next power of two (in 64-bits) that is strictly greater than A. /// Returns zero on overflow. -inline uint64_t NextPowerOf2(uint64_t A) { +constexpr inline uint64_t NextPowerOf2(uint64_t A) { A |= (A >> 1); A |= (A >> 2); A |= (A >> 4); @@ -692,14 +783,15 @@ /// Returns the power of two which is less than or equal to the given value. /// Essentially, it is a floor operation across the domain of powers of two. -inline uint64_t PowerOf2Floor(uint64_t A) { - if (!A) return 0; +constexpr inline uint64_t PowerOf2Floor(uint64_t A) { + if (!A) + return 0; return 1ull << (63 - countLeadingZeros(A, ZB_Undefined)); } /// Returns the power of two which is greater than or equal to the given value. /// Essentially, it is a ceil operation across the domain of powers of two. -inline uint64_t PowerOf2Ceil(uint64_t A) { +constexpr inline uint64_t PowerOf2Ceil(uint64_t A) { if (!A) return 0; return NextPowerOf2(A - 1); @@ -744,7 +836,8 @@ } /// Returns the integer nearest(Numerator / Denominator). -inline uint64_t divideNearest(uint64_t Numerator, uint64_t Denominator) { +constexpr inline uint64_t divideNearest(uint64_t Numerator, + uint64_t Denominator) { return (Numerator + (Denominator / 2)) / Denominator; } @@ -791,7 +884,8 @@ /// Subtract two unsigned integers, X and Y, of type T and return the absolute /// value of the result. template -std::enable_if_t::value, T> AbsoluteDifference(T X, T Y) { +constexpr std::enable_if_t::value, T> +AbsoluteDifference(T X, T Y) { return X > Y ? (X - Y) : (Y - X); } @@ -877,7 +971,6 @@ /// Use this rather than HUGE_VALF; the latter causes warnings on MSVC. extern const float huge_valf; - /// Add two signed integers, computing the two's complement truncated result, /// returning true if overflow occured. template @@ -957,6 +1050,6 @@ return UX > (static_cast(std::numeric_limits::max())) / UY; } -} // End llvm namespace +} // namespace llvm #endif Index: llvm/lib/Support/MathExtras.cpp =================================================================== --- llvm/lib/Support/MathExtras.cpp +++ llvm/lib/Support/MathExtras.cpp @@ -15,17 +15,17 @@ #ifdef _MSC_VER #include #else -#include +#include #endif namespace llvm { -#if defined(_MSC_VER) - // Visual Studio defines the HUGE_VAL class of macros using purposeful - // constant arithmetic overflow, which it then warns on when encountered. - const float huge_valf = std::numeric_limits::infinity(); +#ifdef _MSC_VER +// Visual Studio defines the HUGE_VAL class of macros using purposeful +// constant arithmetic overflow, which it then warns on when encountered. +const float huge_valf = std::numeric_limits::infinity(); #else - const float huge_valf = HUGE_VALF; +constexpr float huge_valf = HUGE_VALF; #endif } // namespace llvm