diff --git a/libc/src/__support/CPP/optional.h b/libc/src/__support/CPP/optional.h --- a/libc/src/__support/CPP/optional.h +++ b/libc/src/__support/CPP/optional.h @@ -18,19 +18,19 @@ // Trivial in_place_t struct. struct in_place_t { - LIBC_INLINE explicit in_place_t() = default; + LIBC_INLINE_VAR explicit in_place_t() = default; }; // Trivial nullopt_t struct. struct nullopt_t { - LIBC_INLINE explicit nullopt_t() = default; + LIBC_INLINE_VAR explicit nullopt_t() = default; }; // nullopt that can be used and returned. -inline constexpr nullopt_t nullopt{}; +LIBC_INLINE_VAR constexpr nullopt_t nullopt{}; // in_place that can be used in the constructor. -inline constexpr in_place_t in_place{}; +LIBC_INLINE_VAR constexpr in_place_t in_place{}; // This is very simple implementation of the std::optional class. It makes // several assumptions that the underlying type is trivially constructable, diff --git a/libc/src/__support/CPP/span.h b/libc/src/__support/CPP/span.h --- a/libc/src/__support/CPP/span.h +++ b/libc/src/__support/CPP/span.h @@ -13,6 +13,8 @@ #include "array.h" // For array #include "type_traits.h" // For remove_cv_t, enable_if_t, is_same_v, is_const_v +#include "src/__support/macros/attributes.h" + namespace __llvm_libc::cpp { // A trimmed down implementation of std::span. @@ -25,12 +27,12 @@ // - No reverse iterators template class span { template - inline static constexpr bool is_const_view_v = + LIBC_INLINE_VAR static constexpr bool is_const_view_v = !cpp::is_const_v && cpp::is_const_v && cpp::is_same_v>; template - inline static constexpr bool is_compatible_v = + LIBC_INLINE_VAR static constexpr bool is_compatible_v = cpp::is_same_v || is_const_view_v; public: @@ -46,61 +48,67 @@ static constexpr size_type dynamic_extent = -1; - constexpr span() : span_data(nullptr), span_size(0) {} + LIBC_INLINE constexpr span() : span_data(nullptr), span_size(0) {} - constexpr span(pointer first, size_type count) + LIBC_INLINE constexpr span(pointer first, size_type count) : span_data(first), span_size(count) {} - constexpr span(pointer first, pointer end) + LIBC_INLINE constexpr span(pointer first, pointer end) : span_data(first), span_size(end - first) {} template , bool> = true> - constexpr span(U (&arr)[N]) : span_data(arr), span_size(N) {} + LIBC_INLINE constexpr span(U (&arr)[N]) : span_data(arr), span_size(N) {} template , bool> = true> - constexpr span(array &arr) + LIBC_INLINE constexpr span(array &arr) : span_data(arr.data()), span_size(arr.size()) {} template , bool> = true> - constexpr span(span &s) : span_data(s.data()), span_size(s.size()) {} + LIBC_INLINE constexpr span(span &s) + : span_data(s.data()), span_size(s.size()) {} template , bool> = true> - constexpr span &operator=(span &s) { + LIBC_INLINE constexpr span &operator=(span &s) { span_data = s.data(); span_size = s.size(); return *this; } - ~span() = default; - constexpr reference operator[](size_type index) const { + LIBC_INLINE ~span() = default; + + LIBC_INLINE constexpr reference operator[](size_type index) const { return data()[index]; } - constexpr iterator begin() const { return data(); } - constexpr iterator end() const { return data() + size(); } - constexpr reference front() const { return (*this)[0]; } - constexpr reference back() const { return (*this)[size() - 1]; } - constexpr pointer data() const { return span_data; } - constexpr size_type size() const { return span_size; } - constexpr size_type size_bytes() const { return sizeof(T) * size(); } - constexpr bool empty() const { return size() == 0; } - - constexpr span subspan(size_type offset, - size_type count = dynamic_extent) const { + + LIBC_INLINE constexpr iterator begin() const { return data(); } + LIBC_INLINE constexpr iterator end() const { return data() + size(); } + LIBC_INLINE constexpr reference front() const { return (*this)[0]; } + LIBC_INLINE constexpr reference back() const { return (*this)[size() - 1]; } + LIBC_INLINE constexpr pointer data() const { return span_data; } + LIBC_INLINE constexpr size_type size() const { return span_size; } + LIBC_INLINE constexpr size_type size_bytes() const { + return sizeof(T) * size(); + } + LIBC_INLINE constexpr bool empty() const { return size() == 0; } + + LIBC_INLINE constexpr span + subspan(size_type offset, size_type count = dynamic_extent) const { return span(data() + offset, count_to_size(offset, count)); } - constexpr span first(size_type count) const { + LIBC_INLINE constexpr span first(size_type count) const { return subspan(0, count); } - constexpr span last(size_type count) const { + LIBC_INLINE constexpr span last(size_type count) const { return span(data() + (size() - count), count); } private: - constexpr size_type count_to_size(size_type offset, size_type count) const { + LIBC_INLINE constexpr size_type count_to_size(size_type offset, + size_type count) const { if (count == dynamic_extent) { return size() - offset; } diff --git a/libc/src/__support/CPP/string_view.h b/libc/src/__support/CPP/string_view.h --- a/libc/src/__support/CPP/string_view.h +++ b/libc/src/__support/CPP/string_view.h @@ -60,7 +60,7 @@ // special value equal to the maximum value representable by the type // size_type. - inline static constexpr size_t npos = -1; + LIBC_INLINE_VAR static constexpr size_t npos = -1; LIBC_INLINE constexpr string_view() : Data(nullptr), Len(0) {} diff --git a/libc/src/__support/CPP/type_traits.h b/libc/src/__support/CPP/type_traits.h --- a/libc/src/__support/CPP/type_traits.h +++ b/libc/src/__support/CPP/type_traits.h @@ -9,6 +9,8 @@ #ifndef LLVM_LIBC_SRC_SUPPORT_CPP_TYPETRAITS_H #define LLVM_LIBC_SRC_SUPPORT_CPP_TYPETRAITS_H +#include "src/__support/macros/attributes.h" + namespace __llvm_libc { namespace cpp { @@ -23,7 +25,7 @@ template struct integral_constant { using value_type = T; - static constexpr T value = v; + LIBC_INLINE_VAR static constexpr T value = v; }; using true_type = cpp::integral_constant; using false_type = cpp::integral_constant; @@ -39,11 +41,12 @@ template struct is_same : cpp::false_type {}; template struct is_same : cpp::true_type {}; template -inline constexpr bool is_same_v = is_same::value; +LIBC_INLINE_VAR constexpr bool is_same_v = is_same::value; template struct is_const : cpp::false_type {}; template struct is_const : cpp::true_type {}; -template inline constexpr bool is_const_v = is_const::value; +template +LIBC_INLINE_VAR constexpr bool is_const_v = is_const::value; template struct remove_cv : type_identity {}; template struct remove_cv : type_identity {}; @@ -71,7 +74,7 @@ } // namespace details template struct is_integral { - static constexpr bool value = details::is_unqualified_any_of< + LIBC_INLINE_VAR static constexpr bool value = details::is_unqualified_any_of< T, #ifdef __SIZEOF_INT128__ __int128_t, __uint128_t, @@ -80,33 +83,36 @@ unsigned int, long, unsigned long, long long, unsigned long long, bool>(); }; template -inline constexpr bool is_integral_v = is_integral::value; +LIBC_INLINE_VAR constexpr bool is_integral_v = is_integral::value; template struct is_enum { - static constexpr bool value = __is_enum(T); + LIBC_INLINE_VAR static constexpr bool value = __is_enum(T); }; -template inline constexpr bool is_enum_v = is_enum::value; +template +LIBC_INLINE_VAR constexpr bool is_enum_v = is_enum::value; template struct is_pointer : cpp::false_type {}; template struct is_pointer : cpp::true_type {}; template struct is_pointer : cpp::true_type {}; template struct is_pointer : cpp::true_type {}; template struct is_pointer : cpp::true_type {}; -template inline constexpr bool is_pointer_v = is_pointer::value; +template +LIBC_INLINE_VAR constexpr bool is_pointer_v = is_pointer::value; template struct is_floating_point { - static constexpr bool value = + LIBC_INLINE_VAR static constexpr bool value = details::is_unqualified_any_of(); }; template -inline constexpr bool is_floating_point_v = is_floating_point::value; +LIBC_INLINE_VAR constexpr bool is_floating_point_v = + is_floating_point::value; template struct is_arithmetic { - static constexpr bool value = + LIBC_INLINE_VAR static constexpr bool value = is_integral::value || is_floating_point::value; }; template -inline constexpr bool is_arithmetic_v = is_arithmetic::value; +LIBC_INLINE_VAR constexpr bool is_arithmetic_v = is_arithmetic::value; namespace details { template ::value> @@ -119,19 +125,20 @@ } // namespace details template struct is_signed { - static constexpr bool value = details::is_signed::value; - constexpr operator bool() const { return value; } - constexpr bool operator()() const { return value; } + LIBC_INLINE_VAR static constexpr bool value = details::is_signed::value; + LIBC_INLINE constexpr operator bool() const { return value; } + LIBC_INLINE constexpr bool operator()() const { return value; } }; -template inline constexpr bool is_signed_v = is_signed::value; +template +LIBC_INLINE_VAR constexpr bool is_signed_v = is_signed::value; template struct is_unsigned { - static constexpr bool value = details::is_unsigned::value; - constexpr operator bool() const { return value; } - constexpr bool operator()() const { return value; } + LIBC_INLINE_VAR static constexpr bool value = details::is_unsigned::value; + LIBC_INLINE constexpr operator bool() const { return value; } + LIBC_INLINE constexpr bool operator()() const { return value; } }; template -inline constexpr bool is_unsigned_v = is_unsigned::value; +LIBC_INLINE_VAR constexpr bool is_unsigned_v = is_unsigned::value; template struct make_unsigned; template <> struct make_unsigned : type_identity {}; @@ -166,7 +173,8 @@ template struct is_void : is_same::type> {}; -template inline constexpr bool is_void_v = is_void::value; +template +LIBC_INLINE_VAR constexpr bool is_void_v = is_void::value; template T declval(); // Compile time checks on implicit conversions. @@ -176,10 +184,12 @@ } // namespace details template -inline constexpr bool is_convertible_v = false; +LIBC_INLINE_VAR constexpr bool is_convertible_v = false; +// FIXME: This should use LIBC_INLINE_VAR, but clang buggily complains about +// this when LIBC_INLINE_VAR uses [[clang::internal_linkage]]. template -inline constexpr bool +constexpr bool is_convertible_v( declval()))>> = true; diff --git a/libc/src/__support/StringUtil/tables/linux_extension_signals.h b/libc/src/__support/StringUtil/tables/linux_extension_signals.h --- a/libc/src/__support/StringUtil/tables/linux_extension_signals.h +++ b/libc/src/__support/StringUtil/tables/linux_extension_signals.h @@ -18,7 +18,7 @@ // The array being larger than necessary isn't a problem. The MsgMappings will // be set to their default state which maps 0 to an empty string. This will get // filtered out in the MessageMapper building stage. -inline constexpr const MsgTable<3> LINUX_SIGNALS = { +LIBC_INLINE_VAR constexpr const MsgTable<3> LINUX_SIGNALS = { #ifdef SIGSTKFLT MsgMapping(SIGSTKFLT, "Stack fault"), // unused #endif diff --git a/libc/src/__support/StringUtil/tables/linux_platform_errors.h b/libc/src/__support/StringUtil/tables/linux_platform_errors.h --- a/libc/src/__support/StringUtil/tables/linux_platform_errors.h +++ b/libc/src/__support/StringUtil/tables/linux_platform_errors.h @@ -15,7 +15,7 @@ namespace __llvm_libc { -inline constexpr auto PLATFORM_ERRORS = +LIBC_INLINE_VAR constexpr auto PLATFORM_ERRORS = STDC_ERRORS + POSIX_ERRORS + LINUX_ERRORS; } // namespace __llvm_libc diff --git a/libc/src/__support/StringUtil/tables/linux_platform_signals.h b/libc/src/__support/StringUtil/tables/linux_platform_signals.h --- a/libc/src/__support/StringUtil/tables/linux_platform_signals.h +++ b/libc/src/__support/StringUtil/tables/linux_platform_signals.h @@ -15,7 +15,7 @@ namespace __llvm_libc { -inline constexpr auto PLATFORM_SIGNALS = +LIBC_INLINE_VAR constexpr auto PLATFORM_SIGNALS = STDC_SIGNALS + POSIX_SIGNALS + LINUX_SIGNALS; } // namespace __llvm_libc diff --git a/libc/src/__support/StringUtil/tables/minimal_platform_errors.h b/libc/src/__support/StringUtil/tables/minimal_platform_errors.h --- a/libc/src/__support/StringUtil/tables/minimal_platform_errors.h +++ b/libc/src/__support/StringUtil/tables/minimal_platform_errors.h @@ -13,7 +13,7 @@ namespace __llvm_libc { -inline constexpr auto PLATFORM_ERRORS = STDC_ERRORS; +LIBC_INLINE_VAR constexpr auto PLATFORM_ERRORS = STDC_ERRORS; } // namespace __llvm_libc diff --git a/libc/src/__support/StringUtil/tables/minimal_platform_signals.h b/libc/src/__support/StringUtil/tables/minimal_platform_signals.h --- a/libc/src/__support/StringUtil/tables/minimal_platform_signals.h +++ b/libc/src/__support/StringUtil/tables/minimal_platform_signals.h @@ -13,7 +13,7 @@ namespace __llvm_libc { -inline constexpr auto PLATFORM_SIGNALS = STDC_SIGNALS; +LIBC_INLINE_VAR constexpr auto PLATFORM_SIGNALS = STDC_SIGNALS; } // namespace __llvm_libc diff --git a/libc/src/__support/StringUtil/tables/posix_errors.h b/libc/src/__support/StringUtil/tables/posix_errors.h --- a/libc/src/__support/StringUtil/tables/posix_errors.h +++ b/libc/src/__support/StringUtil/tables/posix_errors.h @@ -15,7 +15,7 @@ namespace __llvm_libc { -inline constexpr MsgTable<76> POSIX_ERRORS = { +LIBC_INLINE_VAR constexpr MsgTable<76> POSIX_ERRORS = { MsgMapping(EPERM, "Operation not permitted"), MsgMapping(ENOENT, "No such file or directory"), MsgMapping(ESRCH, "No such process"), diff --git a/libc/src/__support/StringUtil/tables/posix_signals.h b/libc/src/__support/StringUtil/tables/posix_signals.h --- a/libc/src/__support/StringUtil/tables/posix_signals.h +++ b/libc/src/__support/StringUtil/tables/posix_signals.h @@ -16,7 +16,7 @@ namespace __llvm_libc { -inline constexpr MsgTable<22> POSIX_SIGNALS = { +LIBC_INLINE_VAR constexpr MsgTable<22> POSIX_SIGNALS = { MsgMapping(SIGHUP, "Hangup"), MsgMapping(SIGQUIT, "Quit"), MsgMapping(SIGTRAP, "Trace/breakpoint trap"), diff --git a/libc/src/__support/StringUtil/tables/signal_table.h b/libc/src/__support/StringUtil/tables/signal_table.h --- a/libc/src/__support/StringUtil/tables/signal_table.h +++ b/libc/src/__support/StringUtil/tables/signal_table.h @@ -26,7 +26,7 @@ namespace __llvm_libc::internal { -inline constexpr auto PLATFORM_SIGNALS = []() { +LIBC_INLINE_VAR constexpr auto PLATFORM_SIGNALS = []() { if constexpr (USE_LINUX_PLATFORM_SIGNALS) { return STDC_SIGNALS + POSIX_SIGNALS + LINUX_SIGNALS; } else { diff --git a/libc/src/__support/StringUtil/tables/stdc_errors.h b/libc/src/__support/StringUtil/tables/stdc_errors.h --- a/libc/src/__support/StringUtil/tables/stdc_errors.h +++ b/libc/src/__support/StringUtil/tables/stdc_errors.h @@ -15,7 +15,7 @@ namespace __llvm_libc { -inline constexpr const MsgTable<4> STDC_ERRORS = { +LIBC_INLINE_VAR constexpr const MsgTable<4> STDC_ERRORS = { MsgMapping(0, "Success"), MsgMapping(EDOM, "Numerical argument out of domain"), MsgMapping(ERANGE, "Numerical result out of range"), diff --git a/libc/src/__support/StringUtil/tables/stdc_signals.h b/libc/src/__support/StringUtil/tables/stdc_signals.h --- a/libc/src/__support/StringUtil/tables/stdc_signals.h +++ b/libc/src/__support/StringUtil/tables/stdc_signals.h @@ -15,7 +15,7 @@ namespace __llvm_libc { -inline constexpr const MsgTable<6> STDC_SIGNALS = { +LIBC_INLINE_VAR constexpr const MsgTable<6> STDC_SIGNALS = { MsgMapping(SIGINT, "Interrupt"), MsgMapping(SIGILL, "Illegal instruction"), MsgMapping(SIGABRT, "Aborted"), diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h --- a/libc/src/__support/UInt.h +++ b/libc/src/__support/UInt.h @@ -27,20 +27,20 @@ static_assert(Bits > 0 && Bits % 64 == 0, "Number of bits in BigInt should be a multiple of 64."); - static constexpr size_t WORDCOUNT = Bits / 64; + static LIBC_INLINE_VAR constexpr size_t WORDCOUNT = Bits / 64; uint64_t val[WORDCOUNT]{}; - static constexpr uint64_t MASK32 = 0xFFFFFFFFu; + static LIBC_INLINE_VAR constexpr uint64_t MASK32 = 0xFFFFFFFFu; - static constexpr uint64_t low(uint64_t v) { return v & MASK32; } - static constexpr uint64_t high(uint64_t v) { return (v >> 32) & MASK32; } + static LIBC_INLINE constexpr uint64_t low(uint64_t v) { return v & MASK32; } + static LIBC_INLINE constexpr uint64_t high(uint64_t v) { return (v >> 32) & MASK32; } - constexpr BigInt() = default; + LIBC_INLINE constexpr BigInt() = default; - constexpr BigInt(const BigInt &other) = default; + LIBC_INLINE constexpr BigInt(const BigInt &other) = default; template - constexpr BigInt(const BigInt &other) { + LIBC_INLINE constexpr BigInt(const BigInt &other) { if (OtherBits >= Bits) { for (size_t i = 0; i < WORDCOUNT; ++i) val[i] = other[i]; @@ -60,7 +60,7 @@ // Construct a BigInt from a C array. template = 0> - constexpr BigInt(const uint64_t (&nums)[N]) { + LIBC_INLINE constexpr BigInt(const uint64_t (&nums)[N]) { size_t min_wordcount = N < WORDCOUNT ? N : WORDCOUNT; size_t i = 0; for (; i < min_wordcount; ++i) @@ -74,7 +74,7 @@ // Initialize the first word to |v| and the rest to 0. template && sizeof(T) <= 16>> - constexpr BigInt(T v) { + LIBC_INLINE constexpr BigInt(T v) { val[0] = static_cast(v); if constexpr (Bits == 64) @@ -93,7 +93,7 @@ } } - constexpr explicit BigInt(const cpp::array &words) { + LIBC_INLINE constexpr explicit BigInt(const cpp::array &words) { for (size_t i = 0; i < WORDCOUNT; ++i) val[i] = words[i]; } @@ -101,7 +101,7 @@ template && sizeof(T) <= 16 && !cpp::is_same_v>> - constexpr explicit operator T() const { + LIBC_INLINE constexpr explicit operator T() const { if constexpr (sizeof(T) <= 8) return static_cast(val[0]); @@ -120,11 +120,11 @@ } } - constexpr explicit operator bool() const { return !is_zero(); } + LIBC_INLINE constexpr explicit operator bool() const { return !is_zero(); } BigInt &operator=(const BigInt &other) = default; - constexpr bool is_zero() const { + LIBC_INLINE constexpr bool is_zero() const { for (size_t i = 0; i < WORDCOUNT; ++i) { if (val[i] != 0) return false; @@ -134,7 +134,7 @@ // Add x to this number and store the result in this number. // Returns the carry value produced by the addition operation. - constexpr uint64_t add(const BigInt &x) { + LIBC_INLINE constexpr uint64_t add(const BigInt &x) { SumCarry s{0, 0}; for (size_t i = 0; i < WORDCOUNT; ++i) { s = add_with_carry_const(val[i], x.val[i], s.carry); @@ -143,7 +143,7 @@ return s.carry; } - BigInt operator+(const BigInt &other) const { + LIBC_INLINE constexpr BigInt operator+(const BigInt &other) const { BigInt result; SumCarry s{0, 0}; for (size_t i = 0; i < WORDCOUNT; ++i) { @@ -155,7 +155,7 @@ // This will only apply when initializing a variable from constant values, so // it will always use the constexpr version of add_with_carry. - constexpr BigInt operator+(BigInt &&other) const { + LIBC_INLINE constexpr BigInt operator+(BigInt &&other) const { BigInt result; SumCarry s{0, 0}; for (size_t i = 0; i < WORDCOUNT; ++i) { @@ -165,7 +165,7 @@ return result; } - constexpr BigInt & + LIBC_INLINE constexpr BigInt & operator+=(const BigInt &other) { add(other); // Returned carry value is ignored. return *this; @@ -173,7 +173,7 @@ // Subtract x to this number and store the result in this number. // Returns the carry value produced by the subtraction operation. - constexpr uint64_t sub(const BigInt &x) { + LIBC_INLINE constexpr uint64_t sub(const BigInt &x) { DiffBorrow d{0, 0}; for (size_t i = 0; i < WORDCOUNT; ++i) { d = sub_with_borrow_const(val[i], x.val[i], d.borrow); @@ -182,7 +182,7 @@ return d.borrow; } - BigInt operator-(const BigInt &other) const { + LIBC_INLINE BigInt operator-(const BigInt &other) const { BigInt result; DiffBorrow d{0, 0}; for (size_t i = 0; i < WORDCOUNT; ++i) { @@ -192,7 +192,7 @@ return result; } - constexpr BigInt operator-(BigInt &&other) const { + LIBC_INLINE constexpr BigInt operator-(BigInt &&other) const { BigInt result; DiffBorrow d{0, 0}; for (size_t i = 0; i < WORDCOUNT; ++i) { @@ -202,7 +202,7 @@ return result; } - constexpr BigInt & + LIBC_INLINE constexpr BigInt & operator-=(const BigInt &other) { // TODO(lntue): Set overflow flag / errno when carry is true. sub(other); @@ -215,7 +215,7 @@ // the operations using 64-bit numbers. This ensures that we don't lose the // carry bits. // Returns the carry value produced by the multiplication operation. - constexpr uint64_t mul(uint64_t x) { + LIBC_INLINE constexpr uint64_t mul(uint64_t x) { BigInt<128, Signed> partial_sum(0); uint64_t carry = 0; for (size_t i = 0; i < WORDCOUNT; ++i) { @@ -230,7 +230,7 @@ return partial_sum.val[1]; } - constexpr BigInt + LIBC_INLINE constexpr BigInt operator*(const BigInt &other) const { if constexpr (Signed) { BigInt a(*this); @@ -271,7 +271,7 @@ // Return the full product, only unsigned for now. template - constexpr BigInt + LIBC_INLINE constexpr BigInt ful_mul(const BigInt &other) const { BigInt result(0); BigInt<128, Signed> partial_sum(0); @@ -317,7 +317,7 @@ // 256 4 16 10 3 // 512 8 64 36 7 constexpr BigInt - quick_mul_hi(const BigInt &other) const { + LIBC_INLINE quick_mul_hi(const BigInt &other) const { BigInt result(0); BigInt<128, Signed> partial_sum(0); uint64_t carry = 0; @@ -346,7 +346,7 @@ // pow takes a power and sets this to its starting value to that power. Zero // to the zeroth power returns 1. - constexpr void pow_n(uint64_t power) { + LIBC_INLINE constexpr void pow_n(uint64_t power) { BigInt result = 1; BigInt cur_power = *this; @@ -364,7 +364,7 @@ // div takes another BigInt of the same size and divides this by it. The value // of this will be set to the quotient, and the return value is the remainder. - constexpr optional> + LIBC_INLINE constexpr optional> div(const BigInt &other) { BigInt remainder(0); if (*this < other) { @@ -407,7 +407,7 @@ // Since the remainder of each division step < x < 2^32, the computation of // each step is now properly contained within uint64_t. // And finally we perform some extra alignment steps for the remaining bits. - constexpr optional> div_uint32_times_pow_2(uint32_t x, + LIBC_INLINE constexpr optional> div_uint32_times_pow_2(uint32_t x, size_t e) { BigInt remainder(0); @@ -515,32 +515,32 @@ return remainder; } - constexpr BigInt + LIBC_INLINE constexpr BigInt operator/(const BigInt &other) const { BigInt result(*this); result.div(other); return result; } - constexpr BigInt & + LIBC_INLINE constexpr BigInt & operator/=(const BigInt &other) { div(other); return *this; } - constexpr BigInt + LIBC_INLINE constexpr BigInt operator%(const BigInt &other) const { BigInt result(*this); return *result.div(other); } - constexpr BigInt & + LIBC_INLINE constexpr BigInt & operator*=(const BigInt &other) { *this = *this * other; return *this; } - constexpr uint64_t clz() { + LIBC_INLINE constexpr uint64_t clz() { uint64_t leading_zeroes = 0; for (size_t i = WORDCOUNT; i > 0; --i) { if (val[i - 1] == 0) { @@ -553,7 +553,7 @@ return leading_zeroes; } - constexpr void shift_left(size_t s) { + LIBC_INLINE constexpr void shift_left(size_t s) { #ifdef __SIZEOF_INT128__ if constexpr (Bits == 128) { // Use builtin 128 bits if available; @@ -596,18 +596,18 @@ } } - constexpr BigInt operator<<(size_t s) const { + LIBC_INLINE constexpr BigInt operator<<(size_t s) const { BigInt result(*this); result.shift_left(s); return result; } - constexpr BigInt &operator<<=(size_t s) { + LIBC_INLINE constexpr BigInt &operator<<=(size_t s) { shift_left(s); return *this; } - constexpr void shift_right(size_t s) { + LIBC_INLINE constexpr void shift_right(size_t s) { #ifdef __SIZEOF_INT128__ if constexpr (Bits == 128) { // Use builtin 128 bits if available; @@ -660,18 +660,18 @@ } } - constexpr BigInt operator>>(size_t s) const { + LIBC_INLINE constexpr BigInt operator>>(size_t s) const { BigInt result(*this); result.shift_right(s); return result; } - constexpr BigInt &operator>>=(size_t s) { + LIBC_INLINE constexpr BigInt &operator>>=(size_t s) { shift_right(s); return *this; } - constexpr BigInt + LIBC_INLINE constexpr BigInt operator&(const BigInt &other) const { BigInt result; for (size_t i = 0; i < WORDCOUNT; ++i) @@ -679,14 +679,14 @@ return result; } - constexpr BigInt & + LIBC_INLINE constexpr BigInt & operator&=(const BigInt &other) { for (size_t i = 0; i < WORDCOUNT; ++i) val[i] &= other.val[i]; return *this; } - constexpr BigInt + LIBC_INLINE constexpr BigInt operator|(const BigInt &other) const { BigInt result; for (size_t i = 0; i < WORDCOUNT; ++i) @@ -694,14 +694,14 @@ return result; } - constexpr BigInt & + LIBC_INLINE constexpr BigInt & operator|=(const BigInt &other) { for (size_t i = 0; i < WORDCOUNT; ++i) val[i] |= other.val[i]; return *this; } - constexpr BigInt + LIBC_INLINE constexpr BigInt operator^(const BigInt &other) const { BigInt result; for (size_t i = 0; i < WORDCOUNT; ++i) @@ -709,27 +709,27 @@ return result; } - constexpr BigInt & + LIBC_INLINE constexpr BigInt & operator^=(const BigInt &other) { for (size_t i = 0; i < WORDCOUNT; ++i) val[i] ^= other.val[i]; return *this; } - constexpr BigInt operator~() const { + LIBC_INLINE constexpr BigInt operator~() const { BigInt result; for (size_t i = 0; i < WORDCOUNT; ++i) result.val[i] = ~val[i]; return result; } - constexpr BigInt operator-() const { + LIBC_INLINE constexpr BigInt operator-() const { BigInt result = ~(*this); result.add(BigInt(1)); return result; } - constexpr bool operator==(const BigInt &other) const { + LIBC_INLINE constexpr bool operator==(const BigInt &other) const { for (size_t i = 0; i < WORDCOUNT; ++i) { if (val[i] != other.val[i]) return false; @@ -737,7 +737,7 @@ return true; } - constexpr bool operator!=(const BigInt &other) const { + LIBC_INLINE constexpr bool operator!=(const BigInt &other) const { for (size_t i = 0; i < WORDCOUNT; ++i) { if (val[i] != other.val[i]) return true; @@ -745,7 +745,7 @@ return false; } - constexpr bool operator>(const BigInt &other) const { + LIBC_INLINE constexpr bool operator>(const BigInt &other) const { if constexpr (Signed) { // Check for different signs; bool a_sign = val[WORDCOUNT - 1] >> 63; @@ -766,7 +766,7 @@ return false; } - constexpr bool operator>=(const BigInt &other) const { + LIBC_INLINE constexpr bool operator>=(const BigInt &other) const { if constexpr (Signed) { // Check for different signs; bool a_sign = val[WORDCOUNT - 1] >> 63; @@ -787,7 +787,7 @@ return true; } - constexpr bool operator<(const BigInt &other) const { + LIBC_INLINE constexpr bool operator<(const BigInt &other) const { if constexpr (Signed) { // Check for different signs; bool a_sign = val[WORDCOUNT - 1] >> 63; @@ -809,7 +809,7 @@ return false; } - constexpr bool operator<=(const BigInt &other) const { + LIBC_INLINE constexpr bool operator<=(const BigInt &other) const { if constexpr (Signed) { // Check for different signs; bool a_sign = val[WORDCOUNT - 1] >> 63; @@ -830,26 +830,26 @@ return true; } - constexpr BigInt &operator++() { + LIBC_INLINE constexpr BigInt &operator++() { BigInt one(1); add(one); return *this; } - constexpr BigInt operator++(int) { + LIBC_INLINE constexpr BigInt operator++(int) { BigInt oldval(*this); BigInt one(1); add(one); return oldval; } - constexpr BigInt &operator--() { + LIBC_INLINE constexpr BigInt &operator--() { BigInt one(1); sub(one); return *this; } - constexpr BigInt operator--(int) { + LIBC_INLINE constexpr BigInt operator--(int) { BigInt oldval(*this); BigInt one(1); sub(one); @@ -857,14 +857,14 @@ } // Return the i-th 64-bit word of the number. - constexpr const uint64_t &operator[](size_t i) const { return val[i]; } + LIBC_INLINE constexpr const uint64_t &operator[](size_t i) const { return val[i]; } // Return the i-th 64-bit word of the number. - constexpr uint64_t &operator[](size_t i) { return val[i]; } + LIBC_INLINE constexpr uint64_t &operator[](size_t i) { return val[i]; } - uint64_t *data() { return val; } + LIBC_INLINE uint64_t *data() { return val; } - const uint64_t *data() const { return val; } + LIBC_INLINE const uint64_t *data() const { return val; } }; template using UInt = BigInt; @@ -874,7 +874,7 @@ // Provides limits of U/Int<128>. template <> class numeric_limits> { public: - static constexpr UInt<128> max() { + LIBC_INLINE static constexpr UInt<128> max() { return UInt<128>({0xffff'ffff'ffff'ffff, 0xffff'ffff'ffff'ffff}); } static constexpr UInt<128> min() { return UInt<128>(0); } @@ -882,10 +882,10 @@ template <> class numeric_limits> { public: - static constexpr Int<128> max() { + LIBC_INLINE static constexpr Int<128> max() { return Int<128>({0xffff'ffff'ffff'ffff, 0x7fff'ffff'ffff'ffff}); } - static constexpr Int<128> min() { + LIBC_INLINE static constexpr Int<128> min() { return Int<128>({0, 0x8000'0000'0000'0000}); } }; diff --git a/libc/src/__support/threads/linux/thread.cpp b/libc/src/__support/threads/linux/thread.cpp --- a/libc/src/__support/threads/linux/thread.cpp +++ b/libc/src/__support/threads/linux/thread.cpp @@ -120,7 +120,7 @@ // This must always be inlined as we may be freeing the calling threads stack in // which case a normal return from the top the stack would cause an invalid // memory read. -static __attribute__((always_inline)) inline void +[[gnu::always_inline]] LIBC_INLINE void free_stack(void *stack, size_t stacksize, size_t guardsize) { uintptr_t stackaddr = reinterpret_cast(stack); stackaddr -= guardsize; @@ -144,7 +144,7 @@ // This must always be inlined as we may be freeing the calling threads stack in // which case a normal return from the top the stack would cause an invalid // memory read. -static __attribute__((always_inline)) inline void +[[gnu::always_inline]] LIBC_INLINE void cleanup_thread_resources(ThreadAttributes *attrib) { // Cleanup the TLS before the stack as the TLS information is stored on // the stack. @@ -153,7 +153,7 @@ free_stack(attrib->stack, attrib->stacksize, attrib->guardsize); } -__attribute__((always_inline)) inline uintptr_t get_start_args_addr() { +[[gnu::always_inline]] LIBC_INLINE uintptr_t get_start_args_addr() { // NOTE: For __builtin_frame_address to work reliably across compilers, // architectures and various optimization levels, the TU including this file // should be compiled with -fno-omit-frame-pointer. @@ -176,7 +176,7 @@ #endif } -__attribute__((noinline)) static void start_thread() { +[[gnu::noinline]] LIBC_INLINE void start_thread() { auto *start_args = reinterpret_cast(get_start_args_addr()); auto *attrib = start_args->thread_attrib; self.attrib = attrib; diff --git a/libc/src/stdio/printf_core/write_int_converter.h b/libc/src/stdio/printf_core/write_int_converter.h --- a/libc/src/stdio/printf_core/write_int_converter.h +++ b/libc/src/stdio/printf_core/write_int_converter.h @@ -19,7 +19,8 @@ namespace __llvm_libc { namespace printf_core { -int inline convert_write_int(Writer *writer, const FormatSection &to_conv) { +LIBC_INLINE int convert_write_int(Writer *writer, + const FormatSection &to_conv) { // This is an additional check added by LLVM-libc. The reason it returns -3 is // because printf uses negative return values for errors, and -1 and -2 are diff --git a/libc/src/string/memory_utils/op_aarch64.h b/libc/src/string/memory_utils/op_aarch64.h --- a/libc/src/string/memory_utils/op_aarch64.h +++ b/libc/src/string/memory_utils/op_aarch64.h @@ -26,7 +26,7 @@ namespace __llvm_libc::aarch64 { -static inline constexpr bool kNeon = LLVM_LIBC_IS_DEFINED(__ARM_NEON); +LIBC_INLINE_VAR constexpr bool kNeon = LLVM_LIBC_IS_DEFINED(__ARM_NEON); namespace neon { @@ -52,7 +52,7 @@ } }; -LIBC_INLINE static bool hasZva() { +LIBC_INLINE bool hasZva() { uint64_t zva_val; asm("mrs %[zva_val], dczid_el0" : [zva_val] "=r"(zva_val)); // DC ZVA is permitted if DZP, bit [4] is zero. diff --git a/libc/src/string/memory_utils/op_generic.h b/libc/src/string/memory_utils/op_generic.h --- a/libc/src/string/memory_utils/op_generic.h +++ b/libc/src/string/memory_utils/op_generic.h @@ -318,24 +318,24 @@ // Same as load above but with an offset to the pointer. // Making the offset explicit hints the compiler to use relevant addressing mode // consistently. -template LIBC_INLINE static T load(CPtr ptr, size_t offset) { +template LIBC_INLINE T load(CPtr ptr, size_t offset) { return ::__llvm_libc::load(ptr + offset); } // Same as above but also makes sure the loaded value is in big endian format. // This is useful when implementing lexicograhic comparisons as big endian // scalar comparison directly maps to lexicographic byte comparisons. -template LIBC_INLINE static T load_be(CPtr ptr, size_t offset) { +template LIBC_INLINE T load_be(CPtr ptr, size_t offset) { return Endian::to_big_endian(load(ptr, offset)); } // Equality: returns true iff values at locations (p1 + offset) and (p2 + // offset) compare equal. -template static bool eq(CPtr p1, CPtr p2, size_t offset); +template LIBC_INLINE bool eq(CPtr p1, CPtr p2, size_t offset); // Not equals: returns non-zero iff values at locations (p1 + offset) and (p2 + // offset) differ. -template static uint32_t neq(CPtr p1, CPtr p2, size_t offset); +template LIBC_INLINE uint32_t neq(CPtr p1, CPtr p2, size_t offset); // Lexicographic comparison: // - returns 0 iff values at locations (p1 + offset) and (p2 + offset) compare @@ -345,7 +345,7 @@ // - returns a positive value if value at location (p1 + offset) is // lexicographically greater than value at (p2 + offset). template -static MemcmpReturnType cmp(CPtr p1, CPtr p2, size_t offset); +LIBC_INLINE MemcmpReturnType cmp(CPtr p1, CPtr p2, size_t offset); // Lexicographic comparison of non-equal values: // - returns a negative value if value at location (p1 + offset) is @@ -353,7 +353,7 @@ // - returns a positive value if value at location (p1 + offset) is // lexicographically greater than value at (p2 + offset). template -static MemcmpReturnType cmp_neq(CPtr p1, CPtr p2, size_t offset); +LIBC_INLINE MemcmpReturnType cmp_neq(CPtr p1, CPtr p2, size_t offset); /////////////////////////////////////////////////////////////////////////////// // Memcmp implementation @@ -563,6 +563,7 @@ } template <> LIBC_INLINE MemcmpReturnType cmp_neq(CPtr p1, CPtr p2, size_t offset); + } // namespace __llvm_libc::generic #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_GENERIC_H diff --git a/libc/src/string/memory_utils/op_x86.h b/libc/src/string/memory_utils/op_x86.h --- a/libc/src/string/memory_utils/op_x86.h +++ b/libc/src/string/memory_utils/op_x86.h @@ -50,7 +50,7 @@ /////////////////////////////////////////////////////////////////////////////// // Memcpy repmovsb implementation struct Memcpy { - static void repmovsb(void *dst, const void *src, size_t count) { + LIBC_INLINE static void repmovsb(void *dst, const void *src, size_t count) { asm volatile("rep movsb" : "+D"(dst), "+S"(src), "+c"(count) : : "memory"); } }; diff --git a/libc/src/string/memory_utils/strcmp_implementations.h b/libc/src/string/memory_utils/strcmp_implementations.h --- a/libc/src/string/memory_utils/strcmp_implementations.h +++ b/libc/src/string/memory_utils/strcmp_implementations.h @@ -14,8 +14,8 @@ namespace __llvm_libc { template -constexpr static int strcmp_implementation(const char *left, const char *right, - Comp &&comp) { +LIBC_INLINE constexpr int +strcmp_implementation(const char *left, const char *right, Comp &&comp) { // TODO: Look at benefits for comparing words at a time. for (; *left && !comp(*left, *right); ++left, ++right) ; @@ -24,8 +24,9 @@ } template -constexpr static int strncmp_implementation(const char *left, const char *right, - size_t n, Comp &&comp) { +LIBC_INLINE constexpr int strncmp_implementation(const char *left, + const char *right, size_t n, + Comp &&comp) { if (n == 0) return 0; diff --git a/libc/src/string/memory_utils/strstr_implementations.h b/libc/src/string/memory_utils/strstr_implementations.h --- a/libc/src/string/memory_utils/strstr_implementations.h +++ b/libc/src/string/memory_utils/strstr_implementations.h @@ -16,8 +16,8 @@ namespace __llvm_libc { template -constexpr static char *strstr_implementation(const char *haystack, - const char *needle, Comp &&comp) { +LIBC_INLINE constexpr char * +strstr_implementation(const char *haystack, const char *needle, Comp &&comp) { void *result = memmem_implementation( static_cast(haystack), internal::string_length(haystack), static_cast(needle), internal::string_length(needle), comp); diff --git a/libc/src/string/memory_utils/utils.h b/libc/src/string/memory_utils/utils.h --- a/libc/src/string/memory_utils/utils.h +++ b/libc/src/string/memory_utils/utils.h @@ -24,48 +24,50 @@ // Allows compile time error reporting in `if constexpr` branches. template -static void deferred_static_assert(const char *msg) { +LIBC_INLINE void deferred_static_assert(const char *msg) { static_assert(flag, "compilation error"); (void)msg; } // Return whether `value` is zero or a power of two. -static constexpr bool is_power2_or_zero(size_t value) { +LIBC_INLINE constexpr bool is_power2_or_zero(size_t value) { return (value & (value - 1U)) == 0; } // Return whether `value` is a power of two. -static constexpr bool is_power2(size_t value) { +LIBC_INLINE constexpr bool is_power2(size_t value) { return value && is_power2_or_zero(value); } // Compile time version of log2 that handles 0. -static constexpr size_t log2s(size_t value) { +LIBC_INLINE constexpr size_t log2s(size_t value) { return (value == 0 || value == 1) ? 0 : 1 + log2s(value / 2); } // Returns the first power of two preceding value or value if it is already a // power of two (or 0 when value is 0). -static constexpr size_t le_power2(size_t value) { +LIBC_INLINE constexpr size_t le_power2(size_t value) { return value == 0 ? value : 1ULL << log2s(value); } // Returns the first power of two following value or value if it is already a // power of two (or 0 when value is 0). -static constexpr size_t ge_power2(size_t value) { +LIBC_INLINE constexpr size_t ge_power2(size_t value) { return is_power2_or_zero(value) ? value : 1ULL << (log2s(value) + 1); } // Returns the number of bytes to substract from ptr to get to the previous // multiple of alignment. If ptr is already aligned returns 0. -template uintptr_t distance_to_align_down(const void *ptr) { +template +LIBC_INLINE uintptr_t distance_to_align_down(const void *ptr) { static_assert(is_power2(alignment), "alignment must be a power of 2"); return reinterpret_cast(ptr) & (alignment - 1U); } // Returns the number of bytes to add to ptr to get to the next multiple of // alignment. If ptr is already aligned returns 0. -template uintptr_t distance_to_align_up(const void *ptr) { +template +LIBC_INLINE uintptr_t distance_to_align_up(const void *ptr) { static_assert(is_power2(alignment), "alignment must be a power of 2"); // The logic is not straightforward and involves unsigned modulo arithmetic // but the generated code is as fast as it can be. @@ -75,12 +77,13 @@ // Returns the number of bytes to add to ptr to get to the next multiple of // alignment. If ptr is already aligned returns alignment. template -uintptr_t distance_to_next_aligned(const void *ptr) { +LIBC_INLINE uintptr_t distance_to_next_aligned(const void *ptr) { return alignment - distance_to_align_down(ptr); } // Returns the same pointer but notifies the compiler that it is aligned. -template static T *assume_aligned(T *ptr) { +template +LIBC_INLINE T *assume_aligned(T *ptr) { return reinterpret_cast(__builtin_assume_aligned(ptr, alignment)); } diff --git a/libc/src/string/string_utils.h b/libc/src/string/string_utils.h --- a/libc/src/string/string_utils.h +++ b/libc/src/string/string_utils.h @@ -225,7 +225,8 @@ } template -constexpr static char *strchr_implementation(const char *src, int c) { +LIBC_INLINE constexpr static char *strchr_implementation(const char *src, + int c) { char ch = static_cast(c); for (; *src && *src != ch; ++src) ; @@ -233,7 +234,8 @@ return *src == ch ? const_cast(src) : ret; } -constexpr static char *strrchr_implementation(const char *src, int c) { +LIBC_INLINE constexpr static char *strrchr_implementation(const char *src, + int c) { char ch = static_cast(c); char *last_occurrence = nullptr; for (; *src; ++src) {