diff --git a/libc/src/__support/CPP/array.h b/libc/src/__support/CPP/array.h --- a/libc/src/__support/CPP/array.h +++ b/libc/src/__support/CPP/array.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SRC_SUPPORT_CPP_ARRAY_H #define LLVM_LIBC_SRC_SUPPORT_CPP_ARRAY_H +#include "src/__support/macros/attributes.h" #include // For size_t. namespace __llvm_libc { @@ -22,28 +23,30 @@ using iterator = T *; using const_iterator = const T *; - constexpr T *data() { return Data; } - constexpr const T *data() const { return Data; } + LIBC_INLINE constexpr T *data() { return Data; } + LIBC_INLINE constexpr const T *data() const { return Data; } - constexpr T &front() { return Data[0]; } - constexpr T &front() const { return Data[0]; } + LIBC_INLINE constexpr T &front() { return Data[0]; } + LIBC_INLINE constexpr T &front() const { return Data[0]; } - constexpr T &back() { return Data[N - 1]; } - constexpr T &back() const { return Data[N - 1]; } + LIBC_INLINE constexpr T &back() { return Data[N - 1]; } + LIBC_INLINE constexpr T &back() const { return Data[N - 1]; } - constexpr T &operator[](size_t Index) { return Data[Index]; } + LIBC_INLINE constexpr T &operator[](size_t Index) { return Data[Index]; } - constexpr const T &operator[](size_t Index) const { return Data[Index]; } + LIBC_INLINE constexpr const T &operator[](size_t Index) const { + return Data[Index]; + } - constexpr size_t size() const { return N; } + LIBC_INLINE constexpr size_t size() const { return N; } - constexpr bool empty() const { return N == 0; } + LIBC_INLINE constexpr bool empty() const { return N == 0; } - constexpr iterator begin() { return Data; } - constexpr const_iterator begin() const { return Data; } + LIBC_INLINE constexpr iterator begin() { return Data; } + LIBC_INLINE constexpr const_iterator begin() const { return Data; } - constexpr iterator end() { return Data + N; } - const_iterator end() const { return Data + N; } + LIBC_INLINE constexpr iterator end() { return Data + N; } + LIBC_INLINE const_iterator end() const { return Data + N; } }; } // namespace cpp diff --git a/libc/src/__support/CPP/bit.h b/libc/src/__support/CPP/bit.h --- a/libc/src/__support/CPP/bit.h +++ b/libc/src/__support/CPP/bit.h @@ -10,6 +10,7 @@ #define LLVM_LIBC_SUPPORT_CPP_BIT_H #include "src/__support/CPP/type_traits.h" +#include "src/__support/macros/attributes.h" #include "src/__support/macros/config.h" // LIBC_HAS_BUILTIN namespace __llvm_libc::cpp { @@ -24,7 +25,8 @@ // This function guarantees the bitcast to be optimized away by the compiler for // GCC >= 8 and Clang >= 6. -template constexpr To bit_cast(const From &from) { +template +LIBC_INLINE constexpr To bit_cast(const From &from) { static_assert(sizeof(To) == sizeof(From), "To and From must be of same size"); static_assert(cpp::is_trivially_copyable::value && cpp::is_trivially_copyable::value, diff --git a/libc/src/__support/CPP/bitset.h b/libc/src/__support/CPP/bitset.h --- a/libc/src/__support/CPP/bitset.h +++ b/libc/src/__support/CPP/bitset.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SRC_SUPPORT_CPP_BITSET_H #define LLVM_LIBC_SRC_SUPPORT_CPP_BITSET_H +#include "src/__support/macros/attributes.h" #include // For size_t. namespace __llvm_libc::cpp { @@ -17,27 +18,27 @@ static_assert(NumberOfBits != 0, "Cannot create a __llvm_libc::cpp::bitset of size 0."); - constexpr void set(size_t Index) { + LIBC_INLINE constexpr void set(size_t Index) { Data[Index / BITS_PER_UNIT] |= mask(Index); } - constexpr void reset() { + LIBC_INLINE constexpr void reset() { for (size_t i = 0; i < NUMBER_OF_UNITS; ++i) Data[i] = 0; } - constexpr bool test(size_t Index) const { + LIBC_INLINE constexpr bool test(size_t Index) const { return Data[Index / BITS_PER_UNIT] & mask(Index); } - constexpr void flip() { + LIBC_INLINE constexpr void flip() { for (size_t i = 0; i < NUMBER_OF_UNITS; ++i) Data[i] = ~Data[i]; } // This function sets all bits in the range from Start to End (inclusive) to // true. It assumes that Start <= End. - constexpr void set_range(size_t Start, size_t End) { + LIBC_INLINE constexpr void set_range(size_t Start, size_t End) { size_t start_index = Start / BITS_PER_UNIT; size_t end_index = End / BITS_PER_UNIT; @@ -64,7 +65,7 @@ } } - constexpr bool operator==(const bitset &other) { + LIBC_INLINE constexpr bool operator==(const bitset &other) { for (size_t i = 0; i < NUMBER_OF_UNITS; ++i) { if (Data[i] != other.Data[i]) return false; @@ -78,7 +79,7 @@ static constexpr size_t NUMBER_OF_UNITS = (NumberOfBits + BITS_PER_UNIT - 1) / BITS_PER_UNIT; - static constexpr size_t mask(size_t Index) { + LIBC_INLINE static constexpr size_t mask(size_t Index) { return size_t{1} << (Index % BITS_PER_UNIT); } size_t Data[NUMBER_OF_UNITS] = {0}; diff --git a/libc/src/__support/CPP/cstddef.h b/libc/src/__support/CPP/cstddef.h --- a/libc/src/__support/CPP/cstddef.h +++ b/libc/src/__support/CPP/cstddef.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SRC_SUPPORT_CPP_BYTE_H #define LLVM_LIBC_SRC_SUPPORT_CPP_BYTE_H +#include "src/__support/macros/attributes.h" #include "type_traits.h" // For enable_if_t, is_integral_v. namespace __llvm_libc::cpp { @@ -16,45 +17,51 @@ enum class byte : unsigned char {}; template -constexpr enable_if_t, byte> +LIBC_INLINE constexpr enable_if_t, byte> operator>>(byte b, IntegerType shift) noexcept { return static_cast(static_cast(b) >> shift); } template -constexpr enable_if_t, byte &> +LIBC_INLINE constexpr enable_if_t, byte &> operator>>=(byte &b, IntegerType shift) noexcept { return b = b >> shift; } template -constexpr enable_if_t, byte> +LIBC_INLINE constexpr enable_if_t, byte> operator<<(byte b, IntegerType shift) noexcept { return static_cast(static_cast(b) << shift); } template -constexpr enable_if_t, byte &> +LIBC_INLINE constexpr enable_if_t, byte &> operator<<=(byte &b, IntegerType shift) noexcept { return b = b << shift; } -constexpr byte operator|(byte l, byte r) noexcept { +LIBC_INLINE constexpr byte operator|(byte l, byte r) noexcept { return static_cast(static_cast(l) | static_cast(r)); } -constexpr byte &operator|=(byte &l, byte r) noexcept { return l = l | r; } -constexpr byte operator&(byte l, byte r) noexcept { +LIBC_INLINE constexpr byte &operator|=(byte &l, byte r) noexcept { + return l = l | r; +} +LIBC_INLINE constexpr byte operator&(byte l, byte r) noexcept { return static_cast(static_cast(l) & static_cast(r)); } -constexpr byte &operator&=(byte &l, byte r) noexcept { return l = l & r; } -constexpr byte operator^(byte l, byte r) noexcept { +LIBC_INLINE constexpr byte &operator&=(byte &l, byte r) noexcept { + return l = l & r; +} +LIBC_INLINE constexpr byte operator^(byte l, byte r) noexcept { return static_cast(static_cast(l) ^ static_cast(r)); } -constexpr byte &operator^=(byte &l, byte r) noexcept { return l = l ^ r; } -constexpr byte operator~(byte b) noexcept { +LIBC_INLINE constexpr byte &operator^=(byte &l, byte r) noexcept { + return l = l ^ r; +} +LIBC_INLINE constexpr byte operator~(byte b) noexcept { return static_cast(~static_cast(b)); } template -constexpr enable_if_t, IntegerType> +LIBC_INLINE constexpr enable_if_t, IntegerType> to_integer(byte b) noexcept { return static_cast(b); } diff --git a/libc/src/__support/CPP/new.h b/libc/src/__support/CPP/new.h --- a/libc/src/__support/CPP/new.h +++ b/libc/src/__support/CPP/new.h @@ -27,14 +27,16 @@ class AllocChecker { bool success = false; - AllocChecker &operator=(bool status) { + + LIBC_INLINE AllocChecker &operator=(bool status) { success = status; return *this; } public: - AllocChecker() = default; - operator bool() const { return success; } + LIBC_INLINE AllocChecker() = default; + + LIBC_INLINE operator bool() const { return success; } LIBC_INLINE static void *alloc(size_t s, AllocChecker &ac) { void *mem = ::malloc(s); diff --git a/libc/src/__support/CPP/optional.h b/libc/src/__support/CPP/optional.h --- a/libc/src/__support/CPP/optional.h +++ b/libc/src/__support/CPP/optional.h @@ -18,12 +18,12 @@ // Trivial in_place_t struct. struct in_place_t { - LIBC_INLINE_VAR explicit in_place_t() = default; + LIBC_INLINE constexpr explicit in_place_t() = default; }; // Trivial nullopt_t struct. struct nullopt_t { - LIBC_INLINE_VAR explicit nullopt_t() = default; + LIBC_INLINE constexpr explicit nullopt_t() = default; }; // nullopt that can be used and returned. diff --git a/libc/src/__support/CPP/span.h b/libc/src/__support/CPP/span.h --- a/libc/src/__support/CPP/span.h +++ b/libc/src/__support/CPP/span.h @@ -46,7 +46,7 @@ using const_reference = const T &; using iterator = T *; - static constexpr size_type dynamic_extent = -1; + LIBC_INLINE_VAR static constexpr size_type dynamic_extent = -1; LIBC_INLINE constexpr span() : span_data(nullptr), span_size(0) {} diff --git a/libc/src/__support/CPP/string_view.h b/libc/src/__support/CPP/string_view.h --- a/libc/src/__support/CPP/string_view.h +++ b/libc/src/__support/CPP/string_view.h @@ -26,7 +26,7 @@ const char *Data; size_t Len; - static size_t min(size_t A, size_t B) { return A <= B ? A : B; } + LIBC_INLINE static size_t min(size_t A, size_t B) { return A <= B ? A : B; } LIBC_INLINE static int compareMemory(const char *Lhs, const char *Rhs, size_t Length) { diff --git a/libc/src/__support/CPP/type_traits.h b/libc/src/__support/CPP/type_traits.h --- a/libc/src/__support/CPP/type_traits.h +++ b/libc/src/__support/CPP/type_traits.h @@ -68,7 +68,8 @@ template using remove_cvref_t = typename remove_cvref::type; namespace details { -template constexpr bool is_unqualified_any_of() { +template +LIBC_INLINE constexpr bool is_unqualified_any_of() { return (... || is_same_v, Args>); } } // namespace details diff --git a/libc/src/__support/common.h b/libc/src/__support/common.h --- a/libc/src/__support/common.h +++ b/libc/src/__support/common.h @@ -39,7 +39,7 @@ namespace __llvm_libc { namespace internal { -constexpr bool same_string(char const *lhs, char const *rhs) { +LIBC_INLINE constexpr bool same_string(char const *lhs, char const *rhs) { for (; *lhs || *rhs; ++lhs, ++rhs) if (*lhs != *rhs) return false; diff --git a/libc/src/__support/ctype_utils.h b/libc/src/__support/ctype_utils.h --- a/libc/src/__support/ctype_utils.h +++ b/libc/src/__support/ctype_utils.h @@ -9,6 +9,8 @@ #ifndef LLVM_LIBC_SRC_SUPPORT_CTYPE_UTILS_H #define LLVM_LIBC_SRC_SUPPORT_CTYPE_UTILS_H +#include "src/__support/macros/attributes.h" + namespace __llvm_libc { namespace internal { @@ -18,25 +20,35 @@ // of a function call by inlining them. // ------------------------------------------------------ -static constexpr bool isalpha(unsigned ch) { return (ch | 32) - 'a' < 26; } +LIBC_INLINE static constexpr bool isalpha(unsigned ch) { + return (ch | 32) - 'a' < 26; +} -static constexpr bool isdigit(unsigned ch) { return (ch - '0') < 10; } +LIBC_INLINE static constexpr bool isdigit(unsigned ch) { + return (ch - '0') < 10; +} -static constexpr bool isalnum(unsigned ch) { +LIBC_INLINE static constexpr bool isalnum(unsigned ch) { return isalpha(ch) || isdigit(ch); } -static constexpr bool isgraph(unsigned ch) { return 0x20 < ch && ch < 0x7f; } +LIBC_INLINE static constexpr bool isgraph(unsigned ch) { + return 0x20 < ch && ch < 0x7f; +} -static constexpr bool islower(unsigned ch) { return (ch - 'a') < 26; } +LIBC_INLINE static constexpr bool islower(unsigned ch) { + return (ch - 'a') < 26; +} -static constexpr bool isupper(unsigned ch) { return (ch - 'A') < 26; } +LIBC_INLINE static constexpr bool isupper(unsigned ch) { + return (ch - 'A') < 26; +} -static constexpr bool isspace(unsigned ch) { +LIBC_INLINE static constexpr bool isspace(unsigned ch) { return ch == ' ' || (ch - '\t') < 5; } -static constexpr int tolower(int ch) { +LIBC_INLINE static constexpr int tolower(int ch) { if (isupper(ch)) return ch + ('a' - 'A'); return ch; diff --git a/libc/src/string/memory_utils/inline_memmem.h b/libc/src/string/memory_utils/inline_memmem.h --- a/libc/src/string/memory_utils/inline_memmem.h +++ b/libc/src/string/memory_utils/inline_memmem.h @@ -9,14 +9,16 @@ #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_MEMMEM_H #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_MEMMEM_H +#include "src/__support/macros/attributes.h" + #include namespace __llvm_libc { template -constexpr static void *inline_memmem(const void *haystack, size_t haystack_len, - const void *needle, size_t needle_len, - Comp &&comp) { +LIBC_INLINE constexpr static void * +inline_memmem(const void *haystack, size_t haystack_len, const void *needle, + size_t needle_len, Comp &&comp) { // TODO: simple brute force implementation. This can be // improved upon using well known string matching algorithms. if (!needle_len) diff --git a/libc/src/string/memory_utils/utils.h b/libc/src/string/memory_utils/utils.h --- a/libc/src/string/memory_utils/utils.h +++ b/libc/src/string/memory_utils/utils.h @@ -82,8 +82,7 @@ } // Returns the same pointer but notifies the compiler that it is aligned. -template -LIBC_INLINE T *assume_aligned(T *ptr) { +template LIBC_INLINE T *assume_aligned(T *ptr) { return reinterpret_cast(__builtin_assume_aligned(ptr, alignment)); } @@ -117,10 +116,10 @@ #ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE __builtin_memcpy_inline(dst, src, Size); #else -// In memory functions `memcpy_inline` is instantiated several times with -// different value of the Size parameter. This doesn't play well with GCC's -// Value Range Analysis that wrongly detects out of bounds accesses. We disable -// the 'array-bounds' warning for the purpose of this function. + // In memory functions `memcpy_inline` is instantiated several times with + // different value of the Size parameter. This doesn't play well with GCC's + // Value Range Analysis that wrongly detects out of bounds accesses. We + // disable the 'array-bounds' warning for the purpose of this function. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Warray-bounds" for (size_t i = 0; i < Size; ++i) @@ -139,20 +138,21 @@ // Can only be constructed from a T. template , bool> = 0> - StrictIntegralType(U value) : value(value) {} + LIBC_INLINE StrictIntegralType(U value) : value(value) {} // Allows using the type in an if statement. - explicit operator bool() const { return value; } + LIBC_INLINE explicit operator bool() const { return value; } // If type is unsigned (bcmp) we allow bitwise OR operations. - StrictIntegralType operator|(const StrictIntegralType &Rhs) const { + LIBC_INLINE StrictIntegralType + operator|(const StrictIntegralType &Rhs) const { static_assert(!cpp::is_signed_v); return value | Rhs.value; } // For interation with the C API we allow explicit conversion back to the // `int` type. - explicit operator int() const { + LIBC_INLINE explicit operator int() const { // bit_cast makes sure that T and int have the same size. return cpp::bit_cast(value); } @@ -252,7 +252,7 @@ // be aligned. // e.g. load_aligned(ptr); template -ValueType load_aligned(CPtr src) { +LIBC_INLINE ValueType load_aligned(CPtr src) { static_assert(sizeof(ValueType) >= (sizeof(T) + ... + sizeof(TS))); const ValueType value = load(assume_aligned(src)); if constexpr (sizeof...(TS) > 0) { @@ -271,14 +271,14 @@ // Alias for loading a 'uint32_t'. template -auto load32_aligned(CPtr src, size_t offset) { +LIBC_INLINE auto load32_aligned(CPtr src, size_t offset) { static_assert((sizeof(T) + ... + sizeof(TS)) == sizeof(uint32_t)); return load_aligned(src + offset); } // Alias for loading a 'uint64_t'. template -auto load64_aligned(CPtr src, size_t offset) { +LIBC_INLINE auto load64_aligned(CPtr src, size_t offset) { static_assert((sizeof(T) + ... + sizeof(TS)) == sizeof(uint64_t)); return load_aligned(src + offset); } @@ -287,7 +287,7 @@ // to be aligned. // e.g. store_aligned(value, ptr); template -void store_aligned(ValueType value, Ptr dst) { +LIBC_INLINE void store_aligned(ValueType value, Ptr dst) { static_assert(sizeof(ValueType) >= (sizeof(T) + ... + sizeof(TS))); constexpr size_t shift = sizeof(T) * 8; if constexpr (Endian::IS_LITTLE) { @@ -306,14 +306,14 @@ // Alias for storing a 'uint32_t'. template -void store32_aligned(uint32_t value, Ptr dst, size_t offset) { +LIBC_INLINE void store32_aligned(uint32_t value, Ptr dst, size_t offset) { static_assert((sizeof(T) + ... + sizeof(TS)) == sizeof(uint32_t)); store_aligned(value, dst + offset); } // Alias for storing a 'uint64_t'. template -void store64_aligned(uint64_t value, Ptr dst, size_t offset) { +LIBC_INLINE void store64_aligned(uint64_t value, Ptr dst, size_t offset) { static_assert((sizeof(T) + ... + sizeof(TS)) == sizeof(uint64_t)); store_aligned(value, dst + offset); } @@ -340,7 +340,7 @@ // Same as align_p1_to_next_boundary above but with a single pointer instead. template -void align_to_next_boundary(T1 *&p1, size_t &count) { +LIBC_INLINE void align_to_next_boundary(T1 *&p1, size_t &count) { CPtr dummy; align_p1_to_next_boundary(p1, dummy, count); } @@ -351,8 +351,8 @@ // Same as align_p1_to_next_boundary but allows for aligning p2 instead of p1. // Precondition: &p1 != &p2 template -void align_to_next_boundary(T1 *__restrict &p1, T2 *__restrict &p2, - size_t &count) { +LIBC_INLINE void align_to_next_boundary(T1 *__restrict &p1, T2 *__restrict &p2, + size_t &count) { if constexpr (AlignOn == Arg::P1) align_p1_to_next_boundary(p1, p2, count); else if constexpr (AlignOn == Arg::P2) @@ -362,7 +362,8 @@ } template struct AlignHelper { - AlignHelper(CPtr ptr) : offset_(distance_to_next_aligned(ptr)) {} + LIBC_INLINE AlignHelper(CPtr ptr) + : offset_(distance_to_next_aligned(ptr)) {} LIBC_INLINE bool not_aligned() const { return offset_ != SIZE; } LIBC_INLINE uintptr_t offset() const { return offset_; } diff --git a/libc/src/string/string_utils.h b/libc/src/string/string_utils.h --- a/libc/src/string/string_utils.h +++ b/libc/src/string/string_utils.h @@ -23,7 +23,7 @@ namespace __llvm_libc { namespace internal { -template constexpr Word repeat_byte(Word byte) { +template LIBC_INLINE constexpr Word repeat_byte(Word byte) { constexpr size_t BITS_IN_BYTE = 8; constexpr size_t BYTE_MASK = 0xff; Word result = 0; @@ -49,7 +49,7 @@ // with the inverse of the original byte. This means that any byte that had the // high bit set will no longer have it set, narrowing the list of bytes which // result in non-zero values to just the zero byte. -template constexpr bool has_zeroes(Word block) { +template LIBC_INLINE constexpr bool has_zeroes(Word block) { constexpr Word LOW_BITS = repeat_byte(0x01); constexpr Word HIGH_BITS = repeat_byte(0x80); Word subtracted = block - LOW_BITS;