[libc] Make string functions buildable with GCC

[gchatelet]

@sivachandra can you have a look ?

[gchatelet]

rebase

[sivachandra]

libc/src/string/memory_utils/bcmp_implementations.h

103

What was the problem with these constexpr conditionals? Because the inline_* functions are now potentially excluded by the preprocessor, they could be invisible at compile time?

[gchatelet]

libc/src/string/memory_utils/bcmp_implementations.h

103

avx512 functions requires a specific ABI. When compiling op_x86.h without avx512 support, GCC complains about these avx512 specific functions not having the right ABI -> I had to selectively exclude them using the preprocessor. Now these functions do not exist anymore when not compiling for avx512, so we cannot refer to them in the XXX_implementations_h files.

Using if constexpr would fail since the code still needs to be valid (even if disabled) so I had to use the preprocessor here as well.

This is far from ideal, it adds a lot of visual clutter.

Another option would be to disable the body of the function in op_x86.h instead of the function itself. So the function would still exist and could be referred to through if constexpr constructs.
I'll give it a try.

[gchatelet]

• Disable function body instead of the function itself.

[gchatelet]

@sivachandra let me know if you prefer this version or the previous one.

[gchatelet]

• Prevent warning about unused variables.

[sivachandra]

In D139939#4000931, @gchatelet wrote:

@sivachandra let me know if you prefer this version or the previous one.

LGTM; Could we make the alternate paths simply a failing static_assert?

[gchatelet]

In D139939#4004243, @sivachandra wrote:

In D139939#4000931, @gchatelet wrote:

@sivachandra let me know if you prefer this version or the previous one.

LGTM; Could we make the alternate paths simply a failing static_assert?

I wish I could but this would result in a compilation failure.
There's a proposal to support this use case but it's not currently implemented.
https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p2593r0.html

Let me know if you prefer the preprocessor solution that adds more safety at the cost of less readable code.

[sivachandra]

In D139939#4004721, @gchatelet wrote:

Let me know if you prefer the preprocessor solution that adds more safety at the cost of less readable code.

This is fine, thanks!

[gchatelet]

FTR https://github.com/llvm/llvm-project/issues/59368

libc/src/__support/compiler_features.h

 // Compiler feature-detection.
 // clang.llvm.org/docs/LanguageExtensions.html#has-feature-and-has-extension
 #if defined(LLVM_LIBC_COMPILER_CLANG)
 #define LLVM_LIBC_HAS_FEATURE(FEATURE) __has_feature(FEATURE)
 #else
 #define LLVM_LIBC_HAS_FEATURE(FEATURE) 0
 #endif
 
+#if defined(LLVM_LIBC_COMPILER_CLANG)
+#define LLVM_LIBC_LOOP_NOUNROLL _Pragma("nounroll")
+#elif defined(LLVM_LIBC_COMPILER_GCC)
+#define LLVM_LIBC_LOOP_NOUNROLL _Pragma("GCC unroll 0")
+#else
+#define LLVM_LIBC_LOOP_NOUNROLL
+#endif
+
 #endif // LLVM_LIBC_SUPPORT_COMPILER_FEATURES_H

libc/src/string/memory_utils/bcmp_implementations.h

 #include "src/string/memory_utils/op_x86.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
 [[maybe_unused]] static inline BcmpReturnType
 inline_bcmp_embedded_tiny(CPtr p1, CPtr p2, size_t count) {
-#pragma nounroll
+  LLVM_LIBC_LOOP_NOUNROLL
   for (size_t offset = 0; offset < count; ++offset)
     if (auto value = generic::Bcmp<1>::block(p1 + offset, p2 + offset))
       return value;
   return BcmpReturnType::ZERO();
 }
 
 #if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64)
 [[maybe_unused]] static inline BcmpReturnType
[[maybe_unused]] static inline BcmpReturnType inline_bcmp_x86(CPtr p1, CPtr p2,
   if (count == 2)
     return generic::Bcmp<2>::block(p1, p2);
   if (count <= 4)
     return generic::Bcmp<2>::head_tail(p1, p2, count);
   if (count <= 8)
     return generic::Bcmp<4>::head_tail(p1, p2, count);
   if (count <= 16)
     return generic::Bcmp<8>::head_tail(p1, p2, count);
   if constexpr (x86::kAvx512BW)
     return inline_bcmp_x86_avx512bw_gt16(p1, p2, count);
   else if constexpr (x86::kAvx2)
     return inline_bcmp_x86_avx2_gt16(p1, p2, count);
   else if constexpr (x86::kSse2)
     return inline_bcmp_x86_sse2_gt16(p1, p2, count);
   else
     return inline_bcmp_generic_gt16(p1, p2, count);
 }

libc/src/string/memory_utils/memcmp_implementations.h

 #include "src/string/memory_utils/op_x86.h"
 #include "src/string/memory_utils/utils.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 [[maybe_unused]] static inline MemcmpReturnType
 inline_memcmp_embedded_tiny(CPtr p1, CPtr p2, size_t count) {
-#pragma nounroll
+  LLVM_LIBC_LOOP_NOUNROLL
   for (size_t offset = 0; offset < count; ++offset)
     if (auto value = generic::Memcmp<1>::block(p1 + offset, p2 + offset))
       return value;
   return MemcmpReturnType::ZERO();
 }
 
 #if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64)
 [[maybe_unused]] static inline MemcmpReturnType
if (count <= 128)
     return x86::avx512bw::Memcmp<64>::head_tail(p1, p2, count);
   if (unlikely(count >= 384)) {
     if (auto value = x86::avx512bw::Memcmp<64>::block(p1, p2))
       return value;
     align_to_next_boundary<64, Arg::P1>(p1, p2, count);
   }
   return x86::avx512bw::Memcmp<64>::loop_and_tail(p1, p2, count);
 }
+
 #endif // defined(LLVM_LIBC_ARCH_X86)
 
 #if defined(LLVM_LIBC_ARCH_AARCH64)
 [[maybe_unused]] static inline MemcmpReturnType
 inline_memcmp_aarch64_neon_gt16(CPtr p1, CPtr p2, size_t count) {
   if (unlikely(count >= 128)) { // [128, ∞]
     if (auto value = generic::Memcmp<16>::block(p1, p2))
       return value;

libc/src/string/memory_utils/memcpy_implementations.h

 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
 [[maybe_unused]] static inline void
 inline_memcpy_embedded_tiny(Ptr __restrict dst, CPtr __restrict src,
                             size_t count) {
-#pragma nounroll
+  LLVM_LIBC_LOOP_NOUNROLL
   for (size_t offset = 0; offset < count; ++offset)
     builtin::Memcpy<1>::block(dst + offset, src + offset);
 }
 
 #if defined(LLVM_LIBC_ARCH_X86)
 [[maybe_unused]] static inline void
 inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) {
   if (count == 0)

libc/src/string/memory_utils/memmove_implementations.h

 
 namespace __llvm_libc {
 
 [[maybe_unused]] static inline void
 inline_memmove_embedded_tiny(Ptr dst, CPtr src, size_t count) {
   if ((count == 0) || (dst == src))
     return;
   if (dst < src) {
-#pragma nounroll
+    LLVM_LIBC_LOOP_NOUNROLL
     for (size_t offset = 0; offset < count; ++offset)
       builtin::Memcpy<1>::block(dst + offset, src + offset);
   } else {
-#pragma nounroll
+    LLVM_LIBC_LOOP_NOUNROLL
     for (ptrdiff_t offset = count - 1; offset >= 0; --offset)
       builtin::Memcpy<1>::block(dst + offset, src + offset);
   }
 }
 
 template <size_t MaxSize>
 [[maybe_unused]] static inline void inline_memmove_generic(Ptr dst, CPtr src,
                                                            size_t count) {

libc/src/string/memory_utils/memset_implementations.h

 #include "src/string/memory_utils/utils.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
 [[maybe_unused]] inline static void
 inline_memset_embedded_tiny(Ptr dst, uint8_t value, size_t count) {
-#pragma nounroll
+  LLVM_LIBC_LOOP_NOUNROLL
   for (size_t offset = 0; offset < count; ++offset)
     generic::Memset<1, 1>::block(dst + offset, value);
 }
 
 #if defined(LLVM_LIBC_ARCH_X86)
 template <size_t MaxSize>
 [[maybe_unused]] inline static void inline_memset_x86(Ptr dst, uint8_t value,
                                                       size_t count) {

libc/src/string/memory_utils/op_builtin.h

 ///////////////////////////////////////////////////////////////////////////////
 // Memcpy
 template <size_t Size> struct Memcpy {
   static constexpr size_t SIZE = Size;
   static inline void block(Ptr __restrict dst, CPtr __restrict src) {
 #ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE
     return __builtin_memcpy_inline(dst, src, SIZE);
 #else
-    deferred_static_assert("Missing __builtin_memcpy_inline");
-    (void)dst;
-    (void)src;
+    // The codegen may be suboptimal.
+    for (size_t i = 0; i < Size; ++i)
+      dst[i] = src[i];
 #endif
   }
 
   static inline void tail(Ptr __restrict dst, CPtr __restrict src,
                           size_t count) {
     block(dst + count - SIZE, src + count - SIZE);
   }
 

libc/src/string/memory_utils/op_generic.h

 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_GENERIC_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_GENERIC_H
 
 #include "src/__support/CPP/array.h"
 #include "src/__support/CPP/type_traits.h"
 #include "src/__support/common.h"
+#include "src/__support/compiler_features.h"
 #include "src/__support/endian.h"
 #include "src/string/memory_utils/op_builtin.h"
 #include "src/string/memory_utils/utils.h"
 
 #include <stdint.h>
 
 namespace __llvm_libc::generic {
 
template <typename T> struct ScalarType {
   static inline void store(Ptr dst, Type value) {
     ::__llvm_libc::store<Type>(dst, value);
   }
   static inline Type splat(uint8_t value) {
     return Type(~0) / Type(0xFF) * Type(value);
   }
 };
 
+// GCC can only take literals as __vector_size__ argument so we have to use
+// template specialization.
+template <size_t Size> struct VectorValueType {};
+template <> struct VectorValueType<1> {
+  using type = uint8_t __attribute__((__vector_size__(1)));
+};
+template <> struct VectorValueType<2> {
+  using type = uint8_t __attribute__((__vector_size__(2)));
+};
+template <> struct VectorValueType<4> {
+  using type = uint8_t __attribute__((__vector_size__(4)));
+};
+template <> struct VectorValueType<8> {
+  using type = uint8_t __attribute__((__vector_size__(8)));
+};
+template <> struct VectorValueType<16> {
+  using type = uint8_t __attribute__((__vector_size__(16)));
+};
+template <> struct VectorValueType<32> {
+  using type = uint8_t __attribute__((__vector_size__(32)));
+};
+template <> struct VectorValueType<64> {
+  using type = uint8_t __attribute__((__vector_size__(64)));
+};
+
 // Implements load, store and splat for vector types.
 template <size_t Size> struct VectorType {
-  using Type = uint8_t __attribute__((__vector_size__(Size)));
+  using Type = typename VectorValueType<Size>::type;
   static inline Type load(CPtr src) { return ::__llvm_libc::load<Type>(src); }
   static inline void store(Ptr dst, Type value) {
     ::__llvm_libc::store<Type>(dst, value);
   }
   static inline Type splat(uint8_t value) {
     Type Out;
     // This for loop is optimized out for vector types.
     for (size_t i = 0; i < Size; ++i)
template <size_t Size, size_t MaxSize> struct Memmove {
   // [___________________LLLLLLLL_________]
   // [_________________SSSSSSSS___________]
   // [_______________________SSSSSSSS_____]
   static inline void loop_and_tail_forward(Ptr dst, CPtr src, size_t count) {
     static_assert(Size > 1, "a loop of size 1 does not need tail");
     const size_t tail_offset = count - Size;
     const auto tail_value = T::load(src + tail_offset);
     size_t offset = 0;
-#pragma nounroll
+    LLVM_LIBC_LOOP_NOUNROLL
     do {
       block(dst + offset, src + offset);
       offset += Size;
     } while (offset < count - Size);
     T::store(dst + tail_offset, tail_value);
   }
 
   // Move backward suitable when dst > src. We load the head bytes before
template <size_t Size, size_t MaxSize> struct Memmove {
   // [___________________SSSSSSSS_________]
   // [_________LLLLLLLL___________________]
   // [___________SSSSSSSS_________________]
   // [_____SSSSSSSS_______________________]
   static inline void loop_and_tail_backward(Ptr dst, CPtr src, size_t count) {
     static_assert(Size > 1, "a loop of size 1 does not need tail");
     const auto head_value = T::load(src);
     ptrdiff_t offset = count - Size;
-#pragma nounroll
+    LLVM_LIBC_LOOP_NOUNROLL
     do {
       block(dst + offset, src + offset);
       offset -= Size;
     } while (offset >= 0);
     T::store(dst, head_value);
   }
 };
 
 } // namespace __llvm_libc::generic
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_GENERIC_H

libc/src/string/memory_utils/op_x86.h

do {
       offset += Size;
     } while (offset < count - Size);
     return tail(p1, p2, count);
   }
 };
 
 namespace sse2 {
 static inline BcmpReturnType bcmp16(CPtr p1, CPtr p2) {
+#if defined(__SSE2__)
   using T = char __attribute__((__vector_size__(16)));
   // A mask indicating which bytes differ after loading 16 bytes from p1 and p2.
   const int mask =
       _mm_movemask_epi8(cpp::bit_cast<__m128i>(load<T>(p1) != load<T>(p2)));
   return static_cast<uint32_t>(mask);
+#else
+  (void)p1;
+  (void)p2;
+  return BcmpReturnType::ZERO();
+#endif // defined(__SSE2__)
 }
 template <size_t Size> using Bcmp = BcmpImpl<Size, 16, bcmp16>;
 } // namespace sse2
 
 namespace avx2 {
 static inline BcmpReturnType bcmp32(CPtr p1, CPtr p2) {
+#if defined(__AVX2__)
   using T = char __attribute__((__vector_size__(32)));
   // A mask indicating which bytes differ after loading 32 bytes from p1 and p2.
   const int mask =
       _mm256_movemask_epi8(cpp::bit_cast<__m256i>(load<T>(p1) != load<T>(p2)));
   // _mm256_movemask_epi8 returns an int but it is to be interpreted as a 32-bit
   // mask.
   return static_cast<uint32_t>(mask);
+#else
+  (void)p1;
+  (void)p2;
+  return BcmpReturnType::ZERO();
+#endif // defined(__AVX2__)
 }
 template <size_t Size> using Bcmp = BcmpImpl<Size, 32, bcmp32>;
 } // namespace avx2
 
 namespace avx512bw {
 static inline BcmpReturnType bcmp64(CPtr p1, CPtr p2) {
+#if defined(__AVX512BW__)
   using T = char __attribute__((__vector_size__(64)));
   // A mask indicating which bytes differ after loading 64 bytes from p1 and p2.
-  const uint64_t mask = _mm512_cmpneq_epi8_mask(load<T>(p1), load<T>(p2));
+  const uint64_t mask = _mm512_cmpneq_epi8_mask(
+      cpp::bit_cast<__m512i>(load<T>(p1)), cpp::bit_cast<__m512i>(load<T>(p2)));
   const bool mask_is_set = mask != 0;
   return static_cast<uint32_t>(mask_is_set);
+#else
+  (void)p1;
+  (void)p2;
+  return BcmpReturnType::ZERO();
+#endif // defined(__AVX512BW__)
 }
 template <size_t Size> using Bcmp = BcmpImpl<Size, 64, bcmp64>;
 } // namespace avx512bw
 
 // Assuming that the mask is non zero, the index of the first mismatching byte
 // is the number of trailing zeros in the mask. Trailing zeros and not leading
 // zeros because the x86 architecture is little endian.
 static inline MemcmpReturnType char_diff_no_zero(CPtr p1, CPtr p2,
do {
       offset += Size;
     } while (offset < count - Size);
     return tail(p1, p2, count);
   }
 };
 
 namespace sse2 {
 static inline MemcmpReturnType memcmp16(CPtr p1, CPtr p2) {
+#if defined(__SSE2__)
   using T = char __attribute__((__vector_size__(16)));
   // A mask indicating which bytes differ after loading 16 bytes from p1 and p2.
   if (int mask =
           _mm_movemask_epi8(cpp::bit_cast<__m128i>(load<T>(p1) != load<T>(p2))))
     return char_diff_no_zero(p1, p2, mask);
   return MemcmpReturnType::ZERO();
+#else
+  (void)p1;
+  (void)p2;
+  return MemcmpReturnType::ZERO();
+#endif // defined(__SSE2__)
 }
 template <size_t Size> using Memcmp = MemcmpImpl<Size, 16, memcmp16, bcmp16>;
 } // namespace sse2
 
 namespace avx2 {
 static inline MemcmpReturnType memcmp32(CPtr p1, CPtr p2) {
+#if defined(__AVX2__)
   using T = char __attribute__((__vector_size__(32)));
   // A mask indicating which bytes differ after loading 32 bytes from p1 and p2.
   if (int mask = _mm256_movemask_epi8(
           cpp::bit_cast<__m256i>(load<T>(p1) != load<T>(p2))))
     return char_diff_no_zero(p1, p2, mask);
   return MemcmpReturnType::ZERO();
+#else
+  (void)p1;
+  (void)p2;
+  return MemcmpReturnType::ZERO();
+#endif // defined(__AVX2__)
 }
 template <size_t Size> using Memcmp = MemcmpImpl<Size, 32, memcmp32, bcmp32>;
 } // namespace avx2
 
 namespace avx512bw {
 static inline MemcmpReturnType memcmp64(CPtr p1, CPtr p2) {
+#if defined(__AVX512BW__)
   using T = char __attribute__((__vector_size__(64)));
   // A mask indicating which bytes differ after loading 64 bytes from p1 and p2.
-  if (uint64_t mask = _mm512_cmpneq_epi8_mask(load<T>(p1), load<T>(p2)))
+  if (uint64_t mask =
+          _mm512_cmpneq_epi8_mask(cpp::bit_cast<__m512i>(load<T>(p1)),
+                                  cpp::bit_cast<__m512i>(load<T>(p2))))
     return char_diff_no_zero(p1, p2, mask);
   return MemcmpReturnType::ZERO();
+#else
+  (void)p1;
+  (void)p2;
+  return MemcmpReturnType::ZERO();
+#endif // defined(__AVX512BW__)
 }
 template <size_t Size> using Memcmp = MemcmpImpl<Size, 64, memcmp64, bcmp64>;
 } // namespace avx512bw
 
 } // namespace __llvm_libc::x86
 
 #endif // LLVM_LIBC_ARCH_X86_64
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_X86_H