[libc] Switch to new implementation of mem* functions

[gchatelet]

Summary

The new framework makes it explicit which processor feature is being
used and allows for easier per platform customization:

• ARM cpu now uses trivial implementations to reduce code size.
• Memcmp, Bcmp and Memmove have been optimized for x86
• Bcmp has been optimized for aarch64.

This is a reland of https://reviews.llvm.org/D135134 (b3f1d58)

[courbet]

libc/src/string/memory_utils/memcpy_implementations.h

127

why is ARM always embedded_tiny ? Shouldn't that be orthogonal ? I could imagine some tiny aarch64 processors...

libc/src/string/memory_utils/memset_implementations.h

108

ditto

[gchatelet]

libc/src/string/memory_utils/memcpy_implementations.h

127

Fair enough, for now this is not customizable but it is expected to be in the future.

There has been some discussions on discourse but we need to better define what would be a reasonable default implementation.

Also it is unclear whether we want to customize per function or for the library as a whole

I will hold this patch until I'm clear whether this will impact our users or not.

[gchatelet]

libc/src/string/memory_utils/memcpy_implementations.h

127

@sivachandra confirmed that ARM support is minimal for now and that we don't have active users . aarch64 is solely targeted at ARM servers. Of course this can change but I think the patch is safe to land as it is now. Let's refine the needs and iterate in subsequent patches.

[gchatelet]

Using b3f1d58a131eb546aaf1ac165c77ccb89c40d758 as base fo diff

[gchatelet]

@courbet use this link to check the diff with the previously submitted patch.
Also let's wait for D136865 to be in before submitting this one.

[courbet]

libc/src/string/memory_utils/memcmp_implementations.h

97

This is unfortunate :(

This should be caught by unit tests, can you add one ?

[gchatelet]

libc/src/string/memory_utils/memcmp_implementations.h

97

Yep this was done in D136865

[gchatelet]

I'm submitting this patch once again. 🤞

libc/src/stdio/printf_core/string_writer.cpp

void StringWriter::write(cpp::string_view new_string) {
   }
 }
 
 void StringWriter::write(char new_char, size_t len) {
   if (len > available_capacity)
     len = available_capacity;
 
   if (len > 0) {
-    inline_memset(cur_buffer, new_char, len);
+    inline_memset(cur_buffer, static_cast<uint8_t>(new_char), len);
     cur_buffer += len;
     available_capacity -= len;
   }
 }
 
 void StringWriter::write(char new_char) {
   if (1 > available_capacity)
     return;

libc/src/string/bcmp.cpp

 //===-- Implementation of bcmp --------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/bcmp.h"
 #include "src/__support/common.h"
 #include "src/string/memory_utils/bcmp_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(int, bcmp,
                    (const void *lhs, const void *rhs, size_t count)) {
-  return inline_bcmp(static_cast<const char *>(lhs),
-                     static_cast<const char *>(rhs), count);
+  return inline_bcmp(lhs, rhs, count);
 }
 
 } // namespace __llvm_libc

libc/src/string/memcmp.cpp

 #include "src/string/memory_utils/memcmp_implementations.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(int, memcmp,
                    (const void *lhs, const void *rhs, size_t count)) {
-  return inline_memcmp(static_cast<const char *>(lhs),
-                       static_cast<const char *>(rhs), count);
+  return inline_memcmp(lhs, rhs, count);
 }
 
 } // namespace __llvm_libc

libc/src/string/memcpy.cpp

 #include "src/__support/common.h"
 #include "src/string/memory_utils/memcpy_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(void *, memcpy,
                    (void *__restrict dst, const void *__restrict src,
                     size_t size)) {
-  inline_memcpy(reinterpret_cast<char *>(dst),
-                reinterpret_cast<const char *>(src), size);
+  inline_memcpy(dst, src, size);
   return dst;
 }
 
 } // namespace __llvm_libc

libc/src/string/memmove.cpp

 //===-- Implementation of memmove -----------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/memmove.h"
 
 #include "src/__support/common.h"
-#include "src/__support/integer_operations.h"
-#include "src/string/memory_utils/elements.h"
+#include "src/string/memory_utils/op_aarch64.h"
+#include "src/string/memory_utils/op_builtin.h"
+#include "src/string/memory_utils/op_generic.h"
+#include "src/string/memory_utils/op_x86.h"
 #include <stddef.h> // size_t, ptrdiff_t
 
+#include <stdio.h>
+
 namespace __llvm_libc {
 
-static inline void inline_memmove(char *dst, const char *src, size_t count) {
-  using namespace __llvm_libc::scalar;
+[[maybe_unused]] static inline void
+inline_memmove_embedded_tiny(Ptr dst, CPtr src, size_t count) {
+  if ((count == 0) || (dst == src))
+    return;
+  if (dst < src) {
+#pragma nounroll
+    for (size_t offset = 0; offset < count; ++offset)
+      builtin::Memcpy<1>::block(dst + offset, src + offset);
+  } else {
+#pragma nounroll
+    for (ptrdiff_t offset = count - 1; offset >= 0; --offset)
+      builtin::Memcpy<1>::block(dst + offset, src + offset);
+  }
+}
+
+template <size_t MaxSize>
+[[maybe_unused]] static inline void inline_memmove_generic(Ptr dst, CPtr src,
+                                                           size_t count) {
   if (count == 0)
     return;
   if (count == 1)
-    return move<_1>(dst, src);
+    return generic::Memmove<1, MaxSize>::block(dst, src);
   if (count <= 4)
-    return move<HeadTail<_2>>(dst, src, count);
+    return generic::Memmove<2, MaxSize>::head_tail(dst, src, count);
   if (count <= 8)
-    return move<HeadTail<_4>>(dst, src, count);
+    return generic::Memmove<4, MaxSize>::head_tail(dst, src, count);
   if (count <= 16)
-    return move<HeadTail<_8>>(dst, src, count);
+    return generic::Memmove<8, MaxSize>::head_tail(dst, src, count);
   if (count <= 32)
-    return move<HeadTail<_16>>(dst, src, count);
+    return generic::Memmove<16, MaxSize>::head_tail(dst, src, count);
   if (count <= 64)
-    return move<HeadTail<_32>>(dst, src, count);
+    return generic::Memmove<32, MaxSize>::head_tail(dst, src, count);
   if (count <= 128)
-    return move<HeadTail<_64>>(dst, src, count);
+    return generic::Memmove<64, MaxSize>::head_tail(dst, src, count);
+  if (dst < src) {
+    generic::Memmove<32, MaxSize>::template align_forward<Arg::Src>(dst, src,
+                                                                    count);
+    return generic::Memmove<64, MaxSize>::loop_and_tail_forward(dst, src,
+                                                                count);
+  } else {
+    generic::Memmove<32, MaxSize>::template align_backward<Arg::Src>(dst, src,
+                                                                     count);
+    return generic::Memmove<64, MaxSize>::loop_and_tail_backward(dst, src,
+                                                                 count);
+  }
+}
 
-  using AlignedMoveLoop = Align<_16, Arg::Src>::Then<Loop<_64>>;
-  if (dst < src)
-    return move<AlignedMoveLoop>(dst, src, count);
-  else if (dst > src)
-    return move_backward<AlignedMoveLoop>(dst, src, count);
+static inline void inline_memmove(Ptr dst, CPtr src, size_t count) {
+#if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64)
+#if defined(LLVM_LIBC_ARCH_X86)
+  static constexpr size_t kMaxSize = x86::kAvx512F ? 64
+                                     : x86::kAvx   ? 32
+                                     : x86::kSse2  ? 16
+                                                   : 8;
+#elif defined(LLVM_LIBC_ARCH_AARCH64)
+  static constexpr size_t kMaxSize = aarch64::kNeon ? 16 : 8;
+#endif
+  // return inline_memmove_generic<kMaxSize>(dst, src, count);
+  if (count == 0)
+    return;
+  if (count == 1)
+    return generic::Memmove<1, kMaxSize>::block(dst, src);
+  if (count <= 4)
+    return generic::Memmove<2, kMaxSize>::head_tail(dst, src, count);
+  if (count <= 8)
+    return generic::Memmove<4, kMaxSize>::head_tail(dst, src, count);
+  if (count <= 16)
+    return generic::Memmove<8, kMaxSize>::head_tail(dst, src, count);
+  if (count <= 32)
+    return generic::Memmove<16, kMaxSize>::head_tail(dst, src, count);
+  if (count <= 64)
+    return generic::Memmove<32, kMaxSize>::head_tail(dst, src, count);
+  if (count <= 128)
+    return generic::Memmove<64, kMaxSize>::head_tail(dst, src, count);
+  if (dst < src) {
+    generic::Memmove<32, kMaxSize>::align_forward<Arg::Src>(dst, src, count);
+    return generic::Memmove<64, kMaxSize>::loop_and_tail_forward(dst, src,
+                                                                 count);
+  } else {
+    generic::Memmove<32, kMaxSize>::align_backward<Arg::Src>(dst, src, count);
+    return generic::Memmove<64, kMaxSize>::loop_and_tail_backward(dst, src,
+                                                                  count);
+  }
+#elif defined(LLVM_LIBC_ARCH_ARM)
+  return inline_memmove_embedded_tiny(dst, src, count);
+#else
+#error "Unsupported platform"
+#endif
 }
 
 LLVM_LIBC_FUNCTION(void *, memmove,
                    (void *dst, const void *src, size_t count)) {
-  inline_memmove(reinterpret_cast<char *>(dst),
-                 reinterpret_cast<const char *>(src), count);
+  inline_memmove(reinterpret_cast<Ptr>(dst), reinterpret_cast<CPtr>(src),
+                 count);
   return dst;
 }
 
 } // namespace __llvm_libc

libc/src/string/memory_utils/bcmp_implementations.h

 //===-- Implementation of bcmp --------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BCMP_IMPLEMENTATIONS_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BCMP_IMPLEMENTATIONS_H
 
 #include "src/__support/architectures.h"
 #include "src/__support/common.h"
-#include "src/string/memory_utils/elements.h"
+#include "src/string/memory_utils/op_aarch64.h"
+#include "src/string/memory_utils/op_builtin.h"
+#include "src/string/memory_utils/op_generic.h"
+#include "src/string/memory_utils/op_x86.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
-// Fixed-size difference between 'lhs' and 'rhs'.
-template <typename Element> bool differs(const char *lhs, const char *rhs) {
-  return !Element::equals(lhs, rhs);
+[[maybe_unused]] static inline BcmpReturnType
+inline_bcmp_embedded_tiny(CPtr p1, CPtr p2, size_t count) {
+#pragma nounroll
+  for (size_t offset = 0; offset < count; ++offset)
+    if (auto value = generic::Bcmp<1>::block(p1 + offset, p2 + offset))
+      return value;
+  return BcmpReturnType::ZERO();
 }
-// Runtime-size difference between 'lhs' and 'rhs'.
-template <typename Element>
-bool differs(const char *lhs, const char *rhs, size_t size) {
-  return !Element::equals(lhs, rhs, size);
+
+#if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64)
+[[maybe_unused]] static inline BcmpReturnType
+inline_bcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) {
+  if (count < 256)
+    return generic::Bcmp<16>::loop_and_tail(p1, p2, count);
+  if (auto value = generic::Bcmp<64>::block(p1, p2))
+    return value;
+  align_to_next_boundary<64, Arg::P1>(p1, p2, count);
+  return generic::Bcmp<64>::loop_and_tail(p1, p2, count);
 }
+#endif // defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64)
 
-static inline int inline_bcmp(const char *lhs, const char *rhs, size_t count) {
 #if defined(LLVM_LIBC_ARCH_X86)
-  using namespace ::__llvm_libc::x86;
-#elif defined(LLVM_LIBC_ARCH_AARCH64)
-  using namespace ::__llvm_libc::aarch64;
-#else
-  using namespace ::__llvm_libc::scalar;
-#endif
+[[maybe_unused]] static inline BcmpReturnType
+inline_bcmp_x86_sse2_gt16(CPtr p1, CPtr p2, size_t count) {
+  if (count <= 32)
+    return x86::sse2::Bcmp<16>::head_tail(p1, p2, count);
+  if (count < 256)
+    return x86::sse2::Bcmp<16>::loop_and_tail(p1, p2, count);
+  if (auto value = x86::sse2::Bcmp<16>::block(p1, p2))
+    return value;
+  align_to_next_boundary<16, Arg::P1>(p1, p2, count);
+  return x86::sse2::Bcmp<64>::loop_and_tail(p1, p2, count);
+}
+
+[[maybe_unused]] static inline BcmpReturnType
+inline_bcmp_x86_avx2_gt16(CPtr p1, CPtr p2, size_t count) {
+  if (count <= 32)
+    return x86::sse2::Bcmp<16>::head_tail(p1, p2, count);
+  if (count <= 64)
+    return x86::avx2::Bcmp<32>::head_tail(p1, p2, count);
+  if (count <= 128)
+    return x86::avx2::Bcmp<64>::head_tail(p1, p2, count);
+  if (unlikely(count >= 256)) {
+    if (auto value = x86::avx2::Bcmp<64>::block(p1, p2))
+      return value;
+    align_to_next_boundary<64, Arg::P1>(p1, p2, count);
+  }
+  return x86::avx2::Bcmp<64>::loop_and_tail(p1, p2, count);
+}
+
+[[maybe_unused]] static inline BcmpReturnType
+inline_bcmp_x86_avx512bw_gt16(CPtr p1, CPtr p2, size_t count) {
+  if (count <= 32)
+    return x86::sse2::Bcmp<16>::head_tail(p1, p2, count);
+  if (count <= 64)
+    return x86::avx2::Bcmp<32>::head_tail(p1, p2, count);
+  if (count <= 128)
+    return x86::avx512bw::Bcmp<64>::head_tail(p1, p2, count);
+  if (unlikely(count >= 256)) {
+    if (auto value = x86::avx512bw::Bcmp<64>::block(p1, p2))
+      return value;
+    align_to_next_boundary<64, Arg::P1>(p1, p2, count);
+  }
+  return x86::avx512bw::Bcmp<64>::loop_and_tail(p1, p2, count);
+}
+
+[[maybe_unused]] static inline BcmpReturnType inline_bcmp_x86(CPtr p1, CPtr p2,
+                                                              size_t count) {
   if (count == 0)
-    return 0;
+    return BcmpReturnType::ZERO();
   if (count == 1)
-    return differs<_1>(lhs, rhs);
+    return generic::Bcmp<1>::block(p1, p2);
   if (count == 2)
-    return differs<_2>(lhs, rhs);
-  if (count == 3)
-    return differs<_3>(lhs, rhs);
+    return generic::Bcmp<2>::block(p1, p2);
+  if (count <= 4)
+    return generic::Bcmp<2>::head_tail(p1, p2, count);
   if (count <= 8)
-    return differs<HeadTail<_4>>(lhs, rhs, count);
+    return generic::Bcmp<4>::head_tail(p1, p2, count);
   if (count <= 16)
-    return differs<HeadTail<_8>>(lhs, rhs, count);
-  if (count <= 32)
-    return differs<HeadTail<_16>>(lhs, rhs, count);
+    return generic::Bcmp<8>::head_tail(p1, p2, count);
+  if constexpr (x86::kAvx512BW)
+    return inline_bcmp_x86_avx512bw_gt16(p1, p2, count);
+  else if constexpr (x86::kAvx2)
+    return inline_bcmp_x86_avx2_gt16(p1, p2, count);
+  else if constexpr (x86::kSse2)
+    return inline_bcmp_x86_sse2_gt16(p1, p2, count);
+  else
+    return inline_bcmp_generic_gt16(p1, p2, count);
+}
+#endif // defined(LLVM_LIBC_ARCH_X86)
+
+#if defined(LLVM_LIBC_ARCH_AARCH64)
+[[maybe_unused]] static inline BcmpReturnType
+inline_bcmp_aarch64(CPtr p1, CPtr p2, size_t count) {
+  if (likely(count <= 32)) {
+    if (unlikely(count >= 16)) {
+      return generic::Bcmp<16>::head_tail(p1, p2, count);
+    }
+    switch (count) {
+    case 0:
+      return BcmpReturnType::ZERO();
+    case 1:
+      return generic::Bcmp<1>::block(p1, p2);
+    case 2:
+      return generic::Bcmp<2>::block(p1, p2);
+    case 3:
+      return generic::Bcmp<2>::head_tail(p1, p2, count);
+    case 4:
+      return generic::Bcmp<4>::block(p1, p2);
+    case 5:
+    case 6:
+    case 7:
+      return generic::Bcmp<4>::head_tail(p1, p2, count);
+    case 8:
+      return generic::Bcmp<8>::block(p1, p2);
+    case 9:
+    case 10:
+    case 11:
+    case 12:
+    case 13:
+    case 14:
+    case 15:
+      return generic::Bcmp<8>::head_tail(p1, p2, count);
+    }
+  }
+
   if (count <= 64)
-    return differs<HeadTail<_32>>(lhs, rhs, count);
-  if (count <= 128)
-    return differs<HeadTail<_64>>(lhs, rhs, count);
-  return differs<Align<_32>::Then<Loop<_32>>>(lhs, rhs, count);
+    return generic::Bcmp<32>::head_tail(p1, p2, count);
+
+  // Aligned loop if > 256, otherwise normal loop
+  if (count > 256) {
+    if (auto value = generic::Bcmp<32>::block(p1, p2))
+      return value;
+    align_to_next_boundary<16, Arg::P1>(p1, p2, count);
+  }
+  return generic::Bcmp<32>::loop_and_tail(p1, p2, count);
+}
+#endif // defined(LLVM_LIBC_ARCH_AARCH64)
+
+static inline BcmpReturnType inline_bcmp(CPtr p1, CPtr p2, size_t count) {
+#if defined(LLVM_LIBC_ARCH_X86)
+  return inline_bcmp_x86(p1, p2, count);
+#elif defined(LLVM_LIBC_ARCH_AARCH64)
+  return inline_bcmp_aarch64(p1, p2, count);
+#elif defined(LLVM_LIBC_ARCH_ARM)
+  return inline_bcmp_embedded_tiny(p1, p2, count);
+#else
+#error "Unsupported platform"
+#endif
+}
+
+static inline int inline_bcmp(const void *p1, const void *p2, size_t count) {
+  return static_cast<int>(inline_bcmp(reinterpret_cast<CPtr>(p1),
+                                      reinterpret_cast<CPtr>(p2), count));
 }
 
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BCMP_IMPLEMENTATIONS_H

libc/src/string/memory_utils/bzero_implementations.h

 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BZERO_IMPLEMENTATIONS_H
 
 #include "src/string/memory_utils/memset_implementations.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
-inline static void inline_bzero(char *dst, size_t count) {
+inline static void inline_bzero(Ptr dst, size_t count) {
   inline_memset(dst, 0, count);
 }
 
+inline static void inline_bzero(void *dst, size_t count) {
+  inline_bzero(reinterpret_cast<Ptr>(dst), count);
+}
+
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BZERO_IMPLEMENTATIONS_H

libc/src/string/memory_utils/memcmp_implementations.h

 //===-- Implementation of memcmp ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP_IMPLEMENTATIONS_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP_IMPLEMENTATIONS_H
 
 #include "src/__support/architectures.h"
 #include "src/__support/common.h"
-#include "src/string/memory_utils/elements.h"
+#include "src/string/memory_utils/op_aarch64.h"
+#include "src/string/memory_utils/op_builtin.h"
+#include "src/string/memory_utils/op_generic.h"
+#include "src/string/memory_utils/op_x86.h"
+#include "src/string/memory_utils/utils.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
+[[maybe_unused]] static inline MemcmpReturnType
+inline_memcmp_embedded_tiny(CPtr p1, CPtr p2, size_t count) {
+#pragma nounroll
+  for (size_t offset = 0; offset < count; ++offset)
+    if (auto value = generic::Memcmp<1>::block(p1 + offset, p2 + offset))
+      return value;
+  return MemcmpReturnType::ZERO();
+}
+
+#if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64)
+[[maybe_unused]] static inline MemcmpReturnType
+inline_memcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) {
+  if (unlikely(count >= 384)) {
+    if (auto value = generic::Memcmp<16>::block(p1, p2))
+      return value;
+    align_to_next_boundary<16, Arg::P1>(p1, p2, count);
+  }
+  return generic::Memcmp<16>::loop_and_tail(p1, p2, count);
+}
+#endif // defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64)
 
-static inline int inline_memcmp(const char *lhs, const char *rhs,
-                                size_t count) {
 #if defined(LLVM_LIBC_ARCH_X86)
-  /////////////////////////////////////////////////////////////////////////////
-  // LLVM_LIBC_ARCH_X86
-  /////////////////////////////////////////////////////////////////////////////
-  using namespace __llvm_libc::x86;
-  if (count == 0)
-    return 0;
-  if (count == 1)
-    return three_way_compare<_1>(lhs, rhs);
-  if (count == 2)
-    return three_way_compare<_2>(lhs, rhs);
-  if (count == 3)
-    return three_way_compare<_3>(lhs, rhs);
-  if (count <= 8)
-    return three_way_compare<HeadTail<_4>>(lhs, rhs, count);
-  if (count <= 16)
-    return three_way_compare<HeadTail<_8>>(lhs, rhs, count);
+[[maybe_unused]] static inline MemcmpReturnType
+inline_memcmp_x86_sse2_gt16(CPtr p1, CPtr p2, size_t count) {
+  if (unlikely(count >= 384)) {
+    if (auto value = x86::sse2::Memcmp<16>::block(p1, p2))
+      return value;
+    align_to_next_boundary<16, Arg::P1>(p1, p2, count);
+  }
+  return x86::sse2::Memcmp<16>::loop_and_tail(p1, p2, count);
+}
+
+[[maybe_unused]] static inline MemcmpReturnType
+inline_memcmp_x86_avx2_gt16(CPtr p1, CPtr p2, size_t count) {
   if (count <= 32)
-    return three_way_compare<HeadTail<_16>>(lhs, rhs, count);
+    return x86::sse2::Memcmp<16>::head_tail(p1, p2, count);
   if (count <= 64)
-    return three_way_compare<HeadTail<_32>>(lhs, rhs, count);
+    return x86::avx2::Memcmp<32>::head_tail(p1, p2, count);
   if (count <= 128)
-    return three_way_compare<HeadTail<_64>>(lhs, rhs, count);
-  return three_way_compare<Align<_32>::Then<Loop<_32>>>(lhs, rhs, count);
-#elif defined(LLVM_LIBC_ARCH_AARCH64)
-  /////////////////////////////////////////////////////////////////////////////
-  // LLVM_LIBC_ARCH_AARCH64
-  /////////////////////////////////////////////////////////////////////////////
-  using namespace ::__llvm_libc::aarch64;
-  if (count == 0) // [0, 0]
-    return 0;
-  if (count == 1) // [1, 1]
-    return three_way_compare<_1>(lhs, rhs);
-  if (count == 2) // [2, 2]
-    return three_way_compare<_2>(lhs, rhs);
-  if (count == 3) // [3, 3]
-    return three_way_compare<_3>(lhs, rhs);
-  if (count < 8) // [4, 7]
-    return three_way_compare<HeadTail<_4>>(lhs, rhs, count);
-  if (count < 16) // [8, 15]
-    return three_way_compare<HeadTail<_8>>(lhs, rhs, count);
-  if (unlikely(count >= 128)) // [128, ∞]
-    return three_way_compare<Align<_16>::Then<Loop<_32>>>(lhs, rhs, count);
-  if (!equals<_16>(lhs, rhs)) // [16, 16]
-    return three_way_compare<_16>(lhs, rhs);
+    return x86::avx2::Memcmp<64>::head_tail(p1, p2, count);
+  if (unlikely(count >= 384)) {
+    if (auto value = x86::avx2::Memcmp<32>::block(p1, p2))
+      return value;
+    align_to_next_boundary<32, Arg::P1>(p1, p2, count);
+  }
+  return x86::avx2::Memcmp<32>::loop_and_tail(p1, p2, count);
+}
+
+[[maybe_unused]] static inline MemcmpReturnType
+inline_memcmp_x86_avx512bw_gt16(CPtr p1, CPtr p2, size_t count) {
+  if (count <= 32)
+    return x86::sse2::Memcmp<16>::head_tail(p1, p2, count);
+  if (count <= 64)
+    return x86::avx2::Memcmp<32>::head_tail(p1, p2, count);
+  if (count <= 128)
+    return x86::avx512bw::Memcmp<64>::head_tail(p1, p2, count);
+  if (unlikely(count >= 384)) {
+    if (auto value = x86::avx512bw::Memcmp<64>::block(p1, p2))
+      return value;
+    align_to_next_boundary<64, Arg::P1>(p1, p2, count);
+  }
+  return x86::avx512bw::Memcmp<64>::loop_and_tail(p1, p2, count);
+}
+#endif // defined(LLVM_LIBC_ARCH_X86)
+
+#if defined(LLVM_LIBC_ARCH_AARCH64)
+[[maybe_unused]] static inline MemcmpReturnType
+inline_memcmp_aarch64_neon_gt16(CPtr p1, CPtr p2, size_t count) {
+  if (unlikely(count >= 128)) { // [128, ∞]
+    if (auto value = generic::Memcmp<16>::block(p1, p2))
+      return value;
+    align_to_next_boundary<16, Arg::P1>(p1, p2, count);
+    return generic::Memcmp<32>::loop_and_tail(p1, p2, count);
+  }
+  if (generic::Bcmp<16>::block(p1, p2)) // [16, 16]
+    return generic::Memcmp<16>::block(p1, p2);
   if (count < 32) // [17, 31]
-    return three_way_compare<Tail<_16>>(lhs, rhs, count);
-  if (!equals<Skip<16>::Then<_16>>(lhs, rhs)) // [32, 32]
-    return three_way_compare<Skip<16>::Then<_16>>(lhs, rhs);
+    return generic::Memcmp<16>::tail(p1, p2, count);
+  if (generic::Bcmp<16>::block(p1 + 16, p2 + 16)) // [32, 32]
+    return generic::Memcmp<16>::block(p1 + 16, p2 + 16);
   if (count < 64) // [33, 63]
-    return three_way_compare<Tail<_32>>(lhs, rhs, count);
+    return generic::Memcmp<32>::tail(p1, p2, count);
   // [64, 127]
-  return three_way_compare<Skip<32>::Then<Loop<_16>>>(lhs, rhs, count);
-#else
-  /////////////////////////////////////////////////////////////////////////////
-  // Default
-  /////////////////////////////////////////////////////////////////////////////
-  using namespace ::__llvm_libc::scalar;
+  return generic::Memcmp<16>::loop_and_tail(p1 + 32, p2 + 32, count - 32);
+}
+#endif // defined(LLVM_LIBC_ARCH_AARCH64)
 
+static inline MemcmpReturnType inline_memcmp(CPtr p1, CPtr p2, size_t count) {
+#if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64)
   if (count == 0)
-    return 0;
+    return MemcmpReturnType::ZERO();
   if (count == 1)
-    return three_way_compare<_1>(lhs, rhs);
+    return generic::Memcmp<1>::block(p1, p2);
   if (count == 2)
-    return three_way_compare<_2>(lhs, rhs);
+    return generic::Memcmp<2>::block(p1, p2);
   if (count == 3)
-    return three_way_compare<_3>(lhs, rhs);
+    return generic::Memcmp<3>::block(p1, p2);
   if (count <= 8)
-    return three_way_compare<HeadTail<_4>>(lhs, rhs, count);
+    return generic::Memcmp<4>::head_tail(p1, p2, count);
   if (count <= 16)
-    return three_way_compare<HeadTail<_8>>(lhs, rhs, count);
-  if (count <= 32)
-    return three_way_compare<HeadTail<_16>>(lhs, rhs, count);
-  if (count <= 64)
-    return three_way_compare<HeadTail<_32>>(lhs, rhs, count);
-  if (count <= 128)
-    return three_way_compare<HeadTail<_64>>(lhs, rhs, count);
-  return three_way_compare<Align<_32>::Then<Loop<_32>>>(lhs, rhs, count);
+    return generic::Memcmp<8>::head_tail(p1, p2, count);
+#if defined(LLVM_LIBC_ARCH_X86)
+  if constexpr (x86::kAvx512BW)
+    return inline_memcmp_x86_avx512bw_gt16(p1, p2, count);
+  else if constexpr (x86::kAvx2)
+    return inline_memcmp_x86_avx2_gt16(p1, p2, count);
+  else if constexpr (x86::kSse2)
+    return inline_memcmp_x86_sse2_gt16(p1, p2, count);
+  else
+    return inline_memcmp_generic_gt16(p1, p2, count);
+#elif defined(LLVM_LIBC_ARCH_AARCH64)
+  if constexpr (aarch64::kNeon)
+    return inline_memcmp_aarch64_neon_gt16(p1, p2, count);
+  else
+    return inline_memcmp_generic_gt16(p1, p2, count);
 #endif
+#elif defined(LLVM_LIBC_ARCH_ARM)
+  return inline_memcmp_embedded_tiny(p1, p2, count);
+#else
+#error "Unsupported platform"
+#endif
+}
+
+static inline int inline_memcmp(const void *p1, const void *p2, size_t count) {
+  return static_cast<int>(inline_memcmp(reinterpret_cast<CPtr>(p1),
+                                        reinterpret_cast<CPtr>(p2), count));
 }
 
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP_IMPLEMENTATIONS_H

libc/src/string/memory_utils/memcpy_implementations.h

 //===-- Memcpy implementation -----------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
 
 #include "src/__support/architectures.h"
 #include "src/__support/common.h"
-#include "src/string/memory_utils/elements.h"
+#include "src/string/memory_utils/op_aarch64.h"
+#include "src/string/memory_utils/op_builtin.h"
+#include "src/string/memory_utils/op_generic.h"
+#include "src/string/memory_utils/op_x86.h"
 #include "src/string/memory_utils/utils.h"
 
 #include <stddef.h> // size_t
 
-// Design rationale
-// ================
-//
-// Using a profiler to observe size distributions for calls into libc
-// functions, it was found most operations act on a small number of bytes.
-// This makes it important to favor small sizes.
-//
-// The tests for `count` are in ascending order so the cost of branching is
-// proportional to the cost of copying.
-//
-// The function is written in C++ for several reasons:
-// - The compiler can __see__ the code, this is useful when performing Profile
-//   Guided Optimization as the optimized code can take advantage of branching
-//   probabilities.
-// - It also allows for easier customization and favors testing multiple
-//   implementation parameters.
-// - As compilers and processors get better, the generated code is improved
-//   with little change on the code side.
-
 namespace __llvm_libc {
 
-static inline void inline_memcpy(char *__restrict dst,
-                                 const char *__restrict src, size_t count) {
-  using namespace __llvm_libc::builtin;
-#if defined(LLVM_LIBC_ARCH_X86)
-  /////////////////////////////////////////////////////////////////////////////
-  // LLVM_LIBC_ARCH_X86
-  /////////////////////////////////////////////////////////////////////////////
-
-  // Whether to use only rep;movsb.
-  constexpr bool USE_ONLY_REP_MOVSB =
-      LLVM_LIBC_IS_DEFINED(LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB);
-
-  // kRepMovsBSize == -1 : Only CopyAligned is used.
-  // kRepMovsBSize ==  0 : Only RepMovsb is used.
-  // else CopyAligned is used up to kRepMovsBSize and then RepMovsb.
-  constexpr size_t REP_MOVS_B_SIZE =
-#if defined(LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE)
-      LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE;
-#else
-      -1;
-#endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
-
-  // Whether target supports AVX instructions.
-  constexpr bool HAS_AVX = LLVM_LIBC_IS_DEFINED(__AVX__);
-
-#if defined(__AVX__)
-  using LoopBlockSize = _64;
-#else
-  using LoopBlockSize = _32;
-#endif
-
-  if (USE_ONLY_REP_MOVSB)
-    return copy<x86::Accelerator>(dst, src, count);
+[[maybe_unused]] static inline void
+inline_memcpy_embedded_tiny(Ptr __restrict dst, CPtr __restrict src,
+                            size_t count) {
+#pragma nounroll
+  for (size_t offset = 0; offset < count; ++offset)
+    builtin::Memcpy<1>::block(dst + offset, src + offset);
+}
 
+#if defined(LLVM_LIBC_ARCH_X86)
+[[maybe_unused]] static inline void
+inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) {
   if (count == 0)
     return;
   if (count == 1)
-    return copy<_1>(dst, src);
+    return builtin::Memcpy<1>::block(dst, src);
   if (count == 2)
-    return copy<_2>(dst, src);
+    return builtin::Memcpy<2>::block(dst, src);
   if (count == 3)
-    return copy<_3>(dst, src);
+    return builtin::Memcpy<3>::block(dst, src);
   if (count == 4)
-    return copy<_4>(dst, src);
+    return builtin::Memcpy<4>::block(dst, src);
   if (count < 8)
-    return copy<HeadTail<_4>>(dst, src, count);
+    return builtin::Memcpy<4>::head_tail(dst, src, count);
   if (count < 16)
-    return copy<HeadTail<_8>>(dst, src, count);
+    return builtin::Memcpy<8>::head_tail(dst, src, count);
   if (count < 32)
-    return copy<HeadTail<_16>>(dst, src, count);
+    return builtin::Memcpy<16>::head_tail(dst, src, count);
   if (count < 64)
-    return copy<HeadTail<_32>>(dst, src, count);
+    return builtin::Memcpy<32>::head_tail(dst, src, count);
   if (count < 128)
-    return copy<HeadTail<_64>>(dst, src, count);
-  if (HAS_AVX && count < 256)
-    return copy<HeadTail<_128>>(dst, src, count);
-  if (count <= REP_MOVS_B_SIZE)
-    return copy<Align<_32, Arg::Dst>::Then<Loop<LoopBlockSize>>>(dst, src,
-                                                                 count);
-  return copy<x86::Accelerator>(dst, src, count);
-#elif defined(LLVM_LIBC_ARCH_AARCH64)
-  /////////////////////////////////////////////////////////////////////////////
-  // LLVM_LIBC_ARCH_AARCH64
-  /////////////////////////////////////////////////////////////////////////////
+    return builtin::Memcpy<64>::head_tail(dst, src, count);
+  if (x86::kAvx && count < 256)
+    return builtin::Memcpy<128>::head_tail(dst, src, count);
+  builtin::Memcpy<32>::block(dst, src);
+  align_to_next_boundary<32, Arg::Dst>(dst, src, count);
+  static constexpr size_t kBlockSize = x86::kAvx ? 64 : 32;
+  return builtin::Memcpy<kBlockSize>::loop_and_tail(dst, src, count);
+}
+
+[[maybe_unused]] static inline void
+inline_memcpy_x86_maybe_interpose_repmovsb(Ptr __restrict dst,
+                                           CPtr __restrict src, size_t count) {
+  // Whether to use rep;movsb exclusively, not at all, or only above a certain
+  // threshold.
+  // TODO: Use only a single preprocessor definition to simplify the code.
+#ifndef LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
+#define LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE -1
+#endif
+
+  static constexpr bool kUseOnlyRepMovsb =
+      LLVM_LIBC_IS_DEFINED(LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB);
+  static constexpr size_t kRepMovsbThreshold =
+      LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE;
+  if constexpr (kUseOnlyRepMovsb)
+    return x86::Memcpy::repmovsb(dst, src, count);
+  else if constexpr (kRepMovsbThreshold >= 0) {
+    if (unlikely(count >= kRepMovsbThreshold))
+      return x86::Memcpy::repmovsb(dst, src, count);
+    else
+      return inline_memcpy_x86(dst, src, count);
+  } else {
+    return inline_memcpy_x86(dst, src, count);
+  }
+}
+#endif // defined(LLVM_LIBC_ARCH_X86)
+
+#if defined(LLVM_LIBC_ARCH_AARCH64)
+[[maybe_unused]] static inline void
+inline_memcpy_aarch64(Ptr __restrict dst, CPtr __restrict src, size_t count) {
   if (count == 0)
     return;
   if (count == 1)
-    return copy<_1>(dst, src);
+    return builtin::Memcpy<1>::block(dst, src);
   if (count == 2)
-    return copy<_2>(dst, src);
+    return builtin::Memcpy<2>::block(dst, src);
   if (count == 3)
-    return copy<_3>(dst, src);
+    return builtin::Memcpy<3>::block(dst, src);
   if (count == 4)
-    return copy<_4>(dst, src);
+    return builtin::Memcpy<4>::block(dst, src);
   if (count < 8)
-    return copy<HeadTail<_4>>(dst, src, count);
+    return builtin::Memcpy<4>::head_tail(dst, src, count);
   if (count < 16)
-    return copy<HeadTail<_8>>(dst, src, count);
+    return builtin::Memcpy<8>::head_tail(dst, src, count);
   if (count < 32)
-    return copy<HeadTail<_16>>(dst, src, count);
+    return builtin::Memcpy<16>::head_tail(dst, src, count);
   if (count < 64)
-    return copy<HeadTail<_32>>(dst, src, count);
+    return builtin::Memcpy<32>::head_tail(dst, src, count);
   if (count < 128)
-    return copy<HeadTail<_64>>(dst, src, count);
-  return copy<Align<_16, Arg::Src>::Then<Loop<_64>>>(dst, src, count);
+    return builtin::Memcpy<64>::head_tail(dst, src, count);
+  builtin::Memcpy<16>::block(dst, src);
+  align_to_next_boundary<16, Arg::Src>(dst, src, count);
+  return builtin::Memcpy<64>::loop_and_tail(dst, src, count);
+}
+#endif // defined(LLVM_LIBC_ARCH_AARCH64)
+
+static inline void inline_memcpy(Ptr __restrict dst, CPtr __restrict src,
+                                 size_t count) {
+  using namespace __llvm_libc::builtin;
+#if defined(LLVM_LIBC_ARCH_X86)
+  return inline_memcpy_x86_maybe_interpose_repmovsb(dst, src, count);
+#elif defined(LLVM_LIBC_ARCH_AARCH64)
+  return inline_memcpy_aarch64(dst, src, count);
+#elif defined(LLVM_LIBC_ARCH_ARM)
+  return inline_memcpy_embedded_tiny(dst, src, count);
 #else
-  /////////////////////////////////////////////////////////////////////////////
-  // Default
-  /////////////////////////////////////////////////////////////////////////////
-  if (count == 0)
-    return;
-  if (count == 1)
-    return copy<_1>(dst, src);
-  if (count == 2)
-    return copy<_2>(dst, src);
-  if (count == 3)
-    return copy<_3>(dst, src);
-  if (count == 4)
-    return copy<_4>(dst, src);
-  if (count < 8)
-    return copy<HeadTail<_4>>(dst, src, count);
-  if (count < 16)
-    return copy<HeadTail<_8>>(dst, src, count);
-  if (count < 32)
-    return copy<HeadTail<_16>>(dst, src, count);
-  if (count < 64)
-    return copy<HeadTail<_32>>(dst, src, count);
-  if (count < 128)
-    return copy<HeadTail<_64>>(dst, src, count);
-  return copy<Align<_32, Arg::Src>::Then<Loop<_32>>>(dst, src, count);
+#error "Unsupported platform"
 #endif
 }
 
+static inline void inline_memcpy(void *__restrict dst,
+                                 const void *__restrict src, size_t count) {
+  inline_memcpy(reinterpret_cast<Ptr>(dst), reinterpret_cast<CPtr>(src), count);
+}
+
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H

libc/src/string/memory_utils/memset_implementations.h

 //===-- Implementation of memset and bzero --------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H
 
 #include "src/__support/architectures.h"
-#include "src/string/memory_utils/elements.h"
+#include "src/string/memory_utils/op_aarch64.h"
+#include "src/string/memory_utils/op_builtin.h"
+#include "src/string/memory_utils/op_generic.h"
+#include "src/string/memory_utils/op_x86.h"
 #include "src/string/memory_utils/utils.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
-// A general purpose implementation assuming cheap unaligned writes for sizes:
-// 1, 2, 4, 8, 16, 32 and 64 Bytes. Note that some architecture can't store 32
-// or 64 Bytes at a time, the compiler will expand them as needed.
-//
-// This implementation is subject to change as we benchmark more processors. We
-// may also want to customize it for processors with specialized instructions
-// that performs better (e.g. `rep stosb`).
-//
-// A note on the apparent discrepancy in the use of 32 vs 64 Bytes writes.
-// We want to balance two things here:
-//  - The number of redundant writes (when using `SetBlockOverlap`),
-//  - The number of conditionals for sizes <=128 (~90% of memset calls are for
-//    such sizes).
-//
-// For the range 64-128:
-//  - SetBlockOverlap<64> uses no conditionals but always writes 128 Bytes this
-//  is wasteful near 65 but efficient toward 128.
-//  - SetAlignedBlocks<32> would consume between 3 and 4 conditionals and write
-//  96 or 128 Bytes.
-//  - Another approach could be to use an hybrid approach copy<64>+Overlap<32>
-//  for 65-96 and copy<96>+Overlap<32> for 97-128
-//
-// Benchmarks showed that redundant writes were cheap (for Intel X86) but
-// conditional were expensive, even on processor that do not support writing 64B
-// at a time (pre-AVX512F). We also want to favor short functions that allow
-// more hot code to fit in the iL1 cache.
-//
-// Above 128 we have to use conditionals since we don't know the upper bound in
-// advance. SetAlignedBlocks<64> may waste up to 63 Bytes, SetAlignedBlocks<32>
-// may waste up to 31 Bytes. Benchmarks showed that SetAlignedBlocks<64> was not
-// superior for sizes that mattered.
-inline static void inline_memset(char *dst, unsigned char value, size_t count) {
+[[maybe_unused]] inline static void
+inline_memset_embedded_tiny(Ptr dst, uint8_t value, size_t count) {
+#pragma nounroll
+  for (size_t offset = 0; offset < count; ++offset)
+    generic::Memset<1, 1>::block(dst + offset, value);
+}
+
 #if defined(LLVM_LIBC_ARCH_X86)
-  /////////////////////////////////////////////////////////////////////////////
-  // LLVM_LIBC_ARCH_X86
-  /////////////////////////////////////////////////////////////////////////////
-  using namespace __llvm_libc::x86;
+template <size_t MaxSize>
+[[maybe_unused]] inline static void inline_memset_x86(Ptr dst, uint8_t value,
+                                                      size_t count) {
   if (count == 0)
     return;
   if (count == 1)
-    return splat_set<_1>(dst, value);
+    return generic::Memset<1, MaxSize>::block(dst, value);
   if (count == 2)
-    return splat_set<_2>(dst, value);
+    return generic::Memset<2, MaxSize>::block(dst, value);
   if (count == 3)
-    return splat_set<_3>(dst, value);
+    return generic::Memset<3, MaxSize>::block(dst, value);
   if (count <= 8)
-    return splat_set<HeadTail<_4>>(dst, value, count);
+    return generic::Memset<4, MaxSize>::head_tail(dst, value, count);
   if (count <= 16)
-    return splat_set<HeadTail<_8>>(dst, value, count);
+    return generic::Memset<8, MaxSize>::head_tail(dst, value, count);
   if (count <= 32)
-    return splat_set<HeadTail<_16>>(dst, value, count);
+    return generic::Memset<16, MaxSize>::head_tail(dst, value, count);
   if (count <= 64)
-    return splat_set<HeadTail<_32>>(dst, value, count);
+    return generic::Memset<32, MaxSize>::head_tail(dst, value, count);
   if (count <= 128)
-    return splat_set<HeadTail<_64>>(dst, value, count);
-  return splat_set<Align<_32, Arg::Dst>::Then<Loop<_32>>>(dst, value, count);
-#elif defined(LLVM_LIBC_ARCH_AARCH64)
-  /////////////////////////////////////////////////////////////////////////////
-  // LLVM_LIBC_ARCH_AARCH64
-  /////////////////////////////////////////////////////////////////////////////
-  using namespace __llvm_libc::aarch64_memset;
+    return generic::Memset<64, MaxSize>::head_tail(dst, value, count);
+  // Aligned loop
+  generic::Memset<32, MaxSize>::block(dst, value);
+  align_to_next_boundary<32>(dst, count);
+  return generic::Memset<32, MaxSize>::loop_and_tail(dst, value, count);
+}
+#endif // defined(LLVM_LIBC_ARCH_X86)
+
+#if defined(LLVM_LIBC_ARCH_AARCH64)
+template <size_t MaxSize>
+[[maybe_unused]] inline static void
+inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) {
   if (count == 0)
     return;
   if (count <= 3) {
-    splat_set<_1>(dst, value);
+    generic::Memset<1, MaxSize>::block(dst, value);
     if (count > 1)
-      splat_set<Tail<_2>>(dst, value, count);
+      generic::Memset<2, MaxSize>::tail(dst, value, count);
     return;
   }
   if (count <= 8)
-    return splat_set<HeadTail<_4>>(dst, value, count);
+    return generic::Memset<4, MaxSize>::head_tail(dst, value, count);
   if (count <= 16)
-    return splat_set<HeadTail<_8>>(dst, value, count);
+    return generic::Memset<8, MaxSize>::head_tail(dst, value, count);
   if (count <= 32)
-    return splat_set<HeadTail<_16>>(dst, value, count);
+    return generic::Memset<16, MaxSize>::head_tail(dst, value, count);
   if (count <= (32 + 64)) {
-    splat_set<_32>(dst, value);
+    generic::Memset<32, MaxSize>::block(dst, value);
     if (count <= 64)
-      return splat_set<Tail<_32>>(dst, value, count);
-    splat_set<Skip<32>::Then<_32>>(dst, value);
-    splat_set<Tail<_32>>(dst, value, count);
+      return generic::Memset<32, MaxSize>::tail(dst, value, count);
+    generic::Memset<32, MaxSize>::block(dst + 32, value);
+    generic::Memset<32, MaxSize>::tail(dst, value, count);
     return;
   }
-  if (count >= 448 && value == 0 && hasZva())
-    return splat_set<Align<_64, Arg::P1>::Then<Loop<Zva64, _64>>>(dst, 0,
-                                                                  count);
-  else
-    return splat_set<Align<_16, Arg::P1>::Then<Loop<_64>>>(dst, value, count);
-#else
-  /////////////////////////////////////////////////////////////////////////////
-  // Default
-  /////////////////////////////////////////////////////////////////////////////
-  using namespace ::__llvm_libc::scalar;
+  if (count >= 448 && value == 0 && aarch64::neon::hasZva()) {
+    generic::Memset<64, MaxSize>::block(dst, 0);
+    align_to_next_boundary<64>(dst, count);
+    return aarch64::neon::BzeroCacheLine<64>::loop_and_tail(dst, 0, count);
+  } else {
+    generic::Memset<16, MaxSize>::block(dst, value);
+    align_to_next_boundary<16>(dst, count);
+    return generic::Memset<64, MaxSize>::loop_and_tail(dst, value, count);
+  }
+}
+#endif // defined(LLVM_LIBC_ARCH_AARCH64)
 
-  if (count == 0)
-    return;
-  if (count == 1)
-    return splat_set<_1>(dst, value);
-  if (count == 2)
-    return splat_set<_2>(dst, value);
-  if (count == 3)
-    return splat_set<_3>(dst, value);
-  if (count <= 8)
-    return splat_set<HeadTail<_4>>(dst, value, count);
-  if (count <= 16)
-    return splat_set<HeadTail<_8>>(dst, value, count);
-  if (count <= 32)
-    return splat_set<HeadTail<_16>>(dst, value, count);
-  if (count <= 64)
-    return splat_set<HeadTail<_32>>(dst, value, count);
-  if (count <= 128)
-    return splat_set<HeadTail<_64>>(dst, value, count);
-  return splat_set<Align<_32, Arg::Dst>::Then<Loop<_32>>>(dst, value, count);
+inline static void inline_memset(Ptr dst, uint8_t value, size_t count) {
+#if defined(LLVM_LIBC_ARCH_X86)
+  static constexpr size_t kMaxSize = x86::kAvx512F ? 64
+                                     : x86::kAvx   ? 32
+                                     : x86::kSse2  ? 16
+                                                   : 8;
+  return inline_memset_x86<kMaxSize>(dst, value, count);
+#elif defined(LLVM_LIBC_ARCH_AARCH64)
+  static constexpr size_t kMaxSize = aarch64::kNeon ? 16 : 8;
+  return inline_memset_aarch64<kMaxSize>(dst, value, count);
+#elif defined(LLVM_LIBC_ARCH_ARM)
+  return inline_memset_embedded_tiny(dst, value, count);
+#else
+#error "Unsupported platform"
 #endif
 }
 
+inline static void inline_memset(void *dst, uint8_t value, size_t count) {
+  inline_memset(reinterpret_cast<Ptr>(dst), value, count);
+}
+
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H

libc/src/string/memory_utils/op_x86.h

 static inline constexpr bool kAvx = LLVM_LIBC_IS_DEFINED(__AVX__);
 static inline constexpr bool kAvx2 = LLVM_LIBC_IS_DEFINED(__AVX2__);
 static inline constexpr bool kAvx512F = LLVM_LIBC_IS_DEFINED(__AVX512F__);
 static inline constexpr bool kAvx512BW = LLVM_LIBC_IS_DEFINED(__AVX512BW__);
 
 ///////////////////////////////////////////////////////////////////////////////
 // Memcpy repmovsb implementation
 struct Memcpy {
-  static void repmovsb(char *dst, const char *src, size_t count) {
+  static void repmovsb(void *dst, const void *src, size_t count) {
     asm volatile("rep movsb" : "+D"(dst), "+S"(src), "+c"(count) : : "memory");
   }
 };
 
 ///////////////////////////////////////////////////////////////////////////////
 // Bcmp
 
 // Base implementation for the Bcmp specializations.

libc/src/string/mempcpy.cpp

 #include "src/string/memory_utils/memcpy_implementations.h"
 
 #include "src/__support/common.h"
 #include <stddef.h> // For size_t.
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(void *, mempcpy,
-                   (void *__restrict dest, const void *__restrict src,
+                   (void *__restrict dst, const void *__restrict src,
                     size_t count)) {
-  char *result = reinterpret_cast<char *>(dest);
-  inline_memcpy(result, reinterpret_cast<const char *>(src), count);
-  return result + count;
+  inline_memcpy(dst, src, count);
+  return reinterpret_cast<char *>(dst) + count;
 }
 
 } // namespace __llvm_libc

libc/src/string/memset.cpp

 //===-- Implementation of memset ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/memset.h"
 #include "src/__support/common.h"
 #include "src/string/memory_utils/memset_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(void *, memset, (void *dst, int value, size_t count)) {
-  inline_memset(reinterpret_cast<char *>(dst),
-                static_cast<unsigned char>(value), count);
+  inline_memset(dst, static_cast<uint8_t>(value), count);
   return dst;
 }
 
 } // namespace __llvm_libc