[libc] Non-temporal memory function support

[gchatelet]

Summary

Enhanced version of the framework to support non-temporal loads and stores.

[gchatelet]

• Fix documentation for address.h

[tschuett]

libc/src/string/memory_utils/address.h

33 ↗

(On Diff #430011)

Is enum class or prefix cuter?

[gchatelet]

• Use enum class for Temporal and Aligned attributes

[gchatelet]

libc/src/string/memory_utils/address.h

33 ↗

(On Diff #430011)

Done, thx for the suggestion.

[gchatelet]

• rebase

[gchatelet]

• rebase and squash

[gchatelet]

• fix type mismatch for inline_memset

[gchatelet]

• Add more alignement tests

[gchatelet]

• Factor some code in address

[gchatelet]

• Use a more principled approach to common alignment

[gchatelet]

• Simplify gcd in address

[gchatelet]

• Update patch after submission of D125966

[gchatelet]

• Rename ScalarBackend

[gchatelet]

rebase and address comments

[gchatelet]

rebase

[gchatelet]

rebase

[gchatelet]

rebase

[gchatelet]

remove unrelated change

[gchatelet]

rebase

[gchatelet]

rebase

[gchatelet]

This work was based on llvm libc framework v2 that was too complicated.

libc/src/stdio/printf_core/parser.h

#ifndef LLVM_LIBC_PRINTF_DISABLE_INDEX_MODE
   // TODO: Look into object stores for optimization.
 
 #endif // LLVM_LIBC_PRINTF_DISABLE_INDEX_MODE
 
 public:
 #ifndef LLVM_LIBC_PRINTF_DISABLE_INDEX_MODE
   Parser(const char *__restrict new_str, internal::ArgList &args)
       : str(new_str), args_cur(args), args_start(args) {
-    inline_memset(reinterpret_cast<char *>(desc_arr), 0,
-                  DESC_ARR_LEN * sizeof(TypeDesc));
+    inline_memset(desc_arr, ubyte::ZERO, sizeof(desc_arr));
   }
 #else
   Parser(const char *__restrict new_str, internal::ArgList &args)
       : str(new_str), args_cur(args) {}
 #endif // LLVM_LIBC_PRINTF_DISABLE_INDEX_MODE
 
   // get_next_section will parse the format string until it has a fully
   // specified format section. This can either be a raw format section with no

libc/src/stdio/printf_core/writer.cpp

int Writer::write(const char *new_string, size_t length) {
   chars_written += length;
   return raw_write(output, new_string, length);
 }
 
 int Writer::write_chars(char new_char, size_t length) {
   constexpr size_t BUFF_SIZE = 8;
   char buff[BUFF_SIZE];
   int result;
-  inline_memset(buff, new_char, BUFF_SIZE);
+  inline_memset(buff, static_cast<ubyte>(new_char), BUFF_SIZE);
   while (length > BUFF_SIZE) {
     result = write(buff, BUFF_SIZE);
     if (result < 0)
       return result;
     length -= BUFF_SIZE;
   }
   return write(buff, length);
 }
 
 } // namespace printf_core
 } // namespace __llvm_libc

libc/src/string/bcmp.cpp

 //===-- Implementation of bcmp --------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/bcmp.h"
+
 #include "src/__support/common.h"
 #include "src/string/memory_utils/bcmp_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(int, bcmp,
                    (const void *lhs, const void *rhs, size_t count)) {
-  return inline_bcmp(static_cast<const char *>(lhs),
-                     static_cast<const char *>(rhs), count);
+  return inline_bcmp(lhs, rhs, count);
 }
 
 } // namespace __llvm_libc

libc/src/string/bzero.cpp

 //===-- Implementation of bzero -------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/bzero.h"
+
 #include "src/__support/common.h"
 #include "src/string/memory_utils/memset_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(void, bzero, (void *ptr, size_t count)) {
-  inline_memset(reinterpret_cast<char *>(ptr), 0, count);
+  inline_memset(ptr, ubyte::ZERO, count);
 }
 
 } // namespace __llvm_libc

libc/src/string/memcmp.cpp

 //===-- Implementation of memcmp ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/memcmp.h"
+
+#include "src/__support/common.h"
 #include "src/string/memory_utils/memcmp_implementations.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(int, memcmp,
                    (const void *lhs, const void *rhs, size_t count)) {
-  return inline_memcmp(static_cast<const char *>(lhs),
-                       static_cast<const char *>(rhs), count);
+  return inline_memcmp(lhs, rhs, count);
 }
 
 } // namespace __llvm_libc

libc/src/string/memcpy.cpp

 //===-- Implementation of memcpy ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/memcpy.h"
+
 #include "src/__support/common.h"
 #include "src/string/memory_utils/memcpy_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(void *, memcpy,
                    (void *__restrict dst, const void *__restrict src,
                     size_t size)) {
-  inline_memcpy(reinterpret_cast<char *>(dst),
-                reinterpret_cast<const char *>(src), size);
+  inline_memcpy(dst, src, size);
   return dst;
 }
 
 } // namespace __llvm_libc

libc/src/string/memmove.cpp

 //===-- Implementation of memmove -----------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/memmove.h"
 
 #include "src/__support/common.h"
-#include "src/__support/integer_operations.h"
-#include "src/string/memory_utils/elements.h"
+#include "src/string/memory_utils/algorithm.h"
+#include "src/string/memory_utils/backends.h"
 #include <stddef.h> // size_t, ptrdiff_t
 
 namespace __llvm_libc {
 
-static inline void inline_memmove(char *dst, const char *src, size_t count) {
-  using namespace __llvm_libc::scalar;
+static inline void inline_memmove(DstAddr<1> dst, SrcAddr<1> src,
+                                  size_t count) {
+  using namespace scalar;
   if (count == 0)
     return;
   if (count == 1)
-    return move<_1>(dst, src);
+    return _1::move(dst, src);
   if (count <= 4)
-    return move<HeadTail<_2>>(dst, src, count);
+    return HeadTail<_2>::move(dst, src, count);
   if (count <= 8)
-    return move<HeadTail<_4>>(dst, src, count);
+    return HeadTail<_4>::move(dst, src, count);
   if (count <= 16)
-    return move<HeadTail<_8>>(dst, src, count);
+    return HeadTail<_8>::move(dst, src, count);
   if (count <= 32)
-    return move<HeadTail<_16>>(dst, src, count);
+    return HeadTail<_16>::move(dst, src, count);
   if (count <= 64)
-    return move<HeadTail<_32>>(dst, src, count);
+    return HeadTail<_32>::move(dst, src, count);
   if (count <= 128)
-    return move<HeadTail<_64>>(dst, src, count);
+    return HeadTail<_64>::move(dst, src, count);
 
   using AlignedMoveLoop = Align<_16, Arg::Src>::Then<Loop<_64>>;
-  if (dst < src)
-    return move<AlignedMoveLoop>(dst, src, count);
-  else if (dst > src)
-    return move_backward<AlignedMoveLoop>(dst, src, count);
+  if (dst.ptr() < src.ptr())
+    return AlignedMoveLoop::move(dst, src, count);
+  else if (dst.ptr() > src.ptr())
+    return AlignedMoveLoop::move_backward(dst, src, count);
 }
 
 LLVM_LIBC_FUNCTION(void *, memmove,
                    (void *dst, const void *src, size_t count)) {
-  inline_memmove(reinterpret_cast<char *>(dst),
-                 reinterpret_cast<const char *>(src), count);
+  inline_memmove(dst, src, count);
   return dst;
 }
 
 } // namespace __llvm_libc

libc/src/string/memory_utils/bcmp_implementations.h

 //===-- Implementation of bcmp --------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BCMP_IMPLEMENTATIONS_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BCMP_IMPLEMENTATIONS_H
 
 #include "src/__support/architectures.h"
-#include "src/__support/common.h"
-#include "src/string/memory_utils/elements.h"
+#include "src/string/memory_utils/algorithm.h"
+#include "src/string/memory_utils/backends.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
-// Fixed-size difference between 'lhs' and 'rhs'.
-template <typename Element> bool differs(const char *lhs, const char *rhs) {
-  return !Element::equals(lhs, rhs);
-}
-// Runtime-size difference between 'lhs' and 'rhs'.
-template <typename Element>
-bool differs(const char *lhs, const char *rhs, size_t size) {
-  return !Element::equals(lhs, rhs, size);
-}
-
-static inline int inline_bcmp(const char *lhs, const char *rhs, size_t count) {
+static inline uint64_t inline_bcmp_uint64_t(SrcAddr<1> lhs, SrcAddr<1> rhs,
+                                            size_t count) {
 #if defined(LLVM_LIBC_ARCH_X86)
-  using namespace ::__llvm_libc::x86;
+  using namespace x86;
 #elif defined(LLVM_LIBC_ARCH_AARCH64)
-  using namespace ::__llvm_libc::aarch64;
+  using namespace aarch64;
 #else
-  using namespace ::__llvm_libc::scalar;
+  using namespace scalar;
 #endif
   if (count == 0)
     return 0;
   if (count == 1)
-    return differs<_1>(lhs, rhs);
+    return _1::isDifferent(lhs, rhs);
   if (count == 2)
-    return differs<_2>(lhs, rhs);
+    return _2::isDifferent(lhs, rhs);
   if (count == 3)
-    return differs<_3>(lhs, rhs);
+    return _3::isDifferent(lhs, rhs);
   if (count <= 8)
-    return differs<HeadTail<_4>>(lhs, rhs, count);
+    return HeadTail<_4>::isDifferent(lhs, rhs, count);
   if (count <= 16)
-    return differs<HeadTail<_8>>(lhs, rhs, count);
+    return HeadTail<_8>::isDifferent(lhs, rhs, count);
   if (count <= 32)
-    return differs<HeadTail<_16>>(lhs, rhs, count);
+    return HeadTail<_16>::isDifferent(lhs, rhs, count);
   if (count <= 64)
-    return differs<HeadTail<_32>>(lhs, rhs, count);
+    return HeadTail<_32>::isDifferent(lhs, rhs, count);
   if (count <= 128)
-    return differs<HeadTail<_64>>(lhs, rhs, count);
-  return differs<Align<_32>::Then<Loop<_32>>>(lhs, rhs, count);
+    return HeadTail<_64>::isDifferent(lhs, rhs, count);
+  return Align<_32>::Then<Loop<_32>>::isDifferent(lhs, rhs, count);
+}
+
+static inline int inline_bcmp(SrcAddr<1> lhs, SrcAddr<1> rhs, size_t count) {
+  return !!inline_bcmp_uint64_t(lhs, rhs, count);
 }
 
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BCMP_IMPLEMENTATIONS_H

libc/src/string/memory_utils/memcmp_implementations.h

 //===-- Implementation of memcmp ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP_IMPLEMENTATIONS_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP_IMPLEMENTATIONS_H
 
 #include "src/__support/architectures.h"
-#include "src/__support/common.h"
-#include "src/string/memory_utils/elements.h"
+#include "src/string/memory_utils/algorithm.h"
+#include "src/string/memory_utils/backends.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
-static inline int inline_memcmp(const char *lhs, const char *rhs,
-                                size_t count) {
+static inline int inline_memcmp(SrcAddr<1> lhs, SrcAddr<1> rhs, size_t count) {
 #if defined(LLVM_LIBC_ARCH_X86)
   /////////////////////////////////////////////////////////////////////////////
   // LLVM_LIBC_ARCH_X86
   /////////////////////////////////////////////////////////////////////////////
-  using namespace __llvm_libc::x86;
+  using namespace x86;
   if (count == 0)
     return 0;
   if (count == 1)
-    return three_way_compare<_1>(lhs, rhs);
+    return _1::threeWayCmp(lhs, rhs);
   if (count == 2)
-    return three_way_compare<_2>(lhs, rhs);
+    return _2::threeWayCmp(lhs, rhs);
   if (count == 3)
-    return three_way_compare<_3>(lhs, rhs);
+    return _3::threeWayCmp(lhs, rhs);
   if (count <= 8)
-    return three_way_compare<HeadTail<_4>>(lhs, rhs, count);
+    return HeadTail<_4>::threeWayCmp(lhs, rhs, count);
   if (count <= 16)
-    return three_way_compare<HeadTail<_8>>(lhs, rhs, count);
+    return HeadTail<_8>::threeWayCmp(lhs, rhs, count);
   if (count <= 32)
-    return three_way_compare<HeadTail<_16>>(lhs, rhs, count);
+    return HeadTail<_16>::threeWayCmp(lhs, rhs, count);
   if (count <= 64)
-    return three_way_compare<HeadTail<_32>>(lhs, rhs, count);
+    return HeadTail<_32>::threeWayCmp(lhs, rhs, count);
   if (count <= 128)
-    return three_way_compare<HeadTail<_64>>(lhs, rhs, count);
-  return three_way_compare<Align<_32>::Then<Loop<_32>>>(lhs, rhs, count);
+    return HeadTail<_64>::threeWayCmp(lhs, rhs, count);
+  return Align<_32>::Then<Loop<_32>>::threeWayCmp(lhs, rhs, count);
 #elif defined(LLVM_LIBC_ARCH_AARCH64)
   /////////////////////////////////////////////////////////////////////////////
   // LLVM_LIBC_ARCH_AARCH64
   /////////////////////////////////////////////////////////////////////////////
-  using namespace ::__llvm_libc::aarch64;
+  using namespace aarch64;
   if (count == 0) // [0, 0]
     return 0;
   if (count == 1) // [1, 1]
-    return three_way_compare<_1>(lhs, rhs);
+    return _1::threeWayCmp(lhs, rhs);
   if (count == 2) // [2, 2]
-    return three_way_compare<_2>(lhs, rhs);
+    return _2::threeWayCmp(lhs, rhs);
   if (count == 3) // [3, 3]
-    return three_way_compare<_3>(lhs, rhs);
+    return _3::threeWayCmp(lhs, rhs);
   if (count < 8) // [4, 7]
-    return three_way_compare<HeadTail<_4>>(lhs, rhs, count);
+    return HeadTail<_4>::threeWayCmp(lhs, rhs, count);
   if (count < 16) // [8, 15]
-    return three_way_compare<HeadTail<_8>>(lhs, rhs, count);
+    return HeadTail<_8>::threeWayCmp(lhs, rhs, count);
   if (unlikely(count >= 128)) // [128, ∞]
-    return three_way_compare<Align<_16>::Then<Loop<_32>>>(lhs, rhs, count);
-  if (!equals<_16>(lhs, rhs)) // [16, 16]
-    return three_way_compare<_16>(lhs, rhs);
+    return Align<_16>::Then<Loop<_32>>::threeWayCmp(lhs, rhs, count);
+  if (_16::isDifferent(lhs, rhs)) // [16, 16]
+    return _16::threeWayCmp(lhs, rhs);
   if (count < 32) // [17, 31]
-    return three_way_compare<Tail<_16>>(lhs, rhs, count);
-  if (!equals<Skip<16>::Then<_16>>(lhs, rhs)) // [32, 32]
-    return three_way_compare<Skip<16>::Then<_16>>(lhs, rhs);
+    return Tail<_16>::threeWayCmp(lhs, rhs, count);
+  if (Skip<16>::Then<_16>::isDifferent(lhs, rhs)) // [32, 32]
+    return Skip<16>::Then<_16>::threeWayCmp(lhs, rhs);
   if (count < 64) // [33, 63]
-    return three_way_compare<Tail<_32>>(lhs, rhs, count);
+    return Tail<_32>::threeWayCmp(lhs, rhs, count);
   // [64, 127]
-  return three_way_compare<Skip<32>::Then<Loop<_16>>>(lhs, rhs, count);
+  return Skip<32>::Then<Loop<_16>>::threeWayCmp(lhs, rhs, count);
 #else
   /////////////////////////////////////////////////////////////////////////////
   // Default
   /////////////////////////////////////////////////////////////////////////////
-  using namespace ::__llvm_libc::scalar;
-
+  using namespace scalar;
   if (count == 0)
     return 0;
   if (count == 1)
-    return three_way_compare<_1>(lhs, rhs);
+    return _1::threeWayCmp(lhs, rhs);
   if (count == 2)
-    return three_way_compare<_2>(lhs, rhs);
+    return _2::threeWayCmp(lhs, rhs);
   if (count == 3)
-    return three_way_compare<_3>(lhs, rhs);
+    return _3::threeWayCmp(lhs, rhs);
   if (count <= 8)
-    return three_way_compare<HeadTail<_4>>(lhs, rhs, count);
+    return HeadTail<_4>::threeWayCmp(lhs, rhs, count);
   if (count <= 16)
-    return three_way_compare<HeadTail<_8>>(lhs, rhs, count);
+    return HeadTail<_8>::threeWayCmp(lhs, rhs, count);
   if (count <= 32)
-    return three_way_compare<HeadTail<_16>>(lhs, rhs, count);
+    return HeadTail<_16>::threeWayCmp(lhs, rhs, count);
   if (count <= 64)
-    return three_way_compare<HeadTail<_32>>(lhs, rhs, count);
+    return HeadTail<_32>::threeWayCmp(lhs, rhs, count);
   if (count <= 128)
-    return three_way_compare<HeadTail<_64>>(lhs, rhs, count);
-  return three_way_compare<Align<_32>::Then<Loop<_32>>>(lhs, rhs, count);
+    return HeadTail<_64>::threeWayCmp(lhs, rhs, count);
+  return Align<_32>::Then<Loop<_32>>::threeWayCmp(lhs, rhs, count);
 #endif
 }
 
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP_IMPLEMENTATIONS_H

libc/src/string/memory_utils/memcpy_implementations.h

 //===-- Memcpy implementation -----------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
 
+#include "src/__support/CPP/TypeTraits.h" // cpp::ConditionalType
 #include "src/__support/architectures.h"
-#include "src/__support/common.h"
-#include "src/string/memory_utils/elements.h"
-#include "src/string/memory_utils/utils.h"
+#include "src/__support/common.h" // LLVM_LIBC_IS_DEFINED
+#include "src/string/memory_utils/algorithm.h"
+#include "src/string/memory_utils/backends.h"
 
 #include <stddef.h> // size_t
 
 // Design rationale
 // ================
 //
 // Using a profiler to observe size distributions for calls into libc
 // functions, it was found most operations act on a small number of bytes.
 // This makes it important to favor small sizes.
 //
 // The tests for `count` are in ascending order so the cost of branching is
 // proportional to the cost of copying.
 //
 // The function is written in C++ for several reasons:
 // - The compiler can __see__ the code, this is useful when performing Profile
 //   Guided Optimization as the optimized code can take advantage of branching
 //   probabilities.
 // - It also allows for easier customization and favors testing multiple
 //   implementation parameters.
 // - As compilers and processors get better, the generated code is improved
 //   with little change on the code side.
 
 namespace __llvm_libc {
 
-static inline void inline_memcpy(char *__restrict dst,
-                                 const char *__restrict src, size_t count) {
-  using namespace __llvm_libc::builtin;
+static inline void inline_memcpy(DstAddr<1> dst, SrcAddr<1> src, size_t count) {
 #if defined(LLVM_LIBC_ARCH_X86)
   /////////////////////////////////////////////////////////////////////////////
   // LLVM_LIBC_ARCH_X86
   /////////////////////////////////////////////////////////////////////////////
-
   // Whether to use only rep;movsb.
   constexpr bool USE_ONLY_REP_MOVSB =
       LLVM_LIBC_IS_DEFINED(LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB);
 
   // kRepMovsBSize == -1 : Only CopyAligned is used.
   // kRepMovsBSize ==  0 : Only RepMovsb is used.
   // else CopyAligned is used up to kRepMovsBSize and then RepMovsb.
   constexpr size_t REP_MOVS_B_SIZE =
 #if defined(LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE)
       LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE;
 #else
       -1;
 #endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
 
+  using namespace x86;
+
   // Whether target supports AVX instructions.
   constexpr bool HAS_AVX = LLVM_LIBC_IS_DEFINED(__AVX__);
-
-#if defined(__AVX__)
-  using LoopBlockSize = _64;
-#else
-  using LoopBlockSize = _32;
-#endif
-
   if (USE_ONLY_REP_MOVSB)
-    return copy<x86::Accelerator>(dst, src, count);
-
+    return repmovsb(as<void>(dst), as<const void>(src), count);
   if (count == 0)
     return;
   if (count == 1)
-    return copy<_1>(dst, src);
+    return _1::copy(dst, src);
   if (count == 2)
-    return copy<_2>(dst, src);
+    return _2::copy(dst, src);
   if (count == 3)
-    return copy<_3>(dst, src);
+    return _3::copy(dst, src);
   if (count == 4)
-    return copy<_4>(dst, src);
+    return _4::copy(dst, src);
   if (count < 8)
-    return copy<HeadTail<_4>>(dst, src, count);
+    return HeadTail<_4>::copy(dst, src, count);
   if (count < 16)
-    return copy<HeadTail<_8>>(dst, src, count);
+    return HeadTail<_8>::copy(dst, src, count);
   if (count < 32)
-    return copy<HeadTail<_16>>(dst, src, count);
+    return HeadTail<_16>::copy(dst, src, count);
   if (count < 64)
-    return copy<HeadTail<_32>>(dst, src, count);
+    return HeadTail<_32>::copy(dst, src, count);
   if (count < 128)
-    return copy<HeadTail<_64>>(dst, src, count);
+    return HeadTail<_64>::copy(dst, src, count);
   if (HAS_AVX && count < 256)
-    return copy<HeadTail<_128>>(dst, src, count);
+    return HeadTail<_128>::copy(dst, src, count);
+  using LoopBlock = cpp::ConditionalType<HAS_AVX, _64, _32>;
   if (count <= REP_MOVS_B_SIZE)
-    return copy<Align<_32, Arg::Dst>::Then<Loop<LoopBlockSize>>>(dst, src,
-                                                                 count);
-  return copy<x86::Accelerator>(dst, src, count);
+    return Align<_32, Arg::Dst>::Then<Loop<LoopBlock>>::copy(dst, src, count);
+  return repmovsb(as<void>(dst), as<const void>(src), count);
+
 #elif defined(LLVM_LIBC_ARCH_AARCH64)
   /////////////////////////////////////////////////////////////////////////////
   // LLVM_LIBC_ARCH_AARCH64
   /////////////////////////////////////////////////////////////////////////////
+  using namespace scalar;
   if (count == 0)
     return;
   if (count == 1)
-    return copy<_1>(dst, src);
+    return _1::copy(dst, src);
   if (count == 2)
-    return copy<_2>(dst, src);
+    return _2::copy(dst, src);
   if (count == 3)
-    return copy<_3>(dst, src);
+    return _3::copy(dst, src);
   if (count == 4)
-    return copy<_4>(dst, src);
+    return _4::copy(dst, src);
   if (count < 8)
-    return copy<HeadTail<_4>>(dst, src, count);
+    return HeadTail<_4>::copy(dst, src, count);
   if (count < 16)
-    return copy<HeadTail<_8>>(dst, src, count);
+    return HeadTail<_8>::copy(dst, src, count);
   if (count < 32)
-    return copy<HeadTail<_16>>(dst, src, count);
+    return HeadTail<_16>::copy(dst, src, count);
   if (count < 64)
-    return copy<HeadTail<_32>>(dst, src, count);
+    return HeadTail<_32>::copy(dst, src, count);
   if (count < 128)
-    return copy<HeadTail<_64>>(dst, src, count);
-  return copy<Align<_16, Arg::Src>::Then<Loop<_64>>>(dst, src, count);
+    return HeadTail<_64>::copy(dst, src, count);
+  return Align<_16, Arg::Src>::Then<Loop<_64>>::copy(dst, src, count);
 #else
   /////////////////////////////////////////////////////////////////////////////
   // Default
   /////////////////////////////////////////////////////////////////////////////
+  using namespace scalar;
   if (count == 0)
     return;
   if (count == 1)
-    return copy<_1>(dst, src);
+    return _1::copy(dst, src);
   if (count == 2)
-    return copy<_2>(dst, src);
+    return _2::copy(dst, src);
   if (count == 3)
-    return copy<_3>(dst, src);
+    return _3::copy(dst, src);
   if (count == 4)
-    return copy<_4>(dst, src);
+    return _4::copy(dst, src);
   if (count < 8)
-    return copy<HeadTail<_4>>(dst, src, count);
+    return HeadTail<_4>::copy(dst, src, count);
   if (count < 16)
-    return copy<HeadTail<_8>>(dst, src, count);
+    return HeadTail<_8>::copy(dst, src, count);
   if (count < 32)
-    return copy<HeadTail<_16>>(dst, src, count);
+    return HeadTail<_16>::copy(dst, src, count);
   if (count < 64)
-    return copy<HeadTail<_32>>(dst, src, count);
+    return HeadTail<_32>::copy(dst, src, count);
   if (count < 128)
-    return copy<HeadTail<_64>>(dst, src, count);
-  return copy<Align<_32, Arg::Src>::Then<Loop<_32>>>(dst, src, count);
+    return HeadTail<_64>::copy(dst, src, count);
+  return Align<_32, Arg::Src>::Then<Loop<_32>>::copy(dst, src, count);
 #endif
 }
 
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H

libc/src/string/memory_utils/memset_implementations.h

 //===-- Implementation of memset and bzero --------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H
 
 #include "src/__support/architectures.h"
-#include "src/string/memory_utils/elements.h"
-#include "src/string/memory_utils/utils.h"
+#include "src/string/memory_utils/algorithm.h"
+#include "src/string/memory_utils/backends.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
 // A general purpose implementation assuming cheap unaligned writes for sizes:
 // 1, 2, 4, 8, 16, 32 and 64 Bytes. Note that some architecture can't store 32
 // or 64 Bytes at a time, the compiler will expand them as needed.

 // conditional were expensive, even on processor that do not support writing 64B
 // at a time (pre-AVX512F). We also want to favor short functions that allow
 // more hot code to fit in the iL1 cache.
 //
 // Above 128 we have to use conditionals since we don't know the upper bound in
 // advance. SetAlignedBlocks<64> may waste up to 63 Bytes, SetAlignedBlocks<32>
 // may waste up to 31 Bytes. Benchmarks showed that SetAlignedBlocks<64> was not
 // superior for sizes that mattered.
-inline static void inline_memset(char *dst, unsigned char value, size_t count) {
+inline static void inline_memset(DstAddr<1> dst, ubyte value, size_t count) {
 #if defined(LLVM_LIBC_ARCH_X86)
   /////////////////////////////////////////////////////////////////////////////
   // LLVM_LIBC_ARCH_X86
   /////////////////////////////////////////////////////////////////////////////
-  using namespace __llvm_libc::x86;
+  using namespace x86;
   if (count == 0)
     return;
   if (count == 1)
-    return splat_set<_1>(dst, value);
+    return _1::set(dst, value);
   if (count == 2)
-    return splat_set<_2>(dst, value);
+    return _2::set(dst, value);
   if (count == 3)
-    return splat_set<_3>(dst, value);
+    return _3::set(dst, value);
   if (count <= 8)
-    return splat_set<HeadTail<_4>>(dst, value, count);
+    return HeadTail<_4>::set(dst, value, count);
   if (count <= 16)
-    return splat_set<HeadTail<_8>>(dst, value, count);
+    return HeadTail<_8>::set(dst, value, count);
   if (count <= 32)
-    return splat_set<HeadTail<_16>>(dst, value, count);
+    return HeadTail<_16>::set(dst, value, count);
   if (count <= 64)
-    return splat_set<HeadTail<_32>>(dst, value, count);
+    return HeadTail<_32>::set(dst, value, count);
   if (count <= 128)
-    return splat_set<HeadTail<_64>>(dst, value, count);
-  return splat_set<Align<_32, Arg::Dst>::Then<Loop<_32>>>(dst, value, count);
+    return HeadTail<_64>::set(dst, value, count);
+  return Align<_32, Arg::Dst>::Then<Loop<_32>>::set(dst, value, count);
 #elif defined(LLVM_LIBC_ARCH_AARCH64)
   /////////////////////////////////////////////////////////////////////////////
   // LLVM_LIBC_ARCH_AARCH64
   /////////////////////////////////////////////////////////////////////////////
-  using namespace __llvm_libc::aarch64_memset;
+  using namespace aarch64;
   if (count == 0)
     return;
   if (count <= 3) {
-    splat_set<_1>(dst, value);
+    _1::set(dst, value);
     if (count > 1)
-      splat_set<Tail<_2>>(dst, value, count);
+      Tail<_2>::set(dst, value, count);
     return;
   }
   if (count <= 8)
-    return splat_set<HeadTail<_4>>(dst, value, count);
+    return HeadTail<_4>::set(dst, value, count);
   if (count <= 16)
-    return splat_set<HeadTail<_8>>(dst, value, count);
+    return HeadTail<_8>::set(dst, value, count);
   if (count <= 32)
-    return splat_set<HeadTail<_16>>(dst, value, count);
+    return HeadTail<_16>::set(dst, value, count);
   if (count <= 96) {
-    splat_set<_32>(dst, value);
+    _32::set(dst, value);
     if (count <= 64)
-      return splat_set<Tail<_32>>(dst, value, count);
-    splat_set<Skip<32>::Then<_32>>(dst, value);
-    splat_set<Tail<_32>>(dst, value, count);
+      return Tail<_32>::set(dst, value, count);
+    Skip<32>::Then<_32>::set(dst, value);
+    Tail<_32>::set(dst, value, count);
     return;
   }
-  if (count < 448 || value != 0 || !AArch64ZVA(dst, count))
-    return splat_set<Align<_16, Arg::_1>::Then<Loop<_64>>>(dst, value, count);
+  if (count >= 448 && value == ubyte::ZERO && hasZva())
+    return Align<_64, Arg::_1>::Then<Loop<Zva64>>::set(dst, ubyte::ZERO, count);
+  else
+    return Align<_16, Arg::_1>::Then<Loop<_64>>::set(dst, value, count);
 #else
   /////////////////////////////////////////////////////////////////////////////
   // Default
   /////////////////////////////////////////////////////////////////////////////
-  using namespace ::__llvm_libc::scalar;
-
+  using namespace scalar;
   if (count == 0)
     return;
   if (count == 1)
-    return splat_set<_1>(dst, value);
+    return _1::set(dst, value);
   if (count == 2)
-    return splat_set<_2>(dst, value);
+    return _2::set(dst, value);
   if (count == 3)
-    return splat_set<_3>(dst, value);
+    return _3::set(dst, value);
   if (count <= 8)
-    return splat_set<HeadTail<_4>>(dst, value, count);
+    return HeadTail<_4>::set(dst, value, count);
   if (count <= 16)
-    return splat_set<HeadTail<_8>>(dst, value, count);
+    return HeadTail<_8>::set(dst, value, count);
   if (count <= 32)
-    return splat_set<HeadTail<_16>>(dst, value, count);
+    return HeadTail<_16>::set(dst, value, count);
   if (count <= 64)
-    return splat_set<HeadTail<_32>>(dst, value, count);
+    return HeadTail<_32>::set(dst, value, count);
   if (count <= 128)
-    return splat_set<HeadTail<_64>>(dst, value, count);
-  return splat_set<Align<_32, Arg::Dst>::Then<Loop<_32>>>(dst, value, count);
+    return HeadTail<_64>::set(dst, value, count);
+  return Align<_32, Arg::Dst>::Then<Loop<_32>>::set(dst, value, count);
 #endif
 }
 
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H

libc/src/string/mempcpy.cpp

 #include "src/string/memory_utils/memcpy_implementations.h"
 
 #include "src/__support/common.h"
 #include <stddef.h> // For size_t.
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(void *, mempcpy,
-                   (void *__restrict dest, const void *__restrict src,
+                   (void *__restrict dst, const void *__restrict src,
                     size_t count)) {
-  char *result = reinterpret_cast<char *>(dest);
-  inline_memcpy(result, reinterpret_cast<const char *>(src), count);
-  return result + count;
+  inline_memcpy(dst, src, count);
+  return reinterpret_cast<char *>(dst) + count;
 }
 
 } // namespace __llvm_libc

libc/src/string/memset.cpp

 //===-- Implementation of memset ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/memset.h"
+
 #include "src/__support/common.h"
 #include "src/string/memory_utils/memset_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(void *, memset, (void *dst, int value, size_t count)) {
-  inline_memset(reinterpret_cast<char *>(dst),
-                static_cast<unsigned char>(value), count);
+  inline_memset(dst, static_cast<ubyte>(value), count);
   return dst;
 }
 
 } // namespace __llvm_libc

libc/src/string/stpncpy.cpp

 //===-- Implementation of stpncpy -----------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/stpncpy.h"
-#include "src/string/memory_utils/memset_implementations.h"
 
 #include "src/__support/common.h"
+#include "src/string/memory_utils/memset_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(char *, stpncpy,
                    (char *__restrict dest, const char *__restrict src,
                     size_t n)) {
   size_t i;
   // Copy up until \0 is found.
   for (i = 0; i < n && src[i] != '\0'; ++i)
     dest[i] = src[i];
   // When n>strlen(src), n-strlen(src) \0 are appended.
   if (n > i)
-    inline_memset(dest + i, 0, n - i);
+    inline_memset(dest + i, ubyte::ZERO, n - i);
   return dest + i;
 }
 
 } // namespace __llvm_libc

libc/src/string/string_utils.h

 
 static inline size_t strlcpy(char *__restrict dst, const char *__restrict src,
                              size_t size) {
   size_t len = internal::string_length(src);
   if (!size)
     return len;
   size_t n = len < size - 1 ? len : size - 1;
   inline_memcpy(dst, src, n);
-  inline_memset(dst + n, 0, size - n);
+  inline_memset(dst + n, ubyte::ZERO, size - n);
   return len;
 }
 
 } // namespace internal
 } // namespace __llvm_libc
 
 #endif //  LIBC_SRC_STRING_STRING_UTILS_H

libc/test/src/string/memcmp_test.cpp

TEST(LlvmLibcMemcmpTest, Sweep) {
   const auto reset = [](char *const ptr) {
     for (size_t i = 0; i < K_MAX_SIZE; ++i)
       ptr[i] = 'a';
   };
 
   reset(lhs);
   reset(rhs);
   for (size_t i = 0; i < K_MAX_SIZE; ++i)
-    EXPECT_EQ(__llvm_libc::memcmp(lhs, rhs, i), 0);
+    ASSERT_EQ(__llvm_libc::memcmp(lhs, rhs, i), 0);
 
   reset(lhs);
   reset(rhs);
   for (size_t i = 0; i < K_MAX_SIZE; ++i) {
     rhs[i] = 'z';
-    EXPECT_LT(__llvm_libc::memcmp(lhs, rhs, K_MAX_SIZE), 0);
+    ASSERT_LT(__llvm_libc::memcmp(lhs, rhs, K_MAX_SIZE), 0);
     rhs[i] = 'a';
   }
 }

utils/bazel/llvm-project-overlay/libc/BUILD.bazel

no_sanitize_features = [
     "-msan",
     "-tsan",
     "-ubsan",
 ]
 
 cc_library(
     name = "string_memory_utils",
     hdrs = [
-        "src/string/memory_utils/elements.h",
-        "src/string/memory_utils/elements_aarch64.h",
-        "src/string/memory_utils/elements_x86.h",
+        "src/string/memory_utils/address.h",
+        "src/string/memory_utils/algorithm.h",
+        "src/string/memory_utils/backends.h",
+        "src/string/memory_utils/backend_scalar.h",
+        "src/string/memory_utils/backend_x86.h",
+        "src/string/memory_utils/backend_aarch64.h",
+        "src/string/memory_utils/sized_op.h",
         "src/string/memory_utils/utils.h",
     ],
     textual_hdrs = [
         "src/string/memory_utils/bcmp_implementations.h",
         "src/string/memory_utils/memcmp_implementations.h",
         "src/string/memory_utils/memcpy_implementations.h",
         "src/string/memory_utils/memset_implementations.h",
     ],
     deps = [
         ":__support_common",
         ":__support_cpp_bit",
+        ":__support_cpp_type_traits",
         ":libc_root",
     ],
 )
 
 cc_library(
     name = "string_utils",
     hdrs = ["src/string/string_utils.h"],
     deps = [