[libc] Non-temporal memory function support

[gchatelet]

Summary

Enhanced version of the framework to support non-temporal loads and stores.

[gchatelet]

• Fix documentation for address.h

[tschuett]

libc/src/string/memory_utils/address.h

34

Is enum class or prefix cuter?

[gchatelet]

• Use enum class for Temporal and Aligned attributes

[gchatelet]

libc/src/string/memory_utils/address.h

34

Done, thx for the suggestion.

[gchatelet]

• rebase

[gchatelet]

• rebase and squash

[gchatelet]

• fix type mismatch for inline_memset

[gchatelet]

• Add more alignement tests

[gchatelet]

• Factor some code in address

[gchatelet]

• Use a more principled approach to common alignment

[gchatelet]

• Simplify gcd in address

[gchatelet]

• Update patch after submission of D125966

[gchatelet]

• Rename ScalarBackend

[gchatelet]

rebase and address comments

[gchatelet]

rebase

[gchatelet]

rebase

[gchatelet]

rebase

[gchatelet]

remove unrelated change

[gchatelet]

rebase

[gchatelet]

rebase

[gchatelet]

This work was based on llvm libc framework v2 that was too complicated.

libc/src/__support/CPP/TypeTraits.h

 #define LLVM_LIBC_SRC_SUPPORT_CPP_TYPETRAITS_H
 
 #include "UInt.h"
 
 namespace __llvm_libc {
 namespace cpp {
 
 template <bool B, typename T> struct EnableIf;
-template <typename T> struct EnableIf<true, T> { typedef T Type; };
+template <typename T> struct EnableIf<true, T> {
+  typedef T Type;
+};
 
 template <bool B, typename T>
 using EnableIfType = typename EnableIf<B, T>::Type;
 
 struct TrueValue {
   static constexpr bool Value = true;
 };
 
 struct FalseValue {
   static constexpr bool Value = false;
 };
 
-template <typename T> struct TypeIdentity { typedef T Type; };
+template <typename T> struct TypeIdentity {
+  typedef T Type;
+};
 
 template <typename T1, typename T2> struct IsSame : public FalseValue {};
 template <typename T> struct IsSame<T, T> : public TrueValue {};
 template <typename T1, typename T2>
 static constexpr bool IsSameV = IsSame<T1, T2>::Value;
 
 template <typename T> struct RemoveCV : public TypeIdentity<T> {};
 template <typename T> struct RemoveCV<const T> : public TypeIdentity<T> {};
static constexpr bool Value =
       IsSameV<unsigned long long, TypeNoCV> || IsSameV<bool, TypeNoCV> ||
       IsSameV<UInt<128>, TypeNoCV>
 #ifdef __SIZEOF_INT128__
       || IsSameV<__uint128_t, TypeNoCV> || IsSameV<__int128_t, TypeNoCV>
 #endif
       ;
 };
 
+template <typename Type> struct IsEnum {
+  static constexpr bool Value = __is_enum(Type);
+};
+
 template <typename T> struct IsPointerTypeNoCV : public FalseValue {};
 template <typename T> struct IsPointerTypeNoCV<T *> : public TrueValue {};
 template <typename T> struct IsPointerType {
   static constexpr bool Value = IsPointerTypeNoCV<RemoveCVType<T>>::Value;
 };
 
 template <typename Type> struct IsFloatingPointType {
   using TypeNoCV = RemoveCVType<Type>;
   static constexpr bool Value = IsSame<float, TypeNoCV>::Value ||
                                 IsSame<double, TypeNoCV>::Value ||
                                 IsSame<long double, TypeNoCV>::Value;
 };
 
 template <typename Type> struct IsArithmetic {
   static constexpr bool Value =
       IsIntegral<Type>::Value || IsFloatingPointType<Type>::Value;
 };
 
+// Compile time type selection.
+template <bool _, class TrueT, class FalseT> struct Conditional {
+  using type = TrueT;
+};
+template <class TrueT, class FalseT> struct Conditional<false, TrueT, FalseT> {
+  using type = FalseT;
+};
+template <bool Cond, typename TrueT, typename FalseT>
+using ConditionalType = typename Conditional<Cond, TrueT, FalseT>::type;
+
 } // namespace cpp
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_SUPPORT_CPP_TYPETRAITS_H

libc/src/__support/common.h

 #define LLVM_LIBC_SUPPORT_COMMON_H
 
 #define LIBC_INLINE_ASM __asm__ __volatile__
 
 #define likely(x) __builtin_expect(!!(x), 1)
 #define unlikely(x) __builtin_expect(x, 0)
 #define UNUSED __attribute__((unused))
 
+#ifdef __has_attribute
+#define LIBC_HAVE_ATTRIBUTE(x) __has_attribute(x)
+#else
+#define LIBC_HAVE_ATTRIBUTE(x) 0
+#endif
+
+#if LIBC_HAVE_ATTRIBUTE(aligned) || (defined(__GNUC__) && !defined(__clang__))
+#define LIBC_ATTRIBUTE_FUNC_ALIGN(bytes) __attribute__((aligned(bytes)))
+#else
+#define LIBC_ATTRIBUTE_FUNC_ALIGN(bytes)
+#endif
+
 #ifndef LLVM_LIBC_FUNCTION_ATTR
 #define LLVM_LIBC_FUNCTION_ATTR
 #endif
 
 // MacOS needs to be excluded because it does not support aliasing.
 #if defined(LLVM_LIBC_PUBLIC_PACKAGING) && (!defined(__APPLE__))
 #define LLVM_LIBC_FUNCTION(type, name, arglist)                                \
   LLVM_LIBC_FUNCTION_ATTR decltype(__llvm_libc::name)                          \

libc/src/string/bcmp.cpp

 //===-- Implementation of bcmp --------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/bcmp.h"
+
 #include "src/__support/common.h"
 #include "src/string/memory_utils/bcmp_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(int, bcmp,
                    (const void *lhs, const void *rhs, size_t count)) {
-  return inline_bcmp(static_cast<const char *>(lhs),
-                     static_cast<const char *>(rhs), count);
+  return inline_bcmp(lhs, rhs, count);
 }
 
 } // namespace __llvm_libc

libc/src/string/bzero.cpp

 //===-- Implementation of bzero -------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/bzero.h"
+
 #include "src/__support/common.h"
 #include "src/string/memory_utils/memset_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(void, bzero, (void *ptr, size_t count)) {
-  inline_memset(reinterpret_cast<char *>(ptr), 0, count);
+  inline_memset(ptr, ubyte::ZERO, count);
 }
 
 } // namespace __llvm_libc

libc/src/string/memcmp.cpp

 //===-- Implementation of memcmp ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/memcmp.h"
+
+#include "src/__support/common.h"
 #include "src/string/memory_utils/memcmp_implementations.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(int, memcmp,
                    (const void *lhs, const void *rhs, size_t count)) {
-  return inline_memcmp(static_cast<const char *>(lhs),
-                       static_cast<const char *>(rhs), count);
+  return inline_memcmp(lhs, rhs, count);
 }
 
 } // namespace __llvm_libc

libc/src/string/memcpy.cpp

 //===-- Implementation of memcpy ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/memcpy.h"
+
 #include "src/__support/common.h"
 #include "src/string/memory_utils/memcpy_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(void *, memcpy,
                    (void *__restrict dst, const void *__restrict src,
                     size_t size)) {
-  inline_memcpy(reinterpret_cast<char *>(dst),
-                reinterpret_cast<const char *>(src), size);
+  inline_memcpy(dst, src, size);
   return dst;
 }
 
 } // namespace __llvm_libc

libc/src/string/memmove.cpp

 //===-- Implementation of memmove -----------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/memmove.h"
 
 #include "src/__support/common.h"
-#include "src/__support/integer_operations.h"
-#include "src/string/memory_utils/elements.h"
+#include "src/string/memory_utils/algorithm.h"
+#include "src/string/memory_utils/backends.h"
 #include <stddef.h> // size_t, ptrdiff_t
 
 namespace __llvm_libc {
 
-static inline void inline_memmove(char *dst, const char *src, size_t count) {
-  using namespace __llvm_libc::scalar;
+static inline void inline_memmove(DstAddr<1> dst, SrcAddr<1> src,
+                                  size_t count) {
+  using namespace scalar;
   if (count == 0)
     return;
   if (count == 1)
-    return move<_1>(dst, src);
+    return _1::move(dst, src);
   if (count <= 4)
-    return move<HeadTail<_2>>(dst, src, count);
+    return HeadTail<_2>::move(dst, src, count);
   if (count <= 8)
-    return move<HeadTail<_4>>(dst, src, count);
+    return HeadTail<_4>::move(dst, src, count);
   if (count <= 16)
-    return move<HeadTail<_8>>(dst, src, count);
+    return HeadTail<_8>::move(dst, src, count);
   if (count <= 32)
-    return move<HeadTail<_16>>(dst, src, count);
+    return HeadTail<_16>::move(dst, src, count);
   if (count <= 64)
-    return move<HeadTail<_32>>(dst, src, count);
+    return HeadTail<_32>::move(dst, src, count);
   if (count <= 128)
-    return move<HeadTail<_64>>(dst, src, count);
+    return HeadTail<_64>::move(dst, src, count);
 
   using AlignedMoveLoop = Align<_16, Arg::Src>::Then<Loop<_64>>;
-  if (dst < src)
-    return move<AlignedMoveLoop>(dst, src, count);
-  else if (dst > src)
-    return move_backward<AlignedMoveLoop>(dst, src, count);
+  if (dst.ptr() < src.ptr())
+    return AlignedMoveLoop::move(dst, src, count);
+  else if (dst.ptr() > src.ptr())
+    return AlignedMoveLoop::move_backward(dst, src, count);
 }
 
 LLVM_LIBC_FUNCTION(void *, memmove,
                    (void *dst, const void *src, size_t count)) {
-  inline_memmove(reinterpret_cast<char *>(dst),
-                 reinterpret_cast<const char *>(src), count);
+  inline_memmove(dst, src, count);
   return dst;
 }
 
 } // namespace __llvm_libc

libc/src/string/memory_utils/address.h

+//===-- Strongly typed address with alignment and access semantics --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_COMMON_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_COMMON_H
+
+#include "src/__support/CPP/TypeTraits.h"  // cpp::ConditionalType
+#include "src/string/memory_utils/utils.h" // is_power2
+#include <stddef.h>                        // size_t
+#include <stdint.h> // uint8_t, uint16_t, uint32_t, uint64_t
+
+namespace __llvm_libc {
+
+// Utility to enable static_assert(false) in templates.
+template <bool flag = false>
+static void deferred_static_assert(const char *msg) {
+  static_assert(flag, "compilation error");
+}
+
+// A non-coercible type to represent raw data.
+enum class ubyte : unsigned char { ZERO = 0 };
+
+// Address attribute instructing whether the underlying load / store operations
+// are temporal or non-temporal.
+enum class Temporal { NO, YES };
+
+// Address attribute instructing whether the underlying load / store operations
+// are aligned or unaligned.
+enum class Aligned { NO, YES };
+
+// Address attribute to discriminate between readable and writable addresses.
+enum class Permission { Read, Write };
+
+// Address is semantically equivalent to a pointer but also conveys compile time
+// information that helps with instructions selection (aligned/unaligned,
+// temporal/non-temporal).
+template <size_t Alignment, Permission P, Temporal TS> struct Address {
+  static_assert(is_power2(Alignment));
+  static constexpr bool IS_ADDRESS_TYPE = true;
+  static constexpr size_t ALIGNMENT = Alignment;
+  static constexpr Permission PERMISSION = P;
+  static constexpr Temporal TEMPORAL = TS;
+  static constexpr bool IS_READ = P == Permission::Read;
+  static constexpr bool IS_WRITE = P == Permission::Write;
+  using value_type = cpp::ConditionalType<!IS_WRITE, const ubyte, ubyte>;
+  using void_type = cpp::ConditionalType<!IS_WRITE, const void, void>;
+
+  Address(void_type *ptr) : ptr_(reinterpret_cast<value_type *>(ptr)) {}
+
+  value_type *ptr() const {
+    return reinterpret_cast<value_type *>(
+        __builtin_assume_aligned(ptr_, ALIGNMENT));
+  }
+
+  value_type *const ptr_;
+};
+
+// Lowers the address to a pointer of type T.
+template <typename T, typename AddrT> static T *as(AddrT addr) {
+  static_assert(AddrT::IS_ADDRESS_TYPE);
+  return reinterpret_cast<T *>(addr.ptr());
+}
+
+// Offsets the address by a compile time amount, this allows propagating
+// alignment whenever possible.
+template <size_t Offset, typename AddrT> static auto offsetAddr(AddrT addr) {
+  static_assert(AddrT::IS_ADDRESS_TYPE);
+  constexpr size_t AL = Offset < AddrT::ALIGNMENT ? Offset : AddrT::ALIGNMENT;
+  return Address<AL, AddrT::PERMISSION, AddrT::TEMPORAL>(addr.ptr_ + Offset);
+}
+
+// Offsets the address by a runtime amount but assuming that the resulting
+// address will be Alignment aligned.
+template <size_t Alignment, typename AddrT>
+static auto offsetAddrAssumeAligned(AddrT addr, size_t offset) {
+  static_assert(AddrT::IS_ADDRESS_TYPE);
+  return Address<Alignment, AddrT::PERMISSION, AddrT::TEMPORAL>(addr.ptr_ +
+                                                                offset);
+}
+
+// Offsets the address by a runtime amount that is assumed to be a multiple of
+// Offset. This allows to propagate the address alignement whenever possible.
+template <size_t Offset, typename AddrT>
+static auto offsetAddrMultiplesOf(AddrT addr, ptrdiff_t offset) {
+  static_assert(AddrT::IS_ADDRESS_TYPE);
+  constexpr size_t AL = Offset < AddrT::ALIGNMENT ? Offset : AddrT::ALIGNMENT;
+  return offsetAddrAssumeAligned<AL>(addr, offset);
+}
+
+// User friendly aliases for common address types.
+template <size_t Alignment>
+using SrcAddr = Address<Alignment, Permission::Read, Temporal::YES>;
+template <size_t Alignment>
+using DstAddr = Address<Alignment, Permission::Write, Temporal::YES>;
+template <size_t Alignment>
+using NtSrcAddr = Address<Alignment, Permission::Read, Temporal::NO>;
+template <size_t Alignment>
+using NtDstAddr = Address<Alignment, Permission::Write, Temporal::NO>;
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_COMMON_H

libc/src/string/memory_utils/algorithm.h

+//===-- Algorithms to compose sized memory operations ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Higher order primitives that build upon the SizedOp facility.
+// They constitute the basic blocks for composing memory functions.
+// This file defines the following operations:
+// - Skip
+// - Tail
+// - HeadTail
+// - Loop
+// - Align
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ALGORITHM_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ALGORITHM_H
+
+#include "src/string/memory_utils/address.h" // Address
+#include "src/string/memory_utils/utils.h"   // offset_to_next_aligned
+
+#include <stddef.h> // ptrdiff_t
+
+namespace __llvm_libc {
+
+// An operation that allows to skip the specified amount of bytes.
+template <ptrdiff_t Bytes> struct Skip {
+  template <typename NextT> struct Then {
+    template <typename DstAddrT>
+    static inline void set(DstAddrT dst, ubyte value) {
+      NextT::set(offsetAddr<Bytes>(dst), value);
+    }
+
+    template <typename SrcAddrT1, typename SrcAddrT2>
+    static inline uint64_t different(SrcAddrT1 src1, SrcAddrT2 src2) {
+      return NextT::different(offsetAddr<Bytes>(src1), offsetAddr<Bytes>(src2));
+    }
+
+    template <typename SrcAddrT1, typename SrcAddrT2>
+    static inline int32_t three_way_cmp(SrcAddrT1 src1, SrcAddrT2 src2) {
+      return NextT::three_way_cmp(offsetAddr<Bytes>(src1),
+                                  offsetAddr<Bytes>(src2));
+    }
+
+    template <typename SrcAddrT1, typename SrcAddrT2>
+    static inline int32_t three_way_cmp(SrcAddrT1 src1, SrcAddrT2 src2,
+                                        size_t runtime_size) {
+      return NextT::three_way_cmp(offsetAddr<Bytes>(src1),
+                                  offsetAddr<Bytes>(src2),
+                                  runtime_size - Bytes);
+    }
+  };
+};
+
+// Compute the address of a tail operation.
+// Because of the runtime size, we loose the alignment information.
+template <size_t Size, typename AddrT>
+static auto tailAddr(AddrT addr, size_t runtime_size) {
+  static_assert(AddrT::IS_ADDRESS_TYPE);
+  return offsetAddrAssumeAligned<1>(addr, runtime_size - Size);
+}
+
+// Perform the operation on the last 'Size' bytes of the buffer.
+//
+// e.g. with
+// [1234567812345678123]
+// [__XXXXXXXXXXXXXX___]
+// [________XXXXXXXX___]
+//
+// Precondition: `runtime_size >= Size`.
+template <typename SizedOp> struct Tail {
+  static constexpr size_t SIZE = SizedOp::SIZE;
+  template <typename DstAddrT, typename SrcAddrT>
+  static inline void copy(DstAddrT dst, SrcAddrT src, size_t runtime_size) {
+    SizedOp::copy(tailAddr<SIZE>(dst, runtime_size),
+                  tailAddr<SIZE>(src, runtime_size));
+  }
+
+  template <typename DstAddrT, typename SrcAddrT>
+  static inline void move(DstAddrT dst, SrcAddrT src, size_t runtime_size) {
+    SizedOp::move(tailAddr<SIZE>(dst, runtime_size),
+                  tailAddr<SIZE>(src, runtime_size));
+  }
+
+  template <typename DstAddrT>
+  static inline void set(DstAddrT dst, ubyte value, size_t runtime_size) {
+    SizedOp::set(tailAddr<SIZE>(dst, runtime_size), value);
+  }
+
+  template <typename SrcAddrT1, typename SrcAddrT2>
+  static inline uint64_t different(SrcAddrT1 src1, SrcAddrT2 src2,
+                                   size_t runtime_size) {
+    return SizedOp::different(tailAddr<SIZE>(src1, runtime_size),
+                              tailAddr<SIZE>(src2, runtime_size));
+  }
+
+  template <typename SrcAddrT1, typename SrcAddrT2>
+  static inline int32_t three_way_cmp(SrcAddrT1 src1, SrcAddrT2 src2,
+                                      size_t runtime_size) {
+    return SizedOp::three_way_cmp(tailAddr<SIZE>(src1, runtime_size),
+                                  tailAddr<SIZE>(src2, runtime_size));
+  }
+};
+
+// Perform the operation on the first and last bytes of the buffer.
+// This is useful for overlapping operations.
+//
+// e.g. with
+// [1234567812345678123]
+// [__XXXXXXXXXXXXXX___]
+// [__XXXXXXXX_________]
+// [________XXXXXXXX___]
+//
+// Precondition: `runtime_size >= Size && runtime_size <= 2 x Size`.
+template <typename SizedOp> struct HeadTail {
+  template <typename DstAddrT, typename SrcAddrT>
+  static inline void copy(DstAddrT dst, SrcAddrT src, size_t runtime_size) {
+    SizedOp::copy(dst, src);
+    Tail<SizedOp>::copy(dst, src, runtime_size);
+  }
+
+  template <typename DstAddrT, typename SrcAddrT>
+  static inline void move(DstAddrT dst, SrcAddrT src, size_t runtime_size) {
+    static constexpr size_t SIZE = SizedOp::SIZE;
+    auto head = SizedOp::load(src);
+    auto tail = SizedOp::load(tailAddr<SIZE>(src, runtime_size));
+    SizedOp::store(tailAddr<SIZE>(dst, runtime_size), tail);
+    SizedOp::store(dst, head);
+  }
+
+  template <typename DstAddrT>
+  static inline void set(DstAddrT dst, ubyte value, size_t runtime_size) {
+    SizedOp::set(dst, value);
+    Tail<SizedOp>::set(dst, value, runtime_size);
+  }
+
+  template <typename SrcAddrT1, typename SrcAddrT2>
+  static inline uint64_t different(SrcAddrT1 src1, SrcAddrT2 src2,
+                                   size_t runtime_size) {
+    if (const auto res = SizedOp::different(src1, src2))
+      return res;
+    return Tail<SizedOp>::different(src1, src2, runtime_size);
+  }
+
+  template <typename SrcAddrT1, typename SrcAddrT2>
+  static inline int32_t three_way_cmp(SrcAddrT1 src1, SrcAddrT2 src2,
+                                      size_t runtime_size) {
+    if (const auto res = SizedOp::three_way_cmp(src1, src2))
+      return res;
+    return Tail<SizedOp>::three_way_cmp(src1, src2, runtime_size);
+  }
+};
+
+// Simple loop ending with a Tail operation.
+//
+// e.g. with
+// [12345678123456781234567812345678]
+// [__XXXXXXXXXXXXXXXXXXXXXXXXXXXX___]
+// [__XXXXXXXX_______________________]
+// [__________XXXXXXXX_______________]
+// [__________________XXXXXXXX_______]
+// [______________________XXXXXXXX___]
+//
+// Precondition:
+// - runtime_size >= Size
+template <typename SizedOp> struct Loop {
+  static constexpr size_t SIZE = SizedOp::SIZE;
+
+  template <typename DstAddrT, typename SrcAddrT>
+  static inline void copy(DstAddrT dst, SrcAddrT src, size_t runtime_size) {
+    size_t offset = 0;
+    do {
+      SizedOp::copy(offsetAddrMultiplesOf<SIZE>(dst, offset),
+                    offsetAddrMultiplesOf<SIZE>(src, offset));
+      offset += SIZE;
+    } while (offset < runtime_size - SIZE);
+    Tail<SizedOp>::copy(dst, src, runtime_size);
+  }
+
+  // Move forward suitable when dst < src. We load the tail bytes before
+  // handling the loop.
+  //
+  // e.g. Moving two bytes
+  // [   |       |       |       |       |]
+  // [___XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX___]
+  // [_________________________LLLLLLLL___]
+  // [___LLLLLLLL_________________________]
+  // [_SSSSSSSS___________________________]
+  // [___________LLLLLLLL_________________]
+  // [_________SSSSSSSS___________________]
+  // [___________________LLLLLLLL_________]
+  // [_________________SSSSSSSS___________]
+  // [_______________________SSSSSSSS_____]
+  template <typename DstAddrT, typename SrcAddrT>
+  static inline void move(DstAddrT dst, SrcAddrT src, size_t runtime_size) {
+    const auto tail_value = SizedOp::load(tailAddr<SIZE>(src, runtime_size));
+    size_t offset = 0;
+    do {
+      SizedOp::move(offsetAddrMultiplesOf<SIZE>(dst, offset),
+                    offsetAddrMultiplesOf<SIZE>(src, offset));
+      offset += SIZE;
+    } while (offset < runtime_size - SIZE);
+    SizedOp::store(tailAddr<SIZE>(dst, runtime_size), tail_value);
+  }
+
+  // Move backward suitable when dst > src. We load the head bytes before
+  // handling the loop.
+  //
+  // e.g. Moving two bytes
+  // [   |       |       |       |       |]
+  // [___XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX___]
+  // [___LLLLLLLL_________________________]
+  // [_________________________LLLLLLLL___]
+  // [___________________________SSSSSSSS_]
+  // [_________________LLLLLLLL___________]
+  // [___________________SSSSSSSS_________]
+  // [_________LLLLLLLL___________________]
+  // [___________SSSSSSSS_________________]
+  // [_____SSSSSSSS_______________________]
+  template <typename DstAddrT, typename SrcAddrT>
+  static inline void move_backward(DstAddrT dst, SrcAddrT src,
+                                   size_t runtime_size) {
+    const auto head_value = SizedOp::load(src);
+    ptrdiff_t offset = runtime_size - SIZE;
+    do {
+      SizedOp::move(offsetAddrMultiplesOf<SIZE>(dst, offset),
+                    offsetAddrMultiplesOf<SIZE>(src, offset));
+      offset -= SIZE;
+    } while (offset >= 0);
+    SizedOp::store(dst, head_value);
+  }
+
+  template <typename DstAddrT>
+  static inline void set(DstAddrT dst, ubyte value, size_t runtime_size) {
+    size_t offset = 0;
+    do {
+      SizedOp::set(offsetAddrMultiplesOf<SIZE>(dst, offset), value);
+      offset += SIZE;
+    } while (offset < runtime_size - SIZE);
+    Tail<SizedOp>::set(dst, value, runtime_size);
+  }
+
+  template <typename SrcAddrT1, typename SrcAddrT2>
+  static inline uint64_t different(SrcAddrT1 src1, SrcAddrT2 src2,
+                                   size_t runtime_size) {
+    size_t offset = 0;
+    do {
+      if (auto res =
+              SizedOp::different(offsetAddrMultiplesOf<SIZE>(src1, offset),
+                                 offsetAddrMultiplesOf<SIZE>(src2, offset)))
+        return res;
+      offset += SIZE;
+    } while (offset < runtime_size - SIZE);
+    return Tail<SizedOp>::different(src1, src2, runtime_size);
+  }
+
+  template <typename SrcAddrT1, typename SrcAddrT2>
+  static inline int32_t three_way_cmp(SrcAddrT1 src1, SrcAddrT2 src2,
+                                      size_t runtime_size) {
+    size_t offset = 0;
+    do {
+      if (auto res =
+              SizedOp::three_way_cmp(offsetAddrMultiplesOf<SIZE>(src1, offset),
+                                     offsetAddrMultiplesOf<SIZE>(src2, offset)))
+        return res;
+      offset += SIZE;
+    } while (offset < runtime_size - SIZE);
+    return Tail<SizedOp>::three_way_cmp(src1, src2, runtime_size);
+  }
+};
+
+// Aligns and calls the subsequent NextT operation
+//
+// e.g. A 16-byte Destination Aligned 32-byte Loop Copy can be written as:
+// Align<16, Arg::Dst>::Then<Loop<32>>::copy(dst, src, runtime_size);
+enum class Arg { _1, _2, Dst = _1, Src = _2, Lhs = _1, Rhs = _2 };
+template <typename SizedOp, Arg AlignOn = Arg::_1> struct Align {
+  template <typename NextT> struct Then {
+    template <typename DstAddrT, typename SrcAddrT>
+    static inline void copy(DstAddrT dst, SrcAddrT src, size_t runtime_size) {
+      SizedOp::copy(dst, src);
+      auto aligned = align(dst, src, runtime_size);
+      NextT::copy(aligned.arg1, aligned.arg2, aligned.size);
+    }
+
+    // Move forward suitable when dst < src. The alignment is performed with
+    // an HeadTail operation of size ∈ [Alignment, 2 x Alignment].
+    //
+    // e.g. Moving two bytes and making sure src is then aligned.
+    // [  |       |       |       |      ]
+    // [____XXXXXXXXXXXXXXXXXXXXXXXXXXXX_]
+    // [____LLLLLLLL_____________________]
+    // [___________LLLLLLLL______________]
+    // [_SSSSSSSS________________________]
+    // [________SSSSSSSS_________________]
+    //
+    // e.g. Moving two bytes and making sure dst is then aligned.
+    // [  |       |       |       |      ]
+    // [____XXXXXXXXXXXXXXXXXXXXXXXXXXXX_]
+    // [____LLLLLLLL_____________________]
+    // [______LLLLLLLL___________________]
+    // [_SSSSSSSS________________________]
+    // [___SSSSSSSS______________________]
+    template <typename DstAddrT, typename SrcAddrT>
+    static inline void move(DstAddrT dst, SrcAddrT src, size_t runtime_size) {
+      auto aligned_after_begin = align(dst, src, runtime_size);
+      // We move pointers forward by Size so we can perform HeadTail.
+      auto aligned = aligned_after_begin.stepForward();
+      HeadTail<SizedOp>::move(dst, src, runtime_size - aligned.size);
+      NextT::move(aligned.arg1, aligned.arg2, aligned.size);
+    }
+
+    // Move backward suitable when dst > src. The alignment is performed with
+    // an HeadTail operation of size ∈ [Alignment, 2 x Alignment].
+    //
+    // e.g. Moving two bytes backward and making sure src is then aligned.
+    // [  |       |       |       |      ]
+    // [____XXXXXXXXXXXXXXXXXXXXXXXX_____]
+    // [ _________________LLLLLLLL_______]
+    // [ ___________________LLLLLLLL_____]
+    // [____________________SSSSSSSS_____]
+    // [______________________SSSSSSSS___]
+    //
+    // e.g. Moving two bytes and making sure dst is then aligned.
+    // [  |       |       |       |      ]
+    // [____XXXXXXXXXXXXXXXXXXXXXXXX_____]
+    // [ _______________LLLLLLLL_________]
+    // [ ___________________LLLLLLLL_____]
+    // [__________________SSSSSSSS_______]
+    // [______________________SSSSSSSS___]
+    template <typename DstAddrT, typename SrcAddrT>
+    static inline void move_backward(DstAddrT dst, SrcAddrT src,
+                                     size_t runtime_size) {
+      const auto dst_end = offsetAddrAssumeAligned<1>(dst, runtime_size);
+      const auto src_end = offsetAddrAssumeAligned<1>(src, runtime_size);
+      auto aligned_after_end = align(dst_end, src_end, 0);
+      // We move pointers back by 2 x Size so we can perform HeadTail.
+      auto aligned = aligned_after_end.stepBack().stepBack();
+      HeadTail<SizedOp>::move(aligned.arg1, aligned.arg2, aligned.size);
+      NextT::move_backward(dst, src, runtime_size - aligned.size);
+    }
+
+    template <typename DstAddrT>
+    static inline void set(DstAddrT dst, ubyte value, size_t runtime_size) {
+      SizedOp::set(dst, value);
+      DstAddrT _(nullptr);
+      auto aligned = align(dst, _, runtime_size);
+      NextT::set(aligned.arg1, value, aligned.size);
+    }
+
+    template <typename SrcAddrT1, typename SrcAddrT2>
+    static inline uint64_t different(SrcAddrT1 src1, SrcAddrT2 src2,
+                                     size_t runtime_size) {
+      if (const auto res = SizedOp::different(src1, src2))
+        return res;
+      auto aligned = align(src1, src2, runtime_size);
+      return NextT::different(aligned.arg1, aligned.arg2, aligned.size);
+    }
+
+    template <typename SrcAddrT1, typename SrcAddrT2>
+    static inline int32_t three_way_cmp(SrcAddrT1 src1, SrcAddrT2 src2,
+                                        size_t runtime_size) {
+      if (const auto res = SizedOp::three_way_cmp(src1, src2))
+        return res;
+      auto aligned = align(src1, src2, runtime_size);
+      return NextT::three_way_cmp(aligned.arg1, aligned.arg2, aligned.size);
+    }
+  };
+
+private:
+  static constexpr size_t SIZE = SizedOp::SIZE;
+  static_assert(SIZE > 1);
+
+  template <typename Arg1AddrT, typename Arg2AddrT> struct Aligned {
+    Arg1AddrT arg1;
+    Arg2AddrT arg2;
+    size_t size;
+
+    Aligned stepForward() const {
+      return Aligned{offsetAddrMultiplesOf<SIZE>(arg1, SIZE),
+                     offsetAddrMultiplesOf<SIZE>(arg2, SIZE), size - SIZE};
+    }
+
+    Aligned stepBack() const {
+      return Aligned{offsetAddrMultiplesOf<SIZE>(arg1, -SIZE),
+                     offsetAddrMultiplesOf<SIZE>(arg2, -SIZE), size + SIZE};
+    }
+  };
+
+  template <typename Arg1AddrT, typename Arg2AddrT>
+  static auto makeAligned(Arg1AddrT arg1, Arg2AddrT arg2, size_t size) {
+    return Aligned<Arg1AddrT, Arg2AddrT>{arg1, arg2, size};
+  }
+
+  template <typename Arg1AddrT, typename Arg2AddrT>
+  static auto align(Arg1AddrT arg1, Arg2AddrT arg2, size_t runtime_size) {
+    static_assert(Arg1AddrT::IS_ADDRESS_TYPE);
+    static_assert(Arg2AddrT::IS_ADDRESS_TYPE);
+    if constexpr (AlignOn == Arg::_1) {
+      auto offset = offset_to_next_aligned<SIZE>(arg1.ptr_);
+      return makeAligned(offsetAddrAssumeAligned<SIZE>(arg1, offset),
+                         offsetAddrAssumeAligned<1>(arg2, offset),
+                         runtime_size - offset);
+    } else if constexpr (AlignOn == Arg::_2) {
+      auto offset = offset_to_next_aligned<SIZE>(arg2.ptr_);
+      return makeAligned(offsetAddrAssumeAligned<1>(arg1, offset),
+                         offsetAddrAssumeAligned<SIZE>(arg2, offset),
+                         runtime_size - offset);
+    } else {
+      deferred_static_assert("AlignOn must be either Arg::_1 or Arg::_2");
+    }
+  }
+};
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ALGORITHM_H

libc/src/string/memory_utils/backend_aarch64.h

+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BACKEND_AARCH64_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BACKEND_AARCH64_H
+
+#if !defined(LLVM_LIBC_ARCH_AARCH64)
+#include "src/string/memory_utils/backend_scalar.h"
+
+#ifdef __ARM_NEON
+#include <arm_neon.h>
+#endif
+
+namespace __llvm_libc {
+
+struct Aarch64Backend : public ScalarBackend {
+  static constexpr bool IS_BACKEND_TYPE = true;
+
+  template <typename T, Temporal TS, Aligned AS,
+            cpp::EnableIfType<ScalarBackend::IsScalarType<T>, bool> = true>
+  static inline T load(const T *src) {
+    return ScalarBackend::template load<T, TS, AS>(src);
+  }
+};
+
+// Implementation of the SizedOp abstraction for the set operation.
+struct Zva64 {
+  static constexpr size_t SIZE = 64;
+
+  template <typename DstAddrT>
+  static inline void set(DstAddrT dst, ubyte value) {
+#if __SIZEOF_POINTER__ == 4
+    asm("dc zva, %w[dst]" : : [dst] "r"(dst) : "memory");
+#else
+    asm("dc zva, %[dst]" : : [dst] "r"(dst) : "memory");
+#endif
+  }
+};
+
+inline static bool hasZva() {
+  uint64_t zva_val;
+  asm("mrs %[zva_val], dczid_el0" : [zva_val] "=r"(zva_val));
+  // DC ZVA is permitted if DZP, bit [4] is zero.
+  // BS, bits [3:0] is log2 of the block size in words.
+  // So the next line checks whether the instruction is permitted and block size
+  // is 16 words (i.e. 64 bytes).
+  return (zva_val & 0b11111) == 0b00100;
+}
+
+namespace aarch64 {
+using _1 = SizedOp<Aarch64Backend, 1>;
+using _2 = SizedOp<Aarch64Backend, 2>;
+using _3 = SizedOp<Aarch64Backend, 3>;
+using _4 = SizedOp<Aarch64Backend, 4>;
+using _8 = SizedOp<Aarch64Backend, 8>;
+using _16 = SizedOp<Aarch64Backend, 16>;
+using _32 = SizedOp<Aarch64Backend, 32>;
+using _64 = SizedOp<Aarch64Backend, 64>;
+using _128 = SizedOp<Aarch64Backend, 128>;
+} // namespace aarch64
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_ARCH_AARCH64
+
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BACKEND_AARCH64_H

libc/src/string/memory_utils/backend_scalar.h

+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BACKEND_SCALAR_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BACKEND_SCALAR_H
+
+#include "src/__support/CPP/TypeTraits.h" // ConditionalType, EnableIfType
+#include "src/__support/endian.h"
+
+namespace __llvm_libc {
+
+struct ScalarBackend {
+  static constexpr bool IS_BACKEND_TYPE = true;
+
+  template <typename T>
+  static constexpr bool IsScalarType =
+      cpp::IsSameV<T, uint8_t> || cpp::IsSameV<T, uint16_t> ||
+      cpp::IsSameV<T, uint32_t> || cpp::IsSameV<T, uint64_t>;
+
+  template <typename T, Temporal TS, Aligned AS>
+  static inline T load(const T *src) {
+    static_assert(IsScalarType<T>);
+    static_assert(TS == Temporal::YES,
+                  "Scalar load does not support non-temporal access");
+    return *src;
+  }
+
+  template <typename T, Temporal TS, Aligned AS>
+  static inline void store(T *dst, T value) {
+    static_assert(IsScalarType<T>);
+    static_assert(TS == Temporal::YES,
+                  "Scalar store does not support non-temporal access");
+    *dst = value;
+  }
+
+  template <typename T> static inline T splat(ubyte value) {
+    static_assert(IsScalarType<T>);
+    return (T(~0) / T(0xFF)) * T(value);
+  }
+
+  template <typename T> static inline uint64_t not_equals(T v1, T v2) {
+    static_assert(IsScalarType<T>);
+    return v1 ^ v2;
+  }
+
+  template <typename T> static inline int32_t three_way_cmp(T v1, T v2) {
+    deferred_static_assert("not implemented");
+  }
+
+  template <size_t Size>
+  using getNextType = cpp::ConditionalType<
+      Size >= 8, uint64_t,
+      cpp::ConditionalType<Size >= 4, uint32_t,
+                           cpp::ConditionalType<Size >= 2, uint16_t, uint8_t>>>;
+};
+
+template <>
+int32_t inline ScalarBackend::three_way_cmp<uint8_t>(uint8_t a, uint8_t b) {
+  const int16_t la = Endian::to_big_endian(a);
+  const int16_t lb = Endian::to_big_endian(b);
+  return la - lb;
+}
+template <>
+int32_t inline ScalarBackend::three_way_cmp<uint16_t>(uint16_t a, uint16_t b) {
+  const int32_t la = Endian::to_big_endian(a);
+  const int32_t lb = Endian::to_big_endian(b);
+  return la - lb;
+}
+template <>
+int32_t inline ScalarBackend::three_way_cmp<uint32_t>(uint32_t a, uint32_t b) {
+  const uint32_t la = Endian::to_big_endian(a);
+  const uint32_t lb = Endian::to_big_endian(b);
+  return la > lb ? 1 : la < lb ? -1 : 0;
+}
+template <>
+int32_t inline ScalarBackend::three_way_cmp<uint64_t>(uint64_t a, uint64_t b) {
+  const uint64_t la = Endian::to_big_endian(a);
+  const uint64_t lb = Endian::to_big_endian(b);
+  return la > lb ? 1 : la < lb ? -1 : 0;
+}
+
+namespace scalar {
+using _1 = SizedOp<ScalarBackend, 1>;
+using _2 = SizedOp<ScalarBackend, 2>;
+using _3 = SizedOp<ScalarBackend, 3>;
+using _4 = SizedOp<ScalarBackend, 4>;
+using _8 = SizedOp<ScalarBackend, 8>;
+using _16 = SizedOp<ScalarBackend, 16>;
+using _32 = SizedOp<ScalarBackend, 32>;
+using _64 = SizedOp<ScalarBackend, 64>;
+using _128 = SizedOp<ScalarBackend, 128>;
+} // namespace scalar
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BACKEND_SCALAR_H

libc/src/string/memory_utils/backend_x86.h

+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BACKEND_X86_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BACKEND_X86_H
+
+#if defined(LLVM_LIBC_ARCH_X86)
+#include "src/__support/CPP/TypeTraits.h" // ConditionalType, EnableIfType
+#include "src/string/memory_utils/backend_scalar.h"
+
+#ifdef __SSE2__
+#include <immintrin.h>
+#endif //  __SSE2__
+
+#if defined(__SSE2__)
+#define HAS_M128 true
+#else
+#define HAS_M128 false
+#endif
+
+#if defined(__AVX2__)
+#define HAS_M256 true
+#else
+#define HAS_M256 false
+#endif
+
+#if defined(__AVX512F__) and defined(__AVX512BW__)
+#define HAS_M512 true
+#else
+#define HAS_M512 false
+#endif
+
+namespace __llvm_libc {
+struct X86Backend : public ScalarBackend {
+  static constexpr bool IS_BACKEND_TYPE = true;
+
+  template <typename T, Temporal TS, Aligned AS,
+            cpp::EnableIfType<ScalarBackend::IsScalarType<T>, bool> = true>
+  static inline T load(const T *src) {
+    return ScalarBackend::template load<T, TS, AS>(src);
+  }
+
+  template <typename T, Temporal TS, Aligned AS,
+            cpp::EnableIfType<!ScalarBackend::IsScalarType<T>, bool> = true>
+  static inline T load(const T *src) {
+    deferred_static_assert("X86 non-temporal load needs aligned access");
+    return {};
+  }
+
+  template <typename T, Temporal TS, Aligned AS,
+            cpp::EnableIfType<ScalarBackend::IsScalarType<T>, bool> = true>
+  static inline void store(T *dst, T value) {
+    ScalarBackend::template store<T, TS, AS>(dst, value);
+  }
+
+  template <typename T, Temporal TS, Aligned AS,
+            cpp::EnableIfType<!ScalarBackend::IsScalarType<T>, bool> = true>
+  static inline void store(T *dst, T value) {
+    deferred_static_assert("X86 non-temporal store needs aligned access");
+  }
+
+  template <typename T> static inline T splat(ubyte value) {
+    return ScalarBackend::template splat<T>(value);
+  }
+
+  template <typename T> static inline uint64_t not_equals(T v1, T v2) {
+    return ScalarBackend::template not_equals<T>(v1, v2);
+  }
+
+  template <typename T,
+            cpp::EnableIfType<ScalarBackend::IsScalarType<T>, bool> = true>
+  static inline int32_t three_way_cmp(T v1, T v2) {
+    return ScalarBackend::template three_way_cmp<T>(v1, v2);
+  }
+
+  template <typename T,
+            cpp::EnableIfType<!ScalarBackend::IsScalarType<T>, bool> = true>
+  static inline int32_t three_way_cmp(T v1, T v2) {
+    return char_diff(reinterpret_cast<char *>(&v1),
+                     reinterpret_cast<char *>(&v2), not_equals(v1, v2));
+  }
+
+  template <size_t Size>
+  using getNextType = cpp::ConditionalType<
+      (HAS_M512 && Size >= 64), __m512i,
+      cpp::ConditionalType<
+          (HAS_M256 && Size >= 32), __m256i,
+          cpp::ConditionalType<(HAS_M128 && Size >= 16), __m128i,
+                               ScalarBackend::getNextType<Size>>>>;
+
+private:
+  static inline int32_t char_diff(const char *a, const char *b, uint64_t mask) {
+    const size_t diff_index = __builtin_ctzll(mask);
+    const int16_t ca = (unsigned char)a[diff_index];
+    const int16_t cb = (unsigned char)b[diff_index];
+    return ca - cb;
+  }
+};
+
+static inline void repmovsb(void *dst, const void *src, size_t runtime_size) {
+  asm volatile("rep movsb"
+               : "+D"(dst), "+S"(src), "+c"(runtime_size)
+               :
+               : "memory");
+}
+
+#define SPECIALIZE_LOAD(T, OS, AS, INTRISIC)                                   \
+  template <> inline T X86Backend::load<T, OS, AS>(const T *src) {             \
+    return INTRISIC(const_cast<T *>(src));                                     \
+  }
+#define SPECIALIZE_STORE(T, OS, AS, INTRISIC)                                  \
+  template <> inline void X86Backend::store<T, OS, AS>(T * dst, T value) {     \
+    INTRISIC(dst, value);                                                      \
+  }
+
+#if HAS_M128
+SPECIALIZE_LOAD(__m128i, Temporal::YES, Aligned::YES, _mm_load_si128)
+SPECIALIZE_LOAD(__m128i, Temporal::YES, Aligned::NO, _mm_loadu_si128)
+SPECIALIZE_LOAD(__m128i, Temporal::NO, Aligned::YES, _mm_stream_load_si128)
+SPECIALIZE_STORE(__m128i, Temporal::YES, Aligned::YES, _mm_store_si128)
+SPECIALIZE_STORE(__m128i, Temporal::YES, Aligned::NO, _mm_storeu_si128)
+SPECIALIZE_STORE(__m128i, Temporal::NO, Aligned::YES, _mm_stream_si128)
+template <> inline __m128i X86Backend::splat<__m128i>(ubyte value) {
+  return _mm_set1_epi8(__builtin_bit_cast(char, value));
+}
+template <>
+inline uint64_t X86Backend::not_equals<__m128i>(__m128i a, __m128i b) {
+  using T = char __attribute__((__vector_size__(16)));
+  return _mm_movemask_epi8(T(a) != T(b));
+}
+#endif // HAS_M128
+
+#if HAS_M256
+SPECIALIZE_LOAD(__m256i, Temporal::YES, Aligned::YES, _mm256_load_si256)
+SPECIALIZE_LOAD(__m256i, Temporal::YES, Aligned::NO, _mm256_loadu_si256)
+SPECIALIZE_LOAD(__m256i, Temporal::NO, Aligned::YES, _mm256_stream_load_si256)
+SPECIALIZE_STORE(__m256i, Temporal::YES, Aligned::YES, _mm256_store_si256)
+SPECIALIZE_STORE(__m256i, Temporal::YES, Aligned::NO, _mm256_storeu_si256)
+SPECIALIZE_STORE(__m256i, Temporal::NO, Aligned::YES, _mm256_stream_si256)
+template <> inline __m256i X86Backend::splat<__m256i>(ubyte value) {
+  return _mm256_set1_epi8(__builtin_bit_cast(char, value));
+}
+template <>
+inline uint64_t X86Backend::not_equals<__m256i>(__m256i a, __m256i b) {
+  using T = char __attribute__((__vector_size__(32)));
+  return _mm256_movemask_epi8(T(a) != T(b));
+}
+#endif // HAS_M256
+
+#if HAS_M512
+SPECIALIZE_LOAD(__m512i, Temporal::YES, Aligned::YES, _mm512_load_si512)
+SPECIALIZE_LOAD(__m512i, Temporal::YES, Aligned::NO, _mm512_loadu_si512)
+SPECIALIZE_LOAD(__m512i, Temporal::NO, Aligned::YES, _mm512_stream_load_si512)
+SPECIALIZE_STORE(__m512i, Temporal::YES, Aligned::YES, _mm512_store_si512)
+SPECIALIZE_STORE(__m512i, Temporal::YES, Aligned::NO, _mm512_storeu_si512)
+SPECIALIZE_STORE(__m512i, Temporal::NO, Aligned::YES, _mm512_stream_si512)
+template <> inline __m512i X86Backend::splat<__m512i>(ubyte value) {
+  return _mm512_broadcastb_epi8(_mm_set1_epi8(__builtin_bit_cast(char, value)));
+}
+template <>
+inline uint64_t X86Backend::not_equals<__m512i>(__m512i a, __m512i b) {
+  return _mm512_cmpneq_epi8_mask(a, b);
+}
+#endif // HAS_M512
+
+namespace x86 {
+using _1 = SizedOp<X86Backend, 1>;
+using _2 = SizedOp<X86Backend, 2>;
+using _3 = SizedOp<X86Backend, 3>;
+using _4 = SizedOp<X86Backend, 4>;
+using _8 = SizedOp<X86Backend, 8>;
+using _16 = SizedOp<X86Backend, 16>;
+using _32 = SizedOp<X86Backend, 32>;
+using _64 = SizedOp<X86Backend, 64>;
+using _128 = SizedOp<X86Backend, 128>;
+} // namespace x86
+
+} // namespace __llvm_libc
+
+#endif // defined(LLVM_LIBC_ARCH_X86)
+
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BACKEND_X86_H

libc/src/string/memory_utils/backends.h

+//===-- Elementary operations to compose memory primitives ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the concept of a Backend.
+// It constitutes the lowest level of the framework and is akin to instruction
+// selection. It defines how to implement aligned/unaligned,
+// temporal/non-temporal loads and stores for a particular architecture as well
+// as efficient ways to fill and compare types.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BACKENDS_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BACKENDS_H
+
+#include "src/string/memory_utils/address.h"  // Temporal, Aligned
+#include "src/string/memory_utils/sized_op.h" // SizedOp
+#include <stddef.h>                           // size_t
+#include <stdint.h>                           // uint##_t
+
+namespace __llvm_libc {
+
+// Backends must implement the following interface.
+struct NoBackend {
+  static constexpr bool IS_BACKEND_TYPE = true;
+  // load
+  template <typename T, Temporal, Aligned> static T load(const T *src);
+
+  // store
+  template <typename T, Temporal, Aligned> static void store(T *dst, T value);
+
+  // return a T filled with `value`
+  template <typename T> static T splat(ubyte value);
+
+  // returns zero iff v1 == v2
+  template <typename T> static uint64_t not_equals(T v1, T v2);
+
+  // returns zero iff v1 == v2, -1 if v1 < v2 and 1 otherwise.
+  template <typename T> static int32_t three_way_diff(T v1, T v2);
+
+  template <size_t Size> using getNextType = void;
+};
+
+} // namespace __llvm_libc
+
+// We inline all backend implementations here to simplify the build system.
+// Each file need to be guarded with the appropriate LLVM_LIBC_ARCH_XXX ifdef.
+#include "src/string/memory_utils/backend_aarch64.h"
+#include "src/string/memory_utils/backend_scalar.h"
+#include "src/string/memory_utils/backend_x86.h"
+
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BACKENDS_H

libc/src/string/memory_utils/bcmp_implementations.h

 //===-- Implementation of bcmp --------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BCMP_IMPLEMENTATIONS_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BCMP_IMPLEMENTATIONS_H
 
 #include "src/__support/architectures.h"
-#include "src/__support/common.h"
-#include "src/string/memory_utils/elements.h"
+#include "src/string/memory_utils/algorithm.h"
+#include "src/string/memory_utils/backends.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
-// Fixed-size difference between 'lhs' and 'rhs'.
-template <typename Element> bool differs(const char *lhs, const char *rhs) {
-  return !Element::equals(lhs, rhs);
-}
-// Runtime-size difference between 'lhs' and 'rhs'.
-template <typename Element>
-bool differs(const char *lhs, const char *rhs, size_t size) {
-  return !Element::equals(lhs, rhs, size);
-}
-
-static inline int inline_bcmp(const char *lhs, const char *rhs, size_t count) {
+static inline uint64_t inline_bcmp_uint64_t(SrcAddr<1> lhs, SrcAddr<1> rhs,
+                                            size_t count) {
 #if defined(LLVM_LIBC_ARCH_X86)
-  using namespace ::__llvm_libc::x86;
+  using namespace x86;
 #elif defined(LLVM_LIBC_ARCH_AARCH64)
-  using namespace ::__llvm_libc::aarch64;
+  using namespace aarch64;
 #else
-  using namespace ::__llvm_libc::scalar;
+  using namespace scalar;
 #endif
   if (count == 0)
     return 0;
   if (count == 1)
-    return differs<_1>(lhs, rhs);
+    return _1::different(lhs, rhs);
   if (count == 2)
-    return differs<_2>(lhs, rhs);
+    return _2::different(lhs, rhs);
   if (count == 3)
-    return differs<_3>(lhs, rhs);
+    return _3::different(lhs, rhs);
   if (count <= 8)
-    return differs<HeadTail<_4>>(lhs, rhs, count);
+    return HeadTail<_4>::different(lhs, rhs, count);
   if (count <= 16)
-    return differs<HeadTail<_8>>(lhs, rhs, count);
+    return HeadTail<_8>::different(lhs, rhs, count);
   if (count <= 32)
-    return differs<HeadTail<_16>>(lhs, rhs, count);
+    return HeadTail<_16>::different(lhs, rhs, count);
   if (count <= 64)
-    return differs<HeadTail<_32>>(lhs, rhs, count);
+    return HeadTail<_32>::different(lhs, rhs, count);
   if (count <= 128)
-    return differs<HeadTail<_64>>(lhs, rhs, count);
-  return differs<Align<_32>::Then<Loop<_32>>>(lhs, rhs, count);
+    return HeadTail<_64>::different(lhs, rhs, count);
+  return Align<_32>::Then<Loop<_32>>::different(lhs, rhs, count);
+}
+
+static inline int inline_bcmp(SrcAddr<1> lhs, SrcAddr<1> rhs, size_t count) {
+  return !!inline_bcmp_uint64_t(lhs, rhs, count);
 }
 
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BCMP_IMPLEMENTATIONS_H

libc/src/string/memory_utils/memcmp_implementations.h

 //===-- Implementation of memcmp ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP_IMPLEMENTATIONS_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP_IMPLEMENTATIONS_H
 
 #include "src/__support/architectures.h"
-#include "src/__support/common.h"
-#include "src/string/memory_utils/elements.h"
+#include "src/string/memory_utils/algorithm.h"
+#include "src/string/memory_utils/backends.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
-static inline int inline_memcmp(const char *lhs, const char *rhs,
-                                size_t count) {
+static inline int inline_memcmp(SrcAddr<1> lhs, SrcAddr<1> rhs, size_t count) {
 #if defined(LLVM_LIBC_ARCH_X86)
   /////////////////////////////////////////////////////////////////////////////
   // LLVM_LIBC_ARCH_X86
   /////////////////////////////////////////////////////////////////////////////
-  using namespace __llvm_libc::x86;
+  using namespace x86;
   if (count == 0)
     return 0;
   if (count == 1)
-    return three_way_compare<_1>(lhs, rhs);
+    return _1::three_way_cmp(lhs, rhs);
   if (count == 2)
-    return three_way_compare<_2>(lhs, rhs);
+    return _2::three_way_cmp(lhs, rhs);
   if (count == 3)
-    return three_way_compare<_3>(lhs, rhs);
+    return _3::three_way_cmp(lhs, rhs);
   if (count <= 8)
-    return three_way_compare<HeadTail<_4>>(lhs, rhs, count);
+    return HeadTail<_4>::three_way_cmp(lhs, rhs, count);
   if (count <= 16)
-    return three_way_compare<HeadTail<_8>>(lhs, rhs, count);
+    return HeadTail<_8>::three_way_cmp(lhs, rhs, count);
   if (count <= 32)
-    return three_way_compare<HeadTail<_16>>(lhs, rhs, count);
+    return HeadTail<_16>::three_way_cmp(lhs, rhs, count);
   if (count <= 64)
-    return three_way_compare<HeadTail<_32>>(lhs, rhs, count);
+    return HeadTail<_32>::three_way_cmp(lhs, rhs, count);
   if (count <= 128)
-    return three_way_compare<HeadTail<_64>>(lhs, rhs, count);
-  return three_way_compare<Align<_32>::Then<Loop<_32>>>(lhs, rhs, count);
+    return HeadTail<_64>::three_way_cmp(lhs, rhs, count);
+  return Align<_32>::Then<Loop<_32>>::three_way_cmp(lhs, rhs, count);
 #elif defined(LLVM_LIBC_ARCH_AARCH64)
   /////////////////////////////////////////////////////////////////////////////
   // LLVM_LIBC_ARCH_AARCH64
   /////////////////////////////////////////////////////////////////////////////
-  using namespace ::__llvm_libc::aarch64;
+  using namespace aarch64;
   if (count == 0) // [0, 0]
     return 0;
   if (count == 1) // [1, 1]
-    return three_way_compare<_1>(lhs, rhs);
+    return _1::three_way_cmp(lhs, rhs);
   if (count == 2) // [2, 2]
-    return three_way_compare<_2>(lhs, rhs);
+    return _2::three_way_cmp(lhs, rhs);
   if (count == 3) // [3, 3]
-    return three_way_compare<_3>(lhs, rhs);
+    return _3::three_way_cmp(lhs, rhs);
   if (count < 8) // [4, 7]
-    return three_way_compare<HeadTail<_4>>(lhs, rhs, count);
+    return HeadTail<_4>::three_way_cmp(lhs, rhs, count);
   if (count < 16) // [8, 15]
-    return three_way_compare<HeadTail<_8>>(lhs, rhs, count);
+    return HeadTail<_8>::three_way_cmp(lhs, rhs, count);
   if (unlikely(count >= 128)) // [128, ∞]
-    return three_way_compare<Align<_16>::Then<Loop<_32>>>(lhs, rhs, count);
-  if (!equals<_16>(lhs, rhs)) // [16, 16]
-    return three_way_compare<_16>(lhs, rhs);
+    return Align<_16>::Then<Loop<_32>>::three_way_cmp(lhs, rhs, count);
+  if (_16::different(lhs, rhs)) // [16, 16]
+    return _16::three_way_cmp(lhs, rhs);
   if (count < 32) // [17, 31]
-    return three_way_compare<Tail<_16>>(lhs, rhs, count);
-  if (!equals<Skip<16>::Then<_16>>(lhs, rhs)) // [32, 32]
-    return three_way_compare<Skip<16>::Then<_16>>(lhs, rhs);
+    return Tail<_16>::three_way_cmp(lhs, rhs, count);
+  if (Skip<16>::Then<_16>::different(lhs, rhs)) // [32, 32]
+    return Skip<16>::Then<_16>::three_way_cmp(lhs, rhs);
   if (count < 64) // [33, 63]
-    return three_way_compare<Tail<_32>>(lhs, rhs, count);
+    return Tail<_32>::three_way_cmp(lhs, rhs, count);
   // [64, 127]
-  return three_way_compare<Skip<32>::Then<Loop<_16>>>(lhs, rhs, count);
+  return Skip<32>::Then<Loop<_16>>::three_way_cmp(lhs, rhs, count);
 #else
   /////////////////////////////////////////////////////////////////////////////
   // Default
   /////////////////////////////////////////////////////////////////////////////
-  using namespace ::__llvm_libc::scalar;
-
+  using namespace scalar;
   if (count == 0)
     return 0;
   if (count == 1)
-    return three_way_compare<_1>(lhs, rhs);
+    return _1::three_way_cmp(lhs, rhs);
   if (count == 2)
-    return three_way_compare<_2>(lhs, rhs);
+    return _2::three_way_cmp(lhs, rhs);
   if (count == 3)
-    return three_way_compare<_3>(lhs, rhs);
+    return _3::three_way_cmp(lhs, rhs);
   if (count <= 8)
-    return three_way_compare<HeadTail<_4>>(lhs, rhs, count);
+    return HeadTail<_4>::three_way_cmp(lhs, rhs, count);
   if (count <= 16)
-    return three_way_compare<HeadTail<_8>>(lhs, rhs, count);
+    return HeadTail<_8>::three_way_cmp(lhs, rhs, count);
   if (count <= 32)
-    return three_way_compare<HeadTail<_16>>(lhs, rhs, count);
+    return HeadTail<_16>::three_way_cmp(lhs, rhs, count);
   if (count <= 64)
-    return three_way_compare<HeadTail<_32>>(lhs, rhs, count);
+    return HeadTail<_32>::three_way_cmp(lhs, rhs, count);
   if (count <= 128)
-    return three_way_compare<HeadTail<_64>>(lhs, rhs, count);
-  return three_way_compare<Align<_32>::Then<Loop<_32>>>(lhs, rhs, count);
+    return HeadTail<_64>::three_way_cmp(lhs, rhs, count);
+  return Align<_32>::Then<Loop<_32>>::three_way_cmp(lhs, rhs, count);
 #endif
 }
 
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP_IMPLEMENTATIONS_H

libc/src/string/memory_utils/memcpy_implementations.h

 //===-- Memcpy implementation -----------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
 
+#include "src/__support/CPP/TypeTraits.h" // cpp::ConditionalType
 #include "src/__support/architectures.h"
-#include "src/__support/common.h"
-#include "src/string/memory_utils/elements.h"
-#include "src/string/memory_utils/utils.h"
+#include "src/__support/common.h" // LLVM_LIBC_IS_DEFINED
+#include "src/string/memory_utils/algorithm.h"
+#include "src/string/memory_utils/backends.h"
 
 #include <stddef.h> // size_t
 
 // Design rationale
 // ================
 //
 // Using a profiler to observe size distributions for calls into libc
 // functions, it was found most operations act on a small number of bytes.
 // This makes it important to favor small sizes.
 //
 // The tests for `count` are in ascending order so the cost of branching is
 // proportional to the cost of copying.
 //
 // The function is written in C++ for several reasons:
 // - The compiler can __see__ the code, this is useful when performing Profile
 //   Guided Optimization as the optimized code can take advantage of branching
 //   probabilities.
 // - It also allows for easier customization and favors testing multiple
 //   implementation parameters.
 // - As compilers and processors get better, the generated code is improved
 //   with little change on the code side.
 
 namespace __llvm_libc {
 
-static inline void inline_memcpy(char *__restrict dst,
-                                 const char *__restrict src, size_t count) {
+static inline void inline_memcpy(DstAddr<1> dst, SrcAddr<1> src, size_t count) {
 #if defined(LLVM_LIBC_ARCH_X86)
   /////////////////////////////////////////////////////////////////////////////
   // LLVM_LIBC_ARCH_X86
   /////////////////////////////////////////////////////////////////////////////
-  using namespace __llvm_libc::x86;
-
+  // using x86 = Algorithms<X86Backend>;
   // Whether to use only rep;movsb.
   constexpr bool USE_ONLY_REP_MOVSB =
       LLVM_LIBC_IS_DEFINED(LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB);
 
   // kRepMovsBSize == -1 : Only CopyAligned is used.
   // kRepMovsBSize ==  0 : Only RepMovsb is used.
   // else CopyAligned is used up to kRepMovsBSize and then RepMovsb.
   constexpr size_t REP_MOVS_B_SIZE =
 #if defined(LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE)
       LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE;
 #else
       -1;
 #endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
 
+  using namespace x86;
+
   // Whether target supports AVX instructions.
   constexpr bool HAS_AVX = LLVM_LIBC_IS_DEFINED(__AVX__);
-
-#if defined(__AVX__)
-  using LoopBlockSize = _64;
-#else
-  using LoopBlockSize = _32;
-#endif
-
   if (USE_ONLY_REP_MOVSB)
-    return copy<Accelerator>(dst, src, count);
-
+    return repmovsb(as<void>(dst), as<const void>(src), count);
   if (count == 0)
     return;
   if (count == 1)
-    return copy<_1>(dst, src);
+    return _1::copy(dst, src);
   if (count == 2)
-    return copy<_2>(dst, src);
+    return _2::copy(dst, src);
   if (count == 3)
-    return copy<_3>(dst, src);
+    return _3::copy(dst, src);
   if (count == 4)
-    return copy<_4>(dst, src);
+    return _4::copy(dst, src);
   if (count < 8)
-    return copy<HeadTail<_4>>(dst, src, count);
+    return HeadTail<_4>::copy(dst, src, count);
   if (count < 16)
-    return copy<HeadTail<_8>>(dst, src, count);
+    return HeadTail<_8>::copy(dst, src, count);
   if (count < 32)
-    return copy<HeadTail<_16>>(dst, src, count);
+    return HeadTail<_16>::copy(dst, src, count);
   if (count < 64)
-    return copy<HeadTail<_32>>(dst, src, count);
+    return HeadTail<_32>::copy(dst, src, count);
   if (count < 128)
-    return copy<HeadTail<_64>>(dst, src, count);
+    return HeadTail<_64>::copy(dst, src, count);
   if (HAS_AVX && count < 256)
-    return copy<HeadTail<_128>>(dst, src, count);
+    return HeadTail<_128>::copy(dst, src, count);
+  using LoopBlock = cpp::ConditionalType<HAS_AVX, _64, _32>;
   if (count <= REP_MOVS_B_SIZE)
-    return copy<Align<_32, Arg::Dst>::Then<Loop<LoopBlockSize>>>(dst, src,
-                                                                 count);
-  return copy<Accelerator>(dst, src, count);
+    return Align<_32, Arg::Dst>::Then<Loop<LoopBlock>>::copy(dst, src, count);
+  return repmovsb(as<void>(dst), as<const void>(src), count);
+
 #elif defined(LLVM_LIBC_ARCH_AARCH64)
   /////////////////////////////////////////////////////////////////////////////
   // LLVM_LIBC_ARCH_AARCH64
   /////////////////////////////////////////////////////////////////////////////
-  using namespace __llvm_libc::scalar;
+  using namespace scalar;
   if (count == 0)
     return;
   if (count == 1)
-    return copy<_1>(dst, src);
+    return _1::copy(dst, src);
   if (count == 2)
-    return copy<_2>(dst, src);
+    return _2::copy(dst, src);
   if (count == 3)
-    return copy<_3>(dst, src);
+    return _3::copy(dst, src);
   if (count == 4)
-    return copy<_4>(dst, src);
+    return _4::copy(dst, src);
   if (count < 8)
-    return copy<HeadTail<_4>>(dst, src, count);
+    return HeadTail<_4>::copy(dst, src, count);
   if (count < 16)
-    return copy<HeadTail<_8>>(dst, src, count);
+    return HeadTail<_8>::copy(dst, src, count);
   if (count < 32)
-    return copy<HeadTail<_16>>(dst, src, count);
+    return HeadTail<_16>::copy(dst, src, count);
   if (count < 64)
-    return copy<HeadTail<_32>>(dst, src, count);
+    return HeadTail<_32>::copy(dst, src, count);
   if (count < 128)
-    return copy<HeadTail<_64>>(dst, src, count);
-  return copy<Align<_16, Arg::Src>::Then<Loop<_64>>>(dst, src, count);
+    return HeadTail<_64>::copy(dst, src, count);
+  return Align<_16, Arg::Src>::Then<Loop<_64>>::copy(dst, src, count);
 #else
   /////////////////////////////////////////////////////////////////////////////
   // Default
   /////////////////////////////////////////////////////////////////////////////
-  using namespace __llvm_libc::scalar;
+  using namespace scalar;
   if (count == 0)
     return;
   if (count == 1)
-    return copy<_1>(dst, src);
+    return _1::copy(dst, src);
   if (count == 2)
-    return copy<_2>(dst, src);
+    return _2::copy(dst, src);
   if (count == 3)
-    return copy<_3>(dst, src);
+    return _3::copy(dst, src);
   if (count == 4)
-    return copy<_4>(dst, src);
+    return _4::copy(dst, src);
   if (count < 8)
-    return copy<HeadTail<_4>>(dst, src, count);
+    return HeadTail<_4>::copy(dst, src, count);
   if (count < 16)
-    return copy<HeadTail<_8>>(dst, src, count);
+    return HeadTail<_8>::copy(dst, src, count);
   if (count < 32)
-    return copy<HeadTail<_16>>(dst, src, count);
+    return HeadTail<_16>::copy(dst, src, count);
   if (count < 64)
-    return copy<HeadTail<_32>>(dst, src, count);
+    return HeadTail<_32>::copy(dst, src, count);
   if (count < 128)
-    return copy<HeadTail<_64>>(dst, src, count);
-  return copy<Align<_32, Arg::Src>::Then<Loop<_32>>>(dst, src, count);
+    return HeadTail<_64>::copy(dst, src, count);
+  return Align<_32, Arg::Src>::Then<Loop<_32>>::copy(dst, src, count);
 #endif
 }
 
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H

libc/src/string/memory_utils/memset_implementations.h

 //===-- Implementation of memset and bzero --------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H
 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H
 
 #include "src/__support/architectures.h"
-#include "src/string/memory_utils/elements.h"
-#include "src/string/memory_utils/utils.h"
+#include "src/string/memory_utils/algorithm.h"
+#include "src/string/memory_utils/backends.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {
 
 // A general purpose implementation assuming cheap unaligned writes for sizes:
 // 1, 2, 4, 8, 16, 32 and 64 Bytes. Note that some architecture can't store 32
 // or 64 Bytes at a time, the compiler will expand them as needed.

 // conditional were expensive, even on processor that do not support writing 64B
 // at a time (pre-AVX512F). We also want to favor short functions that allow
 // more hot code to fit in the iL1 cache.
 //
 // Above 128 we have to use conditionals since we don't know the upper bound in
 // advance. SetAlignedBlocks<64> may waste up to 63 Bytes, SetAlignedBlocks<32>
 // may waste up to 31 Bytes. Benchmarks showed that SetAlignedBlocks<64> was not
 // superior for sizes that mattered.
-inline static void inline_memset(char *dst, unsigned char value, size_t count) {
+inline static void inline_memset(DstAddr<1> dst, ubyte value, size_t count) {
 #if defined(LLVM_LIBC_ARCH_X86)
   /////////////////////////////////////////////////////////////////////////////
   // LLVM_LIBC_ARCH_X86
   /////////////////////////////////////////////////////////////////////////////
-  using namespace __llvm_libc::x86;
+  using namespace x86;
   if (count == 0)
     return;
   if (count == 1)
-    return splat_set<_1>(dst, value);
+    return _1::set(dst, value);
   if (count == 2)
-    return splat_set<_2>(dst, value);
+    return _2::set(dst, value);
   if (count == 3)
-    return splat_set<_3>(dst, value);
+    return _3::set(dst, value);
   if (count <= 8)
-    return splat_set<HeadTail<_4>>(dst, value, count);
+    return HeadTail<_4>::set(dst, value, count);
   if (count <= 16)
-    return splat_set<HeadTail<_8>>(dst, value, count);
+    return HeadTail<_8>::set(dst, value, count);
   if (count <= 32)
-    return splat_set<HeadTail<_16>>(dst, value, count);
+    return HeadTail<_16>::set(dst, value, count);
   if (count <= 64)
-    return splat_set<HeadTail<_32>>(dst, value, count);
+    return HeadTail<_32>::set(dst, value, count);
   if (count <= 128)
-    return splat_set<HeadTail<_64>>(dst, value, count);
-  return splat_set<Align<_32, Arg::Dst>::Then<Loop<_32>>>(dst, value, count);
+    return HeadTail<_64>::set(dst, value, count);
+  return Align<_32, Arg::Dst>::Then<Loop<_32>>::set(dst, value, count);
 #elif defined(LLVM_LIBC_ARCH_AARCH64)
   /////////////////////////////////////////////////////////////////////////////
   // LLVM_LIBC_ARCH_AARCH64
   /////////////////////////////////////////////////////////////////////////////
-  using namespace __llvm_libc::aarch64_memset;
+  using namespace aarch64;
   if (count == 0)
     return;
   if (count <= 3) {
-    splat_set<_1>(dst, value);
+    _1::set(dst, value);
     if (count > 1)
-      splat_set<Tail<_2>>(dst, value, count);
+      Tail<_2>::set(dst, value, count);
     return;
   }
   if (count <= 8)
-    return splat_set<HeadTail<_4>>(dst, value, count);
+    return HeadTail<_4>::set(dst, value, count);
   if (count <= 16)
-    return splat_set<HeadTail<_8>>(dst, value, count);
+    return HeadTail<_8>::set(dst, value, count);
   if (count <= 32)
-    return splat_set<HeadTail<_16>>(dst, value, count);
+    return HeadTail<_16>::set(dst, value, count);
   if (count <= 96) {
-    splat_set<_32>(dst, value);
+    _32::set(dst, value);
     if (count <= 64)
-      return splat_set<Tail<_32>>(dst, value, count);
-    splat_set<Skip<32>::Then<_32>>(dst, value);
-    splat_set<Tail<_32>>(dst, value, count);
+      return Tail<_32>::set(dst, value, count);
+    Skip<32>::Then<_32>::set(dst, value);
+    Tail<_32>::set(dst, value, count);
     return;
   }
-  if (count < 448 || value != 0 || !AArch64ZVA(dst, count))
-    return splat_set<Align<_16, Arg::_1>::Then<Loop<_64>>>(dst, value, count);
+  if (count >= 448 && value == ubyte::ZERO && hasZva())
+    return Align<_64, Arg::_1>::Then<Loop<Zva64>>::set(dst, ubyte::ZERO, count);
+  else
+    return Align<_16, Arg::_1>::Then<Loop<_64>>::set(dst, value, count);
 #else
   /////////////////////////////////////////////////////////////////////////////
   // Default
   /////////////////////////////////////////////////////////////////////////////
-  using namespace ::__llvm_libc::scalar;
-
+  using namespace scalar;
   if (count == 0)
     return;
   if (count == 1)
-    return splat_set<_1>(dst, value);
+    return _1::set(dst, value);
   if (count == 2)
-    return splat_set<_2>(dst, value);
+    return _2::set(dst, value);
   if (count == 3)
-    return splat_set<_3>(dst, value);
+    return _3::set(dst, value);
   if (count <= 8)
-    return splat_set<HeadTail<_4>>(dst, value, count);
+    return HeadTail<_4>::set(dst, value, count);
   if (count <= 16)
-    return splat_set<HeadTail<_8>>(dst, value, count);
+    return HeadTail<_8>::set(dst, value, count);
   if (count <= 32)
-    return splat_set<HeadTail<_16>>(dst, value, count);
+    return HeadTail<_16>::set(dst, value, count);
   if (count <= 64)
-    return splat_set<HeadTail<_32>>(dst, value, count);
+    return HeadTail<_32>::set(dst, value, count);
   if (count <= 128)
-    return splat_set<HeadTail<_64>>(dst, value, count);
-  return splat_set<Align<_32, Arg::Dst>::Then<Loop<_32>>>(dst, value, count);
+    return HeadTail<_64>::set(dst, value, count);
+  return Align<_32, Arg::Dst>::Then<Loop<_32>>::set(dst, value, count);
 #endif
 }
 
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H

libc/src/string/memory_utils/sized_op.h

+//===-- Sized Operations --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the SizedOp struct that serves as the middle end of the
+// framework. It implements sized memory operations by breaking them down into
+// simpler types which availability is described in the Backend. It also
+// provides a way to load and store sized chunks of memory (necessary for the
+// move operation). SizedOp are the building blocks of higher order algorithms
+// like HeadTail, Align or Loop.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_SIZED_OP_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_SIZED_OP_H
+
+#include <stddef.h> // size_t
+
+#ifndef LLVM_LIBC_USE_BUILTIN_MEMCPY_INLINE
+#define LLVM_LIBC_USE_BUILTIN_MEMCPY_INLINE                                    \
+  __has_builtin(__builtin_memcpy_inline)
+#endif // LLVM_LIBC_USE_BUILTIN_MEMCPY_INLINE
+
+namespace __llvm_libc {
+
+template <typename Backend, size_t Size> struct SizedOp {
+  static constexpr size_t SIZE = Size;
+
+private:
+  static_assert(Backend::IS_BACKEND_TYPE);
+  static_assert(Size > 0);
+  using type = typename Backend::template getNextType<Size>;
+  static constexpr size_t TYPE_SIZE = sizeof(type);
+  static_assert(Size >= TYPE_SIZE);
+  static constexpr size_t NEXT_SIZE = Size - TYPE_SIZE;
+  using NextBlock = SizedOp<Backend, NEXT_SIZE>;
+
+  template <typename T, typename AddrT> static constexpr Aligned isAligned() {
+    static_assert(AddrT::IS_ADDRESS_TYPE);
+    return AddrT::ALIGNMENT > 1 && AddrT::ALIGNMENT >= sizeof(T) ? Aligned::YES
+                                                                 : Aligned::NO;
+  }
+
+  template <typename SrcAddrT> static inline auto loadType(SrcAddrT src) {
+    static_assert(SrcAddrT::IS_ADDRESS_TYPE && SrcAddrT::IS_READ);
+    constexpr auto AS = isAligned<type, SrcAddrT>();
+    constexpr auto TS = SrcAddrT::TEMPORAL;
+    return Backend::template load<type, TS, AS>(as<const type>(src));
+  }
+
+  template <typename DstAddrT>
+  static inline void storeType(type value, DstAddrT dst) {
+    static_assert(DstAddrT::IS_ADDRESS_TYPE && DstAddrT::IS_WRITE);
+    constexpr auto AS = isAligned<type, DstAddrT>();
+    constexpr auto TS = DstAddrT::TEMPORAL;
+    return Backend::template store<type, TS, AS>(as<type>(dst), value);
+  }
+
+  struct Value {
+    alignas(alignof(type)) ubyte payload[Size];
+  };
+
+public:
+  template <typename DstAddrT, typename SrcAddrT>
+  static inline void copy(DstAddrT dst, SrcAddrT src) {
+    static_assert(DstAddrT::IS_ADDRESS_TYPE && DstAddrT::IS_WRITE);
+    static_assert(SrcAddrT::IS_ADDRESS_TYPE && SrcAddrT::IS_READ);
+    if constexpr (LLVM_LIBC_USE_BUILTIN_MEMCPY_INLINE &&
+                  DstAddrT::TEMPORAL == Temporal::YES &&
+                  SrcAddrT::TEMPORAL == Temporal::YES) {
+      // delegate optimized copy to compiler.
+      __builtin_memcpy_inline(dst.ptr(), src.ptr(), Size);
+      return;
+    }
+    storeType(loadType(src), dst);
+    if constexpr (NEXT_SIZE)
+      NextBlock::copy(offsetAddr<TYPE_SIZE>(dst), offsetAddr<TYPE_SIZE>(src));
+  }
+
+  template <typename DstAddrT, typename SrcAddrT>
+  static inline void move(DstAddrT dst, SrcAddrT src) {
+    const auto payload = loadType(src);
+    if constexpr (NEXT_SIZE)
+      NextBlock::move(offsetAddr<TYPE_SIZE>(dst), offsetAddr<TYPE_SIZE>(src));
+    storeType(payload, dst);
+  }
+
+  template <typename DstAddrT>
+  static inline void set(DstAddrT dst, ubyte value) {
+    storeType(Backend::template splat<type>(value), dst);
+    if constexpr (NEXT_SIZE)
+      NextBlock::set(offsetAddr<TYPE_SIZE>(dst), value);
+  }
+
+  template <typename SrcAddrT1, typename SrcAddrT2>
+  static inline uint64_t different(SrcAddrT1 src1, SrcAddrT2 src2) {
+    const uint64_t current =
+        Backend::template not_equals<type>(loadType(src1), loadType(src2));
+    if constexpr (NEXT_SIZE) {
+      return current | (NextBlock::different(offsetAddr<TYPE_SIZE>(src1),
+                                             offsetAddr<TYPE_SIZE>(src2)));
+    } else {
+      return current;
+    }
+  }
+
+  template <typename SrcAddrT1, typename SrcAddrT2>
+  static inline int32_t three_way_cmp(SrcAddrT1 src1, SrcAddrT2 src2) {
+    const auto a = loadType(src1);
+    const auto b = loadType(src2);
+    if (Backend::template not_equals<type>(a, b))
+      return Backend::template three_way_cmp<type>(a, b);
+    // if (int32_t res = three_way_cmp<T>(a, b)) {
+    //   return res;
+    // }
+    if constexpr (NEXT_SIZE)
+      return NextBlock::three_way_cmp(offsetAddr<TYPE_SIZE>(src1),
+                                      offsetAddr<TYPE_SIZE>(src2));
+    return 0;
+  }
+
+  template <typename SrcAddrT> static Value load(SrcAddrT src) {
+    Value output;
+    copy(DstAddr<alignof(type)>(output.payload), src);
+    return output;
+  }
+
+  template <typename DstAddrT> static void store(DstAddrT dst, Value value) {
+    copy(dst, SrcAddr<alignof(type)>(value.payload));
+  }
+};
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_SIZED_OP_H

libc/src/string/mempcpy.cpp

 #include "src/string/memory_utils/memcpy_implementations.h"
 
 #include "src/__support/common.h"
 #include <stddef.h> // For size_t.
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(void *, mempcpy,
-                   (void *__restrict dest, const void *__restrict src,
+                   (void *__restrict dst, const void *__restrict src,
                     size_t count)) {
-  char *result = reinterpret_cast<char *>(dest);
-  inline_memcpy(result, reinterpret_cast<const char *>(src), count);
-  return result + count;
+  inline_memcpy(dst, src, count);
+  return reinterpret_cast<char *>(dst) + count;
 }
 
 } // namespace __llvm_libc

libc/src/string/memset.cpp

 //===-- Implementation of memset ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/memset.h"
+
 #include "src/__support/common.h"
 #include "src/string/memory_utils/memset_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(void *, memset, (void *dst, int value, size_t count)) {
-  inline_memset(reinterpret_cast<char *>(dst),
-                static_cast<unsigned char>(value), count);
+  inline_memset(dst, static_cast<ubyte>(value), count);
   return dst;
 }
 
 } // namespace __llvm_libc

libc/src/string/stpncpy.cpp

 //===-- Implementation of stpncpy -----------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/stpncpy.h"
-#include "src/string/memory_utils/memset_implementations.h"
 
 #include "src/__support/common.h"
+#include "src/string/memory_utils/memset_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(char *, stpncpy,
                    (char *__restrict dest, const char *__restrict src,
                     size_t n)) {
   size_t i;
   // Copy up until \0 is found.
   for (i = 0; i < n && src[i] != '\0'; ++i)
     dest[i] = src[i];
   // When n>strlen(src), n-strlen(src) \0 are appended.
   if (n > i)
-    inline_memset(dest + i, 0, n - i);
+    inline_memset(dest + i, ubyte::ZERO, n - i);
   return dest + i;
 }
 
 } // namespace __llvm_libc

libc/test/src/string/memory_utils/CMakeLists.txt

 add_libc_unittest(
   utils_test
   SUITE
     libc_string_unittests
   SRCS
+    address_test.cpp
+    algorithm_test.cpp
+    backend_test.cpp
     elements_test.cpp
     memory_access_test.cpp
     utils_test.cpp
   DEPENDS
     libc.src.string.memory_utils.memory_utils
     libc.src.__support.CPP.array
     libc.src.__support.CPP.array_ref
   COMPILE_OPTIONS
     ${LIBC_COMPILE_OPTIONS_NATIVE}
     -ffreestanding
 )

libc/test/src/string/memory_utils/address_test.cpp

+#include "utils/UnitTest/Test.h"
+#include <src/string/memory_utils/address.h>
+
+namespace __llvm_libc {
+
+TEST(LlvmLibcAddress, AliasAreAddresses) {
+  ASSERT_TRUE(SrcAddr<1>::IS_ADDRESS_TYPE);
+  ASSERT_TRUE(DstAddr<1>::IS_ADDRESS_TYPE);
+  ASSERT_TRUE(NtSrcAddr<1>::IS_ADDRESS_TYPE);
+  ASSERT_TRUE(NtDstAddr<1>::IS_ADDRESS_TYPE);
+}
+
+TEST(LlvmLibcAddress, AliasHaveRightPermissions) {
+  ASSERT_TRUE(SrcAddr<1>::IS_READ);
+  ASSERT_TRUE(NtSrcAddr<1>::IS_READ);
+  ASSERT_TRUE(DstAddr<1>::IS_WRITE);
+  ASSERT_TRUE(NtDstAddr<1>::IS_WRITE);
+}
+
+TEST(LlvmLibcAddress, AliasHaveRightSemantic) {
+  ASSERT_EQ(SrcAddr<1>::TEMPORAL, Temporal::YES);
+  ASSERT_EQ(DstAddr<1>::TEMPORAL, Temporal::YES);
+  ASSERT_EQ(NtSrcAddr<1>::TEMPORAL, Temporal::NO);
+  ASSERT_EQ(NtDstAddr<1>::TEMPORAL, Temporal::NO);
+}
+
+TEST(LlvmLibcAddress, AliasHaveRightAlignment) {
+  ASSERT_EQ(SrcAddr<1>::ALIGNMENT, size_t(1));
+  ASSERT_EQ(SrcAddr<4>::ALIGNMENT, size_t(4));
+}
+
+TEST(LlvmLibcAddress, NarrowAlignment) {
+  // Address 16-byte aligned, offset by 4.
+  ASSERT_EQ(offsetAddr<4>(SrcAddr<16>(nullptr)).ALIGNMENT, 4UL);
+  // Address 8-byte aligned, offset by 8.
+  ASSERT_EQ(offsetAddr<8>(SrcAddr<8>(nullptr)).ALIGNMENT, 8UL);
+  // Address 4-byte aligned, offset by 16.
+  ASSERT_EQ(offsetAddr<16>(SrcAddr<4>(nullptr)).ALIGNMENT, 4UL);
+}
+
+TEST(LlvmLibcAddress, OffsetAddr) {
+  ubyte a;
+  SrcAddr<1> addr(&a);
+  ASSERT_EQ((const void *)offsetAddr<4>(addr).ptr(), (const void *)(&a + 4));
+  ASSERT_EQ((const void *)offsetAddr<32>(addr).ptr(), (const void *)(&a + 32));
+}
+
+TEST(LlvmLibcAddress, AssumeAligned) {
+  SrcAddr<16> addr(nullptr);
+  ASSERT_EQ(offsetAddrAssumeAligned<8>(addr, 0).ALIGNMENT, 8UL);
+  ASSERT_EQ(offsetAddrAssumeAligned<1>(addr, 0).ALIGNMENT, 1UL);
+  ASSERT_EQ(offsetAddrMultiplesOf<4>(addr, 0).ALIGNMENT, 4UL);
+  ASSERT_EQ(offsetAddrMultiplesOf<32>(addr, 0).ALIGNMENT, 16UL);
+}
+
+TEST(LlvmLibcAddress, offsetAddrAssumeAligned) {
+  ubyte a;
+  SrcAddr<1> addr(&a);
+  ASSERT_EQ((const void *)offsetAddrAssumeAligned<1>(addr, 17).ptr(),
+            (const void *)(&a + 17));
+}
+
+TEST(LlvmLibcAddress, offsetAddrMultiplesOf) {
+  ubyte a;
+  SrcAddr<1> addr(&a);
+  ASSERT_EQ((const void *)offsetAddrMultiplesOf<4>(addr, 16).ptr(),
+            (const void *)(&a + 16));
+}
+
+} // namespace __llvm_libc

libc/test/src/string/memory_utils/algorithm_test.cpp

+
+#define LLVM_LIBC_USE_BUILTIN_MEMCPY_INLINE 0
+
+#include "utils/UnitTest/Test.h"
+#include <src/__support/CPP/Array.h>
+#include <src/string/memory_utils/algorithm.h>
+#include <src/string/memory_utils/backends.h>
+
+#include <sstream>
+
+namespace __llvm_libc {
+
+struct alignas(64) Buffer : cpp::Array<char, 128> {
+  bool contains(const char *ptr) const {
+    return ptr >= data() && ptr < (data() + size());
+  }
+  size_t getOffset(const char *ptr) const { return ptr - data(); }
+  void fill(char c) {
+    for (auto itr = begin(); itr != end(); ++itr)
+      *itr = c;
+  }
+};
+
+static Buffer buffer1;
+static Buffer buffer2;
+static std::ostringstream LOG;
+
+struct TestBackend {
+  static constexpr bool IS_BACKEND_TYPE = true;
+
+  template <typename T> static void log(const char *Action, const char *ptr) {
+    LOG << Action << "<" << sizeof(T) << "> ";
+    if (buffer1.contains(ptr))
+      LOG << "a[" << buffer1.getOffset(ptr) << "]";
+    else if (buffer2.contains(ptr))
+      LOG << "b[" << buffer2.getOffset(ptr) << "]";
+    LOG << "\n";
+  }
+
+  template <typename T, Temporal TS, Aligned AS> static T load(const T *src) {
+    log<T>((AS == Aligned::YES ? "LdA" : "LdU"),
+           reinterpret_cast<const char *>(src));
+    return ScalarBackend::load<T, TS, AS>(src);
+  }
+
+  template <typename T, Temporal TS, Aligned AS>
+  static void store(T *dst, T value) {
+    log<T>((AS == Aligned::YES ? "StA" : "StU"),
+           reinterpret_cast<const char *>(dst));
+    ScalarBackend::store<T, TS, AS>(dst, value);
+  }
+
+  template <typename T> static inline T splat(ubyte value) {
+    LOG << "Splat<" << sizeof(T) << "> " << (unsigned)value << '\n';
+    return ScalarBackend::splat<T>(value);
+  }
+
+  template <typename T> static inline uint64_t not_equals(T v1, T v2) {
+    LOG << "Neq<" << sizeof(T) << ">\n";
+    return ScalarBackend::not_equals<T>(v1, v2);
+  }
+
+  template <typename T> static inline int32_t three_way_cmp(T v1, T v2) {
+    LOG << "Diff<" << sizeof(T) << ">\n";
+    return ScalarBackend::three_way_cmp<T>(v1, v2);
+  }
+
+  template <size_t Size> using getNextType = ScalarBackend::getNextType<Size>;
+};
+
+struct LlvmLibcAlgorithm : public testing::Test {
+  void SetUp() override {
+    LOG = std::ostringstream();
+    LOG << '\n';
+  }
+
+  void fillEqual() {
+    buffer1.fill('a');
+    buffer2.fill('a');
+  }
+
+  void fillDifferent() {
+    buffer1.fill('a');
+    buffer2.fill('b');
+  }
+
+  const char *getTrace() {
+    trace_ = LOG.str();
+    return trace_.c_str();
+  }
+
+  const char *stripComments(const char *expected) {
+    expected_.clear();
+    std::stringstream ss(expected);
+    std::string line;
+    while (std::getline(ss, line, '\n')) {
+      const auto pos = line.find('#');
+      if (pos == std::string::npos) {
+        expected_ += line;
+      } else {
+        auto log = line.substr(0, pos);
+        while (!log.empty() && std::isspace(log.back()))
+          log.pop_back();
+        expected_ += log;
+      }
+      expected_ += '\n';
+    }
+    return expected_.c_str();
+  }
+
+  template <size_t Align = 1> SrcAddr<Align> buf1(size_t offset = 0) const {
+    return buffer1.data() + offset;
+  }
+  template <size_t Align = 1> SrcAddr<Align> buf2(size_t offset = 0) const {
+    return buffer2.data() + offset;
+  }
+  template <size_t Align = 1> DstAddr<Align> dst(size_t offset = 0) const {
+    return buffer1.data() + offset;
+  }
+  template <size_t Align = 1> SrcAddr<Align> src(size_t offset = 0) const {
+    return buffer2.data() + offset;
+  }
+
+private:
+  std::string trace_;
+  std::string expected_;
+};
+
+using _8 = SizedOp<TestBackend, 8>;
+
+///////////////////////////////////////////////////////////////////////////////
+//// Testing fixed fized forward operations
+///////////////////////////////////////////////////////////////////////////////
+
+TEST_F(LlvmLibcAlgorithm, copy_1) {
+  SizedOp<TestBackend, 1>::copy(dst(), src());
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<1> b[0]
+StU<1> a[0]
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, copy_15) {
+  SizedOp<TestBackend, 15>::copy(dst(), src());
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> b[0]
+StU<8> a[0]
+LdU<4> b[8]
+StU<4> a[8]
+LdU<2> b[12]
+StU<2> a[12]
+LdU<1> b[14]
+StU<1> a[14]
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, copy_16) {
+  SizedOp<TestBackend, 16>::copy(dst(), src());
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> b[0]
+StU<8> a[0]
+LdU<8> b[8]
+StU<8> a[8]
+)"));
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+TEST_F(LlvmLibcAlgorithm, move_1) {
+  SizedOp<TestBackend, 1>::move(dst(), src());
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<1> b[0]
+StU<1> a[0]
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, move_15) {
+  SizedOp<TestBackend, 15>::move(dst(), src());
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> b[0]
+LdU<4> b[8]
+LdU<2> b[12]
+LdU<1> b[14]
+StU<1> a[14]
+StU<2> a[12]
+StU<4> a[8]
+StU<8> a[0]
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, move_16) {
+  SizedOp<TestBackend, 16>::move(dst(), src());
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> b[0]
+LdU<8> b[8]
+StU<8> a[8]
+StU<8> a[0]
+)"));
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+TEST_F(LlvmLibcAlgorithm, set_1) {
+  SizedOp<TestBackend, 1>::set(dst(), ubyte{42});
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+Splat<1> 42
+StU<1> a[0]
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, set_15) {
+  SizedOp<TestBackend, 15>::set(dst(), ubyte{42});
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+Splat<8> 42
+StU<8> a[0]
+Splat<4> 42
+StU<4> a[8]
+Splat<2> 42
+StU<2> a[12]
+Splat<1> 42
+StU<1> a[14]
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, set_16) {
+  SizedOp<TestBackend, 16>::set(dst(), ubyte{42});
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+Splat<8> 42
+StU<8> a[0]
+Splat<8> 42
+StU<8> a[8]
+)"));
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+TEST_F(LlvmLibcAlgorithm, different_1) {
+  fillEqual();
+  SizedOp<TestBackend, 1>::different(buf1(), buf2());
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<1> a[0]
+LdU<1> b[0]
+Neq<1>
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, different_15) {
+  fillEqual();
+  SizedOp<TestBackend, 15>::different(buf1(), buf2());
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> a[0]
+LdU<8> b[0]
+Neq<8>
+LdU<4> a[8]
+LdU<4> b[8]
+Neq<4>
+LdU<2> a[12]
+LdU<2> b[12]
+Neq<2>
+LdU<1> a[14]
+LdU<1> b[14]
+Neq<1>
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, different_15_no_shortcircuit) {
+  fillDifferent();
+  SizedOp<TestBackend, 15>::different(buf1(), buf2());
+  // If buffer compare different we continue to aggregate.
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> a[0]
+LdU<8> b[0]
+Neq<8>
+LdU<4> a[8]
+LdU<4> b[8]
+Neq<4>
+LdU<2> a[12]
+LdU<2> b[12]
+Neq<2>
+LdU<1> a[14]
+LdU<1> b[14]
+Neq<1>
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, different_16) {
+  fillEqual();
+  SizedOp<TestBackend, 16>::different(buf1(), buf2());
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> a[0]
+LdU<8> b[0]
+Neq<8>
+LdU<8> a[8]
+LdU<8> b[8]
+Neq<8>
+)"));
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+TEST_F(LlvmLibcAlgorithm, three_way_cmp_eq_1) {
+  fillEqual();
+  SizedOp<TestBackend, 1>::three_way_cmp(buf1(), buf2());
+  // Buffer compare equal, returning 0 and no call to Diff.
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<1> a[0]
+LdU<1> b[0]
+Neq<1>
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, three_way_cmp_eq_15) {
+  fillEqual();
+  SizedOp<TestBackend, 15>::three_way_cmp(buf1(), buf2());
+  // Buffer compare equal, returning 0 and no call to Diff.
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> a[0]
+LdU<8> b[0]
+Neq<8>
+LdU<4> a[8]
+LdU<4> b[8]
+Neq<4>
+LdU<2> a[12]
+LdU<2> b[12]
+Neq<2>
+LdU<1> a[14]
+LdU<1> b[14]
+Neq<1>
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, three_way_cmp_neq_15_shortcircuit) {
+  fillDifferent();
+  SizedOp<TestBackend, 16>::three_way_cmp(buf1(), buf2());
+  // If buffer compare different we stop early.
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> a[0]
+LdU<8> b[0]
+Neq<8>
+Diff<8>
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, three_way_cmp_eq_16) {
+  fillEqual();
+  SizedOp<TestBackend, 16>::three_way_cmp(buf1(), buf2());
+  // Buffer compare equal, returning 0 and no call to Diff.
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> a[0]
+LdU<8> b[0]
+Neq<8>
+LdU<8> a[8]
+LdU<8> b[8]
+Neq<8>
+)"));
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//// Testing tail operations
+///////////////////////////////////////////////////////////////////////////////
+
+TEST_F(LlvmLibcAlgorithm, tail_copy_8) {
+  Tail<_8>::copy(dst(), src(), 16);
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> b[8]
+StU<8> a[8]
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, tail_move_8) {
+  Tail<_8>::move(dst(), src(), 16);
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> b[8]
+StU<8> a[8]
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, tail_set_8) {
+  Tail<_8>::set(dst(), ubyte{42}, 16);
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+Splat<8> 42
+StU<8> a[8]
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, tail_different_8) {
+  fillEqual();
+  Tail<_8>::different(buf1(), buf2(), 16);
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> a[8]
+LdU<8> b[8]
+Neq<8>
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, tail_three_way_cmp_8) {
+  fillEqual();
+  Tail<_8>::three_way_cmp(buf1(), buf2(), 16);
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> a[8]
+LdU<8> b[8]
+Neq<8>
+)"));
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//// Testing HeadTail operations
+///////////////////////////////////////////////////////////////////////////////
+
+TEST_F(LlvmLibcAlgorithm, head_tail_copy_8) {
+  HeadTail<_8>::copy(dst(), src(), 16);
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> b[0]
+StU<8> a[0]
+LdU<8> b[8]
+StU<8> a[8]
+)"));
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//// Testing Loop operations
+///////////////////////////////////////////////////////////////////////////////
+
+TEST_F(LlvmLibcAlgorithm, loop_copy_one_iteration_and_tail) {
+  Loop<_8>::copy(dst(), src(), 10);
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> b[0]
+StU<8> a[0] # covers 0-7
+LdU<8> b[2]
+StU<8> a[2] # covers 2-9
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, loop_copy_two_iteration_and_tail) {
+  Loop<_8>::copy(dst(), src(), 17);
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> b[0]
+StU<8> a[0] # covers 0-7
+LdU<8> b[8]
+StU<8> a[8] # covers 8-15
+LdU<8> b[9]
+StU<8> a[9] # covers 9-16
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, loop_with_one_turn_is_inefficient_but_ok) {
+  Loop<_8>::copy(dst(), src(), 8);
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> b[0]
+StU<8> a[0] # first iteration covers 0-7
+LdU<8> b[0] # tail also covers 0-7 but since Loop is supposed to be used
+StU<8> a[0] # with a sufficient number of iterations the tail cost is amortised
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, loop_with_round_number_of_turn) {
+  Loop<_8>::copy(dst(), src(), 24);
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> b[0]
+StU<8> a[0] # first iteration covers 0-7
+LdU<8> b[8]
+StU<8> a[8] # second iteration covers 8-15
+LdU<8> b[16]
+StU<8> a[16]
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, dst_aligned_loop) {
+  Loop<_8>::copy(dst<16>(), src(), 23);
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> b[0]
+StA<8> a[0] # store is aligned on 16B
+LdU<8> b[8]
+StA<8> a[8] # subsequent stores are aligned
+LdU<8> b[15]
+StU<8> a[15] # Tail is always unaligned
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, aligned_loop) {
+  Loop<_8>::copy(dst<16>(), src<8>(), 23);
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdA<8> b[0] # load is aligned on 8B
+StA<8> a[0] # store is aligned on 16B
+LdA<8> b[8] # subsequent loads are aligned
+StA<8> a[8] # subsequent stores are aligned
+LdU<8> b[15] # Tail is always unaligned
+StU<8> a[15] # Tail is always unaligned
+)"));
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//// Testing Align operations
+///////////////////////////////////////////////////////////////////////////////
+
+TEST_F(LlvmLibcAlgorithm, align_dst_copy_8) {
+  Align<_8, Arg::Dst>::Then<Loop<_8>>::copy(dst(2), src(3), 31);
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> b[3]
+StU<8> a[2] # First store covers unaligned bytes
+LdU<8> b[9]
+StA<8> a[8] # First aligned store
+LdU<8> b[17]
+StA<8> a[16] # Subsequent stores are aligned
+LdU<8> b[25]
+StA<8> a[24] # Subsequent stores are aligned
+LdU<8> b[26]
+StU<8> a[25] # Last store covers remaining bytes
+)"));
+}
+
+TEST_F(LlvmLibcAlgorithm, align_src_copy_8) {
+  Align<_8, Arg::Src>::Then<Loop<_8>>::copy(dst(2), src(3), 31);
+  EXPECT_STREQ(getTrace(), stripComments(R"(
+LdU<8> b[3] # First load covers unaligned bytes
+StU<8> a[2]
+LdA<8> b[8] # First aligned load
+StU<8> a[7]
+LdA<8> b[16] # Subsequent loads are aligned
+StU<8> a[15]
+LdA<8> b[24] # Subsequent loads are aligned
+StU<8> a[23]
+LdU<8> b[26] # Last load covers remaining bytes
+StU<8> a[25]
+)"));
+}
+
+} // namespace __llvm_libc

libc/test/src/string/memory_utils/backend_test.cpp

+
+#include "src/__support/CPP/Array.h"
+#include "src/__support/CPP/ArrayRef.h"
+#include "src/__support/CPP/Bit.h"
+#include "src/__support/architectures.h"
+#include "src/string/memory_utils/backends.h"
+#include "utils/UnitTest/Test.h"
+#include <string.h>
+
+namespace __llvm_libc {
+
+template <size_t Size> using Buffer = cpp::Array<char, Size>;
+
+static char GetRandomChar() {
+  static constexpr const uint64_t a = 1103515245;
+  static constexpr const uint64_t c = 12345;
+  static constexpr const uint64_t m = 1ULL << 31;
+  static uint64_t seed = 123456789;
+  seed = (a * seed + c) % m;
+  return seed;
+}
+
+static void Randomize(cpp::MutableArrayRef<char> buffer) {
+  for (auto &current : buffer)
+    current = GetRandomChar();
+}
+
+template <size_t Size> static Buffer<Size> GetRandomBuffer() {
+  Buffer<Size> buffer;
+  Randomize(buffer);
+  return buffer;
+}
+
+template <typename Backend, size_t Size> struct Conf {
+  static_assert(Backend::IS_BACKEND_TYPE);
+  using BufferT = Buffer<Size>;
+  using T = typename Backend::template getNextType<Size>;
+  static_assert(sizeof(T) == Size);
+  static constexpr size_t SIZE = Size;
+
+  static BufferT splat(ubyte value) {
+    return bit_cast<BufferT>(Backend::template splat<T>(value));
+  }
+
+  static uint64_t not_equals(const BufferT &v1, const BufferT &v2) {
+    return Backend::template not_equals<T>(bit_cast<T>(v1), bit_cast<T>(v2));
+  }
+
+  static int32_t three_way_cmp(const BufferT &v1, const BufferT &v2) {
+    return Backend::template three_way_cmp<T>(bit_cast<T>(v1), bit_cast<T>(v2));
+  }
+};
+
+using FunctionTypes = testing::TypeList< //
+#if defined(LLVM_LIBC_ARCH_X86)          //
+    Conf<X86Backend, 1>,                 //
+    Conf<X86Backend, 2>,                 //
+    Conf<X86Backend, 4>,                 //
+    Conf<X86Backend, 8>,                 //
+#if HAS_M128
+    Conf<X86Backend, 16>, //
+#endif
+#if HAS_M256
+    Conf<X86Backend, 32>, //
+#endif
+#if HAS_M512
+    Conf<X86Backend, 64>, //
+#endif
+#endif                      // defined(LLVM_LIBC_ARCH_X86)
+    Conf<ScalarBackend, 1>, //
+    Conf<ScalarBackend, 2>, //
+    Conf<ScalarBackend, 4>, //
+    Conf<ScalarBackend, 8>  //
+    >;
+
+TYPED_TEST(LlvmLibcMemoryBackend, splat, FunctionTypes) {
+  for (auto value : cpp::Array<uint8_t, 3>{0u, 1u, 255u}) {
+    alignas(64) const auto stored = ParamType::splat(bit_cast<ubyte>(value));
+    for (size_t i = 0; i < ParamType::SIZE; ++i)
+      EXPECT_EQ(bit_cast<uint8_t>(stored[i]), value);
+  }
+}
+
+TYPED_TEST(LlvmLibcMemoryBackend, not_equals, FunctionTypes) {
+  alignas(64) const auto a = GetRandomBuffer<ParamType::SIZE>();
+  EXPECT_EQ(ParamType::not_equals(a, a), 0UL);
+  for (size_t i = 0; i < a.size(); ++i) {
+    alignas(64) auto b = a;
+    ++b[i];
+    EXPECT_NE(ParamType::not_equals(a, b), 0UL);
+    EXPECT_NE(ParamType::not_equals(b, a), 0UL);
+  }
+}
+
+TYPED_TEST(LlvmLibcMemoryBackend, three_way_cmp, FunctionTypes) {
+  alignas(64) const auto a = GetRandomBuffer<ParamType::SIZE>();
+  EXPECT_EQ(ParamType::three_way_cmp(a, a), 0);
+  for (size_t i = 0; i < a.size(); ++i) {
+    alignas(64) auto b = a;
+    ++b[i];
+    const auto cmp = memcmp(&a, &b, sizeof(a));
+    ASSERT_NE(cmp, 0);
+    if (cmp > 0) {
+      EXPECT_GT(ParamType::three_way_cmp(a, b), 0);
+      EXPECT_LT(ParamType::three_way_cmp(b, a), 0);
+    } else {
+      EXPECT_LT(ParamType::three_way_cmp(a, b), 0);
+      EXPECT_GT(ParamType::three_way_cmp(b, a), 0);
+    }
+  }
+}
+
+template <typename Backend, size_t Size, Temporal TS, Aligned AS>
+struct LoadStoreConf {
+  static_assert(Backend::IS_BACKEND_TYPE);
+  using BufferT = Buffer<Size>;
+  using T = typename Backend::template getNextType<Size>;
+  static_assert(sizeof(T) == Size);
+  static constexpr size_t SIZE = Size;
+
+  static BufferT load(const BufferT &ref) {
+    const auto *ptr = bit_cast<const T *>(ref.data());
+    const T value = Backend::template load<T, TS, AS>(ptr);
+    return bit_cast<BufferT>(value);
+  }
+
+  static void store(BufferT &ref, const BufferT value) {
+    auto *ptr = bit_cast<T *>(ref.data());
+    Backend::template store<T, TS, AS>(ptr, bit_cast<T>(value));
+  }
+};
+
+using LoadStoreTypes = testing::TypeList<                      //
+#if defined(LLVM_LIBC_ARCH_X86)                                //
+    LoadStoreConf<X86Backend, 1, Temporal::YES, Aligned::NO>,  //
+    LoadStoreConf<X86Backend, 1, Temporal::YES, Aligned::YES>, //
+    LoadStoreConf<X86Backend, 2, Temporal::YES, Aligned::NO>,  //
+    LoadStoreConf<X86Backend, 2, Temporal::YES, Aligned::YES>, //
+    LoadStoreConf<X86Backend, 4, Temporal::YES, Aligned::NO>,  //
+    LoadStoreConf<X86Backend, 4, Temporal::YES, Aligned::YES>, //
+    LoadStoreConf<X86Backend, 8, Temporal::YES, Aligned::NO>,  //
+    LoadStoreConf<X86Backend, 8, Temporal::YES, Aligned::YES>, //
+#if HAS_M128
+    LoadStoreConf<X86Backend, 16, Temporal::YES, Aligned::NO>,  //
+    LoadStoreConf<X86Backend, 16, Temporal::YES, Aligned::YES>, //
+    LoadStoreConf<X86Backend, 16, Temporal::NO, Aligned::YES>,  //
+#endif
+#if HAS_M256
+    LoadStoreConf<X86Backend, 32, Temporal::YES, Aligned::NO>,  //
+    LoadStoreConf<X86Backend, 32, Temporal::YES, Aligned::YES>, //
+    LoadStoreConf<X86Backend, 32, Temporal::NO, Aligned::YES>,  //
+#endif
+#if HAS_M512
+    LoadStoreConf<X86Backend, 64, Temporal::YES, Aligned::NO>,  //
+    LoadStoreConf<X86Backend, 64, Temporal::YES, Aligned::YES>, //
+    LoadStoreConf<X86Backend, 64, Temporal::NO, Aligned::YES>,  //
+#endif
+#endif // defined(LLVM_LIBC_ARCH_X86)
+    LoadStoreConf<ScalarBackend, 1, Temporal::YES, Aligned::NO>,  //
+    LoadStoreConf<ScalarBackend, 1, Temporal::YES, Aligned::YES>, //
+    LoadStoreConf<ScalarBackend, 2, Temporal::YES, Aligned::NO>,  //
+    LoadStoreConf<ScalarBackend, 2, Temporal::YES, Aligned::YES>, //
+    LoadStoreConf<ScalarBackend, 4, Temporal::YES, Aligned::NO>,  //
+    LoadStoreConf<ScalarBackend, 4, Temporal::YES, Aligned::YES>, //
+    LoadStoreConf<ScalarBackend, 8, Temporal::YES, Aligned::NO>,  //
+    LoadStoreConf<ScalarBackend, 8, Temporal::YES, Aligned::YES>  //
+    >;
+
+TYPED_TEST(LlvmLibcMemoryBackend, load, LoadStoreTypes) {
+  alignas(64) const auto expected = GetRandomBuffer<ParamType::SIZE>();
+  const auto loaded = ParamType::load(expected);
+  for (size_t i = 0; i < ParamType::SIZE; ++i)
+    EXPECT_EQ(loaded[i], expected[i]);
+}
+
+TYPED_TEST(LlvmLibcMemoryBackend, store, LoadStoreTypes) {
+  alignas(64) const auto expected = GetRandomBuffer<ParamType::SIZE>();
+  alignas(64) typename ParamType::BufferT stored;
+  ParamType::store(stored, expected);
+  for (size_t i = 0; i < ParamType::SIZE; ++i)
+    EXPECT_EQ(stored[i], expected[i]);
+}
+
+} // namespace __llvm_libc

libc/utils/UnitTest/LibcTest.h

protected:
   // of type promotion.
   template <typename ValType,
             cpp::EnableIfType<cpp::IsIntegral<ValType>::Value, int> = 0>
   bool test(TestCondition Cond, ValType LHS, ValType RHS, const char *LHSStr,
             const char *RHSStr, const char *File, unsigned long Line) {
     return internal::test(Ctx, Cond, LHS, RHS, LHSStr, RHSStr, File, Line);
   }
 
+  template <typename ValType,
+            cpp::EnableIfType<cpp::IsEnum<ValType>::Value, int> = 0>
+  bool test(TestCondition Cond, ValType LHS, ValType RHS, const char *LHSStr,
+            const char *RHSStr, const char *File, unsigned long Line) {
+    return internal::test(Ctx, Cond, (long long)LHS, (long long)RHS, LHSStr,
+                          RHSStr, File, Line);
+  }
+
   template <
       typename ValType,
       cpp::EnableIfType<cpp::IsPointerType<ValType>::Value, ValType> = nullptr>
   bool test(TestCondition Cond, ValType LHS, ValType RHS, const char *LHSStr,
             const char *RHSStr, const char *File, unsigned long Line) {
     return internal::test(Ctx, Cond, (unsigned long long)LHS,
                           (unsigned long long)RHS, LHSStr, RHSStr, File, Line);
   }

utils/bazel/llvm-project-overlay/libc/BUILD.bazel

deps = [
         ":__support_cpp_bitset",
         ":libc_root",
     ],
 )
 
 cc_library(
     name = "string_memory_utils",
     hdrs = [
-        "src/string/memory_utils/elements.h",
-        "src/string/memory_utils/elements_aarch64.h",
-        "src/string/memory_utils/elements_x86.h",
+        "src/string/memory_utils/address.h",
+        "src/string/memory_utils/algorithm.h",
+        "src/string/memory_utils/backends.h",
+        "src/string/memory_utils/backend_scalar.h",
+        "src/string/memory_utils/backend_x86.h",
+        "src/string/memory_utils/backend_aarch64.h",
+        "src/string/memory_utils/sized_op.h",
         "src/string/memory_utils/utils.h",
     ],
     textual_hdrs = [
         "src/string/memory_utils/bcmp_implementations.h",
         "src/string/memory_utils/memcmp_implementations.h",
         "src/string/memory_utils/memcpy_implementations.h",
         "src/string/memory_utils/memset_implementations.h",
     ],
     deps = [
         ":__support_common",
         ":__support_cpp_bit",
+        ":__support_cpp_type_traits",
         ":libc_root",
     ],
 )
 
 libc_function(
     name = "memchr",
     srcs = ["src/string/memchr.cpp"],
     hdrs = ["src/string/memchr.h"],