diff --git a/libc/src/string/memory_utils/CMakeLists.txt b/libc/src/string/memory_utils/CMakeLists.txt --- a/libc/src/string/memory_utils/CMakeLists.txt +++ b/libc/src/string/memory_utils/CMakeLists.txt @@ -4,7 +4,6 @@ HDRS bcmp_implementations.h bzero_implementations.h - elements.h memcmp_implementations.h memcpy_implementations.h memset_implementations.h diff --git a/libc/src/string/memory_utils/elements.h b/libc/src/string/memory_utils/elements.h deleted file mode 100644 --- a/libc/src/string/memory_utils/elements.h +++ /dev/null @@ -1,774 +0,0 @@ -//===-- Elementary operations to compose memory primitives ----------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_H -#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_H - -#include // size_t -#include // uint8_t, uint16_t, uint32_t, uint64_t - -#include "src/__support/endian.h" -#include "src/string/memory_utils/utils.h" - -namespace __llvm_libc { - -// Elementary Operations -// -------------------------------- -// We define abstract elementary operations acting on fixed chunks of memory. -// These are low level building blocks that are meant to be assembled to compose -// higher order abstractions. Each function is defined twice: once with -// fixed-size operations, and once with runtime-size operations. - -// Fixed-size copy from 'src' to 'dst'. -template -void copy(char *__restrict dst, const char *__restrict src) { - Element::copy(dst, src); -} -// Runtime-size copy from 'src' to 'dst'. -template -void copy(char *__restrict dst, const char *__restrict src, size_t size) { - Element::copy(dst, src, size); -} - -// Fixed-size move from 'src' to 'dst'. -template void move(char *dst, const char *src) { - Element::move(dst, src); -} -// Runtime-size move from 'src' to 'dst'. -template void move(char *dst, const char *src, size_t size) { - Element::move(dst, src, size); -} -// Runtime-size move from 'src' to 'dst'. -template -void move_backward(char *dst, const char *src, size_t size) { - Element::move_backward(dst, src, size); -} - -// Fixed-size equality between 'lhs' and 'rhs'. -template bool equals(const char *lhs, const char *rhs) { - return Element::equals(lhs, rhs); -} -// Runtime-size equality between 'lhs' and 'rhs'. -template -bool equals(const char *lhs, const char *rhs, size_t size) { - return Element::equals(lhs, rhs, size); -} - -// Fixed-size three-way comparison between 'lhs' and 'rhs'. -template -int three_way_compare(const char *lhs, const char *rhs) { - return Element::three_way_compare(lhs, rhs); -} -// Runtime-size three-way comparison between 'lhs' and 'rhs'. -template -int three_way_compare(const char *lhs, const char *rhs, size_t size) { - return Element::three_way_compare(lhs, rhs, size); -} - -// Fixed-size initialization. -template -void splat_set(char *dst, const unsigned char value) { - Element::splat_set(dst, value); -} -// Runtime-size initialization. -template -void splat_set(char *dst, const unsigned char value, size_t size) { - Element::splat_set(dst, value, size); -} - -// Stack placeholder for Move operations. -template struct Storage { char bytes[Element::SIZE]; }; - -// Fixed-size Higher-Order Operations -// ---------------------------------- -// - Repeated: Repeat the operation several times in a row. -// - Chained: Chain the operation of several types. - -// Repeat the operation several times in a row. -template struct Repeated { - static constexpr size_t SIZE = ElementCount * Element::SIZE; - - static void copy(char *__restrict dst, const char *__restrict src) { - for (size_t i = 0; i < ElementCount; ++i) { - const size_t offset = i * Element::SIZE; - Element::copy(dst + offset, src + offset); - } - } - - static void move(char *dst, const char *src) { - const auto value = load(src); - store(dst, value); - } - - static bool equals(const char *lhs, const char *rhs) { - for (size_t i = 0; i < ElementCount; ++i) { - const size_t offset = i * Element::SIZE; - if (!Element::equals(lhs + offset, rhs + offset)) - return false; - } - return true; - } - - static int three_way_compare(const char *lhs, const char *rhs) { - for (size_t i = 0; i < ElementCount; ++i) { - const size_t offset = i * Element::SIZE; - // We make the assumption that 'equals' is cheaper than - // 'three_way_compare'. - if (Element::equals(lhs + offset, rhs + offset)) - continue; - return Element::three_way_compare(lhs + offset, rhs + offset); - } - return 0; - } - - static void splat_set(char *dst, const unsigned char value) { - for (size_t i = 0; i < ElementCount; ++i) { - const size_t offset = i * Element::SIZE; - Element::splat_set(dst + offset, value); - } - } - - static Storage load(const char *ptr) { - Storage value; - copy(reinterpret_cast(&value), ptr); - return value; - } - - static void store(char *ptr, Storage value) { - copy(ptr, reinterpret_cast(&value)); - } -}; - -template struct Repeated { - static void move(char *, const char *) {} -}; - -// Chain the operation of several types. -// For instance, to handle a 3 bytes operation, one can use: -// Chained::Operation(); -template struct Chained; - -template struct Chained { - static constexpr size_t SIZE = Head::SIZE + Chained::SIZE; - - static void copy(char *__restrict dst, const char *__restrict src) { - Chained::copy(dst + Head::SIZE, src + Head::SIZE); - __llvm_libc::copy(dst, src); - } - - static void move(char *dst, const char *src) { - const auto value = Head::load(src); - Chained::move(dst + Head::SIZE, src + Head::SIZE); - Head::store(dst, value); - } - - static bool equals(const char *lhs, const char *rhs) { - if (!__llvm_libc::equals(lhs, rhs)) - return false; - return Chained::equals(lhs + Head::SIZE, rhs + Head::SIZE); - } - - static int three_way_compare(const char *lhs, const char *rhs) { - if (__llvm_libc::equals(lhs, rhs)) - return Chained::three_way_compare(lhs + Head::SIZE, - rhs + Head::SIZE); - return __llvm_libc::three_way_compare(lhs, rhs); - } - - static void splat_set(char *dst, const unsigned char value) { - Chained::splat_set(dst + Head::SIZE, value); - __llvm_libc::splat_set(dst, value); - } -}; - -template <> struct Chained<> { - static constexpr size_t SIZE = 0; - static void copy(char *__restrict, const char *__restrict) {} - static void move(char *, const char *) {} - static bool equals(const char *, const char *) { return true; } - static int three_way_compare(const char *, const char *) { return 0; } - static void splat_set(char *, const unsigned char) {} -}; - -// Overlap ElementA and ElementB so they span Size bytes. -template -struct Overlap { - static constexpr size_t SIZE = Size; - static_assert(ElementB::SIZE <= ElementA::SIZE, "ElementB too big"); - static_assert(ElementA::SIZE <= Size, "ElementA too big"); - static_assert((ElementA::SIZE + ElementB::SIZE) >= Size, - "Elements too small to overlap"); - static constexpr size_t OFFSET = SIZE - ElementB::SIZE; - - static void copy(char *__restrict dst, const char *__restrict src) { - ElementA::copy(dst, src); - ElementB::copy(dst + OFFSET, src + OFFSET); - } - - static void move(char *dst, const char *src) { - const auto value_a = ElementA::load(src); - const auto value_b = ElementB::load(src + OFFSET); - ElementB::store(dst + OFFSET, value_b); - ElementA::store(dst, value_a); - } - - static bool equals(const char *lhs, const char *rhs) { - if (!ElementA::equals(lhs, rhs)) - return false; - if (!ElementB::equals(lhs + OFFSET, rhs + OFFSET)) - return false; - return true; - } - - static int three_way_compare(const char *lhs, const char *rhs) { - if (!ElementA::equals(lhs, rhs)) - return ElementA::three_way_compare(lhs, rhs); - if (!ElementB::equals(lhs + OFFSET, rhs + OFFSET)) - return ElementB::three_way_compare(lhs + OFFSET, rhs + OFFSET); - return 0; - } - - static void splat_set(char *dst, const unsigned char value) { - ElementA::splat_set(dst, value); - ElementB::splat_set(dst + OFFSET, value); - } -}; - -// Runtime-size Higher-Order Operations -// ------------------------------------ -// - Tail: Perform the operation on the last 'T::SIZE' bytes of the buffer. -// - HeadTail: Perform the operation on the first and last 'T::SIZE' bytes -// of the buffer. -// - Loop: Perform a loop of fixed-sized operations. - -// Perform the operation on the last 'T::SIZE' bytes of the buffer. -// -// e.g. with -// [1234567812345678123] -// [__XXXXXXXXXXXXXX___] -// [________XXXXXXXX___] -// -// Precondition: `size >= T::SIZE`. -template struct Tail { - static void copy(char *__restrict dst, const char *__restrict src, - size_t size) { - return T::copy(dst + offset(size), src + offset(size)); - } - - static bool equals(const char *lhs, const char *rhs, size_t size) { - return T::equals(lhs + offset(size), rhs + offset(size)); - } - - static int three_way_compare(const char *lhs, const char *rhs, size_t size) { - return T::three_way_compare(lhs + offset(size), rhs + offset(size)); - } - - static void splat_set(char *dst, const unsigned char value, size_t size) { - return T::splat_set(dst + offset(size), value); - } - - static size_t offset(size_t size) { return size - T::SIZE; } -}; - -// Perform the operation on the first and last 'T::SIZE' bytes of the buffer. -// This is useful for overlapping operations. -// -// e.g. with -// [1234567812345678123] -// [__XXXXXXXXXXXXXX___] -// [__XXXXXXXX_________] -// [________XXXXXXXX___] -// -// Precondition: `size >= T::SIZE && size <= 2 x T::SIZE`. -template struct HeadTail { - static void copy(char *__restrict dst, const char *__restrict src, - size_t size) { - T::copy(dst, src); - Tail::copy(dst, src, size); - } - - static void move(char *dst, const char *src, size_t size) { - const size_t offset = Tail::offset(size); - const auto head_value = T::load(src); - const auto tail_value = T::load(src + offset); - T::store(dst + offset, tail_value); - T::store(dst, head_value); - } - - static bool equals(const char *lhs, const char *rhs, size_t size) { - if (!T::equals(lhs, rhs)) - return false; - return Tail::equals(lhs, rhs, size); - } - - static int three_way_compare(const char *lhs, const char *rhs, size_t size) { - if (!T::equals(lhs, rhs)) - return T::three_way_compare(lhs, rhs); - return Tail::three_way_compare(lhs, rhs, size); - } - - static void splat_set(char *dst, const unsigned char value, size_t size) { - T::splat_set(dst, value); - Tail::splat_set(dst, value, size); - } -}; - -// Simple loop ending with a Tail operation. -// -// e.g. with -// [12345678123456781234567812345678] -// [__XXXXXXXXXXXXXXXXXXXXXXXXXXXX___] -// [__XXXXXXXX_______________________] -// [__________XXXXXXXX_______________] -// [__________________XXXXXXXX_______] -// [______________________XXXXXXXX___] -// -// Precondition: -// - size >= T::SIZE -template struct Loop { - static_assert(T::SIZE == TailT::SIZE, - "Tail type must have the same size as T"); - - static void copy(char *__restrict dst, const char *__restrict src, - size_t size) { - size_t offset = 0; - do { - T::copy(dst + offset, src + offset); - offset += T::SIZE; - } while (offset < size - T::SIZE); - Tail::copy(dst, src, size); - } - - // Move forward suitable when dst < src. We load the tail bytes before - // handling the loop. - // - // e.g. Moving two bytes - // [ | | | | |] - // [___XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX___] - // [_________________________LLLLLLLL___] - // [___LLLLLLLL_________________________] - // [_SSSSSSSS___________________________] - // [___________LLLLLLLL_________________] - // [_________SSSSSSSS___________________] - // [___________________LLLLLLLL_________] - // [_________________SSSSSSSS___________] - // [_______________________SSSSSSSS_____] - static void move(char *dst, const char *src, size_t size) { - const size_t tail_offset = Tail::offset(size); - const auto tail_value = TailT::load(src + tail_offset); - size_t offset = 0; - do { - T::move(dst + offset, src + offset); - offset += T::SIZE; - } while (offset < size - T::SIZE); - TailT::store(dst + tail_offset, tail_value); - } - - // Move forward suitable when dst > src. We load the head bytes before - // handling the loop. - // - // e.g. Moving two bytes - // [ | | | | |] - // [___XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX___] - // [___LLLLLLLL_________________________] - // [_________________________LLLLLLLL___] - // [___________________________SSSSSSSS_] - // [_________________LLLLLLLL___________] - // [___________________SSSSSSSS_________] - // [_________LLLLLLLL___________________] - // [___________SSSSSSSS_________________] - // [_____SSSSSSSS_______________________] - static void move_backward(char *dst, const char *src, size_t size) { - const auto head_value = TailT::load(src); - ptrdiff_t offset = size - T::SIZE; - do { - T::move(dst + offset, src + offset); - offset -= T::SIZE; - } while (offset >= 0); - TailT::store(dst, head_value); - } - - static bool equals(const char *lhs, const char *rhs, size_t size) { - size_t offset = 0; - do { - if (!T::equals(lhs + offset, rhs + offset)) - return false; - offset += T::SIZE; - } while (offset < size - T::SIZE); - return Tail::equals(lhs, rhs, size); - } - - static int three_way_compare(const char *lhs, const char *rhs, size_t size) { - size_t offset = 0; - do { - if (!T::equals(lhs + offset, rhs + offset)) - return T::three_way_compare(lhs + offset, rhs + offset); - offset += T::SIZE; - } while (offset < size - T::SIZE); - return Tail::three_way_compare(lhs, rhs, size); - } - - static void splat_set(char *dst, const unsigned char value, size_t size) { - size_t offset = 0; - do { - T::splat_set(dst + offset, value); - offset += T::SIZE; - } while (offset < size - T::SIZE); - Tail::splat_set(dst, value, size); - } -}; - -namespace internal { - -template struct ArgSelector {}; - -template <> struct ArgSelector { - template - static T1 *__restrict &Select(T1 *__restrict &p1ref, T2 *__restrict &) { - return p1ref; - } -}; - -template <> struct ArgSelector { - template - static T2 *__restrict &Select(T1 *__restrict &, T2 *__restrict &p2ref) { - return p2ref; - } -}; - -// Provides a specialized bump function that adjusts pointers and size so first -// argument (resp. second argument) gets aligned to Alignment. -// We make sure the compiler knows about the adjusted pointer alignment. -// The 'additional_bumps' parameter allows to reach previous / next aligned -// pointers. -template struct Align { - template - static void bump(T1 *__restrict &p1ref, T2 *__restrict &p2ref, size_t &size, - int additional_bumps = 0) { - auto &aligned_ptr = ArgSelector::Select(p1ref, p2ref); - auto offset = distance_to_align_up(aligned_ptr); - adjust(offset + additional_bumps * Alignment, p1ref, p2ref, size); - aligned_ptr = assume_aligned(aligned_ptr); - } -}; - -} // namespace internal - -// An alignment operation that: -// - executes the 'AlignmentT' operation -// - bumps 'dst' or 'src' (resp. 'lhs' or 'rhs') pointers so that the selected -// pointer gets aligned, size is decreased accordingly. -// - calls the 'NextT' operation. -// -// e.g. A 16-byte Destination Aligned 32-byte Loop Copy can be written as: -// copy::Then>>(dst, src, count); -template struct Align { -private: - static constexpr size_t ALIGNMENT = AlignmentT::SIZE; - static_assert(ALIGNMENT > 1, "Alignment must be more than 1"); - static_assert(is_power2(ALIGNMENT), "Alignment must be a power of 2"); - -public: - template struct Then { - static void copy(char *__restrict dst, const char *__restrict src, - size_t size) { - AlignmentT::copy(dst, src); - internal::Align::bump(dst, src, size); - NextT::copy(dst, src, size); - } - - // Move forward suitable when dst < src. The alignment is performed with an - // HeadTail operation of size ∈ [Alignment, 2 x Alignment]. - // - // e.g. Moving two bytes and making sure src is then aligned. - // [ | | | | ] - // [____XXXXXXXXXXXXXXXXXXXXXXXXXXXX_] - // [____LLLLLLLL_____________________] - // [___________LLLLLLLL______________] - // [_SSSSSSSS________________________] - // [________SSSSSSSS_________________] - // - // e.g. Moving two bytes and making sure dst is then aligned. - // [ | | | | ] - // [____XXXXXXXXXXXXXXXXXXXXXXXXXXXX_] - // [____LLLLLLLL_____________________] - // [______LLLLLLLL___________________] - // [_SSSSSSSS________________________] - // [___SSSSSSSS______________________] - static void move(char *dst, const char *src, size_t size) { - char *next_dst = dst; - const char *next_src = src; - size_t next_size = size; - internal::Align::bump(next_dst, next_src, next_size, - 1); - HeadTail::move(dst, src, size - next_size); - NextT::move(next_dst, next_src, next_size); - } - - // Move backward suitable when dst > src. The alignment is performed with an - // HeadTail operation of size ∈ [Alignment, 2 x Alignment]. - // - // e.g. Moving two bytes backward and making sure src is then aligned. - // [ | | | | ] - // [____XXXXXXXXXXXXXXXXXXXXXXXX_____] - // [ _________________LLLLLLLL_______] - // [ ___________________LLLLLLLL_____] - // [____________________SSSSSSSS_____] - // [______________________SSSSSSSS___] - // - // e.g. Moving two bytes and making sure dst is then aligned. - // [ | | | | ] - // [____XXXXXXXXXXXXXXXXXXXXXXXX_____] - // [ _______________LLLLLLLL_________] - // [ ___________________LLLLLLLL_____] - // [__________________SSSSSSSS_______] - // [______________________SSSSSSSS___] - static void move_backward(char *dst, const char *src, size_t size) { - char *headtail_dst = dst + size; - const char *headtail_src = src + size; - size_t headtail_size = 0; - internal::Align::bump(headtail_dst, headtail_src, - headtail_size, -2); - HeadTail::move(headtail_dst, headtail_src, headtail_size); - NextT::move_backward(dst, src, size - headtail_size); - } - - static bool equals(const char *lhs, const char *rhs, size_t size) { - if (!AlignmentT::equals(lhs, rhs)) - return false; - internal::Align::bump(lhs, rhs, size); - return NextT::equals(lhs, rhs, size); - } - - static int three_way_compare(const char *lhs, const char *rhs, - size_t size) { - if (!AlignmentT::equals(lhs, rhs)) - return AlignmentT::three_way_compare(lhs, rhs); - internal::Align::bump(lhs, rhs, size); - return NextT::three_way_compare(lhs, rhs, size); - } - - static void splat_set(char *dst, const unsigned char value, size_t size) { - AlignmentT::splat_set(dst, value); - char *dummy = nullptr; - internal::Align::bump(dst, dummy, size); - NextT::splat_set(dst, value, size); - } - }; -}; - -// An operation that allows to skip the specified amount of bytes. -template struct Skip { - template struct Then { - static void copy(char *__restrict dst, const char *__restrict src, - size_t size) { - NextT::copy(dst + Bytes, src + Bytes, size - Bytes); - } - - static void copy(char *__restrict dst, const char *__restrict src) { - NextT::copy(dst + Bytes, src + Bytes); - } - - static bool equals(const char *lhs, const char *rhs, size_t size) { - return NextT::equals(lhs + Bytes, rhs + Bytes, size - Bytes); - } - - static bool equals(const char *lhs, const char *rhs) { - return NextT::equals(lhs + Bytes, rhs + Bytes); - } - - static int three_way_compare(const char *lhs, const char *rhs, - size_t size) { - return NextT::three_way_compare(lhs + Bytes, rhs + Bytes, size - Bytes); - } - - static int three_way_compare(const char *lhs, const char *rhs) { - return NextT::three_way_compare(lhs + Bytes, rhs + Bytes); - } - - static void splat_set(char *dst, const unsigned char value, size_t size) { - NextT::splat_set(dst + Bytes, value, size - Bytes); - } - - static void splat_set(char *dst, const unsigned char value) { - NextT::splat_set(dst + Bytes, value); - } - }; -}; - -// Fixed-size Builtin Operations -// ----------------------------- -// Note: Do not use 'builtin' right now as it requires the implementation of the -// `_inline` versions of all the builtins. Theoretically, Clang can still turn -// them into calls to the C library leading to reentrancy problems. -namespace builtin { - -#ifndef __has_builtin -#define __has_builtin(x) 0 // Compatibility with non-clang compilers. -#endif - -template struct Builtin { - static constexpr size_t SIZE = Size; - - static void copy(char *__restrict dst, const char *__restrict src) { -#if LLVM_LIBC_HAVE_MEMORY_SANITIZER || LLVM_LIBC_HAVE_ADDRESS_SANITIZER - for_loop_copy(dst, src); -#elif __has_builtin(__builtin_memcpy_inline) - // __builtin_memcpy_inline guarantees to never call external functions. - // Unfortunately it is not widely available. - __builtin_memcpy_inline(dst, src, SIZE); -#else - for_loop_copy(dst, src); -#endif - } - - static void move(char *dst, const char *src) { -#if LLVM_LIBC_HAVE_MEMORY_SANITIZER || LLVM_LIBC_HAVE_ADDRESS_SANITIZER - for_loop_move(dst, src); -#elif __has_builtin(__builtin_memmove) - __builtin_memmove(dst, src, SIZE); -#else - for_loop_move(dst, src); -#endif - } - -#if __has_builtin(__builtin_memcmp_inline) -#define LLVM_LIBC_MEMCMP __builtin_memcmp_inline -#else -#define LLVM_LIBC_MEMCMP __builtin_memcmp -#endif - - static bool equals(const char *lhs, const char *rhs) { - return LLVM_LIBC_MEMCMP(lhs, rhs, SIZE) == 0; - } - - static int three_way_compare(const char *lhs, const char *rhs) { - return LLVM_LIBC_MEMCMP(lhs, rhs, SIZE); - } - - static void splat_set(char *dst, const unsigned char value) { - __builtin_memset(dst, value, SIZE); - } - -private: - // Copies `SIZE` bytes from `src` to `dst` using a for loop. - // This code requires the use of `-fno-builtin-memcpy` to prevent the compiler - // from turning the for-loop back into `__builtin_memcpy`. - static void for_loop_copy(char *__restrict dst, const char *__restrict src) { - for (size_t i = 0; i < SIZE; ++i) - dst[i] = src[i]; - } - - static void for_loop_move(char *dst, const char *src) { - for (size_t i = 0; i < SIZE; ++i) - dst[i] = src[i]; - } -}; - -using _1 = Builtin<1>; -using _2 = Builtin<2>; -using _3 = Builtin<3>; -using _4 = Builtin<4>; -using _8 = Builtin<8>; -using _16 = Builtin<16>; -using _32 = Builtin<32>; -using _64 = Builtin<64>; -using _128 = Builtin<128>; - -} // namespace builtin - -// Fixed-size Scalar Operations -// ---------------------------- -namespace scalar { - -// The Scalar type makes use of simple sized integers. -template struct Scalar { - static constexpr size_t SIZE = sizeof(T); - - static void copy(char *__restrict dst, const char *__restrict src) { - store(dst, load(src)); - } - - static void move(char *dst, const char *src) { store(dst, load(src)); } - - static bool equals(const char *lhs, const char *rhs) { - return load(lhs) == load(rhs); - } - - static int three_way_compare(const char *lhs, const char *rhs) { - return scalar_three_way_compare(load(lhs), load(rhs)); - } - - static void splat_set(char *dst, const unsigned char value) { - store(dst, get_splatted_value(value)); - } - - static int scalar_three_way_compare(T a, T b); - - static T load(const char *ptr) { - T value; - builtin::Builtin::copy(reinterpret_cast(&value), ptr); - return value; - } - static void store(char *ptr, T value) { - builtin::Builtin::copy(ptr, reinterpret_cast(&value)); - } - -private: - static T get_splatted_value(const unsigned char value) { - return T(~0) / T(0xFF) * T(value); - } -}; - -template <> -inline int Scalar::scalar_three_way_compare(uint8_t a, uint8_t b) { - const int16_t la = Endian::to_big_endian(a); - const int16_t lb = Endian::to_big_endian(b); - return la - lb; -} -template <> -inline int Scalar::scalar_three_way_compare(uint16_t a, uint16_t b) { - const int32_t la = Endian::to_big_endian(a); - const int32_t lb = Endian::to_big_endian(b); - return la - lb; -} -template <> -inline int Scalar::scalar_three_way_compare(uint32_t a, uint32_t b) { - const uint32_t la = Endian::to_big_endian(a); - const uint32_t lb = Endian::to_big_endian(b); - return la > lb ? 1 : la < lb ? -1 : 0; -} -template <> -inline int Scalar::scalar_three_way_compare(uint64_t a, uint64_t b) { - const uint64_t la = Endian::to_big_endian(a); - const uint64_t lb = Endian::to_big_endian(b); - return la > lb ? 1 : la < lb ? -1 : 0; -} - -using UINT8 = Scalar; // 1 Byte -using UINT16 = Scalar; // 2 Bytes -using UINT32 = Scalar; // 4 Bytes -using UINT64 = Scalar; // 8 Bytes - -using _1 = UINT8; -using _2 = UINT16; -using _3 = Chained; -using _4 = UINT32; -using _8 = UINT64; -using _16 = Repeated<_8, 2>; -using _32 = Repeated<_8, 4>; -using _64 = Repeated<_8, 8>; -using _128 = Repeated<_8, 16>; - -} // namespace scalar -} // namespace __llvm_libc - -#include -#include - -#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_H diff --git a/libc/src/string/memory_utils/elements_aarch64.h b/libc/src/string/memory_utils/elements_aarch64.h deleted file mode 100644 --- a/libc/src/string/memory_utils/elements_aarch64.h +++ /dev/null @@ -1,130 +0,0 @@ -//===-- Elementary operations for aarch64 --------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_AARCH64_H -#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_AARCH64_H - -#include "src/__support/architectures.h" - -#if defined(LLVM_LIBC_ARCH_AARCH64) - -#include -#include // size_t -#include // uint8_t, uint16_t, uint32_t, uint64_t - -#ifdef __ARM_NEON -#include -#endif - -namespace __llvm_libc { -namespace aarch64_memset { -#ifdef __ARM_NEON -struct Splat8 { - static constexpr size_t SIZE = 8; - static void splat_set(char *dst, const unsigned char value) { - vst1_u8((uint8_t *)dst, vdup_n_u8(value)); - } -}; - -struct Splat16 { - static constexpr size_t SIZE = 16; - static void splat_set(char *dst, const unsigned char value) { - vst1q_u8((uint8_t *)dst, vdupq_n_u8(value)); - } -}; - -using _8 = Splat8; -using _16 = Splat16; -#else -using _8 = __llvm_libc::scalar::_8; -using _16 = Repeated<_8, 2>; -#endif // __ARM_NEON - -using _1 = __llvm_libc::scalar::_1; -using _2 = __llvm_libc::scalar::_2; -using _3 = __llvm_libc::scalar::_3; -using _4 = __llvm_libc::scalar::_4; -using _32 = Chained<_16, _16>; -using _64 = Chained<_32, _32>; - -struct Zva64 { - static constexpr size_t SIZE = 64; - - static void splat_set(char *dst, const unsigned char) { -#if __SIZEOF_POINTER__ == 4 - asm("dc zva, %w[dst]" : : [dst] "r"(dst) : "memory"); -#else - asm("dc zva, %[dst]" : : [dst] "r"(dst) : "memory"); -#endif - } -}; - -inline static bool hasZva() { - uint64_t zva_val; - asm("mrs %[zva_val], dczid_el0" : [zva_val] "=r"(zva_val)); - // DC ZVA is permitted if DZP, bit [4] is zero. - // BS, bits [3:0] is log2 of the block size in words. - // So the next line checks whether the instruction is permitted and block size - // is 16 words (i.e. 64 bytes). - return (zva_val & 0b11111) == 0b00100; -} - -} // namespace aarch64_memset - -namespace aarch64 { - -using _1 = __llvm_libc::scalar::_1; -using _2 = __llvm_libc::scalar::_2; -using _3 = __llvm_libc::scalar::_3; -using _4 = __llvm_libc::scalar::_4; -using _8 = __llvm_libc::scalar::_8; -using _16 = __llvm_libc::scalar::_16; - -#ifdef __ARM_NEON -struct N32 { - static constexpr size_t SIZE = 32; - static bool equals(const char *lhs, const char *rhs) { - uint8x16_t l_0 = vld1q_u8((const uint8_t *)lhs); - uint8x16_t r_0 = vld1q_u8((const uint8_t *)rhs); - uint8x16_t l_1 = vld1q_u8((const uint8_t *)(lhs + 16)); - uint8x16_t r_1 = vld1q_u8((const uint8_t *)(rhs + 16)); - uint8x16_t temp = vpmaxq_u8(veorq_u8(l_0, r_0), veorq_u8(l_1, r_1)); - uint64_t res = - vgetq_lane_u64(vreinterpretq_u64_u8(vpmaxq_u8(temp, temp)), 0); - return res == 0; - } - static int three_way_compare(const char *lhs, const char *rhs) { - uint8x16_t l_0 = vld1q_u8((const uint8_t *)lhs); - uint8x16_t r_0 = vld1q_u8((const uint8_t *)rhs); - uint8x16_t l_1 = vld1q_u8((const uint8_t *)(lhs + 16)); - uint8x16_t r_1 = vld1q_u8((const uint8_t *)(rhs + 16)); - uint8x16_t temp = vpmaxq_u8(veorq_u8(l_0, r_0), veorq_u8(l_1, r_1)); - uint64_t res = - vgetq_lane_u64(vreinterpretq_u64_u8(vpmaxq_u8(temp, temp)), 0); - if (res == 0) - return 0; - size_t index = (__builtin_ctzl(res) >> 3) << 2; - uint32_t l = *((const uint32_t *)(lhs + index)); - uint32_t r = *((const uint32_t *)(rhs + index)); - return __llvm_libc::scalar::_4::scalar_three_way_compare(l, r); - } -}; - -using _32 = N32; -using _64 = Repeated<_32, 2>; -#else -using _32 = __llvm_libc::scalar::_32; -using _64 = __llvm_libc::scalar::_64; -#endif // __ARM_NEON - -} // namespace aarch64 -} // namespace __llvm_libc - -#endif // defined(LLVM_LIBC_ARCH_AARCH64) - -#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_AARCH64_H diff --git a/libc/src/string/memory_utils/elements_x86.h b/libc/src/string/memory_utils/elements_x86.h deleted file mode 100644 --- a/libc/src/string/memory_utils/elements_x86.h +++ /dev/null @@ -1,189 +0,0 @@ -//===-- Elementary operations for x86 -------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_X86_H -#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_X86_H - -#include "src/__support/CPP/bit.h" -#include "src/__support/architectures.h" - -#if defined(LLVM_LIBC_ARCH_X86) - -#include // size_t -#include // uint8_t, uint16_t, uint32_t, uint64_t - -#ifdef __SSE2__ -#include -#endif // __SSE2__ - -#include "src/string/memory_utils/elements.h" // __llvm_libc::scalar - -// Fixed-size Vector Operations -// ---------------------------- - -namespace __llvm_libc { -namespace x86 { - -#ifdef __SSE2__ -template struct Vector : public Base { - static void copy(char *__restrict dst, const char *__restrict src) { - Base::store(dst, Base::load(src)); - } - - static void move(char *dst, const char *src) { - Base::store(dst, Base::load(src)); - } - - static bool equals(const char *a, const char *b) { - return Base::not_equal_mask(Base::load(a), Base::load(b)) == 0; - } - - static int three_way_compare(const char *a, const char *b) { - const auto mask = Base::not_equal_mask(Base::load(a), Base::load(b)); - if (!mask) - return 0; - return char_diff(a, b, mask); - } - - static void splat_set(char *dst, const unsigned char value) { - Base::store(dst, Base::get_splatted_value(value)); - } - - static int char_diff(const char *a, const char *b, uint64_t mask) { - const size_t diff_index = __builtin_ctzll(mask); - const int ca = (unsigned char)a[diff_index]; - const int cb = (unsigned char)b[diff_index]; - return ca - cb; - } -}; - -struct M128 { - static constexpr size_t SIZE = 16; - using T = char __attribute__((__vector_size__(SIZE))); - static uint16_t mask(T value) { - // NOLINTNEXTLINE(llvmlibc-callee-namespace) - return static_cast( - _mm_movemask_epi8(cpp::bit_cast<__m128i>(value))); - } - static uint16_t not_equal_mask(T a, T b) { return mask(a != b); } - static T load(const char *ptr) { - // NOLINTNEXTLINE(llvmlibc-callee-namespace) - return cpp::bit_cast( - _mm_loadu_si128(reinterpret_cast<__m128i_u const *>(ptr))); - } - static void store(char *ptr, T value) { - // NOLINTNEXTLINE(llvmlibc-callee-namespace) - return _mm_storeu_si128(reinterpret_cast<__m128i_u *>(ptr), - cpp::bit_cast<__m128i>(value)); - } - static T get_splatted_value(const char v) { - const T splatted = {v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v}; - return splatted; - } -}; - -using Vector128 = Vector; // 16 Bytes - -#ifdef __AVX2__ -struct M256 { - static constexpr size_t SIZE = 32; - using T = char __attribute__((__vector_size__(SIZE))); - static uint32_t mask(T value) { - // NOLINTNEXTLINE(llvmlibc-callee-namespace) - return _mm256_movemask_epi8(cpp::bit_cast<__m256i>(value)); - } - static uint32_t not_equal_mask(T a, T b) { return mask(a != b); } - static T load(const char *ptr) { - // NOLINTNEXTLINE(llvmlibc-callee-namespace) - return cpp::bit_cast( - _mm256_loadu_si256(reinterpret_cast<__m256i const *>(ptr))); - } - static void store(char *ptr, T value) { - // NOLINTNEXTLINE(llvmlibc-callee-namespace) - return _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr), - cpp::bit_cast<__m256i>(value)); - } - static T get_splatted_value(const char v) { - const T splatted = {v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, - v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v}; - return splatted; - } -}; - -using Vector256 = Vector; // 32 Bytes - -#if defined(__AVX512F__) and defined(__AVX512BW__) -struct M512 { - static constexpr size_t SIZE = 64; - using T = char __attribute__((__vector_size__(SIZE))); - static uint64_t not_equal_mask(T a, T b) { - // NOLINTNEXTLINE(llvmlibc-callee-namespace) - return _mm512_cmpneq_epi8_mask(cpp::bit_cast<__m512i>(a), - cpp::bit_cast<__m512i>(b)); - } - static T load(const char *ptr) { - // NOLINTNEXTLINE(llvmlibc-callee-namespace) - return cpp::bit_cast(_mm512_loadu_epi8(ptr)); - } - static void store(char *ptr, T value) { - // NOLINTNEXTLINE(llvmlibc-callee-namespace) - return _mm512_storeu_epi8(ptr, cpp::bit_cast<__m512i>(value)); - } - static T get_splatted_value(const char v) { - const T splatted = {v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, - v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, - v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, - v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v}; - return splatted; - } -}; -using Vector512 = Vector; - -#endif // defined(__AVX512F__) and defined(__AVX512BW__) -#endif // __AVX2__ -#endif // __SSE2__ - -using _1 = __llvm_libc::scalar::_1; -using _2 = __llvm_libc::scalar::_2; -using _3 = __llvm_libc::scalar::_3; -using _4 = __llvm_libc::scalar::_4; -using _8 = __llvm_libc::scalar::_8; -#if defined(__AVX512F__) && defined(__AVX512BW__) -using _16 = __llvm_libc::x86::Vector128; -using _32 = __llvm_libc::x86::Vector256; -using _64 = __llvm_libc::x86::Vector512; -using _128 = __llvm_libc::Repeated<_64, 2>; -#elif defined(__AVX2__) -using _16 = __llvm_libc::x86::Vector128; -using _32 = __llvm_libc::x86::Vector256; -using _64 = __llvm_libc::Repeated<_32, 2>; -using _128 = __llvm_libc::Repeated<_32, 4>; -#elif defined(__SSE2__) -using _16 = __llvm_libc::x86::Vector128; -using _32 = __llvm_libc::Repeated<_16, 2>; -using _64 = __llvm_libc::Repeated<_16, 4>; -using _128 = __llvm_libc::Repeated<_16, 8>; -#else -using _16 = __llvm_libc::Repeated<_8, 2>; -using _32 = __llvm_libc::Repeated<_8, 4>; -using _64 = __llvm_libc::Repeated<_8, 8>; -using _128 = __llvm_libc::Repeated<_8, 16>; -#endif - -struct Accelerator { - static void copy(char *dst, const char *src, size_t count) { - asm volatile("rep movsb" : "+D"(dst), "+S"(src), "+c"(count) : : "memory"); - } -}; - -} // namespace x86 -} // namespace __llvm_libc - -#endif // defined(LLVM_LIBC_ARCH_X86) - -#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_X86_H diff --git a/libc/test/src/string/memory_utils/CMakeLists.txt b/libc/test/src/string/memory_utils/CMakeLists.txt --- a/libc/test/src/string/memory_utils/CMakeLists.txt +++ b/libc/test/src/string/memory_utils/CMakeLists.txt @@ -3,8 +3,6 @@ SUITE libc_string_unittests SRCS - elements_test.cpp - memory_access_test.cpp op_tests.cpp utils_test.cpp COMPILE_OPTIONS diff --git a/libc/test/src/string/memory_utils/elements_test.cpp b/libc/test/src/string/memory_utils/elements_test.cpp deleted file mode 100644 --- a/libc/test/src/string/memory_utils/elements_test.cpp +++ /dev/null @@ -1,137 +0,0 @@ -//===-- Unittests for memory_utils ----------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "src/__support/CPP/array.h" -#include "src/__support/CPP/span.h" -#include "src/string/memory_utils/elements.h" -#include "utils/UnitTest/Test.h" - -namespace __llvm_libc { - -// Registering Types -using FixedSizeTypes = testing::TypeList< -#if defined(__SSE2__) - x86::Vector128, // -#endif // __SSE2__ -#if defined(__AVX2__) - x86::Vector256, // -#endif // __AVX2__ -#if defined(__AVX512F__) and defined(__AVX512BW__) - x86::Vector512, // -#endif // defined(__AVX512F__) and defined(__AVX512BW__) - scalar::UINT8, // - scalar::UINT16, // - scalar::UINT32, // - scalar::UINT64, // - Repeated, // - Repeated, // - Repeated, // - Repeated, // - Repeated, // - Chained, // - Chained, // - builtin::_1, // - builtin::_2, // - builtin::_3, // - builtin::_4, // - builtin::_8 // - >; - -static char GetRandomChar() { - static constexpr const uint64_t a = 1103515245; - static constexpr const uint64_t c = 12345; - static constexpr const uint64_t m = 1ULL << 31; - static uint64_t seed = 123456789; - seed = (a * seed + c) % m; - return seed; -} - -static void Randomize(cpp::span buffer) { - for (auto ¤t : buffer) - current = GetRandomChar(); -} - -template using Buffer = cpp::array; - -template Buffer GetRandomBuffer() { - Buffer buffer; - Randomize(buffer); - return buffer; -} - -TYPED_TEST(LlvmLibcMemoryElements, copy, FixedSizeTypes) { - Buffer Dst; - const auto buffer = GetRandomBuffer(); - copy(Dst.data(), buffer.data()); - for (size_t i = 0; i < ParamType::SIZE; ++i) - EXPECT_EQ(Dst[i], buffer[i]); -} - -template T copy(const T &Input) { - T Output; - for (size_t I = 0; I < Input.size(); ++I) - Output[I] = Input[I]; - return Output; -} - -TYPED_TEST(LlvmLibcMemoryElements, Move, FixedSizeTypes) { - constexpr size_t SIZE = ParamType::SIZE; - using LargeBuffer = cpp::array; - LargeBuffer GroundTruth; - Randomize(GroundTruth); - // Forward, we move the SIZE first bytes from offset 0 to SIZE. - for (size_t Offset = 0; Offset < SIZE; ++Offset) { - LargeBuffer Buffer = copy(GroundTruth); - move(&Buffer[Offset], &Buffer[0]); - for (size_t I = 0; I < SIZE; ++I) - EXPECT_EQ(Buffer[I + Offset], GroundTruth[I]); - } - // Backward, we move the SIZE last bytes from offset 0 to SIZE. - for (size_t Offset = 0; Offset < SIZE; ++Offset) { - LargeBuffer Buffer = copy(GroundTruth); - move(&Buffer[Offset], &Buffer[SIZE]); - for (size_t I = 0; I < SIZE; ++I) - EXPECT_EQ(Buffer[I + Offset], GroundTruth[SIZE + I]); - } -} - -TYPED_TEST(LlvmLibcMemoryElements, Equals, FixedSizeTypes) { - const auto buffer = GetRandomBuffer(); - EXPECT_TRUE(equals(buffer.data(), buffer.data())); -} - -TYPED_TEST(LlvmLibcMemoryElements, three_way_compare, FixedSizeTypes) { - Buffer initial; - for (auto &c : initial) - c = 5; - - // Testing equality - EXPECT_EQ(three_way_compare(initial.data(), initial.data()), 0); - - // Testing all mismatching positions - for (size_t i = 0; i < ParamType::SIZE; ++i) { - auto copy = initial; - ++copy[i]; // copy is now lexicographycally greated than initial - const auto *less = initial.data(); - const auto *greater = copy.data(); - EXPECT_LT(three_way_compare(less, greater), 0); - EXPECT_GT(three_way_compare(greater, less), 0); - } -} - -TYPED_TEST(LlvmLibcMemoryElements, Splat, FixedSizeTypes) { - Buffer Dst; - const cpp::array values = {char(0x00), char(0x7F), char(0xFF)}; - for (char value : values) { - splat_set(Dst.data(), value); - for (size_t i = 0; i < ParamType::SIZE; ++i) - EXPECT_EQ(Dst[i], value); - } -} - -} // namespace __llvm_libc diff --git a/libc/test/src/string/memory_utils/memory_access_test.cpp b/libc/test/src/string/memory_utils/memory_access_test.cpp deleted file mode 100644 --- a/libc/test/src/string/memory_utils/memory_access_test.cpp +++ /dev/null @@ -1,228 +0,0 @@ -//===-- Unittests for memory_utils ----------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#define LLVM_LIBC_UNITTEST_OBSERVE 1 - -#include "src/__support/CPP/array.h" -#include "src/string/memory_utils/elements.h" -#include "utils/UnitTest/Test.h" - -#include -#include - -namespace __llvm_libc { - -static constexpr const size_t kMaxBuffer = 32; - -struct BufferAccess : cpp::array { - BufferAccess() { Reset(); } - void Reset() { - for (auto &value : *this) - value = '0'; - this->operator[](kMaxBuffer) = '\0'; - } - void Touch(ptrdiff_t offset, size_t size) { - if (offset < 0) - return; - for (size_t i = 0; i < size; ++i) - ++(*this)[offset + i]; - } - operator const char *() const { return this->data(); } -}; - -struct Buffer { - ptrdiff_t Offset(const char *ptr) const { - const bool contained = ptr >= data.begin() && ptr < data.end(); - return contained ? ptr - data.begin() : -1; - } - void Reset() { - reads.Reset(); - writes.Reset(); - } - cpp::array data; - BufferAccess __attribute__((aligned(64))) reads; - BufferAccess __attribute__((aligned(64))) writes; -}; - -struct MemoryAccessObserver { - void ObserveRead(const char *ptr, size_t size) { - Buffer1.reads.Touch(Buffer1.Offset(ptr), size); - Buffer2.reads.Touch(Buffer2.Offset(ptr), size); - } - - void ObserveWrite(const char *ptr, size_t size) { - Buffer1.writes.Touch(Buffer1.Offset(ptr), size); - Buffer2.writes.Touch(Buffer2.Offset(ptr), size); - } - - void Reset() { - Buffer1.Reset(); - Buffer2.Reset(); - } - - Buffer Buffer1; - Buffer Buffer2; -}; - -static MemoryAccessObserver Observer; - -template struct TestingElement { - static constexpr size_t SIZE = Size; - - static void copy(char *__restrict dst, const char *__restrict src) { - Observer.ObserveRead(src, SIZE); - Observer.ObserveWrite(dst, SIZE); - } - - static bool equals(const char *lhs, const char *rhs) { - Observer.ObserveRead(lhs, SIZE); - Observer.ObserveRead(rhs, SIZE); - return true; - } - - static int three_way_compare(const char *lhs, const char *rhs) { - Observer.ObserveRead(lhs, SIZE); - Observer.ObserveRead(rhs, SIZE); - return 0; - } - - static void splat_set(char *dst, const unsigned char value) { - Observer.ObserveWrite(dst, SIZE); - } -}; - -using Types = testing::TypeList< - TestingElement<1>, // 1 Byte - TestingElement<2>, // 2 Bytes - TestingElement<4>, // 4 Bytes - Repeated, 3>, // 6 Bytes - Chained, TestingElement<2>, TestingElement<1>> // 7 Bytes - >; - -struct LlvmLibcTestAccessBase : public testing::Test { - - template - void checkOperations(const BufferAccess &expected) { - static const BufferAccess untouched; - - Observer.Reset(); - HigherOrder::copy(dst_ptr() + Offset, src_ptr() + Offset, Size); - ASSERT_STREQ(src().writes, untouched); - ASSERT_STREQ(dst().reads, untouched); - ASSERT_STREQ(src().reads, expected); - ASSERT_STREQ(dst().writes, expected); - Observer.Reset(); - HigherOrder::equals(lhs_ptr() + Offset, rhs_ptr() + Offset, Size); - ASSERT_STREQ(lhs().writes, untouched); - ASSERT_STREQ(rhs().writes, untouched); - ASSERT_STREQ(lhs().reads, expected); - ASSERT_STREQ(rhs().reads, expected); - Observer.Reset(); - HigherOrder::three_way_compare(lhs_ptr() + Offset, rhs_ptr() + Offset, - Size); - ASSERT_STREQ(lhs().writes, untouched); - ASSERT_STREQ(rhs().writes, untouched); - ASSERT_STREQ(lhs().reads, expected); - ASSERT_STREQ(rhs().reads, expected); - Observer.Reset(); - HigherOrder::splat_set(dst_ptr() + Offset, 5, Size); - ASSERT_STREQ(src().reads, untouched); - ASSERT_STREQ(src().writes, untouched); - ASSERT_STREQ(dst().reads, untouched); - ASSERT_STREQ(dst().writes, expected); - } - - void checkMaxAccess(const BufferAccess &expected, int max) { - for (size_t i = 0; i < kMaxBuffer; ++i) { - int value = (int)expected[i] - '0'; - ASSERT_GE(value, 0); - ASSERT_LE(value, max); - } - } - -private: - const Buffer &lhs() const { return Observer.Buffer1; } - const Buffer &rhs() const { return Observer.Buffer2; } - const Buffer &src() const { return Observer.Buffer2; } - const Buffer &dst() const { return Observer.Buffer1; } - Buffer &dst() { return Observer.Buffer1; } - - char *dst_ptr() { return dst().data.begin(); } - const char *src_ptr() { return src().data.begin(); } - const char *lhs_ptr() { return lhs().data.begin(); } - const char *rhs_ptr() { return rhs().data.begin(); } -}; - -template -struct LlvmLibcTestAccessTail : public LlvmLibcTestAccessBase { - - void TearDown() override { - static constexpr size_t Size = 10; - - BufferAccess expected; - expected.Touch(Size - ParamType::SIZE, ParamType::SIZE); - - checkMaxAccess(expected, 1); - checkOperations, Size>(expected); - } -}; -TYPED_TEST_F(LlvmLibcTestAccessTail, Operations, Types) {} - -template -struct LlvmLibcTestAccessHeadTail : public LlvmLibcTestAccessBase { - void TearDown() override { - static constexpr size_t Size = 10; - - BufferAccess expected; - expected.Touch(0, ParamType::SIZE); - expected.Touch(Size - ParamType::SIZE, ParamType::SIZE); - - checkMaxAccess(expected, 2); - checkOperations, Size>(expected); - } -}; -TYPED_TEST_F(LlvmLibcTestAccessHeadTail, Operations, Types) {} - -template -struct LlvmLibcTestAccessLoop : public LlvmLibcTestAccessBase { - void TearDown() override { - static constexpr size_t Size = 20; - - BufferAccess expected; - for (size_t i = 0; i < Size - ParamType::SIZE; i += ParamType::SIZE) - expected.Touch(i, ParamType::SIZE); - expected.Touch(Size - ParamType::SIZE, ParamType::SIZE); - - checkMaxAccess(expected, 2); - checkOperations, Size>(expected); - } -}; -TYPED_TEST_F(LlvmLibcTestAccessLoop, Operations, Types) {} - -template -struct LlvmLibcTestAccessAlignedAccess : public LlvmLibcTestAccessBase { - void TearDown() override { - static constexpr size_t Size = 10; - static constexpr size_t Offset = 2; - using AlignmentT = TestingElement<4>; - - BufferAccess expected; - expected.Touch(Offset, AlignmentT::SIZE); - expected.Touch(AlignmentT::SIZE, ParamType::SIZE); - expected.Touch(Offset + Size - ParamType::SIZE, ParamType::SIZE); - - checkMaxAccess(expected, 3); - checkOperations::Then>, Size, - Offset>(expected); - checkOperations::Then>, Size, - Offset>(expected); - } -}; -TYPED_TEST_F(LlvmLibcTestAccessAlignedAccess, Operations, Types) {} - -} // namespace __llvm_libc diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -1008,9 +1008,6 @@ cc_library( name = "string_memory_utils", hdrs = [ - "src/string/memory_utils/elements_aarch64.h", - "src/string/memory_utils/elements_x86.h", - "src/string/memory_utils/elements.h", "src/string/memory_utils/op_aarch64.h", "src/string/memory_utils/op_builtin.h", "src/string/memory_utils/op_generic.h",