This is an archive of the discontinued LLVM Phabricator instance.

	#TODO(michaelrj): split out the implementations from memory_utils			#TODO(michaelrj): split out the implementations from memory_utils
	add_header_library(			add_header_library(
	memory_utils			memory_utils
	HDRS			HDRS
	bcmp_implementations.h			bcmp_implementations.h
	bzero_implementations.h			bzero_implementations.h
	elements.h
	memcmp_implementations.h			memcmp_implementations.h
	memcpy_implementations.h			memcpy_implementations.h
	memset_implementations.h			memset_implementations.h
	op_aarch64.h			op_aarch64.h
	op_builtin.h			op_builtin.h
	op_generic.h			op_generic.h
	op_x86.h			op_x86.h
	utils.h			utils.h
	Show All 38 Lines

libc/src/string/memory_utils/elements.h

This file was deleted.

	//===-- Elementary operations to compose memory primitives ----------------===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//

	#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_H
	#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_H

	#include <stddef.h> // size_t
	#include <stdint.h> // uint8_t, uint16_t, uint32_t, uint64_t

	#include "src/__support/endian.h"
	#include "src/string/memory_utils/utils.h"

	namespace __llvm_libc {

	// Elementary Operations
	// --------------------------------
	// We define abstract elementary operations acting on fixed chunks of memory.
	// These are low level building blocks that are meant to be assembled to compose
	// higher order abstractions. Each function is defined twice: once with
	// fixed-size operations, and once with runtime-size operations.

	// Fixed-size copy from 'src' to 'dst'.
	template <typename Element>
	void copy(char __restrict dst, const char __restrict src) {
	Element::copy(dst, src);
	}
	// Runtime-size copy from 'src' to 'dst'.
	template <typename Element>
	void copy(char __restrict dst, const char __restrict src, size_t size) {
	Element::copy(dst, src, size);
	}

	// Fixed-size move from 'src' to 'dst'.
	template <typename Element> void move(char dst, const char src) {
	Element::move(dst, src);
	}
	// Runtime-size move from 'src' to 'dst'.
	template <typename Element> void move(char dst, const char src, size_t size) {
	Element::move(dst, src, size);
	}
	// Runtime-size move from 'src' to 'dst'.
	template <typename Element>
	void move_backward(char dst, const char src, size_t size) {
	Element::move_backward(dst, src, size);
	}

	// Fixed-size equality between 'lhs' and 'rhs'.
	template <typename Element> bool equals(const char lhs, const char rhs) {
	return Element::equals(lhs, rhs);
	}
	// Runtime-size equality between 'lhs' and 'rhs'.
	template <typename Element>
	bool equals(const char lhs, const char rhs, size_t size) {
	return Element::equals(lhs, rhs, size);
	}

	// Fixed-size three-way comparison between 'lhs' and 'rhs'.
	template <typename Element>
	int three_way_compare(const char lhs, const char rhs) {
	return Element::three_way_compare(lhs, rhs);
	}
	// Runtime-size three-way comparison between 'lhs' and 'rhs'.
	template <typename Element>
	int three_way_compare(const char lhs, const char rhs, size_t size) {
	return Element::three_way_compare(lhs, rhs, size);
	}

	// Fixed-size initialization.
	template <typename Element>
	void splat_set(char *dst, const unsigned char value) {
	Element::splat_set(dst, value);
	}
	// Runtime-size initialization.
	template <typename Element>
	void splat_set(char *dst, const unsigned char value, size_t size) {
	Element::splat_set(dst, value, size);
	}

	// Stack placeholder for Move operations.
	template <typename Element> struct Storage { char bytes[Element::SIZE]; };

	// Fixed-size Higher-Order Operations
	// ----------------------------------
	// - Repeated<Type, ElementCount>: Repeat the operation several times in a row.
	// - Chained<Types...>: Chain the operation of several types.

	// Repeat the operation several times in a row.
	template <typename Element, size_t ElementCount> struct Repeated {
	static constexpr size_t SIZE = ElementCount * Element::SIZE;

	static void copy(char __restrict dst, const char __restrict src) {
	for (size_t i = 0; i < ElementCount; ++i) {
	const size_t offset = i * Element::SIZE;
	Element::copy(dst + offset, src + offset);
	}
	}

	static void move(char dst, const char src) {
	const auto value = load(src);
	store(dst, value);
	}

	static bool equals(const char lhs, const char rhs) {
	for (size_t i = 0; i < ElementCount; ++i) {
	const size_t offset = i * Element::SIZE;
	if (!Element::equals(lhs + offset, rhs + offset))
	return false;
	}
	return true;
	}

	static int three_way_compare(const char lhs, const char rhs) {
	for (size_t i = 0; i < ElementCount; ++i) {
	const size_t offset = i * Element::SIZE;
	// We make the assumption that 'equals' is cheaper than
	// 'three_way_compare'.
	if (Element::equals(lhs + offset, rhs + offset))
	continue;
	return Element::three_way_compare(lhs + offset, rhs + offset);
	}
	return 0;
	}

	static void splat_set(char *dst, const unsigned char value) {
	for (size_t i = 0; i < ElementCount; ++i) {
	const size_t offset = i * Element::SIZE;
	Element::splat_set(dst + offset, value);
	}
	}

	static Storage<Repeated> load(const char *ptr) {
	Storage<Repeated> value;
	copy(reinterpret_cast<char *>(&value), ptr);
	return value;
	}

	static void store(char *ptr, Storage<Repeated> value) {
	copy(ptr, reinterpret_cast<const char *>(&value));
	}
	};

	template <typename Element> struct Repeated<Element, 0> {
	static void move(char , const char ) {}
	};

	// Chain the operation of several types.
	// For instance, to handle a 3 bytes operation, one can use:
	// Chained<UINT16, UINT8>::Operation();
	template <typename... Types> struct Chained;

	template <typename Head, typename... Tail> struct Chained<Head, Tail...> {
	static constexpr size_t SIZE = Head::SIZE + Chained<Tail...>::SIZE;

	static void copy(char __restrict dst, const char __restrict src) {
	Chained<Tail...>::copy(dst + Head::SIZE, src + Head::SIZE);
	__llvm_libc::copy<Head>(dst, src);
	}

	static void move(char dst, const char src) {
	const auto value = Head::load(src);
	Chained<Tail...>::move(dst + Head::SIZE, src + Head::SIZE);
	Head::store(dst, value);
	}

	static bool equals(const char lhs, const char rhs) {
	if (!__llvm_libc::equals<Head>(lhs, rhs))
	return false;
	return Chained<Tail...>::equals(lhs + Head::SIZE, rhs + Head::SIZE);
	}

	static int three_way_compare(const char lhs, const char rhs) {
	if (__llvm_libc::equals<Head>(lhs, rhs))
	return Chained<Tail...>::three_way_compare(lhs + Head::SIZE,
	rhs + Head::SIZE);
	return __llvm_libc::three_way_compare<Head>(lhs, rhs);
	}

	static void splat_set(char *dst, const unsigned char value) {
	Chained<Tail...>::splat_set(dst + Head::SIZE, value);
	__llvm_libc::splat_set<Head>(dst, value);
	}
	};

	template <> struct Chained<> {
	static constexpr size_t SIZE = 0;
	static void copy(char __restrict, const char __restrict) {}
	static void move(char , const char ) {}
	static bool equals(const char , const char ) { return true; }
	static int three_way_compare(const char , const char ) { return 0; }
	static void splat_set(char *, const unsigned char) {}
	};

	// Overlap ElementA and ElementB so they span Size bytes.
	template <size_t Size, typename ElementA, typename ElementB = ElementA>
	struct Overlap {
	static constexpr size_t SIZE = Size;
	static_assert(ElementB::SIZE <= ElementA::SIZE, "ElementB too big");
	static_assert(ElementA::SIZE <= Size, "ElementA too big");
	static_assert((ElementA::SIZE + ElementB::SIZE) >= Size,
	"Elements too small to overlap");
	static constexpr size_t OFFSET = SIZE - ElementB::SIZE;

	static void copy(char __restrict dst, const char __restrict src) {
	ElementA::copy(dst, src);
	ElementB::copy(dst + OFFSET, src + OFFSET);
	}

	static void move(char dst, const char src) {
	const auto value_a = ElementA::load(src);
	const auto value_b = ElementB::load(src + OFFSET);
	ElementB::store(dst + OFFSET, value_b);
	ElementA::store(dst, value_a);
	}

	static bool equals(const char lhs, const char rhs) {
	if (!ElementA::equals(lhs, rhs))
	return false;
	if (!ElementB::equals(lhs + OFFSET, rhs + OFFSET))
	return false;
	return true;
	}

	static int three_way_compare(const char lhs, const char rhs) {
	if (!ElementA::equals(lhs, rhs))
	return ElementA::three_way_compare(lhs, rhs);
	if (!ElementB::equals(lhs + OFFSET, rhs + OFFSET))
	return ElementB::three_way_compare(lhs + OFFSET, rhs + OFFSET);
	return 0;
	}

	static void splat_set(char *dst, const unsigned char value) {
	ElementA::splat_set(dst, value);
	ElementB::splat_set(dst + OFFSET, value);
	}
	};

	// Runtime-size Higher-Order Operations
	// ------------------------------------
	// - Tail<T>: Perform the operation on the last 'T::SIZE' bytes of the buffer.
	// - HeadTail<T>: Perform the operation on the first and last 'T::SIZE' bytes
	// of the buffer.
	// - Loop<T>: Perform a loop of fixed-sized operations.

	// Perform the operation on the last 'T::SIZE' bytes of the buffer.
	//
	// e.g. with
	// [1234567812345678123]
	// [__XXXXXXXXXXXXXX___]
	// [________XXXXXXXX___]
	//
	// Precondition: `size >= T::SIZE`.
	template <typename T> struct Tail {
	static void copy(char __restrict dst, const char __restrict src,
	size_t size) {
	return T::copy(dst + offset(size), src + offset(size));
	}

	static bool equals(const char lhs, const char rhs, size_t size) {
	return T::equals(lhs + offset(size), rhs + offset(size));
	}

	static int three_way_compare(const char lhs, const char rhs, size_t size) {
	return T::three_way_compare(lhs + offset(size), rhs + offset(size));
	}

	static void splat_set(char *dst, const unsigned char value, size_t size) {
	return T::splat_set(dst + offset(size), value);
	}

	static size_t offset(size_t size) { return size - T::SIZE; }
	};

	// Perform the operation on the first and last 'T::SIZE' bytes of the buffer.
	// This is useful for overlapping operations.
	//
	// e.g. with
	// [1234567812345678123]
	// [__XXXXXXXXXXXXXX___]
	// [__XXXXXXXX_________]
	// [________XXXXXXXX___]
	//
	// Precondition: `size >= T::SIZE && size <= 2 x T::SIZE`.
	template <typename T> struct HeadTail {
	static void copy(char __restrict dst, const char __restrict src,
	size_t size) {
	T::copy(dst, src);
	Tail<T>::copy(dst, src, size);
	}

	static void move(char dst, const char src, size_t size) {
	const size_t offset = Tail<T>::offset(size);
	const auto head_value = T::load(src);
	const auto tail_value = T::load(src + offset);
	T::store(dst + offset, tail_value);
	T::store(dst, head_value);
	}

	static bool equals(const char lhs, const char rhs, size_t size) {
	if (!T::equals(lhs, rhs))
	return false;
	return Tail<T>::equals(lhs, rhs, size);
	}

	static int three_way_compare(const char lhs, const char rhs, size_t size) {
	if (!T::equals(lhs, rhs))
	return T::three_way_compare(lhs, rhs);
	return Tail<T>::three_way_compare(lhs, rhs, size);
	}

	static void splat_set(char *dst, const unsigned char value, size_t size) {
	T::splat_set(dst, value);
	Tail<T>::splat_set(dst, value, size);
	}
	};

	// Simple loop ending with a Tail operation.
	//
	// e.g. with
	// [12345678123456781234567812345678]
	// [__XXXXXXXXXXXXXXXXXXXXXXXXXXXX___]
	// [__XXXXXXXX_______________________]
	// [__________XXXXXXXX_______________]
	// [__________________XXXXXXXX_______]
	// [______________________XXXXXXXX___]
	//
	// Precondition:
	// - size >= T::SIZE
	template <typename T, typename TailT = T> struct Loop {
	static_assert(T::SIZE == TailT::SIZE,
	"Tail type must have the same size as T");

	static void copy(char __restrict dst, const char __restrict src,
	size_t size) {
	size_t offset = 0;
	do {
	T::copy(dst + offset, src + offset);
	offset += T::SIZE;
	} while (offset < size - T::SIZE);
	Tail<TailT>::copy(dst, src, size);
	}

	// Move forward suitable when dst < src. We load the tail bytes before
	// handling the loop.
	//
	// e.g. Moving two bytes
	// [ \| \| \| \| \|]
	// [___XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX___]
	// [_________________________LLLLLLLL___]
	// [___LLLLLLLL_________________________]
	// [_SSSSSSSS___________________________]
	// [___________LLLLLLLL_________________]
	// [_________SSSSSSSS___________________]
	// [___________________LLLLLLLL_________]
	// [_________________SSSSSSSS___________]
	// [_______________________SSSSSSSS_____]
	static void move(char dst, const char src, size_t size) {
	const size_t tail_offset = Tail<T>::offset(size);
	const auto tail_value = TailT::load(src + tail_offset);
	size_t offset = 0;
	do {
	T::move(dst + offset, src + offset);
	offset += T::SIZE;
	} while (offset < size - T::SIZE);
	TailT::store(dst + tail_offset, tail_value);
	}

	// Move forward suitable when dst > src. We load the head bytes before
	// handling the loop.
	//
	// e.g. Moving two bytes
	// [ \| \| \| \| \|]
	// [___XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX___]
	// [___LLLLLLLL_________________________]
	// [_________________________LLLLLLLL___]
	// [___________________________SSSSSSSS_]
	// [_________________LLLLLLLL___________]
	// [___________________SSSSSSSS_________]
	// [_________LLLLLLLL___________________]
	// [___________SSSSSSSS_________________]
	// [_____SSSSSSSS_______________________]
	static void move_backward(char dst, const char src, size_t size) {
	const auto head_value = TailT::load(src);
	ptrdiff_t offset = size - T::SIZE;
	do {
	T::move(dst + offset, src + offset);
	offset -= T::SIZE;
	} while (offset >= 0);
	TailT::store(dst, head_value);
	}

	static bool equals(const char lhs, const char rhs, size_t size) {
	size_t offset = 0;
	do {
	if (!T::equals(lhs + offset, rhs + offset))
	return false;
	offset += T::SIZE;
	} while (offset < size - T::SIZE);
	return Tail<TailT>::equals(lhs, rhs, size);
	}

	static int three_way_compare(const char lhs, const char rhs, size_t size) {
	size_t offset = 0;
	do {
	if (!T::equals(lhs + offset, rhs + offset))
	return T::three_way_compare(lhs + offset, rhs + offset);
	offset += T::SIZE;
	} while (offset < size - T::SIZE);
	return Tail<TailT>::three_way_compare(lhs, rhs, size);
	}

	static void splat_set(char *dst, const unsigned char value, size_t size) {
	size_t offset = 0;
	do {
	T::splat_set(dst + offset, value);
	offset += T::SIZE;
	} while (offset < size - T::SIZE);
	Tail<TailT>::splat_set(dst, value, size);
	}
	};

	namespace internal {

	template <Arg arg> struct ArgSelector {};

	template <> struct ArgSelector<Arg::P1> {
	template <typename T1, typename T2>
	static T1 __restrict &Select(T1 __restrict &p1ref, T2 *__restrict &) {
	return p1ref;
	}
	};

	template <> struct ArgSelector<Arg::P2> {
	template <typename T1, typename T2>
	static T2 __restrict &Select(T1 __restrict &, T2 *__restrict &p2ref) {
	return p2ref;
	}
	};

	// Provides a specialized bump function that adjusts pointers and size so first
	// argument (resp. second argument) gets aligned to Alignment.
	// We make sure the compiler knows about the adjusted pointer alignment.
	// The 'additional_bumps' parameter allows to reach previous / next aligned
	// pointers.
	template <Arg arg, size_t Alignment> struct Align {
	template <typename T1, typename T2>
	static void bump(T1 __restrict &p1ref, T2 __restrict &p2ref, size_t &size,
	int additional_bumps = 0) {
	auto &aligned_ptr = ArgSelector<arg>::Select(p1ref, p2ref);
	auto offset = distance_to_align_up<Alignment>(aligned_ptr);
	adjust(offset + additional_bumps * Alignment, p1ref, p2ref, size);
	aligned_ptr = assume_aligned<Alignment>(aligned_ptr);
	}
	};

	} // namespace internal

	// An alignment operation that:
	// - executes the 'AlignmentT' operation
	// - bumps 'dst' or 'src' (resp. 'lhs' or 'rhs') pointers so that the selected
	// pointer gets aligned, size is decreased accordingly.
	// - calls the 'NextT' operation.
	//
	// e.g. A 16-byte Destination Aligned 32-byte Loop Copy can be written as:
	// copy<Align<_16, Arg::Dst>::Then<Loop<_32>>>(dst, src, count);
	template <typename AlignmentT, Arg AlignOn = Arg::P1> struct Align {
	private:
	static constexpr size_t ALIGNMENT = AlignmentT::SIZE;
	static_assert(ALIGNMENT > 1, "Alignment must be more than 1");
	static_assert(is_power2(ALIGNMENT), "Alignment must be a power of 2");

	public:
	template <typename NextT> struct Then {
	static void copy(char __restrict dst, const char __restrict src,
	size_t size) {
	AlignmentT::copy(dst, src);
	internal::Align<AlignOn, ALIGNMENT>::bump(dst, src, size);
	NextT::copy(dst, src, size);
	}

	// Move forward suitable when dst < src. The alignment is performed with an
	// HeadTail operation of size ∈ [Alignment, 2 x Alignment].
	//
	// e.g. Moving two bytes and making sure src is then aligned.
	// [ \| \| \| \| ]
	// [____XXXXXXXXXXXXXXXXXXXXXXXXXXXX_]
	// [____LLLLLLLL_____________________]
	// [___________LLLLLLLL______________]
	// [_SSSSSSSS________________________]
	// [________SSSSSSSS_________________]
	//
	// e.g. Moving two bytes and making sure dst is then aligned.
	// [ \| \| \| \| ]
	// [____XXXXXXXXXXXXXXXXXXXXXXXXXXXX_]
	// [____LLLLLLLL_____________________]
	// [______LLLLLLLL___________________]
	// [_SSSSSSSS________________________]
	// [___SSSSSSSS______________________]
	static void move(char dst, const char src, size_t size) {
	char *next_dst = dst;
	const char *next_src = src;
	size_t next_size = size;
	internal::Align<AlignOn, ALIGNMENT>::bump(next_dst, next_src, next_size,
	1);
	HeadTail<AlignmentT>::move(dst, src, size - next_size);
	NextT::move(next_dst, next_src, next_size);
	}

	// Move backward suitable when dst > src. The alignment is performed with an
	// HeadTail operation of size ∈ [Alignment, 2 x Alignment].
	//
	// e.g. Moving two bytes backward and making sure src is then aligned.
	// [ \| \| \| \| ]
	// [____XXXXXXXXXXXXXXXXXXXXXXXX_____]
	// [ _________________LLLLLLLL_______]
	// [ ___________________LLLLLLLL_____]
	// [____________________SSSSSSSS_____]
	// [______________________SSSSSSSS___]
	//
	// e.g. Moving two bytes and making sure dst is then aligned.
	// [ \| \| \| \| ]
	// [____XXXXXXXXXXXXXXXXXXXXXXXX_____]
	// [ _______________LLLLLLLL_________]
	// [ ___________________LLLLLLLL_____]
	// [__________________SSSSSSSS_______]
	// [______________________SSSSSSSS___]
	static void move_backward(char dst, const char src, size_t size) {
	char *headtail_dst = dst + size;
	const char *headtail_src = src + size;
	size_t headtail_size = 0;
	internal::Align<AlignOn, ALIGNMENT>::bump(headtail_dst, headtail_src,
	headtail_size, -2);
	HeadTail<AlignmentT>::move(headtail_dst, headtail_src, headtail_size);
	NextT::move_backward(dst, src, size - headtail_size);
	}

	static bool equals(const char lhs, const char rhs, size_t size) {
	if (!AlignmentT::equals(lhs, rhs))
	return false;
	internal::Align<AlignOn, ALIGNMENT>::bump(lhs, rhs, size);
	return NextT::equals(lhs, rhs, size);
	}

	static int three_way_compare(const char lhs, const char rhs,
	size_t size) {
	if (!AlignmentT::equals(lhs, rhs))
	return AlignmentT::three_way_compare(lhs, rhs);
	internal::Align<AlignOn, ALIGNMENT>::bump(lhs, rhs, size);
	return NextT::three_way_compare(lhs, rhs, size);
	}

	static void splat_set(char *dst, const unsigned char value, size_t size) {
	AlignmentT::splat_set(dst, value);
	char *dummy = nullptr;
	internal::Align<Arg::P1, ALIGNMENT>::bump(dst, dummy, size);
	NextT::splat_set(dst, value, size);
	}
	};
	};

	// An operation that allows to skip the specified amount of bytes.
	template <ptrdiff_t Bytes> struct Skip {
	template <typename NextT> struct Then {
	static void copy(char __restrict dst, const char __restrict src,
	size_t size) {
	NextT::copy(dst + Bytes, src + Bytes, size - Bytes);
	}

	static void copy(char __restrict dst, const char __restrict src) {
	NextT::copy(dst + Bytes, src + Bytes);
	}

	static bool equals(const char lhs, const char rhs, size_t size) {
	return NextT::equals(lhs + Bytes, rhs + Bytes, size - Bytes);
	}

	static bool equals(const char lhs, const char rhs) {
	return NextT::equals(lhs + Bytes, rhs + Bytes);
	}

	static int three_way_compare(const char lhs, const char rhs,
	size_t size) {
	return NextT::three_way_compare(lhs + Bytes, rhs + Bytes, size - Bytes);
	}

	static int three_way_compare(const char lhs, const char rhs) {
	return NextT::three_way_compare(lhs + Bytes, rhs + Bytes);
	}

	static void splat_set(char *dst, const unsigned char value, size_t size) {
	NextT::splat_set(dst + Bytes, value, size - Bytes);
	}

	static void splat_set(char *dst, const unsigned char value) {
	NextT::splat_set(dst + Bytes, value);
	}
	};
	};

	// Fixed-size Builtin Operations
	// -----------------------------
	// Note: Do not use 'builtin' right now as it requires the implementation of the
	// `_inline` versions of all the builtins. Theoretically, Clang can still turn
	// them into calls to the C library leading to reentrancy problems.
	namespace builtin {

	#ifndef __has_builtin
	#define __has_builtin(x) 0 // Compatibility with non-clang compilers.
	#endif

	template <size_t Size> struct Builtin {
	static constexpr size_t SIZE = Size;

	static void copy(char __restrict dst, const char __restrict src) {
	#if LLVM_LIBC_HAVE_MEMORY_SANITIZER \|\| LLVM_LIBC_HAVE_ADDRESS_SANITIZER
	for_loop_copy(dst, src);
	#elif __has_builtin(__builtin_memcpy_inline)
	// __builtin_memcpy_inline guarantees to never call external functions.
	// Unfortunately it is not widely available.
	__builtin_memcpy_inline(dst, src, SIZE);
	#else
	for_loop_copy(dst, src);
	#endif
	}

	static void move(char dst, const char src) {
	#if LLVM_LIBC_HAVE_MEMORY_SANITIZER \|\| LLVM_LIBC_HAVE_ADDRESS_SANITIZER
	for_loop_move(dst, src);
	#elif __has_builtin(__builtin_memmove)
	__builtin_memmove(dst, src, SIZE);
	#else
	for_loop_move(dst, src);
	#endif
	}

	#if __has_builtin(__builtin_memcmp_inline)
	#define LLVM_LIBC_MEMCMP __builtin_memcmp_inline
	#else
	#define LLVM_LIBC_MEMCMP __builtin_memcmp
	#endif

	static bool equals(const char lhs, const char rhs) {
	return LLVM_LIBC_MEMCMP(lhs, rhs, SIZE) == 0;
	}

	static int three_way_compare(const char lhs, const char rhs) {
	return LLVM_LIBC_MEMCMP(lhs, rhs, SIZE);
	}

	static void splat_set(char *dst, const unsigned char value) {
	__builtin_memset(dst, value, SIZE);
	}

	private:
	// Copies `SIZE` bytes from `src` to `dst` using a for loop.
	// This code requires the use of `-fno-builtin-memcpy` to prevent the compiler
	// from turning the for-loop back into `__builtin_memcpy`.
	static void for_loop_copy(char __restrict dst, const char __restrict src) {
	for (size_t i = 0; i < SIZE; ++i)
	dst[i] = src[i];
	}

	static void for_loop_move(char dst, const char src) {
	for (size_t i = 0; i < SIZE; ++i)
	dst[i] = src[i];
	}
	};

	using _1 = Builtin<1>;
	using _2 = Builtin<2>;
	using _3 = Builtin<3>;
	using _4 = Builtin<4>;
	using _8 = Builtin<8>;
	using _16 = Builtin<16>;
	using _32 = Builtin<32>;
	using _64 = Builtin<64>;
	using _128 = Builtin<128>;

	} // namespace builtin

	// Fixed-size Scalar Operations
	// ----------------------------
	namespace scalar {

	// The Scalar type makes use of simple sized integers.
	template <typename T> struct Scalar {
	static constexpr size_t SIZE = sizeof(T);

	static void copy(char __restrict dst, const char __restrict src) {
	store(dst, load(src));
	}

	static void move(char dst, const char src) { store(dst, load(src)); }

	static bool equals(const char lhs, const char rhs) {
	return load(lhs) == load(rhs);
	}

	static int three_way_compare(const char lhs, const char rhs) {
	return scalar_three_way_compare(load(lhs), load(rhs));
	}

	static void splat_set(char *dst, const unsigned char value) {
	store(dst, get_splatted_value(value));
	}

	static int scalar_three_way_compare(T a, T b);

	static T load(const char *ptr) {
	T value;
	builtin::Builtin<SIZE>::copy(reinterpret_cast<char *>(&value), ptr);
	return value;
	}
	static void store(char *ptr, T value) {
	builtin::Builtin<SIZE>::copy(ptr, reinterpret_cast<const char *>(&value));
	}

	private:
	static T get_splatted_value(const unsigned char value) {
	return T(~0) / T(0xFF) * T(value);
	}
	};

	template <>
	inline int Scalar<uint8_t>::scalar_three_way_compare(uint8_t a, uint8_t b) {
	const int16_t la = Endian::to_big_endian(a);
	const int16_t lb = Endian::to_big_endian(b);
	return la - lb;
	}
	template <>
	inline int Scalar<uint16_t>::scalar_three_way_compare(uint16_t a, uint16_t b) {
	const int32_t la = Endian::to_big_endian(a);
	const int32_t lb = Endian::to_big_endian(b);
	return la - lb;
	}
	template <>
	inline int Scalar<uint32_t>::scalar_three_way_compare(uint32_t a, uint32_t b) {
	const uint32_t la = Endian::to_big_endian(a);
	const uint32_t lb = Endian::to_big_endian(b);
	return la > lb ? 1 : la < lb ? -1 : 0;
	}
	template <>
	inline int Scalar<uint64_t>::scalar_three_way_compare(uint64_t a, uint64_t b) {
	const uint64_t la = Endian::to_big_endian(a);
	const uint64_t lb = Endian::to_big_endian(b);
	return la > lb ? 1 : la < lb ? -1 : 0;
	}

	using UINT8 = Scalar<uint8_t>; // 1 Byte
	using UINT16 = Scalar<uint16_t>; // 2 Bytes
	using UINT32 = Scalar<uint32_t>; // 4 Bytes
	using UINT64 = Scalar<uint64_t>; // 8 Bytes

	using _1 = UINT8;
	using _2 = UINT16;
	using _3 = Chained<UINT16, UINT8>;
	using _4 = UINT32;
	using _8 = UINT64;
	using _16 = Repeated<_8, 2>;
	using _32 = Repeated<_8, 4>;
	using _64 = Repeated<_8, 8>;
	using _128 = Repeated<_8, 16>;

	} // namespace scalar
	} // namespace __llvm_libc

	#include <src/string/memory_utils/elements_aarch64.h>
	#include <src/string/memory_utils/elements_x86.h>

	#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_H

libc/src/string/memory_utils/elements_aarch64.h

This file was deleted.

	//===-- Elementary operations for aarch64 --------------------------------===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//

	#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_AARCH64_H
	#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_AARCH64_H

	#include "src/__support/architectures.h"

	#if defined(LLVM_LIBC_ARCH_AARCH64)

	#include <src/string/memory_utils/elements.h>
	#include <stddef.h> // size_t
	#include <stdint.h> // uint8_t, uint16_t, uint32_t, uint64_t

	#ifdef __ARM_NEON
	#include <arm_neon.h>
	#endif

	namespace __llvm_libc {
	namespace aarch64_memset {
	#ifdef __ARM_NEON
	struct Splat8 {
	static constexpr size_t SIZE = 8;
	static void splat_set(char *dst, const unsigned char value) {
	vst1_u8((uint8_t *)dst, vdup_n_u8(value));
	}
	};

	struct Splat16 {
	static constexpr size_t SIZE = 16;
	static void splat_set(char *dst, const unsigned char value) {
	vst1q_u8((uint8_t *)dst, vdupq_n_u8(value));
	}
	};

	using _8 = Splat8;
	using _16 = Splat16;
	#else
	using _8 = __llvm_libc::scalar::_8;
	using _16 = Repeated<_8, 2>;
	#endif // __ARM_NEON

	using _1 = __llvm_libc::scalar::_1;
	using _2 = __llvm_libc::scalar::_2;
	using _3 = __llvm_libc::scalar::_3;
	using _4 = __llvm_libc::scalar::_4;
	using _32 = Chained<_16, _16>;
	using _64 = Chained<_32, _32>;

	struct Zva64 {
	static constexpr size_t SIZE = 64;

	static void splat_set(char *dst, const unsigned char) {
	#if __SIZEOF_POINTER__ == 4
	asm("dc zva, %w[dst]" : : [dst] "r"(dst) : "memory");
	#else
	asm("dc zva, %[dst]" : : [dst] "r"(dst) : "memory");
	#endif
	}
	};

	inline static bool hasZva() {
	uint64_t zva_val;
	asm("mrs %[zva_val], dczid_el0" : [zva_val] "=r"(zva_val));
	// DC ZVA is permitted if DZP, bit [4] is zero.
	// BS, bits [3:0] is log2 of the block size in words.
	// So the next line checks whether the instruction is permitted and block size
	// is 16 words (i.e. 64 bytes).
	return (zva_val & 0b11111) == 0b00100;
	}

	} // namespace aarch64_memset

	namespace aarch64 {

	using _1 = __llvm_libc::scalar::_1;
	using _2 = __llvm_libc::scalar::_2;
	using _3 = __llvm_libc::scalar::_3;
	using _4 = __llvm_libc::scalar::_4;
	using _8 = __llvm_libc::scalar::_8;
	using _16 = __llvm_libc::scalar::_16;

	#ifdef __ARM_NEON
	struct N32 {
	static constexpr size_t SIZE = 32;
	static bool equals(const char lhs, const char rhs) {
	uint8x16_t l_0 = vld1q_u8((const uint8_t *)lhs);
	uint8x16_t r_0 = vld1q_u8((const uint8_t *)rhs);
	uint8x16_t l_1 = vld1q_u8((const uint8_t *)(lhs + 16));
	uint8x16_t r_1 = vld1q_u8((const uint8_t *)(rhs + 16));
	uint8x16_t temp = vpmaxq_u8(veorq_u8(l_0, r_0), veorq_u8(l_1, r_1));
	uint64_t res =
	vgetq_lane_u64(vreinterpretq_u64_u8(vpmaxq_u8(temp, temp)), 0);
	return res == 0;
	}
	static int three_way_compare(const char lhs, const char rhs) {
	uint8x16_t l_0 = vld1q_u8((const uint8_t *)lhs);
	uint8x16_t r_0 = vld1q_u8((const uint8_t *)rhs);
	uint8x16_t l_1 = vld1q_u8((const uint8_t *)(lhs + 16));
	uint8x16_t r_1 = vld1q_u8((const uint8_t *)(rhs + 16));
	uint8x16_t temp = vpmaxq_u8(veorq_u8(l_0, r_0), veorq_u8(l_1, r_1));
	uint64_t res =
	vgetq_lane_u64(vreinterpretq_u64_u8(vpmaxq_u8(temp, temp)), 0);
	if (res == 0)
	return 0;
	size_t index = (__builtin_ctzl(res) >> 3) << 2;
	uint32_t l = ((const uint32_t )(lhs + index));
	uint32_t r = ((const uint32_t )(rhs + index));
	return __llvm_libc::scalar::_4::scalar_three_way_compare(l, r);
	}
	};

	using _32 = N32;
	using _64 = Repeated<_32, 2>;
	#else
	using _32 = __llvm_libc::scalar::_32;
	using _64 = __llvm_libc::scalar::_64;
	#endif // __ARM_NEON

	} // namespace aarch64
	} // namespace __llvm_libc

	#endif // defined(LLVM_LIBC_ARCH_AARCH64)

	#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_AARCH64_H

libc/src/string/memory_utils/elements_x86.h

This file was deleted.

	//===-- Elementary operations for x86 -------------------------------------===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//

	#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_X86_H
	#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_X86_H

	#include "src/__support/CPP/bit.h"
	#include "src/__support/architectures.h"

	#if defined(LLVM_LIBC_ARCH_X86)

	#include <stddef.h> // size_t
	#include <stdint.h> // uint8_t, uint16_t, uint32_t, uint64_t

	#ifdef __SSE2__
	#include <immintrin.h>
	#endif // __SSE2__

	#include "src/string/memory_utils/elements.h" // __llvm_libc::scalar

	// Fixed-size Vector Operations
	// ----------------------------

	namespace __llvm_libc {
	namespace x86 {

	#ifdef __SSE2__
	template <typename Base> struct Vector : public Base {
	static void copy(char __restrict dst, const char __restrict src) {
	Base::store(dst, Base::load(src));
	}

	static void move(char dst, const char src) {
	Base::store(dst, Base::load(src));
	}

	static bool equals(const char a, const char b) {
	return Base::not_equal_mask(Base::load(a), Base::load(b)) == 0;
	}

	static int three_way_compare(const char a, const char b) {
	const auto mask = Base::not_equal_mask(Base::load(a), Base::load(b));
	if (!mask)
	return 0;
	return char_diff(a, b, mask);
	}

	static void splat_set(char *dst, const unsigned char value) {
	Base::store(dst, Base::get_splatted_value(value));
	}

	static int char_diff(const char a, const char b, uint64_t mask) {
	const size_t diff_index = __builtin_ctzll(mask);
	const int ca = (unsigned char)a[diff_index];
	const int cb = (unsigned char)b[diff_index];
	return ca - cb;
	}
	};

	struct M128 {
	static constexpr size_t SIZE = 16;
	using T = char __attribute__((__vector_size__(SIZE)));
	static uint16_t mask(T value) {
	// NOLINTNEXTLINE(llvmlibc-callee-namespace)
	return static_cast<uint16_t>(
	_mm_movemask_epi8(cpp::bit_cast<__m128i>(value)));
	}
	static uint16_t not_equal_mask(T a, T b) { return mask(a != b); }
	static T load(const char *ptr) {
	// NOLINTNEXTLINE(llvmlibc-callee-namespace)
	return cpp::bit_cast<T>(
	_mm_loadu_si128(reinterpret_cast<__m128i_u const *>(ptr)));
	}
	static void store(char *ptr, T value) {
	// NOLINTNEXTLINE(llvmlibc-callee-namespace)
	return _mm_storeu_si128(reinterpret_cast<__m128i_u *>(ptr),
	cpp::bit_cast<__m128i>(value));
	}
	static T get_splatted_value(const char v) {
	const T splatted = {v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v};
	return splatted;
	}
	};

	using Vector128 = Vector<M128>; // 16 Bytes

	#ifdef __AVX2__
	struct M256 {
	static constexpr size_t SIZE = 32;
	using T = char __attribute__((__vector_size__(SIZE)));
	static uint32_t mask(T value) {
	// NOLINTNEXTLINE(llvmlibc-callee-namespace)
	return _mm256_movemask_epi8(cpp::bit_cast<__m256i>(value));
	}
	static uint32_t not_equal_mask(T a, T b) { return mask(a != b); }
	static T load(const char *ptr) {
	// NOLINTNEXTLINE(llvmlibc-callee-namespace)
	return cpp::bit_cast<T>(
	_mm256_loadu_si256(reinterpret_cast<__m256i const *>(ptr)));
	}
	static void store(char *ptr, T value) {
	// NOLINTNEXTLINE(llvmlibc-callee-namespace)
	return _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr),
	cpp::bit_cast<__m256i>(value));
	}
	static T get_splatted_value(const char v) {
	const T splatted = {v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v,
	v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v};
	return splatted;
	}
	};

	using Vector256 = Vector<M256>; // 32 Bytes

	#if defined(__AVX512F__) and defined(__AVX512BW__)
	struct M512 {
	static constexpr size_t SIZE = 64;
	using T = char __attribute__((__vector_size__(SIZE)));
	static uint64_t not_equal_mask(T a, T b) {
	// NOLINTNEXTLINE(llvmlibc-callee-namespace)
	return _mm512_cmpneq_epi8_mask(cpp::bit_cast<__m512i>(a),
	cpp::bit_cast<__m512i>(b));
	}
	static T load(const char *ptr) {
	// NOLINTNEXTLINE(llvmlibc-callee-namespace)
	return cpp::bit_cast<T>(_mm512_loadu_epi8(ptr));
	}
	static void store(char *ptr, T value) {
	// NOLINTNEXTLINE(llvmlibc-callee-namespace)
	return _mm512_storeu_epi8(ptr, cpp::bit_cast<__m512i>(value));
	}
	static T get_splatted_value(const char v) {
	const T splatted = {v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v,
	v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v,
	v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v,
	v, v, v, v, v, v, v, v, v, v, v, v, v, v, v, v};
	return splatted;
	}
	};
	using Vector512 = Vector<M512>;

	#endif // defined(__AVX512F__) and defined(__AVX512BW__)
	#endif // __AVX2__
	#endif // __SSE2__

	using _1 = __llvm_libc::scalar::_1;
	using _2 = __llvm_libc::scalar::_2;
	using _3 = __llvm_libc::scalar::_3;
	using _4 = __llvm_libc::scalar::_4;
	using _8 = __llvm_libc::scalar::_8;
	#if defined(__AVX512F__) && defined(__AVX512BW__)
	using _16 = __llvm_libc::x86::Vector128;
	using _32 = __llvm_libc::x86::Vector256;
	using _64 = __llvm_libc::x86::Vector512;
	using _128 = __llvm_libc::Repeated<_64, 2>;
	#elif defined(__AVX2__)
	using _16 = __llvm_libc::x86::Vector128;
	using _32 = __llvm_libc::x86::Vector256;
	using _64 = __llvm_libc::Repeated<_32, 2>;
	using _128 = __llvm_libc::Repeated<_32, 4>;
	#elif defined(__SSE2__)
	using _16 = __llvm_libc::x86::Vector128;
	using _32 = __llvm_libc::Repeated<_16, 2>;
	using _64 = __llvm_libc::Repeated<_16, 4>;
	using _128 = __llvm_libc::Repeated<_16, 8>;
	#else
	using _16 = __llvm_libc::Repeated<_8, 2>;
	using _32 = __llvm_libc::Repeated<_8, 4>;
	using _64 = __llvm_libc::Repeated<_8, 8>;
	using _128 = __llvm_libc::Repeated<_8, 16>;
	#endif

	struct Accelerator {
	static void copy(char dst, const char src, size_t count) {
	asm volatile("rep movsb" : "+D"(dst), "+S"(src), "+c"(count) : : "memory");
	}
	};

	} // namespace x86
	} // namespace __llvm_libc

	#endif // defined(LLVM_LIBC_ARCH_X86)

	#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ELEMENTS_X86_H

libc/test/src/string/memory_utils/CMakeLists.txt

	add_libc_unittest(			add_libc_unittest(
	utils_test			utils_test
	SUITE			SUITE
	libc_string_unittests			libc_string_unittests
	SRCS			SRCS
	elements_test.cpp
	memory_access_test.cpp
	op_tests.cpp			op_tests.cpp
	utils_test.cpp			utils_test.cpp
	COMPILE_OPTIONS			COMPILE_OPTIONS
	${LIBC_COMPILE_OPTIONS_NATIVE}			${LIBC_COMPILE_OPTIONS_NATIVE}
	-ffreestanding			-ffreestanding
	DEPENDS			DEPENDS
	libc.src.string.memory_utils.memory_utils			libc.src.string.memory_utils.memory_utils
	libc.src.__support.CPP.array			libc.src.__support.CPP.array
	libc.src.__support.CPP.span			libc.src.__support.CPP.span
	libc.src.__support.CPP.cstddef			libc.src.__support.CPP.cstddef
	)			)

libc/test/src/string/memory_utils/elements_test.cpp

This file was deleted.

	//===-- Unittests for memory_utils ----------------------------------------===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//

	#include "src/__support/CPP/array.h"
	#include "src/__support/CPP/span.h"
	#include "src/string/memory_utils/elements.h"
	#include "utils/UnitTest/Test.h"

	namespace __llvm_libc {

	// Registering Types
	using FixedSizeTypes = testing::TypeList<
	#if defined(__SSE2__)
	x86::Vector128, //
	#endif // __SSE2__
	#if defined(__AVX2__)
	x86::Vector256, //
	#endif // __AVX2__
	#if defined(__AVX512F__) and defined(__AVX512BW__)
	x86::Vector512, //
	#endif // defined(__AVX512F__) and defined(__AVX512BW__)
	scalar::UINT8, //
	scalar::UINT16, //
	scalar::UINT32, //
	scalar::UINT64, //
	Repeated<scalar::UINT64, 2>, //
	Repeated<scalar::UINT64, 4>, //
	Repeated<scalar::UINT64, 8>, //
	Repeated<scalar::UINT64, 16>, //
	Repeated<scalar::UINT64, 32>, //
	Chained<scalar::UINT16, scalar::UINT8>, //
	Chained<scalar::UINT32, scalar::UINT16, scalar::UINT8>, //
	builtin::_1, //
	builtin::_2, //
	builtin::_3, //
	builtin::_4, //
	builtin::_8 //
	>;

	static char GetRandomChar() {
	static constexpr const uint64_t a = 1103515245;
	static constexpr const uint64_t c = 12345;
	static constexpr const uint64_t m = 1ULL << 31;
	static uint64_t seed = 123456789;
	seed = (a * seed + c) % m;
	return seed;
	}

	static void Randomize(cpp::span<char> buffer) {
	for (auto &current : buffer)
	current = GetRandomChar();
	}

	template <typename Element> using Buffer = cpp::array<char, Element::SIZE>;

	template <typename Element> Buffer<Element> GetRandomBuffer() {
	Buffer<Element> buffer;
	Randomize(buffer);
	return buffer;
	}

	TYPED_TEST(LlvmLibcMemoryElements, copy, FixedSizeTypes) {
	Buffer<ParamType> Dst;
	const auto buffer = GetRandomBuffer<ParamType>();
	copy<ParamType>(Dst.data(), buffer.data());
	for (size_t i = 0; i < ParamType::SIZE; ++i)
	EXPECT_EQ(Dst[i], buffer[i]);
	}

	template <typename T> T copy(const T &Input) {
	T Output;
	for (size_t I = 0; I < Input.size(); ++I)
	Output[I] = Input[I];
	return Output;
	}

	TYPED_TEST(LlvmLibcMemoryElements, Move, FixedSizeTypes) {
	constexpr size_t SIZE = ParamType::SIZE;
	using LargeBuffer = cpp::array<char, SIZE * 2>;
	LargeBuffer GroundTruth;
	Randomize(GroundTruth);
	// Forward, we move the SIZE first bytes from offset 0 to SIZE.
	for (size_t Offset = 0; Offset < SIZE; ++Offset) {
	LargeBuffer Buffer = copy(GroundTruth);
	move<ParamType>(&Buffer[Offset], &Buffer[0]);
	for (size_t I = 0; I < SIZE; ++I)
	EXPECT_EQ(Buffer[I + Offset], GroundTruth[I]);
	}
	// Backward, we move the SIZE last bytes from offset 0 to SIZE.
	for (size_t Offset = 0; Offset < SIZE; ++Offset) {
	LargeBuffer Buffer = copy(GroundTruth);
	move<ParamType>(&Buffer[Offset], &Buffer[SIZE]);
	for (size_t I = 0; I < SIZE; ++I)
	EXPECT_EQ(Buffer[I + Offset], GroundTruth[SIZE + I]);
	}
	}

	TYPED_TEST(LlvmLibcMemoryElements, Equals, FixedSizeTypes) {
	const auto buffer = GetRandomBuffer<ParamType>();
	EXPECT_TRUE(equals<ParamType>(buffer.data(), buffer.data()));
	}

	TYPED_TEST(LlvmLibcMemoryElements, three_way_compare, FixedSizeTypes) {
	Buffer<ParamType> initial;
	for (auto &c : initial)
	c = 5;

	// Testing equality
	EXPECT_EQ(three_way_compare<ParamType>(initial.data(), initial.data()), 0);

	// Testing all mismatching positions
	for (size_t i = 0; i < ParamType::SIZE; ++i) {
	auto copy = initial;
	++copy[i]; // copy is now lexicographycally greated than initial
	const auto *less = initial.data();
	const auto *greater = copy.data();
	EXPECT_LT(three_way_compare<ParamType>(less, greater), 0);
	EXPECT_GT(three_way_compare<ParamType>(greater, less), 0);
	}
	}

	TYPED_TEST(LlvmLibcMemoryElements, Splat, FixedSizeTypes) {
	Buffer<ParamType> Dst;
	const cpp::array<char, 3> values = {char(0x00), char(0x7F), char(0xFF)};
	for (char value : values) {
	splat_set<ParamType>(Dst.data(), value);
	for (size_t i = 0; i < ParamType::SIZE; ++i)
	EXPECT_EQ(Dst[i], value);
	}
	}

	} // namespace __llvm_libc

libc/test/src/string/memory_utils/memory_access_test.cpp

This file was deleted.

	//===-- Unittests for memory_utils ----------------------------------------===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//

	#define LLVM_LIBC_UNITTEST_OBSERVE 1

	#include "src/__support/CPP/array.h"
	#include "src/string/memory_utils/elements.h"
	#include "utils/UnitTest/Test.h"

	#include <stdio.h>
	#include <string.h>

	namespace __llvm_libc {

	static constexpr const size_t kMaxBuffer = 32;

	struct BufferAccess : cpp::array<char, kMaxBuffer + 1> {
	BufferAccess() { Reset(); }
	void Reset() {
	for (auto &value : *this)
	value = '0';
	this->operator[](kMaxBuffer) = '\0';
	}
	void Touch(ptrdiff_t offset, size_t size) {
	if (offset < 0)
	return;
	for (size_t i = 0; i < size; ++i)
	++(*this)[offset + i];
	}
	operator const char *() const { return this->data(); }
	};

	struct Buffer {
	ptrdiff_t Offset(const char *ptr) const {
	const bool contained = ptr >= data.begin() && ptr < data.end();
	return contained ? ptr - data.begin() : -1;
	}
	void Reset() {
	reads.Reset();
	writes.Reset();
	}
	cpp::array<char, kMaxBuffer> data;
	BufferAccess __attribute__((aligned(64))) reads;
	BufferAccess __attribute__((aligned(64))) writes;
	};

	struct MemoryAccessObserver {
	void ObserveRead(const char *ptr, size_t size) {
	Buffer1.reads.Touch(Buffer1.Offset(ptr), size);
	Buffer2.reads.Touch(Buffer2.Offset(ptr), size);
	}

	void ObserveWrite(const char *ptr, size_t size) {
	Buffer1.writes.Touch(Buffer1.Offset(ptr), size);
	Buffer2.writes.Touch(Buffer2.Offset(ptr), size);
	}

	void Reset() {
	Buffer1.Reset();
	Buffer2.Reset();
	}

	Buffer Buffer1;
	Buffer Buffer2;
	};

	static MemoryAccessObserver Observer;

	template <size_t Size> struct TestingElement {
	static constexpr size_t SIZE = Size;

	static void copy(char __restrict dst, const char __restrict src) {
	Observer.ObserveRead(src, SIZE);
	Observer.ObserveWrite(dst, SIZE);
	}

	static bool equals(const char lhs, const char rhs) {
	Observer.ObserveRead(lhs, SIZE);
	Observer.ObserveRead(rhs, SIZE);
	return true;
	}

	static int three_way_compare(const char lhs, const char rhs) {
	Observer.ObserveRead(lhs, SIZE);
	Observer.ObserveRead(rhs, SIZE);
	return 0;
	}

	static void splat_set(char *dst, const unsigned char value) {
	Observer.ObserveWrite(dst, SIZE);
	}
	};

	using Types = testing::TypeList<
	TestingElement<1>, // 1 Byte
	TestingElement<2>, // 2 Bytes
	TestingElement<4>, // 4 Bytes
	Repeated<TestingElement<2>, 3>, // 6 Bytes
	Chained<TestingElement<4>, TestingElement<2>, TestingElement<1>> // 7 Bytes
	>;

	struct LlvmLibcTestAccessBase : public testing::Test {

	template <typename HigherOrder, size_t Size, size_t Offset = 0>
	void checkOperations(const BufferAccess &expected) {
	static const BufferAccess untouched;

	Observer.Reset();
	HigherOrder::copy(dst_ptr() + Offset, src_ptr() + Offset, Size);
	ASSERT_STREQ(src().writes, untouched);
	ASSERT_STREQ(dst().reads, untouched);
	ASSERT_STREQ(src().reads, expected);
	ASSERT_STREQ(dst().writes, expected);
	Observer.Reset();
	HigherOrder::equals(lhs_ptr() + Offset, rhs_ptr() + Offset, Size);
	ASSERT_STREQ(lhs().writes, untouched);
	ASSERT_STREQ(rhs().writes, untouched);
	ASSERT_STREQ(lhs().reads, expected);
	ASSERT_STREQ(rhs().reads, expected);
	Observer.Reset();
	HigherOrder::three_way_compare(lhs_ptr() + Offset, rhs_ptr() + Offset,
	Size);
	ASSERT_STREQ(lhs().writes, untouched);
	ASSERT_STREQ(rhs().writes, untouched);
	ASSERT_STREQ(lhs().reads, expected);
	ASSERT_STREQ(rhs().reads, expected);
	Observer.Reset();
	HigherOrder::splat_set(dst_ptr() + Offset, 5, Size);
	ASSERT_STREQ(src().reads, untouched);
	ASSERT_STREQ(src().writes, untouched);
	ASSERT_STREQ(dst().reads, untouched);
	ASSERT_STREQ(dst().writes, expected);
	}

	void checkMaxAccess(const BufferAccess &expected, int max) {
	for (size_t i = 0; i < kMaxBuffer; ++i) {
	int value = (int)expected[i] - '0';
	ASSERT_GE(value, 0);
	ASSERT_LE(value, max);
	}
	}

	private:
	const Buffer &lhs() const { return Observer.Buffer1; }
	const Buffer &rhs() const { return Observer.Buffer2; }
	const Buffer &src() const { return Observer.Buffer2; }
	const Buffer &dst() const { return Observer.Buffer1; }
	Buffer &dst() { return Observer.Buffer1; }

	char *dst_ptr() { return dst().data.begin(); }
	const char *src_ptr() { return src().data.begin(); }
	const char *lhs_ptr() { return lhs().data.begin(); }
	const char *rhs_ptr() { return rhs().data.begin(); }
	};

	template <typename ParamType>
	struct LlvmLibcTestAccessTail : public LlvmLibcTestAccessBase {

	void TearDown() override {
	static constexpr size_t Size = 10;

	BufferAccess expected;
	expected.Touch(Size - ParamType::SIZE, ParamType::SIZE);

	checkMaxAccess(expected, 1);
	checkOperations<Tail<ParamType>, Size>(expected);
	}
	};
	TYPED_TEST_F(LlvmLibcTestAccessTail, Operations, Types) {}

	template <typename ParamType>
	struct LlvmLibcTestAccessHeadTail : public LlvmLibcTestAccessBase {
	void TearDown() override {
	static constexpr size_t Size = 10;

	BufferAccess expected;
	expected.Touch(0, ParamType::SIZE);
	expected.Touch(Size - ParamType::SIZE, ParamType::SIZE);

	checkMaxAccess(expected, 2);
	checkOperations<HeadTail<ParamType>, Size>(expected);
	}
	};
	TYPED_TEST_F(LlvmLibcTestAccessHeadTail, Operations, Types) {}

	template <typename ParamType>
	struct LlvmLibcTestAccessLoop : public LlvmLibcTestAccessBase {
	void TearDown() override {
	static constexpr size_t Size = 20;

	BufferAccess expected;
	for (size_t i = 0; i < Size - ParamType::SIZE; i += ParamType::SIZE)
	expected.Touch(i, ParamType::SIZE);
	expected.Touch(Size - ParamType::SIZE, ParamType::SIZE);

	checkMaxAccess(expected, 2);
	checkOperations<Loop<ParamType>, Size>(expected);
	}
	};
	TYPED_TEST_F(LlvmLibcTestAccessLoop, Operations, Types) {}

	template <typename ParamType>
	struct LlvmLibcTestAccessAlignedAccess : public LlvmLibcTestAccessBase {
	void TearDown() override {
	static constexpr size_t Size = 10;
	static constexpr size_t Offset = 2;
	using AlignmentT = TestingElement<4>;

	BufferAccess expected;
	expected.Touch(Offset, AlignmentT::SIZE);
	expected.Touch(AlignmentT::SIZE, ParamType::SIZE);
	expected.Touch(Offset + Size - ParamType::SIZE, ParamType::SIZE);

	checkMaxAccess(expected, 3);
	checkOperations<Align<AlignmentT, Arg::P1>::Then<HeadTail<ParamType>>, Size,
	Offset>(expected);
	checkOperations<Align<AlignmentT, Arg::P2>::Then<HeadTail<ParamType>>, Size,
	Offset>(expected);
	}
	};
	TYPED_TEST_F(LlvmLibcTestAccessAlignedAccess, Operations, Types) {}

	} // namespace __llvm_libc

utils/bazel/llvm-project-overlay/libc/BUILD.bazel

Show First 20 Lines • Show All 1,002 Lines • ▼ Show 20 Lines	no_sanitize_features = [
"-msan",		"-msan",
"-tsan",		"-tsan",
"-ubsan",		"-ubsan",
]		]

cc_library(		cc_library(
name = "string_memory_utils",		name = "string_memory_utils",
hdrs = [		hdrs = [
"src/string/memory_utils/elements_aarch64.h",
"src/string/memory_utils/elements_x86.h",
"src/string/memory_utils/elements.h",
"src/string/memory_utils/op_aarch64.h",		"src/string/memory_utils/op_aarch64.h",
"src/string/memory_utils/op_builtin.h",		"src/string/memory_utils/op_builtin.h",
"src/string/memory_utils/op_generic.h",		"src/string/memory_utils/op_generic.h",
"src/string/memory_utils/op_x86.h",		"src/string/memory_utils/op_x86.h",
"src/string/memory_utils/utils.h",		"src/string/memory_utils/utils.h",
],		],
textual_hdrs = [		textual_hdrs = [
"src/string/memory_utils/bcmp_implementations.h",		"src/string/memory_utils/bcmp_implementations.h",
▲ Show 20 Lines • Show All 434 Lines • Show Last 20 Lines

This is an archive of the discontinued LLVM Phabricator instance.

[libc] remove mem functions dead codeClosedPublic

Details

Diff Detail

Event Timeline

Revision Contents

Diff 475741

libc/src/string/memory_utils/CMakeLists.txt

libc/src/string/memory_utils/elements.h

libc/src/string/memory_utils/elements_aarch64.h

libc/src/string/memory_utils/elements_x86.h

libc/test/src/string/memory_utils/CMakeLists.txt

libc/test/src/string/memory_utils/elements_test.cpp

libc/test/src/string/memory_utils/memory_access_test.cpp

utils/bazel/llvm-project-overlay/libc/BUILD.bazel

[libc] remove mem functions dead code
ClosedPublic