diff --git a/libc/src/stdio/printf_core/parser.h b/libc/src/stdio/printf_core/parser.h --- a/libc/src/stdio/printf_core/parser.h +++ b/libc/src/stdio/printf_core/parser.h @@ -59,8 +59,7 @@ #ifndef LLVM_LIBC_PRINTF_DISABLE_INDEX_MODE Parser(const char *__restrict new_str, internal::ArgList &args) : str(new_str), args_cur(args), args_start(args) { - inline_memset(reinterpret_cast(desc_arr), 0, - DESC_ARR_LEN * sizeof(TypeDesc)); + inline_memset(desc_arr, ubyte::ZERO, sizeof(desc_arr)); } #else Parser(const char *__restrict new_str, internal::ArgList &args) diff --git a/libc/src/stdio/printf_core/writer.cpp b/libc/src/stdio/printf_core/writer.cpp --- a/libc/src/stdio/printf_core/writer.cpp +++ b/libc/src/stdio/printf_core/writer.cpp @@ -22,7 +22,7 @@ constexpr size_t BUFF_SIZE = 8; char buff[BUFF_SIZE]; int result; - inline_memset(buff, new_char, BUFF_SIZE); + inline_memset(buff, static_cast(new_char), BUFF_SIZE); while (length > BUFF_SIZE) { result = write(buff, BUFF_SIZE); if (result < 0) diff --git a/libc/src/string/bcmp.cpp b/libc/src/string/bcmp.cpp --- a/libc/src/string/bcmp.cpp +++ b/libc/src/string/bcmp.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "src/string/bcmp.h" + #include "src/__support/common.h" #include "src/string/memory_utils/bcmp_implementations.h" @@ -14,8 +15,7 @@ LLVM_LIBC_FUNCTION(int, bcmp, (const void *lhs, const void *rhs, size_t count)) { - return inline_bcmp(static_cast(lhs), - static_cast(rhs), count); + return inline_bcmp(lhs, rhs, count); } } // namespace __llvm_libc diff --git a/libc/src/string/bzero.cpp b/libc/src/string/bzero.cpp --- a/libc/src/string/bzero.cpp +++ b/libc/src/string/bzero.cpp @@ -7,13 +7,14 @@ //===----------------------------------------------------------------------===// #include "src/string/bzero.h" + #include "src/__support/common.h" #include "src/string/memory_utils/memset_implementations.h" namespace __llvm_libc { LLVM_LIBC_FUNCTION(void, bzero, (void *ptr, size_t count)) { - inline_memset(reinterpret_cast(ptr), 0, count); + inline_memset(ptr, ubyte::ZERO, count); } } // namespace __llvm_libc diff --git a/libc/src/string/memcmp.cpp b/libc/src/string/memcmp.cpp --- a/libc/src/string/memcmp.cpp +++ b/libc/src/string/memcmp.cpp @@ -7,6 +7,8 @@ //===----------------------------------------------------------------------===// #include "src/string/memcmp.h" + +#include "src/__support/common.h" #include "src/string/memory_utils/memcmp_implementations.h" #include // size_t @@ -15,8 +17,7 @@ LLVM_LIBC_FUNCTION(int, memcmp, (const void *lhs, const void *rhs, size_t count)) { - return inline_memcmp(static_cast(lhs), - static_cast(rhs), count); + return inline_memcmp(lhs, rhs, count); } } // namespace __llvm_libc diff --git a/libc/src/string/memcpy.cpp b/libc/src/string/memcpy.cpp --- a/libc/src/string/memcpy.cpp +++ b/libc/src/string/memcpy.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "src/string/memcpy.h" + #include "src/__support/common.h" #include "src/string/memory_utils/memcpy_implementations.h" @@ -15,8 +16,7 @@ LLVM_LIBC_FUNCTION(void *, memcpy, (void *__restrict dst, const void *__restrict src, size_t size)) { - inline_memcpy(reinterpret_cast(dst), - reinterpret_cast(src), size); + inline_memcpy(dst, src, size); return dst; } diff --git a/libc/src/string/memmove.cpp b/libc/src/string/memmove.cpp --- a/libc/src/string/memmove.cpp +++ b/libc/src/string/memmove.cpp @@ -9,42 +9,42 @@ #include "src/string/memmove.h" #include "src/__support/common.h" -#include "src/__support/integer_operations.h" -#include "src/string/memory_utils/elements.h" +#include "src/string/memory_utils/algorithm.h" +#include "src/string/memory_utils/backends.h" #include // size_t, ptrdiff_t namespace __llvm_libc { -static inline void inline_memmove(char *dst, const char *src, size_t count) { - using namespace __llvm_libc::scalar; +static inline void inline_memmove(DstAddr<1> dst, SrcAddr<1> src, + size_t count) { + using namespace scalar; if (count == 0) return; if (count == 1) - return move<_1>(dst, src); + return _1::move(dst, src); if (count <= 4) - return move>(dst, src, count); + return HeadTail<_2>::move(dst, src, count); if (count <= 8) - return move>(dst, src, count); + return HeadTail<_4>::move(dst, src, count); if (count <= 16) - return move>(dst, src, count); + return HeadTail<_8>::move(dst, src, count); if (count <= 32) - return move>(dst, src, count); + return HeadTail<_16>::move(dst, src, count); if (count <= 64) - return move>(dst, src, count); + return HeadTail<_32>::move(dst, src, count); if (count <= 128) - return move>(dst, src, count); + return HeadTail<_64>::move(dst, src, count); using AlignedMoveLoop = Align<_16, Arg::Src>::Then>; - if (dst < src) - return move(dst, src, count); - else if (dst > src) - return move_backward(dst, src, count); + if (dst.ptr() < src.ptr()) + return AlignedMoveLoop::move(dst, src, count); + else if (dst.ptr() > src.ptr()) + return AlignedMoveLoop::move_backward(dst, src, count); } LLVM_LIBC_FUNCTION(void *, memmove, (void *dst, const void *src, size_t count)) { - inline_memmove(reinterpret_cast(dst), - reinterpret_cast(src), count); + inline_memmove(dst, src, count); return dst; } diff --git a/libc/src/string/memory_utils/bcmp_implementations.h b/libc/src/string/memory_utils/bcmp_implementations.h --- a/libc/src/string/memory_utils/bcmp_implementations.h +++ b/libc/src/string/memory_utils/bcmp_implementations.h @@ -10,50 +10,45 @@ #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_BCMP_IMPLEMENTATIONS_H #include "src/__support/architectures.h" -#include "src/__support/common.h" -#include "src/string/memory_utils/elements.h" +#include "src/string/memory_utils/algorithm.h" +#include "src/string/memory_utils/backends.h" #include // size_t namespace __llvm_libc { -// Fixed-size difference between 'lhs' and 'rhs'. -template bool differs(const char *lhs, const char *rhs) { - return !Element::equals(lhs, rhs); -} -// Runtime-size difference between 'lhs' and 'rhs'. -template -bool differs(const char *lhs, const char *rhs, size_t size) { - return !Element::equals(lhs, rhs, size); -} - -static inline int inline_bcmp(const char *lhs, const char *rhs, size_t count) { +static inline uint64_t inline_bcmp_uint64_t(SrcAddr<1> lhs, SrcAddr<1> rhs, + size_t count) { #if defined(LLVM_LIBC_ARCH_X86) - using namespace ::__llvm_libc::x86; + using namespace x86; #elif defined(LLVM_LIBC_ARCH_AARCH64) - using namespace ::__llvm_libc::aarch64; + using namespace aarch64; #else - using namespace ::__llvm_libc::scalar; + using namespace scalar; #endif if (count == 0) return 0; if (count == 1) - return differs<_1>(lhs, rhs); + return _1::isDifferent(lhs, rhs); if (count == 2) - return differs<_2>(lhs, rhs); + return _2::isDifferent(lhs, rhs); if (count == 3) - return differs<_3>(lhs, rhs); + return _3::isDifferent(lhs, rhs); if (count <= 8) - return differs>(lhs, rhs, count); + return HeadTail<_4>::isDifferent(lhs, rhs, count); if (count <= 16) - return differs>(lhs, rhs, count); + return HeadTail<_8>::isDifferent(lhs, rhs, count); if (count <= 32) - return differs>(lhs, rhs, count); + return HeadTail<_16>::isDifferent(lhs, rhs, count); if (count <= 64) - return differs>(lhs, rhs, count); + return HeadTail<_32>::isDifferent(lhs, rhs, count); if (count <= 128) - return differs>(lhs, rhs, count); - return differs::Then>>(lhs, rhs, count); + return HeadTail<_64>::isDifferent(lhs, rhs, count); + return Align<_32>::Then>::isDifferent(lhs, rhs, count); +} + +static inline int inline_bcmp(SrcAddr<1> lhs, SrcAddr<1> rhs, size_t count) { + return !!inline_bcmp_uint64_t(lhs, rhs, count); } } // namespace __llvm_libc diff --git a/libc/src/string/memory_utils/memcmp_implementations.h b/libc/src/string/memory_utils/memcmp_implementations.h --- a/libc/src/string/memory_utils/memcmp_implementations.h +++ b/libc/src/string/memory_utils/memcmp_implementations.h @@ -10,93 +10,91 @@ #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP_IMPLEMENTATIONS_H #include "src/__support/architectures.h" -#include "src/__support/common.h" -#include "src/string/memory_utils/elements.h" +#include "src/string/memory_utils/algorithm.h" +#include "src/string/memory_utils/backends.h" #include // size_t namespace __llvm_libc { -static inline int inline_memcmp(const char *lhs, const char *rhs, - size_t count) { +static inline int inline_memcmp(SrcAddr<1> lhs, SrcAddr<1> rhs, size_t count) { #if defined(LLVM_LIBC_ARCH_X86) ///////////////////////////////////////////////////////////////////////////// // LLVM_LIBC_ARCH_X86 ///////////////////////////////////////////////////////////////////////////// - using namespace __llvm_libc::x86; + using namespace x86; if (count == 0) return 0; if (count == 1) - return three_way_compare<_1>(lhs, rhs); + return _1::threeWayCmp(lhs, rhs); if (count == 2) - return three_way_compare<_2>(lhs, rhs); + return _2::threeWayCmp(lhs, rhs); if (count == 3) - return three_way_compare<_3>(lhs, rhs); + return _3::threeWayCmp(lhs, rhs); if (count <= 8) - return three_way_compare>(lhs, rhs, count); + return HeadTail<_4>::threeWayCmp(lhs, rhs, count); if (count <= 16) - return three_way_compare>(lhs, rhs, count); + return HeadTail<_8>::threeWayCmp(lhs, rhs, count); if (count <= 32) - return three_way_compare>(lhs, rhs, count); + return HeadTail<_16>::threeWayCmp(lhs, rhs, count); if (count <= 64) - return three_way_compare>(lhs, rhs, count); + return HeadTail<_32>::threeWayCmp(lhs, rhs, count); if (count <= 128) - return three_way_compare>(lhs, rhs, count); - return three_way_compare::Then>>(lhs, rhs, count); + return HeadTail<_64>::threeWayCmp(lhs, rhs, count); + return Align<_32>::Then>::threeWayCmp(lhs, rhs, count); #elif defined(LLVM_LIBC_ARCH_AARCH64) ///////////////////////////////////////////////////////////////////////////// // LLVM_LIBC_ARCH_AARCH64 ///////////////////////////////////////////////////////////////////////////// - using namespace ::__llvm_libc::aarch64; + using namespace aarch64; if (count == 0) // [0, 0] return 0; if (count == 1) // [1, 1] - return three_way_compare<_1>(lhs, rhs); + return _1::threeWayCmp(lhs, rhs); if (count == 2) // [2, 2] - return three_way_compare<_2>(lhs, rhs); + return _2::threeWayCmp(lhs, rhs); if (count == 3) // [3, 3] - return three_way_compare<_3>(lhs, rhs); + return _3::threeWayCmp(lhs, rhs); if (count < 8) // [4, 7] - return three_way_compare>(lhs, rhs, count); + return HeadTail<_4>::threeWayCmp(lhs, rhs, count); if (count < 16) // [8, 15] - return three_way_compare>(lhs, rhs, count); + return HeadTail<_8>::threeWayCmp(lhs, rhs, count); if (unlikely(count >= 128)) // [128, ∞] - return three_way_compare::Then>>(lhs, rhs, count); - if (!equals<_16>(lhs, rhs)) // [16, 16] - return three_way_compare<_16>(lhs, rhs); + return Align<_16>::Then>::threeWayCmp(lhs, rhs, count); + if (_16::isDifferent(lhs, rhs)) // [16, 16] + return _16::threeWayCmp(lhs, rhs); if (count < 32) // [17, 31] - return three_way_compare>(lhs, rhs, count); - if (!equals::Then<_16>>(lhs, rhs)) // [32, 32] - return three_way_compare::Then<_16>>(lhs, rhs); + return Tail<_16>::threeWayCmp(lhs, rhs, count); + if (Skip<16>::Then<_16>::isDifferent(lhs, rhs)) // [32, 32] + return Skip<16>::Then<_16>::threeWayCmp(lhs, rhs); if (count < 64) // [33, 63] - return three_way_compare>(lhs, rhs, count); + return Tail<_32>::threeWayCmp(lhs, rhs, count); // [64, 127] - return three_way_compare::Then>>(lhs, rhs, count); + return Skip<32>::Then>::threeWayCmp(lhs, rhs, count); #else ///////////////////////////////////////////////////////////////////////////// // Default ///////////////////////////////////////////////////////////////////////////// - using namespace ::__llvm_libc::scalar; - + using namespace scalar; if (count == 0) return 0; if (count == 1) - return three_way_compare<_1>(lhs, rhs); + return _1::threeWayCmp(lhs, rhs); if (count == 2) - return three_way_compare<_2>(lhs, rhs); + return _2::threeWayCmp(lhs, rhs); if (count == 3) - return three_way_compare<_3>(lhs, rhs); + return _3::threeWayCmp(lhs, rhs); if (count <= 8) - return three_way_compare>(lhs, rhs, count); + return HeadTail<_4>::threeWayCmp(lhs, rhs, count); if (count <= 16) - return three_way_compare>(lhs, rhs, count); + return HeadTail<_8>::threeWayCmp(lhs, rhs, count); if (count <= 32) - return three_way_compare>(lhs, rhs, count); + return HeadTail<_16>::threeWayCmp(lhs, rhs, count); if (count <= 64) - return three_way_compare>(lhs, rhs, count); + return HeadTail<_32>::threeWayCmp(lhs, rhs, count); if (count <= 128) - return three_way_compare>(lhs, rhs, count); - return three_way_compare::Then>>(lhs, rhs, count); + return HeadTail<_64>::threeWayCmp(lhs, rhs, count); + return Align<_32>::Then>::threeWayCmp(lhs, rhs, count); #endif } diff --git a/libc/src/string/memory_utils/memcpy_implementations.h b/libc/src/string/memory_utils/memcpy_implementations.h --- a/libc/src/string/memory_utils/memcpy_implementations.h +++ b/libc/src/string/memory_utils/memcpy_implementations.h @@ -9,10 +9,11 @@ #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H +#include "src/__support/CPP/TypeTraits.h" // cpp::ConditionalType #include "src/__support/architectures.h" -#include "src/__support/common.h" -#include "src/string/memory_utils/elements.h" -#include "src/string/memory_utils/utils.h" +#include "src/__support/common.h" // LLVM_LIBC_IS_DEFINED +#include "src/string/memory_utils/algorithm.h" +#include "src/string/memory_utils/backends.h" #include // size_t @@ -37,14 +38,11 @@ namespace __llvm_libc { -static inline void inline_memcpy(char *__restrict dst, - const char *__restrict src, size_t count) { - using namespace __llvm_libc::builtin; +static inline void inline_memcpy(DstAddr<1> dst, SrcAddr<1> src, size_t count) { #if defined(LLVM_LIBC_ARCH_X86) ///////////////////////////////////////////////////////////////////////////// // LLVM_LIBC_ARCH_X86 ///////////////////////////////////////////////////////////////////////////// - // Whether to use only rep;movsb. constexpr bool USE_ONLY_REP_MOVSB = LLVM_LIBC_IS_DEFINED(LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB); @@ -59,94 +57,91 @@ -1; #endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE + using namespace x86; + // Whether target supports AVX instructions. constexpr bool HAS_AVX = LLVM_LIBC_IS_DEFINED(__AVX__); - -#if defined(__AVX__) - using LoopBlockSize = _64; -#else - using LoopBlockSize = _32; -#endif - if (USE_ONLY_REP_MOVSB) - return copy(dst, src, count); - + return repmovsb(as(dst), as(src), count); if (count == 0) return; if (count == 1) - return copy<_1>(dst, src); + return _1::copy(dst, src); if (count == 2) - return copy<_2>(dst, src); + return _2::copy(dst, src); if (count == 3) - return copy<_3>(dst, src); + return _3::copy(dst, src); if (count == 4) - return copy<_4>(dst, src); + return _4::copy(dst, src); if (count < 8) - return copy>(dst, src, count); + return HeadTail<_4>::copy(dst, src, count); if (count < 16) - return copy>(dst, src, count); + return HeadTail<_8>::copy(dst, src, count); if (count < 32) - return copy>(dst, src, count); + return HeadTail<_16>::copy(dst, src, count); if (count < 64) - return copy>(dst, src, count); + return HeadTail<_32>::copy(dst, src, count); if (count < 128) - return copy>(dst, src, count); + return HeadTail<_64>::copy(dst, src, count); if (HAS_AVX && count < 256) - return copy>(dst, src, count); + return HeadTail<_128>::copy(dst, src, count); + using LoopBlock = cpp::ConditionalType; if (count <= REP_MOVS_B_SIZE) - return copy::Then>>(dst, src, - count); - return copy(dst, src, count); + return Align<_32, Arg::Dst>::Then>::copy(dst, src, count); + return repmovsb(as(dst), as(src), count); + #elif defined(LLVM_LIBC_ARCH_AARCH64) ///////////////////////////////////////////////////////////////////////////// // LLVM_LIBC_ARCH_AARCH64 ///////////////////////////////////////////////////////////////////////////// + using namespace scalar; if (count == 0) return; if (count == 1) - return copy<_1>(dst, src); + return _1::copy(dst, src); if (count == 2) - return copy<_2>(dst, src); + return _2::copy(dst, src); if (count == 3) - return copy<_3>(dst, src); + return _3::copy(dst, src); if (count == 4) - return copy<_4>(dst, src); + return _4::copy(dst, src); if (count < 8) - return copy>(dst, src, count); + return HeadTail<_4>::copy(dst, src, count); if (count < 16) - return copy>(dst, src, count); + return HeadTail<_8>::copy(dst, src, count); if (count < 32) - return copy>(dst, src, count); + return HeadTail<_16>::copy(dst, src, count); if (count < 64) - return copy>(dst, src, count); + return HeadTail<_32>::copy(dst, src, count); if (count < 128) - return copy>(dst, src, count); - return copy::Then>>(dst, src, count); + return HeadTail<_64>::copy(dst, src, count); + return Align<_16, Arg::Src>::Then>::copy(dst, src, count); #else ///////////////////////////////////////////////////////////////////////////// // Default ///////////////////////////////////////////////////////////////////////////// + using namespace scalar; if (count == 0) return; if (count == 1) - return copy<_1>(dst, src); + return _1::copy(dst, src); if (count == 2) - return copy<_2>(dst, src); + return _2::copy(dst, src); if (count == 3) - return copy<_3>(dst, src); + return _3::copy(dst, src); if (count == 4) - return copy<_4>(dst, src); + return _4::copy(dst, src); if (count < 8) - return copy>(dst, src, count); + return HeadTail<_4>::copy(dst, src, count); if (count < 16) - return copy>(dst, src, count); + return HeadTail<_8>::copy(dst, src, count); if (count < 32) - return copy>(dst, src, count); + return HeadTail<_16>::copy(dst, src, count); if (count < 64) - return copy>(dst, src, count); + return HeadTail<_32>::copy(dst, src, count); if (count < 128) - return copy>(dst, src, count); - return copy::Then>>(dst, src, count); + return HeadTail<_64>::copy(dst, src, count); + return Align<_32, Arg::Src>::Then>::copy(dst, src, count); #endif } diff --git a/libc/src/string/memory_utils/memset_implementations.h b/libc/src/string/memory_utils/memset_implementations.h --- a/libc/src/string/memory_utils/memset_implementations.h +++ b/libc/src/string/memory_utils/memset_implementations.h @@ -10,8 +10,8 @@ #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H #include "src/__support/architectures.h" -#include "src/string/memory_utils/elements.h" -#include "src/string/memory_utils/utils.h" +#include "src/string/memory_utils/algorithm.h" +#include "src/string/memory_utils/backends.h" #include // size_t @@ -48,85 +48,86 @@ // advance. SetAlignedBlocks<64> may waste up to 63 Bytes, SetAlignedBlocks<32> // may waste up to 31 Bytes. Benchmarks showed that SetAlignedBlocks<64> was not // superior for sizes that mattered. -inline static void inline_memset(char *dst, unsigned char value, size_t count) { +inline static void inline_memset(DstAddr<1> dst, ubyte value, size_t count) { #if defined(LLVM_LIBC_ARCH_X86) ///////////////////////////////////////////////////////////////////////////// // LLVM_LIBC_ARCH_X86 ///////////////////////////////////////////////////////////////////////////// - using namespace __llvm_libc::x86; + using namespace x86; if (count == 0) return; if (count == 1) - return splat_set<_1>(dst, value); + return _1::set(dst, value); if (count == 2) - return splat_set<_2>(dst, value); + return _2::set(dst, value); if (count == 3) - return splat_set<_3>(dst, value); + return _3::set(dst, value); if (count <= 8) - return splat_set>(dst, value, count); + return HeadTail<_4>::set(dst, value, count); if (count <= 16) - return splat_set>(dst, value, count); + return HeadTail<_8>::set(dst, value, count); if (count <= 32) - return splat_set>(dst, value, count); + return HeadTail<_16>::set(dst, value, count); if (count <= 64) - return splat_set>(dst, value, count); + return HeadTail<_32>::set(dst, value, count); if (count <= 128) - return splat_set>(dst, value, count); - return splat_set::Then>>(dst, value, count); + return HeadTail<_64>::set(dst, value, count); + return Align<_32, Arg::Dst>::Then>::set(dst, value, count); #elif defined(LLVM_LIBC_ARCH_AARCH64) ///////////////////////////////////////////////////////////////////////////// // LLVM_LIBC_ARCH_AARCH64 ///////////////////////////////////////////////////////////////////////////// - using namespace __llvm_libc::aarch64_memset; + using namespace aarch64; if (count == 0) return; if (count <= 3) { - splat_set<_1>(dst, value); + _1::set(dst, value); if (count > 1) - splat_set>(dst, value, count); + Tail<_2>::set(dst, value, count); return; } if (count <= 8) - return splat_set>(dst, value, count); + return HeadTail<_4>::set(dst, value, count); if (count <= 16) - return splat_set>(dst, value, count); + return HeadTail<_8>::set(dst, value, count); if (count <= 32) - return splat_set>(dst, value, count); + return HeadTail<_16>::set(dst, value, count); if (count <= 96) { - splat_set<_32>(dst, value); + _32::set(dst, value); if (count <= 64) - return splat_set>(dst, value, count); - splat_set::Then<_32>>(dst, value); - splat_set>(dst, value, count); + return Tail<_32>::set(dst, value, count); + Skip<32>::Then<_32>::set(dst, value); + Tail<_32>::set(dst, value, count); return; } - if (count < 448 || value != 0 || !AArch64ZVA(dst, count)) - return splat_set::Then>>(dst, value, count); + if (count >= 448 && value == ubyte::ZERO && hasZva()) + return Align<_64, Arg::_1>::Then>::set(dst, ubyte::ZERO, count); + else + return Align<_16, Arg::_1>::Then>::set(dst, value, count); #else ///////////////////////////////////////////////////////////////////////////// // Default ///////////////////////////////////////////////////////////////////////////// - using namespace ::__llvm_libc::scalar; - + using namespace scalar; if (count == 0) return; if (count == 1) - return splat_set<_1>(dst, value); + return _1::set(dst, value); if (count == 2) - return splat_set<_2>(dst, value); + return _2::set(dst, value); if (count == 3) - return splat_set<_3>(dst, value); + return _3::set(dst, value); if (count <= 8) - return splat_set>(dst, value, count); + return HeadTail<_4>::set(dst, value, count); if (count <= 16) - return splat_set>(dst, value, count); + return HeadTail<_8>::set(dst, value, count); if (count <= 32) - return splat_set>(dst, value, count); + return HeadTail<_16>::set(dst, value, count); if (count <= 64) - return splat_set>(dst, value, count); + return HeadTail<_32>::set(dst, value, count); if (count <= 128) - return splat_set>(dst, value, count); - return splat_set::Then>>(dst, value, count); + return HeadTail<_64>::set(dst, value, count); + return Align<_32, Arg::Dst>::Then>::set(dst, value, count); #endif } diff --git a/libc/src/string/mempcpy.cpp b/libc/src/string/mempcpy.cpp --- a/libc/src/string/mempcpy.cpp +++ b/libc/src/string/mempcpy.cpp @@ -15,11 +15,10 @@ namespace __llvm_libc { LLVM_LIBC_FUNCTION(void *, mempcpy, - (void *__restrict dest, const void *__restrict src, + (void *__restrict dst, const void *__restrict src, size_t count)) { - char *result = reinterpret_cast(dest); - inline_memcpy(result, reinterpret_cast(src), count); - return result + count; + inline_memcpy(dst, src, count); + return reinterpret_cast(dst) + count; } } // namespace __llvm_libc diff --git a/libc/src/string/memset.cpp b/libc/src/string/memset.cpp --- a/libc/src/string/memset.cpp +++ b/libc/src/string/memset.cpp @@ -7,14 +7,14 @@ //===----------------------------------------------------------------------===// #include "src/string/memset.h" + #include "src/__support/common.h" #include "src/string/memory_utils/memset_implementations.h" namespace __llvm_libc { LLVM_LIBC_FUNCTION(void *, memset, (void *dst, int value, size_t count)) { - inline_memset(reinterpret_cast(dst), - static_cast(value), count); + inline_memset(dst, static_cast(value), count); return dst; } diff --git a/libc/src/string/stpncpy.cpp b/libc/src/string/stpncpy.cpp --- a/libc/src/string/stpncpy.cpp +++ b/libc/src/string/stpncpy.cpp @@ -7,9 +7,9 @@ //===----------------------------------------------------------------------===// #include "src/string/stpncpy.h" -#include "src/string/memory_utils/memset_implementations.h" #include "src/__support/common.h" +#include "src/string/memory_utils/memset_implementations.h" namespace __llvm_libc { @@ -22,7 +22,7 @@ dest[i] = src[i]; // When n>strlen(src), n-strlen(src) \0 are appended. if (n > i) - inline_memset(dest + i, 0, n - i); + inline_memset(dest + i, ubyte::ZERO, n - i); return dest + i; } diff --git a/libc/src/string/string_utils.h b/libc/src/string/string_utils.h --- a/libc/src/string/string_utils.h +++ b/libc/src/string/string_utils.h @@ -94,7 +94,7 @@ return len; size_t n = len < size - 1 ? len : size - 1; inline_memcpy(dst, src, n); - inline_memset(dst + n, 0, size - n); + inline_memset(dst + n, ubyte::ZERO, size - n); return len; } diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -795,9 +795,13 @@ cc_library( name = "string_memory_utils", hdrs = [ - "src/string/memory_utils/elements.h", - "src/string/memory_utils/elements_aarch64.h", - "src/string/memory_utils/elements_x86.h", + "src/string/memory_utils/address.h", + "src/string/memory_utils/algorithm.h", + "src/string/memory_utils/backends.h", + "src/string/memory_utils/backend_scalar.h", + "src/string/memory_utils/backend_x86.h", + "src/string/memory_utils/backend_aarch64.h", + "src/string/memory_utils/sized_op.h", "src/string/memory_utils/utils.h", ], textual_hdrs = [ @@ -809,6 +813,7 @@ deps = [ ":__support_common", ":__support_cpp_bit", + ":__support_cpp_type_traits", ":libc_root", ], )