diff --git a/libc/src/string/memory_utils/CMakeLists.txt b/libc/src/string/memory_utils/CMakeLists.txt --- a/libc/src/string/memory_utils/CMakeLists.txt +++ b/libc/src/string/memory_utils/CMakeLists.txt @@ -5,6 +5,7 @@ aarch64/bcmp_implementations.h aarch64/memcmp_implementations.h aarch64/memcpy_implementations.h + aarch64/memmove_implementations.h aarch64/memset_implementations.h bcmp_implementations.h bzero_implementations.h @@ -21,11 +22,13 @@ riscv/bcmp_implementations.h riscv/memcmp_implementations.h riscv/memcpy_implementations.h + riscv/memmove_implementations.h riscv/memset_implementations.h utils.h x86_64/bcmp_implementations.h x86_64/memcmp_implementations.h x86_64/memcpy_implementations.h + x86_64/memmove_implementations.h x86_64/memset_implementations.h DEPS libc.src.__support.common diff --git a/libc/src/string/memory_utils/memmove_implementations.h b/libc/src/string/memory_utils/aarch64/memmove_implementations.h copy from libc/src/string/memory_utils/memmove_implementations.h copy to libc/src/string/memory_utils/aarch64/memmove_implementations.h --- a/libc/src/string/memory_utils/memmove_implementations.h +++ b/libc/src/string/memory_utils/aarch64/memmove_implementations.h @@ -1,51 +1,28 @@ -//===-- Memmove implementation ----------------------------------*- C++ -*-===// +//===-- Memmove implementation for aarch64 ----------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// +#ifndef LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMMOVE_IMPLEMENTATIONS_H +#define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMMOVE_IMPLEMENTATIONS_H -#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H -#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H - -#include "src/__support/common.h" -#include "src/__support/macros/optimization.h" -#include "src/string/memory_utils/generic/byte_per_byte.h" -#include "src/string/memory_utils/op_aarch64.h" +#include "src/__support/macros/config.h" // LIBC_INLINE +#include "src/string/memory_utils/op_aarch64.h" // aarch64::kNeon #include "src/string/memory_utils/op_builtin.h" #include "src/string/memory_utils/op_generic.h" -#include "src/string/memory_utils/op_x86.h" -#include // size_t, ptrdiff_t +#include "src/string/memory_utils/utils.h" + +#include // size_t namespace __llvm_libc { -LIBC_INLINE void inline_memmove(Ptr dst, CPtr src, size_t count) { -#if defined(LIBC_TARGET_ARCH_IS_X86) || defined(LIBC_TARGET_ARCH_IS_AARCH64) -#if defined(LIBC_TARGET_ARCH_IS_X86) -#if defined(__AVX512F__) - using uint128_t = generic_v128; - using uint256_t = generic_v256; - using uint512_t = generic_v512; -#elif defined(__AVX__) - using uint128_t = generic_v128; - using uint256_t = generic_v256; - using uint512_t = cpp::array; -#elif defined(__SSE2__) - using uint128_t = generic_v128; - using uint256_t = cpp::array; - using uint512_t = cpp::array; -#else - using uint128_t = cpp::array; - using uint256_t = cpp::array; - using uint512_t = cpp::array; -#endif -#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) +LIBC_INLINE void inline_memmove_aarch64(Ptr dst, CPtr src, size_t count) { static_assert(aarch64::kNeon, "aarch64 supports vector types"); using uint128_t = generic_v128; using uint256_t = generic_v256; using uint512_t = generic_v512; -#endif if (count == 0) return; if (count == 1) @@ -69,16 +46,8 @@ generic::Memmove::align_backward(dst, src, count); return generic::Memmove::loop_and_tail_backward(dst, src, count); } -#else - return inline_memmove_byte_per_byte(dst, src, count); -#endif -} - -LIBC_INLINE void inline_memmove(void *dst, const void *src, size_t count) { - inline_memmove(reinterpret_cast(dst), reinterpret_cast(src), - count); } } // namespace __llvm_libc -#endif /* LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H */ +#endif // LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMMOVE_IMPLEMENTATIONS_H diff --git a/libc/src/string/memory_utils/memmove_implementations.h b/libc/src/string/memory_utils/memmove_implementations.h --- a/libc/src/string/memory_utils/memmove_implementations.h +++ b/libc/src/string/memory_utils/memmove_implementations.h @@ -9,74 +9,28 @@ #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H -#include "src/__support/common.h" -#include "src/__support/macros/optimization.h" -#include "src/string/memory_utils/generic/byte_per_byte.h" -#include "src/string/memory_utils/op_aarch64.h" -#include "src/string/memory_utils/op_builtin.h" -#include "src/string/memory_utils/op_generic.h" -#include "src/string/memory_utils/op_x86.h" #include // size_t, ptrdiff_t -namespace __llvm_libc { - -LIBC_INLINE void inline_memmove(Ptr dst, CPtr src, size_t count) { -#if defined(LIBC_TARGET_ARCH_IS_X86) || defined(LIBC_TARGET_ARCH_IS_AARCH64) #if defined(LIBC_TARGET_ARCH_IS_X86) -#if defined(__AVX512F__) - using uint128_t = generic_v128; - using uint256_t = generic_v256; - using uint512_t = generic_v512; -#elif defined(__AVX__) - using uint128_t = generic_v128; - using uint256_t = generic_v256; - using uint512_t = cpp::array; -#elif defined(__SSE2__) - using uint128_t = generic_v128; - using uint256_t = cpp::array; - using uint512_t = cpp::array; -#else - using uint128_t = cpp::array; - using uint256_t = cpp::array; - using uint512_t = cpp::array; -#endif +#include "src/string/memory_utils/x86_64/memmove_implementations.h" +#define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE inline_memmove_x86 #elif defined(LIBC_TARGET_ARCH_IS_AARCH64) - static_assert(aarch64::kNeon, "aarch64 supports vector types"); - using uint128_t = generic_v128; - using uint256_t = generic_v256; - using uint512_t = generic_v512; -#endif - if (count == 0) - return; - if (count == 1) - return generic::Memmove::block(dst, src); - if (count <= 4) - return generic::Memmove::head_tail(dst, src, count); - if (count <= 8) - return generic::Memmove::head_tail(dst, src, count); - if (count <= 16) - return generic::Memmove::head_tail(dst, src, count); - if (count <= 32) - return generic::Memmove::head_tail(dst, src, count); - if (count <= 64) - return generic::Memmove::head_tail(dst, src, count); - if (count <= 128) - return generic::Memmove::head_tail(dst, src, count); - if (dst < src) { - generic::Memmove::align_forward(dst, src, count); - return generic::Memmove::loop_and_tail_forward(dst, src, count); - } else { - generic::Memmove::align_backward(dst, src, count); - return generic::Memmove::loop_and_tail_backward(dst, src, count); - } +#include "src/string/memory_utils/aarch64/memmove_implementations.h" +#define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE inline_memmove_aarch64 +#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) +#include "src/string/memory_utils/riscv/memmove_implementations.h" +#define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE inline_memmove_riscv #else - return inline_memmove_byte_per_byte(dst, src, count); +// We may want to error instead of defaulting to suboptimal implementation. +#include "src/string/memory_utils/generic/byte_per_byte.h" +#define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE inline_memmove_byte_per_byte #endif -} + +namespace __llvm_libc { LIBC_INLINE void inline_memmove(void *dst, const void *src, size_t count) { - inline_memmove(reinterpret_cast(dst), reinterpret_cast(src), - count); + LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE(reinterpret_cast(dst), + reinterpret_cast(src), count); } } // namespace __llvm_libc diff --git a/libc/src/string/memory_utils/riscv/memmove_implementations.h b/libc/src/string/memory_utils/riscv/memmove_implementations.h new file mode 100644 --- /dev/null +++ b/libc/src/string/memory_utils/riscv/memmove_implementations.h @@ -0,0 +1,33 @@ +//===-- Memmove implementation for riscv ------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LIBC_SRC_STRING_MEMORY_UTILS_RISCV_MEMMOVE_IMPLEMENTATIONS_H +#define LIBC_SRC_STRING_MEMORY_UTILS_RISCV_MEMMOVE_IMPLEMENTATIONS_H + +#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_RISCV64 +#include "src/string/memory_utils/generic/aligned_access.h" +#include "src/string/memory_utils/utils.h" // Ptr, CPtr + +#include // size_t + +namespace __llvm_libc { + +[[maybe_unused]] LIBC_INLINE void +inline_memmove_riscv(Ptr __restrict dst, CPtr __restrict src, size_t count) { +#if defined(LIBC_TARGET_ARCH_IS_RISCV64) + return inline_memmove_aligned_access_64bit(dst, src, count); +#elif defined(LIBC_TARGET_ARCH_IS_RISCV32) + return inline_memmove_aligned_access_32bit(dst, src, count); +#else +#error "Unimplemented" +#endif +} + +} // namespace __llvm_libc + +#endif // LIBC_SRC_STRING_MEMORY_UTILS_RISCV_MEMMOVE_IMPLEMENTATIONS_H diff --git a/libc/src/string/memory_utils/memmove_implementations.h b/libc/src/string/memory_utils/x86_64/memmove_implementations.h copy from libc/src/string/memory_utils/memmove_implementations.h copy to libc/src/string/memory_utils/x86_64/memmove_implementations.h --- a/libc/src/string/memory_utils/memmove_implementations.h +++ b/libc/src/string/memory_utils/x86_64/memmove_implementations.h @@ -1,28 +1,24 @@ -//===-- Memmove implementation ----------------------------------*- C++ -*-===// +//===-- Memmove implementation for x86_64 -----------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// +#ifndef LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMMOVE_IMPLEMENTATIONS_H +#define LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMMOVE_IMPLEMENTATIONS_H -#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H -#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H - -#include "src/__support/common.h" -#include "src/__support/macros/optimization.h" -#include "src/string/memory_utils/generic/byte_per_byte.h" -#include "src/string/memory_utils/op_aarch64.h" +#include "src/__support/macros/config.h" // LIBC_INLINE #include "src/string/memory_utils/op_builtin.h" #include "src/string/memory_utils/op_generic.h" #include "src/string/memory_utils/op_x86.h" -#include // size_t, ptrdiff_t +#include "src/string/memory_utils/utils.h" + +#include // size_t namespace __llvm_libc { -LIBC_INLINE void inline_memmove(Ptr dst, CPtr src, size_t count) { -#if defined(LIBC_TARGET_ARCH_IS_X86) || defined(LIBC_TARGET_ARCH_IS_AARCH64) -#if defined(LIBC_TARGET_ARCH_IS_X86) +LIBC_INLINE void inline_memmove_x86(Ptr dst, CPtr src, size_t count) { #if defined(__AVX512F__) using uint128_t = generic_v128; using uint256_t = generic_v256; @@ -39,12 +35,6 @@ using uint128_t = cpp::array; using uint256_t = cpp::array; using uint512_t = cpp::array; -#endif -#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) - static_assert(aarch64::kNeon, "aarch64 supports vector types"); - using uint128_t = generic_v128; - using uint256_t = generic_v256; - using uint512_t = generic_v512; #endif if (count == 0) return; @@ -69,16 +59,8 @@ generic::Memmove::align_backward(dst, src, count); return generic::Memmove::loop_and_tail_backward(dst, src, count); } -#else - return inline_memmove_byte_per_byte(dst, src, count); -#endif -} - -LIBC_INLINE void inline_memmove(void *dst, const void *src, size_t count) { - inline_memmove(reinterpret_cast(dst), reinterpret_cast(src), - count); } } // namespace __llvm_libc -#endif /* LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H */ +#endif // LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMMOVE_IMPLEMENTATIONS_H diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -2012,6 +2012,7 @@ "src/string/memory_utils/aarch64/bcmp_implementations.h", "src/string/memory_utils/aarch64/memcmp_implementations.h", "src/string/memory_utils/aarch64/memcpy_implementations.h", + "src/string/memory_utils/aarch64/memmove_implementations.h", "src/string/memory_utils/aarch64/memset_implementations.h", "src/string/memory_utils/bcmp_implementations.h", "src/string/memory_utils/bzero_implementations.h", @@ -2025,12 +2026,14 @@ "src/string/memory_utils/riscv/bcmp_implementations.h", "src/string/memory_utils/riscv/memcmp_implementations.h", "src/string/memory_utils/riscv/memcpy_implementations.h", + "src/string/memory_utils/riscv/memmove_implementations.h", "src/string/memory_utils/riscv/memset_implementations.h", "src/string/memory_utils/strcmp_implementations.h", "src/string/memory_utils/strstr_implementations.h", "src/string/memory_utils/x86_64/bcmp_implementations.h", "src/string/memory_utils/x86_64/memcmp_implementations.h", "src/string/memory_utils/x86_64/memcpy_implementations.h", + "src/string/memory_utils/x86_64/memmove_implementations.h", "src/string/memory_utils/x86_64/memset_implementations.h", ], deps = [