diff --git a/libc/src/string/CMakeLists.txt b/libc/src/string/CMakeLists.txt --- a/libc/src/string/CMakeLists.txt +++ b/libc/src/string/CMakeLists.txt @@ -461,7 +461,7 @@ SRCS ${LIBC_SOURCE_DIR}/src/string/memmove.cpp HDRS ${LIBC_SOURCE_DIR}/src/string/memmove.h DEPENDS - .memory_utils.memory_utils + .memory_utils.memcpy_implementation libc.include.string ${ARGN} ) diff --git a/libc/src/string/memmove.cpp b/libc/src/string/memmove.cpp --- a/libc/src/string/memmove.cpp +++ b/libc/src/string/memmove.cpp @@ -7,112 +7,14 @@ //===----------------------------------------------------------------------===// #include "src/string/memmove.h" - -#include "src/__support/common.h" -#include "src/string/memory_utils/op_aarch64.h" -#include "src/string/memory_utils/op_builtin.h" -#include "src/string/memory_utils/op_generic.h" -#include "src/string/memory_utils/op_x86.h" -#include // size_t, ptrdiff_t - -#include +#include "src/string/memory_utils/memmove_implementations.h" +#include // size_t namespace __llvm_libc { -[[maybe_unused]] static inline void -inline_memmove_embedded_tiny(Ptr dst, CPtr src, size_t count) { - if ((count == 0) || (dst == src)) - return; - if (dst < src) { -#pragma nounroll - for (size_t offset = 0; offset < count; ++offset) - builtin::Memcpy<1>::block(dst + offset, src + offset); - } else { -#pragma nounroll - for (ptrdiff_t offset = count - 1; offset >= 0; --offset) - builtin::Memcpy<1>::block(dst + offset, src + offset); - } -} - -template -[[maybe_unused]] static inline void inline_memmove_generic(Ptr dst, CPtr src, - size_t count) { - if (count == 0) - return; - if (count == 1) - return generic::Memmove<1, MaxSize>::block(dst, src); - if (count <= 4) - return generic::Memmove<2, MaxSize>::head_tail(dst, src, count); - if (count <= 8) - return generic::Memmove<4, MaxSize>::head_tail(dst, src, count); - if (count <= 16) - return generic::Memmove<8, MaxSize>::head_tail(dst, src, count); - if (count <= 32) - return generic::Memmove<16, MaxSize>::head_tail(dst, src, count); - if (count <= 64) - return generic::Memmove<32, MaxSize>::head_tail(dst, src, count); - if (count <= 128) - return generic::Memmove<64, MaxSize>::head_tail(dst, src, count); - if (dst < src) { - generic::Memmove<32, MaxSize>::template align_forward(dst, src, - count); - return generic::Memmove<64, MaxSize>::loop_and_tail_forward(dst, src, - count); - } else { - generic::Memmove<32, MaxSize>::template align_backward(dst, src, - count); - return generic::Memmove<64, MaxSize>::loop_and_tail_backward(dst, src, - count); - } -} - -static inline void inline_memmove(Ptr dst, CPtr src, size_t count) { -#if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64) -#if defined(LLVM_LIBC_ARCH_X86) - static constexpr size_t kMaxSize = x86::kAvx512F ? 64 - : x86::kAvx ? 32 - : x86::kSse2 ? 16 - : 8; -#elif defined(LLVM_LIBC_ARCH_AARCH64) - static constexpr size_t kMaxSize = aarch64::kNeon ? 16 : 8; -#endif - // return inline_memmove_generic(dst, src, count); - if (count == 0) - return; - if (count == 1) - return generic::Memmove<1, kMaxSize>::block(dst, src); - if (count <= 4) - return generic::Memmove<2, kMaxSize>::head_tail(dst, src, count); - if (count <= 8) - return generic::Memmove<4, kMaxSize>::head_tail(dst, src, count); - if (count <= 16) - return generic::Memmove<8, kMaxSize>::head_tail(dst, src, count); - if (count <= 32) - return generic::Memmove<16, kMaxSize>::head_tail(dst, src, count); - if (count <= 64) - return generic::Memmove<32, kMaxSize>::head_tail(dst, src, count); - if (count <= 128) - return generic::Memmove<64, kMaxSize>::head_tail(dst, src, count); - if (dst < src) { - generic::Memmove<32, kMaxSize>::align_forward(dst, src, count); - return generic::Memmove<64, kMaxSize>::loop_and_tail_forward(dst, src, - count); - } else { - generic::Memmove<32, kMaxSize>::align_backward(dst, src, count); - return generic::Memmove<64, kMaxSize>::loop_and_tail_backward(dst, src, - count); - } -#elif defined(LLVM_LIBC_ARCH_ARM) - return inline_memmove_embedded_tiny(dst, src, count); -#else -#error "Unsupported platform" -#endif -} - LLVM_LIBC_FUNCTION(void *, memmove, (void *dst, const void *src, size_t count)) { - inline_memmove(reinterpret_cast(dst), reinterpret_cast(src), - count); + inline_memmove(dst, src, count); return dst; } diff --git a/libc/src/string/memory_utils/CMakeLists.txt b/libc/src/string/memory_utils/CMakeLists.txt --- a/libc/src/string/memory_utils/CMakeLists.txt +++ b/libc/src/string/memory_utils/CMakeLists.txt @@ -6,6 +6,7 @@ bzero_implementations.h memcmp_implementations.h memcpy_implementations.h + memmove_implementations.h memset_implementations.h op_aarch64.h op_builtin.h @@ -27,6 +28,14 @@ .memory_utils ) +add_header_library( + memmove_implementation + HDRS + memmove_implementations.h + DEPS + .memory_utils +) + add_header_library( memcmp_implementation HDRS diff --git a/libc/src/string/memmove.cpp b/libc/src/string/memory_utils/memmove_implementations.h copy from libc/src/string/memmove.cpp copy to libc/src/string/memory_utils/memmove_implementations.h --- a/libc/src/string/memmove.cpp +++ b/libc/src/string/memory_utils/memmove_implementations.h @@ -1,4 +1,4 @@ -//===-- Implementation of memmove -----------------------------------------===// +//===-- Memmove implementation ----------------------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,7 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "src/string/memmove.h" +#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H +#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H #include "src/__support/common.h" #include "src/string/memory_utils/op_aarch64.h" @@ -15,8 +16,6 @@ #include "src/string/memory_utils/op_x86.h" #include // size_t, ptrdiff_t -#include - namespace __llvm_libc { [[maybe_unused]] static inline void @@ -109,11 +108,11 @@ #endif } -LLVM_LIBC_FUNCTION(void *, memmove, - (void *dst, const void *src, size_t count)) { +static inline void inline_memmove(void *dst, const void *src, size_t count) { inline_memmove(reinterpret_cast(dst), reinterpret_cast(src), count); - return dst; } } // namespace __llvm_libc + +#endif /* LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H */ diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -1019,6 +1019,7 @@ "src/string/memory_utils/bzero_implementations.h", "src/string/memory_utils/memcmp_implementations.h", "src/string/memory_utils/memcpy_implementations.h", + "src/string/memory_utils/memmove_implementations.h", "src/string/memory_utils/memset_implementations.h", ], deps = [ @@ -1082,7 +1083,6 @@ features = no_sanitize_features, deps = [ ":__support_common", - ":__support_integer_operations", ":string_memory_utils", ], )