[libc] Extract out an architecture independent copy of memcpy implementation.

[sivachandra]

I am trying to get the string functions work on aarch64. While memset and bzero work as is, memcpy does not because of the x86 specific arrangement we have currently. The x86 specific pieces are actually in dead code. So I went ahead and removed them in this patch. If required in future, we should add it back in a fashion which does not break the build for non-x86 architectures.

[sivachandra]

Also, I do not remember everything we discussed in the past. So, feel free to tell me I am doing rubbish.

[gchatelet]

This comment was removed by gchatelet.

[gchatelet]

In D81621#2086447, @sivachandra wrote:

I am trying to get the string functions work on aarch64. While memset and bzero work as is, memcpy does not because of the x86 specific arrangement we have currently. The x86 specific pieces are actually in dead code. So I went ahead and removed them in this patch. If required in future, we should add it back in a fashion which does not break the build for non-x86 architectures.

memset and bzero are generic and can be used as-is for all architectures.
They may need to be tweaked for individual architecture later on but the current code is good enough.

memcpyis a different story. The C++ implementation is quite tied to x86 so I would rather duplicate the code.
Keep the one we have with the reference to AVX and repmovsb for x86 (libc/src/string/x86/memcpy.cpp)
Duplicate it without AVX / Repmovsb for a generic implementation (libc/src/string/memcpy.cpp)
It also sets an example for anyone wanting to provide an architecture specific implementation.

Thx for the -O2 flag!

[sivachandra]

Make an architecture independent copy of memcpy implementation as suggested.

[sivachandra]

In D81621#2090722, @gchatelet wrote:

In D81621#2086447, @sivachandra wrote:

I am trying to get the string functions work on aarch64. While memset and bzero work as is, memcpy does not because of the x86 specific arrangement we have currently. The x86 specific pieces are actually in dead code. So I went ahead and removed them in this patch. If required in future, we should add it back in a fashion which does not break the build for non-x86 architectures.

memset and bzero are generic and can be used as-is for all architectures.
They may need to be tweaked for individual architecture later on but the current code is good enough.

memcpyis a different story. The C++ implementation is quite tied to x86 so I would rather duplicate the code.
Keep the one we have with the reference to AVX and repmovsb for x86 (libc/src/string/x86/memcpy.cpp)
Duplicate it without AVX / Repmovsb for a generic implementation (libc/src/string/memcpy.cpp)
It also sets an example for anyone wanting to provide an architecture specific implementation.

Thx for the -O2 flag!

OK. I have updated as you suggest.

[sivachandra]

In D81621#2090758, @sivachandra wrote:

In D81621#2090722, @gchatelet wrote:

In D81621#2086447, @sivachandra wrote:

I am trying to get the string functions work on aarch64. While memset and bzero work as is, memcpy does not because of the x86 specific arrangement we have currently. The x86 specific pieces are actually in dead code. So I went ahead and removed them in this patch. If required in future, we should add it back in a fashion which does not break the build for non-x86 architectures.

memset and bzero are generic and can be used as-is for all architectures.
They may need to be tweaked for individual architecture later on but the current code is good enough.

memcpyis a different story. The C++ implementation is quite tied to x86 so I would rather duplicate the code.
Keep the one we have with the reference to AVX and repmovsb for x86 (libc/src/string/x86/memcpy.cpp)
Duplicate it without AVX / Repmovsb for a generic implementation (libc/src/string/memcpy.cpp)
It also sets an example for anyone wanting to provide an architecture specific implementation.

Thx for the -O2 flag!

OK. I have updated as you suggest.

I will update the description after hearing back from @gchatelet.

[gchatelet]

Perfect! Thx!

libc/src/string/CMakeLists.txt

compute_flags(flags
     REQUIRE ${ADD_IMPL_REQUIRE}
     REJECT ${ADD_IMPL_REJECT}
   )
   add_entrypoint_object(${impl_name}
     NAME ${name}
     SRCS ${ADD_IMPL_SRCS}
     HDRS ${ADD_IMPL_HDRS}
     DEPENDS ${ADD_IMPL_DEPENDS}
-    COMPILE_OPTIONS ${ADD_IMPL_COMPILE_OPTIONS} ${flags}
+    COMPILE_OPTIONS ${ADD_IMPL_COMPILE_OPTIONS} ${flags} -O2
   )
   get_fq_target_name(${impl_name} fq_target_name)
   set_target_properties(${fq_target_name} PROPERTIES REQUIRE_CPU_FEATURES "${ADD_IMPL_REQUIRE}")
   set_property(GLOBAL APPEND PROPERTY "${name}_implementations" "${fq_target_name}")
 endfunction()
 
 # ------------------------------------------------------------------------------
 # memcpy
 # ------------------------------------------------------------------------------
 
 # include the relevant architecture specific implementations
 if(${LIBC_TARGET_MACHINE} STREQUAL "x86_64")
   set(LIBC_STRING_TARGET_ARCH "x86")
+  set(MEMCPY_SRC ${LIBC_SOURCE_DIR}/src/string/x86/memcpy.cpp)
 else()
   set(LIBC_STRING_TARGET_ARCH ${LIBC_TARGET_MACHINE})
+  set(MEMCPY_SRC ${LIBC_SOURCE_DIR}/src/string/memcpy.cpp)
 endif()
 
 function(add_memcpy memcpy_name)
   add_implementation(memcpy ${memcpy_name}
-    SRCS ${LIBC_SOURCE_DIR}/src/string/${LIBC_STRING_TARGET_ARCH}/memcpy.cpp
+    SRCS ${MEMCPY_SRC}
     HDRS ${LIBC_SOURCE_DIR}/src/string/memcpy.h
     DEPENDS
       .memory_utils.memory_utils
       libc.include.string
     COMPILE_OPTIONS
       -fno-builtin-memcpy
     ${ARGN}
   )
 endfunction()
 
+if(${LIBC_STRING_TARGET_ARCH} STREQUAL "x86")
   add_memcpy(memcpy MARCH native)
+else()
+  add_memcpy(memcpy)
+endif()
 
 # ------------------------------------------------------------------------------
 # memset
 # ------------------------------------------------------------------------------
 
 function(add_memset memset_name)
   add_implementation(memset ${memset_name}
     SRCS ${LIBC_SOURCE_DIR}/src/string/memset.cpp
     HDRS ${LIBC_SOURCE_DIR}/src/string/memset.h
     DEPENDS
       .memory_utils.memory_utils
       libc.include.string
     COMPILE_OPTIONS
       -fno-builtin-memset
     ${ARGN}
   )
 endfunction()
 
+if(${LIBC_STRING_TARGET_ARCH} STREQUAL "x86")
   add_memset(memset MARCH native)
+else()
+  add_memset(memset)
+endif()
 
 # ------------------------------------------------------------------------------
 # bzero
 # ------------------------------------------------------------------------------
 
 function(add_bzero bzero_name)
   add_implementation(bzero ${bzero_name}
     SRCS ${LIBC_SOURCE_DIR}/src/string/bzero.cpp
     HDRS ${LIBC_SOURCE_DIR}/src/string/bzero.h
     DEPENDS
       .memory_utils.memory_utils
       libc.include.string
     COMPILE_OPTIONS
       -fno-builtin-memset
       -fno-builtin-bzero
     ${ARGN}
   )
 endfunction()
 
+if(${LIBC_STRING_TARGET_ARCH} STREQUAL "x86")
   add_bzero(bzero MARCH native)
+else()
+  add_bzero(bzero)
+endif()
 
 # ------------------------------------------------------------------------------
 # Add all other relevant implementations for the native target.
 # ------------------------------------------------------------------------------
 
 if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_STRING_TARGET_ARCH})
   include(${LIBC_STRING_TARGET_ARCH}/CMakeLists.txt)
 endif()

libc/src/string/memcpy.cpp

+//===-- Implementation of memcpy ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/memcpy.h"
+#include "src/__support/common.h"
+#include "src/string/memory_utils/memcpy_utils.h"
+
+namespace __llvm_libc {
+
+// Design rationale
+// ================
+//
+// Using a profiler to observe size distributions for calls into libc
+// functions, it was found most operations act on a small number of bytes.
+// This makes it important to favor small sizes.
+//
+// The tests for `count` are in ascending order so the cost of branching is
+// proportional to the cost of copying.
+//
+// The function is written in C++ for several reasons:
+// - The compiler can __see__ the code, this is useful when performing Profile
+//   Guided Optimization as the optimized code can take advantage of branching
+//   probabilities.
+// - It also allows for easier customization and favors testing multiple
+//   implementation parameters.
+// - As compilers and processors get better, the generated code is improved
+//   with little change on the code side.
+static void memcpy_impl(char *__restrict dst, const char *__restrict src,
+                        size_t count) {
+  if (count == 0)
+    return;
+  if (count == 1)
+    return CopyBlock<1>(dst, src);
+  if (count == 2)
+    return CopyBlock<2>(dst, src);
+  if (count == 3)
+    return CopyBlock<3>(dst, src);
+  if (count == 4)
+    return CopyBlock<4>(dst, src);
+  if (count < 8)
+    return CopyBlockOverlap<4>(dst, src, count);
+  if (count == 8)
+    return CopyBlock<8>(dst, src);
+  if (count < 16)
+    return CopyBlockOverlap<8>(dst, src, count);
+  if (count == 16)
+    return CopyBlock<16>(dst, src);
+  if (count < 32)
+    return CopyBlockOverlap<16>(dst, src, count);
+  if (count < 64)
+    return CopyBlockOverlap<32>(dst, src, count);
+  if (count < 128)
+    return CopyBlockOverlap<64>(dst, src, count);
+  return CopyAlignedBlocks<32>(dst, src, count);
+}
+
+void *LLVM_LIBC_ENTRYPOINT(memcpy)(void *__restrict dst,
+                                   const void *__restrict src, size_t size) {
+  memcpy_impl(reinterpret_cast<char *>(dst),
+              reinterpret_cast<const char *>(src), size);
+  return dst;
+}
+
+} // namespace __llvm_libc