[libc] Allow customization of memcpy via flags.

[gchatelet]

Summary

• Adds LLVM_LIBC_IS_DEFINED macro to libc/src/__support/common.h
• Adds a few knobs to memcpy to help with experimentations:
  • LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB replaces the implementation with a single call to rep;movsb
  • LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE customizes where the usage of rep;movsb

[sivachandra]

Few inline nits.

libc/src/__support/common.h

42

In other places, we have named such namespaces as internal.

45

Just pointing out, we cannot use the C++ standard library. So, this comment should either be removed or updated suitably.

libc/src/string/x86/memcpy.cpp

91

Can we use the early return pattern for this as well without the else?

if (kUseOnlyRepMovSb)
  return CopyRepMovsb(...);

[tschuett]

There is -nostdinc++.

[gchatelet]

• Address comments

libc/src/__support/common.h

#define LLVM_LIBC_FUNCTION(type, name, arglist)                                \
   LLVM_LIBC_FUNCTION_ATTR decltype(__llvm_libc::name)                          \
       __##name##_impl__ __asm__(#name);                                        \
   decltype(__llvm_libc::name) name [[gnu::alias(#name)]];                      \
   type __##name##_impl__ arglist
 #else
 #define LLVM_LIBC_FUNCTION(type, name, arglist) type name arglist
 #endif
 
+namespace __llvm_libc {
+namespace internal {
+constexpr bool same_string(char const *lhs, char const *rhs) {
+  for (; *lhs || *rhs; ++lhs, ++rhs)
+    if (*lhs != *rhs)
+      return false;
+  return true;
+}
+} // namespace internal
+} // namespace __llvm_libc
+
+// LLVM_LIBC_IS_DEFINED checks whether a particular macro is defined.
+// Usage: constexpr bool kUseAvx = LLVM_LIBC_IS_DEFINED(__AVX__);
+//
+// This works by comparing the stringified version of the macro with and without
+// evaluation. If FOO is not undefined both stringifications yield "FOO". If FOO
+// is defined, one stringification yields "FOO" while the other yields its
+// stringified value "1".
+#define LLVM_LIBC_IS_DEFINED(macro)                                            \
+  !__llvm_libc::internal::same_string(                                         \
+      LLVM_LIBC_IS_DEFINED__EVAL_AND_STRINGIZE(macro), #macro)
+#define LLVM_LIBC_IS_DEFINED__EVAL_AND_STRINGIZE(s) #s
+
 #endif // LLVM_LIBC_SUPPORT_COMMON_H

libc/src/string/x86/memcpy.cpp

 //===-- Implementation of memcpy ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/memcpy.h"
 #include "src/__support/common.h"
 #include "src/string/memory_utils/memcpy_utils.h"
 
 namespace __llvm_libc {
 
+// Whether to use only rep;movsb.
+constexpr bool kUseOnlyRepMovsb =
+    LLVM_LIBC_IS_DEFINED(LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB);
+
+// kRepMovsBSize == -1 : Only CopyAligned is used.
+// kRepMovsBSize ==  0 : Only RepMovsb is used.
+// else CopyAligned is used up to kRepMovsBSize and then RepMovsb.
+constexpr size_t kRepMovsBSize =
+#ifdef LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
+    LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE;
+#else
+    -1;
+#endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
+
+// Whether target supports AVX instructions.
+constexpr bool kHasAvx = LLVM_LIBC_IS_DEFINED(__AVX__);
+
+// The chunk size used for the loop copy strategy.
+constexpr size_t kLoopCopyBlockSize = kHasAvx ? 64 : 32;
+
 static void CopyRepMovsb(char *__restrict dst, const char *__restrict src,
                          size_t count) {
   // FIXME: Add MSVC support with
   // #include <intrin.h>
   // __movsb(reinterpret_cast<unsigned char *>(dst),
   //         reinterpret_cast<const unsigned char *>(src), count);
   asm volatile("rep movsb" : "+D"(dst), "+S"(src), "+c"(count) : : "memory");
 }
 
-#if defined(__AVX__)
-#define BEST_SIZE 64
-#else
-#define BEST_SIZE 32
-#endif
-
 // Design rationale
 // ================
 //
 // Using a profiler to observe size distributions for calls into libc
 // functions, it was found most operations act on a small number of bytes.
 // This makes it important to favor small sizes.
 //
 // The tests for `count` are in ascending order so the cost of branching is
 // proportional to the cost of copying.
 //
 // The function is written in C++ for several reasons:
 // - The compiler can __see__ the code, this is useful when performing Profile
 //   Guided Optimization as the optimized code can take advantage of branching
 //   probabilities.
 // - It also allows for easier customization and favors testing multiple
 //   implementation parameters.
 // - As compilers and processors get better, the generated code is improved
 //   with little change on the code side.
 static void memcpy_x86(char *__restrict dst, const char *__restrict src,
                        size_t count) {
+  if (kUseOnlyRepMovsb)
+    return CopyRepMovsb(dst, src, count);
+
   if (count == 0)
     return;
   if (count == 1)
     return CopyBlock<1>(dst, src);
   if (count == 2)
     return CopyBlock<2>(dst, src);
   if (count == 3)
     return CopyBlock<3>(dst, src);
   if (count == 4)
     return CopyBlock<4>(dst, src);
   if (count < 8)
     return CopyBlockOverlap<4>(dst, src, count);
   if (count < 16)
     return CopyBlockOverlap<8>(dst, src, count);
   if (count < 32)
     return CopyBlockOverlap<16>(dst, src, count);
   if (count < 64)
     return CopyBlockOverlap<32>(dst, src, count);
   if (count < 128)
     return CopyBlockOverlap<64>(dst, src, count);
-#if defined(__AVX__)
-  if (count < 256)
+  if (kHasAvx && count < 256)
     return CopyBlockOverlap<128>(dst, src, count);
-#endif
-  // kRepMovsBSize == -1 : Only CopyAligned is used.
-  // kRepMovsBSize ==  0 : Only RepMovsb is used.
-  // else CopyAligned is used to to kRepMovsBSize and then RepMovsb.
-  constexpr size_t kRepMovsBSize = -1;
   if (count <= kRepMovsBSize)
-    return CopyAlignedBlocks<BEST_SIZE>(dst, src, count);
+    return CopyAlignedBlocks<kLoopCopyBlockSize>(dst, src, count);
   return CopyRepMovsb(dst, src, count);
 }
 
 LLVM_LIBC_FUNCTION(void *, memcpy,
                    (void *__restrict dst, const void *__restrict src,
                     size_t size)) {
   memcpy_x86(reinterpret_cast<char *>(dst), reinterpret_cast<const char *>(src),
              size);
   return dst;
 }
 
 } // namespace __llvm_libc