[libc][NFC] Allow memset (and bzero) to be inlined

[gchatelet]

Summary

This allows shipping individual functions without also having to provide
memset or bzero at the expense of bigger functions.

Similar to D113097.

[gchatelet]

• Also fix CMakeLists.txt

[michaelrj]

libc/src/string/memory_utils/memset_implementations.h

8–10

Should the new inline files have header guards?

[gchatelet]

• Also fix CMakeLists.txt
• add header guard, rename function and filename.

[gchatelet]

• rebase

[gchatelet]

I've added the header guard and renamed the file and function, so I think this is good to go.

libc/src/string/CMakeLists.txt

 endif()
 
 # ------------------------------------------------------------------------------
 # memset
 # ------------------------------------------------------------------------------
 
 function(add_memset memset_name)
   add_implementation(memset ${memset_name}
-    SRCS ${MEMSET_SRC}
+    SRCS ${LIBC_SOURCE_DIR}/src/string/memset.cpp
     HDRS ${LIBC_SOURCE_DIR}/src/string/memset.h
     DEPENDS
       .memory_utils.memory_utils
       libc.include.string
     COMPILE_OPTIONS
       -fno-builtin-memset
     ${ARGN}
   )
 endfunction()
 
 if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
-  set(MEMSET_SRC ${LIBC_SOURCE_DIR}/src/string/memset.cpp)
   add_memset(memset_x86_64_opt_sse2   COMPILE_OPTIONS -march=k8             REQUIRE SSE2)
   add_memset(memset_x86_64_opt_sse4   COMPILE_OPTIONS -march=nehalem        REQUIRE SSE4_2)
   add_memset(memset_x86_64_opt_avx2   COMPILE_OPTIONS -march=haswell        REQUIRE AVX2)
   add_memset(memset_x86_64_opt_avx512 COMPILE_OPTIONS -march=skylake-avx512 REQUIRE AVX512F)
   add_memset(memset_opt_host          COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE})
   add_memset(memset)
 elseif(${LIBC_TARGET_ARCHITECTURE_IS_AARCH64})
-  set(MEMSET_SRC ${LIBC_SOURCE_DIR}/src/string/aarch64/memset.cpp)
   add_memset(memset_opt_host          COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE}
                                       COMPILE_OPTIONS "SHELL:-mllvm --tail-merge-threshold=0")
   add_memset(memset                   COMPILE_OPTIONS "SHELL:-mllvm --tail-merge-threshold=0")
 else()
-  set(MEMSET_SRC ${LIBC_SOURCE_DIR}/src/string/memset.cpp)
   add_memset(memset_opt_host          COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE})
   add_memset(memset)
 endif()

libc/src/string/aarch64/memset.cpp

-//===-- Implementation of memset ------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/string/memset.h"
-#include "src/__support/common.h"
-#include "src/string/memory_utils/memset_utils.h"
-
-namespace __llvm_libc {
-
-using namespace __llvm_libc::aarch64_memset;
-
-inline static void AArch64Memset(char *dst, int value, size_t count) {
-  if (count == 0)
-    return;
-  if (count <= 3) {
-    SplatSet<_1>(dst, value);
-    if (count > 1)
-      SplatSet<Tail<_2>>(dst, value, count);
-    return;
-  }
-  if (count <= 8)
-    return SplatSet<HeadTail<_4>>(dst, value, count);
-  if (count <= 16)
-    return SplatSet<HeadTail<_8>>(dst, value, count);
-  if (count <= 32)
-    return SplatSet<HeadTail<_16>>(dst, value, count);
-  if (count <= 96) {
-    SplatSet<_32>(dst, value);
-    if (count <= 64)
-      return SplatSet<Tail<_32>>(dst, value, count);
-    SplatSet<Skip<32>::Then<_32>>(dst, value);
-    SplatSet<Tail<_32>>(dst, value, count);
-    return;
-  }
-  if (count < 448 || value != 0 || !AArch64ZVA(dst, count))
-    return SplatSet<Align<_16, Arg::_1>::Then<Loop<_64>>>(dst, value, count);
-}
-
-LLVM_LIBC_FUNCTION(void *, memset, (void *dst, int value, size_t count)) {
-  AArch64Memset((char *)dst, value, count);
-  return dst;
-}
-
-} // namespace __llvm_libc

libc/src/string/bzero.cpp

 //===-- Implementation of bzero -------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/bzero.h"
 #include "src/__support/common.h"
-#include "src/string/memory_utils/memset_utils.h"
+#include "src/string/memory_utils/memset_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(void, bzero, (void *ptr, size_t count)) {
-  GeneralPurposeMemset(reinterpret_cast<char *>(ptr), 0, count);
+  inline_memset(reinterpret_cast<char *>(ptr), 0, count);
 }
 
 } // namespace __llvm_libc

libc/src/string/memory_utils/memset_implementations.h

-//===-- Memset utils --------------------------------------------*- C++ -*-===//
+//===-- Implementation of memset and bzero --------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_UTILS_H
-#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_UTILS_H
+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H
 
 #include "src/__support/architectures.h"
 #include "src/string/memory_utils/elements.h"
 #include "src/string/memory_utils/utils.h"
 
 #include <stddef.h> // size_t
 
 namespace __llvm_libc {

 // conditional were expensive, even on processor that do not support writing 64B
 // at a time (pre-AVX512F). We also want to favor short functions that allow
 // more hot code to fit in the iL1 cache.
 //
 // Above 128 we have to use conditionals since we don't know the upper bound in
 // advance. SetAlignedBlocks<64> may waste up to 63 Bytes, SetAlignedBlocks<32>
 // may waste up to 31 Bytes. Benchmarks showed that SetAlignedBlocks<64> was not
 // superior for sizes that mattered.
-inline static void GeneralPurposeMemset(char *dst, unsigned char value,
-                                        size_t count) {
+inline static void inline_memset(char *dst, unsigned char value, size_t count) {
 #if defined(LLVM_LIBC_ARCH_X86)
-  using namespace ::__llvm_libc::x86;
+  /////////////////////////////////////////////////////////////////////////////
+  // LLVM_LIBC_ARCH_X86
+  /////////////////////////////////////////////////////////////////////////////
+  using namespace __llvm_libc::x86;
+  if (count == 0)
+    return;
+  if (count == 1)
+    return SplatSet<_1>(dst, value);
+  if (count == 2)
+    return SplatSet<_2>(dst, value);
+  if (count == 3)
+    return SplatSet<_3>(dst, value);
+  if (count <= 8)
+    return SplatSet<HeadTail<_4>>(dst, value, count);
+  if (count <= 16)
+    return SplatSet<HeadTail<_8>>(dst, value, count);
+  if (count <= 32)
+    return SplatSet<HeadTail<_16>>(dst, value, count);
+  if (count <= 64)
+    return SplatSet<HeadTail<_32>>(dst, value, count);
+  if (count <= 128)
+    return SplatSet<HeadTail<_64>>(dst, value, count);
+  return SplatSet<Align<_32, Arg::Dst>::Then<Loop<_32>>>(dst, value, count);
+#elif defined(LLVM_LIBC_ARCH_AARCH64)
+  /////////////////////////////////////////////////////////////////////////////
+  // LLVM_LIBC_ARCH_AARCH64
+  /////////////////////////////////////////////////////////////////////////////
+  using namespace __llvm_libc::aarch64_memset;
+  if (count == 0)
+    return;
+  if (count <= 3) {
+    SplatSet<_1>(dst, value);
+    if (count > 1)
+      SplatSet<Tail<_2>>(dst, value, count);
+    return;
+  }
+  if (count <= 8)
+    return SplatSet<HeadTail<_4>>(dst, value, count);
+  if (count <= 16)
+    return SplatSet<HeadTail<_8>>(dst, value, count);
+  if (count <= 32)
+    return SplatSet<HeadTail<_16>>(dst, value, count);
+  if (count <= 96) {
+    SplatSet<_32>(dst, value);
+    if (count <= 64)
+      return SplatSet<Tail<_32>>(dst, value, count);
+    SplatSet<Skip<32>::Then<_32>>(dst, value);
+    SplatSet<Tail<_32>>(dst, value, count);
+    return;
+  }
+  if (count < 448 || value != 0 || !AArch64ZVA(dst, count))
+    return SplatSet<Align<_16, Arg::_1>::Then<Loop<_64>>>(dst, value, count);
 #else
+  /////////////////////////////////////////////////////////////////////////////
+  // Default
+  /////////////////////////////////////////////////////////////////////////////
   using namespace ::__llvm_libc::scalar;
-#endif
 
   if (count == 0)
     return;
   if (count == 1)
     return SplatSet<_1>(dst, value);
   if (count == 2)
     return SplatSet<_2>(dst, value);
   if (count == 3)
     return SplatSet<_3>(dst, value);
   if (count <= 8)
     return SplatSet<HeadTail<_4>>(dst, value, count);
   if (count <= 16)
     return SplatSet<HeadTail<_8>>(dst, value, count);
   if (count <= 32)
     return SplatSet<HeadTail<_16>>(dst, value, count);
   if (count <= 64)
     return SplatSet<HeadTail<_32>>(dst, value, count);
   if (count <= 128)
     return SplatSet<HeadTail<_64>>(dst, value, count);
   return SplatSet<Align<_32, Arg::Dst>::Then<Loop<_32>>>(dst, value, count);
+#endif
 }
 
 } // namespace __llvm_libc
 
-#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_UTILS_H
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H

libc/src/string/memset.cpp

 //===-- Implementation of memset ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/string/memset.h"
 #include "src/__support/common.h"
-#include "src/string/memory_utils/memset_utils.h"
+#include "src/string/memory_utils/memset_implementations.h"
 
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(void *, memset, (void *dst, int value, size_t count)) {
-  GeneralPurposeMemset(reinterpret_cast<char *>(dst),
-                       static_cast<unsigned char>(value), count);
+  inline_memset(reinterpret_cast<char *>(dst),
   return dst;
 }
 
 } // namespace __llvm_libc