[libc] Add redirected malloc using system-libc.

[cheng.w]

Summary

1. Add a redirected malloc referred to [1].

It is a temporary scheme to make memory management functions work.
[2] is a recent report published by mimalloc, in which leading memory
allocators were evaluated.

2. Add an optional argument REDIRECTED to add_libc_unittest.

Unit tests do not depend on libllvmlibc.a static library which
links libllvmlibc_redirectors.so shared library. [3]

[1] https://reviews.llvm.org/D69020
[2] https://www.microsoft.com/en-us/research/uploads/prod/2019/06/mimalloc-tr-v1.pdf
[3] https://github.com/llvm/llvm-project/blob/main/libc/docs/redirectors_schematic.svg

[cheng.w]

libc/cmake/modules/LLVMLibCTestRules.cmake

148–163

Also tried here:

if(LIBC_UNITTEST_REDIRECTED)
#   find_library(
#     libc_redirectors
#     llvmlibc_redirectors
#     PATHS ${LIBC_BUILD_DIR}/lib
#   )
    target_link_libraries(
      ${fq_target_name}
      PRIVATE
#     ${libc_redirectors}
+     llvmlibc_redirectors
    )
    add_dependencies(
      ${fq_target_name}
      llvmlibc_redirectors
    )
endif()

But malloc unit test failed for passing -nostdlib below to linker.

libc/cmake/modules/LLVMLibCLibraryRules.cmake, line: 142
target_link_libraries(
  ${target_name}
  -nostdlib -lc -lm
)

Would it work fine after the TODO(libc/cmake/modules/LLVMLibCLibraryRules.cmake, line: 133) fixed?

[sivachandra]

Hi,

Sorry for the delay here. The reason for the delay is that, for LLVM-libc's malloc, the plan to is to use Scudo's standalone malloc from here: https://github.com/llvm/llvm-project/tree/main/compiler-rt/lib/scudo/standalone

The high level plan is to keep the the standalone Scudo sources in the compiler-rt area but pull them via some kind of CMake machinery to package them for LLVM libc. This way, we don't add target dependency on compiler-rt, but if they break LLVM libc in some manner, the LLVM libc bots will catch the failures.

I had planned to pick this up in the second half of February. But, if this is something which interests you, go for it.

Thanks,
Siva Chandra

[cheng.w]

Good to hear this plan. 🤗

libc/cmake/modules/LLVMLibCTestRules.cmake

function(get_object_files_for_test result skipped_entrypoints_list)
   list(REMOVE_DUPLICATES skipped_list)
   set(${skipped_entrypoints_list} ${skipped_list} PARENT_SCOPE)
 endfunction(get_object_files_for_test)
 
 # Rule to add a libc unittest.
 # Usage
 #    add_libc_unittest(
 #      <target name>
+#      [REDIRECTED] <Specified if dependencies need libllvmlibc_redirectors.so>
 #      SUITE <name of the suite this test belongs to>
 #      SRCS  <list of .cpp files for the test>
 #      HDRS  <list of .h files for the test>
 #      DEPENDS <list of dependencies>
 #      COMPILE_OPTIONS <list of special compile options for this target>
 #    )
 function(add_libc_unittest target_name)
   if(NOT LLVM_INCLUDE_TESTS)
     return()
   endif()
 
   cmake_parse_arguments(
     "LIBC_UNITTEST"
-    "" # No optional arguments
+    "REDIRECTED" # Optional arguments
     "SUITE" # Single value arguments
     "SRCS;HDRS;DEPENDS;COMPILE_OPTIONS" # Multi-value arguments
     ${ARGN}
   )
   if(NOT LIBC_UNITTEST_SRCS)
     message(FATAL_ERROR "'add_libc_unittest' target requires a SRCS list of .cpp "
                         "files.")
   endif()
function(add_libc_unittest target_name)
   set_target_properties(${fq_target_name}
     PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 
   add_dependencies(
     ${fq_target_name}
     ${fq_deps_list}
   )
 
+  if(LIBC_UNITTEST_REDIRECTED)
+    find_library(
+      libc_redirectors
+      llvmlibc_redirectors
+      PATHS ${LIBC_BUILD_DIR}/lib
+    )
+    target_link_libraries(
+      ${fq_target_name}
+      PRIVATE
+      ${libc_redirectors}
+    )
+    add_dependencies(
+      ${fq_target_name}
+      llvmlibc_redirectors
+    )
+  endif()
   target_link_libraries(${fq_target_name} PRIVATE LibcUnitTest libc_test_utils)
 
   add_custom_command(
     TARGET ${fq_target_name}
     POST_BUILD
     COMMAND $<TARGET_FILE:${fq_target_name}>
   )
   if(LIBC_UNITTEST_SUITE)

libc/config/linux/aarch64/entrypoints.txt

set(TARGET_LIBC_ENTRYPOINTS
     libc.src.fenv.fesetround
     libc.src.fenv.feraiseexcept
     libc.src.fenv.fetestexcept
 
     # stdlib.h entrypoints
     libc.src.stdlib.abs
     libc.src.stdlib.labs
     libc.src.stdlib.llabs
+    libc.src.stdlib.malloc
 
     # string.h entrypoints
     libc.src.string.bzero
     libc.src.string.memchr
     libc.src.string.memcmp
     libc.src.string.memcpy
     libc.src.string.memmove
     libc.src.string.memset

libc/config/linux/api.td

 def StdIOAPI : PublicAPI<"stdio.h"> {
   let TypeDeclarations = [
     SizeT,
     FILE,
   ];
 }
 
 def StdlibAPI : PublicAPI<"stdlib.h"> {
+  let TypeDeclarations = [
+    SizeT,
+  ];
 }
 
 def TimeAPI : PublicAPI<"time.h"> {
   let TypeDeclarations = [
     StructTm,
     TimeT,
   ];
 

libc/config/linux/x86_64/entrypoints.txt

set(TARGET_LIBC_ENTRYPOINTS
     libc.src.signal.signal
 
     # stdlib.h entrypoints
     libc.src.stdlib._Exit
     libc.src.stdlib.abort
     libc.src.stdlib.abs
     libc.src.stdlib.labs
     libc.src.stdlib.llabs
+    libc.src.stdlib.malloc
 
     # string.h entrypoints
     libc.src.string.bzero
     libc.src.string.memchr
     libc.src.string.memcmp
     libc.src.string.memcpy
     libc.src.string.memmove
     libc.src.string.memrchr

libc/lib/CMakeLists.txt

 add_entrypoint_library(
   llvmlibc
   DEPENDS
   ${TARGET_LIBC_ENTRYPOINTS}
 )
 
 add_entrypoint_library(
   llvmlibm
   DEPENDS
   ${TARGET_LIBM_ENTRYPOINTS}
 )
+
+add_redirector_library(
+  llvmlibc_redirectors
+  DEPENDS
+    malloc_redirector
+)

libc/spec/stdc.td

HeaderSpec StdIO = HeaderSpec<
               ArgSpec<FILERestrictedPtr>]
           >,
       ]
   >;
 
   HeaderSpec StdLib = HeaderSpec<
       "stdlib.h",
       [], // Macros
-      [], // Types
+      [ // Types
+          SizeTType,
+      ],
       [], // Enumerations
       [
           FunctionSpec<"abort", RetValSpec<NoReturn>, [ArgSpec<VoidType>]>,
           FunctionSpec<"abs", RetValSpec<IntType>, [ArgSpec<IntType>]>,
           FunctionSpec<"labs", RetValSpec<LongType>, [ArgSpec<LongType>]>,
           FunctionSpec<"llabs", RetValSpec<LongLongType>, [ArgSpec<LongLongType>]>,
+          FunctionSpec<"malloc", RetValSpec<VoidPtr>, [ArgSpec<SizeTType>]>,
           FunctionSpec<"_Exit", RetValSpec<NoReturn>, [ArgSpec<IntType>]>,
       ]
   >;
 
   HeaderSpec Errno = HeaderSpec<
       "errno.h",
       [
         Macro<"errno">,

libc/src/stdlib/CMakeLists.txt

add_entrypoint_object(
   llabs
   SRCS
     llabs.cpp
   HDRS
     llabs.h
   DEPENDS
     .abs_utils
 )
+
+add_entrypoint_object(
+  malloc
+  REDIRECTED
+  SRCS
+    malloc.cpp
+  HDRS
+    malloc.h
+)
+
+add_redirector_object(
+  malloc_redirector
+  SRC
+    malloc_redirector.cpp
+)

libc/src/stdlib/malloc.h

+//===-- Implementation header for malloc ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STDLIB_MALLOC_H
+#define LLVM_LIBC_SRC_STDLIB_MALLOC_H
+
+#include <stddef.h> // size_t
+
+namespace __llvm_libc {
+
+void *malloc(size_t size);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STDLIB_MALLOC_H

libc/src/stdlib/malloc.cpp

+//===-- Implementation of malloc ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// TODO: Add implementation of malloc and remove malloc redirector.
+
+#include "src/stdlib/malloc.h"
+#include "src/__support/common.h"
+#include <stddef.h> // size_t
+
+namespace __llvm_libc {
+
+void *__malloc_redirector(size_t size);
+
+LLVM_LIBC_FUNCTION(void *, malloc, (size_t size)) {
+  return __malloc_redirector(size);
+}
+
+} // namespace __llvm_libc

libc/src/stdlib/malloc_redirector.cpp

+//===-- Implementation of malloc redirector -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// TODO: Remove the malloc redirector once LLVM-libc has the implementation
+// for malloc.
+
+#include <cstdlib>  // malloc
+#include <stddef.h> // size_t
+
+namespace __llvm_libc {
+
+void *__malloc_redirector(size_t size) { return ::malloc(size); }
+
+} // namespace __llvm_libc

libc/test/src/stdlib/CMakeLists.txt

add_libc_unittest(
   llabs_test
   SUITE
     libc_stdlib_unittests
   SRCS
     llabs_test.cpp
   DEPENDS
     libc.src.stdlib.llabs
 )
+
+add_libc_unittest(
+  malloc_test
+  REDIRECTED
+  SUITE
+    libc_stdlib_unittests
+  SRCS
+    malloc_test.cpp
+  DEPENDS
+    libc.src.stdlib.malloc
+    libc.src.stdlib.free
+)

libc/test/src/stdlib/malloc_test.cpp

+//===-- Unittests for malloc ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/stdlib/free.h"
+#include "src/stdlib/malloc.h"
+#include "utils/UnitTest/Test.h"
+
+TEST(LlvmLibcMallocTest, MallocInt) {
+  int *pi = static_cast<int *>(__llvm_libc::malloc(sizeof(int)));
+  *pi = 1;
+  EXPECT_EQ(*pi, 1);
+  __llvm_libc::free(pi);
+}
+
+// Making sure the result returned from malloc(0) could be freed.
+TEST(LlvmLibcMallocTest, MallocZero) {
+  void *p = __llvm_libc::malloc(0);
+  __llvm_libc::free(p);
+}