diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt --- a/libc/CMakeLists.txt +++ b/libc/CMakeLists.txt @@ -21,6 +21,7 @@ include(CMakeParseArguments) include(LLVMLibCRules) +include(LLVMLibCCheckCpuFeatures) add_subdirectory(src) add_subdirectory(config) diff --git a/libc/cmake/modules/LLVMLibCCheckCpuFeatures.cmake b/libc/cmake/modules/LLVMLibCCheckCpuFeatures.cmake --- a/libc/cmake/modules/LLVMLibCCheckCpuFeatures.cmake +++ b/libc/cmake/modules/LLVMLibCCheckCpuFeatures.cmake @@ -69,15 +69,19 @@ _generate_flags_for_up_to(${feature} ${flag_name}) set(${flag_name} ${${flag_name}} PARENT_SCOPE) endforeach() + # Populates the CPU_FEATURE_OPT_LEVELS with all opt levels. + list(TRANSFORM opt_levels PREPEND "CPU_FEATURE_OPT_") + list(TRANSFORM opt_levels TOUPPER) + set(CPU_FEATURE_OPT_LEVELS ${opt_levels} PARENT_SCOPE) endfunction() _generate_opt_levels() #------------------------------------------------------------------------------ # Host cpu feature introspection -# -# Populates a HOST_CPU_FEATURES list containing the available CPU_FEATURE. #------------------------------------------------------------------------------ + +# Populates a HOST_CPU_FEATURES list containing the available CPU_FEATURE. function(_check_host_cpu_feature feature) string(TOLOWER ${feature} lowercase_feature) try_run( @@ -97,3 +101,33 @@ foreach(feature IN LISTS ALL_CPU_FEATURES) _check_host_cpu_feature(${feature}) endforeach() + +# Populates a HOST_CPU_FEATURE_OPT_LEVEL and HOST_CPU_FEATURE_OPT_FLAGS. +function(_check_host_opt_level) + if(HOST_CPU_FEATURES) + list(GET HOST_CPU_FEATURES -1 selected_feature) + else() + set(selected_feature NONE) + endif() + set(HOST_CPU_FEATURE_OPT_LEVEL "CPU_FEATURE_OPT_${selected_feature}" PARENT_SCOPE) + set(HOST_CPU_FEATURE_OPT_FLAGS ${${HOST_CPU_FEATURE_OPT_LEVEL}_FLAGS} PARENT_SCOPE) +endfunction() + +_check_host_opt_level() + +# Convenient function to generate unique name for a particular function, architecture +# and optimization level. +function(generate_function_name function arch opt_level output) + if(NOT function) + message(FATAL_ERROR "function name must not be empty") + endif() + if(NOT arch) + message(FATAL_ERROR "architecture name must not be empty") + endif() + if(NOT ${opt_level} IN_LIST CPU_FEATURE_OPT_LEVELS) + message(FATAL_ERROR "Invalid opt_level, should be one of ${CPU_FEATURE_OPT_LEVELS} was ${opt_level}") + endif() + string(REPLACE "CPU_FEATURE_" "" short_opt ${opt_level}) + string(TOLOWER ${short_opt} lower_short_opt) + set(${output} ${function}_${arch}_${lower_short_opt} PARENT_SCOPE) +endfunction() diff --git a/libc/cmake/modules/LLVMLibCRules.cmake b/libc/cmake/modules/LLVMLibCRules.cmake --- a/libc/cmake/modules/LLVMLibCRules.cmake +++ b/libc/cmake/modules/LLVMLibCRules.cmake @@ -105,13 +105,15 @@ # SRCS # HDRS # DEPENDS +# COMPILE_OPTIONS +# COMPILE_DEFINITIONS # ) function(add_entrypoint_object target_name) cmake_parse_arguments( "ADD_ENTRYPOINT_OBJ" "REDIRECTED" # Optional argument "NAME" # Single value arguments - "SRCS;HDRS;DEPENDS" # Multi value arguments + "SRCS;HDRS;DEPENDS;COMPILE_OPTIONS;COMPILE_DEFINITIONS" # Multi value arguments ${ARGN} ) if(NOT ADD_ENTRYPOINT_OBJ_SRCS) @@ -145,6 +147,18 @@ PRIVATE "${LIBC_BUILD_DIR}/include;${LIBC_SOURCE_DIR};${LIBC_BUILD_DIR}" ) + if(ADD_ENTRYPOINT_OBJ_COMPILE_OPTIONS) + target_compile_options( + ${target_name}_objects + PRIVATE ${ADD_ENTRYPOINT_OBJ_COMPILE_OPTIONS} + ) + endif() + if(ADD_ENTRYPOINT_OBJ_COMPILE_DEFINITIONS) + target_compile_definitions( + ${target_name}_objects + PRIVATE ${ADD_ENTRYPOINT_OBJ_COMPILE_DEFINITIONS} + ) + endif() add_dependencies( ${target_name}_objects support_common_h @@ -301,6 +315,8 @@ # SRCS # HDRS # DEPENDS +# COMPILE_OPTIONS +# COMPILE_DEFINITIONS # ) function(add_libc_unittest target_name) if(NOT LLVM_INCLUDE_TESTS) @@ -311,7 +327,7 @@ "LIBC_UNITTEST" "" # No optional arguments "SUITE" # Single value arguments - "SRCS;HDRS;DEPENDS" # Multi-value arguments + "SRCS;HDRS;DEPENDS;COMPILE_OPTIONS;COMPILE_DEFINITIONS" # Multi-value arguments ${ARGN} ) if(NOT LIBC_UNITTEST_SRCS) @@ -349,7 +365,18 @@ ${LIBC_BUILD_DIR} ${LIBC_BUILD_DIR}/include ) - + if(LIBC_UNITTEST_COMPILE_OPTIONS) + target_compile_options( + ${target_name} + PRIVATE ${LIBC_UNITTEST_COMPILE_OPTIONS} + ) + endif() + if(LIBC_UNITTEST_COMPILE_DEFINITIONS) + target_compile_definitions( + ${target_name} + PRIVATE ${LIBC_UNITTEST_COMPILE_DEFINITIONS} + ) + endif() if(library_deps) target_link_libraries(${target_name} PRIVATE ${library_deps}) endif() diff --git a/libc/src/string/CMakeLists.txt b/libc/src/string/CMakeLists.txt --- a/libc/src/string/CMakeLists.txt +++ b/libc/src/string/CMakeLists.txt @@ -1,17 +1,5 @@ add_subdirectory(memory_utils) -# Checking that the raw object does not contain any undefined symbols. -function(check_no_undefined_symbols target_name) -if(CMAKE_BUILD_TYPE MATCHES RELEASE) -add_custom_command( - TARGET ${target_name} - POST_BUILD - COMMENT "Check that the generated code is valid" - COMMAND ! (nm --undefined-only $ | grep U && echo "Implementation contains undefined symbols") -) -endif(CMAKE_BUILD_TYPE MATCHES RELEASE) -endfunction(check_no_undefined_symbols) - add_entrypoint_object( strcat SRCS @@ -45,48 +33,38 @@ memcpy_arch_specific_${LIBC_TARGET_MACHINE}.h.inc ) -add_custom_target(all_memcpy_implementations) -function(compile_memcpy_with_flags suffix) - cmake_parse_arguments( - "COMPILE_MEMCPY" - "" # No optional argument - "" # No single value arguments - "FLAGS" # Multi value arguments - ${ARGN} - ) - set(target_name memcpy_${LIBC_TARGET_MACHINE}_${suffix}) - add_library( - ${target_name} - OBJECT - memcpy_arch_specific - memcpy.cpp +foreach(opt_level ${CPU_FEATURE_OPT_LEVELS}) + generate_function_name(memcpy ${LIBC_TARGET_MACHINE} ${opt_level} memcpy_name) + add_entrypoint_object( + ${memcpy_name} + SRCS memcpy.cpp + HDRS memcpy.h + DEPENDS + string_h memory_utils + memcpy_arch_specific + COMPILE_OPTIONS + -fno-builtin-memcpy + ${${opt_level}_FLAGS} + COMPILE_DEFINITIONS + MEMCPY_NAME=${memcpy_name} ) - target_compile_definitions(${target_name} PRIVATE MEMCPY_NAME=${target_name}) - target_compile_options(${target_name} PRIVATE -fno-builtin-memcpy ${COMPILE_MEMCPY_FLAGS}) - target_include_directories(${target_name} PRIVATE "${LIBC_BUILD_DIR}/include;${LIBC_SOURCE_DIR};${LIBC_BUILD_DIR}") - add_dependencies(all_memcpy_implementations ${target_name}) - check_no_undefined_symbols(${target_name}) -endfunction(compile_memcpy_with_flags) - -if(${LIBC_TARGET_MACHINE} MATCHES x86|x86_64) - compile_memcpy_with_flags(unopt FLAGS -mno-sse) - compile_memcpy_with_flags(sse FLAGS -msse -mno-sse2) - compile_memcpy_with_flags(sse2 FLAGS -msse2 -mno-avx) - compile_memcpy_with_flags(avx FLAGS -mavx -mno-avx2) - compile_memcpy_with_flags(avx512 FLAGS -mavx512f) -endif() - -add_entrypoint_object( - memcpy - SRCS - memcpy_entrypoint.cpp - HDRS - memcpy.h - DEPENDS - string_h - memory_utils - memcpy_arch_specific -) -set_property(SOURCE memcpy.cpp PROPERTY COMPILE_OPTIONS -fno-builtin-memcpy) -check_no_undefined_symbols(memcpy) + if(${opt_level} STREQUAL ${HOST_CPU_FEATURE_OPT_LEVEL}) + add_entrypoint_object( + memcpy + NAME ${memcpy_name} + SRCS memcpy.cpp + HDRS memcpy.h + DEPENDS + string_h + memory_utils + memcpy_arch_specific + COMPILE_OPTIONS + -fno-builtin-memcpy + ${HOST_CPU_FEATURE_OPT_FLAGS} + COMPILE_DEFINITIONS + MEMCPY_NAME=${memcpy_name} + DECLARE_LLVM_LIBC_ENTRYPOINT + ) + endif() +endforeach() diff --git a/libc/src/string/memcpy.cpp b/libc/src/string/memcpy.cpp --- a/libc/src/string/memcpy.cpp +++ b/libc/src/string/memcpy.cpp @@ -6,4 +6,17 @@ // //===----------------------------------------------------------------------===// +#include "src/string/memcpy.h" +#include "src/__support/common.h" #include "src/string/memcpy_arch_specific.h" + +namespace __llvm_libc { + +#ifdef DECLARE_LLVM_LIBC_ENTRYPOINT +void *LLVM_LIBC_ENTRYPOINT(memcpy)(void *__restrict dst, + const void *__restrict src, size_t size) { + return MEMCPY_NAME(dst, src, size); +} +#endif // DECLARE_LLVM_LIBC_ENTRYPOINT + +} // namespace __llvm_libc diff --git a/libc/src/string/memcpy_arch_specific.h.def b/libc/src/string/memcpy_arch_specific.h.def --- a/libc/src/string/memcpy_arch_specific.h.def +++ b/libc/src/string/memcpy_arch_specific.h.def @@ -56,6 +56,10 @@ CopyGE128(dst, src, count); } +#ifndef MEMCPY_NAME +#error "MEMCPY_NAME is not defined" +#endif // MEMCPY_NAME + void * MEMCPY_NAME(void *__restrict dst, const void *__restrict src, size_t size) { memcpy_no_return(reinterpret_cast(dst), diff --git a/libc/src/string/memcpy_entrypoint.cpp b/libc/src/string/memcpy_entrypoint.cpp deleted file mode 100644 --- a/libc/src/string/memcpy_entrypoint.cpp +++ /dev/null @@ -1,20 +0,0 @@ -//===--------------------- Implementation of memcpy -----------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "src/string/memcpy.h" -#include "src/__support/common.h" -#include "src/string/memcpy_arch_specific.h" - -namespace __llvm_libc { - -void *LLVM_LIBC_ENTRYPOINT(memcpy)(void *__restrict dst, - const void *__restrict src, size_t size) { - return MEMCPY_NAME(dst, src, size); -} - -} // namespace __llvm_libc diff --git a/libc/test/src/string/CMakeLists.txt b/libc/test/src/string/CMakeLists.txt --- a/libc/test/src/string/CMakeLists.txt +++ b/libc/test/src/string/CMakeLists.txt @@ -23,12 +23,23 @@ strcpy ) -add_libc_unittest( - memcpy_test - SUITE - libc_string_unittests - SRCS - memcpy_test.cpp - DEPENDS - memcpy -) +# Tests all implementations of memcpy. +foreach(opt_level ${CPU_FEATURE_OPT_LEVELS}) + # FIXME: remove implementations that can't be tested on the host. + # This needs a notion of opt_level compatibility. + generate_function_name(memcpy ${LIBC_TARGET_MACHINE} ${opt_level} memcpy_name) + add_libc_unittest( + ${memcpy_name}_test + SUITE + libc_string_unittests + SRCS + memcpy_test.cpp + DEPENDS + ${memcpy_name} + COMPILE_OPTIONS + ${HOST_CPU_FEATURE_OPT_FLAGS} + COMPILE_DEFINITIONS + MEMCPY_NAME=${memcpy_name} + ) +endforeach() + diff --git a/libc/test/src/string/memcpy_test.cpp b/libc/test/src/string/memcpy_test.cpp --- a/libc/test/src/string/memcpy_test.cpp +++ b/libc/test/src/string/memcpy_test.cpp @@ -6,10 +6,19 @@ // //===----------------------------------------------------------------------===// -#include "src/string/memcpy.h" #include "utils/CPP/ArrayRef.h" #include "utils/UnitTest/Test.h" +#ifndef MEMCPY_NAME +#error "MEMCPY_NAME is not defined" +#endif // MEMCPY_NAME + +namespace __llvm_libc { + +void *MEMCPY_NAME(void *__restrict, const void *__restrict, size_t); + +} // namespace __llvm_libc + using __llvm_libc::cpp::Array; using __llvm_libc::cpp::ArrayRef; using __llvm_libc::cpp::MutableArrayRef; @@ -34,7 +43,7 @@ auto buffer = dirty; const char *const src = groundtruth.data(); char *const dst = &buffer[align]; - __llvm_libc::memcpy(dst, src, count); + __llvm_libc::MEMCPY_NAME(dst, src, count); // Everything before copy is untouched. for (size_t i = 0; i < align; ++i) ASSERT_EQ(buffer[i], dirty[i]);