Index: cmake/config-ix.cmake =================================================================== --- cmake/config-ix.cmake +++ cmake/config-ix.cmake @@ -29,6 +29,7 @@ check_cxx_compiler_flag(-ftls-model=initial-exec COMPILER_RT_HAS_FTLS_MODEL_INITIAL_EXEC) check_cxx_compiler_flag(-fno-lto COMPILER_RT_HAS_FNO_LTO_FLAG) check_cxx_compiler_flag("-Werror -msse3" COMPILER_RT_HAS_MSSE3_FLAG) +check_cxx_compiler_flag("-Werror -msse4.2" COMPILER_RT_HAS_MSSE4_2_FLAG) check_cxx_compiler_flag(--sysroot=. COMPILER_RT_HAS_SYSROOT_FLAG) if(NOT WIN32 AND NOT CYGWIN) Index: lib/scudo/CMakeLists.txt =================================================================== --- lib/scudo/CMakeLists.txt +++ lib/scudo/CMakeLists.txt @@ -3,16 +3,23 @@ include_directories(..) set(SCUDO_CFLAGS ${SANITIZER_COMMON_CFLAGS}) +# SANITIZER_COMMON_CFLAGS include -fno-builtin, but we actually want builtins! +list(APPEND SCUDO_CFLAGS -fbuiltin) append_rtti_flag(OFF SCUDO_CFLAGS) set(SCUDO_SOURCES scudo_allocator.cpp scudo_flags.cpp + scudo_crc32.cpp scudo_interceptors.cpp scudo_new_delete.cpp scudo_termination.cpp scudo_utils.cpp) +if (COMPILER_RT_HAS_MSSE4_2_FLAG) + set_source_files_properties(scudo_crc32.cpp PROPERTIES COMPILE_FLAGS -msse4.2) +endif() + if(COMPILER_RT_HAS_SCUDO) foreach(arch ${SCUDO_SUPPORTED_ARCH}) add_compiler_rt_runtime(clang_rt.scudo Index: lib/scudo/scudo_allocator.cpp =================================================================== --- lib/scudo/scudo_allocator.cpp +++ lib/scudo/scudo_allocator.cpp @@ -15,6 +15,7 @@ //===----------------------------------------------------------------------===// #include "scudo_allocator.h" +#include "scudo_crc32.h" #include "scudo_utils.h" #include "sanitizer_common/sanitizer_allocator_interface.h" @@ -25,22 +26,6 @@ #include -// Hardware CRC32 is supported at compilation via the following: -// - for i386 & x86_64: -msse4.2 -// - for ARM & AArch64: -march=armv8-a+crc -// An additional check must be performed at runtime as well to make sure the -// emitted instructions are valid on the target host. -#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) -# ifdef __SSE4_2__ -# include -# define HW_CRC32 FIRST_32_SECOND_64(_mm_crc32_u32, _mm_crc32_u64) -# endif -# ifdef __ARM_FEATURE_CRC32 -# include -# define HW_CRC32 FIRST_32_SECOND_64(__crc32cw, __crc32cd) -# endif -#endif - namespace __scudo { #if SANITIZER_CAN_USE_ALLOCATOR64 @@ -84,10 +69,6 @@ // Global static cookie, initialized at start-up. static uptr Cookie; -enum : u8 { - CRC32Software = 0, - CRC32Hardware = 1, -}; // We default to software CRC32 if the alternatives are not supported, either // at compilation or at runtime. static atomic_uint8_t HashAlgorithm = { CRC32Software }; @@ -97,18 +78,11 @@ // the checksumming function if available. INLINE u32 hashUptrs(uptr Pointer, uptr *Array, uptr ArraySize, u8 HashType) { u32 Crc; -#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) - if (HashType == CRC32Hardware) { - Crc = HW_CRC32(Cookie, Pointer); - for (uptr i = 0; i < ArraySize; i++) - Crc = HW_CRC32(Crc, Array[i]); - return Crc; - } -#endif - Crc = computeCRC32(Cookie, Pointer); + Crc = computeCRC32(Cookie, Pointer, HashType); for (uptr i = 0; i < ArraySize; i++) - Crc = computeCRC32(Crc, Array[i]); + Crc = computeCRC32(Crc, Array[i], HashType); return Crc; + } struct ScudoChunk : UnpackedHeader { Index: lib/scudo/scudo_crc32.h =================================================================== --- /dev/null +++ lib/scudo/scudo_crc32.h @@ -0,0 +1,30 @@ +//===-- scudo_crc32.h -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Header for scudo_crc32.cpp. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CRC32_H_ +#define SCUDO_CRC32_H_ + +#include "sanitizer_common/sanitizer_internal_defs.h" + +namespace __scudo { + +enum : u8 { + CRC32Software = 0, + CRC32Hardware = 1, +}; + +u32 computeCRC32(u32 Crc, uptr Data, u8 HashType); + +} // namespace __scudo + +#endif // SCUDO_CRC32_H_ Index: lib/scudo/scudo_crc32.cpp =================================================================== --- /dev/null +++ lib/scudo/scudo_crc32.cpp @@ -0,0 +1,53 @@ +//===-- scudo_crc32.cpp -----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// CRC32 function leveraging hardware specific instructions. This has to be +/// kept separated to restrict the use of compiler specific flags to this file. +/// +//===----------------------------------------------------------------------===// + +// Hardware CRC32 is supported at compilation via the following: +// - for i386 & x86_64: -msse4.2 +// - for ARM & AArch64: -march=armv8-a+crc or -mcrc +// An additional check must be performed at runtime as well to make sure the +// emitted instructions are valid on the target host. +#include "scudo_crc32.h" +#include "scudo_utils.h" + +#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +# ifdef __SSE4_2__ +# include +# define CRC32_INTRINSIC FIRST_32_SECOND_64(_mm_crc32_u32, _mm_crc32_u64) +# endif +# ifdef __ARM_FEATURE_CRC32 +# include +# define CRC32_INTRINSIC FIRST_32_SECOND_64(__crc32cw, __crc32cd) +# endif +#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) + +namespace __scudo { + +#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +INLINE u32 computeHardwareCRC32(u32 Crc, uptr Data) { + return CRC32_INTRINSIC(Crc, Data); +} + +u32 computeCRC32(u32 Crc, uptr Data, u8 HashType) { + if (HashType == CRC32Hardware) { + return computeHardwareCRC32(Crc, Data); + } + return computeSoftwareCRC32(Crc, Data); +} +#else +u32 computeCRC32(u32 Crc, uptr Data, u8 HashType) { + return computeSoftwareCRC32(Crc, Data); +} +#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) + +} // namespace __scudo Index: lib/scudo/scudo_utils.h =================================================================== --- lib/scudo/scudo_utils.h +++ lib/scudo/scudo_utils.h @@ -53,8 +53,8 @@ u64 State[2]; }; -// Software CRC32 functions, to be used when SSE 4.2 support is not detected. -u32 computeCRC32(u32 Crc, uptr Data); +// Software CRC32 functions, to be used when hardware support is not detected. +u32 computeSoftwareCRC32(u32 Crc, uptr Data); } // namespace __scudo Index: lib/scudo/scudo_utils.cpp =================================================================== --- lib/scudo/scudo_utils.cpp +++ lib/scudo/scudo_utils.cpp @@ -185,8 +185,7 @@ 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d }; -u32 computeCRC32(u32 Crc, uptr Data) -{ +u32 computeSoftwareCRC32(u32 Crc, uptr Data) { for (uptr i = 0; i < sizeof(Data); i++) { Crc = CRC32Table[(Crc ^ Data) & 0xff] ^ (Crc >> 8); Data >>= 8;