Index: lib/scudo/scudo_allocator.cpp =================================================================== --- lib/scudo/scudo_allocator.cpp +++ lib/scudo/scudo_allocator.cpp @@ -15,7 +15,6 @@ //===----------------------------------------------------------------------===// #include "scudo_allocator.h" -#include "scudo_crc32.h" #include "scudo_utils.h" #include "sanitizer_common/sanitizer_allocator_interface.h" @@ -73,15 +72,20 @@ // at compilation or at runtime. static atomic_uint8_t HashAlgorithm = { CRC32Software }; -// Helper function that will compute the chunk checksum, being passed all the -// the needed information as uptrs. It will opt for the hardware version of -// the checksumming function if available. -INLINE u32 hashUptrs(uptr Pointer, uptr *Array, uptr ArraySize, u8 HashType) { - u32 Crc; - Crc = computeCRC32(Cookie, Pointer, HashType); - for (uptr i = 0; i < ArraySize; i++) - Crc = computeCRC32(Crc, Array[i], HashType); - return Crc; +SANITIZER_WEAK_ATTRIBUTE u32 computeHardwareCRC32(u32 Crc, uptr Data); + +INLINE u32 computeCRC32(u32 Crc, uptr Data, u8 HashType) { + // If SSE4.2 is defined here, it was enabled everywhere, as opposed to only + // for scudo_crc32.cpp. This means that other SSE instructions were likely + // emitted at other places, and as a result there is no reason to not use + // the hardware version of the CRC32. +#if defined(__SSE4_2__) + return computeHardwareCRC32(Crc, Data); +#else + if (computeHardwareCRC32 && HashType == CRC32Hardware) + return computeHardwareCRC32(Crc, Data); + return computeSoftwareCRC32(Crc, Data); +#endif // defined(__SSE4_2__) } struct ScudoChunk : UnpackedHeader { @@ -108,11 +112,11 @@ ZeroChecksumHeader.Checksum = 0; uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)]; memcpy(&HeaderHolder, &ZeroChecksumHeader, sizeof(HeaderHolder)); - u32 Hash = hashUptrs(reinterpret_cast(this), - HeaderHolder, - ARRAY_SIZE(HeaderHolder), - atomic_load_relaxed(&HashAlgorithm)); - return static_cast(Hash); + u8 HashType = atomic_load_relaxed(&HashAlgorithm); + u32 Crc = computeCRC32(Cookie, reinterpret_cast(this), HashType); + for (uptr i = 0; i < ARRAY_SIZE(HeaderHolder); i++) + Crc = computeCRC32(Crc, HeaderHolder[i], HashType); + return static_cast(Crc); } // Checks the validity of a chunk by verifying its checksum. Index: lib/scudo/scudo_crc32.h =================================================================== --- lib/scudo/scudo_crc32.h +++ /dev/null @@ -1,30 +0,0 @@ -//===-- scudo_crc32.h -------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// Header for scudo_crc32.cpp. -/// -//===----------------------------------------------------------------------===// - -#ifndef SCUDO_CRC32_H_ -#define SCUDO_CRC32_H_ - -#include "sanitizer_common/sanitizer_internal_defs.h" - -namespace __scudo { - -enum : u8 { - CRC32Software = 0, - CRC32Hardware = 1, -}; - -u32 computeCRC32(u32 Crc, uptr Data, u8 HashType); - -} // namespace __scudo - -#endif // SCUDO_CRC32_H_ Index: lib/scudo/scudo_crc32.cpp =================================================================== --- lib/scudo/scudo_crc32.cpp +++ lib/scudo/scudo_crc32.cpp @@ -12,13 +12,13 @@ /// //===----------------------------------------------------------------------===// +#include "sanitizer_common/sanitizer_internal_defs.h" + // Hardware CRC32 is supported at compilation via the following: // - for i386 & x86_64: -msse4.2 // - for ARM & AArch64: -march=armv8-a+crc or -mcrc // An additional check must be performed at runtime as well to make sure the // emitted instructions are valid on the target host. -#include "scudo_crc32.h" -#include "scudo_utils.h" #if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) # ifdef __SSE4_2__ @@ -34,20 +34,9 @@ namespace __scudo { #if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) -INLINE u32 computeHardwareCRC32(u32 Crc, uptr Data) { +u32 computeHardwareCRC32(u32 Crc, uptr Data) { return CRC32_INTRINSIC(Crc, Data); } - -u32 computeCRC32(u32 Crc, uptr Data, u8 HashType) { - if (HashType == CRC32Hardware) { - return computeHardwareCRC32(Crc, Data); - } - return computeSoftwareCRC32(Crc, Data); -} -#else -u32 computeCRC32(u32 Crc, uptr Data, u8 HashType) { - return computeSoftwareCRC32(Crc, Data); -} #endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) } // namespace __scudo Index: lib/scudo/scudo_utils.h =================================================================== --- lib/scudo/scudo_utils.h +++ lib/scudo/scudo_utils.h @@ -53,7 +53,11 @@ u64 State[2]; }; -// Software CRC32 functions, to be used when hardware support is not detected. +enum : u8 { + CRC32Software = 0, + CRC32Hardware = 1, +}; + u32 computeSoftwareCRC32(u32 Crc, uptr Data); } // namespace __scudo Index: lib/scudo/scudo_utils.cpp =================================================================== --- lib/scudo/scudo_utils.cpp +++ lib/scudo/scudo_utils.cpp @@ -20,8 +20,9 @@ #if defined(__x86_64__) || defined(__i386__) # include #endif - -#include +#if defined(__arm__) || defined(__aarch64__) +# include +#endif // TODO(kostyak): remove __sanitizer *Printf uses in favor for our own less // complicated string formatting code. The following is a @@ -97,6 +98,20 @@ } return false; } +#elif defined(__arm__) || defined(__aarch64__) +// For ARM and AArch64, hardware CRC32 support is indicated in the +// AT_HWVAL auxiliary vector. +bool testCPUFeature(CPUFeature Feature) { + static uptr HWCap = getauxval(AT_HWCAP); + + switch (Feature) { + case CRC32CPUFeature: + return !!(HWCap & HWCAP_CRC32); + default: + break; + } + return false; +} #else bool testCPUFeature(CPUFeature Feature) { return false;