diff --git a/compiler-rt/lib/scudo/standalone/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/CMakeLists.txt index 3b696385745a..3313283de06f 100644 --- a/compiler-rt/lib/scudo/standalone/CMakeLists.txt +++ b/compiler-rt/lib/scudo/standalone/CMakeLists.txt @@ -1,99 +1,100 @@ add_compiler_rt_component(scudo_standalone) include_directories(../..) set(SCUDO_CFLAGS) list(APPEND SCUDO_CFLAGS -Wall -nostdinc++) # Remove -stdlib= which is unused when passing -nostdinc++. string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) append_list_if(COMPILER_RT_HAS_FFREESTANDING_FLAG -ffreestanding SCUDO_CFLAGS) append_list_if(COMPILER_RT_HAS_FVISIBILITY_HIDDEN_FLAG -fvisibility=hidden SCUDO_CFLAGS) if(COMPILER_RT_DEBUG) list(APPEND SCUDO_CFLAGS -O0) else() list(APPEND SCUDO_CFLAGS -O3) endif() set(SCUDO_LINK_FLAGS) list(APPEND SCUDO_LINK_FLAGS -Wl,-z,defs,-z,now,-z,relro) append_list_if(COMPILER_RT_HAS_NODEFAULTLIBS_FLAG -nodefaultlibs SCUDO_LINK_FLAGS) if(ANDROID) # Put the shared library in the global group. For more details, see # android-changes-for-ndk-developers.md#changes-to-library-search-order append_list_if(COMPILER_RT_HAS_Z_GLOBAL -Wl,-z,global SCUDO_LINK_FLAGS) endif() set(SCUDO_SOURCES checksum.cc crc32_hw.cc common.cc flags.cc flags_parser.cc fuchsia.cc linux.cc report.cc secondary.cc string_utils.cc) # Enable the SSE 4.2 instruction set for crc32_hw.cc, if available. if (COMPILER_RT_HAS_MSSE4_2_FLAG) set_source_files_properties(crc32_hw.cc PROPERTIES COMPILE_FLAGS -msse4.2) endif() # Enable the AArch64 CRC32 feature for crc32_hw.cc, if available. # Note that it is enabled by default starting with armv8.1-a. if (COMPILER_RT_HAS_MCRC_FLAG) set_source_files_properties(crc32_hw.cc PROPERTIES COMPILE_FLAGS -mcrc) endif() set(SCUDO_HEADERS atomic_helpers.h bytemap.h checksum.h + chunk.h flags.h flags_parser.h fuchsia.h interface.h internal_defs.h linux.h list.h mutex.h platform.h quarantine.h release.h report.h secondary.h size_class_map.h stats.h string_utils.h vector.h) if(COMPILER_RT_HAS_SCUDO_STANDALONE) add_compiler_rt_object_libraries(RTScudoStandalone ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH} SOURCES ${SCUDO_SOURCES} ADDITIONAL_HEADERS ${SCUDO_HEADERS} CFLAGS ${SCUDO_CFLAGS}) add_compiler_rt_runtime(clang_rt.scudo_standalone STATIC ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH} SOURCES ${SCUDO_SOURCES} ADDITIONAL_HEADERS ${SCUDO_HEADERS} CFLAGS ${SCUDO_CFLAGS} PARENT_TARGET scudo_standalone) if(COMPILER_RT_INCLUDE_TESTS) add_subdirectory(tests) endif() endif() diff --git a/compiler-rt/lib/scudo/standalone/checksum.cc b/compiler-rt/lib/scudo/standalone/checksum.cc index ff6462bcdb1d..0896d5bdccd5 100644 --- a/compiler-rt/lib/scudo/standalone/checksum.cc +++ b/compiler-rt/lib/scudo/standalone/checksum.cc @@ -1,70 +1,70 @@ //===-- checksum.cc ---------------------------------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "checksum.h" #include "atomic_helpers.h" #if defined(__x86_64__) || defined(__i386__) #include #elif defined(__arm__) || defined(__aarch64__) #if SCUDO_FUCHSIA #include #include #else #include #endif #endif namespace scudo { -atomic_u8 HashAlgorithm = {BSDChecksum}; +Checksum HashAlgorithm = {Checksum::BSD}; #if defined(__x86_64__) || defined(__i386__) // i386 and x86_64 specific code to detect CRC32 hardware support via CPUID. // CRC32 requires the SSE 4.2 instruction set. #ifndef bit_SSE4_2 #define bit_SSE4_2 bit_SSE42 // clang and gcc have different defines. #endif bool hasHardwareCRC32() { u32 Eax, Ebx = 0, Ecx = 0, Edx = 0; __get_cpuid(0, &Eax, &Ebx, &Ecx, &Edx); const bool IsIntel = (Ebx == signature_INTEL_ebx) && (Edx == signature_INTEL_edx) && (Ecx == signature_INTEL_ecx); const bool IsAMD = (Ebx == signature_AMD_ebx) && (Edx == signature_AMD_edx) && (Ecx == signature_AMD_ecx); if (!IsIntel && !IsAMD) return false; __get_cpuid(1, &Eax, &Ebx, &Ecx, &Edx); return !!(Ecx & bit_SSE4_2); } #elif defined(__arm__) || defined(__aarch64__) #ifndef AT_HWCAP #define AT_HWCAP 16 #endif #ifndef HWCAP_CRC32 #define HWCAP_CRC32 (1U << 7) // HWCAP_CRC32 is missing on older platforms. #endif bool hasHardwareCRC32() { #if SCUDO_FUCHSIA u32 HWCap; const zx_status_t Status = zx_system_get_features(ZX_FEATURE_KIND_CPU, &HWCap); if (Status != ZX_OK) return false; return !!(HWCap & ZX_ARM64_FEATURE_ISA_CRC32); #else return !!(getauxval(AT_HWCAP) & HWCAP_CRC32); #endif // SCUDO_FUCHSIA } #endif // defined(__x86_64__) || defined(__i386__) } // namespace scudo diff --git a/compiler-rt/lib/scudo/standalone/checksum.h b/compiler-rt/lib/scudo/standalone/checksum.h index 7c4afcd965f7..092342fd6efb 100644 --- a/compiler-rt/lib/scudo/standalone/checksum.h +++ b/compiler-rt/lib/scudo/standalone/checksum.h @@ -1,54 +1,54 @@ //===-- checksum.h ----------------------------------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef SCUDO_CHECKSUM_H_ #define SCUDO_CHECKSUM_H_ #include "internal_defs.h" // Hardware CRC32 is supported at compilation via the following: // - for i386 & x86_64: -msse4.2 // - for ARM & AArch64: -march=armv8-a+crc or -mcrc // An additional check must be performed at runtime as well to make sure the // emitted instructions are valid on the target host. #ifdef __SSE4_2__ #include #define CRC32_INTRINSIC FIRST_32_SECOND_64(_mm_crc32_u32, _mm_crc32_u64) #endif #ifdef __ARM_FEATURE_CRC32 #include #define CRC32_INTRINSIC FIRST_32_SECOND_64(__crc32cw, __crc32cd) #endif namespace scudo { -enum ChecksumType : u8 { - BSDChecksum = 0, +enum class Checksum : u8 { + BSD = 0, HardwareCRC32 = 1, }; // BSD checksum, unlike a software CRC32, doesn't use any array lookup. We save // significantly on memory accesses, as well as 1K of CRC32 table, on platforms // that do no support hardware CRC32. The checksum itself is 16-bit, which is at // odds with CRC32, but enough for our needs. INLINE u16 computeBSDChecksum(u16 Sum, uptr Data) { for (u8 I = 0; I < sizeof(Data); I++) { Sum = static_cast((Sum >> 1) | ((Sum & 1) << 15)); Sum = static_cast(Sum + (Data & 0xff)); Data >>= 8; } return Sum; } bool hasHardwareCRC32(); WEAK u32 computeHardwareCRC32(u32 Crc, uptr Data); } // namespace scudo #endif // SCUDO_CHECKSUM_H_ diff --git a/compiler-rt/lib/scudo/standalone/chunk.h b/compiler-rt/lib/scudo/standalone/chunk.h new file mode 100644 index 000000000000..ec0b0ee8b214 --- /dev/null +++ b/compiler-rt/lib/scudo/standalone/chunk.h @@ -0,0 +1,162 @@ +//===-- chunk.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CHUNK_H_ +#define SCUDO_CHUNK_H_ + +#include "platform.h" + +#include "atomic_helpers.h" +#include "checksum.h" +#include "common.h" +#include "report.h" + +namespace scudo { + +extern Checksum HashAlgorithm; + +INLINE u16 computeChecksum(u32 Seed, uptr Value, uptr *Array, uptr ArraySize) { + // If the hardware CRC32 feature is defined here, it was enabled everywhere, + // as opposed to only for crc32_hw.cc. This means that other hardware specific + // instructions were likely emitted at other places, and as a result there is + // no reason to not use it here. +#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) + u32 Crc = static_cast(CRC32_INTRINSIC(Seed, Value)); + for (uptr I = 0; I < ArraySize; I++) + Crc = static_cast(CRC32_INTRINSIC(Crc, Array[I])); + return static_cast((Crc & 0xffff) ^ (Crc >> 16)); +#else + if (HashAlgorithm == Checksum::HardwareCRC32) { + u32 Crc = computeHardwareCRC32(Seed, Value); + for (uptr I = 0; I < ArraySize; I++) + Crc = computeHardwareCRC32(Crc, Array[I]); + return static_cast((Crc & 0xffff) ^ (Crc >> 16)); + } else { + u16 Checksum = computeBSDChecksum(static_cast(Seed & 0xffff), Value); + for (uptr I = 0; I < ArraySize; I++) + Checksum = computeBSDChecksum(Checksum, Array[I]); + return Checksum; + } +#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +} + +namespace Chunk { + +// Note that in an ideal world, `State` and `Origin` should be `enum class`, and +// the associated `UnpackedHeader` fields of their respective enum class type +// but https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61414 prevents it from +// happening, as it will error, complaining the number of bits is not enough. +enum Origin : u8 { + Malloc = 0, + New = 1, + NewArray = 2, + Memalign = 3, +}; + +enum State : u8 { Available = 0, Allocated = 1, Quarantined = 2 }; + +typedef u64 PackedHeader; +// Update the 'Mask' constants to reflect changes in this structure. +struct UnpackedHeader { + u64 Checksum : 16; + u64 ClassId : 8; + u64 SizeOrUnusedBytes : 20; + u8 State : 2; + u8 Origin : 2; + u64 Offset : 16; +}; +typedef atomic_u64 AtomicPackedHeader; +COMPILER_CHECK(sizeof(UnpackedHeader) == sizeof(PackedHeader)); + +// Those constants are required to silence some -Werror=conversion errors when +// assigning values to the related bitfield variables. +constexpr uptr ChecksumMask = (1UL << 16) - 1; +constexpr uptr ClassIdMask = (1UL << 8) - 1; +constexpr uptr SizeOrUnusedBytesMask = (1UL << 20) - 1; +constexpr uptr StateMask = (1UL << 2) - 1; +constexpr uptr OriginMask = (1UL << 2) - 1; +constexpr uptr OffsetMask = (1UL << 16) - 1; + +constexpr uptr getHeaderSize() { + return roundUpTo(sizeof(PackedHeader), 1U << SCUDO_MIN_ALIGNMENT_LOG); +} + +INLINE AtomicPackedHeader *getAtomicHeader(void *Ptr) { + return reinterpret_cast(reinterpret_cast(Ptr) - + getHeaderSize()); +} + +INLINE +const AtomicPackedHeader *getConstAtomicHeader(const void *Ptr) { + return reinterpret_cast( + reinterpret_cast(Ptr) - getHeaderSize()); +} + +INLINE void *getBlockBegin(const void *Ptr, UnpackedHeader *Header) { + return reinterpret_cast(reinterpret_cast(Ptr) - + getHeaderSize() - + (Header->Offset << SCUDO_MIN_ALIGNMENT_LOG)); +} + +// We do not need a cryptographically strong hash for the checksum, but a CRC +// type function that can alert us in the event a header is invalid or +// corrupted. Ideally slightly better than a simple xor of all fields. +static INLINE u16 computeHeaderChecksum(u32 Cookie, const void *Ptr, + UnpackedHeader *Header) { + UnpackedHeader ZeroChecksumHeader = *Header; + ZeroChecksumHeader.Checksum = 0; + uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)]; + memcpy(&HeaderHolder, &ZeroChecksumHeader, sizeof(HeaderHolder)); + return computeChecksum(Cookie, reinterpret_cast(Ptr), HeaderHolder, + ARRAY_SIZE(HeaderHolder)); +} + +INLINE void storeHeader(u32 Cookie, void *Ptr, + UnpackedHeader *NewUnpackedHeader) { + NewUnpackedHeader->Checksum = + computeHeaderChecksum(Cookie, Ptr, NewUnpackedHeader); + PackedHeader NewPackedHeader = bit_cast(*NewUnpackedHeader); + atomic_store_relaxed(getAtomicHeader(Ptr), NewPackedHeader); +} + +INLINE +void loadHeader(u32 Cookie, const void *Ptr, + UnpackedHeader *NewUnpackedHeader) { + PackedHeader NewPackedHeader = atomic_load_relaxed(getConstAtomicHeader(Ptr)); + *NewUnpackedHeader = bit_cast(NewPackedHeader); + if (UNLIKELY(NewUnpackedHeader->Checksum != + computeHeaderChecksum(Cookie, Ptr, NewUnpackedHeader))) + reportHeaderCorruption(const_cast(Ptr)); +} + +INLINE void compareExchangeHeader(u32 Cookie, void *Ptr, + UnpackedHeader *NewUnpackedHeader, + UnpackedHeader *OldUnpackedHeader) { + NewUnpackedHeader->Checksum = + computeHeaderChecksum(Cookie, Ptr, NewUnpackedHeader); + PackedHeader NewPackedHeader = bit_cast(*NewUnpackedHeader); + PackedHeader OldPackedHeader = bit_cast(*OldUnpackedHeader); + if (UNLIKELY(!atomic_compare_exchange_strong( + getAtomicHeader(Ptr), &OldPackedHeader, NewPackedHeader, + memory_order_relaxed))) + reportHeaderRace(Ptr); +} + +INLINE +bool isValid(u32 Cookie, const void *Ptr, UnpackedHeader *NewUnpackedHeader) { + PackedHeader NewPackedHeader = atomic_load_relaxed(getConstAtomicHeader(Ptr)); + *NewUnpackedHeader = bit_cast(NewPackedHeader); + return NewUnpackedHeader->Checksum == + computeHeaderChecksum(Cookie, Ptr, NewUnpackedHeader); +} + +} // namespace Chunk + +} // namespace scudo + +#endif // SCUDO_CHUNK_H_ diff --git a/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt index 548f371bf1f2..cbd87f393f43 100644 --- a/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt +++ b/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt @@ -1,70 +1,71 @@ include_directories(..) add_custom_target(ScudoUnitTests) set_target_properties(ScudoUnitTests PROPERTIES FOLDER "Compiler-RT Tests") set(SCUDO_UNITTEST_CFLAGS ${COMPILER_RT_UNITTEST_CFLAGS} ${COMPILER_RT_GTEST_CFLAGS} -I${COMPILER_RT_SOURCE_DIR}/include -I${COMPILER_RT_SOURCE_DIR}/lib -I${COMPILER_RT_SOURCE_DIR}/lib/scudo/standalone -DGTEST_HAS_RTTI=0) set(SCUDO_TEST_ARCH ${SCUDO_STANDALONE_SUPPORTED_ARCH}) # gtests requires c++ set(LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS}) foreach(lib ${SANITIZER_TEST_CXX_LIBRARIES}) list(APPEND LINK_FLAGS -l${lib}) endforeach() list(APPEND LINK_FLAGS -pthread) set(TEST_HEADERS) foreach (header ${SCUDO_HEADERS}) list(APPEND TEST_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../${header}) endforeach() # add_scudo_unittest( # SOURCES # HEADERS ) macro(add_scudo_unittest testname) cmake_parse_arguments(TEST "" "" "SOURCES;HEADERS" ${ARGN}) if(COMPILER_RT_HAS_SCUDO_STANDALONE) foreach(arch ${SCUDO_TEST_ARCH}) set(ScudoUnitTestsObjects) add_library("RTScudoStandalone.test.${arch}" STATIC $) generate_compiler_rt_tests(ScudoUnitTestsObjects ScudoUnitTests "${testname}-${arch}-Test" ${arch} SOURCES ${TEST_SOURCES} ${COMPILER_RT_GTEST_SOURCE} COMPILE_DEPS ${TEST_HEADERS} DEPS gtest scudo_standalone RUNTIME RTScudoStandalone.test.${arch} CFLAGS ${SCUDO_UNITTEST_CFLAGS} LINK_FLAGS ${LINK_FLAGS}) endforeach() endif() endmacro() set(SCUDO_UNIT_TEST_SOURCES atomic_test.cc bytemap_test.cc checksum_test.cc + chunk_test.cc flags_test.cc list_test.cc map_test.cc mutex_test.cc quarantine_test.cc release_test.cc report_test.cc secondary_test.cc size_class_map_test.cc stats_test.cc strings_test.cc vector_test.cc scudo_unit_test_main.cc) add_scudo_unittest(ScudoUnitTest SOURCES ${SCUDO_UNIT_TEST_SOURCES}) diff --git a/compiler-rt/lib/scudo/standalone/tests/chunk_test.cc b/compiler-rt/lib/scudo/standalone/tests/chunk_test.cc new file mode 100644 index 000000000000..fea975b847b6 --- /dev/null +++ b/compiler-rt/lib/scudo/standalone/tests/chunk_test.cc @@ -0,0 +1,82 @@ +//===-- chunk_test.cc -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "chunk.h" + +#include "gtest/gtest.h" + +#include + +static constexpr scudo::uptr HeaderSize = scudo::Chunk::getHeaderSize(); +static constexpr scudo::u32 Cookie = 0x41424344U; +static constexpr scudo::u32 InvalidCookie = 0x11223344U; + +static void initChecksum(void) { + if (&scudo::computeHardwareCRC32 && scudo::hasHardwareCRC32()) + scudo::HashAlgorithm = scudo::Checksum::HardwareCRC32; +} + +TEST(ScudoChunkTest, ChunkBasic) { + initChecksum(); + const scudo::uptr Size = 0x100U; + scudo::Chunk::UnpackedHeader Header = {}; + void *Block = malloc(HeaderSize + Size); + void *P = reinterpret_cast(reinterpret_cast(Block) + + HeaderSize); + scudo::Chunk::storeHeader(Cookie, P, &Header); + memset(P, 'A', Size); + EXPECT_EQ(scudo::Chunk::getBlockBegin(P, &Header), Block); + scudo::Chunk::loadHeader(Cookie, P, &Header); + EXPECT_TRUE(scudo::Chunk::isValid(Cookie, P, &Header)); + EXPECT_FALSE(scudo::Chunk::isValid(InvalidCookie, P, &Header)); + EXPECT_DEATH(scudo::Chunk::loadHeader(InvalidCookie, P, &Header), ""); + free(Block); +} + +TEST(ScudoChunkTest, ChunkCmpXchg) { + initChecksum(); + const scudo::uptr Size = 0x100U; + scudo::Chunk::UnpackedHeader OldHeader = {}; + OldHeader.Origin = scudo::Chunk::Origin::Malloc; + OldHeader.ClassId = 0x42U; + OldHeader.SizeOrUnusedBytes = Size; + OldHeader.State = scudo::Chunk::State::Allocated; + void *Block = malloc(HeaderSize + Size); + void *P = reinterpret_cast(reinterpret_cast(Block) + + HeaderSize); + scudo::Chunk::storeHeader(Cookie, P, &OldHeader); + memset(P, 'A', Size); + scudo::Chunk::UnpackedHeader NewHeader = OldHeader; + NewHeader.State = scudo::Chunk::State::Quarantined; + scudo::Chunk::compareExchangeHeader(Cookie, P, &NewHeader, &OldHeader); + NewHeader = {}; + EXPECT_TRUE(scudo::Chunk::isValid(Cookie, P, &NewHeader)); + EXPECT_EQ(NewHeader.State, scudo::Chunk::State::Quarantined); + EXPECT_FALSE(scudo::Chunk::isValid(InvalidCookie, P, &NewHeader)); + free(Block); +} + +TEST(ScudoChunkTest, CorruptHeader) { + initChecksum(); + const scudo::uptr Size = 0x100U; + scudo::Chunk::UnpackedHeader Header = {}; + void *Block = malloc(HeaderSize + Size); + void *P = reinterpret_cast(reinterpret_cast(Block) + + HeaderSize); + scudo::Chunk::storeHeader(Cookie, P, &Header); + memset(P, 'A', Size); + EXPECT_EQ(scudo::Chunk::getBlockBegin(P, &Header), Block); + scudo::Chunk::loadHeader(Cookie, P, &Header); + // Simulate a couple of corrupted bits per byte of header data. + for (scudo::uptr I = 0; I < sizeof(scudo::Chunk::PackedHeader); I++) { + *(reinterpret_cast(Block) + I) ^= 0x42U; + EXPECT_DEATH(scudo::Chunk::loadHeader(Cookie, P, &Header), ""); + *(reinterpret_cast(Block) + I) ^= 0x42U; + } + free(Block); +}