diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -816,8 +816,12 @@ } if (SanArgs.needsTsanRt() && SanArgs.linkRuntimes()) SharedRuntimes.push_back("tsan"); - if (SanArgs.needsHwasanRt() && SanArgs.linkRuntimes()) - SharedRuntimes.push_back("hwasan"); + if (SanArgs.needsHwasanRt() && SanArgs.linkRuntimes()) { + if (SanArgs.needsHwasanAliasesRt()) + SharedRuntimes.push_back("hwasan_aliases"); + else + SharedRuntimes.push_back("hwasan"); + } } // The stats_client library is also statically linked into DSOs. @@ -847,9 +851,15 @@ } if (!SanArgs.needsSharedRt() && SanArgs.needsHwasanRt() && SanArgs.linkRuntimes()) { - StaticRuntimes.push_back("hwasan"); - if (SanArgs.linkCXXRuntimes()) - StaticRuntimes.push_back("hwasan_cxx"); + if (SanArgs.needsHwasanAliasesRt()) { + StaticRuntimes.push_back("hwasan_aliases"); + if (SanArgs.linkCXXRuntimes()) + StaticRuntimes.push_back("hwasan_aliases_cxx"); + } else { + StaticRuntimes.push_back("hwasan"); + if (SanArgs.linkCXXRuntimes()) + StaticRuntimes.push_back("hwasan_cxx"); + } } if (SanArgs.needsDfsanRt() && SanArgs.linkRuntimes()) StaticRuntimes.push_back("dfsan"); diff --git a/compiler-rt/lib/hwasan/CMakeLists.txt b/compiler-rt/lib/hwasan/CMakeLists.txt --- a/compiler-rt/lib/hwasan/CMakeLists.txt +++ b/compiler-rt/lib/hwasan/CMakeLists.txt @@ -93,6 +93,24 @@ CFLAGS ${HWASAN_DYNAMIC_CFLAGS} DEFS ${HWASAN_DEFINITIONS}) +# Compile a different runtime for x86 aliasing mode. +set(HWASAN_ALIASES_RTL_CFLAGS ${HWASAN_RTL_CFLAGS}) +list(APPEND HWASAN_ALIASES_RTL_CFLAGS -DHWASAN_ALIASING_MODE) +set(HWASAN_ALIASES_DYNAMIC_CFLAGS ${HWASAN_DYNAMIC_CFLAGS}) +list(APPEND HWASAN_ALIASES_DYNAMIC_CFLAGS -DHWASAN_ALIASING_MODE) +add_compiler_rt_object_libraries(RTHwasanAliases + ARCHS ${HWASAN_SUPPORTED_ARCH} + SOURCES ${HWASAN_RTL_SOURCES} + ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS} + CFLAGS ${HWASAN_ALIASES_RTL_CFLAGS} + DEFS ${HWASAN_DEFINITIONS}) +add_compiler_rt_object_libraries(RTHwasanAliases_dynamic + ARCHS ${HWASAN_SUPPORTED_ARCH} + SOURCES ${HWASAN_RTL_SOURCES} ${HWASAN_RTL_CXX_SOURCES} + ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS} + CFLAGS ${HWASAN_ALIASES_DYNAMIC_CFLAGS} + DEFS ${HWASAN_DEFINITIONS}) + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "") add_compiler_rt_object_libraries(RTHwasan_dynamic_version_script_dummy ARCHS ${HWASAN_SUPPORTED_ARCH} @@ -100,47 +118,61 @@ CFLAGS ${HWASAN_DYNAMIC_CFLAGS} DEFS ${HWASAN_DEFINITIONS}) -foreach(arch ${HWASAN_SUPPORTED_ARCH}) - add_compiler_rt_runtime(clang_rt.hwasan +# If use_aliases is TRUE, adds the HWASan runtime built with alias support. +# Otherwise adds the runtime without alias support. +function(add_hwasan_runtimes arch use_aliases) + set(hwasan_object_lib RTHwasan) + set(hwasan_object_dyn_lib RTHwasan_dynamic) + set(hwasan_runtime clang_rt.hwasan) + set(hwasan_rtl_flags ${HWASAN_RTL_CFLAGS}) + set(hwasan_dyn_flags ${HWASAN_DYNAMIC_CFLAGS}) + if(use_aliases) + set(hwasan_object_lib RTHwasanAliases) + set(hwasan_object_dyn_lib RTHwasanAliases_dynamic) + set(hwasan_runtime clang_rt.hwasan_aliases) + set(hwasan_rtl_flags ${HWASAN_ALIASES_RTL_CFLAGS}) + set(hwasan_dyn_flags ${HWASAN_ALIASES_DYNAMIC_CFLAGS}) + endif() + add_compiler_rt_runtime(${hwasan_runtime} STATIC ARCHS ${arch} - OBJECT_LIBS RTHwasan + OBJECT_LIBS ${hwasan_object_lib} RTInterception RTSanitizerCommon RTSanitizerCommonLibc RTSanitizerCommonCoverage RTSanitizerCommonSymbolizer RTUbsan - CFLAGS ${HWASAN_RTL_CFLAGS} + CFLAGS ${hwasan_rtl_flags} PARENT_TARGET hwasan) - add_compiler_rt_runtime(clang_rt.hwasan_cxx + add_compiler_rt_runtime(${hwasan_runtime}_cxx STATIC ARCHS ${arch} OBJECT_LIBS RTHwasan_cxx RTUbsan_cxx - CFLAGS ${HWASAN_RTL_CFLAGS} + CFLAGS ${hwasan_rtl_flags} PARENT_TARGET hwasan) if (UNIX) - add_sanitizer_rt_version_list(clang_rt.hwasan-dynamic-${arch} - LIBS clang_rt.hwasan-${arch} clang_rt.hwasan_cxx-${arch} + add_sanitizer_rt_version_list(${hwasan_runtime}-dynamic-${arch} + LIBS ${hwasan_runtime}-${arch} ${hwasan_runtime}_cxx-${arch} EXTRA hwasan.syms.extra) set(VERSION_SCRIPT_FLAG - -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/clang_rt.hwasan-dynamic-${arch}.vers) + -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/${hwasan_runtime}-dynamic-${arch}.vers) set_property(SOURCE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp APPEND PROPERTY - OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/clang_rt.hwasan-dynamic-${arch}.vers) + OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${hwasan_runtime}-dynamic-${arch}.vers) else() set(VERSION_SCRIPT_FLAG) endif() - add_compiler_rt_runtime(clang_rt.hwasan + add_compiler_rt_runtime(${hwasan_runtime} SHARED ARCHS ${arch} OBJECT_LIBS - RTHwasan_dynamic + ${hwasan_object_dyn_lib} RTInterception RTSanitizerCommon RTSanitizerCommonLibc @@ -154,7 +186,7 @@ # add_dependencies(clang_rt.asan-dynamic-${arch} clang_rt.asan-dynamic-${arch}-version-list) # generates an order-only dependency in ninja. RTHwasan_dynamic_version_script_dummy - CFLAGS ${HWASAN_DYNAMIC_CFLAGS} + CFLAGS ${hwasan_dyn_flags} LINK_FLAGS ${HWASAN_DYNAMIC_LINK_FLAGS} ${VERSION_SCRIPT_FLAG} LINK_LIBS ${HWASAN_DYNAMIC_LIBS} @@ -162,14 +194,21 @@ PARENT_TARGET hwasan) if(SANITIZER_USE_SYMBOLS) - add_sanitizer_rt_symbols(clang_rt.hwasan + add_sanitizer_rt_symbols(${hwasan_runtime} ARCHS ${arch} EXTRA hwasan.syms.extra) - add_sanitizer_rt_symbols(clang_rt.hwasan_cxx + add_sanitizer_rt_symbols(${hwasan_runtime}_cxx ARCHS ${arch} EXTRA hwasan.syms.extra) - add_dependencies(hwasan clang_rt.hwasan-${arch}-symbols - clang_rt.hwasan_cxx-${arch}-symbols) + add_dependencies(hwasan ${hwasan_runtime}-${arch}-symbols + ${hwasan_runtime}_cxx-${arch}-symbols) + endif() +endfunction() + +foreach(arch ${HWASAN_SUPPORTED_ARCH}) + add_hwasan_runtimes(${arch} FALSE) + if(${arch} MATCHES "x86_64") + add_hwasan_runtimes(${arch} TRUE) endif() endforeach() diff --git a/compiler-rt/lib/hwasan/hwasan.h b/compiler-rt/lib/hwasan/hwasan.h --- a/compiler-rt/lib/hwasan/hwasan.h +++ b/compiler-rt/lib/hwasan/hwasan.h @@ -36,7 +36,10 @@ typedef u8 tag_t; -#if defined(__x86_64__) +#if defined(HWASAN_ALIASING_MODE) +# if !defined(__x86_64__) +# error Aliasing mode is only supported on x86_64 +# endif // Tags are done in middle bits using userspace aliasing. constexpr unsigned kAddressTagShift = 39; constexpr unsigned kTagBits = 3; @@ -49,12 +52,16 @@ // simpler/faster shadow calculation. constexpr unsigned kTaggableRegionCheckShift = __sanitizer::Max(kAddressTagShift + kTagBits + 1U, 44U); +#elif defined(__x86_64__) +// Tags are done in upper bits using Intel LAM. +constexpr unsigned kAddressTagShift = 57; +constexpr unsigned kTagBits = 6; #else // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address // translation and can be used to store a tag. constexpr unsigned kAddressTagShift = 56; constexpr unsigned kTagBits = 8; -#endif // defined(__x86_64__) +#endif // defined(HWASAN_ALIASING_MODE) // Mask for extracting tag bits from the lower 8 bits. constexpr uptr kTagMask = (1UL << kTagBits) - 1; diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.h b/compiler-rt/lib/hwasan/hwasan_allocator.h --- a/compiler-rt/lib/hwasan/hwasan_allocator.h +++ b/compiler-rt/lib/hwasan/hwasan_allocator.h @@ -58,7 +58,7 @@ struct AP64 { static const uptr kSpaceBeg = ~0ULL; -#if defined(__x86_64__) +#if defined(HWASAN_ALIASING_MODE) static const uptr kSpaceSize = 1ULL << kAddressTagShift; #else static const uptr kSpaceSize = 0x2000000000ULL; @@ -110,7 +110,7 @@ void GetAllocatorStats(AllocatorStatCounters s); inline bool InTaggableRegion(uptr addr) { -#if defined(__x86_64__) +#if defined(HWASAN_ALIASING_MODE) // Aliases are mapped next to shadow so that the upper bits match the shadow // base. return (addr >> kTaggableRegionCheckShift) == diff --git a/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp b/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp --- a/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp +++ b/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp @@ -119,12 +119,12 @@ void InitShadowGOT() {} uptr FindDynamicShadowStart(uptr shadow_size_bytes) { -#if defined(__x86_64__) +# if defined(HWASAN_ALIASING_MODE) constexpr uptr kAliasSize = 1ULL << kAddressTagShift; constexpr uptr kNumAliases = 1ULL << kTagBits; return MapDynamicShadowAndAliases(shadow_size_bytes, kAliasSize, kNumAliases, RingBufferSize()); -#endif +# endif return MapDynamicShadow(shadow_size_bytes, kShadowScale, kShadowBaseAlignment, kHighMemEnd); } diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cpp b/compiler-rt/lib/hwasan/hwasan_linux.cpp --- a/compiler-rt/lib/hwasan/hwasan_linux.cpp +++ b/compiler-rt/lib/hwasan/hwasan_linux.cpp @@ -76,7 +76,7 @@ uptr kHighMemStart; uptr kHighMemEnd; -uptr kAliasRegionStart; // Always 0 on non-x86. +uptr kAliasRegionStart; // Always 0 when aliases aren't used. static void PrintRange(uptr start, uptr end, const char *name) { Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name); @@ -125,33 +125,50 @@ if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0), &local_errno) && local_errno == EINVAL) { -#if SANITIZER_ANDROID || defined(__x86_64__) +# if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE) // Some older Android kernels have the tagged pointer ABI on // unconditionally, and hence don't have the tagged-addr prctl while still // allow the ABI. // If targeting Android and the prctl is not around we assume this is the // case. return; -#else +# else if (flags()->fail_without_syscall_abi) { Printf( "FATAL: " "HWAddressSanitizer requires a kernel with tagged address ABI.\n"); Die(); } -#endif +# endif } // Turn on the tagged address ABI. if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) || - !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) && - flags()->fail_without_syscall_abi) { - Printf( - "FATAL: HWAddressSanitizer failed to enable tagged address syscall " - "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` " - "configuration.\n"); - Die(); + !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0))) { +# if defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE) + // Try the new prctl API for Intel LAM. The API is based on a currently + // unsubmitted patch to the Linux kernel (as of May 2021) and is thus + // subject to change. Patch is here: + // https://lore.kernel.org/linux-mm/20210205151631.43511-12-kirill.shutemov@linux.intel.com/ + int tag_bits = kTagBits; + int tag_shift = kAddressTagShift; + if (!internal_iserror( + internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, + reinterpret_cast(&tag_bits), + reinterpret_cast(&tag_shift), 0))) { + CHECK_EQ(tag_bits, kTagBits); + CHECK_EQ(tag_shift, kAddressTagShift); + return; + } +# endif // defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE) + if (flags()->fail_without_syscall_abi) { + Printf( + "FATAL: HWAddressSanitizer failed to enable tagged address syscall " + "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` " + "configuration.\n"); + Die(); + } } #undef PR_SET_TAGGED_ADDR_CTRL #undef PR_GET_TAGGED_ADDR_CTRL @@ -181,7 +198,7 @@ // High memory starts where allocated shadow allows. kHighMemStart = ShadowToMem(kHighShadowStart); -#if defined(__x86_64__) +# if defined(HWASAN_ALIASING_MODE) constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1); kAliasRegionStart = __hwasan_shadow_memory_dynamic_address + kAliasRegionOffset; @@ -191,7 +208,7 @@ CHECK_EQ( (kAliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift, __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift); -#endif +# endif // Check the sanity of the defined memory ranges (there might be gaps). CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0); @@ -236,9 +253,11 @@ } bool MemIsApp(uptr p) { -#if !defined(__x86_64__) // Memory outside the alias range has non-zero tags. +// Memory outside the alias range has non-zero tags. +# if !defined(HWASAN_ALIASING_MODE) CHECK(GetTagFromPointer(p) == 0); -#endif +# endif + return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd); }