diff --git a/compiler-rt/lib/hwasan/CMakeLists.txt b/compiler-rt/lib/hwasan/CMakeLists.txt --- a/compiler-rt/lib/hwasan/CMakeLists.txt +++ b/compiler-rt/lib/hwasan/CMakeLists.txt @@ -93,6 +93,24 @@ CFLAGS ${HWASAN_DYNAMIC_CFLAGS} DEFS ${HWASAN_DEFINITIONS}) +# Compile a different runtime for x86 LAM mode. +set(HWASAN_LAM_RTL_CFLAGS ${HWASAN_RTL_CFLAGS}) +list(APPEND HWASAN_LAM_RTL_CFLAGS -DINTEL_LAM) +set(HWASAN_LAM_DYNAMIC_CFLAGS ${HWASAN_DYNAMIC_CFLAGS}) +list(APPEND HWASAN_LAM_DYNAMIC_CFLAGS -DINTEL_LAM) +add_compiler_rt_object_libraries(RTHwasanLam + ARCHS ${HWASAN_SUPPORTED_ARCH} + SOURCES ${HWASAN_RTL_SOURCES} + ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS} + CFLAGS ${HWASAN_LAM_RTL_CFLAGS} + DEFS ${HWASAN_DEFINITIONS}) +add_compiler_rt_object_libraries(RTHwasanLam_dynamic + ARCHS ${HWASAN_SUPPORTED_ARCH} + SOURCES ${HWASAN_RTL_SOURCES} ${HWASAN_RTL_CXX_SOURCES} + ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS} + CFLAGS ${HWASAN_LAM_DYNAMIC_CFLAGS} + DEFS ${HWASAN_DEFINITIONS}) + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "") add_compiler_rt_object_libraries(RTHwasan_dynamic_version_script_dummy ARCHS ${HWASAN_SUPPORTED_ARCH} @@ -100,47 +118,61 @@ CFLAGS ${HWASAN_DYNAMIC_CFLAGS} DEFS ${HWASAN_DEFINITIONS}) -foreach(arch ${HWASAN_SUPPORTED_ARCH}) - add_compiler_rt_runtime(clang_rt.hwasan +# If use_lam is TRUE, adds the HWASan runtime built with LAM support. +# Otherwise adds the runtime without LAM support. +function(add_hwasan_runtimes arch use_lam) + set(hwasan_object_lib RTHwasan) + set(hwasan_object_dyn_lib RTHwasan_dynamic) + set(hwasan_runtime clang_rt.hwasan) + set(hwasan_rtl_flags ${HWASAN_RTL_CFLAGS}) + set(hwasan_dyn_flags ${HWASAN_DYNAMIC_CFLAGS}) + if(use_lam) + set(hwasan_object_lib RTHwasanLam) + set(hwasan_object_dyn_lib RTHwasanLam_dynamic) + set(hwasan_runtime clang_rt.hwasan_lam) + set(hwasan_rtl_flags ${HWASAN_LAM_RTL_CFLAGS}) + set(hwasan_dyn_flags ${HWASAN_LAM_DYNAMIC_CFLAGS}) + endif() + add_compiler_rt_runtime(${hwasan_runtime} STATIC ARCHS ${arch} - OBJECT_LIBS RTHwasan + OBJECT_LIBS ${hwasan_object_lib} RTInterception RTSanitizerCommon RTSanitizerCommonLibc RTSanitizerCommonCoverage RTSanitizerCommonSymbolizer RTUbsan - CFLAGS ${HWASAN_RTL_CFLAGS} + CFLAGS ${hwasan_rtl_flags} PARENT_TARGET hwasan) - add_compiler_rt_runtime(clang_rt.hwasan_cxx + add_compiler_rt_runtime(${hwasan_runtime}_cxx STATIC ARCHS ${arch} OBJECT_LIBS RTHwasan_cxx RTUbsan_cxx - CFLAGS ${HWASAN_RTL_CFLAGS} + CFLAGS ${hwasan_rtl_flags} PARENT_TARGET hwasan) if (UNIX) - add_sanitizer_rt_version_list(clang_rt.hwasan-dynamic-${arch} - LIBS clang_rt.hwasan-${arch} clang_rt.hwasan_cxx-${arch} + add_sanitizer_rt_version_list(${hwasan_runtime}-dynamic-${arch} + LIBS ${hwasan_runtime}-${arch} ${hwasan_runtime}_cxx-${arch} EXTRA hwasan.syms.extra) set(VERSION_SCRIPT_FLAG - -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/clang_rt.hwasan-dynamic-${arch}.vers) + -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/${hwasan_runtime}-dynamic-${arch}.vers) set_property(SOURCE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp APPEND PROPERTY - OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/clang_rt.hwasan-dynamic-${arch}.vers) + OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${hwasan_runtime}-dynamic-${arch}.vers) else() set(VERSION_SCRIPT_FLAG) endif() - add_compiler_rt_runtime(clang_rt.hwasan + add_compiler_rt_runtime(${hwasan_runtime} SHARED ARCHS ${arch} OBJECT_LIBS - RTHwasan_dynamic + ${hwasan_object_dyn_lib} RTInterception RTSanitizerCommon RTSanitizerCommonLibc @@ -154,7 +186,7 @@ # add_dependencies(clang_rt.asan-dynamic-${arch} clang_rt.asan-dynamic-${arch}-version-list) # generates an order-only dependency in ninja. RTHwasan_dynamic_version_script_dummy - CFLAGS ${HWASAN_DYNAMIC_CFLAGS} + CFLAGS ${hwasan_dyn_flags} LINK_FLAGS ${HWASAN_DYNAMIC_LINK_FLAGS} ${VERSION_SCRIPT_FLAG} LINK_LIBS ${HWASAN_DYNAMIC_LIBS} @@ -162,14 +194,21 @@ PARENT_TARGET hwasan) if(SANITIZER_USE_SYMBOLS) - add_sanitizer_rt_symbols(clang_rt.hwasan + add_sanitizer_rt_symbols(${hwasan_runtime} ARCHS ${arch} EXTRA hwasan.syms.extra) - add_sanitizer_rt_symbols(clang_rt.hwasan_cxx + add_sanitizer_rt_symbols(${hwasan_runtime}_cxx ARCHS ${arch} EXTRA hwasan.syms.extra) - add_dependencies(hwasan clang_rt.hwasan-${arch}-symbols - clang_rt.hwasan_cxx-${arch}-symbols) + add_dependencies(hwasan ${hwasan_runtime}-${arch}-symbols + ${hwasan_runtime}_cxx-${arch}-symbols) + endif() +endfunction() + +foreach(arch ${HWASAN_SUPPORTED_ARCH}) + add_hwasan_runtimes(${arch} FALSE) + if(${arch} MATCHES "x86_64") + add_hwasan_runtimes(${arch} TRUE) endif() endforeach() diff --git a/compiler-rt/lib/hwasan/hwasan.h b/compiler-rt/lib/hwasan/hwasan.h --- a/compiler-rt/lib/hwasan/hwasan.h +++ b/compiler-rt/lib/hwasan/hwasan.h @@ -37,6 +37,11 @@ typedef u8 tag_t; #if defined(__x86_64__) +# if defined(INTEL_LAM) +// Tags are done in upper bits using Intel LAM. +constexpr unsigned kAddressTagShift = 57; +constexpr unsigned kTagBits = 6; +# else // Tags are done in middle bits using userspace aliasing. constexpr unsigned kAddressTagShift = 39; constexpr unsigned kTagBits = 3; @@ -49,6 +54,7 @@ // simpler/faster shadow calculation. constexpr unsigned kTaggableRegionCheckShift = __sanitizer::Max(kAddressTagShift + kTagBits + 1U, 44U); +# endif // defined(INTEL_LAM) #else // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address // translation and can be used to store a tag. diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.h b/compiler-rt/lib/hwasan/hwasan_allocator.h --- a/compiler-rt/lib/hwasan/hwasan_allocator.h +++ b/compiler-rt/lib/hwasan/hwasan_allocator.h @@ -58,7 +58,7 @@ struct AP64 { static const uptr kSpaceBeg = ~0ULL; -#if defined(__x86_64__) +#if defined(__x86_64__) && !defined(INTEL_LAM) static const uptr kSpaceSize = 1ULL << kAddressTagShift; #else static const uptr kSpaceSize = 0x2000000000ULL; @@ -110,7 +110,7 @@ void GetAllocatorStats(AllocatorStatCounters s); inline bool InTaggableRegion(uptr addr) { -#if defined(__x86_64__) +#if defined(__x86_64__) && !defined(INTEL_LAM) // Aliases are mapped next to shadow so that the upper bits match the shadow // base. return (addr >> kTaggableRegionCheckShift) == diff --git a/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp b/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp --- a/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp +++ b/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp @@ -119,12 +119,12 @@ void InitShadowGOT() {} uptr FindDynamicShadowStart(uptr shadow_size_bytes) { -#if defined(__x86_64__) +# if defined(__x86_64__) && !defined(INTEL_LAM) constexpr uptr kAliasSize = 1ULL << kAddressTagShift; constexpr uptr kNumAliases = 1ULL << kTagBits; return MapDynamicShadowAndAliases(shadow_size_bytes, kAliasSize, kNumAliases, RingBufferSize()); -#endif +# endif return MapDynamicShadow(shadow_size_bytes, kShadowScale, kShadowBaseAlignment, kHighMemEnd); } diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cpp b/compiler-rt/lib/hwasan/hwasan_linux.cpp --- a/compiler-rt/lib/hwasan/hwasan_linux.cpp +++ b/compiler-rt/lib/hwasan/hwasan_linux.cpp @@ -76,7 +76,7 @@ uptr kHighMemStart; uptr kHighMemEnd; -uptr kAliasRegionStart; // Always 0 on non-x86. +uptr kAliasRegionStart; // Always 0 when aliases aren't used. static void PrintRange(uptr start, uptr end, const char *name) { Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name); @@ -125,33 +125,50 @@ if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0), &local_errno) && local_errno == EINVAL) { -#if SANITIZER_ANDROID || defined(__x86_64__) +# if SANITIZER_ANDROID || (defined(__x86_64__) && !defined(INTEL_LAM)) // Some older Android kernels have the tagged pointer ABI on // unconditionally, and hence don't have the tagged-addr prctl while still // allow the ABI. // If targeting Android and the prctl is not around we assume this is the // case. return; -#else +# else if (flags()->fail_without_syscall_abi) { Printf( "FATAL: " "HWAddressSanitizer requires a kernel with tagged address ABI.\n"); Die(); } -#endif +# endif } // Turn on the tagged address ABI. if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) || - !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) && - flags()->fail_without_syscall_abi) { - Printf( - "FATAL: HWAddressSanitizer failed to enable tagged address syscall " - "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` " - "configuration.\n"); - Die(); + !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0))) { +# if defined(INTEL_LAM) + // Try the new prctl API for Intel LAM. The API is based on a currently + // unsubmitted patch to the Linux kernel (as of May 2021) and is thus + // subject to change. Patch is here: + // https://lore.kernel.org/linux-mm/20210205151631.43511-12-kirill.shutemov@linux.intel.com/ + int tag_bits = kTagBits; + int tag_shift = kAddressTagShift; + if (!internal_iserror( + internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, + reinterpret_cast(&tag_bits), + reinterpret_cast(&tag_shift), 0))) { + CHECK_EQ(tag_bits, kTagBits); + CHECK_EQ(tag_shift, kAddressTagShift); + return; + } +# endif // defined(INTEL_LAM) + if (flags()->fail_without_syscall_abi) { + Printf( + "FATAL: HWAddressSanitizer failed to enable tagged address syscall " + "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` " + "configuration.\n"); + Die(); + } } #undef PR_SET_TAGGED_ADDR_CTRL #undef PR_GET_TAGGED_ADDR_CTRL @@ -181,7 +198,7 @@ // High memory starts where allocated shadow allows. kHighMemStart = ShadowToMem(kHighShadowStart); -#if defined(__x86_64__) +# if defined(__x86_64__) && !defined(INTEL_LAM) constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1); kAliasRegionStart = __hwasan_shadow_memory_dynamic_address + kAliasRegionOffset; @@ -191,7 +208,7 @@ CHECK_EQ( (kAliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift, __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift); -#endif +# endif // Check the sanity of the defined memory ranges (there might be gaps). CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0);