Hardware-assisted AddressSanitizer (compiler-rt)

[eugenis]

Summary

Runtime library for HWASan, initial commit.
Does not randomize tags yet, does not handle stack or globals.

[krytarowski]

compiler-rt/lib/hwasan/hwasan_linux.cc

87

I have a local draft MSan patch adjusting similar code for NetBSD. It's right no incomplete as I need to add proper definition of INTERCEPT_FUNCTION(_lwp_exit); instead of calling it for each thread.

diff --git a/lib/msan/msan_linux.cc b/lib/msan/msan_linux.cc
index 4e6321fcb..d68509672 100644
--- a/lib/msan/msan_linux.cc
+++ b/lib/msan/msan_linux.cc
@@ -30,6 +30,7 @@
 #include <sys/time.h>
 #include <sys/resource.h>
 
+#include "interception/interception.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_procmaps.h"
 
@@ -174,13 +175,28 @@ void InstallAtExitHandler() {
 
 // ---------------------- TSD ---------------- {{{1
 
+#if !SANITIZER_NETBSD
 static pthread_key_t tsd_key;
+#endif
+
 static bool tsd_key_inited = false;
 
+#if SANITIZER_NETBSD
+INTERCEPTOR(void, _lwp_exit) {
+  CHECK(tsd_key_inited);
+  MsanTSDDtor(GetCurrentThread());
+  REAL(_lwp_exit)();
+}
+#endif
+
 void MsanTSDInit(void (*destructor)(void *tsd)) {
   CHECK(!tsd_key_inited);
   tsd_key_inited = true;
+#if SANITIZER_NETBSD
+  INTERCEPT_FUNCTION(_lwp_exit);
+#else
   CHECK_EQ(0, pthread_key_create(&tsd_key, destructor));
+#endif
 }
 
 static THREADLOCAL MsanThread* msan_current_thread;
@@ -195,16 +211,20 @@ void SetCurrentThread(MsanThread *t) {
   msan_current_thread = t;
   // Make sure that MsanTSDDtor gets called at the end.
   CHECK(tsd_key_inited);
+#if !SANITIZER_NETBSD
   pthread_setspecific(tsd_key, (void *)t);
+#endif
 }
 
 void MsanTSDDtor(void *tsd) {
+#if !SANITIZER_NETBSD
   MsanThread *t = (MsanThread*)tsd;
   if (t->destructor_iterations_ > 1) {
     t->destructor_iterations_--;
     CHECK_EQ(0, pthread_setspecific(tsd_key, tsd));
     return;
   }
+#endif
   msan_current_thread = nullptr;
   // Make sure that signal handler can not see a stale current thread pointer.
   atomic_signal_fence(memory_order_seq_cst);

[eugenis]

Note that hwasan stores current thread pointer in the TSD slot, because it needs to run on Android where thread-local (i.e. __thread) variables don't really work.
We plan to do the same with MSan in the future, but I don't know when.

[kcc]

compiler-rt/lib/hwasan/hwasan_interceptors.cc

138

Do we expect any of this code to (ever? any time soon?) work on FREEBSD? NETBSD?
Do those OSes exist for aarch64?

[eugenis]

compiler-rt/lib/hwasan/hwasan_interceptors.cc

138

At least FreeBSD does.

[krytarowski]

compiler-rt/lib/hwasan/hwasan_interceptors.cc

138

NetBSD works on evbarm64.

https://twitter.com/rsh/status/938372595962994688

[eugenis]

switch to 1-to-16 shadow

[kcc]

My top level comment: can we delete all non-aarch64 code?
The arch owners can reinstate it if needed, but they will only need it if/when they have the TBI feature in HW.

compiler-rt/lib/hwasan/hwasan.cc

61

seems to be unused (nothings sets it to non zero), please remove.

100

can we not have any of these #ifdefs?

182

Is this one relevant?

195

do we need this?
For asan we did, but probably not for hwasan

compiler-rt/lib/hwasan/hwasan_allocator.cc

73

oy! Please remove all of this non-aarch64 stuff.
We won't need non-aarch64 any time soon.

compiler-rt/lib/hwasan/hwasan_flags.inc

25

seems unused, remove

compiler-rt/lib/hwasan/hwasan_interceptors.cc

479

mips?

compiler-rt/lib/hwasan/hwasan_linux.cc

39

Note for future, feel free to ignore now: we should try to have the zero shadow base

107

What's the use case for #else?

[eugenis]

Addressed review comments.
Implemented halt_on_error=1.
Fixed tests to pass on unmodified kernels (see disable_allocator_tagging flag).

[eugenis]

compiler-rt/lib/hwasan/hwasan.cc

61

I've implemented halt_on_error=0

100

don't we want hwasan + ubsan and hwasan + lsan?

195

we don't

[kcc]

LGTM, let's iterate from here.

compiler-rt/cmake/config-ix.cmake

 set(ALL_FUZZER_SUPPORTED_ARCH x86_64)
 
 if(APPLE)
   set(ALL_LSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${MIPS64} ${ARM64})
 else()
   set(ALL_LSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${MIPS64} ${ARM64} ${ARM32} ${PPC64})
 endif()
 set(ALL_MSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64})
+set(ALL_HWASAN_SUPPORTED_ARCH ${ARM64})
 set(ALL_PROFILE_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${PPC64}
     ${MIPS32} ${MIPS64} ${S390X})
 set(ALL_TSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64})
 set(ALL_UBSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64}
     ${MIPS32} ${MIPS64} ${PPC64} ${S390X})
 set(ALL_SAFESTACK_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM64} ${MIPS32} ${MIPS64})
 set(ALL_CFI_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${MIPS64})
 set(ALL_ESAN_SUPPORTED_ARCH ${X86_64} ${MIPS64})
list_intersect(DFSAN_SUPPORTED_ARCH
     ALL_DFSAN_SUPPORTED_ARCH
     SANITIZER_COMMON_SUPPORTED_ARCH)
   list_intersect(LSAN_SUPPORTED_ARCH
     ALL_LSAN_SUPPORTED_ARCH
     SANITIZER_COMMON_SUPPORTED_ARCH)
   list_intersect(MSAN_SUPPORTED_ARCH
     ALL_MSAN_SUPPORTED_ARCH
     SANITIZER_COMMON_SUPPORTED_ARCH)
+  list_intersect(HWASAN_SUPPORTED_ARCH
+    ALL_HWASAN_SUPPORTED_ARCH
+    SANITIZER_COMMON_SUPPORTED_ARCH)
   list_intersect(PROFILE_SUPPORTED_ARCH
     ALL_PROFILE_SUPPORTED_ARCH
     SANITIZER_COMMON_SUPPORTED_ARCH)
   list_intersect(TSAN_SUPPORTED_ARCH
     ALL_TSAN_SUPPORTED_ARCH
     SANITIZER_COMMON_SUPPORTED_ARCH)
   list_intersect(UBSAN_SUPPORTED_ARCH
     ALL_UBSAN_SUPPORTED_ARCH
filter_available_targets(LSAN_COMMON_SUPPORTED_ARCH
     ${SANITIZER_COMMON_SUPPORTED_ARCH})
   filter_available_targets(UBSAN_COMMON_SUPPORTED_ARCH
     ${SANITIZER_COMMON_SUPPORTED_ARCH})
   filter_available_targets(ASAN_SUPPORTED_ARCH ${ALL_ASAN_SUPPORTED_ARCH})
   filter_available_targets(FUZZER_SUPPORTED_ARCH ${ALL_FUZZER_SUPPORTED_ARCH})
   filter_available_targets(DFSAN_SUPPORTED_ARCH ${ALL_DFSAN_SUPPORTED_ARCH})
   filter_available_targets(LSAN_SUPPORTED_ARCH ${ALL_LSAN_SUPPORTED_ARCH})
   filter_available_targets(MSAN_SUPPORTED_ARCH ${ALL_MSAN_SUPPORTED_ARCH})
+  filter_available_targets(HWASAN_SUPPORTED_ARCH ${ALL_HWASAN_SUPPORTED_ARCH})
   filter_available_targets(PROFILE_SUPPORTED_ARCH ${ALL_PROFILE_SUPPORTED_ARCH})
   filter_available_targets(TSAN_SUPPORTED_ARCH ${ALL_TSAN_SUPPORTED_ARCH})
   filter_available_targets(UBSAN_SUPPORTED_ARCH ${ALL_UBSAN_SUPPORTED_ARCH})
   filter_available_targets(SAFESTACK_SUPPORTED_ARCH
     ${ALL_SAFESTACK_SUPPORTED_ARCH})
   filter_available_targets(CFI_SUPPORTED_ARCH ${ALL_CFI_SUPPORTED_ARCH})
   filter_available_targets(ESAN_SUPPORTED_ARCH ${ALL_ESAN_SUPPORTED_ARCH})
   filter_available_targets(SCUDO_SUPPORTED_ARCH ${ALL_SCUDO_SUPPORTED_ARCH})

 message(STATUS "Compiler-RT supported architectures: ${COMPILER_RT_SUPPORTED_ARCH}")
 
 if(ANDROID)
   set(OS_NAME "Android")
 else()
   set(OS_NAME "${CMAKE_SYSTEM_NAME}")
 endif()
 
-set(ALL_SANITIZERS asan;dfsan;msan;tsan;safestack;cfi;esan;scudo;ubsan_minimal)
+set(ALL_SANITIZERS asan;dfsan;msan;hwasan;tsan;safestack;cfi;esan;scudo;ubsan_minimal)
 set(COMPILER_RT_SANITIZERS_TO_BUILD all CACHE STRING
     "sanitizers to build if supported on the target (all;${ALL_SANITIZERS})")
 list_replace(COMPILER_RT_SANITIZERS_TO_BUILD all "${ALL_SANITIZERS}")
 
 if (SANITIZER_COMMON_SUPPORTED_ARCH AND NOT LLVM_USE_SANITIZER AND
     (OS_NAME MATCHES "Android|Darwin|Linux|FreeBSD|NetBSD|Fuchsia" OR
     (OS_NAME MATCHES "Windows" AND (NOT MINGW AND NOT CYGWIN))))
   set(COMPILER_RT_HAS_SANITIZER_COMMON TRUE)

 
 if (COMPILER_RT_HAS_SANITIZER_COMMON AND MSAN_SUPPORTED_ARCH AND
     OS_NAME MATCHES "Linux|NetBSD")
   set(COMPILER_RT_HAS_MSAN TRUE)
 else()
   set(COMPILER_RT_HAS_MSAN FALSE)
 endif()
 
+if (COMPILER_RT_HAS_SANITIZER_COMMON AND HWASAN_SUPPORTED_ARCH AND
+    OS_NAME MATCHES "Linux|Android")
+  set(COMPILER_RT_HAS_HWASAN TRUE)
+else()
+  set(COMPILER_RT_HAS_HWASAN FALSE)
+endif()
+
 if (PROFILE_SUPPORTED_ARCH AND NOT LLVM_USE_SANITIZER AND
     OS_NAME MATCHES "Darwin|Linux|FreeBSD|Windows|Android")
   set(COMPILER_RT_HAS_PROFILE TRUE)
 else()
   set(COMPILER_RT_HAS_PROFILE FALSE)
 endif()
 
 if (COMPILER_RT_HAS_SANITIZER_COMMON AND TSAN_SUPPORTED_ARCH AND

compiler-rt/include/CMakeLists.txt

 if (COMPILER_RT_BUILD_SANITIZERS)
   set(SANITIZER_HEADERS
     sanitizer/allocator_interface.h
     sanitizer/asan_interface.h
     sanitizer/common_interface_defs.h
     sanitizer/coverage_interface.h
     sanitizer/dfsan_interface.h
     sanitizer/esan_interface.h
+    sanitizer/hwasan_interface.h
     sanitizer/linux_syscall_hooks.h
     sanitizer/lsan_interface.h
     sanitizer/msan_interface.h
     sanitizer/tsan_interface.h
     sanitizer/tsan_interface_atomic.h)
 endif(COMPILER_RT_BUILD_SANITIZERS)
 
 if (COMPILER_RT_BUILD_XRAY)

compiler-rt/include/sanitizer/hwasan_interface.h

+//===-- sanitizer/asan_interface.h ------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Public interface header.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_HWASAN_INTERFACE_H
+#define SANITIZER_HWASAN_INTERFACE_H
+
+#include <sanitizer/common_interface_defs.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+  // This function may be optionally provided by user and should return
+  // a string containing HWASan runtime options. See asan_flags.h for details.
+  const char* __hwasan_default_options();
+
+  void __hwasan_enable_allocator_tagging();
+  void __hwasan_disable_allocator_tagging();
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // SANITIZER_HWASAN_INTERFACE_H

compiler-rt/lib/hwasan/.clang-format

+BasedOnStyle: Google

compiler-rt/lib/hwasan/CMakeLists.txt

+include_directories(..)
+
+# Runtime library sources and build flags.
+set(HWASAN_RTL_SOURCES
+  hwasan.cc
+  hwasan_allocator.cc
+  hwasan_interceptors.cc
+  hwasan_linux.cc
+  hwasan_report.cc
+  hwasan_thread.cc
+  hwasan_poisoning.cc
+  )
+
+set(HWASAN_RTL_CXX_SOURCES
+  hwasan_new_delete.cc)
+
+
+set(HWASAN_RTL_CFLAGS ${SANITIZER_COMMON_CFLAGS})
+append_rtti_flag(OFF HWASAN_RTL_CFLAGS)
+append_list_if(COMPILER_RT_HAS_FPIE_FLAG -fPIE HWASAN_RTL_CFLAGS)
+# Prevent clang from generating libc calls.
+append_list_if(COMPILER_RT_HAS_FFREESTANDING_FLAG -ffreestanding HWASAN_RTL_CFLAGS)
+
+set(HWASAN_DYNAMIC_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS})
+
+if(ANDROID)
+# On Android, -z global does not do what it is documented to do.
+# On Android, -z global moves the library ahead in the lookup order,
+# placing it right after the LD_PRELOADs. This is used to compensate for the fact
+# that Android linker does not look at the dependencies of the main executable
+# that aren't dependencies of the current DSO when resolving symbols from said DSO.
+# As a net result, this allows running ASan executables without LD_PRELOAD-ing the
+# ASan runtime library.
+# The above is applicable to L MR1 or newer.
+  if (COMPILER_RT_HAS_Z_GLOBAL)
+    list(APPEND HWASAN_DYNAMIC_LINK_FLAGS -Wl,-z,global)
+  endif()
+endif()
+
+set(HWASAN_DYNAMIC_CFLAGS ${HWASAN_RTL_CFLAGS})
+append_list_if(COMPILER_RT_HAS_FTLS_MODEL_INITIAL_EXEC
+  -ftls-model=initial-exec HWASAN_DYNAMIC_CFLAGS)
+append_list_if(MSVC /DEBUG HWASAN_DYNAMIC_LINK_FLAGS)
+
+set(HWASAN_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARY} ${SANITIZER_COMMON_LINK_LIBS})
+
+append_list_if(COMPILER_RT_HAS_LIBDL dl HWASAN_DYNAMIC_LIBS)
+append_list_if(COMPILER_RT_HAS_LIBRT rt HWASAN_DYNAMIC_LIBS)
+append_list_if(COMPILER_RT_HAS_LIBM m HWASAN_DYNAMIC_LIBS)
+append_list_if(COMPILER_RT_HAS_LIBPTHREAD pthread HWASAN_DYNAMIC_LIBS)
+append_list_if(COMPILER_RT_HAS_LIBLOG log HWASAN_DYNAMIC_LIBS)
+
+# Static runtime library.
+add_compiler_rt_component(hwasan)
+
+add_compiler_rt_object_libraries(RTHwasan
+  ARCHS ${HWASAN_SUPPORTED_ARCH}
+  SOURCES ${HWASAN_RTL_SOURCES} CFLAGS ${HWASAN_RTL_CFLAGS})
+add_compiler_rt_object_libraries(RTHwasan_cxx
+  ARCHS ${HWASAN_SUPPORTED_ARCH}
+  SOURCES ${HWASAN_RTL_CXX_SOURCES} CFLAGS ${HWASAN_RTL_CFLAGS})
+add_compiler_rt_object_libraries(RTHwasan_dynamic
+  ARCHS ${HWASAN_SUPPORTED_ARCH}
+  SOURCES ${HWASAN_RTL_SOURCES} ${TSAN_RTL_CXX_SOURCES}
+  CFLAGS ${HWASAN_DYNAMIC_CFLAGS})
+
+file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cc "")
+add_compiler_rt_object_libraries(RTHwasan_dynamic_version_script_dummy
+  ARCHS ${HWASAN_SUPPORTED_ARCH}
+  SOURCES ${CMAKE_CURRENT_BINARY_DIR}/dummy.cc
+  CFLAGS ${HWASAN_DYNAMIC_CFLAGS})
+
+foreach(arch ${HWASAN_SUPPORTED_ARCH})
+  add_compiler_rt_runtime(clang_rt.hwasan
+    STATIC
+    ARCHS ${arch}
+    OBJECT_LIBS RTHwasan
+                RTInterception
+                RTSanitizerCommon
+                RTSanitizerCommonLibc
+    RTUbsan
+    CFLAGS ${HWASAN_RTL_CFLAGS}
+    PARENT_TARGET hwasan)
+  add_compiler_rt_runtime(clang_rt.hwasan_cxx
+    STATIC
+    ARCHS ${arch}
+    OBJECT_LIBS RTHwasan_cxx
+                RTUbsan_cxx
+    CFLAGS ${HWASAN_RTL_CFLAGS}
+    PARENT_TARGET hwasan)
+
+  if (UNIX)
+    add_sanitizer_rt_version_list(clang_rt.hwasan-dynamic-${arch}
+                                  LIBS clang_rt.hwasan-${arch} clang_rt.hwasan_cxx-${arch}
+                                  EXTRA hwasan.syms.extra)
+    set(VERSION_SCRIPT_FLAG
+         -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/clang_rt.hwasan-dynamic-${arch}.vers)
+    set_property(SOURCE
+      ${CMAKE_CURRENT_BINARY_DIR}/dummy.cc
+      APPEND PROPERTY
+      OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/clang_rt.hwasan-dynamic-${arch}.vers)
+  else()
+    set(VERSION_SCRIPT_FLAG)
+  endif()
+
+
+  add_compiler_rt_runtime(clang_rt.hwasan
+    SHARED
+    ARCHS ${arch}
+    OBJECT_LIBS
+            RTHwasan_dynamic
+            RTInterception
+            RTSanitizerCommon
+            RTSanitizerCommonLibc
+      RTUbsan
+            # The only purpose of RTHWAsan_dynamic_version_script_dummy is to
+            # carry a dependency of the shared runtime on the version script.
+            # Replacing it with a straightforward
+            # add_dependencies(clang_rt.asan-dynamic-${arch} clang_rt.asan-dynamic-${arch}-version-list)
+            # generates an order-only dependency in ninja.
+            RTHwasan_dynamic_version_script_dummy
+    CFLAGS ${HWASAN_DYNAMIC_CFLAGS}
+    LINK_FLAGS ${HWASAN_DYNAMIC_LINK_FLAGS}
+              ${VERSION_SCRIPT_FLAG}
+    LINK_LIBS ${HWASAN_DYNAMIC_LIBS}
+    DEFS ${ASAN_DYNAMIC_DEFINITIONS}
+    PARENT_TARGET hwasan)
+
+  if(UNIX)
+    add_sanitizer_rt_symbols(clang_rt.hwasan
+      ARCHS ${arch}
+      EXTRA hwasan.syms.extra)
+    add_sanitizer_rt_symbols(clang_rt.hwasan_cxx
+      ARCHS ${arch}
+      EXTRA hwasan.syms.extra)
+    add_dependencies(hwasan clang_rt.hwasan-${arch}-symbols
+                          clang_rt.hwasan_cxx-${arch}-symbols)
+  endif()
+endforeach()
+
+add_compiler_rt_resource_file(hwasan_blacklist hwasan_blacklist.txt hwasan)
+
+# if(COMPILER_RT_INCLUDE_TESTS)
+#   add_subdirectory(tests)
+# endif()

compiler-rt/lib/hwasan/hwasan.h

+//===-- hwasan.h --------------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Private Hwasan header.
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_H
+#define HWASAN_H
+
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "hwasan_interface_internal.h"
+#include "hwasan_flags.h"
+#include "ubsan/ubsan_platform.h"
+
+#ifndef HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
+# define HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE 1
+#endif
+
+#ifndef HWASAN_CONTAINS_UBSAN
+# define HWASAN_CONTAINS_UBSAN CAN_SANITIZE_UB
+#endif
+
+typedef u8 tag_t;
+
+// Reasonable values are 4 (for 1/16th shadow) and 6 (for 1/64th).
+const uptr kShadowScale = 4;
+const uptr kShadowAlignment = 1UL << kShadowScale;
+
+#define MEM_TO_SHADOW_OFFSET(mem) ((uptr)(mem) >> kShadowScale)
+#define MEM_TO_SHADOW(mem)         \
+  (((uptr)(mem) >> kShadowScale) + \
+   __hwasan_shadow_memory_dynamic_address_internal)
+#define SHADOW_TO_MEM(shadow)                                       \
+  (((uptr)(shadow)-__hwasan_shadow_memory_dynamic_address_internal) \
+   << kShadowScale)
+
+#define MEM_IS_APP(mem) true
+
+// TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address
+// translation and can be used to store a tag.
+const unsigned kAddressTagShift = 56;
+const uptr kAddressTagMask = 0xFFUL << kAddressTagShift;
+
+static inline tag_t GetTagFromPointer(uptr p) {
+  return p >> kAddressTagShift;
+}
+
+static inline uptr GetAddressFromPointer(uptr p) {
+  return p & ~kAddressTagMask;
+}
+
+static inline void * GetAddressFromPointer(const void *p) {
+  return (void *)((uptr)p & ~kAddressTagMask);
+}
+
+static inline uptr AddTagToPointer(uptr p, tag_t tag) {
+  return (p & ~kAddressTagMask) | ((uptr)tag << kAddressTagShift);
+}
+
+namespace __hwasan {
+
+extern int hwasan_inited;
+extern bool hwasan_init_is_running;
+extern int hwasan_report_count;
+
+bool ProtectRange(uptr beg, uptr end);
+bool InitShadow();
+char *GetProcSelfMaps();
+void InitializeInterceptors();
+
+void HwasanAllocatorInit();
+void HwasanAllocatorThreadFinish();
+void HwasanDeallocate(StackTrace *stack, void *ptr);
+
+void *hwasan_malloc(uptr size, StackTrace *stack);
+void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack);
+void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack);
+void *hwasan_valloc(uptr size, StackTrace *stack);
+void *hwasan_pvalloc(uptr size, StackTrace *stack);
+void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack);
+void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack);
+int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
+                        StackTrace *stack);
+
+void InstallTrapHandler();
+void InstallAtExitHandler();
+
+const char *GetStackOriginDescr(u32 id, uptr *pc);
+
+void EnterSymbolizer();
+void ExitSymbolizer();
+bool IsInSymbolizer();
+
+struct SymbolizerScope {
+  SymbolizerScope() { EnterSymbolizer(); }
+  ~SymbolizerScope() { ExitSymbolizer(); }
+};
+
+void PrintWarning(uptr pc, uptr bp);
+
+void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp,
+                   void *context, bool request_fast_unwind);
+
+void ReportInvalidAccess(StackTrace *stack, u32 origin);
+void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size,
+                       bool is_store);
+void ReportStats();
+void ReportAtExitStatistics();
+void DescribeMemoryRange(const void *x, uptr size);
+void ReportInvalidAccessInsideAddressRange(const char *what, const void *start, uptr size,
+                                 uptr offset);
+
+// Returns a "chained" origin id, pointing to the given stack trace followed by
+// the previous origin id.
+u32 ChainOrigin(u32 id, StackTrace *stack);
+
+const int STACK_TRACE_TAG_POISON = StackTrace::TAG_CUSTOM + 1;
+
+#define GET_MALLOC_STACK_TRACE                                            \
+  BufferedStackTrace stack;                                               \
+  if (hwasan_inited)                                                       \
+  GetStackTrace(&stack, common_flags()->malloc_context_size,              \
+                StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \
+                common_flags()->fast_unwind_on_malloc)
+
+#define GET_FATAL_STACK_TRACE_PC_BP(pc, bp)              \
+  BufferedStackTrace stack;                              \
+  if (hwasan_inited)                                       \
+  GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \
+                common_flags()->fast_unwind_on_fatal)
+
+class ScopedThreadLocalStateBackup {
+ public:
+  ScopedThreadLocalStateBackup() { Backup(); }
+  ~ScopedThreadLocalStateBackup() { Restore(); }
+  void Backup();
+  void Restore();
+ private:
+  u64 va_arg_overflow_size_tls;
+};
+
+void HwasanTSDInit(void (*destructor)(void *tsd));
+void *HwasanTSDGet();
+void HwasanTSDSet(void *tsd);
+void HwasanTSDDtor(void *tsd);
+
+void HwasanOnDeadlySignal(int signo, void *info, void *context);
+
+}  // namespace __hwasan
+
+#define HWASAN_MALLOC_HOOK(ptr, size)       \
+  do {                                    \
+    if (&__sanitizer_malloc_hook) {       \
+      __sanitizer_malloc_hook(ptr, size); \
+    }                                     \
+    RunMallocHooks(ptr, size);            \
+  } while (false)
+#define HWASAN_FREE_HOOK(ptr)       \
+  do {                            \
+    if (&__sanitizer_free_hook) { \
+      __sanitizer_free_hook(ptr); \
+    }                             \
+    RunFreeHooks(ptr);            \
+  } while (false)
+
+#endif  // HWASAN_H

compiler-rt/lib/hwasan/hwasan.cc

+//===-- hwasan.cc -----------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// HWAddressSanitizer runtime.
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "ubsan/ubsan_flags.h"
+#include "ubsan/ubsan_init.h"
+
+// ACHTUNG! No system header includes in this file.
+
+using namespace __sanitizer;
+
+namespace __hwasan {
+
+void EnterSymbolizer() {
+  HwasanThread *t = GetCurrentThread();
+  CHECK(t);
+  t->EnterSymbolizer();
+}
+void ExitSymbolizer() {
+  HwasanThread *t = GetCurrentThread();
+  CHECK(t);
+  t->LeaveSymbolizer();
+}
+bool IsInSymbolizer() {
+  HwasanThread *t = GetCurrentThread();
+  return t && t->InSymbolizer();
+}
+
+static Flags hwasan_flags;
+
+Flags *flags() {
+  return &hwasan_flags;
+}
+
+int hwasan_inited = 0;
+bool hwasan_init_is_running;
+
+int hwasan_report_count = 0;
+
+void Flags::SetDefaults() {
+#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
+#include "hwasan_flags.inc"
+#undef HWASAN_FLAG
+}
+
+static void RegisterHwasanFlags(FlagParser *parser, Flags *f) {
+#define HWASAN_FLAG(Type, Name, DefaultValue, Description) \
+  RegisterFlag(parser, #Name, Description, &f->Name);
+#include "hwasan_flags.inc"
+#undef HWASAN_FLAG
+}
+
+static void InitializeFlags() {
+  SetCommonFlagsDefaults();
+  {
+    CommonFlags cf;
+    cf.CopyFrom(*common_flags());
+    cf.external_symbolizer_path = GetEnv("HWASAN_SYMBOLIZER_PATH");
+    cf.malloc_context_size = 20;
+    cf.handle_ioctl = true;
+    // FIXME: test and enable.
+    cf.check_printf = false;
+    cf.intercept_tls_get_addr = true;
+    cf.exitcode = 99;
+    cf.handle_sigill = kHandleSignalExclusive;
+    OverrideCommonFlags(cf);
+  }
+
+  Flags *f = flags();
+  f->SetDefaults();
+
+  FlagParser parser;
+  RegisterHwasanFlags(&parser, f);
+  RegisterCommonFlags(&parser);
+
+#if HWASAN_CONTAINS_UBSAN
+  __ubsan::Flags *uf = __ubsan::flags();
+  uf->SetDefaults();
+
+  FlagParser ubsan_parser;
+  __ubsan::RegisterUbsanFlags(&ubsan_parser, uf);
+  RegisterCommonFlags(&ubsan_parser);
+#endif
+
+  // Override from user-specified string.
+  if (__hwasan_default_options)
+    parser.ParseString(__hwasan_default_options());
+#if HWASAN_CONTAINS_UBSAN
+  const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
+  ubsan_parser.ParseString(ubsan_default_options);
+#endif
+
+  const char *hwasan_options = GetEnv("HWASAN_OPTIONS");
+  parser.ParseString(hwasan_options);
+#if HWASAN_CONTAINS_UBSAN
+  ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS"));
+#endif
+  VPrintf(1, "HWASAN_OPTIONS: %s\n", hwasan_options ? hwasan_options : "<empty>");
+
+  InitializeCommonFlags();
+
+  if (Verbosity()) ReportUnrecognizedFlags();
+
+  if (common_flags()->help) parser.PrintFlagDescriptions();
+}
+
+void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp,
+                   void *context, bool request_fast_unwind) {
+  HwasanThread *t = GetCurrentThread();
+  if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind)) {
+    // Block reports from our interceptors during _Unwind_Backtrace.
+    SymbolizerScope sym_scope;
+    return stack->Unwind(max_s, pc, bp, context, 0, 0, request_fast_unwind);
+  }
+  stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(),
+                request_fast_unwind);
+}
+
+void PrintWarning(uptr pc, uptr bp) {
+  GET_FATAL_STACK_TRACE_PC_BP(pc, bp);
+  ReportInvalidAccess(&stack, 0);
+}
+
+} // namespace __hwasan
+
+// Interface.
+
+using namespace __hwasan;
+
+void __hwasan_init() {
+  CHECK(!hwasan_init_is_running);
+  if (hwasan_inited) return;
+  hwasan_init_is_running = 1;
+  SanitizerToolName = "HWAddressSanitizer";
+
+  InitTlsSize();
+
+  CacheBinaryName();
+  InitializeFlags();
+
+  __sanitizer_set_report_path(common_flags()->log_path);
+
+  InitializeInterceptors();
+  InstallDeadlySignalHandlers(HwasanOnDeadlySignal);
+  InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
+
+  DisableCoreDumperIfNecessary();
+  if (!InitShadow()) {
+    Printf("FATAL: HWAddressSanitizer can not mmap the shadow memory.\n");
+    Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
+    Printf("FATAL: Disabling ASLR is known to cause this error.\n");
+    Printf("FATAL: If running under GDB, try "
+           "'set disable-randomization off'.\n");
+    DumpProcessMap();
+    Die();
+  }
+
+  Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
+
+  InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
+
+  HwasanTSDInit(HwasanTSDDtor);
+
+  HwasanAllocatorInit();
+
+  HwasanThread *main_thread = HwasanThread::Create(nullptr, nullptr);
+  SetCurrentThread(main_thread);
+  main_thread->ThreadStart();
+
+#if HWASAN_CONTAINS_UBSAN
+  __ubsan::InitAsPlugin();
+#endif
+
+  VPrintf(1, "HWAddressSanitizer init done\n");
+
+  hwasan_init_is_running = 0;
+  hwasan_inited = 1;
+}
+
+void __hwasan_print_shadow(const void *x, uptr size) {
+  // FIXME:
+  Printf("FIXME: __hwasan_print_shadow unimplemented\n");
+}
+
+sptr __hwasan_test_shadow(const void *p, uptr sz) {
+  if (sz == 0)
+    return -1;
+  tag_t ptr_tag = GetTagFromPointer((uptr)p);
+  if (ptr_tag == 0)
+    return -1;
+  uptr ptr_raw = GetAddressFromPointer((uptr)p);
+  uptr shadow_first = MEM_TO_SHADOW(ptr_raw);
+  uptr shadow_last = MEM_TO_SHADOW(ptr_raw + sz - 1);
+  for (uptr s = shadow_first; s <= shadow_last; ++s)
+    if (*(tag_t*)s != ptr_tag)
+      return SHADOW_TO_MEM(s) - ptr_raw;
+  return -1;
+}
+
+u16 __sanitizer_unaligned_load16(const uu16 *p) {
+  return *p;
+}
+u32 __sanitizer_unaligned_load32(const uu32 *p) {
+  return *p;
+}
+u64 __sanitizer_unaligned_load64(const uu64 *p) {
+  return *p;
+}
+void __sanitizer_unaligned_store16(uu16 *p, u16 x) {
+  *p = x;
+}
+void __sanitizer_unaligned_store32(uu32 *p, u32 x) {
+  *p = x;
+}
+void __sanitizer_unaligned_store64(uu64 *p, u64 x) {
+  *p = x;
+}
+
+__attribute__((always_inline))
+static void SigIll() {
+#if defined(__aarch64__)
+  asm("hlt #0x1\n\t");
+#elif defined(__x86_64__) || defined(__i386__)
+  asm("ud2\n\t");
+#else
+  // FIXME: not always sigill.
+  __builtin_trap();
+#endif
+  // __builtin_unreachable();
+}
+
+__attribute__((always_inline, nodebug))
+static void CheckAddress(uptr p) {
+  tag_t ptr_tag = GetTagFromPointer(p);
+  uptr ptr_raw = p & ~kAddressTagMask;
+  tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(ptr_raw);
+  if (ptr_tag != mem_tag)
+    SigIll();
+}
+
+__attribute__((always_inline, nodebug))
+static void CheckAddressSized(uptr p, uptr sz) {
+  CHECK_NE(0, sz);
+  tag_t ptr_tag = GetTagFromPointer(p);
+  uptr ptr_raw = p & ~kAddressTagMask;
+  tag_t *shadow_first = (tag_t *)MEM_TO_SHADOW(ptr_raw);
+  tag_t *shadow_last = (tag_t *)MEM_TO_SHADOW(ptr_raw + sz - 1);
+  for (tag_t *t = shadow_first; t <= shadow_last; ++t)
+    if (ptr_tag != *t) SigIll();
+}
+
+void __hwasan_load(uptr p, uptr sz) { CheckAddressSized(p, sz); }
+void __hwasan_load1(uptr p) { CheckAddress(p); }
+void __hwasan_load2(uptr p) { CheckAddress(p); }
+void __hwasan_load4(uptr p) { CheckAddress(p); }
+void __hwasan_load8(uptr p) { CheckAddress(p); }
+void __hwasan_load16(uptr p) { CheckAddress(p); }
+
+void __hwasan_store(uptr p, uptr sz) { CheckAddressSized(p, sz); }
+void __hwasan_store1(uptr p) { CheckAddress(p); }
+void __hwasan_store2(uptr p) { CheckAddress(p); }
+void __hwasan_store4(uptr p) { CheckAddress(p); }
+void __hwasan_store8(uptr p) { CheckAddress(p); }
+void __hwasan_store16(uptr p) { CheckAddress(p); }
+
+#if !SANITIZER_SUPPORTS_WEAK_HOOKS
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+const char* __hwasan_default_options() { return ""; }
+}  // extern "C"
+#endif
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_print_stack_trace() {
+  GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
+  stack.Print();
+}
+} // extern "C"

compiler-rt/lib/hwasan/hwasan.syms.extra

+__hwasan_*
+__ubsan_*

compiler-rt/lib/hwasan/hwasan_allocator.h

+//===-- hwasan_allocator.h ----------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_ALLOCATOR_H
+#define HWASAN_ALLOCATOR_H
+
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace __hwasan {
+
+struct HwasanThreadLocalMallocStorage {
+  uptr quarantine_cache[16];
+  // Allocator cache contains atomic_uint64_t which must be 8-byte aligned.
+  ALIGNED(8) uptr allocator_cache[96 * (512 * 8 + 16)];  // Opaque.
+  void CommitBack();
+
+ private:
+  // These objects are allocated via mmap() and are zero-initialized.
+  HwasanThreadLocalMallocStorage() {}
+};
+
+struct Metadata;
+
+class HwasanChunkView {
+ public:
+  HwasanChunkView() : block_(0), metadata_(nullptr) {}
+  HwasanChunkView(uptr block, Metadata *metadata)
+      : block_(block), metadata_(metadata) {}
+  bool IsValid() const;        // Checks if it points to a valid allocated chunk
+  bool IsAllocated() const;    // Checks if the memory is currently allocated
+  uptr Beg() const;            // First byte of user memory
+  uptr End() const;            // Last byte of user memory
+  uptr UsedSize() const;       // Size requested by the user
+  u32 GetAllocStackId() const;
+  u32 GetFreeStackId() const;
+ private:
+  uptr block_;
+  Metadata *const metadata_;
+};
+
+HwasanChunkView FindHeapChunkByAddress(uptr address);
+
+} // namespace __hwasan
+
+#endif // HWASAN_ALLOCATOR_H

compiler-rt/lib/hwasan/hwasan_allocator.cc

+//===-- hwasan_allocator.cc --------------------------- ---------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// HWAddressSanitizer allocator.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "hwasan.h"
+#include "hwasan_allocator.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+
+namespace __hwasan {
+
+enum {
+  CHUNK_INVALID = 0,
+  CHUNK_FREE = 1,
+  CHUNK_ALLOCATED = 2
+};
+
+struct Metadata {
+  u64 state : 2;
+  u64 requested_size : 62;
+  u32 alloc_context_id;
+  u32 free_context_id;
+};
+
+bool HwasanChunkView::IsValid() const {
+  return metadata_ && metadata_->state != CHUNK_INVALID;
+}
+bool HwasanChunkView::IsAllocated() const {
+  return metadata_ && metadata_->state == CHUNK_ALLOCATED;
+}
+uptr HwasanChunkView::Beg() const {
+  return block_;
+}
+uptr HwasanChunkView::End() const {
+  return Beg() + UsedSize();
+}
+uptr HwasanChunkView::UsedSize() const {
+  return metadata_->requested_size;
+}
+u32 HwasanChunkView::GetAllocStackId() const {
+  return metadata_->alloc_context_id;
+}
+u32 HwasanChunkView::GetFreeStackId() const {
+  return metadata_->free_context_id;
+}
+
+struct HwasanMapUnmapCallback {
+  void OnMap(uptr p, uptr size) const {}
+  void OnUnmap(uptr p, uptr size) const {
+    // We are about to unmap a chunk of user memory.
+    // It can return as user-requested mmap() or another thread stack.
+    // Make it accessible with zero-tagged pointer.
+    TagMemory(p, size, 0);
+  }
+};
+
+#if !defined(__aarch64__)
+#error unsupported platform
+#endif
+
+static const uptr kMaxAllowedMallocSize = 2UL << 30;  // 2G
+static const uptr kRegionSizeLog = 20;
+static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog;
+typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap;
+
+struct AP32 {
+  static const uptr kSpaceBeg = 0;
+  static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
+  static const uptr kMetadataSize = sizeof(Metadata);
+  typedef __sanitizer::CompactSizeClassMap SizeClassMap;
+  static const uptr kRegionSizeLog = __hwasan::kRegionSizeLog;
+  typedef __hwasan::ByteMap ByteMap;
+  typedef HwasanMapUnmapCallback MapUnmapCallback;
+  static const uptr kFlags = 0;
+};
+typedef SizeClassAllocator32<AP32> PrimaryAllocator;
+typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache;
+typedef LargeMmapAllocator<HwasanMapUnmapCallback> SecondaryAllocator;
+typedef CombinedAllocator<PrimaryAllocator, AllocatorCache,
+                          SecondaryAllocator> Allocator;
+
+static Allocator allocator;
+static AllocatorCache fallback_allocator_cache;
+static SpinMutex fallback_mutex;
+static atomic_uint8_t hwasan_allocator_tagging_enabled;
+
+void HwasanAllocatorInit() {
+  atomic_store_relaxed(&hwasan_allocator_tagging_enabled,
+                       !flags()->disable_allocator_tagging);
+  SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+  allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
+}
+
+AllocatorCache *GetAllocatorCache(HwasanThreadLocalMallocStorage *ms) {
+  CHECK(ms);
+  CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
+  return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
+}
+
+void HwasanThreadLocalMallocStorage::CommitBack() {
+  allocator.SwallowCache(GetAllocatorCache(this));
+}
+
+static void *HwasanAllocate(StackTrace *stack, uptr size, uptr alignment,
+                          bool zeroise) {
+  alignment = Max(alignment, kShadowAlignment);
+  size = RoundUpTo(size, kShadowAlignment);
+
+  if (size > kMaxAllowedMallocSize) {
+    Report("WARNING: HWAddressSanitizer failed to allocate %p bytes\n",
+           (void *)size);
+    return Allocator::FailureHandler::OnBadRequest();
+  }
+  HwasanThread *t = GetCurrentThread();
+  void *allocated;
+  if (t) {
+    AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+    allocated = allocator.Allocate(cache, size, alignment);
+  } else {
+    SpinMutexLock l(&fallback_mutex);
+    AllocatorCache *cache = &fallback_allocator_cache;
+    allocated = allocator.Allocate(cache, size, alignment);
+  }
+  Metadata *meta =
+      reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
+  meta->state = CHUNK_ALLOCATED;
+  meta->requested_size = size;
+  meta->alloc_context_id = StackDepotPut(*stack);
+  if (zeroise)
+    internal_memset(allocated, 0, size);
+
+  void *user_ptr = (flags()->tag_in_malloc &&
+                    atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+                       ? (void *)TagMemoryAligned((uptr)allocated, size, 0xBB)
+                       : allocated;
+
+  HWASAN_MALLOC_HOOK(user_ptr, size);
+  return user_ptr;
+}
+
+void HwasanDeallocate(StackTrace *stack, void *user_ptr) {
+  CHECK(user_ptr);
+  HWASAN_FREE_HOOK(user_ptr);
+
+  void *p = GetAddressFromPointer(user_ptr);
+  Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
+  uptr size = meta->requested_size;
+  meta->state = CHUNK_FREE;
+  meta->requested_size = 0;
+  meta->free_context_id = StackDepotPut(*stack);
+  // This memory will not be reused by anyone else, so we are free to keep it
+  // poisoned.
+  if (flags()->tag_in_free &&
+      atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+    TagMemoryAligned((uptr)p, size, 0xBC);
+  HwasanThread *t = GetCurrentThread();
+  if (t) {
+    AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+    allocator.Deallocate(cache, p);
+  } else {
+    SpinMutexLock l(&fallback_mutex);
+    AllocatorCache *cache = &fallback_allocator_cache;
+    allocator.Deallocate(cache, p);
+  }
+}
+
+void *HwasanReallocate(StackTrace *stack, void *user_old_p, uptr new_size,
+                     uptr alignment) {
+  alignment = Max(alignment, kShadowAlignment);
+  new_size = RoundUpTo(new_size, kShadowAlignment);
+
+  void *old_p = GetAddressFromPointer(user_old_p);
+  Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(old_p));
+  uptr old_size = meta->requested_size;
+  uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p);
+  if (new_size <= actually_allocated_size) {
+    // We are not reallocating here.
+    // FIXME: update stack trace for the allocation?
+    meta->requested_size = new_size;
+    if (!atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+      return user_old_p;
+    if (flags()->retag_in_realloc)
+      return (void *)TagMemoryAligned((uptr)old_p, new_size, 0xCC);
+    if (new_size > old_size) {
+      tag_t tag = GetTagFromPointer((uptr)user_old_p);
+      TagMemoryAligned((uptr)old_p + old_size, new_size - old_size, tag);
+    }
+    return user_old_p;
+  }
+  uptr memcpy_size = Min(new_size, old_size);
+  void *new_p = HwasanAllocate(stack, new_size, alignment, false /*zeroise*/);
+  if (new_p) {
+    internal_memcpy(new_p, old_p, memcpy_size);
+    HwasanDeallocate(stack, old_p);
+  }
+  return new_p;
+}
+
+HwasanChunkView FindHeapChunkByAddress(uptr address) {
+  void *block = allocator.GetBlockBegin(reinterpret_cast<void*>(address));
+  if (!block)
+    return HwasanChunkView();
+  Metadata *metadata =
+      reinterpret_cast<Metadata*>(allocator.GetMetaData(block));
+  return HwasanChunkView(reinterpret_cast<uptr>(block), metadata);
+}
+
+static uptr AllocationSize(const void *user_ptr) {
+  const void *p = GetAddressFromPointer(user_ptr);
+  if (!p) return 0;
+  const void *beg = allocator.GetBlockBegin(p);
+  if (beg != p) return 0;
+  Metadata *b = (Metadata *)allocator.GetMetaData(p);
+  return b->requested_size;
+}
+
+void *hwasan_malloc(uptr size, StackTrace *stack) {
+  return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false));
+}
+
+void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) {
+  if (UNLIKELY(CheckForCallocOverflow(size, nmemb)))
+    return SetErrnoOnNull(Allocator::FailureHandler::OnBadRequest());
+  return SetErrnoOnNull(HwasanAllocate(stack, nmemb * size, sizeof(u64), true));
+}
+
+void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) {
+  if (!ptr)
+    return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false));
+  if (size == 0) {
+    HwasanDeallocate(stack, ptr);
+    return nullptr;
+  }
+  return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64)));
+}
+
+void *hwasan_valloc(uptr size, StackTrace *stack) {
+  return SetErrnoOnNull(HwasanAllocate(stack, size, GetPageSizeCached(), false));
+}
+
+void *hwasan_pvalloc(uptr size, StackTrace *stack) {
+  uptr PageSize = GetPageSizeCached();
+  if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
+    errno = errno_ENOMEM;
+    return Allocator::FailureHandler::OnBadRequest();
+  }
+  // pvalloc(0) should allocate one page.
+  size = size ? RoundUpTo(size, PageSize) : PageSize;
+  return SetErrnoOnNull(HwasanAllocate(stack, size, PageSize, false));
+}
+
+void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) {
+  if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
+    errno = errno_EINVAL;
+    return Allocator::FailureHandler::OnBadRequest();
+  }
+  return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false));
+}
+
+void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) {
+  if (UNLIKELY(!IsPowerOfTwo(alignment))) {
+    errno = errno_EINVAL;
+    return Allocator::FailureHandler::OnBadRequest();
+  }
+  return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false));
+}
+
+int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
+                        StackTrace *stack) {
+  if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
+    Allocator::FailureHandler::OnBadRequest();
+    return errno_EINVAL;
+  }
+  void *ptr = HwasanAllocate(stack, size, alignment, false);
+  if (UNLIKELY(!ptr))
+    return errno_ENOMEM;
+  CHECK(IsAligned((uptr)ptr, alignment));
+  *memptr = ptr;
+  return 0;
+}
+
+} // namespace __hwasan
+
+using namespace __hwasan;
+
+void __hwasan_enable_allocator_tagging() {
+  atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 1);
+}
+
+void __hwasan_disable_allocator_tagging() {
+  atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 0);
+}
+
+uptr __sanitizer_get_current_allocated_bytes() {
+  uptr stats[AllocatorStatCount];
+  allocator.GetStats(stats);
+  return stats[AllocatorStatAllocated];
+}
+
+uptr __sanitizer_get_heap_size() {
+  uptr stats[AllocatorStatCount];
+  allocator.GetStats(stats);
+  return stats[AllocatorStatMapped];
+}
+
+uptr __sanitizer_get_free_bytes() { return 1; }
+
+uptr __sanitizer_get_unmapped_bytes() { return 1; }
+
+uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
+
+int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
+
+uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }

compiler-rt/lib/hwasan/hwasan_blacklist.txt

+# Blacklist for HWAddressSanitizer. Turns off instrumentation of particular
+# functions or sources. Use with care. You may set location of blacklist
+# at compile-time using -fsanitize-blacklist=<path> flag.
+
+# Example usage:
+# fun:*bad_function_name*
+# src:file_with_tricky_code.cc

compiler-rt/lib/hwasan/hwasan_flags.h

+//===-- hwasan_flags.h --------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+#ifndef HWASAN_FLAGS_H
+#define HWASAN_FLAGS_H
+
+namespace __hwasan {
+
+struct Flags {
+#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "hwasan_flags.inc"
+#undef HWASAN_FLAG
+
+  void SetDefaults();
+};
+
+Flags *flags();
+
+}  // namespace __hwasan
+
+#endif  // HWASAN_FLAGS_H

compiler-rt/lib/hwasan/hwasan_flags.inc

+//===-- hwasan_flags.inc ------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Hwasan runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef HWASAN_FLAG
+# error "Define HWASAN_FLAG prior to including this file!"
+#endif
+
+// HWASAN_FLAG(Type, Name, DefaultValue, Description)
+// See COMMON_FLAG in sanitizer_flags.inc for more details.
+
+HWASAN_FLAG(bool, tag_in_malloc, true, "")
+HWASAN_FLAG(bool, tag_in_free, true, "")
+HWASAN_FLAG(bool, retag_in_realloc, true, "")
+HWASAN_FLAG(bool, print_stats, false, "")
+HWASAN_FLAG(bool, halt_on_error, true, "")
+HWASAN_FLAG(bool, atexit, false, "")
+
+// Test only flag to disable malloc/realloc/free memory tagging on startup.
+// Tagging can be reenabled with __hwasan_enable_allocator_tagging().
+HWASAN_FLAG(bool, disable_allocator_tagging, false, "")

compiler-rt/lib/hwasan/hwasan_interceptors.cc

+//===-- hwasan_interceptors.cc ----------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Interceptors for standard library functions.
+//
+// FIXME: move as many interceptors as possible into
+// sanitizer_common/sanitizer_common_interceptors.h
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "hwasan.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+#include <stdarg.h>
+// ACHTUNG! No other system header includes in this file.
+// Ideally, we should get rid of stdarg.h as well.
+
+using namespace __hwasan;
+
+using __sanitizer::memory_order;
+using __sanitizer::atomic_load;
+using __sanitizer::atomic_store;
+using __sanitizer::atomic_uintptr_t;
+
+DECLARE_REAL(SIZE_T, strlen, const char *s)
+DECLARE_REAL(SIZE_T, strnlen, const char *s, SIZE_T maxlen)
+DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n)
+DECLARE_REAL(void *, memset, void *dest, int c, uptr n)
+
+bool IsInInterceptorScope() {
+  HwasanThread *t = GetCurrentThread();
+  return t && t->InInterceptorScope();
+}
+
+struct InterceptorScope {
+  InterceptorScope() {
+    HwasanThread *t = GetCurrentThread();
+    if (t)
+      t->EnterInterceptorScope();
+  }
+  ~InterceptorScope() {
+    HwasanThread *t = GetCurrentThread();
+    if (t)
+      t->LeaveInterceptorScope();
+  }
+};
+
+static uptr allocated_for_dlsym;
+static const uptr kDlsymAllocPoolSize = 1024;
+static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
+
+static bool IsInDlsymAllocPool(const void *ptr) {
+  uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+  return off < sizeof(alloc_memory_for_dlsym);
+}
+
+static void *AllocateFromLocalPool(uptr size_in_bytes) {
+  uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
+  void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
+  allocated_for_dlsym += size_in_words;
+  CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
+  return mem;
+}
+
+#define ENSURE_HWASAN_INITED() do { \
+  CHECK(!hwasan_init_is_running); \
+  if (!hwasan_inited) { \
+    __hwasan_init(); \
+  } \
+} while (0)
+
+
+
+#define HWASAN_READ_RANGE(ctx, offset, size) \
+  CHECK_UNPOISONED(offset, size)
+#define HWASAN_WRITE_RANGE(ctx, offset, size) \
+  CHECK_UNPOISONED(offset, size)
+
+
+
+// Check that [x, x+n) range is unpoisoned.
+#define CHECK_UNPOISONED_0(x, n)                                       \
+  do {                                                                 \
+    sptr __offset = __hwasan_test_shadow(x, n);                         \
+    if (__hwasan::IsInSymbolizer()) break;                              \
+    if (__offset >= 0) {                                               \
+      GET_CALLER_PC_BP_SP;                                             \
+      (void)sp;                                                        \
+      ReportInvalidAccessInsideAddressRange(__func__, x, n, __offset); \
+      __hwasan::PrintWarning(pc, bp);                                   \
+      if (__hwasan::flags()->halt_on_error) {                           \
+        Printf("Exiting\n");                                           \
+        Die();                                                         \
+      }                                                                \
+    }                                                                  \
+  } while (0)
+
+// Check that [x, x+n) range is unpoisoned unless we are in a nested
+// interceptor.
+#define CHECK_UNPOISONED(x, n)                             \
+  do {                                                     \
+    if (!IsInInterceptorScope()) CHECK_UNPOISONED_0(x, n); \
+  } while (0)
+
+#define CHECK_UNPOISONED_STRING_OF_LEN(x, len, n)               \
+  CHECK_UNPOISONED((x),                                         \
+    common_flags()->strict_string_checks ? (len) + 1 : (n) )
+
+
+INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) {
+  GET_MALLOC_STACK_TRACE;
+  CHECK_NE(memptr, 0);
+  int res = hwasan_posix_memalign(memptr, alignment, size, &stack);
+  return res;
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void *, memalign, SIZE_T alignment, SIZE_T size) {
+  GET_MALLOC_STACK_TRACE;
+  return hwasan_memalign(alignment, size, &stack);
+}
+#define HWASAN_MAYBE_INTERCEPT_MEMALIGN INTERCEPT_FUNCTION(memalign)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MEMALIGN
+#endif
+
+INTERCEPTOR(void *, aligned_alloc, SIZE_T alignment, SIZE_T size) {
+  GET_MALLOC_STACK_TRACE;
+  return hwasan_aligned_alloc(alignment, size, &stack);
+}
+
+INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) {
+  GET_MALLOC_STACK_TRACE;
+  void *ptr = hwasan_memalign(alignment, size, &stack);
+  if (ptr)
+    DTLS_on_libc_memalign(ptr, size);
+  return ptr;
+}
+
+INTERCEPTOR(void *, valloc, SIZE_T size) {
+  GET_MALLOC_STACK_TRACE;
+  return hwasan_valloc(size, &stack);
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void *, pvalloc, SIZE_T size) {
+  GET_MALLOC_STACK_TRACE;
+  return hwasan_pvalloc(size, &stack);
+}
+#define HWASAN_MAYBE_INTERCEPT_PVALLOC INTERCEPT_FUNCTION(pvalloc)
+#else
+#define HWASAN_MAYBE_INTERCEPT_PVALLOC
+#endif
+
+INTERCEPTOR(void, free, void *ptr) {
+  GET_MALLOC_STACK_TRACE;
+  if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
+  HwasanDeallocate(&stack, ptr);
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void, cfree, void *ptr) {
+  GET_MALLOC_STACK_TRACE;
+  if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
+  HwasanDeallocate(&stack, ptr);
+}
+#define HWASAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree)
+#else
+#define HWASAN_MAYBE_INTERCEPT_CFREE
+#endif
+
+INTERCEPTOR(uptr, malloc_usable_size, void *ptr) {
+  return __sanitizer_get_allocated_size(ptr);
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+// This function actually returns a struct by value, but we can't unpoison a
+// temporary! The following is equivalent on all supported platforms but
+// aarch64 (which uses a different register for sret value).  We have a test
+// to confirm that.
+INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) {
+#ifdef __aarch64__
+  uptr r8;
+  asm volatile("mov %0,x8" : "=r" (r8));
+  sret = reinterpret_cast<__sanitizer_mallinfo*>(r8);
+#endif
+  REAL(memset)(sret, 0, sizeof(*sret));
+}
+#define HWASAN_MAYBE_INTERCEPT_MALLINFO INTERCEPT_FUNCTION(mallinfo)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MALLINFO
+#endif
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(int, mallopt, int cmd, int value) {
+  return -1;
+}
+#define HWASAN_MAYBE_INTERCEPT_MALLOPT INTERCEPT_FUNCTION(mallopt)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MALLOPT
+#endif
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void, malloc_stats, void) {
+  // FIXME: implement, but don't call REAL(malloc_stats)!
+}
+#define HWASAN_MAYBE_INTERCEPT_MALLOC_STATS INTERCEPT_FUNCTION(malloc_stats)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MALLOC_STATS
+#endif
+
+
+INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) {
+  GET_MALLOC_STACK_TRACE;
+  if (UNLIKELY(!hwasan_inited))
+    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
+    return AllocateFromLocalPool(nmemb * size);
+  return hwasan_calloc(nmemb, size, &stack);
+}
+
+INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) {
+  GET_MALLOC_STACK_TRACE;
+  if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
+    uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+    uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
+    void *new_ptr;
+    if (UNLIKELY(!hwasan_inited)) {
+      new_ptr = AllocateFromLocalPool(copy_size);
+    } else {
+      copy_size = size;
+      new_ptr = hwasan_malloc(copy_size, &stack);
+    }
+    internal_memcpy(new_ptr, ptr, copy_size);
+    return new_ptr;
+  }
+  return hwasan_realloc(ptr, size, &stack);
+}
+
+INTERCEPTOR(void *, malloc, SIZE_T size) {
+  GET_MALLOC_STACK_TRACE;
+  if (UNLIKELY(!hwasan_inited))
+    // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
+    return AllocateFromLocalPool(size);
+  return hwasan_malloc(size, &stack);
+}
+
+
+INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags,
+            int fd, OFF_T offset) {
+  if (hwasan_init_is_running)
+    return REAL(mmap)(addr, length, prot, flags, fd, offset);
+  ENSURE_HWASAN_INITED();
+  if (addr && !MEM_IS_APP(addr)) {
+    if (flags & map_fixed) {
+      errno = errno_EINVAL;
+      return (void *)-1;
+    } else {
+      addr = nullptr;
+    }
+  }
+  void *res = REAL(mmap)(addr, length, prot, flags, fd, offset);
+  return res;
+}
+
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags,
+            int fd, OFF64_T offset) {
+  ENSURE_HWASAN_INITED();
+  if (addr && !MEM_IS_APP(addr)) {
+    if (flags & map_fixed) {
+      errno = errno_EINVAL;
+      return (void *)-1;
+    } else {
+      addr = nullptr;
+    }
+  }
+  void *res = REAL(mmap64)(addr, length, prot, flags, fd, offset);
+  return res;
+}
+#define HWASAN_MAYBE_INTERCEPT_MMAP64 INTERCEPT_FUNCTION(mmap64)
+#else
+#define HWASAN_MAYBE_INTERCEPT_MMAP64
+#endif
+
+extern "C" int pthread_attr_init(void *attr);
+extern "C" int pthread_attr_destroy(void *attr);
+
+static void *HwasanThreadStartFunc(void *arg) {
+  HwasanThread *t = (HwasanThread *)arg;
+  SetCurrentThread(t);
+  return t->ThreadStart();
+}
+
+INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*),
+            void * param) {
+  ENSURE_HWASAN_INITED(); // for GetTlsSize()
+  __sanitizer_pthread_attr_t myattr;
+  if (!attr) {
+    pthread_attr_init(&myattr);
+    attr = &myattr;
+  }
+
+  AdjustStackSize(attr);
+
+  HwasanThread *t = HwasanThread::Create(callback, param);
+
+  int res = REAL(pthread_create)(th, attr, HwasanThreadStartFunc, t);
+
+  if (attr == &myattr)
+    pthread_attr_destroy(&myattr);
+  return res;
+}
+
+static void BeforeFork() {
+  StackDepotLockAll();
+}
+
+static void AfterFork() {
+  StackDepotUnlockAll();
+}
+
+INTERCEPTOR(int, fork, void) {
+  ENSURE_HWASAN_INITED();
+  BeforeFork();
+  int pid = REAL(fork)();
+  AfterFork();
+  return pid;
+}
+
+
+struct HwasanInterceptorContext {
+  bool in_interceptor_scope;
+};
+
+namespace __hwasan {
+
+int OnExit() {
+  // FIXME: ask frontend whether we need to return failure.
+  return 0;
+}
+
+} // namespace __hwasan
+
+// A version of CHECK_UNPOISONED using a saved scope value. Used in common
+// interceptors.
+#define CHECK_UNPOISONED_CTX(ctx, x, n)                         \
+  do {                                                          \
+    if (!((HwasanInterceptorContext *)ctx)->in_interceptor_scope) \
+      CHECK_UNPOISONED_0(x, n);                                 \
+  } while (0)
+
+#define HWASAN_INTERCEPT_FUNC(name)                                       \
+  do {                                                                  \
+    if ((!INTERCEPT_FUNCTION(name) || !REAL(name)))                     \
+      VReport(1, "HWAddressSanitizer: failed to intercept '" #name "'\n"); \
+  } while (0)
+
+#define HWASAN_INTERCEPT_FUNC_VER(name, ver)                                    \
+  do {                                                                        \
+    if ((!INTERCEPT_FUNCTION_VER(name, ver) || !REAL(name)))                  \
+      VReport(                                                                \
+          1, "HWAddressSanitizer: failed to intercept '" #name "@@" #ver "'\n"); \
+  } while (0)
+
+#define COMMON_INTERCEPT_FUNCTION(name) HWASAN_INTERCEPT_FUNC(name)
+#define COMMON_INTERCEPT_FUNCTION_VER(name, ver)                          \
+  HWASAN_INTERCEPT_FUNC_VER(name, ver)
+#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
+  CHECK_UNPOISONED_CTX(ctx, ptr, size)
+#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
+  CHECK_UNPOISONED_CTX(ctx, ptr, size)
+#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(ptr, size) \
+  HWASAN_WRITE_RANGE(ctx, ptr, size)
+#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...)                  \
+  if (hwasan_init_is_running) return REAL(func)(__VA_ARGS__);       \
+  ENSURE_HWASAN_INITED();                                           \
+  HwasanInterceptorContext hwasan_ctx = {IsInInterceptorScope()};     \
+  ctx = (void *)&hwasan_ctx;                                        \
+  (void)ctx;                                                      \
+  InterceptorScope interceptor_scope;
+#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \
+  do {                                            \
+  } while (false)
+#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
+  do {                                         \
+  } while (false)
+#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
+  do {                                         \
+  } while (false)
+#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
+  do {                                                      \
+  } while (false)
+#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
+  do {                                                \
+  } while (false)  // FIXME
+#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
+  do {                                                         \
+  } while (false)  // FIXME
+#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name)
+#define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit()
+
+#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end)                           \
+  if (HwasanThread *t = GetCurrentThread()) {                                    \
+    *begin = t->tls_begin();                                                   \
+    *end = t->tls_end();                                                       \
+  } else {                                                                     \
+    *begin = *end = 0;                                                         \
+  }
+
+#include "sanitizer_common/sanitizer_platform_interceptors.h"
+#include "sanitizer_common/sanitizer_common_interceptors.inc"
+#include "sanitizer_common/sanitizer_signal_interceptors.inc"
+
+#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) CHECK_UNPOISONED(p, s)
+#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
+  do {                                       \
+    (void)(p);                               \
+    (void)(s);                               \
+  } while (false)
+#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
+  do {                                       \
+    (void)(p);                               \
+    (void)(s);                               \
+  } while (false)
+#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
+  do {                                        \
+    (void)(p);                                \
+    (void)(s);                                \
+  } while (false)
+#include "sanitizer_common/sanitizer_common_syscalls.inc"
+
+
+
+namespace __hwasan {
+
+void InitializeInterceptors() {
+  static int inited = 0;
+  CHECK_EQ(inited, 0);
+  InitializeCommonInterceptors();
+  InitializeSignalInterceptors();
+
+  INTERCEPT_FUNCTION(mmap);
+  HWASAN_MAYBE_INTERCEPT_MMAP64;
+  INTERCEPT_FUNCTION(posix_memalign);
+  HWASAN_MAYBE_INTERCEPT_MEMALIGN;
+  INTERCEPT_FUNCTION(__libc_memalign);
+  INTERCEPT_FUNCTION(valloc);
+  HWASAN_MAYBE_INTERCEPT_PVALLOC;
+  INTERCEPT_FUNCTION(malloc);
+  INTERCEPT_FUNCTION(calloc);
+  INTERCEPT_FUNCTION(realloc);
+  INTERCEPT_FUNCTION(free);
+  HWASAN_MAYBE_INTERCEPT_CFREE;
+  INTERCEPT_FUNCTION(malloc_usable_size);
+  HWASAN_MAYBE_INTERCEPT_MALLINFO;
+  HWASAN_MAYBE_INTERCEPT_MALLOPT;
+  HWASAN_MAYBE_INTERCEPT_MALLOC_STATS;
+  INTERCEPT_FUNCTION(pthread_create);
+  INTERCEPT_FUNCTION(fork);
+
+  inited = 1;
+}
+} // namespace __hwasan

compiler-rt/lib/hwasan/hwasan_interface_internal.h

+//===-- hwasan_interface_internal.h -------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Private Hwasan interface header.
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_INTERFACE_INTERNAL_H
+#define HWASAN_INTERFACE_INTERNAL_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_init();
+
+using __sanitizer::uptr;
+using __sanitizer::sptr;
+using __sanitizer::uu64;
+using __sanitizer::uu32;
+using __sanitizer::uu16;
+using __sanitizer::u64;
+using __sanitizer::u32;
+using __sanitizer::u16;
+using __sanitizer::u8;
+
+SANITIZER_INTERFACE_ATTRIBUTE
+extern uptr __hwasan_shadow_memory_dynamic_address;
+
+// Hidden alias for internal access.
+__attribute__((visibility("hidden")))
+extern uptr __hwasan_shadow_memory_dynamic_address_internal;
+
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load(uptr, uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load1(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load2(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load4(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load8(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load16(uptr);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store(uptr, uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store1(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store2(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store4(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store8(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store16(uptr);
+
+// Returns the offset of the first tag mismatch or -1 if the whole range is
+// good.
+SANITIZER_INTERFACE_ATTRIBUTE
+sptr __hwasan_test_shadow(const void *x, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+/* OPTIONAL */ const char* __hwasan_default_options();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_print_shadow(const void *x, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u16 __sanitizer_unaligned_load16(const uu16 *p);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u32 __sanitizer_unaligned_load32(const uu32 *p);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+u64 __sanitizer_unaligned_load64(const uu64 *p);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store16(uu16 *p, u16 x);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store32(uu32 *p, u32 x);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_unaligned_store64(uu64 *p, u64 x);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_enable_allocator_tagging();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_disable_allocator_tagging();
+
+}  // extern "C"
+
+#endif  // HWASAN_INTERFACE_INTERNAL_H

compiler-rt/lib/hwasan/hwasan_linux.cc

+//===-- hwasan_linux.cc -----------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Linux-, NetBSD- and FreeBSD-specific code.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
+
+#include "hwasan.h"
+#include "hwasan_thread.h"
+
+#include <elf.h>
+#include <link.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <unistd.h>
+#include <unwind.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+
+uptr __hwasan_shadow_memory_dynamic_address;
+
+__attribute__((alias("__hwasan_shadow_memory_dynamic_address")))
+extern uptr __hwasan_shadow_memory_dynamic_address_internal;
+
+namespace __hwasan {
+
+bool InitShadow() {
+  const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
+  uptr shadow_size = MEM_TO_SHADOW_OFFSET(maxVirtualAddress) + 1;
+  __hwasan_shadow_memory_dynamic_address =
+      reinterpret_cast<uptr>(MmapNoReserveOrDie(shadow_size, "shadow"));
+  return true;
+}
+
+static void HwasanAtExit(void) {
+  if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
+    ReportStats();
+  if (hwasan_report_count > 0) {
+    // ReportAtExitStatistics();
+    if (common_flags()->exitcode)
+      internal__exit(common_flags()->exitcode);
+  }
+}
+
+void InstallAtExitHandler() {
+  atexit(HwasanAtExit);
+}
+
+// ---------------------- TSD ---------------- {{{1
+
+static pthread_key_t tsd_key;
+static bool tsd_key_inited = false;
+
+void HwasanTSDInit(void (*destructor)(void *tsd)) {
+  CHECK(!tsd_key_inited);
+  tsd_key_inited = true;
+  CHECK_EQ(0, pthread_key_create(&tsd_key, destructor));
+}
+
+HwasanThread *GetCurrentThread() {
+  return (HwasanThread*)pthread_getspecific(tsd_key);
+}
+
+void SetCurrentThread(HwasanThread *t) {
+  // Make sure that HwasanTSDDtor gets called at the end.
+  CHECK(tsd_key_inited);
+  // Make sure we do not reset the current HwasanThread.
+  CHECK_EQ(0, pthread_getspecific(tsd_key));
+  pthread_setspecific(tsd_key, (void *)t);
+}
+
+void HwasanTSDDtor(void *tsd) {
+  HwasanThread *t = (HwasanThread*)tsd;
+  if (t->destructor_iterations_ > 1) {
+    t->destructor_iterations_--;
+    CHECK_EQ(0, pthread_setspecific(tsd_key, tsd));
+    return;
+  }
+  // Make sure that signal handler can not see a stale current thread pointer.
+  atomic_signal_fence(memory_order_seq_cst);
+  HwasanThread::TSDDtor(tsd);
+}
+
+struct AccessInfo {
+  uptr addr;
+  uptr size;
+  bool is_store;
+  bool is_load;
+};
+
+#if defined(__aarch64__)
+static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
+  AccessInfo ai;
+  uptr pc = (uptr)info->si_addr;
+
+  struct {
+    uptr addr;
+    unsigned size;
+    bool is_store;
+  } handlers[] = {
+      {(uptr)&__hwasan_load1, 1, false},   {(uptr)&__hwasan_load2, 2, false},
+      {(uptr)&__hwasan_load4, 4, false},   {(uptr)&__hwasan_load8, 8, false},
+      {(uptr)&__hwasan_load16, 16, false},  {(uptr)&__hwasan_load, 0, false},
+      {(uptr)&__hwasan_store1, 1, true},  {(uptr)&__hwasan_store2, 2, true},
+      {(uptr)&__hwasan_store4, 4, true},  {(uptr)&__hwasan_store8, 8, true},
+      {(uptr)&__hwasan_store16, 16, true}, {(uptr)&__hwasan_store, 0, true}};
+  int best = -1;
+  uptr best_distance = 0;
+  for (size_t i = 0; i < sizeof(handlers) / sizeof(handlers[0]); ++i) {
+    uptr handler = handlers[i].addr;
+    // Don't accept pc == handler: HLT is never the first instruction.
+    if (pc <= handler) continue;
+    uptr distance = pc - handler;
+    if (distance > 256) continue;
+    if (best == -1 || best_distance > distance) {
+      best = i;
+      best_distance = distance;
+    }
+  }
+
+  // Not ours.
+  if (best == -1)
+    return AccessInfo{0, 0, false, false};
+
+  ai.is_store = handlers[best].is_store;
+  ai.is_load = !handlers[best].is_store;
+  ai.size = handlers[best].size;
+
+  ai.addr = uc->uc_mcontext.regs[0];
+  if (ai.size == 0)
+    ai.size = uc->uc_mcontext.regs[1];
+  return ai;
+}
+#else
+static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
+  return AccessInfo{0, 0, false, false};
+}
+#endif
+
+static void HwasanOnSIGILL(int signo, siginfo_t *info, ucontext_t *uc) {
+  SignalContext sig{info, uc};
+  AccessInfo ai = GetAccessInfo(info, uc);
+  if (!ai.is_store && !ai.is_load)
+    return;
+
+  InternalScopedBuffer<BufferedStackTrace> stack_buffer(1);
+  BufferedStackTrace *stack = stack_buffer.data();
+  stack->Reset();
+  GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, uc,
+                common_flags()->fast_unwind_on_fatal);
+
+  ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store);
+
+  ++hwasan_report_count;
+  if (flags()->halt_on_error)
+    Die();
+  else
+    uc->uc_mcontext.pc += 4;
+}
+
+static void OnStackUnwind(const SignalContext &sig, const void *,
+                          BufferedStackTrace *stack) {
+  GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context,
+                common_flags()->fast_unwind_on_fatal);
+}
+
+void HwasanOnDeadlySignal(int signo, void *info, void *context) {
+  // Probably a tag mismatch.
+  // FIXME: detect pc range in __hwasan_load* or __hwasan_store*.
+  if (signo == SIGILL)
+    HwasanOnSIGILL(signo, (siginfo_t *)info, (ucontext_t*)context);
+  else
+    HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
+}
+
+
+} // namespace __hwasan
+
+#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD

compiler-rt/lib/hwasan/hwasan_new_delete.cc

+//===-- hwasan_new_delete.cc ------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Interceptors for operators new and delete.
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+
+#if HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
+
+#include <stddef.h>
+
+using namespace __hwasan;  // NOLINT
+
+// Fake std::nothrow_t to avoid including <new>.
+namespace std {
+  struct nothrow_t {};
+}  // namespace std
+
+
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+#define OPERATOR_NEW_BODY(nothrow) \
+  GET_MALLOC_STACK_TRACE; \
+  void *res = hwasan_malloc(size, &stack);\
+  if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\
+  return res
+
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size, std::nothrow_t const&) {
+  OPERATOR_NEW_BODY(true /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size, std::nothrow_t const&) {
+  OPERATOR_NEW_BODY(true /*nothrow*/);
+}
+
+#define OPERATOR_DELETE_BODY \
+  GET_MALLOC_STACK_TRACE; \
+  if (ptr) HwasanDeallocate(&stack, ptr)
+
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, std::nothrow_t const&) {
+  OPERATOR_DELETE_BODY;
+}
+
+#endif // HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE

compiler-rt/lib/hwasan/hwasan_poisoning.h

+//===-- hwasan_poisoning.h ----------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_POISONING_H
+#define HWASAN_POISONING_H
+
+#include "hwasan.h"
+
+namespace __hwasan {
+uptr TagMemory(uptr p, uptr size, tag_t tag);
+uptr TagMemoryAligned(uptr p, uptr size, tag_t tag);
+
+}  // namespace __hwasan
+
+#endif  // HWASAN_POISONING_H

compiler-rt/lib/hwasan/hwasan_poisoning.cc

+//===-- hwasan_poisoning.cc ---------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "hwasan_poisoning.h"
+
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace __hwasan {
+
+uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
+  CHECK(IsAligned(p, kShadowAlignment));
+  CHECK(IsAligned(size, kShadowAlignment));
+  uptr shadow_start = MEM_TO_SHADOW(p);
+  uptr shadow_size = MEM_TO_SHADOW_OFFSET(size);
+  internal_memset((void *)shadow_start, tag, shadow_size);
+  return AddTagToPointer(p, tag);
+}
+
+uptr TagMemory(uptr p, uptr size, tag_t tag) {
+  uptr start = RoundDownTo(p, kShadowAlignment);
+  uptr end = RoundUpTo(p + size, kShadowAlignment);
+  return TagMemoryAligned(start, end - start, tag);
+}
+
+}  // namespace __hwasan

compiler-rt/lib/hwasan/hwasan_report.cc

+//===-- hwasan_report.cc ----------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Error reporting.
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "hwasan_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_mutex.h"
+#include "sanitizer_common/sanitizer_report_decorator.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+
+using namespace __sanitizer;
+
+namespace __hwasan {
+
+static StackTrace GetStackTraceFromId(u32 id) {
+  CHECK(id);
+  StackTrace res = StackDepotGet(id);
+  CHECK(res.trace);
+  return res;
+}
+
+class Decorator: public __sanitizer::SanitizerCommonDecorator {
+ public:
+  Decorator() : SanitizerCommonDecorator() { }
+  const char *Allocation() { return Magenta(); }
+  const char *Origin() { return Magenta(); }
+  const char *Name() { return Green(); }
+};
+
+struct HeapAddressDescription {
+  uptr addr;
+  u32 alloc_stack_id;
+  u32 free_stack_id;
+
+  void Print() const {
+    Decorator d;
+    if (free_stack_id) {
+      Printf("%sfreed here:%s\n", d.Allocation(), d.Default());
+      GetStackTraceFromId(free_stack_id).Print();
+      Printf("%spreviously allocated here:%s\n", d.Allocation(), d.Default());
+    } else {
+      Printf("%sallocated here:%s\n", d.Allocation(), d.Default());
+    }
+    GetStackTraceFromId(alloc_stack_id).Print();
+  }
+};
+
+bool GetHeapAddressInformation(uptr addr, uptr access_size,
+                               HeapAddressDescription *description) {
+  HwasanChunkView chunk = FindHeapChunkByAddress(addr);
+  if (!chunk.IsValid())
+    return false;
+  description->addr = addr;
+  description->alloc_stack_id = chunk.GetAllocStackId();
+  description->free_stack_id = chunk.GetFreeStackId();
+  return true;
+}
+
+void PrintAddressDescription(uptr addr, uptr access_size) {
+  HeapAddressDescription heap_description;
+  if (GetHeapAddressInformation(addr, access_size, &heap_description)) {
+    heap_description.Print();
+    return;
+  }
+  // We exhausted our possibilities. Bail out.
+  Printf("HWAddressSanitizer can not describe address in more detail.\n");
+}
+
+void ReportInvalidAccess(StackTrace *stack, u32 origin) {
+  ScopedErrorReportLock l;
+
+  Decorator d;
+  Printf("%s", d.Warning());
+  Report("WARNING: HWAddressSanitizer: invalid access\n");
+  Printf("%s", d.Default());
+  stack->Print();
+  ReportErrorSummary("invalid-access", stack);
+}
+
+void ReportStats() {}
+
+void ReportInvalidAccessInsideAddressRange(const char *what, const void *start,
+                                           uptr size, uptr offset) {
+  ScopedErrorReportLock l;
+
+  Decorator d;
+  Printf("%s", d.Warning());
+  Printf("%sTag mismatch in %s%s%s at offset %zu inside [%p, %zu)%s\n",
+         d.Warning(), d.Name(), what, d.Warning(), offset, start, size,
+         d.Default());
+  PrintAddressDescription((uptr)start + offset, 1);
+  // if (__sanitizer::Verbosity())
+  //   DescribeMemoryRange(start, size);
+}
+
+void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size,
+                       bool is_store) {
+  ScopedErrorReportLock l;
+
+  Decorator d;
+  Printf("%s", d.Warning());
+  uptr address = GetAddressFromPointer(addr);
+  Printf("%s of size %zu at %p\n", is_store ? "WRITE" : "READ", access_size,
+         address);
+
+  tag_t ptr_tag = GetTagFromPointer(addr);
+  tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(address);
+  Printf("pointer tag 0x%x\nmemory tag  0x%x\n", ptr_tag, mem_tag);
+  Printf("%s", d.Default());
+
+  stack->Print();
+
+  PrintAddressDescription(address, access_size);
+
+  ReportErrorSummary("tag-mismatch", stack);
+}
+
+
+}  // namespace __hwasan

compiler-rt/lib/hwasan/hwasan_thread.h

+//===-- hwasan_thread.h -------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_THREAD_H
+#define HWASAN_THREAD_H
+
+#include "hwasan_allocator.h"
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace __hwasan {
+
+class HwasanThread {
+ public:
+  static HwasanThread *Create(thread_callback_t start_routine, void *arg);
+  static void TSDDtor(void *tsd);
+  void Destroy();
+
+  void Init();  // Should be called from the thread itself.
+  thread_return_t ThreadStart();
+
+  uptr stack_top() { return stack_top_; }
+  uptr stack_bottom() { return stack_bottom_; }
+  uptr tls_begin() { return tls_begin_; }
+  uptr tls_end() { return tls_end_; }
+  bool IsMainThread() { return start_routine_ == nullptr; }
+
+  bool AddrIsInStack(uptr addr) {
+    return addr >= stack_bottom_ && addr < stack_top_;
+  }
+
+  bool InSignalHandler() { return in_signal_handler_; }
+  void EnterSignalHandler() { in_signal_handler_++; }
+  void LeaveSignalHandler() { in_signal_handler_--; }
+
+  bool InSymbolizer() { return in_symbolizer_; }
+  void EnterSymbolizer() { in_symbolizer_++; }
+  void LeaveSymbolizer() { in_symbolizer_--; }
+
+  bool InInterceptorScope() { return in_interceptor_scope_; }
+  void EnterInterceptorScope() { in_interceptor_scope_++; }
+  void LeaveInterceptorScope() { in_interceptor_scope_--; }
+
+  HwasanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; }
+
+  int destructor_iterations_;
+
+ private:
+  // NOTE: There is no HwasanThread constructor. It is allocated
+  // via mmap() and *must* be valid in zero-initialized state.
+  void SetThreadStackAndTls();
+  void ClearShadowForThreadStackAndTLS();
+  thread_callback_t start_routine_;
+  void *arg_;
+  uptr stack_top_;
+  uptr stack_bottom_;
+  uptr tls_begin_;
+  uptr tls_end_;
+
+  unsigned in_signal_handler_;
+  unsigned in_symbolizer_;
+  unsigned in_interceptor_scope_;
+
+  HwasanThreadLocalMallocStorage malloc_storage_;
+};
+
+HwasanThread *GetCurrentThread();
+void SetCurrentThread(HwasanThread *t);
+
+} // namespace __hwasan
+
+#endif // HWASAN_THREAD_H

compiler-rt/lib/hwasan/hwasan_thread.cc

+
+#include "hwasan.h"
+#include "hwasan_thread.h"
+#include "hwasan_poisoning.h"
+#include "hwasan_interface_internal.h"
+
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+namespace __hwasan {
+
+HwasanThread *HwasanThread::Create(thread_callback_t start_routine,
+                               void *arg) {
+  uptr PageSize = GetPageSizeCached();
+  uptr size = RoundUpTo(sizeof(HwasanThread), PageSize);
+  HwasanThread *thread = (HwasanThread*)MmapOrDie(size, __func__);
+  thread->start_routine_ = start_routine;
+  thread->arg_ = arg;
+  thread->destructor_iterations_ = GetPthreadDestructorIterations();
+
+  return thread;
+}
+
+void HwasanThread::SetThreadStackAndTls() {
+  uptr tls_size = 0;
+  uptr stack_size = 0;
+  GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size,
+                       &tls_begin_, &tls_size);
+  stack_top_ = stack_bottom_ + stack_size;
+  tls_end_ = tls_begin_ + tls_size;
+
+  int local;
+  CHECK(AddrIsInStack((uptr)&local));
+}
+
+void HwasanThread::Init() {
+  SetThreadStackAndTls();
+  CHECK(MEM_IS_APP(stack_bottom_));
+  CHECK(MEM_IS_APP(stack_top_ - 1));
+}
+
+void HwasanThread::TSDDtor(void *tsd) {
+  HwasanThread *t = (HwasanThread*)tsd;
+  t->Destroy();
+}
+
+void HwasanThread::ClearShadowForThreadStackAndTLS() {
+  TagMemory(stack_bottom_, stack_top_ - stack_bottom_, 0);
+  if (tls_begin_ != tls_end_)
+    TagMemory(tls_begin_, tls_end_ - tls_begin_, 0);
+}
+
+void HwasanThread::Destroy() {
+  malloc_storage().CommitBack();
+  ClearShadowForThreadStackAndTLS();
+  uptr size = RoundUpTo(sizeof(HwasanThread), GetPageSizeCached());
+  UnmapOrDie(this, size);
+  DTLS_Destroy();
+}
+
+thread_return_t HwasanThread::ThreadStart() {
+  Init();
+
+  if (!start_routine_) {
+    // start_routine_ == 0 if we're on the main thread or on one of the
+    // OS X libdispatch worker threads. But nobody is supposed to call
+    // ThreadStart() for the worker threads.
+    return 0;
+  }
+
+  thread_return_t res = start_routine_(arg_);
+
+  return res;
+}
+
+} // namespace __hwasan

compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h

 }  // namespace __sanitizer
 
 namespace __asan  { using namespace __sanitizer; }  // NOLINT
 namespace __dsan  { using namespace __sanitizer; }  // NOLINT
 namespace __dfsan { using namespace __sanitizer; }  // NOLINT
 namespace __esan  { using namespace __sanitizer; }  // NOLINT
 namespace __lsan  { using namespace __sanitizer; }  // NOLINT
 namespace __msan  { using namespace __sanitizer; }  // NOLINT
+namespace __hwasan  { using namespace __sanitizer; }  // NOLINT
 namespace __tsan  { using namespace __sanitizer; }  // NOLINT
 namespace __scudo { using namespace __sanitizer; }  // NOLINT
 namespace __ubsan { using namespace __sanitizer; }  // NOLINT
 namespace __xray  { using namespace __sanitizer; }  // NOLINT
 namespace __interception  { using namespace __sanitizer; }  // NOLINT
 
 
 #endif  // SANITIZER_DEFS_H

compiler-rt/test/hwasan/CMakeLists.txt

+set(HWASAN_LIT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+
+set(HWASAN_TESTSUITES)
+
+set(HWASAN_TEST_ARCH ${HWASAN_SUPPORTED_ARCH})
+
+foreach(arch ${HWASAN_TEST_ARCH})
+  set(HWASAN_TEST_TARGET_ARCH ${arch})
+  string(TOLOWER "-${arch}" HWASAN_TEST_CONFIG_SUFFIX)
+  get_test_cc_for_arch(${arch} HWASAN_TEST_TARGET_CC HWASAN_TEST_TARGET_CFLAGS)
+  string(TOUPPER ${arch} ARCH_UPPER_CASE)
+  set(CONFIG_NAME ${ARCH_UPPER_CASE})
+
+  configure_lit_site_cfg(
+    ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
+    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg)
+  list(APPEND HWASAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
+endforeach()
+
+set(HWASAN_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS})
+if(NOT COMPILER_RT_STANDALONE_BUILD)
+  list(APPEND HWASAN_TEST_DEPS hwasan)
+endif()
+
+add_lit_testsuite(check-hwasan "Running the HWAddressSanitizer tests"
+  ${HWASAN_TESTSUITES}
+  DEPENDS ${HWASAN_TEST_DEPS}
+  )
+set_target_properties(check-hwasan PROPERTIES FOLDER "Compiler-RT Misc")

compiler-rt/test/hwasan/TestCases/halt-on-error.cc

+// RUN: %clangxx_hwasan -O0 %s -o %t && not %env_hwasan_opts=halt_on_error=0 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-%os --check-prefix=CHECK
+// REQUIRES: stable-runtime
+
+#include <stdlib.h>
+#include <sanitizer/hwasan_interface.h>
+
+int main() {
+  __hwasan_enable_allocator_tagging();
+  int* volatile x = (int*)malloc(16);
+  free(x);
+  __hwasan_disable_allocator_tagging();
+  return x[2] + ((char *)x)[6] + ((char *)x)[9];
+  // CHECK: READ of size 4 at
+  // CHECK: #0 {{.*}} in __hwasan_load4 {{.*}}hwasan.cc
+  // CHECK: #1 {{.*}} in main {{.*}}halt-on-error.cc:12
+  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in __hwasan_load4
+
+  // CHECK: READ of size 1 at
+  // CHECK: #0 {{.*}} in __hwasan_load1 {{.*}}hwasan.cc
+  // CHECK: #1 {{.*}} in main {{.*}}halt-on-error.cc:12
+  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in __hwasan_load1
+
+  // CHECK: READ of size 1 at
+  // CHECK: #0 {{.*}} in __hwasan_load1 {{.*}}hwasan.cc
+  // CHECK: #1 {{.*}} in main {{.*}}halt-on-error.cc:12
+  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in __hwasan_load1
+
+  // CHECK-NOT: tag-mismatch
+}

compiler-rt/test/hwasan/TestCases/use-after-free.cc

+// RUN: %clangxx_hwasan -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-%os --check-prefix=CHECK
+// RUN: %clangxx_hwasan -O1 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-%os --check-prefix=CHECK
+// RUN: %clangxx_hwasan -O2 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-%os --check-prefix=CHECK
+// RUN: %clangxx_hwasan -O3 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-%os --check-prefix=CHECK
+// REQUIRES: stable-runtime
+
+#include <stdlib.h>
+#include <sanitizer/hwasan_interface.h>
+
+int main() {
+  __hwasan_enable_allocator_tagging();
+  char *x = (char*)malloc(10);
+  free(x);
+  __hwasan_disable_allocator_tagging();
+  return x[5];
+  // CHECK: READ of size 1 at
+  // CHECK: #0 {{.*}} in __hwasan_load1 {{.*}}hwasan.cc
+  // CHECK: #1 {{.*}} in main {{.*}}use-after-free.cc:15
+
+  // CHECK: freed here:
+  // CHECK: #0 {{.*}} in free {{.*}}hwasan_interceptors.cc
+  // CHECK: #1 {{.*}} in main {{.*}}use-after-free.cc:13
+
+  // CHECK: previously allocated here:
+  // CHECK: #0 {{.*}} in __interceptor_malloc {{.*}}hwasan_interceptors.cc
+  // CHECK: #1 {{.*}} in main {{.*}}use-after-free.cc:12
+
+  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in __hwasan_load1
+}

compiler-rt/test/hwasan/lit.cfg

+# -*- Python -*-
+
+import os
+
+# Setup config name.
+config.name = 'HWAddressSanitizer' + getattr(config, 'name_suffix', 'default')
+
+# Setup source root.
+config.test_source_root = os.path.dirname(__file__)
+
+# Setup default compiler flags used with -fsanitize=memory option.
+clang_hwasan_cflags = ["-fsanitize=hwaddress", config.target_cflags] + config.debug_info_flags
+clang_hwasan_cxxflags = config.cxx_mode_flags + clang_hwasan_cflags
+
+def build_invocation(compile_flags):
+  return " " + " ".join([config.clang] + compile_flags) + " "
+
+config.substitutions.append( ("%clang_hwasan ", build_invocation(clang_hwasan_cflags)) )
+config.substitutions.append( ("%clangxx_hwasan ", build_invocation(clang_hwasan_cxxflags)) )
+
+default_hwasan_opts_str = ':'.join(['disable_allocator_tagging=1'] + config.default_sanitizer_opts)
+if default_hwasan_opts_str:
+  config.environment['HWASAN_OPTIONS'] = default_hwasan_opts_str
+  default_hwasan_opts_str += ':'
+config.substitutions.append(('%env_hwasan_opts=',
+                             'env HWASAN_OPTIONS=' + default_hwasan_opts_str))
+
+# Default test suffixes.
+config.suffixes = ['.c', '.cc', '.cpp']
+
+if config.host_os not in ['Linux', 'Android']:
+  config.unsupported = True

compiler-rt/test/hwasan/lit.site.cfg.in

+@LIT_SITE_CFG_IN_HEADER@
+
+# Tool-specific config options.
+config.name_suffix = "@HWASAN_TEST_CONFIG_SUFFIX@"
+config.target_cflags = "@HWASAN_TEST_TARGET_CFLAGS@"
+config.target_arch = "@HWASAN_TEST_TARGET_ARCH@"
+
+# Load common config for all compiler-rt lit tests.
+lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured")
+
+# Load tool-specific config that would do the real work.
+lit_config.load_config(config, "@HWASAN_LIT_SOURCE_DIR@/lit.cfg")