hwasan: Use system allocator to realloc and free untagged pointers in interceptor mode.

[pcc]

Summary

The Android dynamic loader has a non-standard feature that allows
libraries such as the hwasan runtime to interpose symbols even after
the symbol already has a value. The new value of the symbol is used to
relocate libraries loaded after the interposing library, but existing
libraries keep the old value.

What this means in practice is that if we have .so files that depend
on interceptor-mode hwasan without the main executable depending on
it, some of the libraries in the process will be using the hwasan
allocator and some will be using the system allocator, and these
allocators need to interact somehow. For example, if an instrumented
library calls a function such as strdup that allocates memory on
behalf of the caller, the instrumented library can reasonably expect
to be able to call free to deallocate the memory.

We can handle that relatively easily with hwasan by using tag 0 to
represent allocations from the system allocator. If hwasan's realloc
or free functions are passed a pointer with tag 0, the system allocator
is called.

One limitation is that this scheme doesn't work in reverse: if an
instrumented library allocates memory, it must free the memory itself
and cannot pass ownership to a system library. In a future change,
we may want to expose an API for calling the system allocator so
that instrumented libraries can safely transfer ownership of memory
to system libraries.

[eugenis]

LGTM

Mention that this loader feature is enabled by DF_GLOBAL somewhere in the patch description.

One limitation is that this scheme doesn't work in reverse: if an
instrumented library allocates memory, it must free the memory itself
and cannot pass ownership to a system library. In a future change,
we may want to expose an API for calling the system allocator so
that instrumented libraries can safely transfer ownership of memory
to system libraries.

That's unfortunate, but there is not a lot we can do about it. Hopefully, it is not very common in practice.

compiler-rt/trunk/lib/hwasan/hwasan.h

 bool InitShadow();
 void InitThreads();
 void MadviseShadow();
 char *GetProcSelfMaps();
 void InitializeInterceptors();
 
 void HwasanAllocatorInit();
 void HwasanAllocatorThreadFinish();
-void HwasanDeallocate(StackTrace *stack, void *ptr);
 
 void *hwasan_malloc(uptr size, StackTrace *stack);
 void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack);
 void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack);
 void *hwasan_valloc(uptr size, StackTrace *stack);
 void *hwasan_pvalloc(uptr size, StackTrace *stack);
 void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack);
 void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack);
 int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
                         StackTrace *stack);
+void hwasan_free(void *ptr, StackTrace *stack);
 
 void InstallTrapHandler();
 void InstallAtExitHandler();
 
 const char *GetStackOriginDescr(u32 id, uptr *pc);
 const char *GetStackFrameDescr(uptr pc);
 
 void EnterSymbolizer();

compiler-rt/trunk/lib/hwasan/hwasan_allocator.h

 //===-- hwasan_allocator.h --------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of HWAddressSanitizer.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef HWASAN_ALLOCATOR_H
 #define HWASAN_ALLOCATOR_H
 
+#include "interception/interception.h"
 #include "sanitizer_common/sanitizer_allocator.h"
 #include "sanitizer_common/sanitizer_allocator_checks.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
 #include "sanitizer_common/sanitizer_allocator_report.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_ring_buffer.h"
 #include "hwasan_poisoning.h"
 
 #if !defined(__aarch64__) && !defined(__x86_64__)
 #error Unsupported platform
 #endif
 
+#if HWASAN_WITH_INTERCEPTORS
+DECLARE_REAL(void *, realloc, void *ptr, uptr size)
+DECLARE_REAL(void, free, void *ptr)
+#endif
+
 namespace __hwasan {
 
 struct Metadata {
   u32 requested_size : 31;  // sizes are < 2G.
   u32 right_aligned  : 1;
   u32 alloc_context_id;
 };
 

compiler-rt/trunk/lib/hwasan/hwasan_allocator.cc

 #include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 #include "hwasan.h"
 #include "hwasan_allocator.h"
 #include "hwasan_mapping.h"
 #include "hwasan_thread.h"
 #include "hwasan_report.h"
 
+#if HWASAN_WITH_INTERCEPTORS
+DEFINE_REAL(void *, realloc, void *ptr, uptr size)
+DEFINE_REAL(void, free, void *ptr)
+#endif
+
 namespace __hwasan {
 
 static Allocator allocator;
 static AllocatorCache fallback_allocator_cache;
 static SpinMutex fallback_mutex;
 static atomic_uint8_t hwasan_allocator_tagging_enabled;
 
 static const tag_t kFallbackAllocTag = 0xBB;

 static bool PointerAndMemoryTagsMatch(void *tagged_ptr) {
   CHECK(tagged_ptr);
   tag_t ptr_tag = GetTagFromPointer(reinterpret_cast<uptr>(tagged_ptr));
   tag_t mem_tag = *reinterpret_cast<tag_t *>(
       MemToShadow(reinterpret_cast<uptr>(UntagPtr(tagged_ptr))));
   return ptr_tag == mem_tag;
 }
 
-void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
+static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
   CHECK(tagged_ptr);
   HWASAN_FREE_HOOK(tagged_ptr);
 
   if (!PointerAndMemoryTagsMatch(tagged_ptr))
     ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr));
 
   void *untagged_ptr = UntagPtr(tagged_ptr);
   void *aligned_ptr = reinterpret_cast<void *>(
if (auto *ha = t->heap_allocations())
                 free_context_id, static_cast<u32>(orig_size)});
   } else {
     SpinMutexLock l(&fallback_mutex);
     AllocatorCache *cache = &fallback_allocator_cache;
     allocator.Deallocate(cache, aligned_ptr);
   }
 }
 
-void *HwasanReallocate(StackTrace *stack, void *tagged_ptr_old, uptr new_size,
-                     uptr alignment) {
+static void *HwasanReallocate(StackTrace *stack, void *tagged_ptr_old,
+                              uptr new_size, uptr alignment) {
   if (!PointerAndMemoryTagsMatch(tagged_ptr_old))
     ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr_old));
 
   void *tagged_ptr_new =
       HwasanAllocate(stack, new_size, alignment, false /*zeroise*/);
   if (tagged_ptr_old && tagged_ptr_new) {
     void *untagged_ptr_old =  UntagPtr(tagged_ptr_old);
     Metadata *meta =
         reinterpret_cast<Metadata *>(allocator.GetMetaData(untagged_ptr_old));
     internal_memcpy(UntagPtr(tagged_ptr_new), untagged_ptr_old,
                     Min(new_size, static_cast<uptr>(meta->requested_size)));
     HwasanDeallocate(stack, tagged_ptr_old);
   }
   return tagged_ptr_new;
 }
 
-void *HwasanCalloc(StackTrace *stack, uptr nmemb, uptr size) {
+static void *HwasanCalloc(StackTrace *stack, uptr nmemb, uptr size) {
   if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
     if (AllocatorMayReturnNull())
       return nullptr;
     ReportCallocOverflow(nmemb, size, stack);
   }
   return HwasanAllocate(stack, nmemb * size, sizeof(u64), true);
 }
 

 
 void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) {
   return SetErrnoOnNull(HwasanCalloc(stack, nmemb, size));
 }
 
 void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) {
   if (!ptr)
     return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false));
+
+#if HWASAN_WITH_INTERCEPTORS
+  // A tag of 0 means that this is a system allocator allocation, so we must use
+  // the system allocator to realloc it.
+  if (!flags()->disable_allocator_tagging && GetTagFromPointer((uptr)ptr) == 0)
+    return REAL(realloc)(ptr, size);
+#endif
+
   if (size == 0) {
     HwasanDeallocate(stack, ptr);
     return nullptr;
   }
   return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64)));
 }
 
 void *hwasan_valloc(uptr size, StackTrace *stack) {
int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
   if (UNLIKELY(!ptr))
     // OOM error is already taken care of by HwasanAllocate.
     return errno_ENOMEM;
   CHECK(IsAligned((uptr)ptr, alignment));
   *memptr = ptr;
   return 0;
 }
 
+void hwasan_free(void *ptr, StackTrace *stack) {
+#if HWASAN_WITH_INTERCEPTORS
+  // A tag of 0 means that this is a system allocator allocation, so we must use
+  // the system allocator to free it.
+  if (!flags()->disable_allocator_tagging && GetTagFromPointer((uptr)ptr) == 0)
+    return REAL(free)(ptr);
+#endif
+
+  return HwasanDeallocate(stack, ptr);
+}
+
 }  // namespace __hwasan
 
 using namespace __hwasan;
 
 void __hwasan_enable_allocator_tagging() {
   atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 1);
 }
 
 void __hwasan_disable_allocator_tagging() {
+#if HWASAN_WITH_INTERCEPTORS
+  // Allocator tagging must be enabled for the system allocator fallback to work
+  // correctly. This means that we can't disable it at runtime if it was enabled
+  // at startup since that might result in our deallocations going to the system
+  // allocator. If tagging was disabled at startup we avoid this problem by
+  // disabling the fallback altogether.
+  CHECK(flags()->disable_allocator_tagging);
+#endif
+
   atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 0);
 }
 
 uptr __sanitizer_get_current_allocated_bytes() {
   uptr stats[AllocatorStatCount];
   allocator.GetStats(stats);
   return stats[AllocatorStatAllocated];
 }

compiler-rt/trunk/lib/hwasan/hwasan_interceptors.cc

 // Interceptors for standard library functions.
 //
 // FIXME: move as many interceptors as possible into
 // sanitizer_common/sanitizer_common_interceptors.h
 //===----------------------------------------------------------------------===//
 
 #include "interception/interception.h"
 #include "hwasan.h"
+#include "hwasan_allocator.h"
 #include "hwasan_mapping.h"
 #include "hwasan_thread.h"
 #include "hwasan_poisoning.h"
 #include "hwasan_report.h"
 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
 #include "sanitizer_common/sanitizer_allocator.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
 #include "sanitizer_common/sanitizer_allocator_internal.h"

 
 using namespace __hwasan;
 
 using __sanitizer::memory_order;
 using __sanitizer::atomic_load;
 using __sanitizer::atomic_store;
 using __sanitizer::atomic_uintptr_t;
 
-DECLARE_REAL(SIZE_T, strlen, const char *s)
-DECLARE_REAL(SIZE_T, strnlen, const char *s, SIZE_T maxlen)
-DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n)
-DECLARE_REAL(void *, memset, void *dest, int c, uptr n)
-
 bool IsInInterceptorScope() {
   Thread *t = GetCurrentThread();
   return t && t->InInterceptorScope();
 }
 
 struct InterceptorScope {
   InterceptorScope() {
     Thread *t = GetCurrentThread();

 void * __sanitizer_pvalloc(uptr size) {
   GET_MALLOC_STACK_TRACE;
   return hwasan_pvalloc(size, &stack);
 }
 
 void __sanitizer_free(void *ptr) {
   GET_MALLOC_STACK_TRACE;
   if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
-  HwasanDeallocate(&stack, ptr);
+  hwasan_free(ptr, &stack);
 }
 
 void __sanitizer_cfree(void *ptr) {
   GET_MALLOC_STACK_TRACE;
   if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
-  HwasanDeallocate(&stack, ptr);
+  hwasan_free(ptr, &stack);
 }
 
 uptr __sanitizer_malloc_usable_size(const void *ptr) {
   return __sanitizer_get_allocated_size(ptr);
 }
 
 struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() {
   __sanitizer_struct_mallinfo sret;

 void InitializeInterceptors() {
   static int inited = 0;
   CHECK_EQ(inited, 0);
 
   INTERCEPT_FUNCTION(fork);
 
 #if HWASAN_WITH_INTERCEPTORS
   INTERCEPT_FUNCTION(pthread_create);
+  INTERCEPT_FUNCTION(realloc);
+  INTERCEPT_FUNCTION(free);
 #endif
 
   inited = 1;
 }
 } // namespace __hwasan

compiler-rt/trunk/lib/hwasan/hwasan_linux.cc

 #include <sys/time.h>
 #include <unistd.h>
 #include <unwind.h>
 
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_procmaps.h"
 
 #if HWASAN_WITH_INTERCEPTORS && !SANITIZER_ANDROID
+SANITIZER_INTERFACE_ATTRIBUTE
 THREADLOCAL uptr __hwasan_tls;
 #endif
 
 namespace __hwasan {
 
 static void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
   CHECK_EQ((beg % GetMmapGranularity()), 0);
   CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);

compiler-rt/trunk/lib/hwasan/hwasan_new_delete.cc

 }
 INTERCEPTOR_ATTRIBUTE
 void *operator new[](size_t size, std::nothrow_t const&) {
   OPERATOR_NEW_BODY(true /*nothrow*/);
 }
 
 #define OPERATOR_DELETE_BODY \
   GET_MALLOC_STACK_TRACE; \
-  if (ptr) HwasanDeallocate(&stack, ptr)
+  if (ptr) hwasan_free(ptr, &stack)
 
 INTERCEPTOR_ATTRIBUTE
 void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
 INTERCEPTOR_ATTRIBUTE
 void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
 INTERCEPTOR_ATTRIBUTE
 void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; }
 INTERCEPTOR_ATTRIBUTE
 void operator delete[](void *ptr, std::nothrow_t const&) {
   OPERATOR_DELETE_BODY;
 }
 
 #endif // HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE

compiler-rt/trunk/test/hwasan/TestCases/Posix/system-allocator-fallback.cc

+// RUN: %clangxx %s -o %t -ldl
+// RUN: %clangxx_hwasan -shared %s -o %t.so -DSHARED_LIB -shared-libsan -Wl,-rpath,%compiler_rt_libdir
+// RUN: %env_hwasan_opts=disable_allocator_tagging=0 %run %t
+
+#include <stddef.h>
+
+// Test that allocations made by the system allocator can be realloc'd and freed
+// by the hwasan allocator.
+
+typedef void run_test_fn(void *(*system_malloc)(size_t size));
+
+#ifdef SHARED_LIB
+
+// Call the __sanitizer_ versions of these functions so that the test
+// doesn't require the Android dynamic loader.
+extern "C" void *__sanitizer_realloc(void *ptr, size_t size);
+extern "C" void __sanitizer_free(void *ptr);
+
+extern "C" run_test_fn run_test;
+void run_test(void *(*system_malloc)(size_t size)) {
+  void *mem = system_malloc(64);
+  mem = __sanitizer_realloc(mem, 128);
+  __sanitizer_free(mem);
+}
+
+#else
+
+#include <dlfcn.h>
+#include <stdlib.h>
+#include <string>
+
+int main(int argc, char **argv) {
+  std::string path = argv[0];
+  path += ".so";
+  void *lib = dlopen(path.c_str(), RTLD_NOW);
+  if (!lib) {
+    printf("error in dlopen(): %s\n", dlerror());
+    return 1;
+  }
+
+  auto run_test = reinterpret_cast<run_test_fn *>(dlsym(lib, "run_test"));
+  if (!run_test) {
+    printf("failed dlsym\n");
+    return 1;
+  }
+
+  run_test(malloc);
+}
+
+#endif

compiler-rt/trunk/test/hwasan/lit.cfg

 # -*- Python -*-
 
 import os
 
 # Setup config name.
 config.name = 'HWAddressSanitizer' + getattr(config, 'name_suffix', 'default')
 
 # Setup source root.
 config.test_source_root = os.path.dirname(__file__)
 
 # Setup default compiler flags used with -fsanitize=memory option.
-clang_hwasan_cflags = ["-fsanitize=hwaddress", "-mllvm", "-hwasan-generate-tags-with-calls", config.target_cflags] + config.debug_info_flags
+clang_cflags = [config.target_cflags] + config.debug_info_flags
+clang_cxxflags = config.cxx_mode_flags + clang_cflags
+clang_hwasan_cflags = ["-fsanitize=hwaddress", "-mllvm", "-hwasan-generate-tags-with-calls"] + clang_cflags
 clang_hwasan_cxxflags = config.cxx_mode_flags + clang_hwasan_cflags
 
 def build_invocation(compile_flags):
   return " " + " ".join([config.clang] + compile_flags) + " "
 
+config.substitutions.append( ("%clangxx ", build_invocation(clang_cxxflags)) )
 config.substitutions.append( ("%clang_hwasan ", build_invocation(clang_hwasan_cflags)) )
 config.substitutions.append( ("%clangxx_hwasan ", build_invocation(clang_hwasan_cxxflags)) )
+config.substitutions.append( ("%compiler_rt_libdir", config.compiler_rt_libdir) )
 
 default_hwasan_opts_str = ':'.join(['disable_allocator_tagging=1', 'random_tags=0'] + config.default_sanitizer_opts)
 if default_hwasan_opts_str:
   config.environment['HWASAN_OPTIONS'] = default_hwasan_opts_str
   default_hwasan_opts_str += ':'
 config.substitutions.append(('%env_hwasan_opts=',
                              'env HWASAN_OPTIONS=' + default_hwasan_opts_str))
 
 # Default test suffixes.
 config.suffixes = ['.c', '.cc', '.cpp']
 
 if config.host_os not in ['Linux', 'Android']:
   config.unsupported = True