[HWASAN] Init lsan and install at_exit hook

[kstoimenov]

Create by mistake.

[kstoimenov]

Updated.

[vitalybuka]

compiler-rt/lib/hwasan/hwasan_allocator.cpp

246

What is this for?

compiler-rt/lib/hwasan/hwasan_fuchsia.cpp

188

I recommend to extract a patches:

1. flags
2. InstallAtExitCheckLeaks

[vitalybuka]

we will need patches:

1. lsan tests in HWASAN, probably copied from asan
2. hwasan tests in lsan, lsan already run asan tests, we need to enabled hwasan as well, on supported platforms

[kstoimenov]

After pull.

[vitalybuka]

compiler-rt/lib/hwasan/hwasan_allocator.cpp

246

What is this for?

ping

compiler-rt/lib/hwasan/hwasan_fuchsia.cpp

188

also allocator patch

[kstoimenov]

Split.

[kstoimenov]

After moving makefile.

[kstoimenov]

Split out flags support.

[kstoimenov]

Rebase.

[vitalybuka]

compiler-rt/lib/hwasan/hwasan.cpp

90

move into flags patch
we will remove this FIXME in the patch which enables hwasan testing in check-lsan

[vitalybuka]

Don't use period in patch titles

[vitalybuka]

rebase

compiler-rt/lib/hwasan/CMakeLists.txt

add_compiler_rt_runtime(${hwasan_runtime}
     ARCHS ${arch}
     OBJECT_LIBS ${hwasan_object_lib}
                 RTHwasan_preinit
                 RTInterception
                 RTSanitizerCommon
                 RTSanitizerCommonLibc
                 RTSanitizerCommonCoverage
                 RTSanitizerCommonSymbolizer
+                RTLSanCommon
                 RTUbsan
     CFLAGS ${hwasan_rtl_flags}
     PARENT_TARGET hwasan)
   add_compiler_rt_runtime(${hwasan_runtime}_cxx
     STATIC
     ARCHS ${arch}
     OBJECT_LIBS RTHwasan_cxx
                 RTUbsan_cxx
add_compiler_rt_runtime(${hwasan_runtime}
     ARCHS ${arch}
     OBJECT_LIBS
             ${hwasan_object_dyn_lib}
             RTInterception
             RTSanitizerCommon
             RTSanitizerCommonLibc
             RTSanitizerCommonCoverage
             RTSanitizerCommonSymbolizer
+            RTLSanCommon
             RTUbsan
             RTUbsan_cxx
             # The only purpose of RTHWAsan_dynamic_version_script_dummy is to
             # carry a dependency of the shared runtime on the version script.
             # Replacing it with a straightforward
             # add_dependencies(clang_rt.asan-dynamic-${arch} clang_rt.asan-dynamic-${arch}-version-list)
             # generates an order-only dependency in ninja.
             RTHwasan_dynamic_version_script_dummy

compiler-rt/lib/hwasan/hwasan.h

 void HwasanTSDInit();
 void HwasanTSDThreadInit();
 void HwasanAtExit();
 
 void HwasanOnDeadlySignal(int signo, void *info, void *context);
 
 void HwasanInstallAtForkHandler();
 
+void InstallAtExitCheckLeaks();
+
 void UpdateMemoryUsage();
 
 void AppendToErrorMessageBuffer(const char *buffer);
 
 void AndroidTestTlsSlot();
 
 // This is a compiler-generated struct that can be shared between hwasan
 // implementations.

compiler-rt/lib/hwasan/hwasan.cpp

SetCommonFlagsDefaults();
     // FIXME: test and enable.
     cf.check_printf = false;
     cf.intercept_tls_get_addr = true;
     cf.exitcode = 99;
     // 8 shadow pages ~512kB, small enough to cover common stack sizes.
     cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8);
     // Sigtrap is used in error reporting.
     cf.handle_sigtrap = kHandleSignalExclusive;
+    // FIXME: enable once all false positives have been fixed.
+    cf.detect_leaks = false;
 
 #if SANITIZER_ANDROID
     // Let platform handle other signals. It is better at reporting them then we
     // are.
     cf.handle_segv = kHandleSignalNo;
     cf.handle_sigbus = kHandleSignalNo;
     cf.handle_abort = kHandleSignalNo;
     cf.handle_sigill = kHandleSignalNo;
     cf.handle_sigfpe = kHandleSignalNo;
 #endif
     OverrideCommonFlags(cf);
   }
 
   Flags *f = flags();
   f->SetDefaults();
 
   FlagParser parser;
   RegisterHwasanFlags(&parser, f);
   RegisterCommonFlags(&parser);
 
+#if CAN_SANITIZE_LEAKS
+  __lsan::Flags *lf = __lsan::flags();
+  lf->SetDefaults();
+
+  FlagParser lsan_parser;
+  __lsan::RegisterLsanFlags(&lsan_parser, lf);
+  RegisterCommonFlags(&lsan_parser);
+#endif
+
 #if HWASAN_CONTAINS_UBSAN
   __ubsan::Flags *uf = __ubsan::flags();
   uf->SetDefaults();
 
   FlagParser ubsan_parser;
   __ubsan::RegisterUbsanFlags(&ubsan_parser, uf);
   RegisterCommonFlags(&ubsan_parser);
 #endif
 
   // Override from user-specified string.
   if (__hwasan_default_options)
     parser.ParseString(__hwasan_default_options());
 #if HWASAN_CONTAINS_UBSAN
   const char *ubsan_default_options = __ubsan_default_options();
   ubsan_parser.ParseString(ubsan_default_options);
 #endif
 
   parser.ParseStringFromEnv("HWASAN_OPTIONS");
+#if CAN_SANITIZE_LEAKS
+  lsan_parser.ParseStringFromEnv("LSAN_OPTIONS");
+#endif
 #if HWASAN_CONTAINS_UBSAN
   ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
 #endif
 
   InitializeCommonFlags();
 
   if (Verbosity()) ReportUnrecognizedFlags();
 
   if (common_flags()->help) parser.PrintFlagDescriptions();
+  // Flag validation:
+  if (!CAN_SANITIZE_LEAKS && common_flags()->detect_leaks) {
+    Report("%s: detect_leaks is not supported on this platform.\n",
+           SanitizerToolName);
+    Die();
+  }
 }
 
 static void CheckUnwind() {
   GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
   stack.Print();
 }
 
 static void HwasanFormatMemoryUsage(InternalScopedString &s) {
__attribute__((constructor(0))) void __hwasan_init() {
   InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
 
   HwasanTSDInit();
   HwasanTSDThreadInit();
 
   HwasanAllocatorInit();
   HwasanInstallAtForkHandler();
 
+  if (CAN_SANITIZE_LEAKS) {
+    __lsan::InitCommonLsan();
+    InstallAtExitCheckLeaks();
+  }
+
 #if HWASAN_CONTAINS_UBSAN
   __ubsan::InitAsPlugin();
 #endif
 
+  if (CAN_SANITIZE_LEAKS) {
+    __lsan::ScopedInterceptorDisabler disabler;
+    Symbolizer::LateInitialize();
+  } else {
+    Symbolizer::LateInitialize();
+  }
+
   VPrintf(1, "HWAddressSanitizer init done\n");
 
   hwasan_init_is_running = 0;
   hwasan_inited = 1;
 }
 
 void __hwasan_library_loaded(ElfW(Addr) base, const ElfW(Phdr) * phdr,
                              ElfW(Half) phnum) {

compiler-rt/lib/hwasan/hwasan_allocator.h

public:
   bool IsAllocated() const;    // Checks if the memory is currently allocated
   uptr Beg() const;            // First byte of user memory
   uptr End() const;            // Last byte of user memory
   uptr UsedSize() const;       // Size requested by the user
   uptr ActualSize() const;     // Size allocated by the allocator.
   u32 GetAllocStackId() const;
   bool FromSmallHeap() const;
   bool AddrIsInside(uptr addr) const;
+  inline void SetLsanTag(__lsan::ChunkTag tag);
+  inline __lsan::ChunkTag GetLsanTag() const;
 
  private:
   friend class __lsan::LsanMetadata;
   uptr block_;
   Metadata *const metadata_;
 };
 
 HwasanChunkView FindHeapChunkByAddress(uptr address);
+HwasanChunkView FindHeapChunkByAddressFastLocked(uptr address);
 
 // Information about one (de)allocation that happened in the past.
 // These are recorded in a thread-local ring buffer.
 // TODO: this is currently 24 bytes (20 bytes + alignment).
 // Compress it to 16 bytes or extend it to be more useful.
 struct HeapAllocationRecord {
   uptr tagged_addr;
   u32  alloc_context_id;

compiler-rt/lib/hwasan/hwasan_allocator.cpp

 bool HwasanChunkView::FromSmallHeap() const {
   return allocator.FromPrimary(reinterpret_cast<void *>(block_));
 }
 
 bool HwasanChunkView::AddrIsInside(uptr addr) const {
   return (addr >= Beg()) && (addr < Beg() + UsedSize());
 }
 
+inline void HwasanChunkView::SetLsanTag(__lsan::ChunkTag tag) {
+  CHECK(metadata_);
+  metadata_->SetLsanTag(tag);
+}
+inline __lsan::ChunkTag HwasanChunkView::GetLsanTag() const {
+  CHECK(metadata_);
+  return metadata_->GetLsanTag();
+}
+
 inline void Metadata::SetAllocated(u32 stack, u64 size) {
   Thread *t = GetCurrentThread();
   u64 context = t ? t->unique_id() : kMainTid;
   context <<= 32;
   context += stack;
   requested_size_low = size & ((1ul << 32) - 1);
   requested_size_high = size >> 32;
   atomic_store(&alloc_context_id, context, memory_order_relaxed);

 inline u64 Metadata::GetRequestedSize() const {
   return (static_cast<u64>(requested_size_high) << 32) + requested_size_low;
 }
 
 inline u32 Metadata::GetAllocStackId() const {
   return atomic_load(&alloc_context_id, memory_order_relaxed);
 }
 
-static const uptr kChunkHeaderSize = sizeof(HwasanChunkView);
-
 void GetAllocatorStats(AllocatorStatCounters s) {
   allocator.GetStats(s);
 }
 
 inline void Metadata::SetLsanTag(__lsan::ChunkTag tag) {
   lsan_tag = tag;
 }
 
if (flags()->tag_in_malloc && malloc_bisect(stack, orig_size)) {
       }
     } else {
       user_ptr = (void *)TagMemoryAligned((uptr)user_ptr, size, 0);
     }
   }
 
   Metadata *meta =
       reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
+#if CAN_SANITIZE_LEAKS
+  meta->SetLsanTag(__lsan::DisabledInThisThread() ? __lsan::kIgnored
+                                                  : __lsan::kDirectlyLeaked);
+#endif
   meta->SetAllocated(StackDepotPut(*stack), orig_size);
   RunMallocHooks(user_ptr, size);
   return user_ptr;
 }
 
 static bool PointerAndMemoryTagsMatch(void *tagged_ptr) {
   CHECK(tagged_ptr);
   uptr tagged_uptr = reinterpret_cast<uptr>(tagged_ptr);
HwasanChunkView FindHeapChunkByAddress(uptr address) {
   void *block = allocator.GetBlockBegin(reinterpret_cast<void*>(address));
   if (!block)
     return HwasanChunkView();
   Metadata *metadata =
       reinterpret_cast<Metadata*>(allocator.GetMetaData(block));
   return HwasanChunkView(reinterpret_cast<uptr>(block), metadata);
 }
 
+HwasanChunkView FindHeapChunkByAddressFastLocked(uptr address) {
+  void *block =
+      allocator.GetBlockBeginFastLocked(reinterpret_cast<void *>(address));
+  if (!block)
+    return HwasanChunkView();
+  Metadata *metadata =
+      reinterpret_cast<Metadata *>(allocator.GetMetaData(block));
+  return HwasanChunkView(reinterpret_cast<uptr>(block), metadata);
+}
+
 static uptr AllocationSize(const void *tagged_ptr) {
   const void *untagged_ptr = UntagPtr(tagged_ptr);
   if (!untagged_ptr) return 0;
   const void *beg = allocator.GetBlockBegin(untagged_ptr);
   Metadata *b = (Metadata *)allocator.GetMetaData(untagged_ptr);
   if (beg != untagged_ptr) return 0;
   return b->GetRequestedSize();
 }

 
 void GetAllocatorGlobalRange(uptr *begin, uptr *end) {
   *begin = (uptr)&__hwasan::allocator;
   *end = *begin + sizeof(__hwasan::allocator);
 }
 
 uptr PointsIntoChunk(void *p) {
   uptr addr = reinterpret_cast<uptr>(p);
-  __hwasan::HwasanChunkView view = __hwasan::FindHeapChunkByAddress(addr);
-  if (!view.IsAllocated()) 
+  __hwasan::HwasanChunkView view =
+      __hwasan::FindHeapChunkByAddressFastLocked(addr);
+  if (!view.IsAllocated())
     return 0;
   uptr chunk = view.Beg();
   if (view.AddrIsInside(addr))
     return chunk;
   if (IsSpecialCaseOfOperatorNew0(chunk, view.UsedSize(), addr))
     return chunk;
   return 0;
 }
 
 uptr GetUserBegin(uptr chunk) {
-  return __hwasan::FindHeapChunkByAddress(chunk).Beg();
+  return __hwasan::FindHeapChunkByAddressFastLocked(chunk).Beg();
 }
 
 LsanMetadata::LsanMetadata(uptr chunk) {
-  metadata_ = chunk ? reinterpret_cast<__hwasan::Metadata *>(
-                          chunk - __hwasan::kChunkHeaderSize)
+  metadata_ = chunk ? __hwasan::allocator.GetMetaData((void *)chunk)
                     : nullptr;
 }
 
 bool LsanMetadata::allocated() const {
   if (!metadata_)
     return false;
   __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
   return m->IsAllocated();
u32 LsanMetadata::stack_trace_id() const {
   __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
   return m->GetAllocStackId();
 }
 
 void ForEachChunk(ForEachChunkCallback callback, void *arg) {
   __hwasan::allocator.ForEachChunk(callback, arg);
 }
 
+IgnoreObjectResult IgnoreObjectLocked(const void *p) {
+  uptr addr = reinterpret_cast<uptr>(p);
+  __hwasan::HwasanChunkView view = __hwasan::FindHeapChunkByAddressFastLocked(addr);
+  if (!view.IsAllocated() || !view.AddrIsInside(addr)) {
+    return kIgnoreObjectInvalid;
+  }
+  if (view.GetLsanTag() == kIgnored)
+    return kIgnoreObjectAlreadyIgnored;
+  view.SetLsanTag(kIgnored);
+  return kIgnoreObjectSuccess;
+}
+
 }  // namespace __lsan
 
 using namespace __hwasan;
 
 void __hwasan_enable_allocator_tagging() {
   atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 1);
 }
 

compiler-rt/lib/hwasan/hwasan_fuchsia.cpp

 
 // On linux, this just would call `atexit(HwasanAtExit)`. The functions in
 // HwasanAtExit are unimplemented for Fuchsia and effectively no-ops, so this
 // function is unneeded.
 void InstallAtExitHandler() {}
 
 void HwasanInstallAtForkHandler() {}
 
+void InstallAtExitCheckLeaks() {}
+
 void InitializeOsSupport() {
 #ifdef __aarch64__
   uint32_t features = 0;
   CHECK_EQ(zx_system_get_features(ZX_FEATURE_KIND_ADDRESS_TAGGING, &features),
            ZX_OK);
   if (!(features & ZX_ARM64_FEATURE_ADDRESS_TAGGING_TBI) &&
       flags()->fail_without_syscall_abi) {
     Printf(

compiler-rt/lib/hwasan/hwasan_interceptors.cpp

 #undef SIG_BLOCK
 #undef SIG_SETMASK
 
 #  endif  // HWASAN_WITH_INTERCEPTORS
 
 namespace __hwasan {
 
 int OnExit() {
+  if (CAN_SANITIZE_LEAKS && common_flags()->detect_leaks &&
+      __lsan::HasReportedLeaks()) {
+    return common_flags()->exitcode;
+  }
   // FIXME: ask frontend whether we need to return failure.
   return 0;
 }
 
 } // namespace __hwasan
 
 namespace __hwasan {
 

compiler-rt/lib/hwasan/hwasan_linux.cpp

void HwasanInstallAtForkHandler() {
   };
   auto after = []() {
     StackDepotUnlockAll();
     HwasanAllocatorUnlock();
   };
   pthread_atfork(before, after, after);
 }
 
+void InstallAtExitCheckLeaks() {
+  if (CAN_SANITIZE_LEAKS) {
+    if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) {
+      if (flags()->halt_on_error)
+        Atexit(__lsan::DoLeakCheck);
+      else
+        Atexit(__lsan::DoRecoverableLeakCheckVoid);
+    }
+  }
+}
+
 }  // namespace __hwasan
 
 #endif  // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD