[HWASan] Refactor in preparation for x86 aliasing mode. NFC

[morehouse]

Talked with @eugenis offline. I'm going to look into getting rid of "base tags" by only storing the changeable bits in shadow.

[morehouse]

• Avoid base tag complexities by only storing changeable bits in shadow.

[morehouse]

• Remove unused includes.

[vitalybuka]

compiler-rt/lib/hwasan/hwasan_allocator.cpp

162

InTaggableRegion should be the first as likely the cheepest of those checks

245

same

[morehouse]

• Check InTaggableRegion before other checks in malloc/free.

[eugenis]

compiler-rt/lib/hwasan/hwasan_linux.cpp

225

Is this still needed if we mask out the higher non-tag bits?

[morehouse]

compiler-rt/lib/hwasan/hwasan_linux.cpp

225

Yes, tag bits are in userspace, so non-heap pointers are unlikely to have 0 there.

[eugenis]

LGTM

compiler-rt/lib/hwasan/hwasan_linux.cpp

225

ah, it's not because of the "base tag", it's because some app memory is outside of the aliased / taggable range.
Please adjust the comment.

[morehouse]

• Update comment.

compiler-rt/lib/hwasan/hwasan_allocator.h

 //===-- hwasan_allocator.h --------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of HWAddressSanitizer.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef HWASAN_ALLOCATOR_H
 #define HWASAN_ALLOCATOR_H
 
+#include "hwasan.h"
+#include "hwasan_interface_internal.h"
+#include "hwasan_poisoning.h"
 #include "sanitizer_common/sanitizer_allocator.h"
 #include "sanitizer_common/sanitizer_allocator_checks.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
 #include "sanitizer_common/sanitizer_allocator_report.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_ring_buffer.h"
-#include "hwasan_poisoning.h"
 
 #if !defined(__aarch64__) && !defined(__x86_64__)
 #error Unsupported platform
 #endif
 
 namespace __hwasan {
 
 struct Metadata {
struct HeapAllocationRecord {
   u32  free_context_id;
   u32  requested_size;
 };
 
 typedef RingBuffer<HeapAllocationRecord> HeapAllocationsRingBuffer;
 
 void GetAllocatorStats(AllocatorStatCounters s);
 
+inline bool InTaggableRegion(uptr addr) {
+  // TODO: specialize for x86 once we use aliasing mode in the allocator.
+  return true;
+}
+
 } // namespace __hwasan
 
 #endif // HWASAN_ALLOCATOR_H

compiler-rt/lib/hwasan/hwasan_allocator.cpp

 
 static uptr TaggedSize(uptr size) {
   if (!size) size = 1;
   uptr new_size = RoundUpTo(size, kShadowAlignment);
   CHECK_GE(new_size, size);
   return new_size;
 }
 
+static inline tag_t BaseTag(void *untagged_ptr) {
+#if defined(__x86_64__)
+  return GetTagFromPointer(reinterpret_cast<uptr>(untagged_ptr));
+#else
+  return 0;
+#endif
+}
+
 static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment,
                             bool zeroise) {
   if (orig_size > kMaxAllowedMallocSize) {
     if (AllocatorMayReturnNull()) {
       Report("WARNING: HWAddressSanitizer failed to allocate 0x%zx bytes\n",
              orig_size);
       return nullptr;
     }
if (zeroise) {
     internal_memset(allocated, flags()->malloc_fill_byte, fill_size);
   }
   if (size != orig_size) {
     internal_memcpy(reinterpret_cast<u8 *>(allocated) + orig_size, tail_magic,
                     size - orig_size - 1);
   }
 
   void *user_ptr = allocated;
+  tag_t tag = BaseTag(user_ptr);
   // Tagging can only be skipped when both tag_in_malloc and tag_in_free are
   // false. When tag_in_malloc = false and tag_in_free = true malloc needs to
   // retag to 0.
   if ((flags()->tag_in_malloc || flags()->tag_in_free) &&
-      atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) {
+      atomic_load_relaxed(&hwasan_allocator_tagging_enabled) &&
+      InTaggableRegion(reinterpret_cast<uptr>(user_ptr))) {
     if (flags()->tag_in_malloc && malloc_bisect(stack, orig_size)) {
-      tag_t tag = t ? t->GenerateRandomTag() : kFallbackAllocTag;
+      tag |= t ? t->GenerateRandomTag() : kFallbackAllocTag;
       uptr tag_size = orig_size ? orig_size : 1;
       uptr full_granule_size = RoundDownTo(tag_size, kShadowAlignment);
       user_ptr =
           (void *)TagMemoryAligned((uptr)user_ptr, full_granule_size, tag);
       if (full_granule_size != tag_size) {
         u8 *short_granule =
             reinterpret_cast<u8 *>(allocated) + full_granule_size;
         TagMemoryAligned((uptr)short_granule, kShadowAlignment,
                          tag_size % kShadowAlignment);
         short_granule[kShadowAlignment - 1] = tag;
       }
     } else {
-      user_ptr = (void *)TagMemoryAligned((uptr)user_ptr, size, 0);
+      user_ptr = (void *)TagMemoryAligned((uptr)user_ptr, size, tag);
     }
   }
+#if defined(__x86_64__)
+  else {
+    // In aliasing mode our base tag is non-zero, so we must tag the shadow to
+    // avoid mismatches.
+    TagMemoryAligned(reinterpret_cast<uptr>(user_ptr), size, tag);
+  }
+#endif
 
   HWASAN_MALLOC_HOOK(user_ptr, size);
   return user_ptr;
 }
 
 static bool PointerAndMemoryTagsMatch(void *tagged_ptr) {
   CHECK(tagged_ptr);
   uptr tagged_uptr = reinterpret_cast<uptr>(tagged_ptr);
+  if (!InTaggableRegion(tagged_uptr))
+    return true;
   tag_t mem_tag = *reinterpret_cast<tag_t *>(
       MemToShadow(reinterpret_cast<uptr>(UntagPtr(tagged_ptr))));
   return PossiblyShortTagMatches(mem_tag, tagged_uptr, 1);
 }
 
 static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
   CHECK(tagged_ptr);
   HWASAN_FREE_HOOK(tagged_ptr);
 
   if (!PointerAndMemoryTagsMatch(tagged_ptr))
     ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr));
 
-  void *untagged_ptr = UntagPtr(tagged_ptr);
+  void *untagged_ptr = InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr))
+                           ? UntagPtr(tagged_ptr)
+                           : tagged_ptr;
   void *aligned_ptr = reinterpret_cast<void *>(
       RoundDownTo(reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment));
   Metadata *meta =
       reinterpret_cast<Metadata *>(allocator.GetMetaData(aligned_ptr));
   uptr orig_size = meta->get_requested_size();
   u32 free_context_id = StackDepotPut(*stack);
   u32 alloc_context_id = meta->alloc_context_id;
 
static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
   // poisoned.
   Thread *t = GetCurrentThread();
   if (flags()->max_free_fill_size > 0) {
     uptr fill_size =
         Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size);
     internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size);
   }
   if (flags()->tag_in_free && malloc_bisect(stack, 0) &&
-      atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+      atomic_load_relaxed(&hwasan_allocator_tagging_enabled) &&
+      InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr))) {
+    tag_t tag = BaseTag(untagged_ptr);
+    tag |= t ? t->GenerateRandomTag() : kFallbackFreeTag;
     TagMemoryAligned(reinterpret_cast<uptr>(aligned_ptr), TaggedSize(orig_size),
-                     t ? t->GenerateRandomTag() : kFallbackFreeTag);
+                     tag);
+  }
   if (t) {
     allocator.Deallocate(t->allocator_cache(), aligned_ptr);
     if (auto *ha = t->heap_allocations())
       ha->push({reinterpret_cast<uptr>(tagged_ptr), alloc_context_id,
                 free_context_id, static_cast<u32>(orig_size)});
   } else {
     SpinMutexLock l(&fallback_mutex);
     AllocatorCache *cache = &fallback_allocator_cache;

compiler-rt/lib/hwasan/hwasan_checks.h

 //===-- hwasan_checks.h -----------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of HWAddressSanitizer.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef HWASAN_CHECKS_H
 #define HWASAN_CHECKS_H
 
+#include "hwasan_allocator.h"
 #include "hwasan_mapping.h"
 #include "sanitizer_common/sanitizer_common.h"
 
 namespace __hwasan {
 template <unsigned X>
 __attribute__((always_inline)) static void SigTrap(uptr p) {
 #if defined(__aarch64__)
   (void)p;
#endif
   return *(u8 *)(ptr | (kShadowAlignment - 1)) == ptr_tag;
 }
 
 enum class ErrorAction { Abort, Recover };
 enum class AccessType { Load, Store };
 
 template <ErrorAction EA, AccessType AT, unsigned LogSize>
 __attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) {
+  if (!InTaggableRegion(p))
+    return;
   uptr ptr_raw = p & kAddressUntagMask;
   tag_t mem_tag = *(tag_t *)MemToShadow(ptr_raw);
   if (UNLIKELY(!PossiblyShortTagMatches(mem_tag, p, 1 << LogSize))) {
     SigTrap<0x20 * (EA == ErrorAction::Recover) +
             0x10 * (AT == AccessType::Store) + LogSize>(p);
     if (EA == ErrorAction::Abort)
       __builtin_unreachable();
   }
 }
 
 template <ErrorAction EA, AccessType AT>
 __attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p,
                                                                       uptr sz) {
-  if (sz == 0)
+  if (sz == 0 || !InTaggableRegion(p))
     return;
   tag_t ptr_tag = GetTagFromPointer(p);
   uptr ptr_raw = p & kAddressUntagMask;
   tag_t *shadow_first = (tag_t *)MemToShadow(ptr_raw);
   tag_t *shadow_last = (tag_t *)MemToShadow(ptr_raw + sz);
   for (tag_t *t = shadow_first; t < shadow_last; ++t)
     if (UNLIKELY(ptr_tag != *t)) {
       SigTrap<0x20 * (EA == ErrorAction::Recover) +

compiler-rt/lib/hwasan/hwasan_linux.cpp

void InitThreads() {
   ReserveShadowMemoryRange(thread_space_start, thread_space_end - 1,
                            "hwasan threads", /*madvise_shadow*/ false);
   ProtectGap(thread_space_end,
              __hwasan_shadow_memory_dynamic_address - thread_space_end);
   InitThreadList(thread_space_start, thread_space_end - thread_space_start);
 }
 
 bool MemIsApp(uptr p) {
+#if !defined(__x86_64__)  // Aliasing-mode base tags are non-zero.
   CHECK(GetTagFromPointer(p) == 0);
+#endif
   return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
 }
 
 static void HwasanAtExit(void) {
   if (common_flags()->print_module_map)
     DumpProcessMap();
   if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
     ReportStats();