[HWASan] Port HWASan to Linux x86-64 (compiler-rt)

[alekseyshl]

Summary

Porting HWASan to Linux x86-64, the second of the three patches, compiler-rt part.

The approach is similar to ARM case, trap signal is used to communicate
memory tag check failure. int3 instruction is used to generate a signal,
access parameters are stored in nop [eax + offset] instruction immediately
following the int3 one

Had to add HWASan init on malloc because, due to much less interceptors
defined (most other sanitizers intercept much more and get initalized
via one of those interceptors or don't care about malloc), HWASan was not
initialized yet when libstdc++ was trying to allocate memory for its own
fixed-size heap, which led to CHECK-fail in AllocateFromLocalPool.

Also added the CHECK() failure handler with more detailed message and
stack reporting.

[eugenis]

lib/hwasan/hwasan.cc

257 ↗

(On Diff #139200)

It's actually added because the kernel has other plans for low values of brk immediate.

268 ↗

(On Diff #139200)

We could do the same thing with assembly constraint in aarch64 branch to put the pointer in x0. Let's not touch that code for now, but please add a FIXME.

lib/hwasan/hwasan_linux.cc

214 ↗

(On Diff #139200)

This is mostly a copy of the comment above. Please factor out the common part.

[alekseyshl]

• Adjust comments

[alekseyshl]

Comments addressed

compiler-rt/trunk/cmake/config-ix.cmake

 set(ALL_FUZZER_SUPPORTED_ARCH x86_64)
 
 if(APPLE)
   set(ALL_LSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${MIPS64} ${ARM64})
 else()
   set(ALL_LSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${MIPS64} ${ARM64} ${ARM32} ${PPC64})
 endif()
 set(ALL_MSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64})
-set(ALL_HWASAN_SUPPORTED_ARCH ${ARM64})
+set(ALL_HWASAN_SUPPORTED_ARCH ${X86_64} ${ARM64})
 set(ALL_PROFILE_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${PPC64}
     ${MIPS32} ${MIPS64} ${S390X})
 set(ALL_TSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64})
 set(ALL_UBSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64}
     ${MIPS32} ${MIPS64} ${PPC64} ${S390X})
 set(ALL_SAFESTACK_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM64} ${MIPS32} ${MIPS64})
 set(ALL_CFI_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${MIPS64})
 set(ALL_ESAN_SUPPORTED_ARCH ${X86_64} ${MIPS64})

compiler-rt/trunk/lib/hwasan/hwasan.h

GetStackTrace(&stack, common_flags()->malloc_context_size,              \
                 common_flags()->fast_unwind_on_malloc)
 
 #define GET_FATAL_STACK_TRACE_PC_BP(pc, bp)              \
   BufferedStackTrace stack;                              \
   if (hwasan_inited)                                       \
   GetStackTrace(&stack, kStackTraceMax, pc, bp, nullptr, \
                 common_flags()->fast_unwind_on_fatal)
 
+#define GET_FATAL_STACK_TRACE_HERE \
+  GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME())
+
+#define PRINT_CURRENT_STACK_CHECK() \
+  {                                 \
+    GET_FATAL_STACK_TRACE_HERE;     \
+    stack.Print();                  \
+  }
+
 class ScopedThreadLocalStateBackup {
  public:
   ScopedThreadLocalStateBackup() { Backup(); }
   ~ScopedThreadLocalStateBackup() { Restore(); }
   void Backup();
   void Restore();
  private:
   u64 va_arg_overflow_size_tls;

compiler-rt/trunk/lib/hwasan/hwasan.cc

stack->Unwind(max_s, pc, bp, context, t->stack_top(), t->stack_bottom(),
                 request_fast_unwind);
 }
 
 void PrintWarning(uptr pc, uptr bp) {
   GET_FATAL_STACK_TRACE_PC_BP(pc, bp);
   ReportInvalidAccess(&stack, 0);
 }
 
+static void HWAsanCheckFailed(const char *file, int line, const char *cond,
+                              u64 v1, u64 v2) {
+  Report("HWAddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file,
+         line, cond, (uptr)v1, (uptr)v2);
+  PRINT_CURRENT_STACK_CHECK();
+  Die();
+}
+
 } // namespace __hwasan
 
 // Interface.
 
 using namespace __hwasan;
 
 void __hwasan_init() {
   CHECK(!hwasan_init_is_running);
   if (hwasan_inited) return;
   hwasan_init_is_running = 1;
   SanitizerToolName = "HWAddressSanitizer";
 
   InitTlsSize();
 
   CacheBinaryName();
   InitializeFlags();
 
+  // Install tool-specific callbacks in sanitizer_common.
+  SetCheckFailedCallback(HWAsanCheckFailed);
+
   __sanitizer_set_report_path(common_flags()->log_path);
 
   InitializeInterceptors();
   InstallDeadlySignalHandlers(HwasanOnDeadlySignal);
   InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
 
   DisableCoreDumperIfNecessary();
   if (!InitShadow()) {
void __sanitizer_unaligned_store32(uu32 *p, u32 x) {
   *p = x;
 }
 void __sanitizer_unaligned_store64(uu64 *p, u64 x) {
   *p = x;
 }
 
 template<unsigned X>
 __attribute__((always_inline))
-static void SigTrap() {
+static void SigTrap(uptr p) {
 #if defined(__aarch64__)
-  asm("brk %0\n\t" ::"n"(X));
-#elif defined(__x86_64__) || defined(__i386__)
-  asm("ud2\n\t");
+  (void)p;
+  // 0x900 is added to do not interfere with the kernel use of lower values of
+  // brk immediate.
+  // FIXME: Add a constraint to put the pointer into x0, the same as x86 branch.
+  asm("brk %0\n\t" ::"n"(0x900 + X));
+#elif defined(__x86_64__)
+  // INT3 + NOP DWORD ptr [EAX + X] to pass X to our signal handler, 5 bytes
+  // total. The pointer is passed via rdi.
+  // 0x40 is added as a safeguard, to help distinguish our trap from others and
+  // to avoid 0 offsets in the command (otherwise it'll be reduced to a
+  // different nop command, the three bytes one).
+  asm volatile(
+      "int3\n"
+      "nopl %c0(%%rax)\n"
+      :: "n"(0x40 + X), "D"(p));
 #else
   // FIXME: not always sigill.
   __builtin_trap();
 #endif
   // __builtin_unreachable();
 }
 
 enum class ErrorAction { Abort, Recover };
 enum class AccessType { Load, Store };
 
 template <ErrorAction EA, AccessType AT, unsigned LogSize>
 __attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) {
   tag_t ptr_tag = GetTagFromPointer(p);
   uptr ptr_raw = p & ~kAddressTagMask;
   tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(ptr_raw);
   if (UNLIKELY(ptr_tag != mem_tag)) {
-    SigTrap<0x900 + 0x20 * (EA == ErrorAction::Recover) +
-           0x10 * (AT == AccessType::Store) + LogSize>();
+    SigTrap<0x20 * (EA == ErrorAction::Recover) +
+           0x10 * (AT == AccessType::Store) + LogSize>(p);
     if (EA == ErrorAction::Abort) __builtin_unreachable();
   }
 }
 
 template <ErrorAction EA, AccessType AT>
 __attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p,
                                                                       uptr sz) {
   CHECK_NE(0, sz);
   tag_t ptr_tag = GetTagFromPointer(p);
   uptr ptr_raw = p & ~kAddressTagMask;
   tag_t *shadow_first = (tag_t *)MEM_TO_SHADOW(ptr_raw);
   tag_t *shadow_last = (tag_t *)MEM_TO_SHADOW(ptr_raw + sz - 1);
   for (tag_t *t = shadow_first; t <= shadow_last; ++t)
     if (UNLIKELY(ptr_tag != *t)) {
-      SigTrap<0x900 + 0x20 * (EA == ErrorAction::Recover) +
-             0x10 * (AT == AccessType::Store) + 0xf>();
+      SigTrap<0x20 * (EA == ErrorAction::Recover) +
+             0x10 * (AT == AccessType::Store) + 0xf>(p);
       if (EA == ErrorAction::Abort) __builtin_unreachable();
     }
 }
 
 void __hwasan_load(uptr p, uptr sz) {
   CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz);
 }
 void __hwasan_load1(uptr p) {

compiler-rt/trunk/lib/hwasan/hwasan_allocator.cc

struct HwasanMapUnmapCallback {
   void OnUnmap(uptr p, uptr size) const {
     // We are about to unmap a chunk of user memory.
     // It can return as user-requested mmap() or another thread stack.
     // Make it accessible with zero-tagged pointer.
     TagMemory(p, size, 0);
   }
 };
 
-#if !defined(__aarch64__)
-#error unsupported platform
+#if !defined(__aarch64__) && !defined(__x86_64__)
+#error Unsupported platform
 #endif
 
 static const uptr kMaxAllowedMallocSize = 2UL << 30;  // 2G
 static const uptr kRegionSizeLog = 20;
 static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog;
 typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap;
 
 struct AP32 {

compiler-rt/trunk/lib/hwasan/hwasan_interceptors.cc

if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
     internal_memcpy(new_ptr, ptr, copy_size);
     return new_ptr;
   }
   return hwasan_realloc(ptr, size, &stack);
 }
 
 INTERCEPTOR(void *, malloc, SIZE_T size) {
   GET_MALLOC_STACK_TRACE;
+  if (UNLIKELY(!hwasan_init_is_running))
+    ENSURE_HWASAN_INITED();
   if (UNLIKELY(!hwasan_inited))
     // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
     return AllocateFromLocalPool(size);
   return hwasan_malloc(size, &stack);
 }
 
 template <class Mmap>
 static void *mmap_interceptor(Mmap real_mmap, void *addr, SIZE_T sz, int prot,

compiler-rt/trunk/lib/hwasan/hwasan_linux.cc

while (size > step) {
       size -= step;
       void *res = MmapFixedNoAccess(addr, size, "shadow gap");
       if (addr == (uptr)res) return;
     }
   }
 
   Report(
       "ERROR: Failed to protect the shadow gap. "
-      "ASan cannot proceed correctly. ABORTING.\n");
+      "HWASan cannot proceed correctly. ABORTING.\n");
   DumpProcessMap();
   Die();
 }
 
 // LowMem covers as much of the first 4GB as possible.
-const uptr kLowMemEnd = 1UL<<32;
+const uptr kLowMemEnd = 1UL << 32;
 const uptr kLowShadowEnd = kLowMemEnd >> kShadowScale;
 const uptr kLowShadowStart = kLowShadowEnd >> kShadowScale;
 static uptr kHighShadowStart;
 static uptr kHighShadowEnd;
 static uptr kHighMemStart;
 
 bool InitShadow() {
   const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
 
-
   // HighMem covers the upper part of the address space.
   kHighShadowEnd = (maxVirtualAddress >> kShadowScale) + 1;
   kHighShadowStart = Max(kLowMemEnd, kHighShadowEnd >> kShadowScale);
   CHECK(kHighShadowStart < kHighShadowEnd);
 
   kHighMemStart = kHighShadowStart << kShadowScale;
   CHECK(kHighShadowEnd <= kHighMemStart);
 

 struct AccessInfo {
   uptr addr;
   uptr size;
   bool is_store;
   bool is_load;
   bool recover;
 };
 
-#if defined(__aarch64__)
 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
-  // Access type is encoded in BRK immediate as 0x9XY,
-  // where X&1 is 1 for store, 0 for load,
-  // and X&2 is 1 if the error is recoverable.
-  // Valid values of Y are 0 to 4, which are interpreted as log2(access_size),
-  // and 0xF, which means that access size is stored in X1 register.
-  // Access address is always in X0 register.
-  AccessInfo ai;
+  // Access type is passed in a platform dependent way (see below) and encoded
+  // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is
+  // recoverable. Valid values of Y are 0 to 4, which are interpreted as
+  // log2(access_size), and 0xF, which means that access size is passed via
+  // platform dependent register (see below).
+#if defined(__aarch64__)
+  // Access type is encoded in BRK immediate as 0x900 + 0xXY. For Y == 0xF,
+  // access size is stored in X1 register. Access address is always in X0
+  // register.
   uptr pc = (uptr)info->si_addr;
-  unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
+  const unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
   if ((code & 0xff00) != 0x900)
-    return AccessInfo{0, 0, false, false}; // Not ours.
-  bool is_store = code & 0x10;
-  bool recover = code & 0x20;
-  unsigned size_log = code & 0xf;
+    return AccessInfo{}; // Not ours.
+
+  const bool is_store = code & 0x10;
+  const bool recover = code & 0x20;
+  const const uptr addr = uc->uc_mcontext.regs[0];
+  const unsigned size_log = code & 0xf;
   if (size_log > 4 && size_log != 0xf)
-    return AccessInfo{0, 0, false, false}; // Not ours.
+    return AccessInfo{}; // Not ours.
+  const uptr size = size_log == 0xf ? uc->uc_mcontext.regs[1] : 1U << size_log;
+
+#elif defined(__x86_64__)
+  // Access type is encoded in the instruction following INT3 as
+  // NOP DWORD ptr [EAX + 0x40 + 0xXY]. For Y == 0xF, access size is stored in
+  // RSI register. Access address is always in RDI register.
+  uptr pc = (uptr)uc->uc_mcontext.gregs[REG_RIP];
+  uint8_t *nop = (uint8_t*)pc;
+  if (*nop != 0x0f || *(nop + 1) != 0x1f || *(nop + 2) != 0x40  ||
+      *(nop + 3) < 0x40)
+    return AccessInfo{}; // Not ours.
+  const unsigned code = *(nop + 3);
+
+  const bool is_store = code & 0x10;
+  const bool recover = code & 0x20;
+  const uptr addr = uc->uc_mcontext.gregs[REG_RDI];
+  const unsigned size_log = code & 0xf;
+  if (size_log > 4 && size_log != 0xf)
+    return AccessInfo{}; // Not ours.
+  const uptr size =
+      size_log == 0xf ? uc->uc_mcontext.gregs[REG_RSI] : 1U << size_log;
 
-  ai.is_store = is_store;
-  ai.is_load = !is_store;
-  ai.addr = uc->uc_mcontext.regs[0];
-  if (size_log == 0xf)
-    ai.size = uc->uc_mcontext.regs[1];
-  else
-    ai.size = 1U << size_log;
-  ai.recover = recover;
-  return ai;
-}
 #else
-static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
-  return AccessInfo{0, 0, false, false};
-}
+# error Unsupported architecture
 #endif
 
+  return AccessInfo{addr, size, is_store, !is_store, recover};
+}
+
 static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) {
-  SignalContext sig{info, uc};
   AccessInfo ai = GetAccessInfo(info, uc);
   if (!ai.is_store && !ai.is_load)
     return false;
 
   InternalScopedBuffer<BufferedStackTrace> stack_buffer(1);
   BufferedStackTrace *stack = stack_buffer.data();
   stack->Reset();
+  SignalContext sig{info, uc};
   GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, uc,
                 common_flags()->fast_unwind_on_fatal);
 
   ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store);
 
   ++hwasan_report_count;
   if (flags()->halt_on_error || !ai.recover)
     Die();
 
+#if defined(__aarch64__)
   uc->uc_mcontext.pc += 4;
+#elif defined(__x86_64__)
+#else
+# error Unsupported architecture
+#endif
   return true;
 }
 
 static void OnStackUnwind(const SignalContext &sig, const void *,
                           BufferedStackTrace *stack) {
   GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context,
                 common_flags()->fast_unwind_on_fatal);
 }