[HWASan] Introduce non-zero based and dynamic shadow memory (compiler-rt).

[alekseyshl]

Summary

Retire the fixed shadow memory mapping to avoid conflicts with default
process memory mapping (currently manifests on Android).

Tests on AArch64 show <1% performance loss and code size increase,
making it possible to use dynamic shadow memory by default.

For the simplicity and unifirmity sake, use dynamic shadow memory mapping
with base address accessed via ifunc resolver on all supported platforms.

Keep the fixed shadow memory mapping around to be able to run
performance comparison tests later.

Complementing D45840.

[eugenis]

That's a lot of code :(
But I don't see anything that could be thrown or factored out. We don't really need fixed (zero-based) stuff, but it's good to have it around for a while (for benchmarks and such).

lib/hwasan/hwasan_dynamic_shadow.h

2

this file is named differently (and some others, too)

[alekseyshl]

• Fix file comments and check more approriate condition for the PIE/ASLR warning.

[alekseyshl]

Rebase

lib/hwasan/CMakeLists.txt

 include_directories(..)
 
 # Runtime library sources and build flags.
 set(HWASAN_RTL_SOURCES
   hwasan.cc
   hwasan_allocator.cc
+  hwasan_dynamic_shadow.cc
   hwasan_interceptors.cc
   hwasan_linux.cc
+  hwasan_poisoning.cc
   hwasan_report.cc
   hwasan_thread.cc
-  hwasan_poisoning.cc
   )
 
 set(HWASAN_RTL_CXX_SOURCES
   hwasan_new_delete.cc)
 
 
 set(HWASAN_RTL_CFLAGS ${SANITIZER_COMMON_CFLAGS})
 append_rtti_flag(OFF HWASAN_RTL_CFLAGS)

lib/hwasan/hwasan.h

 #endif
 
 #ifndef HWASAN_CONTAINS_UBSAN
 # define HWASAN_CONTAINS_UBSAN CAN_SANITIZE_UB
 #endif
 
 typedef u8 tag_t;
 
-// Reasonable values are 4 (for 1/16th shadow) and 6 (for 1/64th).
-const uptr kShadowScale = 4;
-const uptr kShadowAlignment = 1UL << kShadowScale;
-
-#define MEM_TO_SHADOW_OFFSET(mem) ((uptr)(mem) >> kShadowScale)
-#define MEM_TO_SHADOW(mem) ((uptr)(mem) >> kShadowScale)
-#define SHADOW_TO_MEM(shadow) ((uptr)(shadow) << kShadowScale)
-
-#define MEM_IS_APP(mem) MemIsApp((uptr)(mem))
-
 // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address
 // translation and can be used to store a tag.
 const unsigned kAddressTagShift = 56;
 const uptr kAddressTagMask = 0xFFUL << kAddressTagShift;
 
 static inline tag_t GetTagFromPointer(uptr p) {
   return p >> kAddressTagShift;
 }

 }
 
 namespace __hwasan {
 
 extern int hwasan_inited;
 extern bool hwasan_init_is_running;
 extern int hwasan_report_count;
 
-bool MemIsApp(uptr p);
-
 bool ProtectRange(uptr beg, uptr end);
 bool InitShadow();
 char *GetProcSelfMaps();
 void InitializeInterceptors();
 
 void HwasanAllocatorInit();
 void HwasanAllocatorThreadFinish();
 void HwasanDeallocate(StackTrace *stack, void *ptr);

lib/hwasan/hwasan.cc

 //===-- hwasan.cc -----------------------------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of HWAddressSanitizer.
 //
 // HWAddressSanitizer runtime.
 //===----------------------------------------------------------------------===//
 
 #include "hwasan.h"
+#include "hwasan_mapping.h"
 #include "hwasan_thread.h"
 #include "hwasan_poisoning.h"
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_flag_parser.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_procmaps.h"

 }
 
 } // namespace __hwasan
 
 // Interface.
 
 using namespace __hwasan;
 
+uptr __hwasan_shadow_memory_dynamic_address;  // Global interface symbol.
+
 void __hwasan_init() {
   CHECK(!hwasan_init_is_running);
   if (hwasan_inited) return;
   hwasan_init_is_running = 1;
   SanitizerToolName = "HWAddressSanitizer";
 
   InitTlsSize();
 
   CacheBinaryName();
   InitializeFlags();
 
   // Install tool-specific callbacks in sanitizer_common.
   SetCheckFailedCallback(HWAsanCheckFailed);
 
   __sanitizer_set_report_path(common_flags()->log_path);
 
   InitializeInterceptors();
   InstallDeadlySignalHandlers(HwasanOnDeadlySignal);
   InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
 
   DisableCoreDumperIfNecessary();
   if (!InitShadow()) {
-    Printf("FATAL: HWAddressSanitizer can not mmap the shadow memory.\n");
+    Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n");
+    if (HWASAN_FIXED_MAPPING) {
       Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
       Printf("FATAL: Disabling ASLR is known to cause this error.\n");
       Printf("FATAL: If running under GDB, try "
              "'set disable-randomization off'.\n");
+    }
     DumpProcessMap();
     Die();
   }
 
   Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
 
   InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
 

lib/hwasan/hwasan_allocator.cc

 #include "sanitizer_common/sanitizer_allocator.h"
 #include "sanitizer_common/sanitizer_allocator_checks.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 #include "hwasan.h"
 #include "hwasan_allocator.h"
+#include "hwasan_mapping.h"
 #include "hwasan_thread.h"
 #include "hwasan_poisoning.h"
 
 namespace __hwasan {
 
 enum {
   CHUNK_INVALID = 0,
   CHUNK_FREE = 1,

lib/hwasan/hwasan_dynamic_shadow.h

+//===-- hwasan_dynamic_shadow.h ---------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer. It reserves dynamic shadow memory
+/// region.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_PREMAP_SHADOW_H
+#define HWASAN_PREMAP_SHADOW_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+namespace __hwasan {
+
+uptr FindDynamicShadowStart(uptr shadow_size_bytes);
+
+}  // namespace __hwasan
+
+#endif  // HWASAN_PREMAP_SHADOW_H

lib/hwasan/hwasan_dynamic_shadow.cc

+//===-- hwasan_dynamic_shadow.cc --------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer. It reserves dynamic shadow memory
+/// region and handles ifunc resolver case, when necessary.
+///
+//===----------------------------------------------------------------------===//
+
+#include "hwasan_dynamic_shadow.h"
+#include "hwasan_mapping.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_posix.h"
+
+// The code in this file needs to run in an unrelocated binary. It should not
+// access any external symbol, including its own non-hidden globals.
+
+namespace __hwasan {
+
+static void UnmapFromTo(uptr from, uptr to) {
+  if (to == from)
+    return;
+  CHECK(to >= from);
+  uptr res = internal_munmap(reinterpret_cast<void *>(from), to - from);
+  if (UNLIKELY(internal_iserror(res))) {
+    Report("ERROR: %s failed to unmap 0x%zx (%zd) bytes at address %p\n",
+           SanitizerToolName, to - from, to - from, from);
+    CHECK("unable to unmap" && 0);
+  }
+}
+
+// Returns an address aligned to 8 pages, such that one page on the left and
+// shadow_size_bytes bytes on the right of it are mapped r/o.
+static uptr MapDynamicShadow(uptr shadow_size_bytes) {
+  const uptr granularity = GetMmapGranularity();
+  const uptr alignment = granularity * SHADOW_GRANULARITY;
+  const uptr left_padding = granularity;
+  const uptr shadow_size =
+      RoundUpTo(shadow_size_bytes, granularity);
+  const uptr map_size = shadow_size + left_padding + alignment;
+
+  const uptr map_start = (uptr)MmapNoAccess(map_size);
+  CHECK_NE(map_start, ~(uptr)0);
+
+  const uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
+
+  UnmapFromTo(map_start, shadow_start - left_padding);
+  UnmapFromTo(shadow_start + shadow_size, map_start + map_size);
+
+  return shadow_start;
+}
+
+}  // namespace __hwasan
+
+#if HWASAN_PREMAP_SHADOW
+
+extern "C" {
+
+INTERFACE_ATTRIBUTE void __hwasan_shadow();
+decltype(__hwasan_shadow)* __hwasan_premap_shadow();
+
+}  // extern "C"
+
+namespace __hwasan {
+
+// Conservative upper limit.
+static uptr PremapShadowSize() {
+  return RoundUpTo(GetMaxVirtualAddress() >> kShadowScale,
+                   GetMmapGranularity());
+}
+
+static uptr PremapShadow() {
+  return MapDynamicShadow(PremapShadowSize());
+}
+
+static bool IsPremapShadowAvailable() {
+  const uptr shadow = reinterpret_cast<uptr>(&__hwasan_shadow);
+  const uptr resolver = reinterpret_cast<uptr>(&__hwasan_premap_shadow);
+  // shadow == resolver is how Android KitKat and older handles ifunc.
+  // shadow == 0 just in case.
+  return shadow != 0 && shadow != resolver;
+}
+
+static uptr FindPremappedShadowStart(uptr shadow_size_bytes) {
+  const uptr granularity = GetMmapGranularity();
+  const uptr shadow_start = reinterpret_cast<uptr>(&__hwasan_shadow);
+  const uptr premap_shadow_size = PremapShadowSize();
+  const uptr shadow_size = RoundUpTo(shadow_size_bytes, granularity);
+
+  // We may have mapped too much. Release extra memory.
+  UnmapFromTo(shadow_start + shadow_size, shadow_start + premap_shadow_size);
+  return shadow_start;
+}
+
+}  // namespace __hwasan
+
+extern "C" {
+
+decltype(__hwasan_shadow)* __hwasan_premap_shadow() {
+  // The resolver might be called multiple times. Map the shadow just once.
+  static __sanitizer::uptr shadow = 0;
+  if (!shadow)
+    shadow = __hwasan::PremapShadow();
+  return reinterpret_cast<decltype(__hwasan_shadow)*>(shadow);
+}
+
+// __hwasan_shadow is a "function" that has the same address as the first byte
+// of the shadow mapping.
+INTERFACE_ATTRIBUTE __attribute__((ifunc("__hwasan_premap_shadow")))
+void __hwasan_shadow();
+
+}  // extern "C"
+
+#endif  // HWASAN_PREMAP_SHADOW
+
+namespace __hwasan {
+
+uptr FindDynamicShadowStart(uptr shadow_size_bytes) {
+#if HWASAN_PREMAP_SHADOW
+  if (IsPremapShadowAvailable())
+    return FindPremappedShadowStart(shadow_size_bytes);
+#endif
+  return MapDynamicShadow(shadow_size_bytes);
+}
+
+}  // namespace __hwasan

lib/hwasan/hwasan_interceptors.cc

 // Interceptors for standard library functions.
 //
 // FIXME: move as many interceptors as possible into
 // sanitizer_common/sanitizer_common_interceptors.h
 //===----------------------------------------------------------------------===//
 
 #include "interception/interception.h"
 #include "hwasan.h"
+#include "hwasan_mapping.h"
 #include "hwasan_thread.h"
 #include "hwasan_poisoning.h"
 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
 #include "sanitizer_common/sanitizer_allocator.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
 #include "sanitizer_common/sanitizer_allocator_internal.h"
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"

lib/hwasan/hwasan_interface_internal.h

 //===----------------------------------------------------------------------===//
 
 #ifndef HWASAN_INTERFACE_INTERNAL_H
 #define HWASAN_INTERFACE_INTERNAL_H
 
 #include "sanitizer_common/sanitizer_internal_defs.h"
 
 extern "C" {
+
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_init();
 
 using __sanitizer::uptr;
 using __sanitizer::sptr;
 using __sanitizer::uu64;
 using __sanitizer::uu32;
 using __sanitizer::uu16;
 using __sanitizer::u64;
 using __sanitizer::u32;
 using __sanitizer::u16;
 using __sanitizer::u8;
 
 SANITIZER_INTERFACE_ATTRIBUTE
+extern uptr __hwasan_shadow_memory_dynamic_address;
+
+SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_loadN(uptr, uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_load1(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_load2(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_load4(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE

lib/hwasan/hwasan_linux.cc

-//===-- hwasan_linux.cc -----------------------------------------------------===//
+//===-- hwasan_linux.cc -----------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
-//
-// This file is a part of HWAddressSanitizer.
-//
-// Linux-, NetBSD- and FreeBSD-specific code.
+///
+/// \file
+/// This file is a part of HWAddressSanitizer and contains Linux-, NetBSD- and
+/// FreeBSD-specific code.
+///
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_common/sanitizer_platform.h"
 #if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
 
 #include "hwasan.h"
+#include "hwasan_dynamic_shadow.h"
+#include "hwasan_interface_internal.h"
+#include "hwasan_mapping.h"
 #include "hwasan_thread.h"
 
 #include <elf.h>
 #include <link.h>
 #include <pthread.h>
+#include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <signal.h>
+#include <sys/resource.h>
+#include <sys/time.h>
 #include <unistd.h>
 #include <unwind.h>
-#include <sys/time.h>
-#include <sys/resource.h>
 
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_procmaps.h"
 
 namespace __hwasan {
 
-void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
+static void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
   CHECK_EQ((beg % GetMmapGranularity()), 0);
   CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
   uptr size = end - beg + 1;
   DecreaseTotalMmap(size);  // Don't count the shadow against mmap_limit_mb.
   void *res = MmapFixedNoReserve(beg, size, name);
   if (res != (void *)beg) {
     Report(
         "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
         "Perhaps you're using ulimit -v\n",
         size);
     Abort();
   }
   if (common_flags()->no_huge_pages_for_shadow) NoHugePagesInRegion(beg, size);
   if (common_flags()->use_madv_dontdump) DontDumpShadowMemory(beg, size);
 }
 
 static void ProtectGap(uptr addr, uptr size) {
+  if (!size)
+    return;
   void *res = MmapFixedNoAccess(addr, size, "shadow gap");
-  if (addr == (uptr)res) return;
+  if (addr == (uptr)res)
+    return;
   // A few pages at the start of the address space can not be protected.
   // But we really want to protect as much as possible, to prevent this memory
   // being returned as a result of a non-FIXED mmap().
   if (addr == 0) {
     uptr step = GetMmapGranularity();
     while (size > step) {
       addr += step;
       size -= step;
       void *res = MmapFixedNoAccess(addr, size, "shadow gap");
-      if (addr == (uptr)res) return;
+      if (addr == (uptr)res)
+        return;
     }
   }
 
   Report(
-      "ERROR: Failed to protect the shadow gap. "
-      "HWASan cannot proceed correctly. ABORTING.\n");
+      "ERROR: Failed to protect shadow gap [%p, %p]. "
+      "HWASan cannot proceed correctly. ABORTING.\n", (void *)addr,
+      (void *)(addr + size));
   DumpProcessMap();
   Die();
 }
 
-// LowMem covers as much of the first 4GB as possible.
-const uptr kLowMemEnd = 1UL << 32;
-const uptr kLowShadowEnd = kLowMemEnd >> kShadowScale;
-const uptr kLowShadowStart = kLowShadowEnd >> kShadowScale;
+static uptr kLowMemStart;
+static uptr kLowMemEnd;
+static uptr kLowShadowEnd;
+static uptr kLowShadowStart;
 static uptr kHighShadowStart;
 static uptr kHighShadowEnd;
 static uptr kHighMemStart;
+static uptr kHighMemEnd;
 
-bool InitShadow() {
-  const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
+static void PrintRange(uptr start, uptr end, const char *name) {
+  Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name);
+}
 
+static void PrintAddressSpaceLayout() {
+  PrintRange(kHighMemStart, kHighMemEnd, "HighMem");
+  if (kHighShadowEnd + 1 < kHighMemStart)
+    PrintRange(kHighShadowEnd + 1, kHighMemStart - 1, "ShadowGap");
+  else
+    CHECK_EQ(kHighShadowEnd + 1, kHighMemStart);
+  PrintRange(kHighShadowStart, kHighShadowEnd, "HighShadow");
+  if (SHADOW_OFFSET) {
+    if (kLowShadowEnd + 1 < kHighShadowStart)
+      PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap");
+    else
+      CHECK_EQ(kLowMemEnd + 1, kHighShadowStart);
+    PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow");
+    if (kLowMemEnd + 1 < kLowShadowStart)
+      PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap");
+    else
+      CHECK_EQ(kLowMemEnd + 1, kLowShadowStart);
+    PrintRange(kLowMemStart, kLowMemEnd, "LowMem");
+    CHECK_EQ(0, kLowMemStart);
+  } else {
+    if (kLowMemEnd + 1 < kHighShadowStart)
+      PrintRange(kLowMemEnd + 1, kHighShadowStart - 1, "ShadowGap");
+    else
+      CHECK_EQ(kLowMemEnd + 1, kHighShadowStart);
+    PrintRange(kLowMemStart, kLowMemEnd, "LowMem");
+    CHECK_EQ(kLowShadowEnd + 1, kLowMemStart);
+    PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow");
+    PrintRange(0, kLowShadowStart - 1, "ShadowGap");
+  }
+}
+
+static uptr GetHighMemEnd() {
   // HighMem covers the upper part of the address space.
-  kHighShadowEnd = (maxVirtualAddress >> kShadowScale) + 1;
-  kHighShadowStart = Max(kLowMemEnd, kHighShadowEnd >> kShadowScale);
-  CHECK(kHighShadowStart < kHighShadowEnd);
-
-  kHighMemStart = kHighShadowStart << kShadowScale;
-  CHECK(kHighShadowEnd <= kHighMemStart);
-
-  if (Verbosity()) {
-    Printf("|| `[%p, %p]` || HighMem    ||\n", (void *)kHighMemStart,
-           (void *)maxVirtualAddress);
-    if (kHighMemStart > kHighShadowEnd)
-      Printf("|| `[%p, %p]` || ShadowGap2 ||\n", (void *)kHighShadowEnd,
-             (void *)kHighMemStart);
-    Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowStart,
-           (void *)kHighShadowEnd);
-    if (kHighShadowStart > kLowMemEnd)
-      Printf("|| `[%p, %p]` || ShadowGap2 ||\n", (void *)kHighShadowEnd,
-             (void *)kHighMemStart);
-    Printf("|| `[%p, %p]` || LowMem     ||\n", (void *)kLowShadowEnd,
-           (void *)kLowMemEnd);
-    Printf("|| `[%p, %p]` || LowShadow  ||\n", (void *)kLowShadowStart,
-           (void *)kLowShadowEnd);
-    Printf("|| `[%p, %p]` || ShadowGap1 ||\n", (void *)0,
-           (void *)kLowShadowStart);
-  }
-
-  ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd - 1, "low shadow");
-  ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd - 1, "high shadow");
-  ProtectGap(0, kLowShadowStart);
-  if (kHighShadowStart > kLowMemEnd)
-    ProtectGap(kLowMemEnd, kHighShadowStart - kLowMemEnd);
-  if (kHighMemStart > kHighShadowEnd)
-    ProtectGap(kHighShadowEnd, kHighMemStart - kHighShadowEnd);
+  uptr max_address = GetMaxUserVirtualAddress();
+  if (SHADOW_OFFSET)
+    // Adjust max address to make sure that kHighMemEnd and kHighMemStart are
+    // properly aligned:
+    max_address |= SHADOW_GRANULARITY * GetMmapGranularity() - 1;
+  return max_address;
+}
+
+static void InitializeShadowBaseAddress(uptr shadow_size_bytes) {
+  // Set the shadow memory address to uninitialized.
+  __hwasan_shadow_memory_dynamic_address = kDefaultShadowSentinel;
+  uptr shadow_start = SHADOW_OFFSET;
+  // Detect if a dynamic shadow address must be used and find the available
+  // location when necessary. When dynamic address is used, the macro
+  // kLowShadowBeg expands to __hwasan_shadow_memory_dynamic_address which
+  // was just set to kDefaultShadowSentinel.
+  if (shadow_start == kDefaultShadowSentinel) {
+    __hwasan_shadow_memory_dynamic_address = 0;
+    CHECK_EQ(0, SHADOW_OFFSET);
+    shadow_start = FindDynamicShadowStart(shadow_size_bytes);
+  }
+  // Update the shadow memory address (potentially) used by instrumentation.
+  __hwasan_shadow_memory_dynamic_address = shadow_start;
+}
+
+bool InitShadow() {
+  // Define the entire memory range.
+  kHighMemEnd = GetHighMemEnd();
+
+  // Determine shadow memory base offset.
+  InitializeShadowBaseAddress(MEM_TO_SHADOW_SIZE(kHighMemEnd));
+
+  // Place the low memory first.
+  if (SHADOW_OFFSET) {
+    kLowMemEnd = SHADOW_OFFSET - 1;
+    kLowMemStart = 0;
+  } else {
+    // LowMem covers as much of the first 4GB as possible.
+    kLowMemEnd = (1UL << 32) - 1;
+    kLowMemStart = MEM_TO_SHADOW(kLowMemEnd) + 1;
+  }
+
+  // Define the low shadow based on the already placed low memory.
+  kLowShadowEnd = MEM_TO_SHADOW(kLowMemEnd);
+  kLowShadowStart = SHADOW_OFFSET ? SHADOW_OFFSET : MEM_TO_SHADOW(kLowMemStart);
+
+  // High shadow takes whatever memory is left up there (making sure it is not
+  // interfering with low memory in the fixed case).
+  kHighShadowEnd = MEM_TO_SHADOW(kHighMemEnd);
+  kHighShadowStart = Max(kLowMemEnd, MEM_TO_SHADOW(kHighShadowEnd)) + 1;
+
+  // High memory starts where allocated shadow allows.
+  kHighMemStart = SHADOW_TO_MEM(kHighShadowStart);
+
+  // Check the sanity of the defined memory ranges (there might be gaps).
+  CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0);
+  CHECK_GT(kHighMemStart, kHighShadowEnd);
+  CHECK_GT(kHighShadowEnd, kHighShadowStart);
+  CHECK_GT(kHighShadowStart, kLowMemEnd);
+  CHECK_GT(kLowMemEnd, kLowMemStart);
+  CHECK_GT(kLowShadowEnd, kLowShadowStart);
+  if (SHADOW_OFFSET)
+    CHECK_GT(kLowShadowStart, kLowMemEnd);
+  else
+    CHECK_GT(kLowMemEnd, kLowShadowStart);
+
+  if (Verbosity())
+    PrintAddressSpaceLayout();
+
+  // Reserve shadow memory.
+  ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd, "low shadow");
+  ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd, "high shadow");
+
+  // Protect all the gaps.
+  ProtectGap(0, Min(kLowMemStart, kLowShadowStart));
+  if (SHADOW_OFFSET) {
+    if (kLowMemEnd + 1 < kLowShadowStart)
+      ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1);
+    if (kLowShadowEnd + 1 < kHighShadowStart)
+      ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1);
+  } else {
+    if (kLowMemEnd + 1 < kHighShadowStart)
+      ProtectGap(kLowMemEnd + 1, kHighShadowStart - kLowMemEnd - 1);
+  }
+  if (kHighShadowEnd + 1 < kHighMemStart)
+    ProtectGap(kHighShadowEnd + 1, kHighMemStart - kHighShadowEnd - 1);
 
   return true;
 }
 
 bool MemIsApp(uptr p) {
   CHECK(GetTagFromPointer(p) == 0);
-  return p >= kHighMemStart || (p >= kLowShadowEnd && p < kLowMemEnd);
+  return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
 }
 
 static void HwasanAtExit(void) {
   if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
     ReportStats();
   if (hwasan_report_count > 0) {
     // ReportAtExitStatistics();
     if (common_flags()->exitcode)

lib/hwasan/hwasan_mapping.h

+//===-- hwasan_mapping.h ----------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer and defines memory mapping.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef HWASAN_MAPPING_H
+#define HWASAN_MAPPING_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+// Typical mapping on Linux/x86_64 with fixed shadow mapping:
+// || [0x080000000000, 0x7fffffffffff] || HighMem    ||
+// || [0x008000000000, 0x07ffffffffff] || HighShadow ||
+// || [0x000100000000, 0x007fffffffff] || ShadowGap  ||
+// || [0x000010000000, 0x0000ffffffff] || LowMem     ||
+// || [0x000001000000, 0x00000fffffff] || LowShadow  ||
+// || [0x000000000000, 0x000000ffffff] || ShadowGap  ||
+//
+// and with dynamic shadow mapped at [0x770d59f40000, 0x7f0d59f40000]:
+// || [0x7f0d59f40000, 0x7fffffffffff] || HighMem    ||
+// || [0x7efe2f934000, 0x7f0d59f3ffff] || HighShadow ||
+// || [0x7e7e2f934000, 0x7efe2f933fff] || ShadowGap  ||
+// || [0x770d59f40000, 0x7e7e2f933fff] || LowShadow  ||
+// || [0x000000000000, 0x770d59f3ffff] || LowMem     ||
+
+// Typical mapping on Android/AArch64 (39-bit VMA):
+// || [0x001000000000, 0x007fffffffff] || HighMem    ||
+// || [0x000800000000, 0x000fffffffff] || ShadowGap  ||
+// || [0x000100000000, 0x0007ffffffff] || HighShadow ||
+// || [0x000010000000, 0x0000ffffffff] || LowMem     ||
+// || [0x000001000000, 0x00000fffffff] || LowShadow  ||
+// || [0x000000000000, 0x000000ffffff] || ShadowGap  ||
+//
+// and with dynamic shadow mapped: [0x007477480000, 0x007c77480000]:
+// || [0x007c77480000, 0x007fffffffff] || HighMem    ||
+// || [0x007c3ebc8000, 0x007c7747ffff] || HighShadow ||
+// || [0x007bbebc8000, 0x007c3ebc7fff] || ShadowGap  ||
+// || [0x007477480000, 0x007bbebc7fff] || LowShadow  ||
+// || [0x000000000000, 0x00747747ffff] || LowMem     ||
+
+static constexpr __sanitizer::u64 kDefaultShadowSentinel = ~(__sanitizer::u64)0;
+
+// Reasonable values are 4 (for 1/16th shadow) and 6 (for 1/64th).
+constexpr __sanitizer::uptr kShadowScale = 4;
+constexpr __sanitizer::uptr kShadowAlignment = 1ULL << kShadowScale;
+
+#define HWASAN_FIXED_MAPPING 0
+
+#if HWASAN_FIXED_MAPPING
+# define SHADOW_OFFSET (0)
+# define HWASAN_PREMAP_SHADOW 0
+#else
+# define SHADOW_OFFSET (__hwasan_shadow_memory_dynamic_address)
+# define HWASAN_PREMAP_SHADOW 1
+#endif
+
+#define SHADOW_GRANULARITY (1ULL << kShadowScale)
+
+#define MEM_TO_SHADOW(mem) (((uptr)(mem) >> kShadowScale) + SHADOW_OFFSET)
+#define SHADOW_TO_MEM(shadow) (((uptr)(shadow) - SHADOW_OFFSET) << kShadowScale)
+
+#define MEM_TO_SHADOW_SIZE(size) ((uptr)(size) >> kShadowScale)
+
+#define MEM_IS_APP(mem) MemIsApp((uptr)(mem))
+
+namespace __hwasan {
+
+bool MemIsApp(uptr p);
+
+}  // namespace __hwasan
+
+#endif  // HWASAN_MAPPING_H

lib/hwasan/hwasan_poisoning.cc

 //===-- hwasan_poisoning.cc ---------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of HWAddressSanitizer.
 //
 //===----------------------------------------------------------------------===//
 
 #include "hwasan_poisoning.h"
 
+#include "hwasan_mapping.h"
 #include "interception/interception.h"
 #include "sanitizer_common/sanitizer_common.h"
 
 namespace __hwasan {
 
 uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
   CHECK(IsAligned(p, kShadowAlignment));
   CHECK(IsAligned(size, kShadowAlignment));
   uptr shadow_start = MEM_TO_SHADOW(p);
-  uptr shadow_size = MEM_TO_SHADOW_OFFSET(size);
+  uptr shadow_size = MEM_TO_SHADOW_SIZE(size);
   internal_memset((void *)shadow_start, tag, shadow_size);
   return AddTagToPointer(p, tag);
 }
 
 uptr TagMemory(uptr p, uptr size, tag_t tag) {
   uptr start = RoundDownTo(p, kShadowAlignment);
   uptr end = RoundUpTo(p + size, kShadowAlignment);
   return TagMemoryAligned(start, end - start, tag);
 }
 
 }  // namespace __hwasan

lib/hwasan/hwasan_report.cc

 //===-- hwasan_report.cc ----------------------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of HWAddressSanitizer.
 //
 // Error reporting.
 //===----------------------------------------------------------------------===//
 
 #include "hwasan.h"
 #include "hwasan_allocator.h"
+#include "hwasan_mapping.h"
 #include "sanitizer_common/sanitizer_allocator_internal.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_mutex.h"
 #include "sanitizer_common/sanitizer_report_decorator.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 #include "sanitizer_common/sanitizer_symbolizer.h"
 

lib/hwasan/hwasan_thread.cc

 
 #include "hwasan.h"
+#include "hwasan_mapping.h"
 #include "hwasan_thread.h"
 #include "hwasan_poisoning.h"
 #include "hwasan_interface_internal.h"
 
 #include "sanitizer_common/sanitizer_tls_get_addr.h"
 
 namespace __hwasan {