[hwasan] Inline instrumentation & fixed shadow.

[eugenis]

Summary

This brings CPU overhead on bzip2 down from 5.5x to 2x.

[kcc]

LGTM with one optional question.

compiler-rt/lib/hwasan/hwasan_linux.cc

125

Do you still want to keep it?

[eugenis]

Removed unused variable.

[eugenis]

compiler-rt/lib/hwasan/hwasan_linux.cc

125

you are right, there is no point in keeping it

[eugenis]

update hwasan tests

compiler-rt/lib/hwasan/hwasan.h

 
 typedef u8 tag_t;
 
 // Reasonable values are 4 (for 1/16th shadow) and 6 (for 1/64th).
 const uptr kShadowScale = 4;
 const uptr kShadowAlignment = 1UL << kShadowScale;
 
 #define MEM_TO_SHADOW_OFFSET(mem) ((uptr)(mem) >> kShadowScale)
-#define MEM_TO_SHADOW(mem)         \
-  (((uptr)(mem) >> kShadowScale) + \
-   __hwasan_shadow_memory_dynamic_address_internal)
-#define SHADOW_TO_MEM(shadow)                                       \
-  (((uptr)(shadow)-__hwasan_shadow_memory_dynamic_address_internal) \
-   << kShadowScale)
+#define MEM_TO_SHADOW(mem) ((uptr)(mem) >> kShadowScale)
+#define SHADOW_TO_MEM(shadow) ((uptr)(shadow) << kShadowScale)
 
 #define MEM_IS_APP(mem) true
 
 // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address
 // translation and can be used to store a tag.
 const unsigned kAddressTagShift = 56;
 const uptr kAddressTagMask = 0xFFUL << kAddressTagShift;
 

compiler-rt/lib/hwasan/hwasan.cc

 }
 void __sanitizer_unaligned_store32(uu32 *p, u32 x) {
   *p = x;
 }
 void __sanitizer_unaligned_store64(uu64 *p, u64 x) {
   *p = x;
 }
 
+template<unsigned X>
 __attribute__((always_inline))
 static void SigIll() {
 #if defined(__aarch64__)
-  asm("hlt #0x1\n\t");
+  asm("hlt %0\n\t" ::"n"(X));
 #elif defined(__x86_64__) || defined(__i386__)
   asm("ud2\n\t");
 #else
   // FIXME: not always sigill.
   __builtin_trap();
 #endif
   // __builtin_unreachable();
 }
 
+template<bool IsStore, unsigned LogSize>
 __attribute__((always_inline, nodebug))
 static void CheckAddress(uptr p) {
   tag_t ptr_tag = GetTagFromPointer(p);
   uptr ptr_raw = p & ~kAddressTagMask;
   tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(ptr_raw);
-  if (ptr_tag != mem_tag)
-    SigIll();
+  if (UNLIKELY(ptr_tag != mem_tag)) SigIll<0x100 + 0x10 * IsStore + LogSize>();
 }
 
+template<bool IsStore>
 __attribute__((always_inline, nodebug))
 static void CheckAddressSized(uptr p, uptr sz) {
   CHECK_NE(0, sz);
   tag_t ptr_tag = GetTagFromPointer(p);
   uptr ptr_raw = p & ~kAddressTagMask;
   tag_t *shadow_first = (tag_t *)MEM_TO_SHADOW(ptr_raw);
   tag_t *shadow_last = (tag_t *)MEM_TO_SHADOW(ptr_raw + sz - 1);
   for (tag_t *t = shadow_first; t <= shadow_last; ++t)
-    if (ptr_tag != *t) SigIll();
+    if (UNLIKELY(ptr_tag != *t)) SigIll<0x100 + 0x10 * IsStore + 0xf>();
 }
 
-void __hwasan_load(uptr p, uptr sz) { CheckAddressSized(p, sz); }
-void __hwasan_load1(uptr p) { CheckAddress(p); }
-void __hwasan_load2(uptr p) { CheckAddress(p); }
-void __hwasan_load4(uptr p) { CheckAddress(p); }
-void __hwasan_load8(uptr p) { CheckAddress(p); }
-void __hwasan_load16(uptr p) { CheckAddress(p); }
-
-void __hwasan_store(uptr p, uptr sz) { CheckAddressSized(p, sz); }
-void __hwasan_store1(uptr p) { CheckAddress(p); }
-void __hwasan_store2(uptr p) { CheckAddress(p); }
-void __hwasan_store4(uptr p) { CheckAddress(p); }
-void __hwasan_store8(uptr p) { CheckAddress(p); }
-void __hwasan_store16(uptr p) { CheckAddress(p); }
+void __hwasan_load(uptr p, uptr sz) { CheckAddressSized<false>(p, sz); }
+void __hwasan_load1(uptr p) { CheckAddress<false, 0>(p); }
+void __hwasan_load2(uptr p) { CheckAddress<false, 1>(p); }
+void __hwasan_load4(uptr p) { CheckAddress<false, 2>(p); }
+void __hwasan_load8(uptr p) { CheckAddress<false, 3>(p); }
+void __hwasan_load16(uptr p) { CheckAddress<false, 4>(p); }
+
+void __hwasan_store(uptr p, uptr sz) { CheckAddressSized<true>(p, sz); }
+void __hwasan_store1(uptr p) { CheckAddress<true, 0>(p); }
+void __hwasan_store2(uptr p) { CheckAddress<true, 1>(p); }
+void __hwasan_store4(uptr p) { CheckAddress<true, 2>(p); }
+void __hwasan_store8(uptr p) { CheckAddress<true, 3>(p); }
+void __hwasan_store16(uptr p) { CheckAddress<true, 4>(p); }
 
 #if !SANITIZER_SUPPORTS_WEAK_HOOKS
 extern "C" {
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
 const char* __hwasan_default_options() { return ""; }
 }  // extern "C"
 #endif
 
 extern "C" {
 SANITIZER_INTERFACE_ATTRIBUTE
 void __sanitizer_print_stack_trace() {
   GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
   stack.Print();
 }
 } // extern "C"

compiler-rt/lib/hwasan/hwasan_linux.cc

 
 uptr __hwasan_shadow_memory_dynamic_address;
 
 __attribute__((alias("__hwasan_shadow_memory_dynamic_address")))
 extern uptr __hwasan_shadow_memory_dynamic_address_internal;
 
 namespace __hwasan {
 
+void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
+  CHECK_EQ((beg % GetMmapGranularity()), 0);
+  CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
+  uptr size = end - beg + 1;
+  DecreaseTotalMmap(size);  // Don't count the shadow against mmap_limit_mb.
+  void *res = MmapFixedNoReserve(beg, size, name);
+  if (res != (void *)beg) {
+    Report(
+        "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
+        "Perhaps you're using ulimit -v\n",
+        size);
+    Abort();
+  }
+  if (common_flags()->no_huge_pages_for_shadow) NoHugePagesInRegion(beg, size);
+  if (common_flags()->use_madv_dontdump) DontDumpShadowMemory(beg, size);
+}
+
+static void ProtectGap(uptr addr, uptr size) {
+  void *res = MmapFixedNoAccess(addr, size, "shadow gap");
+  if (addr == (uptr)res) return;
+  // A few pages at the start of the address space can not be protected.
+  // But we really want to protect as much as possible, to prevent this memory
+  // being returned as a result of a non-FIXED mmap().
+  if (addr == 0) {
+    uptr step = GetMmapGranularity();
+    while (size > step) {
+      addr += step;
+      size -= step;
+      void *res = MmapFixedNoAccess(addr, size, "shadow gap");
+      if (addr == (uptr)res) return;
+    }
+  }
+
+  Report(
+      "ERROR: Failed to protect the shadow gap. "
+      "ASan cannot proceed correctly. ABORTING.\n");
+  DumpProcessMap();
+  Die();
+}
+
 bool InitShadow() {
   const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
-  uptr shadow_size = MEM_TO_SHADOW_OFFSET(maxVirtualAddress) + 1;
-  __hwasan_shadow_memory_dynamic_address =
-      reinterpret_cast<uptr>(MmapNoReserveOrDie(shadow_size, "shadow"));
+
+  // LowMem covers as much of the first 4GB as possible.
+  const uptr kLowMemEnd = 1UL<<32;
+  const uptr kLowShadowEnd = kLowMemEnd >> kShadowScale;
+  const uptr kLowShadowStart = kLowShadowEnd >> kShadowScale;
+
+  // HighMem covers the upper part of the address space.
+  const uptr kHighShadowEnd = (maxVirtualAddress >> kShadowScale) + 1;
+  const uptr kHighShadowStart = Max(kLowMemEnd, kHighShadowEnd >> kShadowScale);
+  CHECK(kHighShadowStart < kHighShadowEnd);
+
+  const uptr kHighMemStart = kHighShadowStart << kShadowScale;
+  CHECK(kHighShadowEnd <= kHighMemStart);
+
+  if (Verbosity()) {
+    Printf("|| `[%p, %p]` || HighMem    ||\n", (void *)kHighMemStart,
+           (void *)maxVirtualAddress);
+    if (kHighMemStart > kHighShadowEnd)
+      Printf("|| `[%p, %p]` || ShadowGap2 ||\n", (void *)kHighShadowEnd,
+             (void *)kHighMemStart);
+    Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowStart,
+           (void *)kHighShadowEnd);
+    if (kHighShadowStart > kLowMemEnd)
+      Printf("|| `[%p, %p]` || ShadowGap2 ||\n", (void *)kHighShadowEnd,
+             (void *)kHighMemStart);
+    Printf("|| `[%p, %p]` || LowMem     ||\n", (void *)kLowShadowEnd,
+           (void *)kLowMemEnd);
+    Printf("|| `[%p, %p]` || LowShadow  ||\n", (void *)kLowShadowStart,
+           (void *)kLowShadowEnd);
+    Printf("|| `[%p, %p]` || ShadowGap1 ||\n", (void *)0,
+           (void *)kLowShadowStart);
+  }
+
+  ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd - 1, "low shadow");
+  ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd - 1, "high shadow");
+  ProtectGap(0, kLowShadowStart);
+  if (kHighShadowStart > kLowMemEnd)
+    ProtectGap(kLowMemEnd, kHighShadowStart - kLowMemEnd);
+  if (kHighMemStart > kHighShadowEnd)
+    ProtectGap(kHighShadowEnd, kHighMemStart - kHighShadowEnd);
+
+  __hwasan_shadow_memory_dynamic_address = 0;
+
   return true;
 }
 
 static void HwasanAtExit(void) {
   if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
     ReportStats();
   if (hwasan_report_count > 0) {
     // ReportAtExitStatistics();
struct AccessInfo {
   uptr addr;
   uptr size;
   bool is_store;
   bool is_load;
 };
 
 #if defined(__aarch64__)
 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
+  // Access type is encoded in HLT immediate as 0x1XY,
+  // where X is 1 for store, 0 for load.
+  // Valid values of Y are 0 to 4, which are interpreted as log2(access_size),
+  // and 0xF, which means that access size is stored in X1 register.
+  // Access address is always in X0 register.
   AccessInfo ai;
   uptr pc = (uptr)info->si_addr;
+  unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
+  if ((code & 0xff00) != 0x100)
+    return AccessInfo{0, 0, false, false}; // Not ours.
+  bool is_store = code & 0x10;
+  unsigned size_log = code & 0xff;
+  if (size_log > 4 && size_log != 0xf)
+    return AccessInfo{0, 0, false, false}; // Not ours.
 
-  struct {
-    uptr addr;
-    unsigned size;
-    bool is_store;
-  } handlers[] = {
-      {(uptr)&__hwasan_load1, 1, false},   {(uptr)&__hwasan_load2, 2, false},
-      {(uptr)&__hwasan_load4, 4, false},   {(uptr)&__hwasan_load8, 8, false},
-      {(uptr)&__hwasan_load16, 16, false},  {(uptr)&__hwasan_load, 0, false},
-      {(uptr)&__hwasan_store1, 1, true},  {(uptr)&__hwasan_store2, 2, true},
-      {(uptr)&__hwasan_store4, 4, true},  {(uptr)&__hwasan_store8, 8, true},
-      {(uptr)&__hwasan_store16, 16, true}, {(uptr)&__hwasan_store, 0, true}};
-  int best = -1;
-  uptr best_distance = 0;
-  for (size_t i = 0; i < sizeof(handlers) / sizeof(handlers[0]); ++i) {
-    uptr handler = handlers[i].addr;
-    // Don't accept pc == handler: HLT is never the first instruction.
-    if (pc <= handler) continue;
-    uptr distance = pc - handler;
-    if (distance > 256) continue;
-    if (best == -1 || best_distance > distance) {
-      best = i;
-      best_distance = distance;
-    }
-  }
-
-  // Not ours.
-  if (best == -1)
-    return AccessInfo{0, 0, false, false};
-
-  ai.is_store = handlers[best].is_store;
-  ai.is_load = !handlers[best].is_store;
-  ai.size = handlers[best].size;
-
+  ai.is_store = is_store;
+  ai.is_load = !is_store;
   ai.addr = uc->uc_mcontext.regs[0];
-  if (ai.size == 0)
+  if (size_log == 0xf)
     ai.size = uc->uc_mcontext.regs[1];
+  else
+    ai.size = 1U << size_log;
   return ai;
 }
 #else
 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
   return AccessInfo{0, 0, false, false};
 }
 #endif
 
-static void HwasanOnSIGILL(int signo, siginfo_t *info, ucontext_t *uc) {
+static bool HwasanOnSIGILL(int signo, siginfo_t *info, ucontext_t *uc) {
   SignalContext sig{info, uc};
   AccessInfo ai = GetAccessInfo(info, uc);
   if (!ai.is_store && !ai.is_load)
-    return;
+    return false;
 
   InternalScopedBuffer<BufferedStackTrace> stack_buffer(1);
   BufferedStackTrace *stack = stack_buffer.data();
   stack->Reset();
   GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, uc,
                 common_flags()->fast_unwind_on_fatal);
 
   ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store);
 
   ++hwasan_report_count;
   if (flags()->halt_on_error)
     Die();
-  else
-    uc->uc_mcontext.pc += 4;
+
+  return true;
 }
 
 static void OnStackUnwind(const SignalContext &sig, const void *,
                           BufferedStackTrace *stack) {
   GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context,
                 common_flags()->fast_unwind_on_fatal);
 }
 
 void HwasanOnDeadlySignal(int signo, void *info, void *context) {
   // Probably a tag mismatch.
-  // FIXME: detect pc range in __hwasan_load* or __hwasan_store*.
   if (signo == SIGILL)
-    HwasanOnSIGILL(signo, (siginfo_t *)info, (ucontext_t*)context);
-  else
-    HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
+    if (HwasanOnSIGILL(signo, (siginfo_t *)info, (ucontext_t*)context))
+      return;
+
 }
 
 
 } // namespace __hwasan
 
 #endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD

compiler-rt/test/hwasan/TestCases/halt-on-error.cc

 // RUN: %clangxx_hwasan -O0 %s -o %t && not %env_hwasan_opts=halt_on_error=0 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-%os --check-prefix=CHECK
 // REQUIRES: stable-runtime
 
 #include <stdlib.h>
 #include <sanitizer/hwasan_interface.h>
 
 int main() {
   __hwasan_enable_allocator_tagging();
   int* volatile x = (int*)malloc(16);
   free(x);
   __hwasan_disable_allocator_tagging();
   return x[2] + ((char *)x)[6] + ((char *)x)[9];
   // CHECK: READ of size 4 at
-  // CHECK: #0 {{.*}} in __hwasan_load4 {{.*}}hwasan.cc
-  // CHECK: #1 {{.*}} in main {{.*}}halt-on-error.cc:12
-  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in __hwasan_load4
+  // CHECK: #0 {{.*}} in main {{.*}}halt-on-error.cc:12
+  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in main
 
   // CHECK: READ of size 1 at
-  // CHECK: #0 {{.*}} in __hwasan_load1 {{.*}}hwasan.cc
-  // CHECK: #1 {{.*}} in main {{.*}}halt-on-error.cc:12
-  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in __hwasan_load1
+  // CHECK: #0 {{.*}} in main {{.*}}halt-on-error.cc:12
+  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in main
 
   // CHECK: READ of size 1 at
-  // CHECK: #0 {{.*}} in __hwasan_load1 {{.*}}hwasan.cc
-  // CHECK: #1 {{.*}} in main {{.*}}halt-on-error.cc:12
-  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in __hwasan_load1
+  // CHECK: #0 {{.*}} in main {{.*}}halt-on-error.cc:12
+  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in main
 
   // CHECK-NOT: tag-mismatch
 }

compiler-rt/test/hwasan/TestCases/use-after-free.cc

 // RUN: %clangxx_hwasan -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-%os --check-prefix=CHECK
 // RUN: %clangxx_hwasan -O1 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-%os --check-prefix=CHECK
 // RUN: %clangxx_hwasan -O2 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-%os --check-prefix=CHECK
 // RUN: %clangxx_hwasan -O3 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-%os --check-prefix=CHECK
 // REQUIRES: stable-runtime
 
 #include <stdlib.h>
 #include <sanitizer/hwasan_interface.h>
 
 int main() {
   __hwasan_enable_allocator_tagging();
   char *x = (char*)malloc(10);
   free(x);
   __hwasan_disable_allocator_tagging();
   return x[5];
   // CHECK: READ of size 1 at
-  // CHECK: #0 {{.*}} in __hwasan_load1 {{.*}}hwasan.cc
-  // CHECK: #1 {{.*}} in main {{.*}}use-after-free.cc:15
+  // CHECK: #0 {{.*}} in main {{.*}}use-after-free.cc:15
 
   // CHECK: freed here:
   // CHECK: #0 {{.*}} in free {{.*}}hwasan_interceptors.cc
   // CHECK: #1 {{.*}} in main {{.*}}use-after-free.cc:13
 
   // CHECK: previously allocated here:
   // CHECK: #0 {{.*}} in __interceptor_malloc {{.*}}hwasan_interceptors.cc
   // CHECK: #1 {{.*}} in main {{.*}}use-after-free.cc:12
 
-  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in __hwasan_load1
+  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in main
 }

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/IR/Function.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/InstVisitor.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/LLVMContext.h"

 #define DEBUG_TYPE "hwasan"
 
 static const char *const kHwasanModuleCtorName = "hwasan.module_ctor";
 static const char *const kHwasanInitName = "__hwasan_init";
 
 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
 static const size_t kNumberOfAccessSizes = 5;
 
+static const size_t kShadowScale = 4;
+static const unsigned kPointerTagShift = 56;
+
 static cl::opt<std::string> ClMemoryAccessCallbackPrefix(
     "hwasan-memory-access-callback-prefix",
     cl::desc("Prefix for memory access callbacks"), cl::Hidden,
     cl::init("__hwasan_"));
 
+static cl::opt<bool>
+    ClInstrumentWithCalls("hwasan-instrument-with-calls",
+                cl::desc("instrument reads and writes with callbacks"),
+                cl::Hidden, cl::init(false));
+
 static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
                                        cl::desc("instrument read instructions"),
                                        cl::Hidden, cl::init(true));
 
 static cl::opt<bool> ClInstrumentWrites(
     "hwasan-instrument-writes", cl::desc("instrument write instructions"),
     cl::Hidden, cl::init(true));
 
public:
   HWAddressSanitizer() : FunctionPass(ID) {}
 
   StringRef getPassName() const override { return "HWAddressSanitizer"; }
 
   bool runOnFunction(Function &F) override;
   bool doInitialization(Module &M) override;
 
   void initializeCallbacks(Module &M);
+  void instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
+                                 unsigned AccessSizeIndex,
+                                 Instruction *InsertBefore);
   bool instrumentMemAccess(Instruction *I);
   Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite,
                                    uint64_t *TypeSize, unsigned *Alignment,
                                    Value **MaybeMask);
 
 private:
   LLVMContext *C;
   Type *IntptrTy;

 }
 
 static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
   size_t Res = countTrailingZeros(TypeSize / 8);
   assert(Res < kNumberOfAccessSizes);
   return Res;
 }
 
+void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
+                                                   unsigned AccessSizeIndex,
+                                                   Instruction *InsertBefore) {
+  IRBuilder<> IRB(InsertBefore);
+  Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift), IRB.getInt8Ty());
+  Value *AddrLong =
+      IRB.CreateAnd(PtrLong, ConstantInt::get(PtrLong->getType(),
+                                              ~(0xFFULL << kPointerTagShift)));
+  Value *ShadowLong = IRB.CreateLShr(AddrLong, kShadowScale);
+  Value *MemTag = IRB.CreateLoad(IRB.CreateIntToPtr(ShadowLong, IRB.getInt8PtrTy()));
+  Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
+
+  TerminatorInst *CheckTerm =
+      SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, false,
+                                MDBuilder(*C).createBranchWeights(1, 100000));
+
+  IRB.SetInsertPoint(CheckTerm);
+  // The signal handler will find the data address in x0.
+  InlineAsm *Asm = InlineAsm::get(
+      FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
+      "hlt #" + itostr(0x100 + IsWrite * 0x10 + AccessSizeIndex), "{x0}",
+      /*hasSideEffects=*/true);
+  IRB.CreateCall(Asm, PtrLong);
+}
+
 bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
   DEBUG(dbgs() << "Instrumenting: " << *I << "\n");
   bool IsWrite = false;
   unsigned Alignment = 0;
   uint64_t TypeSize = 0;
   Value *MaybeMask = nullptr;
   Value *Addr =
       isInterestingMemoryAccess(I, &IsWrite, &TypeSize, &Alignment, &MaybeMask);
 
   if (!Addr)
     return false;
 
   if (MaybeMask)
     return false; //FIXME
 
   IRBuilder<> IRB(I);
   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
   if (isPowerOf2_64(TypeSize) &&
-      (TypeSize / 8 <= (1UL << (kNumberOfAccessSizes - 1)))) {
+      (TypeSize / 8 <= (1UL << (kNumberOfAccessSizes - 1))) &&
+      (Alignment >= (1UL << kShadowScale) || Alignment == 0 ||
+       Alignment >= TypeSize / 8)) {
     size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize);
+    if (ClInstrumentWithCalls) {
       IRB.CreateCall(HwasanMemoryAccessCallback[IsWrite][AccessSizeIndex],
                      AddrLong);
     } else {
+      instrumentMemAccessInline(AddrLong, IsWrite, AccessSizeIndex, I);
+    }
+  } else {
     IRB.CreateCall(HwasanMemoryAccessCallbackSized[IsWrite],
                    {AddrLong, ConstantInt::get(IntptrTy, TypeSize / 8)});
   }
 
   return true;
 }
 
 bool HWAddressSanitizer::runOnFunction(Function &F) {

llvm/test/Instrumentation/HWAddressSanitizer/basic.ll

 ; Test basic address sanitizer instrumentation.
 ;
 ; RUN: opt < %s -hwasan -S | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define i8 @test_load8(i8* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; CHECK: call void @__hwasan_load1(i64 %[[A]])
-; CHECK: %[[B:[^ ]*]] = load i8, i8* %a
-; CHECK: ret i8 %[[B]]
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #256", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
+; CHECK: ret i8 %[[G]]
 
 entry:
   %b = load i8, i8* %a, align 4
   ret i8 %b
 }
 
 define i16 @test_load16(i16* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
-; CHECK: call void @__hwasan_load2(i64 %[[A]])
-; CHECK: %[[B:[^ ]*]] = load i16, i16* %a
-; CHECK: ret i16 %[[B]]
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #257", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: %[[G:[^ ]*]] = load i16, i16* %a, align 4
+; CHECK: ret i16 %[[G]]
 
 entry:
   %b = load i16, i16* %a, align 4
   ret i16 %b
 }
 
 define i32 @test_load32(i32* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
-; CHECK: call void @__hwasan_load4(i64 %[[A]])
-; CHECK: %[[B:[^ ]*]] = load i32, i32* %a
-; CHECK: ret i32 %[[B]]
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #258", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: %[[G:[^ ]*]] = load i32, i32* %a, align 4
+; CHECK: ret i32 %[[G]]
 
 entry:
   %b = load i32, i32* %a, align 4
   ret i32 %b
 }
 
 define i64 @test_load64(i64* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
-; CHECK: call void @__hwasan_load8(i64 %[[A]])
-; CHECK: %[[B:[^ ]*]] = load i64, i64* %a
-; CHECK: ret i64 %[[B]]
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #259", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: %[[G:[^ ]*]] = load i64, i64* %a, align 8
+; CHECK: ret i64 %[[G]]
 
 entry:
   %b = load i64, i64* %a, align 8
   ret i64 %b
 }
 
 define i128 @test_load128(i128* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
-; CHECK: call void @__hwasan_load16(i64 %[[A]])
-; CHECK: %[[B:[^ ]*]] = load i128, i128* %a
-; CHECK: ret i128 %[[B]]
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #260", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: %[[G:[^ ]*]] = load i128, i128* %a, align 16
+; CHECK: ret i128 %[[G]]
 
 entry:
   %b = load i128, i128* %a, align 16
   ret i128 %b
 }
 
 define i40 @test_load40(i40* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load40(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i40* %a to i64
 ; CHECK: call void @__hwasan_load(i64 %[[A]], i64 5)
 ; CHECK: %[[B:[^ ]*]] = load i40, i40* %a
 ; CHECK: ret i40 %[[B]]
 
 entry:
   %b = load i40, i40* %a, align 4
   ret i40 %b
 }
 
 define void @test_store8(i8* %a, i8 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; CHECK: call void @__hwasan_store1(i64 %[[A]])
-; CHECK: store i8 %b, i8* %a
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #272", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: store i8 %b, i8* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i8 %b, i8* %a, align 4
   ret void
 }
 
 define void @test_store16(i16* %a, i16 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
-; CHECK: call void @__hwasan_store2(i64 %[[A]])
-; CHECK: store i16 %b, i16* %a
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #273", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: store i16 %b, i16* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i16 %b, i16* %a, align 4
   ret void
 }
 
 define void @test_store32(i32* %a, i32 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
-; CHECK: call void @__hwasan_store4(i64 %[[A]])
-; CHECK: store i32 %b, i32* %a
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #274", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: store i32 %b, i32* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i32 %b, i32* %a, align 4
   ret void
 }
 
 define void @test_store64(i64* %a, i64 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
-; CHECK: call void @__hwasan_store8(i64 %[[A]])
-; CHECK: store i64 %b, i64* %a
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #275", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: store i64 %b, i64* %a, align 8
 ; CHECK: ret void
 
 entry:
-  store i64 %b, i64* %a, align 4
+  store i64 %b, i64* %a, align 8
   ret void
 }
 
 define void @test_store128(i128* %a, i128 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
-; CHECK: call void @__hwasan_store16(i64 %[[A]])
-; CHECK: store i128 %b, i128* %a
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; CHECK: call void asm sideeffect "hlt #276", "{x0}"(i64 %[[A]])
+; CHECK: br label
+
+; CHECK: store i128 %b, i128* %a, align 16
 ; CHECK: ret void
 
 entry:
-  store i128 %b, i128* %a, align 4
+  store i128 %b, i128* %a, align 16
   ret void
 }
 
 define void @test_store40(i40* %a, i40 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store40(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i40* %a to i64
 ; CHECK: call void @__hwasan_store(i64 %[[A]], i64 5)
 ; CHECK: store i40 %b, i40* %a
 ; CHECK: ret void
 
 entry:
   store i40 %b, i40* %a, align 4
   ret void
 }
 
+define void @test_store_unaligned(i64* %a, i64 %b) sanitize_hwaddress {
+; CHECK-LABEL: @test_store_unaligned(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
+; CHECK: call void @__hwasan_store(i64 %[[A]], i64 8)
+; CHECK: store i64 %b, i64* %a, align 4
+; CHECK: ret void
+
+entry:
+  store i64 %b, i64* %a, align 4
+  ret void
+}
+
 define i8 @test_load_noattr(i8* %a) {
 ; CHECK-LABEL: @test_load_noattr(
 ; CHECK-NEXT: entry:
 ; CHECK-NEXT: %[[B:[^ ]*]] = load i8, i8* %a
 ; CHECK-NEXT: ret i8 %[[B]]
 
 entry:
   %b = load i8, i8* %a, align 4

llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll

 ; Test basic address sanitizer instrumentation.
 ;
-; RUN: opt < %s -hwasan -S | FileCheck %s
+; RUN: opt < %s -hwasan -hwasan-instrument-with-calls -S | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define i8 @test_load8(i8* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
 ; CHECK: call void @__hwasan_load1(i64 %[[A]])