[hwasan] Implement -fsanitize-recover=hwaddress.

[eugenis]

Summary

Very similar to AddressSanitizer, with the exception of the error type encoding.

[kcc]

LGTM with a nit

compiler-rt/lib/hwasan/hwasan.cc

254–255

I'd prefer enums to booleans, for better readability

[eugenis]

compiler-rt/lib/hwasan/hwasan.cc

254–255

good idea!

[eugenis]

Replaced booleans with enums.
Added __builtin_unreachable in non-recover variants of callbacks.

[eugenis]

Tweaked a test.

clang/lib/CodeGen/BackendUtil.cpp

PM.add(createAddressSanitizerFunctionPass(
       /*CompileKernel*/ true,
       /*Recover*/ true, /*UseAfterScope*/ false));
   PM.add(createAddressSanitizerModulePass(/*CompileKernel*/true,
                                           /*Recover*/true));
 }
 
 static void addHWAddressSanitizerPasses(const PassManagerBuilder &Builder,
                                             legacy::PassManagerBase &PM) {
-  PM.add(createHWAddressSanitizerPass());
+  const PassManagerBuilderWrapper &BuilderWrapper =
+      static_cast<const PassManagerBuilderWrapper &>(Builder);
+  const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts();
+  bool Recover = CGOpts.SanitizeRecover.has(SanitizerKind::HWAddress);
+  PM.add(createHWAddressSanitizerPass(Recover));
 }
 
 static void addMemorySanitizerPass(const PassManagerBuilder &Builder,
                                    legacy::PassManagerBase &PM) {
   const PassManagerBuilderWrapper &BuilderWrapper =
       static_cast<const PassManagerBuilderWrapper&>(Builder);
   const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts();
   int TrackOrigins = CGOpts.SanitizeMemoryTrackOrigins;

compiler-rt/lib/hwasan/hwasan.cc

 #elif defined(__x86_64__) || defined(__i386__)
   asm("ud2\n\t");
 #else
   // FIXME: not always sigill.
   __builtin_trap();
 #endif
   // __builtin_unreachable();
 }
 
-template<bool IsStore, unsigned LogSize>
-__attribute__((always_inline, nodebug))
-static void CheckAddress(uptr p) {
+enum class ErrorAction { Abort, Recover };
+enum class AccessType { Load, Store };
+
+template <ErrorAction EA, AccessType AT, unsigned LogSize>
+__attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) {
   tag_t ptr_tag = GetTagFromPointer(p);
   uptr ptr_raw = p & ~kAddressTagMask;
   tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(ptr_raw);
-  if (UNLIKELY(ptr_tag != mem_tag)) SigIll<0x100 + 0x10 * IsStore + LogSize>();
+  if (UNLIKELY(ptr_tag != mem_tag)) {
+    SigIll<0x100 + 0x20 * (EA == ErrorAction::Recover) +
+           0x10 * (AT == AccessType::Store) + LogSize>();
+    if (EA == ErrorAction::Abort) __builtin_unreachable();
+  }
 }
 
-template<bool IsStore>
-__attribute__((always_inline, nodebug))
-static void CheckAddressSized(uptr p, uptr sz) {
+template <ErrorAction EA, AccessType AT>
+__attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p,
+                                                                      uptr sz) {
   CHECK_NE(0, sz);
   tag_t ptr_tag = GetTagFromPointer(p);
   uptr ptr_raw = p & ~kAddressTagMask;
   tag_t *shadow_first = (tag_t *)MEM_TO_SHADOW(ptr_raw);
   tag_t *shadow_last = (tag_t *)MEM_TO_SHADOW(ptr_raw + sz - 1);
   for (tag_t *t = shadow_first; t <= shadow_last; ++t)
-    if (UNLIKELY(ptr_tag != *t)) SigIll<0x100 + 0x10 * IsStore + 0xf>();
+    if (UNLIKELY(ptr_tag != *t)) {
+      SigIll<0x100 + 0x20 * (EA == ErrorAction::Recover) +
+             0x10 * (AT == AccessType::Store) + 0xf>();
+      if (EA == ErrorAction::Abort) __builtin_unreachable();
+    }
+}
+
+void __hwasan_load(uptr p, uptr sz) {
+  CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz);
+}
+void __hwasan_load1(uptr p) {
+  CheckAddress<ErrorAction::Abort, AccessType::Load, 0>(p);
+}
+void __hwasan_load2(uptr p) {
+  CheckAddress<ErrorAction::Abort, AccessType::Load, 1>(p);
+}
+void __hwasan_load4(uptr p) {
+  CheckAddress<ErrorAction::Abort, AccessType::Load, 2>(p);
+}
+void __hwasan_load8(uptr p) {
+  CheckAddress<ErrorAction::Abort, AccessType::Load, 3>(p);
+}
+void __hwasan_load16(uptr p) {
+  CheckAddress<ErrorAction::Abort, AccessType::Load, 4>(p);
+}
+
+void __hwasan_load_noabort(uptr p, uptr sz) {
+  CheckAddressSized<ErrorAction::Recover, AccessType::Load>(p, sz);
+}
+void __hwasan_load1_noabort(uptr p) {
+  CheckAddress<ErrorAction::Recover, AccessType::Load, 0>(p);
+}
+void __hwasan_load2_noabort(uptr p) {
+  CheckAddress<ErrorAction::Recover, AccessType::Load, 1>(p);
+}
+void __hwasan_load4_noabort(uptr p) {
+  CheckAddress<ErrorAction::Recover, AccessType::Load, 2>(p);
+}
+void __hwasan_load8_noabort(uptr p) {
+  CheckAddress<ErrorAction::Recover, AccessType::Load, 3>(p);
+}
+void __hwasan_load16_noabort(uptr p) {
+  CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p);
 }
 
-void __hwasan_load(uptr p, uptr sz) { CheckAddressSized<false>(p, sz); }
-void __hwasan_load1(uptr p) { CheckAddress<false, 0>(p); }
-void __hwasan_load2(uptr p) { CheckAddress<false, 1>(p); }
-void __hwasan_load4(uptr p) { CheckAddress<false, 2>(p); }
-void __hwasan_load8(uptr p) { CheckAddress<false, 3>(p); }
-void __hwasan_load16(uptr p) { CheckAddress<false, 4>(p); }
-
-void __hwasan_store(uptr p, uptr sz) { CheckAddressSized<true>(p, sz); }
-void __hwasan_store1(uptr p) { CheckAddress<true, 0>(p); }
-void __hwasan_store2(uptr p) { CheckAddress<true, 1>(p); }
-void __hwasan_store4(uptr p) { CheckAddress<true, 2>(p); }
-void __hwasan_store8(uptr p) { CheckAddress<true, 3>(p); }
-void __hwasan_store16(uptr p) { CheckAddress<true, 4>(p); }
+void __hwasan_store(uptr p, uptr sz) {
+  CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz);
+}
+void __hwasan_store1(uptr p) {
+  CheckAddress<ErrorAction::Abort, AccessType::Store, 0>(p);
+}
+void __hwasan_store2(uptr p) {
+  CheckAddress<ErrorAction::Abort, AccessType::Store, 1>(p);
+}
+void __hwasan_store4(uptr p) {
+  CheckAddress<ErrorAction::Abort, AccessType::Store, 2>(p);
+}
+void __hwasan_store8(uptr p) {
+  CheckAddress<ErrorAction::Abort, AccessType::Store, 3>(p);
+}
+void __hwasan_store16(uptr p) {
+  CheckAddress<ErrorAction::Abort, AccessType::Store, 4>(p);
+}
+
+void __hwasan_store_noabort(uptr p, uptr sz) {
+  CheckAddressSized<ErrorAction::Recover, AccessType::Store>(p, sz);
+}
+void __hwasan_store1_noabort(uptr p) {
+  CheckAddress<ErrorAction::Recover, AccessType::Store, 0>(p);
+}
+void __hwasan_store2_noabort(uptr p) {
+  CheckAddress<ErrorAction::Recover, AccessType::Store, 1>(p);
+}
+void __hwasan_store4_noabort(uptr p) {
+  CheckAddress<ErrorAction::Recover, AccessType::Store, 2>(p);
+}
+void __hwasan_store8_noabort(uptr p) {
+  CheckAddress<ErrorAction::Recover, AccessType::Store, 3>(p);
+}
+void __hwasan_store16_noabort(uptr p) {
+  CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p);
+}
 
 #if !SANITIZER_SUPPORTS_WEAK_HOOKS
 extern "C" {
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
 const char* __hwasan_default_options() { return ""; }
 }  // extern "C"
 #endif
 
 extern "C" {
 SANITIZER_INTERFACE_ATTRIBUTE
 void __sanitizer_print_stack_trace() {
   GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
   stack.Print();
 }
 } // extern "C"

compiler-rt/lib/hwasan/hwasan_interface_internal.h

 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_load4(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_load8(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_load16(uptr);
 
 SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load_noabort(uptr, uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load1_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load2_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load4_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load8_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load16_noabort(uptr);
+
+SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_store(uptr, uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_store1(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_store2(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_store4(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_store8(uptr);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_store16(uptr);
 
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store_noabort(uptr, uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store1_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store2_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store4_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store8_noabort(uptr);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store16_noabort(uptr);
+
 // Returns the offset of the first tag mismatch or -1 if the whole range is
 // good.
 SANITIZER_INTERFACE_ATTRIBUTE
 sptr __hwasan_test_shadow(const void *x, uptr size);
 
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
 /* OPTIONAL */ const char* __hwasan_default_options();
 

compiler-rt/lib/hwasan/hwasan_linux.cc

void HwasanTSDDtor(void *tsd) {
   HwasanThread::TSDDtor(tsd);
 }
 
 struct AccessInfo {
   uptr addr;
   uptr size;
   bool is_store;
   bool is_load;
+  bool recover;
 };
 
 #if defined(__aarch64__)
 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
   // Access type is encoded in HLT immediate as 0x1XY,
-  // where X is 1 for store, 0 for load.
+  // where X&1 is 1 for store, 0 for load,
+  // and X&2 is 1 if the error is recoverable.
   // Valid values of Y are 0 to 4, which are interpreted as log2(access_size),
   // and 0xF, which means that access size is stored in X1 register.
   // Access address is always in X0 register.
   AccessInfo ai;
   uptr pc = (uptr)info->si_addr;
   unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
   if ((code & 0xff00) != 0x100)
     return AccessInfo{0, 0, false, false}; // Not ours.
   bool is_store = code & 0x10;
+  bool recover = code & 0x20;
   unsigned size_log = code & 0xf;
   if (size_log > 4 && size_log != 0xf)
     return AccessInfo{0, 0, false, false}; // Not ours.
 
   ai.is_store = is_store;
   ai.is_load = !is_store;
   ai.addr = uc->uc_mcontext.regs[0];
   if (size_log == 0xf)
     ai.size = uc->uc_mcontext.regs[1];
   else
     ai.size = 1U << size_log;
+  ai.recover = recover;
   return ai;
 }
 #else
 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
   return AccessInfo{0, 0, false, false};
 }
 #endif
 
 static bool HwasanOnSIGILL(int signo, siginfo_t *info, ucontext_t *uc) {
   SignalContext sig{info, uc};
   AccessInfo ai = GetAccessInfo(info, uc);
   if (!ai.is_store && !ai.is_load)
     return false;
 
   InternalScopedBuffer<BufferedStackTrace> stack_buffer(1);
   BufferedStackTrace *stack = stack_buffer.data();
   stack->Reset();
   GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, uc,
                 common_flags()->fast_unwind_on_fatal);
 
   ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store);
 
   ++hwasan_report_count;
-  if (flags()->halt_on_error)
+  if (flags()->halt_on_error || !ai.recover)
     Die();
 
   uc->uc_mcontext.pc += 4;
   return true;
 }
 
 static void OnStackUnwind(const SignalContext &sig, const void *,
                           BufferedStackTrace *stack) {

compiler-rt/test/hwasan/TestCases/halt-on-error.cc

-// RUN: %clangxx_hwasan -O0 %s -o %t && not %env_hwasan_opts=halt_on_error=0 %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_hwasan -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefixes=COMMON
+// RUN: %clangxx_hwasan -O0 %s -o %t && not %env_hwasan_opts=halt_on_error=1 %run %t 2>&1 | FileCheck %s --check-prefix=COMMON
+// RUN: %clangxx_hwasan -O0 %s -o %t && not %env_hwasan_opts=halt_on_error=0 %run %t 2>&1 | FileCheck %s --check-prefix=COMMON
+
+// RUN: %clangxx_hwasan -O0 %s -o %t -fsanitize-recover=hwaddress && not %run %t 2>&1 | FileCheck %s --check-prefix=COMMON
+// RUN: %clangxx_hwasan -O0 %s -o %t -fsanitize-recover=hwaddress && not %env_hwasan_opts=halt_on_error=1 %run %t 2>&1 | FileCheck %s --check-prefix=COMMON
+// RUN: %clangxx_hwasan -O0 %s -o %t -fsanitize-recover=hwaddress && not %env_hwasan_opts=halt_on_error=0 %run %t 2>&1 | FileCheck %s --check-prefixes=COMMON,RECOVER
+
+// RUN: %clangxx_hwasan -mllvm -hwasan-instrument-with-calls=1 -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=COMMON
+// RUN: %clangxx_hwasan -mllvm -hwasan-instrument-with-calls=1 -O0 %s -o %t && not %env_hwasan_opts=halt_on_error=1 %run %t 2>&1 | FileCheck %s --check-prefix=COMMON
+// RUN: %clangxx_hwasan -mllvm -hwasan-instrument-with-calls=1 -O0 %s -o %t && not %env_hwasan_opts=halt_on_error=0 %run %t 2>&1 | FileCheck %s --check-prefix=COMMON
+
+// RUN: %clangxx_hwasan -mllvm -hwasan-instrument-with-calls=1 -O0 %s -o %t -fsanitize-recover=hwaddress && not %run %t 2>&1 | FileCheck %s --check-prefix=COMMON
+// RUN: %clangxx_hwasan -mllvm -hwasan-instrument-with-calls=1 -O0 %s -o %t -fsanitize-recover=hwaddress && not %env_hwasan_opts=halt_on_error=1 %run %t 2>&1 | FileCheck %s --check-prefix=COMMON
+// RUN: %clangxx_hwasan -mllvm -hwasan-instrument-with-calls=1 -O0 %s -o %t -fsanitize-recover=hwaddress && not %env_hwasan_opts=halt_on_error=0 %run %t 2>&1 | FileCheck %s --check-prefixes=COMMON,RECOVER
+
 // REQUIRES: stable-runtime
 
 #include <stdlib.h>
 #include <sanitizer/hwasan_interface.h>
 
 int main() {
   __hwasan_enable_allocator_tagging();
   int* volatile x = (int*)malloc(16);
   free(x);
   __hwasan_disable_allocator_tagging();
   return x[2] + ((char *)x)[6] + ((char *)x)[9];
-  // CHECK: READ of size 4 at
-  // CHECK: #0 {{.*}} in main {{.*}}halt-on-error.cc:12
-  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in main
-
-  // CHECK: READ of size 1 at
-  // CHECK: #0 {{.*}} in main {{.*}}halt-on-error.cc:12
-  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in main
-
-  // CHECK: READ of size 1 at
-  // CHECK: #0 {{.*}} in main {{.*}}halt-on-error.cc:12
-  // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in main
+  // COMMON: READ of size 4 at
+  // When instrumenting with callbacks, main is actually #1, and #0 is __hwasan_load4.
+  // COMMON: #{{.*}} in main {{.*}}halt-on-error.cc:27
+  // COMMON: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in
+
+  // RECOVER: READ of size 1 at
+  // RECOVER: #{{.*}} in main {{.*}}halt-on-error.cc:27
+  // RECOVER: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in
+
+  // RECOVER: READ of size 1 at
+  // RECOVER: #{{.*}} in main {{.*}}halt-on-error.cc:27
+  // RECOVER: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in
 
-  // CHECK-NOT: tag-mismatch
+  // COMMON-NOT: tag-mismatch
 }

llvm/include/llvm/Transforms/Instrumentation.h

 ModulePass *createAddressSanitizerModulePass(bool CompileKernel = false,
                                              bool Recover = false,
                                              bool UseGlobalsGC = true);
 
 // Insert MemorySanitizer instrumentation (detection of uninitialized reads)
 FunctionPass *createMemorySanitizerPass(int TrackOrigins = 0,
                                         bool Recover = false);
 
-FunctionPass *createHWAddressSanitizerPass();
+FunctionPass *createHWAddressSanitizerPass(bool Recover = false);
 
 // Insert ThreadSanitizer (race detection) instrumentation
 FunctionPass *createThreadSanitizerPass();
 
 // Insert DataFlowSanitizer (dynamic data flow analysis) instrumentation
 ModulePass *createDataFlowSanitizerPass(
     const std::vector<std::string> &ABIListFiles = std::vector<std::string>(),
     void *(*getArgTLS)() = nullptr, void *(*getRetValTLS)() = nullptr);

llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

static cl::opt<bool> ClInstrumentWrites(
     "hwasan-instrument-writes", cl::desc("instrument write instructions"),
     cl::Hidden, cl::init(true));
 
 static cl::opt<bool> ClInstrumentAtomics(
     "hwasan-instrument-atomics",
     cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
     cl::init(true));
 
+static cl::opt<bool> ClRecover(
+    "hwasan-recover",
+    cl::desc("Enable recovery mode (continue-after-error)."),
+    cl::Hidden, cl::init(false));
+
 namespace {
 
 /// \brief An instrumentation pass implementing detection of addressability bugs
 /// using tagged pointers.
 class HWAddressSanitizer : public FunctionPass {
 public:
   // Pass identification, replacement for typeid.
   static char ID;
 
-  HWAddressSanitizer() : FunctionPass(ID) {}
+  HWAddressSanitizer(bool Recover = false)
+      : FunctionPass(ID), Recover(Recover || ClRecover) {}
 
   StringRef getPassName() const override { return "HWAddressSanitizer"; }
 
   bool runOnFunction(Function &F) override;
   bool doInitialization(Module &M) override;
 
   void initializeCallbacks(Module &M);
   void instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
                                  unsigned AccessSizeIndex,
                                  Instruction *InsertBefore);
   bool instrumentMemAccess(Instruction *I);
   Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite,
                                    uint64_t *TypeSize, unsigned *Alignment,
                                    Value **MaybeMask);
 
 private:
   LLVMContext *C;
   Type *IntptrTy;
 
+  bool Recover;
+
   Function *HwasanCtorFunction;
 
   Function *HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
   Function *HwasanMemoryAccessCallbackSized[2];
 };
 
 } // end anonymous namespace
 
 char HWAddressSanitizer::ID = 0;
 
 INITIALIZE_PASS_BEGIN(
     HWAddressSanitizer, "hwasan",
     "HWAddressSanitizer: detect memory bugs using tagged addressing.", false, false)
 INITIALIZE_PASS_END(
     HWAddressSanitizer, "hwasan",
     "HWAddressSanitizer: detect memory bugs using tagged addressing.", false, false)
 
-FunctionPass *llvm::createHWAddressSanitizerPass() {
-  return new HWAddressSanitizer();
+FunctionPass *llvm::createHWAddressSanitizerPass(bool Recover) {
+  return new HWAddressSanitizer(Recover);
 }
 
 /// \brief Module-level initialization.
 ///
 /// inserts a call to __hwasan_init to the module's constructor list.
 bool HWAddressSanitizer::doInitialization(Module &M) {
   DEBUG(dbgs() << "Init " << M.getName() << "\n");
   auto &DL = M.getDataLayout();
bool HWAddressSanitizer::doInitialization(Module &M) {
   appendToGlobalCtors(M, HwasanCtorFunction, 0);
   return true;
 }
 
 void HWAddressSanitizer::initializeCallbacks(Module &M) {
   IRBuilder<> IRB(*C);
   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
     const std::string TypeStr = AccessIsWrite ? "store" : "load";
+    const std::string EndingStr = Recover ? "_noabort" : "";
 
     HwasanMemoryAccessCallbackSized[AccessIsWrite] =
         checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-            ClMemoryAccessCallbackPrefix + TypeStr,
+            ClMemoryAccessCallbackPrefix + TypeStr + EndingStr,
             FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false)));
 
     for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
          AccessSizeIndex++) {
       HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
           checkSanitizerInterfaceFunction(M.getOrInsertFunction(
               ClMemoryAccessCallbackPrefix + TypeStr +
-                  itostr(1ULL << AccessSizeIndex),
+                  itostr(1ULL << AccessSizeIndex) + EndingStr,
               FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false)));
     }
   }
 }
 
 Value *HWAddressSanitizer::isInterestingMemoryAccess(Instruction *I,
                                                    bool *IsWrite,
                                                    uint64_t *TypeSize,
void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
   Value *AddrLong =
       IRB.CreateAnd(PtrLong, ConstantInt::get(PtrLong->getType(),
                                               ~(0xFFULL << kPointerTagShift)));
   Value *ShadowLong = IRB.CreateLShr(AddrLong, kShadowScale);
   Value *MemTag = IRB.CreateLoad(IRB.CreateIntToPtr(ShadowLong, IRB.getInt8PtrTy()));
   Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
 
   TerminatorInst *CheckTerm =
-      SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, false,
+      SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, !Recover,
                                 MDBuilder(*C).createBranchWeights(1, 100000));
 
   IRB.SetInsertPoint(CheckTerm);
   // The signal handler will find the data address in x0.
   InlineAsm *Asm = InlineAsm::get(
       FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
-      "hlt #" + itostr(0x100 + IsWrite * 0x10 + AccessSizeIndex), "{x0}",
+      "hlt #" +
+          itostr(0x100 + Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex),
+      "{x0}",
       /*hasSideEffects=*/true);
   IRB.CreateCall(Asm, PtrLong);
 }
 
 bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
   DEBUG(dbgs() << "Instrumenting: " << *I << "\n");
   bool IsWrite = false;
   unsigned Alignment = 0;

llvm/test/Instrumentation/HWAddressSanitizer/basic.ll

 ; Test basic address sanitizer instrumentation.
 ;
-; RUN: opt < %s -hwasan -S | FileCheck %s
+; RUN: opt < %s -hwasan -hwasan-recover=0 -S | FileCheck %s  --check-prefixes=CHECK,ABORT
+; RUN: opt < %s -hwasan -hwasan-recover=1 -S | FileCheck %s  --check-prefixes=CHECK,RECOVER
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define i8 @test_load8(i8* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; CHECK: call void asm sideeffect "hlt #256", "{x0}"(i64 %[[A]])
-; CHECK: br label
+; ABORT: call void asm sideeffect "hlt #256", "{x0}"(i64 %[[A]])
+; ABORT: unreachable
+; RECOVER: call void asm sideeffect "hlt #288", "{x0}"(i64 %[[A]])
+; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
 ; CHECK: ret i8 %[[G]]
 
 entry:
   %b = load i8, i8* %a, align 4
   ret i8 %b
 }
 
 define i16 @test_load16(i16* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; CHECK: call void asm sideeffect "hlt #257", "{x0}"(i64 %[[A]])
-; CHECK: br label
+; ABORT: call void asm sideeffect "hlt #257", "{x0}"(i64 %[[A]])
+; ABORT: unreachable
+; RECOVER: call void asm sideeffect "hlt #289", "{x0}"(i64 %[[A]])
+; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i16, i16* %a, align 4
 ; CHECK: ret i16 %[[G]]
 
 entry:
   %b = load i16, i16* %a, align 4
   ret i16 %b
 }
 
 define i32 @test_load32(i32* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; CHECK: call void asm sideeffect "hlt #258", "{x0}"(i64 %[[A]])
-; CHECK: br label
+; ABORT: call void asm sideeffect "hlt #258", "{x0}"(i64 %[[A]])
+; ABORT: unreachable
+; RECOVER: call void asm sideeffect "hlt #290", "{x0}"(i64 %[[A]])
+; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i32, i32* %a, align 4
 ; CHECK: ret i32 %[[G]]
 
 entry:
   %b = load i32, i32* %a, align 4
   ret i32 %b
 }
 
 define i64 @test_load64(i64* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; CHECK: call void asm sideeffect "hlt #259", "{x0}"(i64 %[[A]])
-; CHECK: br label
+; ABORT: call void asm sideeffect "hlt #259", "{x0}"(i64 %[[A]])
+; ABORT: unreachable
+; RECOVER: call void asm sideeffect "hlt #291", "{x0}"(i64 %[[A]])
+; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i64, i64* %a, align 8
 ; CHECK: ret i64 %[[G]]
 
 entry:
   %b = load i64, i64* %a, align 8
   ret i64 %b
 }
 
 define i128 @test_load128(i128* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; CHECK: call void asm sideeffect "hlt #260", "{x0}"(i64 %[[A]])
-; CHECK: br label
+; ABORT: call void asm sideeffect "hlt #260", "{x0}"(i64 %[[A]])
+; ABORT: unreachable
+; RECOVER: call void asm sideeffect "hlt #292", "{x0}"(i64 %[[A]])
+; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i128, i128* %a, align 16
 ; CHECK: ret i128 %[[G]]
 
 entry:
   %b = load i128, i128* %a, align 16
   ret i128 %b
 }
 
 define i40 @test_load40(i40* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load40(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i40* %a to i64
-; CHECK: call void @__hwasan_load(i64 %[[A]], i64 5)
+; ABORT: call void @__hwasan_load(i64 %[[A]], i64 5)
+; RECOVER: call void @__hwasan_load_noabort(i64 %[[A]], i64 5)
 ; CHECK: %[[B:[^ ]*]] = load i40, i40* %a
 ; CHECK: ret i40 %[[B]]
 
 entry:
   %b = load i40, i40* %a, align 4
   ret i40 %b
 }
 
 define void @test_store8(i8* %a, i8 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; CHECK: call void asm sideeffect "hlt #272", "{x0}"(i64 %[[A]])
-; CHECK: br label
+; ABORT: call void asm sideeffect "hlt #272", "{x0}"(i64 %[[A]])
+; ABORT: unreachable
+; RECOVER: call void asm sideeffect "hlt #304", "{x0}"(i64 %[[A]])
+; RECOVER: br label
 
 ; CHECK: store i8 %b, i8* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i8 %b, i8* %a, align 4
   ret void
 }
 
 define void @test_store16(i16* %a, i16 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; CHECK: call void asm sideeffect "hlt #273", "{x0}"(i64 %[[A]])
-; CHECK: br label
+; ABORT: call void asm sideeffect "hlt #273", "{x0}"(i64 %[[A]])
+; ABORT: unreachable
+; RECOVER: call void asm sideeffect "hlt #305", "{x0}"(i64 %[[A]])
+; RECOVER: br label
 
 ; CHECK: store i16 %b, i16* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i16 %b, i16* %a, align 4
   ret void
 }
 
 define void @test_store32(i32* %a, i32 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; CHECK: call void asm sideeffect "hlt #274", "{x0}"(i64 %[[A]])
-; CHECK: br label
+; ABORT: call void asm sideeffect "hlt #274", "{x0}"(i64 %[[A]])
+; ABORT: unreachable
+; RECOVER: call void asm sideeffect "hlt #306", "{x0}"(i64 %[[A]])
+; RECOVER: br label
 
 ; CHECK: store i32 %b, i32* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i32 %b, i32* %a, align 4
   ret void
 }
 
 define void @test_store64(i64* %a, i64 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; CHECK: call void asm sideeffect "hlt #275", "{x0}"(i64 %[[A]])
-; CHECK: br label
+; ABORT: call void asm sideeffect "hlt #275", "{x0}"(i64 %[[A]])
+; ABORT: unreachable
+; RECOVER: call void asm sideeffect "hlt #307", "{x0}"(i64 %[[A]])
+; RECOVER: br label
 
 ; CHECK: store i64 %b, i64* %a, align 8
 ; CHECK: ret void
 
 entry:
   store i64 %b, i64* %a, align 8
   ret void
 }
 
 define void @test_store128(i128* %a, i128 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; CHECK: call void asm sideeffect "hlt #276", "{x0}"(i64 %[[A]])
-; CHECK: br label
+; ABORT: call void asm sideeffect "hlt #276", "{x0}"(i64 %[[A]])
+; ABORT: unreachable
+; RECOVER: call void asm sideeffect "hlt #308", "{x0}"(i64 %[[A]])
+; RECOVER: br label
 
 ; CHECK: store i128 %b, i128* %a, align 16
 ; CHECK: ret void
 
 entry:
   store i128 %b, i128* %a, align 16
   ret void
 }
 
 define void @test_store40(i40* %a, i40 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store40(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i40* %a to i64
-; CHECK: call void @__hwasan_store(i64 %[[A]], i64 5)
+; ABORT: call void @__hwasan_store(i64 %[[A]], i64 5)
+; RECOVER: call void @__hwasan_store_noabort(i64 %[[A]], i64 5)
 ; CHECK: store i40 %b, i40* %a
 ; CHECK: ret void
 
 entry:
   store i40 %b, i40* %a, align 4
   ret void
 }
 
 define void @test_store_unaligned(i64* %a, i64 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store_unaligned(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
-; CHECK: call void @__hwasan_store(i64 %[[A]], i64 8)
+; ABORT: call void @__hwasan_store(i64 %[[A]], i64 8)
+; RECOVER: call void @__hwasan_store_noabort(i64 %[[A]], i64 8)
 ; CHECK: store i64 %b, i64* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i64 %b, i64* %a, align 4
   ret void
 }
 

llvm/test/Instrumentation/HWAddressSanitizer/with-calls.ll

 ; Test basic address sanitizer instrumentation.
 ;
-; RUN: opt < %s -hwasan -hwasan-instrument-with-calls -S | FileCheck %s
+; RUN: opt < %s -hwasan -hwasan-instrument-with-calls -S | FileCheck %s --check-prefixes=CHECK,ABORT
+; RUN: opt < %s -hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,RECOVER
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define i8 @test_load8(i8* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; CHECK: call void @__hwasan_load1(i64 %[[A]])
+; ABORT: call void @__hwasan_load1(i64 %[[A]])
+; RECOVER: call void @__hwasan_load1_noabort(i64 %[[A]])
 ; CHECK: %[[B:[^ ]*]] = load i8, i8* %a
 ; CHECK: ret i8 %[[B]]
 
 entry:
   %b = load i8, i8* %a, align 4
   ret i8 %b
 }
 
 define i16 @test_load16(i16* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
-; CHECK: call void @__hwasan_load2(i64 %[[A]])
+; ABORT: call void @__hwasan_load2(i64 %[[A]])
+; RECOVER: call void @__hwasan_load2_noabort(i64 %[[A]])
 ; CHECK: %[[B:[^ ]*]] = load i16, i16* %a
 ; CHECK: ret i16 %[[B]]
 
 entry:
   %b = load i16, i16* %a, align 4
   ret i16 %b
 }
 
 define i32 @test_load32(i32* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
-; CHECK: call void @__hwasan_load4(i64 %[[A]])
+; ABORT: call void @__hwasan_load4(i64 %[[A]])
+; RECOVER: call void @__hwasan_load4_noabort(i64 %[[A]])
 ; CHECK: %[[B:[^ ]*]] = load i32, i32* %a
 ; CHECK: ret i32 %[[B]]
 
 entry:
   %b = load i32, i32* %a, align 4
   ret i32 %b
 }
 
 define i64 @test_load64(i64* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
-; CHECK: call void @__hwasan_load8(i64 %[[A]])
+; ABORT: call void @__hwasan_load8(i64 %[[A]])
+; RECOVER: call void @__hwasan_load8_noabort(i64 %[[A]])
 ; CHECK: %[[B:[^ ]*]] = load i64, i64* %a
 ; CHECK: ret i64 %[[B]]
 
 entry:
   %b = load i64, i64* %a, align 8
   ret i64 %b
 }
 
 define i128 @test_load128(i128* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
-; CHECK: call void @__hwasan_load16(i64 %[[A]])
+; ABORT: call void @__hwasan_load16(i64 %[[A]])
+; RECOVER: call void @__hwasan_load16_noabort(i64 %[[A]])
 ; CHECK: %[[B:[^ ]*]] = load i128, i128* %a
 ; CHECK: ret i128 %[[B]]
 
 entry:
   %b = load i128, i128* %a, align 16
   ret i128 %b
 }
 
 define i40 @test_load40(i40* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load40(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i40* %a to i64
-; CHECK: call void @__hwasan_load(i64 %[[A]], i64 5)
+; ABORT: call void @__hwasan_load(i64 %[[A]], i64 5)
+; RECOVER: call void @__hwasan_load_noabort(i64 %[[A]], i64 5)
 ; CHECK: %[[B:[^ ]*]] = load i40, i40* %a
 ; CHECK: ret i40 %[[B]]
 
 entry:
   %b = load i40, i40* %a, align 4
   ret i40 %b
 }
 
 define void @test_store8(i8* %a, i8 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store8(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; CHECK: call void @__hwasan_store1(i64 %[[A]])
+; ABORT: call void @__hwasan_store1(i64 %[[A]])
+; RECOVER: call void @__hwasan_store1_noabort(i64 %[[A]])
 ; CHECK: store i8 %b, i8* %a
 ; CHECK: ret void
 
 entry:
   store i8 %b, i8* %a, align 4
   ret void
 }
 
 define void @test_store16(i16* %a, i16 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
-; CHECK: call void @__hwasan_store2(i64 %[[A]])
+; ABORT: call void @__hwasan_store2(i64 %[[A]])
+; RECOVER: call void @__hwasan_store2_noabort(i64 %[[A]])
 ; CHECK: store i16 %b, i16* %a
 ; CHECK: ret void
 
 entry:
   store i16 %b, i16* %a, align 4
   ret void
 }
 
 define void @test_store32(i32* %a, i32 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
-; CHECK: call void @__hwasan_store4(i64 %[[A]])
+; ABORT: call void @__hwasan_store4(i64 %[[A]])
+; RECOVER: call void @__hwasan_store4_noabort(i64 %[[A]])
 ; CHECK: store i32 %b, i32* %a
 ; CHECK: ret void
 
 entry:
   store i32 %b, i32* %a, align 4
   ret void
 }
 
 define void @test_store64(i64* %a, i64 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
-; CHECK: call void @__hwasan_store8(i64 %[[A]])
+; ABORT: call void @__hwasan_store8(i64 %[[A]])
+; RECOVER: call void @__hwasan_store8_noabort(i64 %[[A]])
 ; CHECK: store i64 %b, i64* %a
 ; CHECK: ret void
 
 entry:
   store i64 %b, i64* %a, align 8
   ret void
 }
 
 define void @test_store128(i128* %a, i128 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
-; CHECK: call void @__hwasan_store16(i64 %[[A]])
+; ABORT: call void @__hwasan_store16(i64 %[[A]])
+; RECOVER: call void @__hwasan_store16_noabort(i64 %[[A]])
 ; CHECK: store i128 %b, i128* %a
 ; CHECK: ret void
 
 entry:
   store i128 %b, i128* %a, align 16
   ret void
 }
 
 define void @test_store40(i40* %a, i40 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store40(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i40* %a to i64
-; CHECK: call void @__hwasan_store(i64 %[[A]], i64 5)
+; ABORT: call void @__hwasan_store(i64 %[[A]], i64 5)
+; RECOVER: call void @__hwasan_store_noabort(i64 %[[A]], i64 5)
 ; CHECK: store i40 %b, i40* %a
 ; CHECK: ret void
 
 entry:
   store i40 %b, i40* %a, align 4
   ret void
 }