hwasan: fix inline instrumentation

[andreyknvl]

Summary

This patch changes hwasan inline instrumentation:

1. Fixes address untagging for shadow address calculation (use 0xFF instead of 0x00 for the top byte).
2. Emits brk instruction instead of hlt for the kernel and user space.
3. Use 0x900 instead of 0x100 for brk immediate (0x100 - 0x800 are unavailable in the kernel).
4. Fixes and adds appropriate tests.

[kcc]

please add a lit test.

[eugenis]

LGTM with a test.
Please don't copy all the tests for different hlt# values, just one would be enough to capture the difference in 0x900.

[eugenis]

Andrey, do you mind switching userspace to brk as well? You'd need to change the decoding in hwasan_linux.cc as well, but it could be even less work than writing a kernel-only lit test.

[andreyknvl]

Switched user space hwasan to brk.
Fixed and added tests.

[andreyknvl]

PTAL

[andreyknvl]

It seems that hwasan_linux.cc is in a different repo (compiler-rt) and is not picked up by svn diff. Do I need to submit that change as a separate patch?

[eugenis]

You can pass multiple repositories as arguments to svn diff, or upload a separate revision, or use git monorepo.

lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

314 ↗

(On Diff #135230)

Could we use 0x900 in the userspace as well? I assume 0x100 is not available in the kernel for some reason. There are no ABI concerns yet.

[andreyknvl]

Use 0x900 instead of 0x100 for brk immediate (0x100 - 0x800 are unavailable in the kernel).

[andreyknvl]

Done, PTAL

[eugenis]

projects/compiler-rt/lib/hwasan/hwasan_linux.cc

191 ↗

(On Diff #135278)

0x9XY

255 ↗

(On Diff #135278)

I don't think we have a SIGTRAP handler.

[andreyknvl]

Install SIGTRAP handler.

[andreyknvl]

PTAL

[eugenis]

projects/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc

221 ↗

(On Diff #135292)

It won't do anything, see GetHandleSignalModeImpl

[andreyknvl]

Try harder to install SIGTRAP handler.

[andreyknvl]

PTAL

[eugenis]

Looks great, thank you!
I can land it for you.

[andreyknvl]

If to land means to commit, then yes, please :)

[eugenis]

r325711.
I had to fix userspace callbacks - they were still doing hlt with 0x100 instead of brk with 0x900.

compiler-rt/trunk/lib/hwasan/hwasan.cc

SetCommonFlagsDefaults();
     cf.CopyFrom(*common_flags());
     cf.external_symbolizer_path = GetEnv("HWASAN_SYMBOLIZER_PATH");
     cf.malloc_context_size = 20;
     cf.handle_ioctl = true;
     // FIXME: test and enable.
     cf.check_printf = false;
     cf.intercept_tls_get_addr = true;
     cf.exitcode = 99;
-    cf.handle_sigill = kHandleSignalExclusive;
+    cf.handle_sigtrap = kHandleSignalExclusive;
     OverrideCommonFlags(cf);
   }
 
   Flags *f = flags();
   f->SetDefaults();
 
   FlagParser parser;
   RegisterHwasanFlags(&parser, f);
void __sanitizer_unaligned_store32(uu32 *p, u32 x) {
   *p = x;
 }
 void __sanitizer_unaligned_store64(uu64 *p, u64 x) {
   *p = x;
 }
 
 template<unsigned X>
 __attribute__((always_inline))
-static void SigIll() {
+static void SigTrap() {
 #if defined(__aarch64__)
-  asm("hlt %0\n\t" ::"n"(X));
+  asm("brk %0\n\t" ::"n"(X));
 #elif defined(__x86_64__) || defined(__i386__)
   asm("ud2\n\t");
 #else
   // FIXME: not always sigill.
   __builtin_trap();
 #endif
   // __builtin_unreachable();
 }
 
 enum class ErrorAction { Abort, Recover };
 enum class AccessType { Load, Store };
 
 template <ErrorAction EA, AccessType AT, unsigned LogSize>
 __attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) {
   tag_t ptr_tag = GetTagFromPointer(p);
   uptr ptr_raw = p & ~kAddressTagMask;
   tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(ptr_raw);
   if (UNLIKELY(ptr_tag != mem_tag)) {
-    SigIll<0x100 + 0x20 * (EA == ErrorAction::Recover) +
+    SigTrap<0x900 + 0x20 * (EA == ErrorAction::Recover) +
            0x10 * (AT == AccessType::Store) + LogSize>();
     if (EA == ErrorAction::Abort) __builtin_unreachable();
   }
 }
 
 template <ErrorAction EA, AccessType AT>
 __attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p,
                                                                       uptr sz) {
   CHECK_NE(0, sz);
   tag_t ptr_tag = GetTagFromPointer(p);
   uptr ptr_raw = p & ~kAddressTagMask;
   tag_t *shadow_first = (tag_t *)MEM_TO_SHADOW(ptr_raw);
   tag_t *shadow_last = (tag_t *)MEM_TO_SHADOW(ptr_raw + sz - 1);
   for (tag_t *t = shadow_first; t <= shadow_last; ++t)
     if (UNLIKELY(ptr_tag != *t)) {
-      SigIll<0x100 + 0x20 * (EA == ErrorAction::Recover) +
+      SigTrap<0x900 + 0x20 * (EA == ErrorAction::Recover) +
              0x10 * (AT == AccessType::Store) + 0xf>();
       if (EA == ErrorAction::Abort) __builtin_unreachable();
     }
 }
 
 void __hwasan_load(uptr p, uptr sz) {
   CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz);
 }

compiler-rt/trunk/lib/hwasan/hwasan_linux.cc

struct AccessInfo {
   uptr size;
   bool is_store;
   bool is_load;
   bool recover;
 };
 
 #if defined(__aarch64__)
 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
-  // Access type is encoded in HLT immediate as 0x1XY,
+  // Access type is encoded in BRK immediate as 0x9XY,
   // where X&1 is 1 for store, 0 for load,
   // and X&2 is 1 if the error is recoverable.
   // Valid values of Y are 0 to 4, which are interpreted as log2(access_size),
   // and 0xF, which means that access size is stored in X1 register.
   // Access address is always in X0 register.
   AccessInfo ai;
   uptr pc = (uptr)info->si_addr;
   unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
-  if ((code & 0xff00) != 0x100)
+  if ((code & 0xff00) != 0x900)
     return AccessInfo{0, 0, false, false}; // Not ours.
   bool is_store = code & 0x10;
   bool recover = code & 0x20;
   unsigned size_log = code & 0xf;
   if (size_log > 4 && size_log != 0xf)
     return AccessInfo{0, 0, false, false}; // Not ours.
 
   ai.is_store = is_store;
   ai.is_load = !is_store;
   ai.addr = uc->uc_mcontext.regs[0];
   if (size_log == 0xf)
     ai.size = uc->uc_mcontext.regs[1];
   else
     ai.size = 1U << size_log;
   ai.recover = recover;
   return ai;
 }
 #else
 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
   return AccessInfo{0, 0, false, false};
 }
 #endif
 
-static bool HwasanOnSIGILL(int signo, siginfo_t *info, ucontext_t *uc) {
+static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) {
   SignalContext sig{info, uc};
   AccessInfo ai = GetAccessInfo(info, uc);
   if (!ai.is_store && !ai.is_load)
     return false;
 
   InternalScopedBuffer<BufferedStackTrace> stack_buffer(1);
   BufferedStackTrace *stack = stack_buffer.data();
   stack->Reset();

 static void OnStackUnwind(const SignalContext &sig, const void *,
                           BufferedStackTrace *stack) {
   GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context,
                 common_flags()->fast_unwind_on_fatal);
 }
 
 void HwasanOnDeadlySignal(int signo, void *info, void *context) {
   // Probably a tag mismatch.
-  if (signo == SIGILL)
-    if (HwasanOnSIGILL(signo, (siginfo_t *)info, (ucontext_t*)context))
+  if (signo == SIGTRAP)
+    if (HwasanOnSIGTRAP(signo, (siginfo_t *)info, (ucontext_t*)context))
       return;
 
   HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
 }
 
 
 } // namespace __hwasan
 
 #endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD

compiler-rt/trunk/lib/sanitizer_common/sanitizer_flags.inc

 COMMON_FLAG(HandleSignalMode, handle_segv, kHandleSignalYes,
             COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGSEGV))
 COMMON_FLAG(HandleSignalMode, handle_sigbus, kHandleSignalYes,
             COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGBUS))
 COMMON_FLAG(HandleSignalMode, handle_abort, kHandleSignalNo,
             COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGABRT))
 COMMON_FLAG(HandleSignalMode, handle_sigill, kHandleSignalNo,
             COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGILL))
+COMMON_FLAG(HandleSignalMode, handle_sigtrap, kHandleSignalNo,
+            COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGTRAP))
 COMMON_FLAG(HandleSignalMode, handle_sigfpe, kHandleSignalYes,
             COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGFPE))
 #undef COMMON_FLAG_HANDLE_SIGNAL_HELP
 COMMON_FLAG(bool, allow_user_segv_handler, true,
             "Deprecated. True has no effect, use handle_sigbus=1. If false, "
             "handle_*=1 will be upgraded to handle_*=2.")
 COMMON_FLAG(bool, use_sigaltstack, true,
             "If set, uses alternate stack for signal handling.")

compiler-rt/trunk/lib/sanitizer_common/sanitizer_linux.cc

 #endif
 
 static HandleSignalMode GetHandleSignalModeImpl(int signum) {
   switch (signum) {
     case SIGABRT:
       return common_flags()->handle_abort;
     case SIGILL:
       return common_flags()->handle_sigill;
+    case SIGTRAP:
+      return common_flags()->handle_sigtrap;
     case SIGFPE:
       return common_flags()->handle_sigfpe;
     case SIGSEGV:
       return common_flags()->handle_segv;
     case SIGBUS:
       return common_flags()->handle_sigbus;
   }
   return kHandleSignalNo;

compiler-rt/trunk/lib/sanitizer_common/sanitizer_mac.cc

 void ListOfModules::fallbackInit() { clear(); }
 
 static HandleSignalMode GetHandleSignalModeImpl(int signum) {
   switch (signum) {
     case SIGABRT:
       return common_flags()->handle_abort;
     case SIGILL:
       return common_flags()->handle_sigill;
+    case SIGTRAP:
+      return common_flags()->handle_sigtrap;
     case SIGFPE:
       return common_flags()->handle_sigfpe;
     case SIGSEGV:
       return common_flags()->handle_segv;
     case SIGBUS:
       return common_flags()->handle_sigbus;
   }
   return kHandleSignalNo;

compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix_libcdep.cc

void InstallDeadlySignalHandlers(SignalHandlerType handler) {
   // This will cause SetAlternateSignalStack to be called twice, but the stack
   // will be actually set only once.
   if (common_flags()->use_sigaltstack) SetAlternateSignalStack();
   MaybeInstallSigaction(SIGSEGV, handler);
   MaybeInstallSigaction(SIGBUS, handler);
   MaybeInstallSigaction(SIGABRT, handler);
   MaybeInstallSigaction(SIGFPE, handler);
   MaybeInstallSigaction(SIGILL, handler);
+  MaybeInstallSigaction(SIGTRAP, handler);
 }
 
 bool SignalContext::IsStackOverflow() const {
   // Access at a reasonable offset above SP, or slightly below it (to account
   // for x86_64 or PowerPC redzone, ARM push of multiple registers, etc) is
   // probably a stack overflow.
 #ifdef __s390__
   // On s390, the fault address in siginfo points to start of the page, not

llvm/trunk/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

public:
   bool instrumentMemAccess(Instruction *I);
   Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite,
                                    uint64_t *TypeSize, unsigned *Alignment,
                                    Value **MaybeMask);
 
   bool isInterestingAlloca(const AllocaInst &AI);
   bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag);
   Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
+  Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
   bool instrumentStack(SmallVectorImpl<AllocaInst *> &Allocas,
                        SmallVectorImpl<Instruction *> &RetVec);
   Value *getNextTagWithCall(IRBuilder<> &IRB);
   Value *getStackBaseTag(IRBuilder<> &IRB);
   Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI,
                      unsigned AllocaNo);
   Value *getUARTag(IRBuilder<> &IRB, Value *StackTag);
 
static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
   return Res;
 }
 
 void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
                                                    unsigned AccessSizeIndex,
                                                    Instruction *InsertBefore) {
   IRBuilder<> IRB(InsertBefore);
   Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift), IRB.getInt8Ty());
-  Value *AddrLong =
-      IRB.CreateAnd(PtrLong, ConstantInt::get(PtrLong->getType(),
-                                              ~(0xFFULL << kPointerTagShift)));
+  Value *AddrLong = untagPointer(IRB, PtrLong);
   Value *ShadowLong = IRB.CreateLShr(AddrLong, kShadowScale);
   if (ClMappingOffset)
     ShadowLong = IRB.CreateAdd(
         ShadowLong, ConstantInt::get(PtrLong->getType(), ClMappingOffset,
                                      /*isSigned=*/false));
   Value *MemTag =
       IRB.CreateLoad(IRB.CreateIntToPtr(ShadowLong, IRB.getInt8PtrTy()));
   Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
 
   TerminatorInst *CheckTerm =
       SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, !Recover,
                                 MDBuilder(*C).createBranchWeights(1, 100000));
 
   IRB.SetInsertPoint(CheckTerm);
   // The signal handler will find the data address in x0.
   InlineAsm *Asm = InlineAsm::get(
       FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
-      "hlt #" +
-          itostr(0x100 + Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex),
+      "brk #" +
+          itostr(0x900 + Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex),
       "{x0}",
       /*hasSideEffects=*/true);
   IRB.CreateCall(Asm, PtrLong);
 }
 
 bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
   DEBUG(dbgs() << "Instrumenting: " << *I << "\n");
   bool IsWrite = false;
Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong,
   } else {
     // Userspace can simply do OR (tag << 56);
     Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift);
     TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag);
   }
   return IRB.CreateIntToPtr(TaggedPtrLong, Ty);
 }
 
+// Remove tag from an address.
+Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
+  Value *UntaggedPtrLong;
+  if (ClEnableKhwasan) {
+    // Kernel addresses have 0xFF in the most significant byte.
+    UntaggedPtrLong = IRB.CreateOr(PtrLong,
+        ConstantInt::get(PtrLong->getType(), 0xFFULL << kPointerTagShift));
+  } else {
+    // Userspace addresses have 0x00.
+    UntaggedPtrLong = IRB.CreateAnd(PtrLong,
+        ConstantInt::get(PtrLong->getType(), ~(0xFFULL << kPointerTagShift)));
+  }
+  return UntaggedPtrLong;
+}
+
 bool HWAddressSanitizer::instrumentStack(
     SmallVectorImpl<AllocaInst *> &Allocas,
     SmallVectorImpl<Instruction *> &RetVec) {
   Function *F = Allocas[0]->getParent()->getParent();
   Instruction *InsertPt = &*F->getEntryBlock().begin();
   IRBuilder<> IRB(InsertPt);
 
   Value *StackTag = getStackBaseTag(IRB);

llvm/trunk/test/Instrumentation/HWAddressSanitizer/atomic.ll

 ; Test basic address sanitizer instrumentation.
 ;
 ; RUN: opt < %s -hwasan -S | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define void @atomicrmw(i64* %ptr) sanitize_hwaddress {
 ; CHECK-LABEL: @atomicrmw(
 ; CHECK: lshr i64 %[[A:[^ ]*]], 56
-; CHECK: call void asm sideeffect "hlt #275", "{x0}"(i64 %[[A]])
+; CHECK: call void asm sideeffect "brk #2323", "{x0}"(i64 %[[A]])
 ; CHECK: atomicrmw add i64* %ptr, i64 1 seq_cst
 ; CHECK: ret void
 
 entry:
   %0 = atomicrmw add i64* %ptr, i64 1 seq_cst
   ret void
 }
 
 define void @cmpxchg(i64* %ptr, i64 %compare_to, i64 %new_value) sanitize_hwaddress {
 ; CHECK-LABEL: @cmpxchg(
 ; CHECK: lshr i64 %[[A:[^ ]*]], 56
-; CHECK: call void asm sideeffect "hlt #275", "{x0}"(i64 %[[A]])
+; CHECK: call void asm sideeffect "brk #2323", "{x0}"(i64 %[[A]])
 ; CHECK: cmpxchg i64* %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
 ; CHECK: ret void
 
 entry:
   %0 = cmpxchg i64* %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
   ret void
 }

llvm/trunk/test/Instrumentation/HWAddressSanitizer/basic.ll

 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #256", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2304", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #288", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2336", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
 ; CHECK: ret i8 %[[G]]
 
 entry:
   %b = load i8, i8* %a, align 4
   ret i8 %b
 }
 
 define i16 @test_load16(i16* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #257", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2305", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #289", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2337", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i16, i16* %a, align 4
 ; CHECK: ret i16 %[[G]]
 
 entry:
   %b = load i16, i16* %a, align 4
   ret i16 %b
 }
 
 define i32 @test_load32(i32* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #258", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2306", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #290", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2338", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i32, i32* %a, align 4
 ; CHECK: ret i32 %[[G]]
 
 entry:
   %b = load i32, i32* %a, align 4
   ret i32 %b
 }
 
 define i64 @test_load64(i64* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #259", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2307", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #291", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2339", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i64, i64* %a, align 8
 ; CHECK: ret i64 %[[G]]
 
 entry:
   %b = load i64, i64* %a, align 8
   ret i64 %b
 }
 
 define i128 @test_load128(i128* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #260", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2308", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #292", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2340", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i128, i128* %a, align 16
 ; CHECK: ret i128 %[[G]]
 
 entry:
   %b = load i128, i128* %a, align 16
   ret i128 %b

 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #272", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2320", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #304", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2352", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: store i8 %b, i8* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i8 %b, i8* %a, align 4
   ret void
 }
 
 define void @test_store16(i16* %a, i16 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store16(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #273", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2321", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #305", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2353", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: store i16 %b, i16* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i16 %b, i16* %a, align 4
   ret void
 }
 
 define void @test_store32(i32* %a, i32 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store32(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #274", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2322", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #306", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2354", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: store i32 %b, i32* %a, align 4
 ; CHECK: ret void
 
 entry:
   store i32 %b, i32* %a, align 4
   ret void
 }
 
 define void @test_store64(i64* %a, i64 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store64(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #275", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2323", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #307", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2355", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: store i64 %b, i64* %a, align 8
 ; CHECK: ret void
 
 entry:
   store i64 %b, i64* %a, align 8
   ret void
 }
 
 define void @test_store128(i128* %a, i128 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store128(
 ; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
 ; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
 ; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
 ; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
 ; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
 ; CHECK: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #276", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2324", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #308", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2356", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: store i128 %b, i128* %a, align 16
 ; CHECK: ret void
 
 entry:
   store i128 %b, i128* %a, align 16
   ret void

llvm/trunk/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll

-; Test basic address sanitizer instrumentation.
+; Test kernel hwasan instrumentation for alloca.
 ;
 ; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 declare void @use32(i32*)
 

llvm/trunk/test/Instrumentation/HWAddressSanitizer/kernel.ll

 ; Test kernel hwasan instrumentation.
 ;
-; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s --allow-empty --check-prefixes=KERNEL
-; RUN: opt < %s -hwasan -hwasan-mapping-offset=12345678 -S | FileCheck %s  --check-prefixes=OFFSET
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s --allow-empty --check-prefixes=INIT
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s  --check-prefixes=CHECK,NOOFFSET
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-mapping-offset=12345678 -S | FileCheck %s  --check-prefixes=CHECK,OFFSET
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=0 -S | FileCheck %s  --check-prefixes=CHECK,NOOFFSET,ABORT
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -S | FileCheck %s  --check-prefixes=CHECK,NOOFFSET,RECOVER
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define i8 @test_load(i8* %a) sanitize_hwaddress {
-; OFFSET-LABEL: @test_load(
-; OFFSET: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; OFFSET: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; OFFSET: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; OFFSET: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; OFFSET: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK-LABEL: @test_load(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = or i64 %[[A]], -72057594037927936
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+
+; NOOFFSET: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+
 ; OFFSET: %[[D1:[^ ]*]] = add i64 %[[D]], 12345678
 ; OFFSET: %[[E:[^ ]*]] = inttoptr i64 %[[D1]] to i8*
-; OFFSET: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
-; OFFSET: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; OFFSET: br i1 %[[F]],
+
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; ABORT: call void asm sideeffect "brk #2304", "{x0}"(i64 %[[A]])
+; ABORT: unreachable
+; RECOVER: call void asm sideeffect "brk #2336", "{x0}"(i64 %[[A]])
+; RECOVER: br label
+
+; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
+; CHECK: ret i8 %[[G]]
 
 entry:
   %b = load i8, i8* %a, align 4
   ret i8 %b
 }
 
-; KERNEL-NOT: call void @__hwasan_init
+; INIT-NOT: call void @__hwasan_init