diff --git a/compiler-rt/lib/hwasan/CMakeLists.txt b/compiler-rt/lib/hwasan/CMakeLists.txt --- a/compiler-rt/lib/hwasan/CMakeLists.txt +++ b/compiler-rt/lib/hwasan/CMakeLists.txt @@ -21,6 +21,7 @@ hwasan.h hwasan_allocator.h hwasan_dynamic_shadow.h + hwasan_tag_mismatch_aarch64.S hwasan_flags.h hwasan_flags.inc hwasan_interface_internal.h diff --git a/compiler-rt/lib/hwasan/hwasan_interface_internal.h b/compiler-rt/lib/hwasan/hwasan_interface_internal.h --- a/compiler-rt/lib/hwasan/hwasan_interface_internal.h +++ b/compiler-rt/lib/hwasan/hwasan_interface_internal.h @@ -99,6 +99,9 @@ SANITIZER_INTERFACE_ATTRIBUTE uptr __hwasan_tag_pointer(uptr p, u8 tag); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_tag_mismatch(uptr addr, u8 ts); + SANITIZER_INTERFACE_ATTRIBUTE u8 __hwasan_generate_tag(); diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cc b/compiler-rt/lib/hwasan/hwasan_linux.cc --- a/compiler-rt/lib/hwasan/hwasan_linux.cc +++ b/compiler-rt/lib/hwasan/hwasan_linux.cc @@ -374,15 +374,27 @@ } static void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, - ucontext_t *uc) { + ucontext_t *uc, uptr *registers_frame = nullptr) { InternalMmapVector stack_buffer(1); BufferedStackTrace *stack = stack_buffer.data(); stack->Reset(); + GetStackTrace(stack, kStackTraceMax, pc, frame, uc, common_flags()->fast_unwind_on_fatal); + // The second stack frame contains the failure __hwasan_check function, as + // we have a stack frame for the registers saved in __hwasan_tag_mismatch that + // we wish to ignore. This (currently) only occurs on AArch64, as x64 + // implementations use SIGTRAP to implement the failure, and thus do not go + // through the stack saver. + if (registers_frame) { + stack->trace++; + stack->size--; + } + bool fatal = flags()->halt_on_error || !ai.recover; - ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal); + ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal, + registers_frame); } static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { @@ -402,8 +414,8 @@ return true; } -extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_tag_mismatch( - uptr addr, uptr access_info) { +extern "C" void HwasanTagMismatch(uptr addr, uptr access_info, + uptr *registers_frame) { AccessInfo ai; ai.is_store = access_info & 0x10; ai.recover = false; @@ -411,7 +423,7 @@ ai.size = 1 << (access_info & 0xf); HandleTagMismatch(ai, (uptr)__builtin_return_address(0), - (uptr)__builtin_frame_address(0), nullptr); + (uptr)__builtin_frame_address(0), nullptr, registers_frame); __builtin_unreachable(); } diff --git a/compiler-rt/lib/hwasan/hwasan_report.h b/compiler-rt/lib/hwasan/hwasan_report.h --- a/compiler-rt/lib/hwasan/hwasan_report.h +++ b/compiler-rt/lib/hwasan/hwasan_report.h @@ -22,11 +22,11 @@ void ReportStats(); void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size, - bool is_store, bool fatal); + bool is_store, bool fatal, uptr *registers_frame); void ReportInvalidFree(StackTrace *stack, uptr addr); void ReportTailOverwritten(StackTrace *stack, uptr addr, uptr orig_size, uptr tail_size, const u8 *expected); - +void ReportRegisters(uptr *registers_frame, uptr pc); void ReportAtExitStatistics(); diff --git a/compiler-rt/lib/hwasan/hwasan_report.cc b/compiler-rt/lib/hwasan/hwasan_report.cc --- a/compiler-rt/lib/hwasan/hwasan_report.cc +++ b/compiler-rt/lib/hwasan/hwasan_report.cc @@ -14,6 +14,7 @@ #include "hwasan.h" #include "hwasan_allocator.h" #include "hwasan_mapping.h" +#include "hwasan_report.h" #include "hwasan_thread.h" #include "hwasan_thread_list.h" #include "sanitizer_common/sanitizer_allocator_internal.h" @@ -389,7 +390,7 @@ } void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, - bool is_store, bool fatal) { + bool is_store, bool fatal, uptr *registers_frame) { ScopedReport R(fatal); SavedStackAllocations current_stack_allocations( GetCurrentThread()->stack_allocations()); @@ -430,7 +431,35 @@ PrintTagsAroundAddr(tag_ptr); + if (registers_frame) + ReportRegisters(registers_frame, pc); + ReportErrorSummary(bug_type, stack); } +// See the frame breakdown defined in __hwasan_fail_entry_point (from +// hwasan_fail_entry_point_aarch64.S). +static const char *kDoubleSpace = " "; +static const char *kSingleSpace = " "; +void ReportRegisters(uptr *frame, uptr pc) { + Printf("Registers where the failure occurred (pc %p):\n", pc); + for (int i = 0; i < 28; i += 4) { + // Note - manually inserting a double or single space here based on the + // number of digits in the register name, as our sanitizer Printf does not + // support padding where the content is left aligned (i.e. the format + // specifier "%-2d" will CHECK fail). + Printf(" x%d%s%016llx", i, (i < 10) ? kDoubleSpace : kSingleSpace, + frame[28 - i]); + Printf(" x%d%s%016llx", i + 1, (i + 1 < 10) ? kDoubleSpace : kSingleSpace, + frame[30 - (i + 1)]); + Printf(" x%d%s%016llx", i + 2, (i + 2 < 10) ? kDoubleSpace : kSingleSpace, + frame[28 - (i + 2)]); + Printf(" x%d%s%016llx\n", i + 3, + (i + 3 < 10) ? kDoubleSpace : kSingleSpace, frame[30 - (i + 3)]); + } + Printf(" x28 %016llx", frame[0]); + Printf(" x29 %016llx", frame[30]); + Printf(" x30 %016llx\n", frame[31]); +} + } // namespace __hwasan diff --git a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S @@ -0,0 +1,100 @@ +// The content of this file is AArch64-only: +#if defined(__aarch64__) +#include "sanitizer_common/sanitizer_asm.h" + +// The responsibility of the HWASan entry point in compiler-rt is to primarily +// readjust the stack from the callee and save the current register values to +// the stack. +// This entry point function should be called from a __hwasan_check_* symbol. +// These are generated during a lowering pass in the backend, and are found in +// AArch64AsmPrinter::EmitHwasanMemaccessSymbols(). Please look there for +// further information. +// The __hwasan_check_* caller of this function should have expanded the stack +// and saved the previous values of x0, x1, x29, and x30. This function will +// "consume" these saved values and treats it as part of its own stack frame. +// In this sense, the __hwasan_check_* callee and this function "share" a stack +// frame. This allows us to omit having unwinding information (.cfi_*) present +// in every __hwasan_check_* function, therefore reducing binary size. This is +// particularly important as hwasan_check_* instances are duplicated in every +// translation unit where HWASan is enabled. +// This function calls HwasanTagMismatch to step back into the C++ code that +// completes the stack unwinding and error printing. This function is is not +// permitted to return. + + +// Frame from __hwasan_check_: +// | ... | +// | ... | +// | Previous stack frames... | +// +=================================+ +// | Return address (x30) for caller | +// | of __hwasan_check_*. | +// +---------------------------------+ +// | Frame address (x29) for caller | +// | of __hwasan_check_* | +// +---------------------------------+ +// | Saved x1, as __hwasan_check_* | +// | clobbers it. | +// +---------------------------------+ +// | Saved x0, likewise above. | +// +---------------------------------+ <-- [x30 / SP] + +// This function takes two arguments: +// * x0: The address of read/write instruction that caused HWASan check fail. +// * x1: The tag size. + +.section .text +.file "hwasan_fail_entry_point_aarch64.S" +.global __hwasan_tag_mismatch +.type __hwasan_tag_mismatch, %function +__hwasan_tag_mismatch: + CFI_STARTPROC + + // Set the CFA to be the return address for caller of __hwasan_check_*. Note + // that we do not emit CFI predicates to describe the contents of this stack + // frame, as this proxy entry point should never be debugged. The contents + // are static and are handled by the unwinder after calling + // __hwasan_tag_mismatch. The frame pointer is already correctly setup + // by __hwasan_check_*. + add x29, sp, #16 + CFI_DEF_CFA(w29, 16) + CFI_OFFSET(w30, -8) + CFI_OFFSET(w29, -16) + + // Allocate the frame for the rest of the registers, Note x0, x1, x29, x30 are + // already saved BELOW the stack pointer, so we need to allocate room for + // them. + stp x2, x3, [sp, #-16]! + stp x4, x5, [sp, #-16]! + stp x6, x7, [sp, #-16]! + stp x8, x9, [sp, #-16]! + stp x10, x11, [sp, #-16]! + stp x12, x13, [sp, #-16]! + stp x14, x15, [sp, #-16]! + stp x16, x17, [sp, #-16]! + stp x18, x19, [sp, #-16]! + stp x20, x21, [sp, #-16]! + stp x22, x23, [sp, #-16]! + stp x24, x25, [sp, #-16]! + stp x26, x27, [sp, #-16]! + str x28, [sp, #-16]! + + // Pass the address of the frame to HwasanTagMismatch, so that it can extract + // the saved registers from this frame without having to worry about finding + // this frame. + mov x2, sp + + bl HwasanTagMismatch + CFI_ENDPROC + +.Lfunc_end0: + .size __hwasan_tag_mismatch, .Lfunc_end0-__hwasan_tag_mismatch + +#if defined(__linux__) +/* We do not need executable stack. */ +.section .note.GNU-stack,"",@progbits +#endif + +.addrsig + +#endif // defined(__aarch64__) diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -19,6 +19,7 @@ #include "AArch64TargetObjectFile.h" #include "InstPrinter/AArch64InstPrinter.h" #include "MCTargetDesc/AArch64AddressingModes.h" +#include "MCTargetDesc/AArch64MCExpr.h" #include "MCTargetDesc/AArch64MCTargetDesc.h" #include "MCTargetDesc/AArch64TargetStreamer.h" #include "Utils/AArch64BaseInfo.h" @@ -266,6 +267,9 @@ MCSymbol *HwasanTagMismatchSym = OutContext.getOrCreateSymbol("__hwasan_tag_mismatch"); + const MCSymbolRefExpr *HwasanTagMismatchRef = + MCSymbolRefExpr::create(HwasanTagMismatchSym, OutContext); + for (auto &P : HwasanMemaccessSymbols) { unsigned Reg = P.first.first; uint32_t AccessInfo = P.first.second; @@ -316,6 +320,22 @@ MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI); OutStreamer->EmitLabel(HandleMismatchSym); + + OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXpre) + .addReg(AArch64::SP) + .addReg(AArch64::FP) + .addReg(AArch64::LR) + .addReg(AArch64::SP) + .addImm(-2), + *STI); + OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXpre) + .addReg(AArch64::SP) + .addReg(AArch64::X0) + .addReg(AArch64::X1) + .addReg(AArch64::SP) + .addImm(-2), + *STI); + if (Reg != AArch64::X0) OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ORRXrs) .addReg(AArch64::X0) @@ -328,10 +348,27 @@ .addImm(AccessInfo) .addImm(0), *STI); + + // Intentionally load the GOT entry and branch to it, rather than possibly + // late binding the function, which may clobber the registers before we have + // a chance to save them. OutStreamer->EmitInstruction( - MCInstBuilder(AArch64::B) - .addExpr(MCSymbolRefExpr::create(HwasanTagMismatchSym, OutContext)), + MCInstBuilder(AArch64::ADRP) + .addReg(AArch64::X16) + .addExpr(AArch64MCExpr::create( + HwasanTagMismatchRef, + AArch64MCExpr::VariantKind::VK_GOT_PAGE, OutContext)), *STI); + OutStreamer->EmitInstruction( + MCInstBuilder(AArch64::LDRXui) + .addReg(AArch64::X16) + .addReg(AArch64::X16) + .addExpr(AArch64MCExpr::create( + HwasanTagMismatchRef, + AArch64MCExpr::VariantKind::VK_GOT_LO12, OutContext)), + *STI); + OutStreamer->EmitInstruction( + MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI); } } diff --git a/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll b/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll --- a/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll +++ b/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll @@ -43,8 +43,13 @@ ; CHECK-NEXT: b.ne .Ltmp0 ; CHECK-NEXT: ret ; CHECK-NEXT: .Ltmp0: +; CHECK-NEXT: stp x29, x30, [sp, #-16]! +; CHECK-NEXT: stp x0, x1, [sp, #-16]! ; CHECK-NEXT: mov x1, #456 -; CHECK-NEXT: b __hwasan_tag_mismatch +; CHECK-NEXT: adrp x16, :got:__hwasan_tag_mismatch +; CHECK-NEXT: ldr x16, [x16, :got_lo12:__hwasan_tag_mismatch] +; CHECK-NEXT: br x16 + ; CHECK: .section .text.hot,"axG",@progbits,__hwasan_check_x1_123,comdat ; CHECK-NEXT: .type __hwasan_check_x1_123,@function @@ -58,6 +63,10 @@ ; CHECK-NEXT: b.ne .Ltmp1 ; CHECK-NEXT: ret ; CHECK-NEXT: .Ltmp1: +; CHECK-NEXT: stp x29, x30, [sp, #-16]! +; CHECK-NEXT: stp x0, x1, [sp, #-16]! ; CHECK-NEXT: mov x0, x1 ; CHECK-NEXT: mov x1, #123 -; CHECK-NEXT: b __hwasan_tag_mismatch +; CHECK-NEXT: adrp x16, :got:__hwasan_tag_mismatch +; CHECK-NEXT: ldr x16, [x16, :got_lo12:__hwasan_tag_mismatch] +; CHECK-NEXT: br x16 diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/hwasan/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/hwasan/BUILD.gn --- a/llvm/utils/gn/secondary/compiler-rt/lib/hwasan/BUILD.gn +++ b/llvm/utils/gn/secondary/compiler-rt/lib/hwasan/BUILD.gn @@ -42,6 +42,7 @@ "hwasan_allocator.h", "hwasan_dynamic_shadow.cc", "hwasan_dynamic_shadow.h", + "hwasan_tag_mismatch_aarch64.S", "hwasan_flags.h", "hwasan_interceptors.cc", "hwasan_interface_internal.h",