Index: compiler-rt/lib/hwasan/CMakeLists.txt =================================================================== --- compiler-rt/lib/hwasan/CMakeLists.txt +++ compiler-rt/lib/hwasan/CMakeLists.txt @@ -7,6 +7,7 @@ hwasan_dynamic_shadow.cpp hwasan_exceptions.cpp hwasan_globals.cpp + hwasan_longjmp_x86_64.S hwasan_interceptors.cpp hwasan_interceptors_vfork.S hwasan_linux.cpp @@ -14,6 +15,7 @@ hwasan_poisoning.cpp hwasan_report.cpp hwasan_setjmp.S + hwasan_setjmp_x86_64.S hwasan_tag_mismatch_aarch64.S hwasan_thread.cpp hwasan_thread_list.cpp Index: compiler-rt/lib/hwasan/hwasan.h =================================================================== --- compiler-rt/lib/hwasan/hwasan.h +++ compiler-rt/lib/hwasan/hwasan.h @@ -52,7 +52,7 @@ // simpler/faster shadow calculation. constexpr unsigned kTaggableRegionCheckShift = __sanitizer::Max(kAddressTagShift + kTagBits + 1U, 44U); -#elif defined(__x86_64__) +#elif defined(__x86_64__) && !defined(__ILP32__) // Tags are done in upper bits using Intel LAM. constexpr unsigned kAddressTagShift = 57; constexpr unsigned kTagBits = 6; @@ -162,7 +162,8 @@ RunFreeHooks(ptr); \ } while (false) -#if HWASAN_WITH_INTERCEPTORS && defined(__aarch64__) +#if HWASAN_WITH_INTERCEPTORS && !defined(__ILP32__) && \ + (defined(__aarch64__) || defined(__x86_64__)) // For both bionic and glibc __sigset_t is an unsigned long. typedef unsigned long __hw_sigset_t; // Setjmp and longjmp implementations are platform specific, and hence the @@ -180,6 +181,7 @@ }; typedef struct __hw_jmp_buf_struct __hw_jmp_buf[1]; typedef struct __hw_jmp_buf_struct __hw_sigjmp_buf[1]; -#endif // HWASAN_WITH_INTERCEPTORS && __aarch64__ +#endif // HWASAN_WITH_INTERCEPTORS && !defined(__ILP32__) && \ + // (defined(__aarch64__) || defined(__x86_64__)) #endif // HWASAN_H Index: compiler-rt/lib/hwasan/hwasan.cpp =================================================================== --- compiler-rt/lib/hwasan/hwasan.cpp +++ compiler-rt/lib/hwasan/hwasan.cpp @@ -495,7 +495,7 @@ Printf("%s\n", s.data()); } -static const u8 kFallbackTag = 0xBB; +static const u8 kFallbackTag = 0xBB & kTagMask; u8 __hwasan_generate_tag() { Thread *t = GetCurrentThread(); Index: compiler-rt/lib/hwasan/hwasan_interceptors.cpp =================================================================== --- compiler-rt/lib/hwasan/hwasan_interceptors.cpp +++ compiler-rt/lib/hwasan/hwasan_interceptors.cpp @@ -303,6 +303,52 @@ #endif // HWASAN_WITH_INTERCEPTORS && __aarch64__ +#if HWASAN_WITH_INTERCEPTORS && SANITIZER_LINUX && \ + defined(__x86_64__) && !defined(__ILP32__) +extern "C" void __hw_x86_64_longjmp(__hw_register_buf env, int retval); +// Get and/or change the set of blocked signals. +extern "C" int sigprocmask(int __how, const __hw_sigset_t *__restrict __set, + __hw_sigset_t *__restrict __oset); +#define SIG_BLOCK 0 +#define SIG_SETMASK 2 +// Refer GLibc-2.32/setjmp/sigjmp.c +// In glibc, this function is called by the `sigsetjmp' macro before doing a +// `__setjmp' on ENV[0].__jmpbuf. Always return zero. +extern "C" int __sigjmp_save(__hw_sigjmp_buf env, int savemask) { + env[0].__mask_was_saved = + (savemask && sigprocmask(SIG_BLOCK, (__hw_sigset_t *)0, + &env[0].__saved_mask) == 0); + return 0; +} + +static void __attribute__((always_inline)) +InternalLongjmp(__hw_register_buf env, int retval) { + // Clear all memory tags on the stack between here and where we're going. + unsigned long long stack_pointer = env[6]; //JB_RSP + + // The stack pointer should never be tagged, so we don't need to clear the + // tag for this function call. + __hwasan_handle_longjmp((void *)stack_pointer); + + __hw_x86_64_longjmp(env, retval); +} + +INTERCEPTOR(void, siglongjmp, __hw_sigjmp_buf env, int val) { + if (env[0].__mask_was_saved) + // Restore the saved signal mask. + (void)sigprocmask(SIG_SETMASK, &env[0].__saved_mask, + (__hw_sigset_t *)0); + InternalLongjmp(env[0].__jmpbuf, val); +} + +INTERCEPTOR(void, longjmp, __hw_jmp_buf env, int val) { + InternalLongjmp(env[0].__jmpbuf, val); +} +#undef SIG_BLOCK +#undef SIG_SETMASK + +#endif // HWASAN_WITH_INTERCEPTORS && LINUX && __x86_64__ && !ILP32 + static void BeforeFork() { StackDepotLockAll(); } Index: compiler-rt/lib/hwasan/hwasan_longjmp_x86_64.S =================================================================== --- /dev/null +++ compiler-rt/lib/hwasan/hwasan_longjmp_x86_64.S @@ -0,0 +1,97 @@ +//===-- hwasan_longjmp_x86_64.S --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// HWAddressSanitizer runtime. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_asm.h" + +#if HWASAN_WITH_INTERCEPTORS && defined(__x86_64__) && !defined(__ILP32__) +#include "sanitizer_common/sanitizer_platform.h" + +// We want to restore the context of the calling function. +// That requires +// 1) No modification of the link register by this function. +// 2) No modification of the stack pointer by this function. +// 3) (no modification of any other saved register, but that's not really going +// to occur, and hence isn't as much of a worry). +// +// There's essentially no way to ensure that the compiler will not modify the +// stack pointer when compiling a C function. +// Hence we have to write this function in assembly. + +.file "hwasan_longjmp_x86_64.S" +.section .text + +.global __hw_x86_64_longjmp +ASM_TYPE_FUNCTION(__hw_x86_64_longjmp) +__hw_x86_64_longjmp: + CFI_STARTPROC + endbr64 + + // Restore callee save registers. + mov (JB_RSP*8)(%rdi),%r8 + mov (JB_RBP*8)(%rdi),%r9 + mov (JB_PC*8)(%rdi),%rdx + + testl $X86_FEATURE_1_SHSTK, %fs:FEATURE_1_OFFSET + jz L_hw_skip_ssp + + // Check and adjust the Shadow-Stack-Pointer. + // Get the current ssp. + rdsspq %rax + // And compare it with the saved ssp value. + subq SHADOW_STACK_POINTER_OFFSET(%rdi), %rax + je L_hw_skip_ssp + // Count the number of frames to adjust and adjust it + // with incssp instruction. The instruction can adjust + // the ssp by [0..255] value only thus use a loop if + // the number of frames is bigger than 255. + negq %rax + shrq $3, %rax + // We saved Shadow-Stack-Pointer of setjmp. Since we are + // restoring Shadow-Stack-Pointer of setjmp's caller, we + // need to unwind shadow stack by one more frame. + addq $1, %rax + + movq $255, %rbx +L_hw_loop: + // rbx = rax < 255 ? rax : 255 + cmpq %rbx, %rax + cmovb %rax, %rbx + incsspq %rbx + subq %rbx, %rax + ja L_hw_loop + +L_hw_skip_ssp: + //LIBC_PROBE (longjmp, 3, LP_SIZE@%RDI_LP, -4@%esi, LP_SIZE@%RDX_LP) + nop + // We add unwind information for the target here. + movq (JB_RBX*8)(%rdi),%rbx + movq (JB_R12*8)(%rdi),%r12 + movq (JB_R13*8)(%rdi),%r13 + movq (JB_R14*8)(%rdi),%r14 + movq (JB_R15*8)(%rdi),%r15 + // Set return value for setjmp. + mov %esi, %eax + mov %r8, %rsp + movq %r9,%rbp + // LIBC_PROBE (longjmp_target, 3, + // LP_SIZE@%RDI_LP, -4@%eax, LP_SIZE@%RDX_LP) + nop + jmpq *%rdx + + CFI_ENDPROC +ASM_SIZE(__hw_x86_64_longjmp) + +#endif + +// We do not need executable stack. +NO_EXEC_STACK_DIRECTIVE Index: compiler-rt/lib/hwasan/hwasan_setjmp_x86_64.S =================================================================== --- /dev/null +++ compiler-rt/lib/hwasan/hwasan_setjmp_x86_64.S @@ -0,0 +1,93 @@ +//===-- hwasan_setjmp_x86_64.S --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// HWAddressSanitizer runtime. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_asm.h" + +#if HWASAN_WITH_INTERCEPTORS && defined(__x86_64__) && !defined(__ILP32__) +#include "sanitizer_common/sanitizer_platform.h" + +// We want to save the context of the calling function. +// That requires +// 1) No modification of the link register by this function. +// 2) No modification of the stack pointer by this function. +// 3) (no modification of any other saved register, but that's not really going +// to occur, and hence isn't as much of a worry). +// +// There's essentially no way to ensure that the compiler will not modify the +// stack pointer when compiling a C function. +// Hence we have to write this function in assembly. + +.section .text +.file "hwasan_setjmp_x86_64.S" + +.global __interceptor_setjmp +ASM_TYPE_FUNCTION(__interceptor_setjmp) +__interceptor_setjmp: + CFI_STARTPROC + endbr64 + xorl %esi, %esi + jmp __interceptor_sigsetjmp + CFI_ENDPROC +ASM_SIZE(__interceptor_setjmp) + +.global __interceptor_sigsetjmp +ASM_TYPE_FUNCTION(__interceptor_sigsetjmp) +__interceptor_sigsetjmp: + CFI_STARTPROC + endbr64 +// Save callee save registers. + movq %rbx, (JB_RBX*8)(%rdi) + movq %rbp, (JB_RBP*8)(%rdi) + movq %r12, (JB_R12*8)(%rdi) + movq %r13, (JB_R13*8)(%rdi) + movq %r14, (JB_R14*8)(%rdi) + movq %r15, (JB_R15*8)(%rdi) + +// Save SP as it will be after we return. + lea 8(%rsp), %rdx + movq %rdx, (JB_RSP*8)(%rdi) + +// Save PC we are returning to now. + mov (%rsp), %rax + movq %rax, (JB_PC*8)(%rdi) + +// Check if Shadow Stack is enabled. + testl $X86_FEATURE_1_SHSTK, %fs:FEATURE_1_OFFSET + jz .L_skip_ssp + +// Get the current Shadow-Stack-Pointer and save it. + rdsspq %rax + movq %rax, SHADOW_STACK_POINTER_OFFSET(%rdi) + +// Make a tail call to __sigjmp_save; it takes the same args. +.L_skip_ssp: + jmp __sigjmp_save + + CFI_ENDPROC +ASM_SIZE(__interceptor_sigsetjmp) + + +.macro ALIAS first second + .globl \second + .equ \second\(), \first +.endm + +ALIAS __interceptor_sigsetjmp, __sigsetjmp +.weak __sigsetjmp + +ALIAS __interceptor_setjmp, _setjmp +.weak _setjmp +#endif + +// We do not need executable stack. +NO_EXEC_STACK_DIRECTIVE Index: compiler-rt/lib/sanitizer_common/sanitizer_platform.h =================================================================== --- compiler-rt/lib/sanitizer_common/sanitizer_platform.h +++ compiler-rt/lib/sanitizer_common/sanitizer_platform.h @@ -139,6 +139,26 @@ # define FIRST_32_SECOND_64(a, b) (a) #endif +// Define register index in setjmp/longjmp buffer. +#if defined(__x86_64__) +#define JB_RBX 0 +#define JB_RBP 1 +#define JB_R12 2 +#define JB_R13 3 +#define JB_R14 4 +#define JB_R15 5 +#define JB_RSP 6 +#define JB_PC 7 +#define JB_SIZE (8*8) + +#define X86_FEATURE_1_IBT (1U << 0) +#define X86_FEATURE_1_SHSTK (1U << 1) +#define X86_FEATURE_1_LAM_U48 (1U << 2) +#define X86_FEATURE_1_LAM_U57 (1U << 3) +#define FEATURE_1_OFFSET 0x48 +#define SHADOW_STACK_POINTER_OFFSET 0x58 +#endif + #if defined(__x86_64__) && !defined(_LP64) # define SANITIZER_X32 1 #else Index: llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp =================================================================== --- llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -69,27 +69,27 @@ static const size_t kDefaultShadowScale = 4; static const uint64_t kDynamicShadowSentinel = std::numeric_limits::max(); -static const unsigned kPointerTagShift = 56; static const unsigned kShadowBaseAlignment = 32; -static cl::opt ClMemoryAccessCallbackPrefix( - "hwasan-memory-access-callback-prefix", - cl::desc("Prefix for memory access callbacks"), cl::Hidden, - cl::init("__hwasan_")); +static cl::opt + ClMemoryAccessCallbackPrefix("hwasan-memory-access-callback-prefix", + cl::desc("Prefix for memory access callbacks"), + cl::Hidden, cl::init("__hwasan_")); -static cl::opt - ClInstrumentWithCalls("hwasan-instrument-with-calls", - cl::desc("instrument reads and writes with callbacks"), - cl::Hidden, cl::init(false)); +static cl::opt ClInstrumentWithCalls( + "hwasan-instrument-with-calls", + cl::desc("instrument reads and writes with callbacks"), cl::Hidden, + cl::init(false)); static cl::opt ClInstrumentReads("hwasan-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true)); -static cl::opt ClInstrumentWrites( - "hwasan-instrument-writes", cl::desc("instrument write instructions"), - cl::Hidden, cl::init(true)); +static cl::opt + ClInstrumentWrites("hwasan-instrument-writes", + cl::desc("instrument write instructions"), cl::Hidden, + cl::init(true)); static cl::opt ClInstrumentAtomics( "hwasan-instrument-atomics", @@ -100,10 +100,10 @@ cl::desc("instrument byval arguments"), cl::Hidden, cl::init(true)); -static cl::opt ClRecover( - "hwasan-recover", - cl::desc("Enable recovery mode (continue-after-error)."), - cl::Hidden, cl::init(false)); +static cl::opt + ClRecover("hwasan-recover", + cl::desc("Enable recovery mode (continue-after-error)."), + cl::Hidden, cl::init(false)); static cl::opt ClInstrumentStack("hwasan-instrument-stack", cl::desc("instrument stack (allocas)"), @@ -130,10 +130,10 @@ cl::desc("don't report bad accesses via pointers with this tag"), cl::Hidden, cl::init(-1)); -static cl::opt ClEnableKhwasan( - "hwasan-kernel", - cl::desc("Enable KernelHWAddressSanitizer instrumentation"), - cl::Hidden, cl::init(false)); +static cl::opt + ClEnableKhwasan("hwasan-kernel", + cl::desc("Enable KernelHWAddressSanitizer instrumentation"), + cl::Hidden, cl::init(false)); // These flags allow to change the shadow mapping and control how shadow memory // is accessed. The shadow mapping looks like: @@ -185,6 +185,15 @@ cl::desc("inline all checks"), cl::Hidden, cl::init(false)); +static cl::opt ClUntagPointer("hwasan-untag-mem-operation", + cl::desc("untag mem operate"), cl::Hidden, + cl::init(false)); + +// TODO: Need to co-related with clang option. +static cl::opt ClUsePageAlias("hwasan-use-page-alias", + cl::desc("hwasan use page alias"), + cl::Hidden, cl::init(true)); + namespace { /// An instrumentation pass implementing detection of addressability bugs @@ -192,10 +201,12 @@ class HWAddressSanitizer { public: explicit HWAddressSanitizer(Module &M, bool CompileKernel = false, - bool Recover = false) : M(M) { + bool Recover = false) + : M(M) { this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover; - this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ? - ClEnableKhwasan : CompileKernel; + this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 + ? ClEnableKhwasan + : CompileKernel; initializeModule(); } @@ -224,6 +235,7 @@ bool isInterestingAlloca(const AllocaInst &AI); bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size); + Value *getTargetTagPtr(IRBuilder<> &IRB, Value *PtrLong, Value *Tag); Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag); Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong); bool instrumentStack( @@ -235,10 +247,11 @@ Value *getNextTagWithCall(IRBuilder<> &IRB); Value *getStackBaseTag(IRBuilder<> &IRB); Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI, - unsigned AllocaNo); + unsigned AllocaNo); Value *getUARTag(IRBuilder<> &IRB, Value *StackTag); Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty); + Value *retagTargetTag(IRBuilder<> &IRB, Value *OldTag); void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord); void instrumentGlobal(GlobalVariable *GV, uint8_t Tag); @@ -246,6 +259,21 @@ void instrumentPersonalityFunctions(); + unsigned getTargetTagShift() { + if (TargetTriple.getArch() == Triple::x86_64) + return 57; + return 56; + } + + uint64_t getTargetTagMask() { + if (TargetTriple.getArch() == Triple::x86_64) + return 0x3FLL; + return 0xFFLL; + } + + unsigned kPointerTagShift; + uint64_t TagMaskByte; + private: LLVMContext *C; Module &M; @@ -484,7 +512,7 @@ // x86_64 uses userspace pointer aliases, currently heap-only with callback // instrumentation only. - UsePageAliases = TargetTriple.getArch() == Triple::x86_64; + UsePageAliases = ClUsePageAlias && (TargetTriple.getArch() == Triple::x86_64); InstrumentWithCalls = UsePageAliases ? true : ClInstrumentWithCalls; InstrumentStack = UsePageAliases ? false : ClInstrumentStack; @@ -498,6 +526,8 @@ Int32Ty = IRB.getInt32Ty(); HwasanCtorFunction = nullptr; + kPointerTagShift = getTargetTagShift(); + TagMaskByte = getTargetTagMask(); // Older versions of Android do not have the required runtime support for // short granules, global or personality function instrumentation. On other @@ -708,9 +738,6 @@ } void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) { - if (TargetTriple.isAArch64() || TargetTriple.getArch() == Triple::x86_64) - return; - IRBuilder<> IRB(I); Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy); Value *UntaggedPtr = @@ -799,28 +826,27 @@ IRB.SetInsertPoint(CheckFailTerm); InlineAsm *Asm; switch (TargetTriple.getArch()) { - case Triple::x86_64: - // The signal handler will find the data address in rdi. - Asm = InlineAsm::get( - FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false), - "int3\nnopl " + - itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) + - "(%rax)", - "{rdi}", - /*hasSideEffects=*/true); - break; - case Triple::aarch64: - case Triple::aarch64_be: - // The signal handler will find the data address in x0. - Asm = InlineAsm::get( - FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false), - "brk #" + - itostr(0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)), - "{x0}", - /*hasSideEffects=*/true); - break; - default: - report_fatal_error("unsupported architecture"); + case Triple::x86_64: + // The signal handler will find the data address in rdi. + Asm = InlineAsm::get( + FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false), + "int3\nnopl " + + itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) + + "(%rax)", + "{rdi}", + /*hasSideEffects=*/true); + break; + case Triple::aarch64: + case Triple::aarch64_be: + // The signal handler will find the data address in x0. + Asm = InlineAsm::get( + FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false), + "brk #" + itostr(0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)), + "{x0}", + /*hasSideEffects=*/true); + break; + default: + report_fatal_error("unsupported architecture"); } IRB.CreateCall(Asm, PtrLong); if (Recover) @@ -851,7 +877,7 @@ LLVM_DEBUG(dbgs() << "Instrumenting: " << O.getInsn() << "\n"); if (O.MaybeMask) - return false; //FIXME + return false; // FIXME IRBuilder<> IRB(O.getInsn()); if (isPowerOf2_64(O.TypeSize) && @@ -870,7 +896,8 @@ {IRB.CreatePointerCast(Addr, IntptrTy), ConstantInt::get(IntptrTy, O.TypeSize / 8)}); } - untagPointerOperand(O.getInsn(), Addr); + if (ClUntagPointer) + untagPointerOperand(O.getInsn(), Addr); return true; } @@ -887,8 +914,8 @@ return SizeInBytes * ArraySize; } -bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, - Value *Tag, size_t Size) { +bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, + size_t Size) { size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment()); if (!UseShortGranules) Size = AlignedSize; @@ -921,7 +948,7 @@ return true; } -static unsigned RetagMask(unsigned AllocaNo) { +static unsigned retagMaskArmv8(unsigned AllocaNo) { // A list of 8-bit numbers that have at most one run of non-zero bits. // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these // masks. @@ -939,6 +966,29 @@ return FastMasks[AllocaNo % (sizeof(FastMasks) / sizeof(FastMasks[0]))]; } +static unsigned retagX86_64(unsigned AllocaNo, uint64_t TagMaskByte) { + return AllocaNo & TagMaskByte; +} + +static unsigned retagMask(unsigned AllocaNo, Triple TargetTriple, + uint64_t TagMaskByte) { + if (TargetTriple.getArch() == Triple::x86_64) + return retagX86_64(AllocaNo, TagMaskByte); + return retagMaskArmv8(AllocaNo); +} + +// The whole pass imply the Tag is 8-bits size, because it is first +// implemented by aarch64 whose tag is happen 1 byte. We should diff +// it frome other targets now. +Value *HWAddressSanitizer::retagTargetTag(IRBuilder<> &IRB, Value *OldTag) { + if (TargetTriple.getArch() == Triple::x86_64) { + Constant *TagMask = ConstantInt::get(IntptrTy, TagMaskByte); + Value *NewTag = IRB.CreateAnd(OldTag, TagMask); + return NewTag; + } else + return OldTag; +} + Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) { return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy); } @@ -964,6 +1014,9 @@ Value *StackTag = IRB.CreateXor(StackPointerLong, IRB.CreateLShr(StackPointerLong, 20), "hwasan.stack.base.tag"); + + StackTag = retagTargetTag(IRB, StackTag); + return StackTag; } @@ -971,34 +1024,51 @@ AllocaInst *AI, unsigned AllocaNo) { if (ClGenerateTagsWithCalls) return getNextTagWithCall(IRB); - return IRB.CreateXor(StackTag, - ConstantInt::get(IntptrTy, RetagMask(AllocaNo))); + + Value *Tag = IRB.CreateXor( + StackTag, ConstantInt::get( + IntptrTy, retagMask(AllocaNo, TargetTriple, TagMaskByte))); + return retagTargetTag(IRB, Tag); } Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) { if (ClUARRetagToZero) return ConstantInt::get(IntptrTy, 0); + if (ClGenerateTagsWithCalls) return getNextTagWithCall(IRB); - return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU)); + + Value *Tag = IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU)); + return retagTargetTag(IRB, Tag); } -// Add a tag to an address. -Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty, - Value *PtrLong, Value *Tag) { - assert(!UsePageAliases); +Value *HWAddressSanitizer::getTargetTagPtr(IRBuilder<> &IRB, Value *PtrLong, + Value *Tag) { Value *TaggedPtrLong; + Value *Mask; + Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift); + // Kernel addresses have 0xFF in the most significant byte. if (CompileKernel) { - // Kernel addresses have 0xFF in the most significant byte. - Value *ShiftedTag = IRB.CreateOr( - IRB.CreateShl(Tag, kPointerTagShift), - ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1)); + // X86_64 can not tag the highest bit. + if (TargetTriple.getArch() == Triple::x86_64) + Mask = ConstantInt::get(IntptrTy, ((1ULL << kPointerTagShift) - 1) | + (1ULL << (64 - 1))); + else + Mask = ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1); + ShiftedTag = IRB.CreateOr(ShiftedTag, Mask); TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag); } else { - // Userspace can simply do OR (tag << 56); - Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift); + // Userspace can simply do OR (tag << kPointerTagShift); TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag); } + return TaggedPtrLong; +} + +// Add a tag to an address. +Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty, + Value *PtrLong, Value *Tag) { + assert(!UsePageAliases); + Value *TaggedPtrLong = getTargetTagPtr(IRB, PtrLong, Tag); return IRB.CreateIntToPtr(TaggedPtrLong, Ty); } @@ -1008,11 +1078,13 @@ Value *UntaggedPtrLong; if (CompileKernel) { // Kernel addresses have 0xFF in the most significant byte. - UntaggedPtrLong = IRB.CreateOr(PtrLong, - ConstantInt::get(PtrLong->getType(), 0xFFULL << kPointerTagShift)); + UntaggedPtrLong = + IRB.CreateOr(PtrLong, ConstantInt::get(PtrLong->getType(), + 0xFFULL << kPointerTagShift)); } else { // Userspace addresses have 0x00. - UntaggedPtrLong = IRB.CreateAnd(PtrLong, + UntaggedPtrLong = IRB.CreateAnd( + PtrLong, ConstantInt::get(PtrLong->getType(), ~(0xFFULL << kPointerTagShift))); } return UntaggedPtrLong; @@ -1034,7 +1106,6 @@ if (ThreadPtrGlobal) return ThreadPtrGlobal; - return nullptr; } @@ -1163,8 +1234,9 @@ // Prepend "tag_offset, N" to the dwarf expression. // Tag offset logically applies to the alloca pointer, and it makes sense // to put it at the beginning of the expression. - SmallVector NewOps = {dwarf::DW_OP_LLVM_tag_offset, - RetagMask(N)}; + SmallVector NewOps = { + dwarf::DW_OP_LLVM_tag_offset, + retagMask(N, TargetTriple, TagMaskByte)}; auto Locations = DDI->location_ops(); unsigned LocNo = std::distance(Locations.begin(), find(Locations, AI)); DDI->setExpression( @@ -1213,9 +1285,9 @@ SmallVector OperandsToInstrument; SmallVector IntrinToInstrument; - SmallVector AllocasToInstrument; - SmallVector RetVec; - SmallVector LandingPadVec; + SmallVector AllocasToInstrument; + SmallVector RetVec; + SmallVector LandingPadVec; DenseMap> AllocaDbgMap; for (auto &BB : F) { for (auto &Inst : BB) { @@ -1459,8 +1531,10 @@ for (GlobalVariable *GV : Globals) { // Skip tag 0 in order to avoid collisions with untagged memory. + Tag &= TagMaskByte; if (Tag == 0) Tag = 1; + instrumentGlobal(GV, Tag++); } } Index: llvm/test/Instrumentation/HWAddressSanitizer/X86/stack.ll =================================================================== --- /dev/null +++ llvm/test/Instrumentation/HWAddressSanitizer/X86/stack.ll @@ -0,0 +1,54 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -hwasan -hwasan-instrument-with-calls -hwasan-instrument-stack -hwasan-use-page-alias=false -S | FileCheck %s + +source_filename = "stack.c" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: noinline nounwind optnone uwtable +define dso_local i32 @main() sanitize_hwaddress { +; CHECK-LABEL: @main( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[DOTHWASAN_SHADOW:%.*]] = call i8* asm "", "=r,0"(i8* null) +; CHECK-NEXT: [[TMP0:%.*]] = call i8* @llvm.frameaddress.p0i8(i32 0) +; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint i8* [[TMP0]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 20 +; CHECK-NEXT: [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]] +; CHECK-NEXT: [[TMP3:%.*]] = and i64 [[HWASAN_STACK_BASE_TAG]], 63 +; CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4 +; CHECK-NEXT: [[LV0:%.*]] = alloca { i32, [12 x i8] }, align 16 +; CHECK-NEXT: [[TMP4:%.*]] = bitcast { i32, [12 x i8] }* [[LV0]] to i32* +; CHECK-NEXT: [[TMP5:%.*]] = xor i64 [[TMP3]], 0 +; CHECK-NEXT: [[TMP6:%.*]] = and i64 [[TMP5]], 63 +; CHECK-NEXT: [[TMP7:%.*]] = ptrtoint i32* [[TMP4]] to i64 +; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[TMP6]], 57 +; CHECK-NEXT: [[TMP9:%.*]] = or i64 [[TMP7]], [[TMP8]] +; CHECK-NEXT: [[LV0_HWASAN:%.*]] = inttoptr i64 [[TMP9]] to i32* +; CHECK-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP6]] to i8 +; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP4]] to i8* +; CHECK-NEXT: call void @__hwasan_tag_memory(i8* [[TMP11]], i8 [[TMP10]], i64 16) +; CHECK-NEXT: [[TMP12:%.*]] = ptrtoint i32* [[RETVAL]] to i64 +; CHECK-NEXT: call void @__hwasan_store4(i64 [[TMP12]]) +; CHECK-NEXT: store i32 0, i32* [[RETVAL]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = ptrtoint i32* [[LV0_HWASAN]] to i64 +; CHECK-NEXT: call void @__hwasan_store4(i64 [[TMP13]]) +; CHECK-NEXT: store i32 12345, i32* [[LV0_HWASAN]], align 4 +; CHECK-NEXT: call void @foo(i32* [[LV0_HWASAN]]) +; CHECK-NEXT: [[TMP14:%.*]] = ptrtoint i32* [[LV0_HWASAN]] to i64 +; CHECK-NEXT: call void @__hwasan_load4(i64 [[TMP14]]) +; CHECK-NEXT: [[TMP15:%.*]] = load i32, i32* [[LV0_HWASAN]], align 4 +; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32* [[TMP4]] to i8* +; CHECK-NEXT: call void @__hwasan_tag_memory(i8* [[TMP16]], i8 0, i64 16) +; CHECK-NEXT: ret i32 [[TMP15]] +; +entry: + %retval = alloca i32, align 4 + %lv0 = alloca i32, align 4 + store i32 0, i32* %retval, align 4 + store i32 12345, i32* %lv0, align 4 + call void @foo(i32* %lv0) + %0 = load i32, i32* %lv0, align 4 + ret i32 %0 +} + +declare dso_local void @foo(i32*)