Index: llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp =================================================================== --- llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -69,7 +69,6 @@ static const size_t kDefaultShadowScale = 4; static const uint64_t kDynamicShadowSentinel = std::numeric_limits::max(); -static const unsigned kPointerTagShift = 56; static const unsigned kShadowBaseAlignment = 32; @@ -186,6 +185,15 @@ cl::desc("inline all checks"), cl::Hidden, cl::init(false)); +static cl::opt ClUntagPointer("hwasan-untag-mem-operation", + cl::desc("untag mem operate"), cl::Hidden, + cl::init(false)); + +// TODO: Need to co-related with clang option. +static cl::opt ClUsePageAlias("hwasan-use-page-alias", + cl::desc("hwasan use page alias"), + cl::Hidden, cl::init(true)); + namespace { /// An instrumentation pass implementing detection of addressability bugs @@ -227,6 +235,7 @@ bool isInterestingAlloca(const AllocaInst &AI); bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size); + Value *getTargetTagPtr(IRBuilder<> &IRB, Value *PtrLong, Value *Tag); Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag); Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong); bool instrumentStack( @@ -242,6 +251,7 @@ Value *getUARTag(IRBuilder<> &IRB, Value *StackTag); Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty); + Value *retagTargetTag(IRBuilder<> &IRB, Value *OldTag); void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord); void instrumentGlobal(GlobalVariable *GV, uint8_t Tag); @@ -249,6 +259,21 @@ void instrumentPersonalityFunctions(); + unsigned getTargetTagShift() { + if (TargetTriple.getArch() == Triple::x86_64) + return 57; + return 56; + } + + uint64_t getTargetTagMask() { + if (TargetTriple.getArch() == Triple::x86_64) + return 0x3FLL; + return 0xFFLL; + } + + unsigned kPointerTagShift; + uint64_t TagMaskByte; + private: LLVMContext *C; Module &M; @@ -487,7 +512,7 @@ // x86_64 uses userspace pointer aliases, currently heap-only with callback // instrumentation only. - UsePageAliases = TargetTriple.getArch() == Triple::x86_64; + UsePageAliases = ClUsePageAlias && (TargetTriple.getArch() == Triple::x86_64); InstrumentWithCalls = UsePageAliases ? true : ClInstrumentWithCalls; InstrumentStack = UsePageAliases ? false : ClInstrumentStack; @@ -501,6 +526,8 @@ Int32Ty = IRB.getInt32Ty(); HwasanCtorFunction = nullptr; + kPointerTagShift = getTargetTagShift(); + TagMaskByte = getTargetTagMask(); // Older versions of Android do not have the required runtime support for // short granules, global or personality function instrumentation. On other @@ -711,9 +738,6 @@ } void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) { - if (TargetTriple.isAArch64() || TargetTriple.getArch() == Triple::x86_64) - return; - IRBuilder<> IRB(I); Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy); Value *UntaggedPtr = @@ -872,7 +896,8 @@ {IRB.CreatePointerCast(Addr, IntptrTy), ConstantInt::get(IntptrTy, O.TypeSize / 8)}); } - untagPointerOperand(O.getInsn(), Addr); + if (ClUntagPointer) + untagPointerOperand(O.getInsn(), Addr); return true; } @@ -923,7 +948,7 @@ return true; } -static unsigned RetagMask(unsigned AllocaNo) { +static unsigned retagMaskArmv8(unsigned AllocaNo) { // A list of 8-bit numbers that have at most one run of non-zero bits. // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these // masks. @@ -941,6 +966,29 @@ return FastMasks[AllocaNo % (sizeof(FastMasks) / sizeof(FastMasks[0]))]; } +static unsigned retagX86_64(unsigned AllocaNo, uint64_t TagMaskByte) { + return AllocaNo & TagMaskByte; +} + +static unsigned retagMask(unsigned AllocaNo, Triple TargetTriple, + uint64_t TagMaskByte) { + if (TargetTriple.getArch() == Triple::x86_64) + return retagX86_64(AllocaNo, TagMaskByte); + return retagMaskArmv8(AllocaNo); +} + +// The whole pass imply the Tag is 8-bits size, because it is first +// implemented by aarch64 whose tag is happen 1 byte. We should diff +// it frome other targets now. +Value *HWAddressSanitizer::retagTargetTag(IRBuilder<> &IRB, Value *OldTag) { + if (TargetTriple.getArch() == Triple::x86_64) { + Constant *TagMask = ConstantInt::get(IntptrTy, TagMaskByte); + Value *NewTag = IRB.CreateAnd(OldTag, TagMask); + return NewTag; + } else + return OldTag; +} + Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) { return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy); } @@ -966,6 +1014,9 @@ Value *StackTag = IRB.CreateXor(StackPointerLong, IRB.CreateLShr(StackPointerLong, 20), "hwasan.stack.base.tag"); + + StackTag = retagTargetTag(IRB, StackTag); + return StackTag; } @@ -973,34 +1024,51 @@ AllocaInst *AI, unsigned AllocaNo) { if (ClGenerateTagsWithCalls) return getNextTagWithCall(IRB); - return IRB.CreateXor(StackTag, - ConstantInt::get(IntptrTy, RetagMask(AllocaNo))); + + Value *Tag = IRB.CreateXor( + StackTag, ConstantInt::get( + IntptrTy, retagMask(AllocaNo, TargetTriple, TagMaskByte))); + return retagTargetTag(IRB, Tag); } Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) { if (ClUARRetagToZero) return ConstantInt::get(IntptrTy, 0); + if (ClGenerateTagsWithCalls) return getNextTagWithCall(IRB); - return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU)); + + Value *Tag = IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU)); + return retagTargetTag(IRB, Tag); } -// Add a tag to an address. -Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty, - Value *PtrLong, Value *Tag) { - assert(!UsePageAliases); +Value *HWAddressSanitizer::getTargetTagPtr(IRBuilder<> &IRB, Value *PtrLong, + Value *Tag) { Value *TaggedPtrLong; + Value *Mask; + Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift); + // Kernel addresses have 0xFF in the most significant byte. if (CompileKernel) { - // Kernel addresses have 0xFF in the most significant byte. - Value *ShiftedTag = IRB.CreateOr( - IRB.CreateShl(Tag, kPointerTagShift), - ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1)); + // X86_64 can not tag the highest bit. + if (TargetTriple.getArch() == Triple::x86_64) + Mask = ConstantInt::get(IntptrTy, ((1ULL << kPointerTagShift) - 1) | + (1ULL << (64 - 1))); + else + Mask = ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1); + ShiftedTag = IRB.CreateOr(ShiftedTag, Mask); TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag); } else { - // Userspace can simply do OR (tag << 56); - Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift); + // Userspace can simply do OR (tag << kPointerTagShift); TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag); } + return TaggedPtrLong; +} + +// Add a tag to an address. +Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty, + Value *PtrLong, Value *Tag) { + assert(!UsePageAliases); + Value *TaggedPtrLong = getTargetTagPtr(IRB, PtrLong, Tag); return IRB.CreateIntToPtr(TaggedPtrLong, Ty); } @@ -1166,8 +1234,9 @@ // Prepend "tag_offset, N" to the dwarf expression. // Tag offset logically applies to the alloca pointer, and it makes sense // to put it at the beginning of the expression. - SmallVector NewOps = {dwarf::DW_OP_LLVM_tag_offset, - RetagMask(N)}; + SmallVector NewOps = { + dwarf::DW_OP_LLVM_tag_offset, + retagMask(N, TargetTriple, TagMaskByte)}; auto Locations = DDI->location_ops(); unsigned LocNo = std::distance(Locations.begin(), find(Locations, AI)); DDI->setExpression( @@ -1462,8 +1531,10 @@ for (GlobalVariable *GV : Globals) { // Skip tag 0 in order to avoid collisions with untagged memory. + Tag &= TagMaskByte; if (Tag == 0) Tag = 1; + instrumentGlobal(GV, Tag++); } } Index: llvm/test/Instrumentation/HWAddressSanitizer/X86/stack.ll =================================================================== --- /dev/null +++ llvm/test/Instrumentation/HWAddressSanitizer/X86/stack.ll @@ -0,0 +1,98 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -hwasan -hwasan-instrument-with-calls -hwasan-instrument-stack -hwasan-use-page-alias=false -S | FileCheck %s +; RUN: opt < %s -hwasan -hwasan-instrument-with-calls -hwasan-instrument-stack -hwasan-use-page-alias=false -hwasan-untag-mem-operation=true -S | FileCheck --check-prefix=CHECK-UNTAG %s + +source_filename = "stack.c" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: noinline nounwind optnone uwtable +define dso_local i32 @main() sanitize_hwaddress { +; CHECK-LABEL: @main( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[DOTHWASAN_SHADOW:%.*]] = call i8* asm "", "=r,0"(i8* null) +; CHECK-NEXT: [[TMP0:%.*]] = call i8* @llvm.frameaddress.p0i8(i32 0) +; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint i8* [[TMP0]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 20 +; CHECK-NEXT: [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]] +; CHECK-NEXT: [[TMP3:%.*]] = and i64 [[HWASAN_STACK_BASE_TAG]], 63 +; CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4 +; CHECK-NEXT: [[LV0:%.*]] = alloca { i32, [12 x i8] }, align 16 +; CHECK-NEXT: [[TMP4:%.*]] = bitcast { i32, [12 x i8] }* [[LV0]] to i32* +; CHECK-NEXT: [[TMP5:%.*]] = xor i64 [[TMP3]], 0 +; CHECK-NEXT: [[TMP6:%.*]] = and i64 [[TMP5]], 63 +; CHECK-NEXT: [[TMP7:%.*]] = ptrtoint i32* [[TMP4]] to i64 +; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[TMP6]], 57 +; CHECK-NEXT: [[TMP9:%.*]] = or i64 [[TMP7]], [[TMP8]] +; CHECK-NEXT: [[LV0_HWASAN:%.*]] = inttoptr i64 [[TMP9]] to i32* +; CHECK-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP6]] to i8 +; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP4]] to i8* +; CHECK-NEXT: call void @__hwasan_tag_memory(i8* [[TMP11]], i8 [[TMP10]], i64 16) +; CHECK-NEXT: [[TMP12:%.*]] = ptrtoint i32* [[RETVAL]] to i64 +; CHECK-NEXT: call void @__hwasan_store4(i64 [[TMP12]]) +; CHECK-NEXT: store i32 0, i32* [[RETVAL]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = ptrtoint i32* [[LV0_HWASAN]] to i64 +; CHECK-NEXT: call void @__hwasan_store4(i64 [[TMP13]]) +; CHECK-NEXT: store i32 12345, i32* [[LV0_HWASAN]], align 4 +; CHECK-NEXT: call void @foo(i32* [[LV0_HWASAN]]) +; CHECK-NEXT: [[TMP14:%.*]] = ptrtoint i32* [[LV0_HWASAN]] to i64 +; CHECK-NEXT: call void @__hwasan_load4(i64 [[TMP14]]) +; CHECK-NEXT: [[TMP15:%.*]] = load i32, i32* [[LV0_HWASAN]], align 4 +; CHECK-NEXT: [[TMP16:%.*]] = bitcast i32* [[TMP4]] to i8* +; CHECK-NEXT: call void @__hwasan_tag_memory(i8* [[TMP16]], i8 0, i64 16) +; CHECK-NEXT: ret i32 [[TMP15]] +; +; CHECK-UNTAG-LABEL: @main( +; CHECK-UNTAG-NEXT: entry: +; CHECK-UNTAG-NEXT: [[DOTHWASAN_SHADOW:%.*]] = call i8* asm "", "=r,0"(i8* null) +; CHECK-UNTAG-NEXT: [[TMP0:%.*]] = call i8* @llvm.frameaddress.p0i8(i32 0) +; CHECK-UNTAG-NEXT: [[TMP1:%.*]] = ptrtoint i8* [[TMP0]] to i64 +; CHECK-UNTAG-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 20 +; CHECK-UNTAG-NEXT: [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]] +; CHECK-UNTAG-NEXT: [[TMP3:%.*]] = and i64 [[HWASAN_STACK_BASE_TAG]], 63 +; CHECK-UNTAG-NEXT: [[RETVAL:%.*]] = alloca i32, align 4 +; CHECK-UNTAG-NEXT: [[LV0:%.*]] = alloca { i32, [12 x i8] }, align 16 +; CHECK-UNTAG-NEXT: [[TMP4:%.*]] = bitcast { i32, [12 x i8] }* [[LV0]] to i32* +; CHECK-UNTAG-NEXT: [[TMP5:%.*]] = xor i64 [[TMP3]], 0 +; CHECK-UNTAG-NEXT: [[TMP6:%.*]] = and i64 [[TMP5]], 63 +; CHECK-UNTAG-NEXT: [[TMP7:%.*]] = ptrtoint i32* [[TMP4]] to i64 +; CHECK-UNTAG-NEXT: [[TMP8:%.*]] = shl i64 [[TMP6]], 57 +; CHECK-UNTAG-NEXT: [[TMP9:%.*]] = or i64 [[TMP7]], [[TMP8]] +; CHECK-UNTAG-NEXT: [[LV0_HWASAN:%.*]] = inttoptr i64 [[TMP9]] to i32* +; CHECK-UNTAG-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP6]] to i8 +; CHECK-UNTAG-NEXT: [[TMP11:%.*]] = bitcast i32* [[TMP4]] to i8* +; CHECK-UNTAG-NEXT: call void @__hwasan_tag_memory(i8* [[TMP11]], i8 [[TMP10]], i64 16) +; CHECK-UNTAG-NEXT: [[TMP12:%.*]] = ptrtoint i32* [[RETVAL]] to i64 +; CHECK-UNTAG-NEXT: call void @__hwasan_store4(i64 [[TMP12]]) +; CHECK-UNTAG-NEXT: [[TMP13:%.*]] = ptrtoint i32* [[RETVAL]] to i64 +; CHECK-UNTAG-NEXT: [[TMP14:%.*]] = and i64 [[TMP13]], 144115188075855871 +; CHECK-UNTAG-NEXT: [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to i32* +; CHECK-UNTAG-NEXT: store i32 0, i32* [[TMP15]], align 4 +; CHECK-UNTAG-NEXT: [[TMP16:%.*]] = ptrtoint i32* [[LV0_HWASAN]] to i64 +; CHECK-UNTAG-NEXT: call void @__hwasan_store4(i64 [[TMP16]]) +; CHECK-UNTAG-NEXT: [[TMP17:%.*]] = ptrtoint i32* [[LV0_HWASAN]] to i64 +; CHECK-UNTAG-NEXT: [[TMP18:%.*]] = and i64 [[TMP17]], 144115188075855871 +; CHECK-UNTAG-NEXT: [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to i32* +; CHECK-UNTAG-NEXT: store i32 12345, i32* [[TMP19]], align 4 +; CHECK-UNTAG-NEXT: call void @foo(i32* [[LV0_HWASAN]]) +; CHECK-UNTAG-NEXT: [[TMP20:%.*]] = ptrtoint i32* [[LV0_HWASAN]] to i64 +; CHECK-UNTAG-NEXT: call void @__hwasan_load4(i64 [[TMP20]]) +; CHECK-UNTAG-NEXT: [[TMP21:%.*]] = ptrtoint i32* [[LV0_HWASAN]] to i64 +; CHECK-UNTAG-NEXT: [[TMP22:%.*]] = and i64 [[TMP21]], 144115188075855871 +; CHECK-UNTAG-NEXT: [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to i32* +; CHECK-UNTAG-NEXT: [[TMP24:%.*]] = load i32, i32* [[TMP23]], align 4 +; CHECK-UNTAG-NEXT: [[TMP25:%.*]] = bitcast i32* [[TMP4]] to i8* +; CHECK-UNTAG-NEXT: call void @__hwasan_tag_memory(i8* [[TMP25]], i8 0, i64 16) +; CHECK-UNTAG-NEXT: ret i32 [[TMP24]] +; +entry: + %retval = alloca i32, align 4 + %lv0 = alloca i32, align 4 + store i32 0, i32* %retval, align 4 + store i32 12345, i32* %lv0, align 4 + call void @foo(i32* %lv0) + %0 = load i32, i32* %lv0, align 4 + ret i32 %0 +} + +declare dso_local void @foo(i32*)