diff --git a/compiler-rt/lib/asan/asan_interface.inc b/compiler-rt/lib/asan/asan_interface.inc --- a/compiler-rt/lib/asan/asan_interface.inc +++ b/compiler-rt/lib/asan/asan_interface.inc @@ -120,6 +120,19 @@ INTERFACE_FUNCTION(__asan_set_shadow_f3) INTERFACE_FUNCTION(__asan_set_shadow_f5) INTERFACE_FUNCTION(__asan_set_shadow_f8) +INTERFACE_FUNCTION(__asan_set_shadow2_00) +INTERFACE_FUNCTION(__asan_set_shadow2_01) +INTERFACE_FUNCTION(__asan_set_shadow2_02) +INTERFACE_FUNCTION(__asan_set_shadow2_03) +INTERFACE_FUNCTION(__asan_set_shadow2_04) +INTERFACE_FUNCTION(__asan_set_shadow2_05) +INTERFACE_FUNCTION(__asan_set_shadow2_06) +INTERFACE_FUNCTION(__asan_set_shadow2_07) +INTERFACE_FUNCTION(__asan_set_shadow2_f1) +INTERFACE_FUNCTION(__asan_set_shadow2_f2) +INTERFACE_FUNCTION(__asan_set_shadow2_f3) +INTERFACE_FUNCTION(__asan_set_shadow2_f5) +INTERFACE_FUNCTION(__asan_set_shadow2_f8) INTERFACE_FUNCTION(__asan_stack_free_0) INTERFACE_FUNCTION(__asan_stack_free_1) INTERFACE_FUNCTION(__asan_stack_free_2) diff --git a/compiler-rt/lib/asan/asan_interface_internal.h b/compiler-rt/lib/asan/asan_interface_internal.h --- a/compiler-rt/lib/asan/asan_interface_internal.h +++ b/compiler-rt/lib/asan/asan_interface_internal.h @@ -115,6 +115,33 @@ SANITIZER_INTERFACE_ATTRIBUTE void __asan_set_shadow_f8(uptr addr, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE + void __asan_set_shadow2_00(uptr addr, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE + void __asan_set_shadow2_01(uptr addr, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE + void __asan_set_shadow2_02(uptr addr, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE + void __asan_set_shadow2_03(uptr addr, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE + void __asan_set_shadow2_04(uptr addr, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE + void __asan_set_shadow2_05(uptr addr, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE + void __asan_set_shadow2_06(uptr addr, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE + void __asan_set_shadow2_07(uptr addr, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE + void __asan_set_shadow2_f1(uptr addr, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE + void __asan_set_shadow2_f2(uptr addr, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE + void __asan_set_shadow2_f3(uptr addr, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE + void __asan_set_shadow2_f5(uptr addr, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE + void __asan_set_shadow2_f8(uptr addr, uptr size); + // These two functions are used by instrumented code in the // use-after-scope mode. They mark memory for local variables as // unaddressable when they leave scope and addressable before the diff --git a/compiler-rt/lib/asan/asan_poisoning.cpp b/compiler-rt/lib/asan/asan_poisoning.cpp --- a/compiler-rt/lib/asan/asan_poisoning.cpp +++ b/compiler-rt/lib/asan/asan_poisoning.cpp @@ -304,6 +304,71 @@ } } +void __asan_set_shadow2_00(uptr addr, uptr size) { + addr += ASAN_SHADOW_OFFSET; + REAL(memset)((void *)addr, 0, size); +} + +void __asan_set_shadow2_01(uptr addr, uptr size) { + addr += ASAN_SHADOW_OFFSET; + REAL(memset)((void *)addr, 0x01, size); +} + +void __asan_set_shadow2_02(uptr addr, uptr size) { + addr += ASAN_SHADOW_OFFSET; + REAL(memset)((void *)addr, 0x02, size); +} + +void __asan_set_shadow2_03(uptr addr, uptr size) { + addr += ASAN_SHADOW_OFFSET; + REAL(memset)((void *)addr, 0x03, size); +} + +void __asan_set_shadow2_04(uptr addr, uptr size) { + addr += ASAN_SHADOW_OFFSET; + REAL(memset)((void *)addr, 0x04, size); +} + +void __asan_set_shadow2_05(uptr addr, uptr size) { + addr += ASAN_SHADOW_OFFSET; + REAL(memset)((void *)addr, 0x05, size); +} + +void __asan_set_shadow2_06(uptr addr, uptr size) { + addr += ASAN_SHADOW_OFFSET; + REAL(memset)((void *)addr, 0x06, size); +} + +void __asan_set_shadow2_07(uptr addr, uptr size) { + addr += ASAN_SHADOW_OFFSET; + REAL(memset)((void *)addr, 0x07, size); +} + +void __asan_set_shadow2_f1(uptr addr, uptr size) { + addr += ASAN_SHADOW_OFFSET; + REAL(memset)((void *)addr, 0xf1, size); +} + +void __asan_set_shadow2_f2(uptr addr, uptr size) { + addr += ASAN_SHADOW_OFFSET; + REAL(memset)((void *)addr, 0xf2, size); +} + +void __asan_set_shadow2_f3(uptr addr, uptr size) { + addr += ASAN_SHADOW_OFFSET; + REAL(memset)((void *)addr, 0xf3, size); +} + +void __asan_set_shadow2_f5(uptr addr, uptr size) { + addr += ASAN_SHADOW_OFFSET; + REAL(memset)((void *)addr, 0xf5, size); +} + +void __asan_set_shadow2_f8(uptr addr, uptr size) { + addr += ASAN_SHADOW_OFFSET; + REAL(memset)((void *)addr, 0xf8, size); +} + void __asan_set_shadow_00(uptr addr, uptr size) { REAL(memset)((void *)addr, 0, size); } diff --git a/compiler-rt/lib/asan/asan_rtl.cpp b/compiler-rt/lib/asan/asan_rtl.cpp --- a/compiler-rt/lib/asan/asan_rtl.cpp +++ b/compiler-rt/lib/asan/asan_rtl.cpp @@ -300,6 +300,19 @@ case 50: __asan_set_shadow_f3(0, 0); break; case 51: __asan_set_shadow_f5(0, 0); break; case 52: __asan_set_shadow_f8(0, 0); break; + case 140: __asan_set_shadow2_00(0, 0); break; + case 141: __asan_set_shadow2_01(0, 0); break; + case 142: __asan_set_shadow2_02(0, 0); break; + case 143: __asan_set_shadow2_03(0, 0); break; + case 144: __asan_set_shadow2_04(0, 0); break; + case 145: __asan_set_shadow2_05(0, 0); break; + case 146: __asan_set_shadow2_06(0, 0); break; + case 147: __asan_set_shadow2_07(0, 0); break; + case 148: __asan_set_shadow2_f1(0, 0); break; + case 149: __asan_set_shadow2_f2(0, 0); break; + case 150: __asan_set_shadow2_f3(0, 0); break; + case 151: __asan_set_shadow2_f5(0, 0); break; + case 152: __asan_set_shadow2_f8(0, 0); break; } // clang-format on } diff --git a/compiler-rt/lib/asan/asan_rtl_x86_64.S b/compiler-rt/lib/asan/asan_rtl_x86_64.S --- a/compiler-rt/lib/asan/asan_rtl_x86_64.S +++ b/compiler-rt/lib/asan/asan_rtl_x86_64.S @@ -27,7 +27,7 @@ #define ASAN_MEMORY_ACCESS_INITIAL_CHECK_ADD(reg, op, s) \ mov %##reg,%r10 ;\ shr $0x3,%r10 ;\ - movsbl ASAN_SHADOW_OFFSET_CONST(%r10),%r10d ;\ + movsbl %gs:(%r10),%r10d ;\ test %r10d,%r10d ;\ jne CLABEL(reg, op, s, add) ;\ RLABEL(reg, op, s, add): ;\ @@ -35,35 +35,29 @@ #define ASAN_MEMORY_ACCESS_EXTRA_CHECK_1(reg, op, i) \ CLABEL(reg, op, 1, i): ;\ - push %rcx ;\ - mov %##reg,%rcx ;\ - and $0x7,%ecx ;\ - cmp %r10d,%ecx ;\ - pop %rcx ;\ + mov %##reg,%r11 ;\ + and $0x7,%r11d ;\ + cmp %r10d,%r11d ;\ jl RLABEL(reg, op, 1, i);\ mov %##reg,%rdi ;\ jmp __asan_report_##op##1_asm ;\ #define ASAN_MEMORY_ACCESS_EXTRA_CHECK_2(reg, op, i) \ CLABEL(reg, op, 2, i): ;\ - push %rcx ;\ - mov %##reg,%rcx ;\ - and $0x7,%ecx ;\ - add $0x1,%ecx ;\ - cmp %r10d,%ecx ;\ - pop %rcx ;\ + mov %##reg,%r11 ;\ + and $0x7,%r11d ;\ + add $0x1,%r11d ;\ + cmp %r10d,%r11d ;\ jl RLABEL(reg, op, 2, i);\ mov %##reg,%rdi ;\ jmp __asan_report_##op##2_asm ;\ #define ASAN_MEMORY_ACCESS_EXTRA_CHECK_4(reg, op, i) \ CLABEL(reg, op, 4, i): ;\ - push %rcx ;\ - mov %##reg,%rcx ;\ - and $0x7,%ecx ;\ - add $0x3,%ecx ;\ - cmp %r10d,%ecx ;\ - pop %rcx ;\ + mov %##reg,%r11 ;\ + and $0x7,%r11d ;\ + add $0x3,%r11d ;\ + cmp %r10d,%r11d ;\ jl RLABEL(reg, op, 4, i);\ mov %##reg,%rdi ;\ jmp __asan_report_##op##4_asm ;\ diff --git a/compiler-rt/lib/asan/asan_shadow_setup.cpp b/compiler-rt/lib/asan/asan_shadow_setup.cpp --- a/compiler-rt/lib/asan/asan_shadow_setup.cpp +++ b/compiler-rt/lib/asan/asan_shadow_setup.cpp @@ -19,6 +19,12 @@ # include "asan_internal.h" # include "asan_mapping.h" +# if SANITIZER_X64 +# include +# include +# include + +# endif namespace __asan { static void ProtectGap(uptr addr, uptr size) { @@ -52,6 +58,18 @@ #endif } +# if SANITIZER_X64 +__attribute__((destructor)) void __chktag_count_report(void) { + void *gs = nullptr; + syscall(SYS_arch_prctl, ARCH_GET_GS, &gs); + + if ((uptr)gs != __asan_shadow_memory_dynamic_address) { + Report("GS %p\n", gs); + Die(); + } +} +# endif + void InitializeShadowMemory() { // Set the shadow memory address to uninitialized. __asan_shadow_memory_dynamic_address = kDefaultShadowSentinel; @@ -68,7 +86,10 @@ } // Update the shadow memory address (potentially) used by instrumentation. __asan_shadow_memory_dynamic_address = shadow_start; - +# if SANITIZER_X64 + syscall(SYS_arch_prctl, ARCH_SET_GS, + (void *)__asan_shadow_memory_dynamic_address); +# endif if (kLowShadowBeg) shadow_start -= GetMmapGranularity(); if (!full_shadow_is_available) diff --git a/compiler-rt/lib/asan/weak_symbols.txt b/compiler-rt/lib/asan/weak_symbols.txt --- a/compiler-rt/lib/asan/weak_symbols.txt +++ b/compiler-rt/lib/asan/weak_symbols.txt @@ -17,3 +17,19 @@ ___asan_set_shadow_f6 ___asan_set_shadow_f7 ___asan_set_shadow_f8 +___asan_set_shadow2_00 +___asan_set_shadow2_01 +___asan_set_shadow2_02 +___asan_set_shadow2_03 +___asan_set_shadow2_04 +___asan_set_shadow2_05 +___asan_set_shadow2_06 +___asan_set_shadow2_07 +___asan_set_shadow2_f1 +___asan_set_shadow2_f2 +___asan_set_shadow2_f3 +___asan_set_shadow2_f4 +___asan_set_shadow2_f5 +___asan_set_shadow2_f6 +___asan_set_shadow2_f7 +___asan_set_shadow2_f8 \ No newline at end of file diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -155,7 +155,7 @@ const char kAsanGenPrefix[] = "___asan_gen_"; const char kODRGenPrefix[] = "__odr_asan_gen_"; const char kSanCovGenPrefix[] = "__sancov_gen_"; -const char kAsanSetShadowPrefix[] = "__asan_set_shadow_"; +const char kAsanSetShadowPrefix[] = "__asan_set_shadow2_"; const char kAsanPoisonStackMemoryName[] = "__asan_poison_stack_memory"; const char kAsanUnpoisonStackMemoryName[] = "__asan_unpoison_stack_memory"; @@ -710,6 +710,7 @@ Value *SizeArgument, uint32_t Exp); void instrumentMemIntrinsic(MemIntrinsic *MI); Value *memToShadow(Value *Shadow, IRBuilder<> &IRB); + Value *memToShadowGS(Value *Shadow, IRBuilder<> &IRB); bool suppressInstrumentationSiteForDebug(int &Instrumented); bool instrumentFunction(Function &F, const TargetLibraryInfo *TLI); bool maybeInsertAsanInitAtFunctionEntry(Function &F); @@ -1228,6 +1229,11 @@ return IRB.CreateAdd(Shadow, ShadowBase); } +Value *AddressSanitizer::memToShadowGS(Value *Shadow, IRBuilder<> &IRB) { + Shadow = IRB.CreateLShr(Shadow, Mapping.Scale); + return Shadow; +} + // Instrument memset/memmove/memcpy void AddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) { InstrumentationIRBuilder IRB(MI); @@ -1722,8 +1728,30 @@ InstrumentationIRBuilder IRB(InsertBefore); size_t AccessSizeIndex = TypeStoreSizeToSizeIndex(TypeStoreSize); const ASanAccessInfo AccessInfo(IsWrite, CompileKernel, AccessSizeIndex); + + const size_t Granularity = 1ULL << Mapping.Scale; if (UseCalls && ClOptimizeCallbacks) { + + if ((TypeStoreSize < 8 * Granularity)) { + Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy); + Type *ShadowTy = + IntegerType::get(*C, std::max(8U, TypeStoreSize >> Mapping.Scale)); + Type *ShadowPtrTy = PointerType::get(ShadowTy, 256); + Value *ShadowPtr = memToShadowGS(AddrLong, IRB); + const uint64_t ShadowAlign = std::max( + Alignment.valueOrOne().value() >> Mapping.Scale, 1); + Value *ShadowValue = IRB.CreateAlignedLoad( + ShadowTy, IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy), + Align(ShadowAlign)); + Value *Cmp = IRB.CreateIsNotNull(ShadowValue); + + Instruction *CrashTerm = SplitBlockAndInsertIfThen( + Cmp, InsertBefore, false, + MDBuilder(*C).createBranchWeights(1, 100000)); + IRB.SetInsertPoint(CrashTerm); + } + const ASanAccessInfo AccessInfo(IsWrite, CompileKernel, AccessSizeIndex); Module *M = IRB.GetInsertBlock()->getParent()->getParent(); IRB.CreateCall( @@ -1746,15 +1774,14 @@ Type *ShadowTy = IntegerType::get(*C, std::max(8U, TypeStoreSize >> Mapping.Scale)); - Type *ShadowPtrTy = PointerType::get(ShadowTy, 0); - Value *ShadowPtr = memToShadow(AddrLong, IRB); + Type *ShadowPtrTy = PointerType::get(ShadowTy, 256); + Value *ShadowPtr = memToShadowGS(AddrLong, IRB); const uint64_t ShadowAlign = std::max(Alignment.valueOrOne().value() >> Mapping.Scale, 1); Value *ShadowValue = IRB.CreateAlignedLoad( ShadowTy, IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy), Align(ShadowAlign)); Value *Cmp = IRB.CreateIsNotNull(ShadowValue); - size_t Granularity = 1ULL << Mapping.Scale; Instruction *CrashTerm = nullptr; if (ClAlwaysSlowPath || (TypeStoreSize < 8 * Granularity)) { @@ -3034,7 +3061,7 @@ Value *Ptr = IRB.CreateAdd(ShadowBase, ConstantInt::get(IntptrTy, i)); Value *Poison = IRB.getIntN(StoreSizeInBytes * 8, Val); IRB.CreateAlignedStore( - Poison, IRB.CreateIntToPtr(Ptr, Poison->getType()->getPointerTo()), + Poison, IRB.CreateIntToPtr(Ptr, Poison->getType()->getPointerTo(256)), Align(1)); i += StoreSizeInBytes; @@ -3438,7 +3465,7 @@ const auto &ShadowAfterScope = GetShadowBytesAfterScope(SVD, L); // Poison the stack red zones at the entry. - Value *ShadowBase = ASan.memToShadow(LocalStackBase, IRB); + Value *ShadowBase = ASan.memToShadowGS(LocalStackBase, IRB); // As mask we must use most poisoned case: red zones and after scope. // As bytes we can use either the same or just red zones only. copyToShadow(ShadowAfterScope, ShadowAfterScope, IRB, ShadowBase);