Index: compiler-rt/lib/asan/CMakeLists.txt =================================================================== --- compiler-rt/lib/asan/CMakeLists.txt +++ compiler-rt/lib/asan/CMakeLists.txt @@ -21,6 +21,7 @@ asan_memory_profile.cc asan_poisoning.cc asan_posix.cc + asan_premap_shadow.cc asan_report.cc asan_rtl.cc asan_shadow_setup.cc Index: compiler-rt/lib/asan/asan_init_version.h =================================================================== --- compiler-rt/lib/asan/asan_init_version.h +++ compiler-rt/lib/asan/asan_init_version.h @@ -15,6 +15,8 @@ #ifndef ASAN_INIT_VERSION_H #define ASAN_INIT_VERSION_H +#include "sanitizer_common/sanitizer_platform.h" + extern "C" { // Every time the ASan ABI changes we also change the version number in the // __asan_init function name. Objects built with incompatible ASan ABI @@ -32,7 +34,12 @@ // v6=>v7: added 'odr_indicator' to __asan_global // v7=>v8: added '__asan_(un)register_image_globals' functions for dead // stripping support on Mach-O platforms +#if SANITIZER_WORDSIZE == 32 && SANITIZER_ANDROID + // v8=>v9: 32-bit Android switched to dynamic shadow + #define __asan_version_mismatch_check __asan_version_mismatch_check_v9 +#else #define __asan_version_mismatch_check __asan_version_mismatch_check_v8 +#endif } #endif // ASAN_INIT_VERSION_H Index: compiler-rt/lib/asan/asan_linux.cc =================================================================== --- compiler-rt/lib/asan/asan_linux.cc +++ compiler-rt/lib/asan/asan_linux.cc @@ -17,6 +17,7 @@ #include "asan_interceptors.h" #include "asan_internal.h" +#include "asan_premap_shadow.h" #include "asan_thread.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_freebsd.h" @@ -81,10 +82,37 @@ return &_DYNAMIC; // defined in link.h } +#if ASAN_PREMAP_SHADOW uptr FindDynamicShadowStart() { - UNREACHABLE("FindDynamicShadowStart is not available"); - return 0; + uptr granularity = GetMmapGranularity(); + uptr shadow_start = reinterpret_cast(&__asan_shadow); + uptr shadow_size = PremapShadowSize(); + UnmapOrDie((void *)(shadow_start - granularity), shadow_size + granularity); + // MmapNoAccess does not touch TotalMmap, but UnmapOrDie decreases it. + // Compensate. + IncreaseTotalMmap(shadow_size + granularity); + return shadow_start; } +#else +uptr FindDynamicShadowStart() { + uptr granularity = GetMmapGranularity(); + uptr alignment = granularity * 8; + uptr left_padding = granularity; + uptr shadow_size = kHighShadowEnd + left_padding; + uptr map_size = shadow_size + alignment; + + uptr map_start = (uptr)MmapNoAccess(map_size); + CHECK_NE(map_start, ~(uptr)0); + + uptr shadow_start = RoundUpTo(map_start, alignment); + UnmapOrDie((void *)map_start, map_size); + // MmapNoAccess does not touch TotalMmap, but UnmapOrDie decreases it. + // Compensate. + IncreaseTotalMmap(map_size); + + return shadow_start; +} +#endif void AsanApplyToGlobals(globals_op_fptr op, const void *needle) { UNIMPLEMENTED(); Index: compiler-rt/lib/asan/asan_mapping.h =================================================================== --- compiler-rt/lib/asan/asan_mapping.h +++ compiler-rt/lib/asan/asan_mapping.h @@ -156,7 +156,7 @@ # define SHADOW_OFFSET (0) #elif SANITIZER_WORDSIZE == 32 # if SANITIZER_ANDROID -# define SHADOW_OFFSET (0) +# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address # elif defined(__mips__) # define SHADOW_OFFSET kMIPS32_ShadowOffset32 # elif SANITIZER_FREEBSD @@ -200,6 +200,12 @@ # endif #endif +#if SANITIZER_ANDROID && defined(__arm__) +# define ASAN_PREMAP_SHADOW 1 +#else +# define ASAN_PREMAP_SHADOW 0 +#endif + #define SHADOW_GRANULARITY (1ULL << SHADOW_SCALE) #define MEM_TO_SHADOW(mem) (((mem) >> SHADOW_SCALE) + (SHADOW_OFFSET)) Index: compiler-rt/lib/asan/asan_premap_shadow.h =================================================================== --- /dev/null +++ compiler-rt/lib/asan/asan_premap_shadow.h @@ -0,0 +1,28 @@ +//===-- asan_mapping.h ------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// Premap shadow range with an ifunc resolver. +//===----------------------------------------------------------------------===// + + +#ifndef ASAN_PREMAP_SHADOW_H +#define ASAN_PREMAP_SHADOW_H + +#if ASAN_PREMAP_SHADOW +namespace __asan { +// Conservative upper limit. +uptr PremapShadowSize(); +} +#endif + +extern "C" INTERFACE_ATTRIBUTE void __asan_shadow(); + +#endif // ASAN_PREMAP_SHADOW_H Index: compiler-rt/lib/asan/asan_premap_shadow.cc =================================================================== --- /dev/null +++ compiler-rt/lib/asan/asan_premap_shadow.cc @@ -0,0 +1,69 @@ +//===-- asan_premap_shadow.cc ---------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer, an address sanity checker. +// +// Reserve shadow memory with an ifunc resolver. +//===----------------------------------------------------------------------===// + +#include "asan_mapping.h" + +#if ASAN_PREMAP_SHADOW + +#include "asan_premap_shadow.h" +#include "sanitizer_common/sanitizer_posix.h" + +namespace __asan { + +// The code in this file needs to run in an unrelocated binary. It may not +// access any external symbol, including its own non-hidden globals. + +// Conservative upper limit. +uptr PremapShadowSize() { + return GetMaxVirtualAddress() >> SHADOW_SCALE; +} + +// Returns an address aligned to 8 pages, such that one page on the left and +// PremapShadowSize() bytes on the right of it are mapped r/o. +uptr PremapShadow() { + uptr granularity = GetMmapGranularity(); + uptr alignment = granularity * 8; + uptr left_padding = granularity; + uptr shadow_size = PremapShadowSize(); + uptr map_size = shadow_size + left_padding + alignment; + + uptr map_start = (uptr)MmapNoAccess(map_size); + CHECK_NE(map_start, ~(uptr)0); + + uptr shadow_start = RoundUpTo(map_start + left_padding, alignment); + uptr shadow_end = shadow_start + shadow_size; + internal_munmap(reinterpret_cast(map_start), + shadow_start - left_padding - map_start); + internal_munmap(reinterpret_cast(shadow_end), + map_start + map_size - shadow_end); + return shadow_start; +} + +} // namespace __asan + +extern "C" { +decltype(__asan_shadow)* __asan_premap_shadow() { + // The resolver may be called multiple times. Map the shadow just once. + static uptr premapped_shadow = 0; + if (!premapped_shadow) premapped_shadow = __asan::PremapShadow(); + return reinterpret_cast(premapped_shadow); +} + +// __asan_shadow is a "function" that has the same address as the first byte of +// the shadow mapping. +INTERFACE_ATTRIBUTE __attribute__((ifunc("__asan_premap_shadow"))) void +__asan_shadow(); +} + +#endif // ASAN_PREMAP_SHADOW Index: compiler-rt/lib/sanitizer_common/sanitizer_common.h =================================================================== --- compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -73,6 +73,7 @@ return PageSizeCached; } uptr GetMmapGranularity(); +uptr GetMaxVirtualAddress(); uptr GetMaxUserVirtualAddress(); // Threads tid_t GetTid(); Index: compiler-rt/lib/sanitizer_common/sanitizer_flags.h =================================================================== --- compiler-rt/lib/sanitizer_common/sanitizer_flags.h +++ compiler-rt/lib/sanitizer_common/sanitizer_flags.h @@ -34,7 +34,9 @@ }; // Functions to get/set global CommonFlags shared by all sanitizer runtimes: -extern CommonFlags common_flags_dont_use; +// FIXME: hidden needed for asan_premap_shadow. Consider building with +// -fvisibility=hidden. +extern CommonFlags common_flags_dont_use __attribute__((visibility("hidden"))); inline const CommonFlags *common_flags() { return &common_flags_dont_use; } Index: compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc =================================================================== --- compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc +++ compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cc @@ -191,6 +191,10 @@ return ShadowBounds.memory_limit - 1; } +uptr GetMaxVirtualAddress() { + return GetMaxUserVirtualAddress(); +} + static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type, bool raw_report, bool die_for_nomem) { size = RoundUpTo(size, PAGE_SIZE); Index: compiler-rt/lib/sanitizer_common/sanitizer_linux.cc =================================================================== --- compiler-rt/lib/sanitizer_common/sanitizer_linux.cc +++ compiler-rt/lib/sanitizer_common/sanitizer_linux.cc @@ -954,7 +954,7 @@ } #endif // SANITIZER_WORDSIZE == 32 -uptr GetMaxUserVirtualAddress() { +uptr GetMaxVirtualAddress() { #if SANITIZER_NETBSD && defined(__x86_64__) return 0x7f7ffffff000ULL; // (0x00007f8000000000 - PAGE_SIZE) #elif SANITIZER_WORDSIZE == 64 @@ -978,15 +978,21 @@ # if defined(__s390__) return (1ULL << 31) - 1; // 0x7fffffff; # else - uptr res = (1ULL << 32) - 1; // 0xffffffff; - if (!common_flags()->full_address_space) - res -= GetKernelAreaSize(); - CHECK_LT(reinterpret_cast(&res), res); - return res; + return (1ULL << 32) - 1; // 0xffffffff; # endif #endif // SANITIZER_WORDSIZE } +uptr GetMaxUserVirtualAddress() { + uptr addr = GetMaxVirtualAddress(); +#if SANITIZER_WORDSIZE == 32 && !defined(__s390__) + if (!common_flags()->full_address_space) + addr -= GetKernelAreaSize(); + CHECK_LT(reinterpret_cast(&addr), addr); +#endif + return addr; +} + uptr GetPageSize() { // Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array. #if SANITIZER_ANDROID Index: compiler-rt/lib/sanitizer_common/sanitizer_mac.cc =================================================================== --- compiler-rt/lib/sanitizer_common/sanitizer_mac.cc +++ compiler-rt/lib/sanitizer_common/sanitizer_mac.cc @@ -864,6 +864,10 @@ #endif // SANITIZER_WORDSIZE } +uptr GetMaxVirtualAddress() { + return GetMaxUserVirtualAddress(); +} + uptr FindAvailableMemoryRange(uptr shadow_size, uptr alignment, uptr left_padding, Index: compiler-rt/lib/sanitizer_common/sanitizer_win.cc =================================================================== --- compiler-rt/lib/sanitizer_common/sanitizer_win.cc +++ compiler-rt/lib/sanitizer_common/sanitizer_win.cc @@ -70,6 +70,10 @@ return (uptr)si.lpMaximumApplicationAddress; } +uptr GetMaxVirtualAddress() { + return GetMaxUserVirtualAddress(); +} + bool FileExists(const char *filename) { return ::GetFileAttributesA(filename) != INVALID_FILE_ATTRIBUTES; } Index: llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp =================================================================== --- llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -136,8 +136,8 @@ static const char *const kAsanPoisonGlobalsName = "__asan_before_dynamic_init"; static const char *const kAsanUnpoisonGlobalsName = "__asan_after_dynamic_init"; static const char *const kAsanInitName = "__asan_init"; -static const char *const kAsanVersionCheckName = - "__asan_version_mismatch_check_v8"; +static const char *const kAsanVersionCheckNamePrefix = + "__asan_version_mismatch_check_v"; static const char *const kAsanPtrCmp = "__sanitizer_ptr_cmp"; static const char *const kAsanPtrSub = "__sanitizer_ptr_sub"; static const char *const kAsanHandleNoReturnName = "__asan_handle_no_return"; @@ -207,6 +207,12 @@ cl::desc("Load shadow address into a local variable for each function"), cl::Hidden, cl::init(false)); +static cl::opt + ClWithIfunc("asan-with-ifunc", + cl::desc("Access dynamic shadow through an ifunc global on " + "platforms that support this"), + cl::Hidden, cl::init(false)); + // This flag limits the number of instructions to be instrumented // in any given BB. Normally, this should be set to unlimited (INT_MAX), // but due to http://llvm.org/bugs/show_bug.cgi?id=12652 we temporary @@ -447,10 +453,14 @@ /// This struct defines the shadow mapping using the rule: /// shadow = (mem >> Scale) ADD-or-OR Offset. +/// If InGlobal is true, then +/// extern char __asan_shadow[]; +/// shadow = (mem >> Scale) + &__asan_shadow struct ShadowMapping { int Scale; uint64_t Offset; bool OrShadowOffset; + bool InGlobal; }; } // end anonymous namespace @@ -472,6 +482,7 @@ TargetTriple.getArch() == Triple::mipsel; bool IsMIPS64 = TargetTriple.getArch() == Triple::mips64 || TargetTriple.getArch() == Triple::mips64el; + bool IsArmOrThumb = TargetTriple.isARM() || TargetTriple.isThumb(); bool IsAArch64 = TargetTriple.getArch() == Triple::aarch64; bool IsWindows = TargetTriple.isOSWindows(); bool IsFuchsia = TargetTriple.isOSFuchsia(); @@ -479,10 +490,8 @@ ShadowMapping Mapping; if (LongSize == 32) { - // Android is always PIE, which means that the beginning of the address - // space is always available. if (IsAndroid) - Mapping.Offset = 0; + Mapping.Offset = kDynamicShadowSentinel; else if (IsMIPS32) Mapping.Offset = kMIPS32_ShadowOffset32; else if (IsFreeBSD) @@ -550,6 +559,7 @@ Mapping.OrShadowOffset = !IsAArch64 && !IsPPC64 && !IsSystemZ && !IsPS4CPU && !(Mapping.Offset & (Mapping.Offset - 1)) && Mapping.Offset != kDynamicShadowSentinel; + Mapping.InGlobal = ClWithIfunc && IsAndroid && IsArmOrThumb; return Mapping; } @@ -672,6 +682,7 @@ DominatorTree *DT; Function *AsanHandleNoReturnFunc; Function *AsanPtrCmpFunction, *AsanPtrSubFunction; + Constant *AsanShadowGlobal; // These arrays is indexed by AccessIsWrite, Experiment and log2(AccessSize). Function *AsanErrorCallback[2][2][kNumberOfAccessSizes]; @@ -744,6 +755,7 @@ size_t MinRedzoneSizeForGlobal() const { return RedzoneSizeForScale(Mapping.Scale); } + int GetAsanVersion(const Module &M) const; GlobalsMetadata GlobalsMD; bool CompileKernel; @@ -1109,6 +1121,11 @@ if (Mapping.Offset == 0) return Shadow; // (Shadow >> scale) | offset Value *ShadowBase; + if (Mapping.InGlobal) + return IRB.CreatePtrToInt( + IRB.CreateGEP(AsanShadowGlobal, + {ConstantInt::get(IntptrTy, 0), Shadow}), + IntptrTy); if (LocalDynamicShadow) ShadowBase = LocalDynamicShadow; else @@ -2156,6 +2173,16 @@ return true; } +int AddressSanitizerModule::GetAsanVersion(const Module &M) const { + int LongSize = M.getDataLayout().getPointerSizeInBits(); + bool isAndroid = Triple(M.getTargetTriple()).isAndroid(); + int Version = 8; + // 32-bit Android is one version ahead because of the switch to dynamic + // shadow. + Version += (LongSize == 32 && isAndroid); + return Version; +} + bool AddressSanitizerModule::runOnModule(Module &M) { C = &(M.getContext()); int LongSize = M.getDataLayout().getPointerSizeInBits(); @@ -2169,9 +2196,11 @@ // Create a module constructor. A destructor is created lazily because not all // platforms, and not all modules need it. + std::string VersionCheckName = + kAsanVersionCheckNamePrefix + std::to_string(GetAsanVersion(M)); std::tie(AsanCtorFunction, std::ignore) = createSanitizerCtorAndInitFunctions( M, kAsanModuleCtorName, kAsanInitName, /*InitArgTypes=*/{}, - /*InitArgs=*/{}, kAsanVersionCheckName); + /*InitArgs=*/{}, VersionCheckName); bool CtorComdat = true; bool Changed = false; @@ -2270,6 +2299,9 @@ EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false), StringRef(""), StringRef(""), /*hasSideEffects=*/true); + if (Mapping.InGlobal) + AsanShadowGlobal = M.getOrInsertGlobal("__asan_shadow", + ArrayType::get(IRB.getInt8Ty(), 0)); } // virtual