Index: lib/asan/asan_rtl.cc =================================================================== --- lib/asan/asan_rtl.cc +++ lib/asan/asan_rtl.cc @@ -420,6 +420,8 @@ __asan_option_detect_stack_use_after_return = flags()->detect_stack_use_after_return; + __sanitizer::InitializePlatformEarly(); + // Re-exec ourselves if we need to set additional env or command line args. MaybeReexec(); Index: lib/asan/asan_shadow_setup.cc =================================================================== --- lib/asan/asan_shadow_setup.cc +++ lib/asan/asan_shadow_setup.cc @@ -31,7 +31,7 @@ CHECK_EQ(((end + 1) % GetMmapGranularity()), 0); uptr size = end - beg + 1; DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb. - if (!MmapFixedNoReserve(beg, size, name)) { + if (!MmapFixedNoReserveShadow(beg, size, name)) { Report( "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. " "Perhaps you're using ulimit -v\n", Index: lib/dfsan/dfsan.cc =================================================================== --- lib/dfsan/dfsan.cc +++ lib/dfsan/dfsan.cc @@ -425,7 +425,7 @@ InitializePlatformEarly(); - if (!MmapFixedNoReserve(ShadowAddr(), UnusedAddr() - ShadowAddr())) + if (!MmapFixedNoReserveShadow(ShadowAddr(), UnusedAddr() - ShadowAddr())) Die(); // Protect the region of memory we don't use, to preserve the one-to-one Index: lib/esan/esan.cpp =================================================================== --- lib/esan/esan.cpp +++ lib/esan/esan.cpp @@ -170,7 +170,8 @@ Map = (uptr)MmapFixedNoAccess(ShadowStart, ShadowEnd- ShadowStart, "shadow"); } else { - if (MmapFixedNoReserve(ShadowStart, ShadowEnd - ShadowStart, "shadow")) + if (MmapFixedNoReserveShadow(ShadowStart, ShadowEnd - ShadowStart, + "shadow")) Map = ShadowStart; } if (Map != ShadowStart) { Index: lib/hwasan/hwasan_linux.cc =================================================================== --- lib/hwasan/hwasan_linux.cc +++ lib/hwasan/hwasan_linux.cc @@ -44,7 +44,7 @@ CHECK_EQ(((end + 1) % GetMmapGranularity()), 0); uptr size = end - beg + 1; DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb. - if (!MmapFixedNoReserve(beg, size, name)) { + if (!MmapFixedNoReserveShadow(beg, size, name)) { Report( "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. " "Perhaps you're using ulimit -v\n", Index: lib/msan/msan_linux.cc =================================================================== --- lib/msan/msan_linux.cc +++ lib/msan/msan_linux.cc @@ -143,7 +143,7 @@ if (map) { if (!CheckMemoryRangeAvailability(start, size)) return false; - if (!MmapFixedNoReserve(start, size, kMemoryLayout[i].name)) + if (!MmapFixedNoReserveShadow(start, size, kMemoryLayout[i].name)) return false; if (common_flags()->use_madv_dontdump) DontDumpShadowMemory(start, size); Index: lib/msan/msan_poisoning.cc =================================================================== --- lib/msan/msan_poisoning.cc +++ lib/msan/msan_poisoning.cc @@ -139,7 +139,7 @@ if (page_end != shadow_end) { REAL(memset)((void *)page_end, 0, shadow_end - page_end); } - if (!MmapFixedNoReserve(page_beg, page_end - page_beg)) + if (!MmapFixedNoReserveShadow(page_beg, page_end - page_beg)) Die(); } } Index: lib/sanitizer_common/sanitizer_common.h =================================================================== --- lib/sanitizer_common/sanitizer_common.h +++ lib/sanitizer_common/sanitizer_common.h @@ -91,6 +91,8 @@ void *MmapOrDieOnFatalError(uptr size, const char *mem_type); bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name = nullptr) WARN_UNUSED_RESULT; +bool MmapFixedNoReserveShadow(uptr fixed_addr, uptr size, + const char *name = nullptr) WARN_UNUSED_RESULT; void *MmapNoReserveOrDie(uptr size, const char *mem_type); void *MmapFixedOrDie(uptr fixed_addr, uptr size); // Behaves just like MmapFixedOrDie, but tolerates out of memory condition, in @@ -897,6 +899,7 @@ bool IsMemoryAccess() const; }; +void InitializePlatformEarly(); void MaybeReexec(); template Index: lib/sanitizer_common/sanitizer_fuchsia.cc =================================================================== --- lib/sanitizer_common/sanitizer_fuchsia.cc +++ lib/sanitizer_common/sanitizer_fuchsia.cc @@ -89,6 +89,7 @@ *stack_top = *stack_bottom + size; } +void InitializePlatformEarly() {} void MaybeReexec() {} void CheckASLR() {} void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {} Index: lib/sanitizer_common/sanitizer_linux.cc =================================================================== --- lib/sanitizer_common/sanitizer_linux.cc +++ lib/sanitizer_common/sanitizer_linux.cc @@ -190,7 +190,11 @@ offset / 4096); #endif } -#endif // !SANITIZER_S390 && !SANITIZER_OPENBSD +uptr internal_mmap_shadow(void *addr, uptr length, int prot, int flags, int fd, + OFF_T offset) { + return internal_mmap(addr, length, prot, flags, fd, offset); +} +#endif // !SANITIZER_S390 && !SANITIZER_OPENBSD #if !SANITIZER_OPENBSD uptr internal_munmap(void *addr, uptr length) { @@ -1972,6 +1976,10 @@ void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); } +void InitializePlatformEarly() { + // Do nothing. +} + void MaybeReexec() { // No need to re-exec on Linux. } Index: lib/sanitizer_common/sanitizer_linux_s390.cc =================================================================== --- lib/sanitizer_common/sanitizer_linux_s390.cc +++ lib/sanitizer_common/sanitizer_linux_s390.cc @@ -55,6 +55,11 @@ # endif } +uptr internal_mmap_shadow(void *addr, uptr length, int prot, int flags, int fd, + OFF_T offset) { + return internal_mmap(addr, length, prot, flags, fd, offset); +} + uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, int *parent_tidptr, void *newtls, int *child_tidptr) { if (!fn || !child_stack) Index: lib/sanitizer_common/sanitizer_mac.h =================================================================== --- lib/sanitizer_common/sanitizer_mac.h +++ lib/sanitizer_common/sanitizer_mac.h @@ -40,6 +40,10 @@ MACOS_VERSION_MOUNTAIN_LION, MACOS_VERSION_MAVERICKS, MACOS_VERSION_YOSEMITE, + MACOS_VERSION_EL_CAPITAN, + MACOS_VERSION_SIERRA, + MACOS_VERSION_HIGH_SIERRA, + MACOS_VERSION_HIGH_SIERRA_DOT_RELEASE_4, MACOS_VERSION_UNKNOWN_NEWER }; Index: lib/sanitizer_common/sanitizer_mac.cc =================================================================== --- lib/sanitizer_common/sanitizer_mac.cc +++ lib/sanitizer_common/sanitizer_mac.cc @@ -115,6 +115,21 @@ return (uptr)mmap(addr, length, prot, flags, fd, offset); } +// XNU on Darwin provides a mmap flag that optimizes allocation/deallocation of +// giant memory regions (i.e. shadow memory regions). +#define kXnuFastMmapFd 0x4 +static bool use_xnu_fast_mmap = false; + +uptr internal_mmap_shadow(void *addr, size_t length, int prot, int flags, + int fd, u64 offset) { + if (fd == -1) { + int fd = VM_MAKE_TAG(VM_MEMORY_SANITIZER); + if (use_xnu_fast_mmap) fd |= kXnuFastMmapFd; + } + if (&__mmap) return (uptr)__mmap(addr, length, prot, flags, fd, offset); + return (uptr)mmap(addr, length, prot, flags, fd, offset); +} + uptr internal_munmap(void *addr, uptr length) { if (&__munmap) return __munmap(addr, length); return munmap(addr, length); @@ -502,23 +517,33 @@ CHECK_NE(sysctl(mib, 2, 0, &len, 0, 0), -1); CHECK_LT(len, maxlen); CHECK_NE(sysctl(mib, 2, version, &len, 0, 0), -1); - switch (version[0]) { - case '9': return MACOS_VERSION_LEOPARD; - case '1': { - switch (version[1]) { - case '0': return MACOS_VERSION_SNOW_LEOPARD; - case '1': return MACOS_VERSION_LION; - case '2': return MACOS_VERSION_MOUNTAIN_LION; - case '3': return MACOS_VERSION_MAVERICKS; - case '4': return MACOS_VERSION_YOSEMITE; - default: - if (IsDigit(version[1])) - return MACOS_VERSION_UNKNOWN_NEWER; - else - return MACOS_VERSION_UNKNOWN; - } - } - default: return MACOS_VERSION_UNKNOWN; + + // Expect .(.) + CHECK_GE(len, 3); + const char *p = version; + int major = internal_simple_strtoll(p, &p, /*base=*/10); + if (*p != '.') return MACOS_VERSION_UNKNOWN; + p += 1; + int minor = internal_simple_strtoll(p, &p, /*base=*/10); + if (*p != '.') return MACOS_VERSION_UNKNOWN; + + switch (major) { + case 9: return MACOS_VERSION_LEOPARD; + case 10: return MACOS_VERSION_SNOW_LEOPARD; + case 11: return MACOS_VERSION_LION; + case 12: return MACOS_VERSION_MOUNTAIN_LION; + case 13: return MACOS_VERSION_MAVERICKS; + case 14: return MACOS_VERSION_YOSEMITE; + case 15: return MACOS_VERSION_EL_CAPITAN; + case 16: return MACOS_VERSION_SIERRA; + case 17: + // Not a typo, 17.5 Darwin Kernel Version maps to High Sierra 10.13.4. + if (minor >= 5) + return MACOS_VERSION_HIGH_SIERRA_DOT_RELEASE_4; + return MACOS_VERSION_HIGH_SIERRA; + default: + if (major < 9) return MACOS_VERSION_UNKNOWN; + return MACOS_VERSION_UNKNOWN_NEWER; } } @@ -661,6 +686,16 @@ void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); } +void InitializePlatformEarly() { + // Only use xnu_fast_mmap when on x86_64 and the OS supports it. + use_xnu_fast_mmap = +#if defined(__x86_64__) + GetMacosVersion() >= MACOS_VERSION_HIGH_SIERRA_DOT_RELEASE_4; +#else + false; +#endif +} + #if !SANITIZER_GO static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES"; LowLevelAllocator allocator_for_env; Index: lib/sanitizer_common/sanitizer_openbsd.cc =================================================================== --- lib/sanitizer_common/sanitizer_openbsd.cc +++ lib/sanitizer_common/sanitizer_openbsd.cc @@ -45,6 +45,11 @@ return (uptr)mmap(addr, length, prot, flags, fd, offset); } +uptr internal_mmap_shadow(void *addr, uptr length, int prot, int flags, int fd, + OFF_T offset) { + return internal_mmap(addr, length, prot, flags, fd, offset); +} + uptr internal_munmap(void *addr, uptr length) { return munmap(addr, length); } int internal_mprotect(void *addr, uptr length, int prot) { Index: lib/sanitizer_common/sanitizer_posix.h =================================================================== --- lib/sanitizer_common/sanitizer_posix.h +++ lib/sanitizer_common/sanitizer_posix.h @@ -38,8 +38,10 @@ uptr internal_write(fd_t fd, const void *buf, uptr count); // Memory -uptr internal_mmap(void *addr, uptr length, int prot, int flags, - int fd, OFF_T offset); +uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd, + OFF_T offset); +uptr internal_mmap_shadow(void *addr, uptr length, int prot, int flags, int fd, + OFF_T offset); uptr internal_munmap(void *addr, uptr length); int internal_mprotect(void *addr, uptr length, int prot); Index: lib/sanitizer_common/sanitizer_posix_libcdep.cc =================================================================== --- lib/sanitizer_common/sanitizer_posix_libcdep.cc +++ lib/sanitizer_common/sanitizer_posix_libcdep.cc @@ -328,15 +328,16 @@ } #endif -bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) { +static bool MmapFixedNoReserveCommon(uptr fixed_addr, uptr size, + const char *name, bool shadow) { int fd = name ? GetNamedMappingFd(name, size) : -1; unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE; if (fd == -1) flags |= MAP_ANON; uptr PageSize = GetPageSizeCached(); - uptr p = internal_mmap((void *)(fixed_addr & ~(PageSize - 1)), - RoundUpTo(size, PageSize), PROT_READ | PROT_WRITE, - flags, fd, 0); + uptr p = (shadow ? internal_mmap_shadow : internal_mmap)( + (void *)(fixed_addr & ~(PageSize - 1)), RoundUpTo(size, PageSize), + PROT_READ | PROT_WRITE, flags, fd, 0); int reserrno; if (internal_iserror(p, &reserrno)) { Report("ERROR: %s failed to " @@ -348,6 +349,14 @@ return true; } +bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) { + return MmapFixedNoReserveCommon(fixed_addr, size, name, /*shadow=*/ false); +} + +bool MmapFixedNoReserveShadow(uptr fixed_addr, uptr size, const char *name) { + return MmapFixedNoReserveCommon(fixed_addr, size, name, /*shadow=*/ true); +} + uptr ReservedAddressRange::Init(uptr size, const char *name, uptr fixed_addr) { // We don't pass `name` along because, when you enable `decorate_proc_maps` // AND actually use a named mapping AND are using a sanitizer intercepting @@ -387,8 +396,8 @@ unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE; if (fd == -1) flags |= MAP_ANON; - return (void *)internal_mmap((void *)fixed_addr, size, PROT_NONE, flags, fd, - 0); + return (void *)internal_mmap_shadow((void *)fixed_addr, size, PROT_NONE, + flags, fd, 0); } void *MmapNoAccess(uptr size) { Index: lib/sanitizer_common/sanitizer_rtems.cc =================================================================== --- lib/sanitizer_common/sanitizer_rtems.cc +++ lib/sanitizer_common/sanitizer_rtems.cc @@ -95,6 +95,7 @@ *tls_addr = *tls_size = 0; } +void InitializePlatformEarly() {} void MaybeReexec() {} void CheckASLR() {} void DisableCoreDumperIfNecessary() {} Index: lib/sanitizer_common/sanitizer_win.cc =================================================================== --- lib/sanitizer_common/sanitizer_win.cc +++ lib/sanitizer_common/sanitizer_win.cc @@ -224,6 +224,10 @@ return true; } +bool MmapFixedNoReserveShadow(uptr fixed_addr, uptr size, const char *name) { + return MmapFixedNoReserve(fixed_addr, size, name); +} + // Memory space mapped by 'MmapFixedOrDie' must have been reserved by // 'MmapFixedNoAccess'. void *MmapFixedOrDie(uptr fixed_addr, uptr size) { @@ -1004,6 +1008,10 @@ // Do nothing. } +void InitializePlatformEarly() { + // Do nothing. +} + void MaybeReexec() { // No need to re-exec on Windows. } Index: lib/tsan/rtl/tsan_platform_posix.cc =================================================================== --- lib/tsan/rtl/tsan_platform_posix.cc +++ lib/tsan/rtl/tsan_platform_posix.cc @@ -54,7 +54,8 @@ #if !SANITIZER_GO void InitializeShadowMemory() { // Map memory shadow. - if (!MmapFixedNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(), "shadow")) { + if (!MmapFixedNoReserveShadow(ShadowBeg(), ShadowEnd() - ShadowBeg(), + "shadow")) { Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n"); Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n"); Die(); @@ -108,7 +109,7 @@ // Map meta shadow. const uptr meta = MetaShadowBeg(); const uptr meta_size = MetaShadowEnd() - meta; - if (!MmapFixedNoReserve(meta, meta_size, "meta shadow")) { + if (!MmapFixedNoReserveShadow(meta, meta_size, "meta shadow")) { Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n"); Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n"); Die(); Index: lib/tsan/rtl/tsan_rtl.cc =================================================================== --- lib/tsan/rtl/tsan_rtl.cc +++ lib/tsan/rtl/tsan_rtl.cc @@ -246,7 +246,8 @@ const uptr kPageSize = GetPageSizeCached(); uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize); uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize); - if (!MmapFixedNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow")) + if (!MmapFixedNoReserveShadow(shadow_begin, shadow_end - shadow_begin, + "shadow")) Die(); // Meta shadow is 2:1, so tread carefully. @@ -259,7 +260,8 @@ if (!data_mapped) { // First call maps data+bss. data_mapped = true; - if (!MmapFixedNoReserve(meta_begin, meta_end - meta_begin, "meta shadow")) + if (!MmapFixedNoReserveShadow(meta_begin, meta_end - meta_begin, + "meta shadow")) Die(); } else { // Mapping continous heap. @@ -270,7 +272,8 @@ return; if (meta_begin < mapped_meta_end) meta_begin = mapped_meta_end; - if (!MmapFixedNoReserve(meta_begin, meta_end - meta_begin, "meta shadow")) + if (!MmapFixedNoReserveShadow(meta_begin, meta_end - meta_begin, + "meta shadow")) Die(); mapped_meta_end = meta_end; } @@ -359,7 +362,9 @@ CheckASLR(); InitializeFlags(&ctx->flags, options); AvoidCVE_2016_2143(); - InitializePlatformEarly(); + __sanitizer::InitializePlatformEarly(); + __tsan::InitializePlatformEarly(); + #if !SANITIZER_GO // Re-exec ourselves if we need to set additional env or command line args. MaybeReexec();