[sanitizer] Split MmapFixedNoReserve into a version that disallows overwriting existing memory and one that allows it

[kubamracek]

Summary

Multiple times during work on sanitizers, I've hit a situation where the shadow memory initialization (done via mmap) would overwrite some existing memory. This patch changes MmapFixedNoReserve to add a safety check to fail when something already occupies the region we're trying to map into, and introduces MmapFixedNoReserveAllowOverwrite which explicitly allows overwriting memory. Same for MmapFixedNoAccess and MmapFixedNoAccessAllowOverwrite.

The patch implements this for Darwin only currently, but if there's a some Linux and Windows API to do the same, I'll be happy to add them.

[kubamracek]

ping

[vitalybuka]

LGTM

lib/sanitizer_common/sanitizer_common.h

98

misaligned

lib/asan/asan_internal.h

 void *AsanTSDGet();
 void AsanTSDSet(void *tsd);
 void PlatformTSDDtor(void *tsd);
 
 void AppendToErrorMessageBuffer(const char *buffer);
 
 void *AsanDlSymNext(const char *sym);
 
-void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name);
-
 // Add convenient macro for interface functions that may be represented as
 // weak hooks.
 #define ASAN_MALLOC_HOOK(ptr, size)                                   \
   do {                                                                \
     if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(ptr, size); \
     RunMallocHooks(ptr, size);                                        \
   } while (false)
 #define ASAN_FREE_HOOK(ptr)                                 \

lib/asan/asan_poisoning.h

if (page_beg >= page_end) {
       REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg);
     } else {
       if (page_beg != shadow_beg) {
         REAL(memset)((void *)shadow_beg, 0, page_beg - shadow_beg);
       }
       if (page_end != shadow_end) {
         REAL(memset)((void *)page_end, 0, shadow_end - page_end);
       }
-      ReserveShadowMemoryRange(page_beg, page_end - 1, nullptr);
+      MmapFixedNoReserveAllowOverwrite(page_beg, page_end - page_beg);
     }
   }
 }
 
 ALWAYS_INLINE void FastPoisonShadowPartialRightRedzone(
     uptr aligned_addr, uptr size, uptr redzone_size, u8 value) {
   DCHECK(CanPoisonMemory());
   bool poison_partial = flags()->poison_partial;

lib/asan/asan_shadow_setup.cc

 
 // ---------------------- mmap -------------------- {{{1
 // Reserve memory range [beg, end].
 // We need to use inclusive range because end+1 may not be representable.
 void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
   CHECK_EQ((beg % GetMmapGranularity()), 0);
   CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
   uptr size = end - beg + 1;
-  DecreaseTotalMmap(size);  // Don't count the shadow against mmap_limit_mb.
   void *res = MmapFixedNoReserve(beg, size, name);
   if (res != (void *)beg) {
     Report(
         "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
         "Perhaps you're using ulimit -v\n",
         size);
     Abort();
   }

lib/sanitizer_common/sanitizer_common.h

 void *MmapOrDie(uptr size, const char *mem_type, bool raw_report = false);
 INLINE void *MmapOrDieQuietly(uptr size, const char *mem_type) {
   return MmapOrDie(size, mem_type, /*raw_report*/ true);
 }
 void UnmapOrDie(void *addr, uptr size);
 // Behaves just like MmapOrDie, but tolerates out of memory condition, in that
 // case returns nullptr.
 void *MmapOrDieOnFatalError(uptr size, const char *mem_type);
+// Maps the specified memory region with "NORESERVE". Fails if any part of the
+// memory region is already used. On success returns 'fixed_addr'.
 void *MmapFixedNoReserve(uptr fixed_addr, uptr size,
                          const char *name = nullptr);
+// Same as MmapFixedNoReserve, but doesn't fail when the memory region or a part
+// of it is already used, instead overwrites any existing mapping.
+void *MmapFixedNoReserveAllowOverwrite(uptr fixed_addr, uptr size,
+                         const char *name = nullptr);
 void *MmapNoReserveOrDie(uptr size, const char *mem_type);
 void *MmapFixedOrDie(uptr fixed_addr, uptr size);
 // Behaves just like MmapFixedOrDie, but tolerates out of memory condition, in
 // that case returns nullptr.
 void *MmapFixedOrDieOnFatalError(uptr fixed_addr, uptr size);
+// Maps the specified memory region and disallows any access to it. Fails if any
+// part of the memory region is already used. On success returns 'fixed_addr'.
 void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name = nullptr);
+// Same as MmapFixedNoAccess, but doesn't fail when the memory region or a part
+// of it is already used, instead overwrites any existing mapping.
+void *MmapFixedNoAccessAllowOverwrite(uptr fixed_addr, uptr size,
+                                      const char *name = nullptr);
 void *MmapNoAccess(uptr size);
 // Map aligned chunk of address space; size and alignment are powers of two.
 // Dies on all but out of memory errors, in the latter case returns nullptr.
 void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
                                    const char *mem_type);
 // Disallow access to a memory range.  Use MmapFixedNoAccess to allocate an
 // unaccessible memory.
 bool MprotectNoAccess(uptr addr, uptr size);

lib/sanitizer_common/sanitizer_linux_libcdep.cc

 extern "C" void android_set_abort_message(const char *msg);
 void SetAbortMessage(const char *str) { android_set_abort_message(str); }
 #else
 void SetAbortMessage(const char *str) {}
 #endif
 
 #endif // SANITIZER_LINUX
 
+#if SANITIZER_ANDROID || SANITIZER_GO
+int GetNamedMappingFd(const char *name, uptr size) {
+  return -1;
+}
+#else
+int GetNamedMappingFd(const char *name, uptr size) {
+  if (!common_flags()->decorate_proc_maps)
+    return -1;
+  char shmname[200];
+  CHECK(internal_strlen(name) < sizeof(shmname) - 10);
+  internal_snprintf(shmname, sizeof(shmname), "%zu [%s]", internal_getpid(),
+                    name);
+  int fd = shm_open(shmname, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU);
+  CHECK_GE(fd, 0);
+  int res = internal_ftruncate(fd, size);
+  CHECK_EQ(0, res);
+  res = shm_unlink(shmname);
+  CHECK_EQ(0, res);
+  return fd;
+}
+#endif
+
+void *MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) {
+  // FIXME: This should disallow overwriting already existing mappings.
+  return MmapFixedNoReserveAllowOverwrite(fixed_addr, size, name);
+}
+
+void *MmapFixedNoReserveAllowOverwrite(uptr fixed_addr, uptr size,
+                         const char *name = nullptr) {
+  int fd = name ? GetNamedMappingFd(name, size) : -1;
+  unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE;
+  if (fd == -1) flags |= MAP_ANON;
+
+  uptr PageSize = GetPageSizeCached();
+  uptr p = internal_mmap((void *)(fixed_addr & ~(PageSize - 1)),
+                         RoundUpTo(size, PageSize), PROT_READ | PROT_WRITE,
+                         flags, fd, 0);
+  int reserrno;
+  if (internal_iserror(p, &reserrno))
+    Report("ERROR: %s failed to "
+           "allocate 0x%zx (%zd) bytes at address %zx (errno: %d)\n",
+           SanitizerToolName, size, size, fixed_addr, reserrno);
+  return (void *)p;
+}
+
+void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) {
+  // FIXME: This should disallow overwriting already existing mappings.
+  return MmapFixedNoAccessAllowOverwrite(fixed_addr, size, name);
+}
+
+void *MmapFixedNoAccessAllowOverwrite(uptr fixed_addr, uptr size,
+                                      const char *name = nullptr) {
+  int fd = name ? GetNamedMappingFd(name, size) : -1;
+  unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE;
+  if (fd == -1) flags |= MAP_ANON;
+
+  return (void *)internal_mmap((void *)fixed_addr, size, PROT_NONE, flags, fd,
+                               0);
+}
+
 } // namespace __sanitizer
 
 #endif // SANITIZER_FREEBSD || SANITIZER_LINUX

lib/sanitizer_common/sanitizer_mac_libcdep.cc

 // This file is shared between various sanitizers' runtime libraries and
 // implements OSX-specific functions.
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_platform.h"
 #if SANITIZER_MAC
 #include "sanitizer_mac.h"
 
+#include <mach/mach.h>
 #include <sys/mman.h>
 
 namespace __sanitizer {
 
 void RestrictMemoryToMaxAddress(uptr max_address) {
   uptr size_to_mmap = GetMaxVirtualAddress() + 1 - max_address;
   void *res = MmapFixedNoAccess(max_address, size_to_mmap, "high gap");
   CHECK(res != MAP_FAILED);
 }
 
+static void *DarwinMmapFixed(uptr fixed_addr, uptr size, int flags,
+                             bool noaccess = false) {
+  uptr PageSize = GetPageSizeCached();
+  vm_address_t p = (vm_address_t)(fixed_addr & ~(PageSize - 1));
+  size = RoundUpTo(size, PageSize);
+  flags |= VM_FLAGS_FIXED | VM_MAKE_TAG(VM_MEMORY_ANALYSIS_TOOL);
+  kern_return_t r = vm_allocate(mach_task_self(), &p, size, flags);
+  if (r != KERN_SUCCESS) {
+    Report(
+        "ERROR: %s failed to allocate 0x%zx (%zd) bytes at address %zx "
+        "(vm_allocate returned: 0x%zx)\n",
+        SanitizerToolName, size, size, p, (uptr)r);
+    return 0;
+  }
+  if (noaccess) {
+    r = vm_protect(mach_task_self(), p, size, 0, VM_PROT_NONE);
+    if (r != KERN_SUCCESS) {
+      Report(
+          "ERROR: %s failed to vm_protect 0x%zx (%zd) bytes at address %zx "
+          "(vm_protect returned: 0x%zx)\n",
+          SanitizerToolName, size, size, p, (uptr)r);
+      return 0;
+    }
+  }
+  return (void *)p;
+}
+
+void *MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) {
+  return DarwinMmapFixed(fixed_addr, size, 0);
+}
+
+void *MmapFixedNoReserveAllowOverwrite(uptr fixed_addr, uptr size,
+                                       const char *name) {
+  return DarwinMmapFixed(fixed_addr, size, VM_FLAGS_OVERWRITE);
+}
+
+void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) {
+  return DarwinMmapFixed(fixed_addr, size, 0, /*noaccess*/ true);
+}
+
+void *MmapFixedNoAccessAllowOverwrite(uptr fixed_addr, uptr size,
+                                      const char *name) {
+  return DarwinMmapFixed(fixed_addr, size, VM_FLAGS_OVERWRITE,
+                         /*noaccess*/ true);
+}
+
 }  // namespace __sanitizer
 
 #endif  // SANITIZER_MAC

lib/sanitizer_common/sanitizer_posix_libcdep.cc

void PrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
   // cached mappings.
   MemoryMappingLayout::CacheMemoryMappings();
   // Same for /proc/self/exe in the symbolizer.
 #if !SANITIZER_GO
   Symbolizer::GetOrInit()->PrepareForSandboxing();
 #endif
 }
 
-#if SANITIZER_ANDROID || SANITIZER_GO
-int GetNamedMappingFd(const char *name, uptr size) {
-  return -1;
-}
-#else
-int GetNamedMappingFd(const char *name, uptr size) {
-  if (!common_flags()->decorate_proc_maps)
-    return -1;
-  char shmname[200];
-  CHECK(internal_strlen(name) < sizeof(shmname) - 10);
-  internal_snprintf(shmname, sizeof(shmname), "%zu [%s]", internal_getpid(),
-                    name);
-  int fd = shm_open(shmname, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU);
-  CHECK_GE(fd, 0);
-  int res = internal_ftruncate(fd, size);
-  CHECK_EQ(0, res);
-  res = shm_unlink(shmname);
-  CHECK_EQ(0, res);
-  return fd;
-}
-#endif
-
-void *MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) {
-  int fd = name ? GetNamedMappingFd(name, size) : -1;
-  unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE;
-  if (fd == -1) flags |= MAP_ANON;
-
-  uptr PageSize = GetPageSizeCached();
-  uptr p = internal_mmap((void *)(fixed_addr & ~(PageSize - 1)),
-                         RoundUpTo(size, PageSize), PROT_READ | PROT_WRITE,
-                         flags, fd, 0);
-  int reserrno;
-  if (internal_iserror(p, &reserrno))
-    Report("ERROR: %s failed to "
-           "allocate 0x%zx (%zd) bytes at address %zx (errno: %d)\n",
-           SanitizerToolName, size, size, fixed_addr, reserrno);
-  IncreaseTotalMmap(size);
-  return (void *)p;
-}
-
-void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) {
-  int fd = name ? GetNamedMappingFd(name, size) : -1;
-  unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE;
-  if (fd == -1) flags |= MAP_ANON;
-
-  return (void *)internal_mmap((void *)fixed_addr, size, PROT_NONE, flags, fd,
-                               0);
-}
-
 void *MmapNoAccess(uptr size) {
   unsigned flags = MAP_PRIVATE | MAP_ANON | MAP_NORESERVE;
   return (void *)internal_mmap(nullptr, size, PROT_NONE, flags, -1, 0);
 }
 
 // This function is defined elsewhere if we intercepted pthread_attr_getstack.
 extern "C" {
 SANITIZER_WEAK_ATTRIBUTE int

lib/sanitizer_common/sanitizer_win.cc

 #endif
   if (p == 0)
     Report("ERROR: %s failed to "
            "allocate %p (%zd) bytes at %p (error code: %d)\n",
            SanitizerToolName, size, size, fixed_addr, GetLastError());
   return p;
 }
 
+void *MmapFixedNoReserveAllowOverwrite(uptr fixed_addr, uptr size,
+                                       const char *name) {
+  // FIXME: This should actually allow overwriting existing memory.
+  return MmapFixedNoReserve(fixed_addr, size, name);
+}
+
 // Memory space mapped by 'MmapFixedOrDie' must have been reserved by
 // 'MmapFixedNoAccess'.
 void *MmapFixedOrDie(uptr fixed_addr, uptr size) {
   void *p = VirtualAlloc((LPVOID)fixed_addr, size,
       MEM_COMMIT, PAGE_READWRITE);
   if (p == 0) {
     char mem_type[30];
     internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
void *res = VirtualAlloc((LPVOID)fixed_addr, size,
                            MEM_RESERVE, PAGE_NOACCESS);
   if (res == 0)
     Report("WARNING: %s failed to "
            "mprotect %p (%zd) bytes at %p (error code: %d)\n",
            SanitizerToolName, size, size, fixed_addr, GetLastError());
   return res;
 }
 
+void *MmapFixedNoAccessAllowOverwrite(uptr fixed_addr, uptr size,
+                                      const char *name) {
+  // FIXME: This should actually allow overwriting existing memory.
+  return MmapFixedNoAccess(fixed_addr, size, name);
+}
+
 void *MmapNoAccess(uptr size) {
   void *res = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_NOACCESS);
   if (res == 0)
     Report("WARNING: %s failed to "
            "mprotect %p (%zd) bytes (error code: %d)\n",
            SanitizerToolName, size, size, GetLastError());
   return res;
 }

lib/tsan/go/buildgo.sh

 elif [ "`uname -a | grep Darwin`" != "" ]; then
   SUFFIX="darwin_amd64"
   OSCFLAGS="-fPIC -Wno-unused-const-variable -Wno-unknown-warning-option -isysroot $(xcodebuild -version -sdk macosx Path) -mmacosx-version-min=10.7"
   OSLDFLAGS="-lpthread -fPIC -fpie -mmacosx-version-min=10.7"
   SRCS="
     $SRCS
     ../rtl/tsan_platform_mac.cc
     ../../sanitizer_common/sanitizer_mac.cc
+    ../../sanitizer_common/sanitizer_mac_libcdep.cc
     ../../sanitizer_common/sanitizer_posix.cc
     ../../sanitizer_common/sanitizer_posix_libcdep.cc
     ../../sanitizer_common/sanitizer_procmaps_mac.cc
   "
 elif [ "`uname -a | grep MINGW`" != "" ]; then
   SUFFIX="windows_amd64"
   OSCFLAGS="-Wno-error=attributes -Wno-attributes -Wno-unused-const-variable -Wno-unknown-warning-option"
   OSLDFLAGS=""