[sanitizer] Implement MapDynamicShadowAndAliases.

[morehouse]

Summary

The function works like MapDynamicShadow, except that it creates aliased
memory to the right of the shadow. The main use case is for HWASan
aliasing mode, which gets fast IsAlias() checks by exploiting the fact
that the upper bits of the shadow base and aliased memory match.

[eugenis]

compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp

927

This should be internal_mremap, in case mremap in libc is instrumented.

[morehouse]

• Use internal_mremap.

[eugenis]

LGTM

compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp

188

I do not think this check is necessary. You can simply pass all the argument to the kernel.

[vitalybuka]

compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp

962

Maybe :)

[morehouse]

• Simplify and improve readability.

compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp

962

I think it is less readable, though doing ~0 is also not super clear. I've changed it to cast -1 to uptr, since this is what mmap(2) says MAP_FAILED is.

[ro]

This patch broke the Solaris buildbots (Solaris/sparcv9 and Solaris/amd64:

FAILED: projects/compiler-rt/lib/sanitizer_common/CMakeFiles/RTSanitizerCommonLibc.i386.dir/sanitizer_linux_libcdep.cpp.o
[...]
/vol/llvm/src/llvm-project/dist/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp: In function ‘__sanitizer::uptr __sanitizer::MremapCreateAlias(__sanitizer::uptr, __sanitizer::uptr, __sanitizer::uptr)’:
/vol/llvm/src/llvm-project/dist/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp:928:26: error: ‘MREMAP_MAYMOVE’ was not declared in this scope
  928 |                          MREMAP_MAYMOVE | MREMAP_FIXED,
      |                          ^~~~~~~~~~~~~~
/vol/llvm/src/llvm-project/dist/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp:928:43: error: ‘MREMAP_FIXED’ was not declared in this scope; did you mean ‘MAP_FIXED’?
  928 |                          MREMAP_MAYMOVE | MREMAP_FIXED,
      |                                           ^~~~~~~~~~~~
      |                                           MAP_FIXED

Despite its name, sanitizer_linux_libcdep.cpp is shared between FreeBSD, Linux, NetBSD, and Solaris, while according to the Linux manpage, mremap is Linux-only.

[morehouse]

In D98369#2647334, @ro wrote:

This patch broke the Solaris buildbots (Solaris/sparcv9 and Solaris/amd64:

FAILED: projects/compiler-rt/lib/sanitizer_common/CMakeFiles/RTSanitizerCommonLibc.i386.dir/sanitizer_linux_libcdep.cpp.o
[...]
/vol/llvm/src/llvm-project/dist/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp: In function ‘__sanitizer::uptr __sanitizer::MremapCreateAlias(__sanitizer::uptr, __sanitizer::uptr, __sanitizer::uptr)’:
/vol/llvm/src/llvm-project/dist/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp:928:26: error: ‘MREMAP_MAYMOVE’ was not declared in this scope
  928 |                          MREMAP_MAYMOVE | MREMAP_FIXED,
      |                          ^~~~~~~~~~~~~~
/vol/llvm/src/llvm-project/dist/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp:928:43: error: ‘MREMAP_FIXED’ was not declared in this scope; did you mean ‘MAP_FIXED’?
  928 |                          MREMAP_MAYMOVE | MREMAP_FIXED,
      |                                           ^~~~~~~~~~~~
      |                                           MAP_FIXED

Despite its name, sanitizer_linux_libcdep.cpp is shared between FreeBSD, Linux, NetBSD, and Solaris, while according to the Linux manpage, mremap is Linux-only.

That's confusing... I've submitted https://github.com/llvm/llvm-project/commit/643d87ebab7882442400fbb983f2b6a268012b50 to fix the build.

[ro]

Indeed. However, the misleading filenames are just the tip of the iceberg. Hardcoding every feature check with platform macros instead of feature tests (preferably detected at build time) feels so 90ies...

Once I'd pulled in an unrelated fix for another unrelated build-breaking bug, the bots are back to normal. Thanks!

compiler-rt/lib/sanitizer_common/sanitizer_common.h

 // be aligned to the mmap granularity * 2^shadow_scale, or to
 // 2^min_shadow_base_alignment if that is larger. The returned address will
 // have max(2^min_shadow_base_alignment, mmap granularity) on the left, and
 // shadow_size_bytes bytes on the right, which on linux is mapped no access.
 // The high_mem_end may be updated if the original shadow size doesn't fit.
 uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
                       uptr min_shadow_base_alignment, uptr &high_mem_end);
 
+// Let S = max(shadow_size, num_aliases * alias_size, ring_buffer_size).
+// Reserves 2*S bytes of address space to the right of the returned address and
+// ring_buffer_size bytes to the left.  The returned address is aligned to 2*S.
+// Also creates num_aliases regions of accessible memory starting at offset S
+// from the returned address.  Each region has size alias_size and is backed by
+// the same physical memory.
+uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
+                                uptr num_aliases, uptr ring_buffer_size);
+
 // Reserve memory range [beg, end]. If madvise_shadow is true then apply
 // madvise (e.g. hugepages, core dumping) requested by options.
 void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name,
                               bool madvise_shadow = true);
 
 // Protect size bytes of memory starting at addr. Also try to protect
 // several pages at the start of the address space as specified by
 // zero_base_shadow_start, at most up to the size or zero_base_max_shadow_start.

compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp

 #endif
 }
 #endif // !SANITIZER_S390
 
 uptr internal_munmap(void *addr, uptr length) {
   return internal_syscall(SYSCALL(munmap), (uptr)addr, length);
 }
 
+uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,
+                     void *new_address) {
+  return internal_syscall(SYSCALL(mremap), (uptr)old_address, old_size,
+                          new_size, flags, (uptr)new_address);
+}
+
 int internal_mprotect(void *addr, uptr length, int prot) {
   return internal_syscall(SYSCALL(mprotect), (uptr)addr, length, prot);
 }
 
 int internal_madvise(uptr addr, uptr length, int advice) {
   return internal_syscall(SYSCALL(madvise), addr, length, advice);
 }
 

compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp

 #if SANITIZER_NETBSD
 #define _RTLD_SOURCE  // for __lwp_gettcb_fast() / __lwp_getprivate_fast()
 #endif
 
 #include <dlfcn.h>  // for dlsym()
 #include <link.h>
 #include <pthread.h>
 #include <signal.h>
+#include <sys/mman.h>
 #include <sys/resource.h>
 #include <syslog.h>
 
 #if !defined(ElfW)
 #define ElfW(type) Elf_##type
 #endif
 
 #if SANITIZER_FREEBSD
uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
   const uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
 
   UnmapFromTo(map_start, shadow_start - left_padding);
   UnmapFromTo(shadow_start + shadow_size, map_start + map_size);
 
   return shadow_start;
 }
 
+static uptr MmapSharedNoReserve(uptr addr, uptr size) {
+  return internal_mmap(
+      reinterpret_cast<void *>(addr), size, PROT_READ | PROT_WRITE,
+      MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+}
+
+static uptr MremapCreateAlias(uptr base_addr, uptr alias_addr,
+                              uptr alias_size) {
+  return internal_mremap(reinterpret_cast<void *>(base_addr), 0, alias_size,
+                         MREMAP_MAYMOVE | MREMAP_FIXED,
+                         reinterpret_cast<void *>(alias_addr));
+}
+
+static void CreateAliases(uptr start_addr, uptr alias_size, uptr num_aliases) {
+  uptr total_size = alias_size * num_aliases;
+  uptr mapped = MmapSharedNoReserve(start_addr, total_size);
+  CHECK_EQ(mapped, start_addr);
+
+  for (uptr i = 1; i < num_aliases; ++i) {
+    uptr alias_addr = start_addr + i * alias_size;
+    CHECK_EQ(MremapCreateAlias(start_addr, alias_addr, alias_size), alias_addr);
+  }
+}
+
+uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
+                                uptr num_aliases, uptr ring_buffer_size) {
+  CHECK_EQ(alias_size & (alias_size - 1), 0);
+  CHECK_EQ(num_aliases & (num_aliases - 1), 0);
+  CHECK_EQ(ring_buffer_size & (ring_buffer_size - 1), 0);
+
+  const uptr granularity = GetMmapGranularity();
+  shadow_size = RoundUpTo(shadow_size, granularity);
+  CHECK_EQ(shadow_size & (shadow_size - 1), 0);
+
+  const uptr alias_region_size = alias_size * num_aliases;
+  const uptr alignment =
+      2 * Max(Max(shadow_size, alias_region_size), ring_buffer_size);
+  const uptr left_padding = ring_buffer_size;
+
+  const uptr right_size = alignment;
+  const uptr map_size = left_padding + 2 * alignment;
+
+  const uptr map_start = reinterpret_cast<uptr>(MmapNoAccess(map_size));
+  CHECK_NE(map_start, static_cast<uptr>(-1));
+  const uptr right_start = RoundUpTo(map_start + left_padding, alignment);
+
+  UnmapFromTo(map_start, right_start - left_padding);
+  UnmapFromTo(right_start + right_size, map_start + map_size);
+
+  CreateAliases(right_start + right_size / 2, alias_size, num_aliases);
+
+  return right_start;
+}
+
 void InitializePlatformCommonFlags(CommonFlags *cf) {
 #if SANITIZER_ANDROID
   if (&__libc_get_static_tls_bounds == nullptr)
     cf->detect_leaks = false;
 #endif
 }
 
 } // namespace __sanitizer
 
 #endif

compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp

uptr internal_mmap(void *addr, size_t length, int prot, int flags,
   return (uptr)mmap(addr, length, prot, flags, fd, offset);
 }
 
 uptr internal_munmap(void *addr, uptr length) {
   if (&__munmap) return __munmap(addr, length);
   return munmap(addr, length);
 }
 
+uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,
+                     void *new_address) {
+  CHECK(false && "internal_mremap is unimplemented on Mac");
+  return 0;
+}
+
 int internal_mprotect(void *addr, uptr length, int prot) {
   return mprotect(addr, length, prot);
 }
 
 int internal_madvise(uptr addr, uptr length, int advice) {
   return madvise((void *)addr, length, advice);
 }
 
if (shadow_start == 0) {
       CHECK(0 && "cannot place shadow after restricting vm");
     }
   }
   CHECK_NE((uptr)0, shadow_start);
   CHECK(IsAligned(shadow_start, alignment));
   return shadow_start;
 }
 
+uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
+                                uptr num_aliases, uptr ring_buffer_size) {
+  CHECK(false && "HWASan aliasing is unimplemented on Mac");
+  return 0;
+}
+
 uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
                               uptr *largest_gap_found,
                               uptr *max_occupied_addr) {
   typedef vm_region_submap_short_info_data_64_t RegionInfo;
   enum { kRegionInfoSize = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64 };
   // Start searching for available memory region past PAGEZERO, which is
   // 4KB on 32-bit and 4GB on 64-bit.
   mach_vm_address_t start_address =

compiler-rt/lib/sanitizer_common/sanitizer_netbsd.cpp

uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
   return (uptr)__mmap(addr, length, prot, flags, fd, 0, offset);
 }
 
 uptr internal_munmap(void *addr, uptr length) {
   DEFINE__REAL(int, munmap, void *a, uptr b);
   return _REAL(munmap, addr, length);
 }
 
+uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,
+                     void *new_address) {
+  CHECK(false && "internal_mremap is unimplemented on NetBSD");
+  return 0;
+}
+
 int internal_mprotect(void *addr, uptr length, int prot) {
   DEFINE__REAL(int, mprotect, void *a, uptr b, int c);
   return _REAL(mprotect, addr, length, prot);
 }
 
 int internal_madvise(uptr addr, uptr length, int advice) {
   DEFINE__REAL(int, madvise, void *a, uptr b, int c);
   return _REAL(madvise, (void *)addr, length, advice);

compiler-rt/lib/sanitizer_common/sanitizer_posix.h

 
 uptr internal_read(fd_t fd, void *buf, uptr count);
 uptr internal_write(fd_t fd, const void *buf, uptr count);
 
 // Memory
 uptr internal_mmap(void *addr, uptr length, int prot, int flags,
                    int fd, u64 offset);
 uptr internal_munmap(void *addr, uptr length);
+uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,
+                     void *new_address);
 int internal_mprotect(void *addr, uptr length, int prot);
 int internal_madvise(uptr addr, uptr length, int advice);
 
 // OS
 uptr internal_filesize(fd_t fd);  // -1 on error.
 uptr internal_stat(const char *path, void *buf);
 uptr internal_lstat(const char *path, void *buf);
 uptr internal_fstat(fd_t fd, void *buf);

compiler-rt/lib/sanitizer_common/sanitizer_win.cpp

while (true) {
     }
 
     // Move to the next region.
     address = (uptr)info.BaseAddress + info.RegionSize;
   }
   return 0;
 }
 
+uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
+                                uptr num_aliases, uptr ring_buffer_size) {
+  CHECK(false && "HWASan aliasing is unimplemented on Windows");
+  return 0;
+}
+
 bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
   MEMORY_BASIC_INFORMATION mbi;
   CHECK(VirtualQuery((void *)range_start, &mbi, sizeof(mbi)));
   return mbi.Protect == PAGE_NOACCESS &&
          (uptr)mbi.BaseAddress + mbi.RegionSize >= range_end;
 }
 
 void *MapFileToMemory(const char *file_name, uptr *buff_size) {