[MSan] Simulate OOM in mmap_interceptor()

[iii]

Summary

Some kernels can provide 16EiB worth of mappings to each process, which
causes mmap test to run for a very long time. In order to make it stop
after a few seconds, make mmap_interceptor() fail when the original
mmap() returns an address which is outside of the application range.

[eugenis]

LGTM

[iii]

Thanks for reviewing! I don't think I have commit access - may I ask you to commit this one for me?

compiler-rt/lib/msan/msan_interceptors.cpp

if (flags()->poison_in_dtor) {
     stack.tag = STACK_TRACE_TAG_POISON;
     PoisonMemory(data, size, &stack);
   }
 }
 
 template <class Mmap>
 static void *mmap_interceptor(Mmap real_mmap, void *addr, SIZE_T length,
                               int prot, int flags, int fd, OFF64_T offset) {
-  if (addr && !MEM_IS_APP(addr)) {
+  SIZE_T rounded_length = RoundUpTo(length, GetPageSize());
+  void *end_addr = (char *)addr + (rounded_length - 1);
+  if (addr && (!MEM_IS_APP(addr) || !MEM_IS_APP(end_addr))) {
     if (flags & map_fixed) {
       errno = errno_EINVAL;
       return (void *)-1;
     } else {
       addr = nullptr;
     }
   }
   void *res = real_mmap(addr, length, prot, flags, fd, offset);
-  if (res != (void *)-1) __msan_unpoison(res, RoundUpTo(length, GetPageSize()));
+  if (res != (void *)-1) {
+    void *end_res = (char *)res + (rounded_length - 1);
+    if (MEM_IS_APP(res) && MEM_IS_APP(end_res)) {
+      __msan_unpoison(res, rounded_length);
+    } else {
+      // Application has attempted to map more memory than is supported by
+      // MSAN. Act as if we ran out of memory.
+      internal_munmap(res, length);
+      errno = errno_ENOMEM;
+      return (void *)-1;
+    }
+  }
   return res;
 }
 
 INTERCEPTOR(int, getrusage, int who, void *usage) {
   ENSURE_MSAN_INITED();
   int res = REAL(getrusage)(who, usage);
   if (res == 0) {
     __msan_unpoison(usage, __sanitizer::struct_rusage_sz);