[asan] Be more careful and verbose when allocating dynamic shadow memory

[kubamracek]

Summary

FindAvailableMemoryRange can currently overwrite existing memory (by restricting the VM below addresses that are already used). This patch adds a check to make sure we don't restrict the VM space too much. We are also now more explicit about why the lookup failed and print out verbose values.

[george.karpenkov]

lib/asan/asan_mac.cc

66

yay logging yay!

75

I don't quite understand semantics here.
max_occupied_vm to me means "maximal size of occupied virtual memory".
Then why it's a problem if a new maximal size is smaller? Shouldn't it be a problem if it's larger?

lib/sanitizer_common/sanitizer_mac.cc

901

Is this assignment necessary? Maximum size is zero unless set otherwise?

[kubamracek]

lib/asan/asan_mac.cc

75

max_occupied_vm is the largest address that it already occupied by something. We're trying to restrict the VM space with new_max_vm being the new VM limit. This check is to make sure we don't restrict too much (which would overwrite memory that is already used).

Suggestions for better name of max_occupied_vm?

lib/sanitizer_common/sanitizer_mac.cc

901

I'd rather not leave garbage in max_vm_address if for some reason the loop below doesn't find any regions at all.

[kubamracek]

lib/sanitizer_common/sanitizer_mac.cc

901

I mean, I don't want the semantics of this (or any) function to only partially (on some paths) initialize pointer-passed values...

[george.karpenkov]

lib/asan/asan_mac.cc

75

largest_occupied_vm? Almost the same, but at least to me does quite have same connotation, but that's a not important nit.. In any case, maybe this should be explained in a doxygen comment for FindAvailableMemoryRange?

[fjricci]

lib/asan/asan_mac.cc

75

max_occupied_addr or max_occupied_vm_addr, perhaps?

[kubamracek]

Renamed variable.

lib/asan/asan_mac.cc

 
 uptr FindDynamicShadowStart() {
   uptr granularity = GetMmapGranularity();
   uptr alignment = 8 * granularity;
   uptr left_padding = granularity;
   uptr space_size = kHighShadowEnd + left_padding;
 
   uptr largest_gap_found = 0;
-  uptr shadow_start = FindAvailableMemoryRange(space_size, alignment,
-                                               granularity, &largest_gap_found);
+  uptr max_occupied_vm = 0;
+  VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
+  uptr shadow_start = FindAvailableMemoryRange(
+      space_size, alignment, granularity, &largest_gap_found, &max_occupied_vm);
   // If the shadow doesn't fit, restrict the address space to make it fit.
   if (shadow_start == 0) {
+    VReport(
+        2, "Shadow doesn't fit, largest_gap_found = %p, max_occupied_vm = %p\n",
+        largest_gap_found, max_occupied_vm);
     uptr new_max_vm = RoundDownTo(largest_gap_found << SHADOW_SCALE, alignment);
+    if (new_max_vm < max_occupied_vm) {
+      Report("Unable to find a memory range for dynamic shadow.\n");
+      Report(
+          "space_size = %p, largest_gap_found = %p, max_occupied_vm = %p, "
+          "new_max_vm = %p\n",
+          space_size, largest_gap_found, max_occupied_vm, new_max_vm);
+      CHECK(0 && "cannot place shadow");
+    }
     RestrictMemoryToMaxAddress(new_max_vm);
     kHighMemEnd = new_max_vm - 1;
     space_size = kHighShadowEnd + left_padding;
-    shadow_start =
-        FindAvailableMemoryRange(space_size, alignment, granularity, nullptr);
+    VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
+    shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity,
+                                            nullptr, nullptr);
+    if (shadow_start == 0) {
+      Report("Unable to find a memory range after restricting VM.\n");
+      CHECK(0 && "cannot place shadow after restricting vm");
+    }
   }
   CHECK_NE((uptr)0, shadow_start);
   CHECK(IsAligned(shadow_start, alignment));
   return shadow_start;
 }
 
 // No-op. Mac does not support static linkage anyway.
 void AsanCheckDynamicRTPrereqs() {}

lib/sanitizer_common/sanitizer_common.h

void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
                                    const char *mem_type);
 // Disallow access to a memory range.  Use MmapFixedNoAccess to allocate an
 // unaccessible memory.
 bool MprotectNoAccess(uptr addr, uptr size);
 bool MprotectReadOnly(uptr addr, uptr size);
 
 // Find an available address space.
 uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
-                              uptr *largest_gap_found);
+                              uptr *largest_gap_found, uptr *max_vm_address);
 
 // Used to check if we can map shadow memory to a fixed location.
 bool MemoryRangeIsAvailable(uptr range_start, uptr range_end);
 // Releases memory pages entirely within the [beg, end] address range. Noop if
 // the provided range does not contain at least one entire page.
 void ReleaseMemoryPagesToOS(uptr beg, uptr end);
 void IncreaseTotalMmap(uptr size);
 void DecreaseTotalMmap(uptr size);

lib/sanitizer_common/sanitizer_linux.cc

Report(
         "RTLD_DEEPBIND from dlopen flags.\n",
         filename, filename);
     Die();
   }
 #endif
 }
 
 uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
-                              uptr *largest_gap_found) {
+                              uptr *largest_gap_found, uptr *max_vm_address) {
   UNREACHABLE("FindAvailableMemoryRange is not available");
   return 0;
 }
 
 bool GetRandom(void *buffer, uptr length, bool blocking) {
   if (!buffer || !length || length > 256)
     return false;
 #if SANITIZER_USE_GETRANDOM

lib/sanitizer_common/sanitizer_mac.cc

#else  // SANITIZER_WORDSIZE == 32
   return (1ULL << 32) - 1;  // 0xffffffff;
 #endif  // SANITIZER_WORDSIZE
 }
 
 uptr GetMaxVirtualAddress() {
   return GetMaxUserVirtualAddress();
 }
 
-uptr FindAvailableMemoryRange(uptr shadow_size,
-                              uptr alignment,
-                              uptr left_padding,
-                              uptr *largest_gap_found) {
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+                              uptr *largest_gap_found, uptr *max_vm_address) {
   typedef vm_region_submap_short_info_data_64_t RegionInfo;
   enum { kRegionInfoSize = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64 };
   // Start searching for available memory region past PAGEZERO, which is
   // 4KB on 32-bit and 4GB on 64-bit.
   mach_vm_address_t start_address =
     (SANITIZER_WORDSIZE == 32) ? 0x000000001000 : 0x000100000000;
 
   mach_vm_address_t address = start_address;
   mach_vm_address_t free_begin = start_address;
   kern_return_t kr = KERN_SUCCESS;
   if (largest_gap_found) *largest_gap_found = 0;
+  if (max_vm_address) *max_vm_address = 0;
   while (kr == KERN_SUCCESS) {
     mach_vm_size_t vmsize = 0;
     natural_t depth = 0;
     RegionInfo vminfo;
     mach_msg_type_number_t count = kRegionInfoSize;
     kr = mach_vm_region_recurse(mach_task_self(), &address, &vmsize, &depth,
                                 (vm_region_info_t)&vminfo, &count);
     if (kr == KERN_INVALID_ADDRESS) {
       // No more regions beyond "address", consider the gap at the end of VM.
       address = GetMaxVirtualAddress() + 1;
       vmsize = 0;
+    } else {
+      if (max_vm_address) *max_vm_address = address + vmsize;
     }
     if (free_begin != address) {
       // We found a free region [free_begin..address-1].
       uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment);
       uptr gap_end = RoundDownTo((uptr)address, alignment);
       uptr gap_size = gap_end > gap_start ? gap_end - gap_start : 0;
-      if (shadow_size < gap_size) {
+      if (size < gap_size) {
         return gap_start;
       }
 
       if (largest_gap_found && *largest_gap_found < gap_size) {
         *largest_gap_found = gap_size;
       }
     }
     // Move to the next region.

lib/sanitizer_common/sanitizer_win.cc

 }
 
 void DontDumpShadowMemory(uptr addr, uptr length) {
   // This is almost useless on 32-bits.
   // FIXME: add madvise-analog when we move to 64-bits.
 }
 
 uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
-                              uptr *largest_gap_found) {
+                              uptr *largest_gap_found, uptr *max_vm_address) {
   uptr address = 0;
   while (true) {
     MEMORY_BASIC_INFORMATION info;
     if (!::VirtualQuery((void*)address, &info, sizeof(info)))
       return 0;
 
     if (info.State == MEM_FREE) {
       uptr shadow_address = RoundUpTo((uptr)info.BaseAddress + left_padding,