[windows] Fix deadlock on mmap failure due to CHECK recursion

[rnk]

Summary

Printing a stacktrace acquires a spinlock, and the sanitizer spinlocks
aren't re-entrant. Avoid the problem by reusing the logic we already
have on Posix.

This failure mode is already exercised by the existing mmap_limit_mb.cc
test case. It will be enabled in a forthcoming change, so I didn't add
standalone tests for this change.

[samsonov]

LGTM

compiler-rt/trunk/lib/sanitizer_common/sanitizer_common.h

 void SortArray(uptr *array, uptr size);
 bool TemplateMatch(const char *templ, const char *str);
 
 // Exit
 void NORETURN Abort();
 void NORETURN Die();
 void NORETURN
 CheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2);
+void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
+                                      error_t err);
 
 // Set the name of the current thread to 'name', return true on succees.
 // The name may be truncated to a system-dependent limit.
 bool SanitizerSetThreadName(const char *name);
 // Get the name of the current thread (no more than max_len bytes),
 // return true on succees. name should have space for at least max_len+1 bytes.
 bool SanitizerGetThreadName(char *name, int max_len);
 

compiler-rt/trunk/lib/sanitizer_common/sanitizer_common.cc

void NORETURN CheckFailed(const char *file, int line, const char *cond,
   if (CheckFailedCallback) {
     CheckFailedCallback(file, line, cond, v1, v2);
   }
   Report("Sanitizer CHECK failed: %s:%d %s (%lld, %lld)\n", file, line, cond,
                                                             v1, v2);
   Die();
 }
 
+void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
+                                      error_t err) {
+  static int recursion_count;
+  if (recursion_count) {
+    // The Report() and CHECK calls below may call mmap recursively and fail.
+    // If we went into recursion, just die.
+    RawWrite("ERROR: Failed to mmap\n");
+    Die();
+  }
+  recursion_count++;
+  Report("ERROR: %s failed to "
+         "allocate 0x%zx (%zd) bytes of %s (error code: %d)\n",
+         SanitizerToolName, size, size, mem_type, err);
+  DumpProcessMap();
+  UNREACHABLE("unable to mmap");
+}
+
 bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
                       uptr *read_len, uptr max_len, error_t *errno_p) {
   uptr PageSize = GetPageSizeCached();
   uptr kMinFileLen = PageSize;
   *buff = nullptr;
   *buff_size = 0;
   *read_len = 0;
   // The files we usually open are not seekable, so try different buffer sizes.

compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix.cc

 }
 
 void *MmapOrDie(uptr size, const char *mem_type) {
   size = RoundUpTo(size, GetPageSizeCached());
   uptr res = internal_mmap(0, size,
                             PROT_READ | PROT_WRITE,
                             MAP_PRIVATE | MAP_ANON, -1, 0);
   int reserrno;
-  if (internal_iserror(res, &reserrno)) {
-    static int recursion_count;
-    if (recursion_count) {
-      // The Report() and CHECK calls below may call mmap recursively and fail.
-      // If we went into recursion, just die.
-      RawWrite("ERROR: Failed to mmap\n");
-      Die();
-    }
-    recursion_count++;
-    Report("ERROR: %s failed to "
-           "allocate 0x%zx (%zd) bytes of %s (errno: %d)\n",
-           SanitizerToolName, size, size, mem_type, reserrno);
-    DumpProcessMap();
-    CHECK("unable to mmap" && 0);
-  }
+  if (internal_iserror(res, &reserrno))
+    ReportMmapFailureAndDie(size, mem_type, reserrno);
   IncreaseTotalMmap(size);
   return (void *)res;
 }
 
 void UnmapOrDie(void *addr, uptr size) {
   if (!addr || !size) return;
   uptr res = internal_munmap(addr, size);
   if (internal_iserror(res)) {

compiler-rt/trunk/lib/sanitizer_common/sanitizer_win.cc

void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
   // including stack guard page) ?
   *stack_top = (uptr)mbi.BaseAddress + mbi.RegionSize;
   *stack_bottom = (uptr)mbi.AllocationBase;
 }
 #endif  // #if !SANITIZER_GO
 
 void *MmapOrDie(uptr size, const char *mem_type) {
   void *rv = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
-  if (rv == 0) {
-    Report("ERROR: %s failed to "
-           "allocate 0x%zx (%zd) bytes of %s (error code: %d)\n",
-           SanitizerToolName, size, size, mem_type, GetLastError());
-    CHECK("unable to mmap" && 0);
-  }
+  if (rv == 0)
+    ReportMmapFailureAndDie(size, mem_type, GetLastError());
   return rv;
 }
 
 void UnmapOrDie(void *addr, uptr size) {
   if (!size || !addr)
     return;
 
   if (VirtualFree(addr, size, MEM_DECOMMIT) == 0) {