[sanitizer] Small tweaks and fixes to allocator related functions

[cryptoad]

Summary

In sanitizer_allocator_primary32.h:

• rounding up in MapWithCallback is not needed as MmapOrDie does it. Note that the 64-bit counterpart doesn't round up, this keeps the behavior consistent;
• since IsAligned exists, use it in AllocateRegion;
• in PopulateFreeList:
  • checking b->Count to be greater than 0 when b->Count() == max_count is redundant when done more than once. Just check that max_count is greater than 0 out of the loop; the compiler (at least on ARM) didn't optimize it;
  • mark the batch creation failure as UNLIKELY;

In sanitizer_allocator_primary64.h:

• in MapWithCallback, mark the failure condition as UNLIKELY;

In sanitizer_posix.h:

• mark a bunch of Mmap related failure conditions as UNLIKELY;
• in MmapAlignedOrDieOnFatalError, we have IsAligned, so use it; rearrange the conditions as one test was redudant;
• in MmapFixedImpl, 30 chars was not large enough to hold the message and a full 64-bit address (or at least a 48-bit usermode address), increase to 40.

lib/sanitizer_common/sanitizer_allocator_primary32.h

s32 ReleaseToOSIntervalMs() const {
     return kReleaseToOSIntervalNever;
   }
 
   void SetReleaseToOSIntervalMs(s32 release_to_os_interval_ms) {
     // This is empty here. Currently only implemented in 64-bit allocator.
   }
 
   void *MapWithCallback(uptr size) {
-    size = RoundUpTo(size, GetPageSizeCached());
     void *res = MmapOrDie(size, "SizeClassAllocator32");
     MapUnmapCallback().OnMap((uptr)res, size);
     return res;
   }
 
   void UnmapWithCallback(uptr beg, uptr size) {
     MapUnmapCallback().OnUnmap(beg, size);
     UnmapOrDie(reinterpret_cast<void *>(beg), size);
private:
   uptr AllocateRegion(AllocatorStats *stat, uptr class_id) {
     CHECK_LT(class_id, kNumClasses);
     uptr res = reinterpret_cast<uptr>(MmapAlignedOrDieOnFatalError(
         kRegionSize, kRegionSize, "SizeClassAllocator32"));
     if (UNLIKELY(!res))
       return 0;
     MapUnmapCallback().OnMap(res, kRegionSize);
     stat->Add(AllocatorStatMapped, kRegionSize);
-    CHECK_EQ(0U, (res & (kRegionSize - 1)));
+    CHECK(IsAligned(res, kRegionSize));
     possible_regions.set(ComputeRegionId(res), static_cast<u8>(class_id));
     return res;
   }
 
   SizeClassInfo *GetSizeClassInfo(uptr class_id) {
     CHECK_LT(class_id, kNumClasses);
     return &size_class_info_array[class_id];
   }
 
   bool PopulateFreeList(AllocatorStats *stat, AllocatorCache *c,
                         SizeClassInfo *sci, uptr class_id) {
     uptr size = ClassIdToSize(class_id);
     uptr reg = AllocateRegion(stat, class_id);
     if (UNLIKELY(!reg))
       return false;
     uptr n_chunks = kRegionSize / (size + kMetadataSize);
     uptr max_count = TransferBatch::MaxCached(class_id);
+    CHECK_GT(max_count, 0);
     TransferBatch *b = nullptr;
     for (uptr i = reg; i < reg + n_chunks * size; i += size) {
       if (!b) {
         b = c->CreateBatch(class_id, this, (TransferBatch*)i);
-        if (!b)
+        if (UNLIKELY(!b))
           return false;
         b->Clear();
       }
       b->Add((void*)i);
       if (b->Count() == max_count) {
-        CHECK_GT(b->Count(), 0);
         sci->free_list.push_back(b);
         b = nullptr;
       }
     }
     if (b) {
       CHECK_GT(b->Count(), 0);
       sci->free_list.push_back(b);
     }
     return true;
   }
 
   ByteMap possible_regions;
   SizeClassInfo size_class_info_array[kNumClasses];
 };

lib/sanitizer_common/sanitizer_allocator_primary64.h

private:
 
   CompactPtrT *GetFreeArray(uptr region_beg) {
     return reinterpret_cast<CompactPtrT *>(region_beg + kRegionSize -
                                            kFreeArraySize);
   }
 
   bool MapWithCallback(uptr beg, uptr size) {
     uptr mapped = reinterpret_cast<uptr>(MmapFixedOrDieOnFatalError(beg, size));
-    if (!mapped)
+    if (UNLIKELY(!mapped))
       return false;
     CHECK_EQ(beg, mapped);
     MapUnmapCallback().OnMap(beg, size);
     return true;
   }
 
   void MapWithCallbackOrDie(uptr beg, uptr size) {
     CHECK_EQ(beg, reinterpret_cast<uptr>(MmapFixedOrDie(beg, size)));

lib/sanitizer_common/sanitizer_posix.cc

 }
 
 void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
   size = RoundUpTo(size, GetPageSizeCached());
   uptr res = internal_mmap(nullptr, size,
                            PROT_READ | PROT_WRITE,
                            MAP_PRIVATE | MAP_ANON, -1, 0);
   int reserrno;
-  if (internal_iserror(res, &reserrno))
+  if (UNLIKELY(internal_iserror(res, &reserrno)))
     ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno, raw_report);
   IncreaseTotalMmap(size);
   return (void *)res;
 }
 
 void UnmapOrDie(void *addr, uptr size) {
   if (!addr || !size) return;
   uptr res = internal_munmap(addr, size);
-  if (internal_iserror(res)) {
+  if (UNLIKELY(internal_iserror(res))) {
     Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n",
            SanitizerToolName, size, size, addr);
     CHECK("unable to unmap" && 0);
   }
   DecreaseTotalMmap(size);
 }
 
 void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
   size = RoundUpTo(size, GetPageSizeCached());
   uptr res = internal_mmap(nullptr, size,
                            PROT_READ | PROT_WRITE,
                            MAP_PRIVATE | MAP_ANON, -1, 0);
   int reserrno;
-  if (internal_iserror(res, &reserrno)) {
+  if (UNLIKELY(internal_iserror(res, &reserrno))) {
     if (reserrno == ENOMEM)
       return nullptr;
     ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno);
   }
   IncreaseTotalMmap(size);
   return (void *)res;
 }
 
 // We want to map a chunk of address space aligned to 'alignment'.
 // We do it by maping a bit more and then unmaping redundant pieces.
 // We probably can do it with fewer syscalls in some OS-dependent way.
 void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
                                    const char *mem_type) {
   CHECK(IsPowerOfTwo(size));
   CHECK(IsPowerOfTwo(alignment));
   uptr map_size = size + alignment;
   uptr map_res = (uptr)MmapOrDieOnFatalError(map_size, mem_type);
-  if (!map_res)
+  if (UNLIKELY(!map_res))
     return nullptr;
   uptr map_end = map_res + map_size;
   uptr res = map_res;
-  if (res & (alignment - 1))  // Not aligned.
-    res = (map_res + alignment) & ~(alignment - 1);
-  uptr end = res + size;
-  if (res != map_res)
+  if (!IsAligned(res, alignment)) {
+    res = (map_res + alignment - 1) & ~(alignment - 1);
     UnmapOrDie((void*)map_res, res - map_res);
+  }
+  uptr end = res + size;
   if (end != map_end)
     UnmapOrDie((void*)end, map_end - end);
   return (void*)res;
 }
 
 void *MmapNoReserveOrDie(uptr size, const char *mem_type) {
   uptr PageSize = GetPageSizeCached();
   uptr p = internal_mmap(nullptr,
                          RoundUpTo(size, PageSize),
                          PROT_READ | PROT_WRITE,
                          MAP_PRIVATE | MAP_ANON | MAP_NORESERVE,
                          -1, 0);
   int reserrno;
-  if (internal_iserror(p, &reserrno))
+  if (UNLIKELY(internal_iserror(p, &reserrno)))
     ReportMmapFailureAndDie(size, mem_type, "allocate noreserve", reserrno);
   IncreaseTotalMmap(size);
   return (void *)p;
 }
 
 void *MmapFixedImpl(uptr fixed_addr, uptr size, bool tolerate_enomem) {
   uptr PageSize = GetPageSizeCached();
   uptr p = internal_mmap((void*)(fixed_addr & ~(PageSize - 1)),
       RoundUpTo(size, PageSize),
       PROT_READ | PROT_WRITE,
       MAP_PRIVATE | MAP_ANON | MAP_FIXED,
       -1, 0);
   int reserrno;
-  if (internal_iserror(p, &reserrno)) {
+  if (UNLIKELY(internal_iserror(p, &reserrno))) {
     if (tolerate_enomem && reserrno == ENOMEM)
       return nullptr;
-    char mem_type[30];
+    char mem_type[40];
     internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx",
                       fixed_addr);
     ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno);
   }
   IncreaseTotalMmap(size);
   return (void *)p;
 }