[hwasan] Increase max allocation size to 1Tb.

[eugenis]

Summary

2Gb is unreasonably low on devices with 12Gb RAM and more.

[eugenis]

This costs 4 bytes per allocation.
We could save some memory by splitting primary and secondary allocator metadata types, but that would require significant code refactoring.

[eugenis]

attribute((packed))

[eugenis]

.

[pcc]

Do we even need a limit at this level any more if the implementation supports up to 2^63 sizes? I guess any address-space imposed limit would be enforced by mmap failing when we go to make a large secondary allocation.

[eugenis]

replaced attribute((packed)) with split storage

[eugenis]

I think it's good to have a limit so that allocation attempts with obviously wrong sizes are rejected with a clear message.

[pcc]

LGTM

In D89750#2340467, @eugenis wrote:

I think it's good to have a limit so that allocation attempts with obviously wrong sizes are rejected with a clear message.

Okay, fair enough.

compiler-rt/lib/hwasan/hwasan_allocator.h

 
 #if !defined(__aarch64__) && !defined(__x86_64__)
 #error Unsupported platform
 #endif
 
 namespace __hwasan {
 
 struct Metadata {
-  u32 requested_size : 31;  // sizes are < 2G.
-  u32 right_aligned  : 1;
+  u32 requested_size_low;
+  u32 requested_size_high : 31;
+  u32 right_aligned : 1;
   u32 alloc_context_id;
+  u64 get_requested_size() {
+    return (static_cast<u64>(requested_size_high) << 32) + requested_size_low;
+  }
+  void set_requested_size(u64 size) {
+    requested_size_low = size & ((1ul << 32) - 1);
+    requested_size_high = size >> 32;
+  }
 };
 
 struct HwasanMapUnmapCallback {
   void OnMap(uptr p, uptr size) const { UpdateMemoryUsage(); }
   void OnUnmap(uptr p, uptr size) const {
     // We are about to unmap a chunk of user memory.
     // It can return as user-requested mmap() or another thread stack.
     // Make it accessible with zero-tagged pointer.
     TagMemory(p, size, 0);
   }
 };
 
-static const uptr kMaxAllowedMallocSize = 2UL << 30;  // 2G
+static const uptr kMaxAllowedMallocSize = 1UL << 40;  // 1T
 
 struct AP64 {
   static const uptr kSpaceBeg = ~0ULL;
   static const uptr kSpaceSize = 0x2000000000ULL;
   static const uptr kMetadataSize = sizeof(Metadata);
   typedef __sanitizer::VeryDenseSizeClassMap SizeClassMap;
   using AddressSpaceView = LocalAddressSpaceView;
   typedef HwasanMapUnmapCallback MapUnmapCallback;

compiler-rt/lib/hwasan/hwasan_allocator.cpp

enum RightAlignMode {
   kRightAlignSometimes,
   kRightAlignAlways
 };
 
 // Initialized in HwasanAllocatorInit, an never changed.
 static ALIGNED(16) u8 tail_magic[kShadowAlignment - 1];
 
 bool HwasanChunkView::IsAllocated() const {
-  return metadata_ && metadata_->alloc_context_id && metadata_->requested_size;
+  return metadata_ && metadata_->alloc_context_id &&
+         metadata_->get_requested_size();
 }
 
 // Aligns the 'addr' right to the granule boundary.
 static uptr AlignRight(uptr addr, uptr requested_size) {
   uptr tail_size = requested_size % kShadowAlignment;
   if (!tail_size) return addr;
   return addr + kShadowAlignment - tail_size;
 }
 
 uptr HwasanChunkView::Beg() const {
   if (metadata_ && metadata_->right_aligned)
-    return AlignRight(block_, metadata_->requested_size);
+    return AlignRight(block_, metadata_->get_requested_size());
   return block_;
 }
 uptr HwasanChunkView::End() const {
   return Beg() + UsedSize();
 }
 uptr HwasanChunkView::UsedSize() const {
-  return metadata_->requested_size;
+  return metadata_->get_requested_size();
 }
 u32 HwasanChunkView::GetAllocStackId() const {
   return metadata_->alloc_context_id;
 }
 
 uptr HwasanChunkView::ActualSize() const {
   return allocator.GetActuallyAllocatedSize(reinterpret_cast<void *>(block_));
 }
static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment,
   if (UNLIKELY(!allocated)) {
     SetAllocatorOutOfMemory();
     if (AllocatorMayReturnNull())
       return nullptr;
     ReportOutOfMemory(size, stack);
   }
   Metadata *meta =
       reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
-  meta->requested_size = static_cast<u32>(orig_size);
+  meta->set_requested_size(orig_size);
   meta->alloc_context_id = StackDepotPut(*stack);
   meta->right_aligned = false;
   if (zeroise) {
     internal_memset(allocated, 0, size);
   } else if (flags()->max_malloc_fill_size > 0) {
     uptr fill_size = Min(size, (uptr)flags()->max_malloc_fill_size);
     internal_memset(allocated, flags()->malloc_fill_byte, fill_size);
   }
static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
   if (!PointerAndMemoryTagsMatch(tagged_ptr))
     ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr));
 
   void *untagged_ptr = UntagPtr(tagged_ptr);
   void *aligned_ptr = reinterpret_cast<void *>(
       RoundDownTo(reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment));
   Metadata *meta =
       reinterpret_cast<Metadata *>(allocator.GetMetaData(aligned_ptr));
-  uptr orig_size = meta->requested_size;
+  uptr orig_size = meta->get_requested_size();
   u32 free_context_id = StackDepotPut(*stack);
   u32 alloc_context_id = meta->alloc_context_id;
 
   // Check tail magic.
   uptr tagged_size = TaggedSize(orig_size);
   if (flags()->free_checks_tail_magic && orig_size &&
       tagged_size != orig_size) {
     uptr tail_size = tagged_size - orig_size - 1;
     CHECK_LT(tail_size, kShadowAlignment);
     void *tail_beg = reinterpret_cast<void *>(
         reinterpret_cast<uptr>(aligned_ptr) + orig_size);
     if (tail_size && internal_memcmp(tail_beg, tail_magic, tail_size))
       ReportTailOverwritten(stack, reinterpret_cast<uptr>(tagged_ptr),
                             orig_size, tail_magic);
   }
 
-  meta->requested_size = 0;
+  meta->set_requested_size(0);
   meta->alloc_context_id = 0;
   // This memory will not be reused by anyone else, so we are free to keep it
   // poisoned.
   Thread *t = GetCurrentThread();
   if (flags()->max_free_fill_size > 0) {
     uptr fill_size =
         Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size);
     internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size);
if (!PointerAndMemoryTagsMatch(tagged_ptr_old))
     ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr_old));
 
   void *tagged_ptr_new =
       HwasanAllocate(stack, new_size, alignment, false /*zeroise*/);
   if (tagged_ptr_old && tagged_ptr_new) {
     void *untagged_ptr_old =  UntagPtr(tagged_ptr_old);
     Metadata *meta =
         reinterpret_cast<Metadata *>(allocator.GetMetaData(untagged_ptr_old));
-    internal_memcpy(UntagPtr(tagged_ptr_new), untagged_ptr_old,
-                    Min(new_size, static_cast<uptr>(meta->requested_size)));
+    internal_memcpy(
+        UntagPtr(tagged_ptr_new), untagged_ptr_old,
+        Min(new_size, static_cast<uptr>(meta->get_requested_size())));
     HwasanDeallocate(stack, tagged_ptr_old);
   }
   return tagged_ptr_new;
 }
 
 static void *HwasanCalloc(StackTrace *stack, uptr nmemb, uptr size) {
   if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
     if (AllocatorMayReturnNull())
static uptr AllocationSize(const void *tagged_ptr) {
   Metadata *b = (Metadata *)allocator.GetMetaData(untagged_ptr);
   if (b->right_aligned) {
     if (beg != reinterpret_cast<void *>(RoundDownTo(
                    reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment)))
       return 0;
   } else {
     if (beg != untagged_ptr) return 0;
   }
-  return b->requested_size;
+  return b->get_requested_size();
 }
 
 void *hwasan_malloc(uptr size, StackTrace *stack) {
   return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false));
 }
 
 void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) {
   return SetErrnoOnNull(HwasanCalloc(stack, nmemb, size));

compiler-rt/test/hwasan/TestCases/allocator_returns_null.cpp

 
 #include "utils.h"
 
 int main(int argc, char **argv) {
   assert(argc == 2);
   const char *action = argv[1];
   untag_fprintf(stderr, "%s:\n", action);
 
-  static const size_t kMaxAllowedMallocSizePlusOne = (2UL << 30) + 1;
+  static const size_t kMaxAllowedMallocSizePlusOne = (1UL << 40) + 1;
 
   void *x = nullptr;
   if (!untag_strcmp(action, "malloc")) {
     x = malloc(kMaxAllowedMallocSizePlusOne);
   } else if (!untag_strcmp(action, "calloc")) {
     x = calloc((kMaxAllowedMallocSizePlusOne / 4) + 1, 4);
   } else if (!untag_strcmp(action, "calloc-overflow")) {
     volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max();

compiler-rt/test/hwasan/TestCases/sizes.cpp

 int main(int argc, char **argv) {
   assert(argc <= 3);
   bool test_size_max = argc == 3 && !untag_strcmp(argv[2], "max");
 
   static const size_t kMaxAllowedMallocSize = 1ULL << 40;
   static const size_t kChunkHeaderSize = 16;
 
   size_t MallocSize = test_size_max ? std::numeric_limits<size_t>::max()
-                                    : kMaxAllowedMallocSize;
+                                    : (kMaxAllowedMallocSize + 1);
 
   if (!untag_strcmp(argv[1], "malloc")) {
     void *p = malloc(MallocSize);
     assert(!p);
     p = malloc(kMaxAllowedMallocSize - kChunkHeaderSize);
     assert(!p);
   } else if (!untag_strcmp(argv[1], "calloc")) {
     // Trigger an overflow in calloc.