[hwasan] Display causes in order of probability.

[fmayer]

Summary

A heap or global buffer that is far away from the faulting address is
unlikely to be the cause, especially if there is a potential
use-after-free as well, so we want to show it after the other
causes.

[fmayer]

Only treat linear overflows as very likely.

[eugenis]

@pcc @hctim
I'd like to have a test that a far overflow is not printed where there is a valid use after free.
You can do it by

• allocating in a loop until you get too allocations that are nearby but not adjacent
• faking the tags so they match (__hwasan_tag_memory can do that)
• deallocating one and touching it

hwasan does not record the original allocation tag anywhere (but it records "free" in the history buffer)

compiler-rt/lib/hwasan/hwasan_report.cpp

318

Make it "Cause:" to match the MTE output:
https://cs.android.com/android/platform/superproject/+/master:system/core/debuggerd/libdebuggerd/tombstone_proto_to_text.cpp;l=263;drc=78f0670ddadb98c270cc0a0115bb4da4e5d80a95

Add an explanation about multiple possible causes somewhere. It seems hard to count them before printing, so I'm ok with the note going to the end of the report.

420

The only way to "describe" a stack location is through the history buffer. ShowCandidate should iterate over it looking for a matching tag and address (but unlike use-after-return, the address would be the suspected overflow address, not the fault address, and the tag would match the current memory tag).

This becomes complicated in the offline symbolization case (see the hwasan_symbolize script). Unsymbolized online reporting would need to print tag and address of the suspected original allocation that can be matched against offline symbolized history.

This is out of scope of this change.

474

I suggest we do not print "far" overflow candidates if there is any other alternative at all (i.e. num_descriptions > 0).

[fmayer]

Add test.

[fmayer]

Added a test. Verified it fails with the previous logic and passes with the new one.

[fmayer]

Remove leftover print in test.

[eugenis]

LGTM with the test changes

compiler-rt/test/hwasan/TestCases/use-after-free-and-overflow.c

23

distance is in bytes, you probably want something more than 1.

40

break if other != nullptr?

46

This can flake if any of the adjacent allocations have a tag of 3.

Maybe iterate over possible tag values until __hwasan_test_shadow both on the left and on the right returns an error?

[fmayer]

Improve test.

compiler-rt/test/hwasan/TestCases/use-after-free-and-overflow.c

46

I think it's easier to just tag the two surrounding bytes to prevent the flake.

[fmayer]

Add comment to test.

[eugenis]

LGTM

compiler-rt/test/hwasan/TestCases/use-after-free-and-overflow.c

20

nit: single statement blocks do not use {} in LLVM

[fmayer]

Address formatting nits.

compiler-rt/lib/hwasan/hwasan_report.cpp

static uptr GetGlobalSizeFromDescriptor(uptr ptr) {
   for (const hwasan_global &global :
        HwasanGlobalsFor(load_bias, phdr_begin, ehdr->e_phnum))
     if (global.addr() <= ptr && ptr < global.addr() + global.size())
       return global.size();
 
   return 0;
 }
 
-void PrintAddressDescription(
-    uptr tagged_addr, uptr access_size,
-    StackAllocationsRingBuffer *current_stack_allocations) {
+static void ShowCandidate(uptr untagged_addr, tag_t *candidate, tag_t *left,
+                          tag_t *right) {
   Decorator d;
-  int num_descriptions_printed = 0;
-  uptr untagged_addr = UntagAddr(tagged_addr);
-
-  // Print some very basic information about the address, if it's a heap.
-  HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr);
-  if (uptr beg = chunk.Beg()) {
-    uptr size = chunk.ActualSize();
-    Printf("%s[%p,%p) is a %s %s heap chunk; "
-           "size: %zd offset: %zd\n%s",
-           d.Location(),
-           beg, beg + size,
-           chunk.FromSmallHeap() ? "small" : "large",
-           chunk.IsAllocated() ? "allocated" : "unallocated",
-           size, untagged_addr - beg,
-           d.Default());
-  }
-
-  // Check if this looks like a heap buffer overflow by scanning
-  // the shadow left and right and looking for the first adjacent
-  // object with a different memory tag. If that tag matches addr_tag,
-  // check the allocator if it has a live chunk there.
-  tag_t addr_tag = GetTagFromPointer(tagged_addr);
-  tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr));
-  tag_t *candidate = nullptr, *left = tag_ptr, *right = tag_ptr;
-  for (int i = 0; i < 1000; i++) {
-    if (TagsEqual(addr_tag, left)) {
-      candidate = left;
-      break;
-    }
-    --left;
-    if (TagsEqual(addr_tag, right)) {
-      candidate = right;
-      break;
-    }
-    ++right;
-  }
-
-  if (candidate) {
-    uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate));
-    HwasanChunkView chunk = FindHeapChunkByAddress(mem);
-    if (chunk.IsAllocated()) {
-      uptr offset;
-      const char *whence;
-      if (untagged_addr < chunk.End() && untagged_addr >= chunk.Beg()) {
-        offset = untagged_addr - chunk.Beg();
-        whence = "inside";
-      } else if (candidate == left) {
-        offset = untagged_addr - chunk.End();
-        whence = "to the right of";
-      } else {
-        offset = chunk.Beg() - untagged_addr;
-        whence = "to the left of";
-      }
-      Printf("%s", d.Location());
-      Printf("%p is located %zd bytes %s %zd-byte region [%p,%p)\n",
-             untagged_addr, offset, whence, chunk.UsedSize(), chunk.Beg(),
-             chunk.End());
-      Printf("%s", d.Allocation());
-      Printf("allocated here:\n");
-      Printf("%s", d.Default());
-      GetStackTraceFromId(chunk.GetAllocStackId()).Print();
-      num_descriptions_printed++;
-    } else {
-      // Check whether the address points into a loaded library. If so, this is
-      // most likely a global variable.
-      const char *module_name;
-      uptr module_address;
-      Symbolizer *sym = Symbolizer::GetOrInit();
-      if (sym->GetModuleNameAndOffsetForPC(mem, &module_name,
-                                           &module_address)) {
-        DataInfo info;
-        if (sym->SymbolizeData(mem, &info) && info.start) {
-          Printf(
-              "%p is located %zd bytes to the %s of %zd-byte global variable "
-              "%s [%p,%p) in %s\n",
-              untagged_addr,
-              candidate == left ? untagged_addr - (info.start + info.size)
-                                : info.start - untagged_addr,
-              candidate == left ? "right" : "left", info.size, info.name,
-              info.start, info.start + info.size, module_name);
-        } else {
-          uptr size = GetGlobalSizeFromDescriptor(mem);
-          if (size == 0)
-            // We couldn't find the size of the global from the descriptors.
-            Printf(
-                "%p is located to the %s of a global variable in (%s+0x%x)\n",
-                untagged_addr, candidate == left ? "right" : "left",
-                module_name, module_address);
-          else
-            Printf(
-                "%p is located to the %s of a %zd-byte global variable in "
-                "(%s+0x%x)\n",
-                untagged_addr, candidate == left ? "right" : "left", size,
-                module_name, module_address);
-        }
-        num_descriptions_printed++;
-      }
-    }
+    Printf("%s", d.Error());
+    Printf("\nCause: heap-buffer-overflow\n");
+    Printf("%s", d.Default());
+    return;
+  }
+  if (sym->GetModuleNameAndOffsetForPC(mem, &module_name, &module_address)) {
+    Printf("%s", d.Error());
+    Printf("\nCause: global-overflow\n");
+    Printf("%s", d.Default());
+    Printf("%s", d.Location());
+        Printf("%p is located to the %s of a global variable in (%s+0x%x)\n",
+               untagged_addr, candidate == left ? "right" : "left", module_name,
+               module_address);
+    Printf("%s", d.Default());
+  }
+}
+
+void PrintAddressDescription(
+    uptr tagged_addr, uptr access_size,
+    StackAllocationsRingBuffer *current_stack_allocations) {
+  Decorator d;
+  int num_descriptions_printed = 0;
+  uptr untagged_addr = UntagAddr(tagged_addr);
+
+  // Print some very basic information about the address, if it's a heap.
+  HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr);
+  if (uptr beg = chunk.Beg()) {
+    uptr size = chunk.ActualSize();
+    Printf("%s[%p,%p) is a %s %s heap chunk; "
+           "size: %zd offset: %zd\n%s",
+           d.Location(),
+           beg, beg + size,
+           chunk.FromSmallHeap() ? "small" : "large",
+           chunk.IsAllocated() ? "allocated" : "unallocated",
+           size, untagged_addr - beg,
+           d.Default());
+  }
+
+  // Check if this looks like a heap buffer overflow by scanning
+  // the shadow left and right and looking for the first adjacent
+  // object with a different memory tag. If that tag matches addr_tag,
+  // check the allocator if it has a live chunk there.
+  tag_t addr_tag = GetTagFromPointer(tagged_addr);
+  tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr));
+  tag_t *candidate = nullptr, *left = tag_ptr, *right = tag_ptr;
+  uptr candidate_distance = 0;
+  for (; candidate_distance < 1000; candidate_distance++) {
+    if (TagsEqual(addr_tag, left)) {
+      candidate = left;
+      break;
+    }
+    --left;
+    if (TagsEqual(addr_tag, right)) {
+      candidate = right;
+      break;
+    }
+    ++right;
+
+  constexpr auto kCloseCandidateDistance = 1;
+
+  if (candidate && candidate_distance <= kCloseCandidateDistance) {
+    ShowCandidate(untagged_addr, candidate, left, right);
+    num_descriptions_printed++;
+
+  hwasanThreadList().VisitAllLiveThreads([&](Thread *t) {
+    if (t->AddrIsInStack(untagged_addr)) {
+      // TODO(fmayer): figure out how to distinguish use-after-return and
+      // stack-buffer-overflow.
+      Printf("%s", d.Error());
+      Printf("\nCause: stack tag-mismatch\n");
+      Printf("%s", d.Location());
+      Printf("Address %p is located in stack of thread T%zd\n", untagged_addr,
+             t->unique_id());
+      Printf("%s", d.Default());
+      t->Announce();
+
+      auto *sa = (t == GetCurrentThread() && current_stack_allocations)
+                     ? current_stack_allocations
+                     : t->stack_allocations();
+      PrintStackAllocations(sa, addr_tag, untagged_addr);
+      num_descriptions_printed++;
     }
+  });
 
   hwasanThreadList().VisitAllLiveThreads([&](Thread *t) {
     // Scan all threads' ring buffers to find if it's a heap-use-after-free.
     HeapAllocationRecord har;
     uptr ring_index, num_matching_addrs, num_matching_addrs_4b;
     if (FindHeapAllocation(t->heap_allocations(), tagged_addr, &har,
                            &ring_index, &num_matching_addrs,
                            &num_matching_addrs_4b)) {
+      Printf("%s", d.Error());
+      Printf("\nCause: use-after-free\n");
       Printf("%s", d.Location());
       Printf("%p is located %zd bytes inside of %zd-byte region [%p,%p)\n",
              untagged_addr, untagged_addr - UntagAddr(har.tagged_addr),
              har.requested_size, UntagAddr(har.tagged_addr),
              UntagAddr(har.tagged_addr) + har.requested_size);
       Printf("%s", d.Allocation());
       Printf("freed by thread T%zd here:\n", t->unique_id());
       Printf("%s", d.Default());
if (FindHeapAllocation(t->heap_allocations(), tagged_addr, &har,
              flags()->heap_history_size);
       Printf("hwasan_dev_note_num_matching_addrs: %zd\n", num_matching_addrs);
       Printf("hwasan_dev_note_num_matching_addrs_4b: %zd\n",
              num_matching_addrs_4b);
 
       t->Announce();
       num_descriptions_printed++;
     }
+  });
 
-    // Very basic check for stack memory.
-    if (t->AddrIsInStack(untagged_addr)) {
-      Printf("%s", d.Location());
-      Printf("Address %p is located in stack of thread T%zd\n", untagged_addr,
-             t->unique_id());
-      Printf("%s", d.Default());
-      t->Announce();
-
-      auto *sa = (t == GetCurrentThread() && current_stack_allocations)
-                     ? current_stack_allocations
-                     : t->stack_allocations();
-      PrintStackAllocations(sa, addr_tag, untagged_addr);
-      num_descriptions_printed++;
-    }
-  });
+  if (candidate && num_descriptions_printed == 0) {
+    ShowCandidate(untagged_addr, candidate, left, right);
 
   // Print the remaining threads, as an extra information, 1 line per thread.
   hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { t->Announce(); });
 
   if (!num_descriptions_printed)
     // We exhausted our possibilities. Bail out.
     Printf("HWAddressSanitizer can not describe address in more detail.\n");
+  if (num_descriptions_printed > 1) {
+    Printf(
+        "There are %d potential causes, printed above in order "
+        "of likeliness.",
+        num_descriptions_printed);
+  }
 }
 
 void ReportStats() {}
 
 static void PrintTagInfoAroundAddr(tag_t *tag_ptr, uptr num_rows,
                                    void (*print_tag)(InternalScopedString &s,
                                                      tag_t *tag)) {
   const uptr row_len = 16;  // better be power of two.

compiler-rt/test/hwasan/TestCases/global.c

 // RUN: %clang_hwasan %s -o %t
 // RUN: %run %t 0
 // RUN: not %run %t 1 2>&1 | FileCheck --check-prefixes=CHECK,RSYM %s
 // RUN: not %env_hwasan_opts=symbolize=0 %run %t 1 2>&1 | FileCheck --check-prefixes=CHECK,RNOSYM %s
 // RUN: not %run %t -1 2>&1 | FileCheck --check-prefixes=CHECK,LSYM %s
 // RUN: not %env_hwasan_opts=symbolize=0 %run %t -1 2>&1 | FileCheck --check-prefixes=CHECK,LNOSYM %s
 
 // REQUIRES: pointer-tagging
 
 int x = 1;
 
 int main(int argc, char **argv) {
+  // CHECK: Cause: global-overflow
   // RSYM: is located 0 bytes to the right of 4-byte global variable x {{.*}} in {{.*}}global.c.tmp
   // RNOSYM: is located to the right of a 4-byte global variable in ({{.*}}global.c.tmp+{{.*}})
   // LSYM: is located 4 bytes to the left of 4-byte global variable x {{.*}} in {{.*}}global.c.tmp
   // LNOSYM: is located to the left of a 4-byte global variable in ({{.*}}global.c.tmp+{{.*}})
   // CHECK-NOT: can not describe
   (&x)[atoi(argv[1])] = 1;
 }

compiler-rt/test/hwasan/TestCases/heap-buffer-overflow.c

int main(int argc, char **argv) {
   sink = x[offset];
 
 #if defined(__x86_64__)
   // Aliasing mode doesn't support the secondary allocator, so we fake a HWASan
   // report instead of disabling the entire test.
   if (size == 1000000) {
     fprintf(stderr, "is a large allocated heap chunk; size: 1003520 offset: %d\n",
             offset);
+    fprintf(stderr, "Cause: heap-buffer-overflow\n");
     fprintf(stderr, "is located %s of 1000000-byte region\n",
             offset == -30 ? "30 bytes to the left" : "0 bytes to the right");
     return -1;
   }
 #endif
 
 // CHECK40: allocated heap chunk; size: 32 offset: 8
+// CHECK40: Cause: heap-buffer-overflow
 // CHECK40: is located 10 bytes to the right of 30-byte region
 //
 // CHECK80: allocated heap chunk; size: 32 offset: 16
+// CHECK80: Cause: heap-buffer-overflow
 // CHECK80: is located 50 bytes to the right of 30-byte region
 //
+// CHECKm30: Cause: heap-buffer-overflow
 // CHECKm30: is located 30 bytes to the left of 30-byte region
 //
 // CHECKMm30: is a large allocated heap chunk; size: 1003520 offset: -30
+// CHECKMm30: Cause: heap-buffer-overflow
 // CHECKMm30: is located 30 bytes to the left of 1000000-byte region
 //
 // CHECKM: is a large allocated heap chunk; size: 1003520 offset: 1000000
+// CHECKM: Cause: heap-buffer-overflow
 // CHECKM: is located 0 bytes to the right of 1000000-byte region
 //
 // CHECK31: tags: [[TAG:..]]/0e (ptr/mem)
+// CHECK31: Cause: heap-buffer-overflow
 // CHECK31: is located 1 bytes to the right of 30-byte region
 // CHECK31: Memory tags around the buggy address
 // CHECK31: [0e]
 // CHECK31: Tags for short granules around the buggy address
 // CHECK31: {{\[}}[[TAG]]]
 //
+// CHECK20: Cause: heap-buffer-overflow
 // CHECK20: is located 10 bytes to the right of 20-byte region [0x{{.*}}0,0x{{.*}}4)
   free(x);
 }

compiler-rt/test/hwasan/TestCases/stack-oob.c

int f() {
   return p[SIZE];
 }
 
 int main() {
   f();
   // CHECK: READ of size 1 at
   // CHECK: #0 {{.*}} in f{{.*}}stack-oob.c:[[@LINE-6]]
 
+  // CHECK: Cause: stack tag-mismatch
   // CHECK: is located in stack of threa
 
   // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in f
 }

compiler-rt/test/hwasan/TestCases/stack-uar.c

 int main() {
   char *p = buggy();
   Unrelated1();
   Unrelated2();
   Unrelated3();
   return *p;
   // CHECK: READ of size 1 at
   // CHECK: #0 {{.*}} in main{{.*}}stack-uar.c:[[@LINE-2]]
+  // CHECK: Cause: stack tag-mismatch
   // CHECK: is located in stack of thread
   // CHECK: Potentially referenced stack objects:
-  // CHECK-NEXT: zzz in buggy {{.*}}stack-uar.c:[[@LINE-19]]
+  // CHECK-NEXT: zzz in buggy {{.*}}stack-uar.c:[[@LINE-20]]
   // CHECK-NEXT: Memory tags around the buggy address
 
   // NOSYM: Previously allocated frames:
   // NOSYM-NEXT: record_addr:0x{{.*}} record:0x{{.*}} ({{.*}}/stack-uar.c.tmp+0x{{.*}}){{$}}
   // NOSYM-NEXT: record_addr:0x{{.*}} record:0x{{.*}} ({{.*}}/stack-uar.c.tmp+0x{{.*}}){{$}}
   // NOSYM-NEXT: record_addr:0x{{.*}} record:0x{{.*}} ({{.*}}/stack-uar.c.tmp+0x{{.*}}){{$}}
   // NOSYM-NEXT: record_addr:0x{{.*}} record:0x{{.*}} ({{.*}}/stack-uar.c.tmp+0x{{.*}}){{$}}
   // NOSYM-NEXT: Memory tags around the buggy address
 
   // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch {{.*}} in main
 }

compiler-rt/test/hwasan/TestCases/thread-uaf.c

void *Deallocate(void *arg) {
   return NULL;
 }
 
 void *Use(void *arg) {
   x[5] = 42;
   // CHECK: ERROR: HWAddressSanitizer: tag-mismatch on address
   // CHECK: WRITE of size 1 {{.*}} in thread T3
   // CHECK: thread-uaf.c:[[@LINE-3]]
+  // CHECK: Cause: use-after-free
   // CHECK: freed by thread T2 here
   // CHECK: in Deallocate
   // CHECK: previously allocated here:
   // CHECK: in Allocate
   // CHECK-DAG: Thread: T2 0x
   // CHECK-DAG: Thread: T3 0x
   // CHECK-DAG: Thread: T0 0x
   // CHECK-DAG: Thread: T1 0x

compiler-rt/test/hwasan/TestCases/use-after-free-and-overflow.c

+// Checks that we do not print a faraway buffer overrun if we find a
+// use-after-free.
+// RUN: %clang_hwasan -O0 %s -o %t
+// RUN: not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK
+// REQUIRES: stable-runtime
+
+#include <sanitizer/hwasan_interface.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define ALLOC_ATTEMPTS 256
+
+char *Untag(void *x) {
+  return (char *)__hwasan_tag_pointer(x, 0);
+}
+
+void *FindMatch(void *ptrs[ALLOC_ATTEMPTS], void *value) {
+  for (int i = 0; i < ALLOC_ATTEMPTS; ++i) {
+    if (!ptrs[i]) {
+      return NULL;
+    }
+    int distance = Untag(value) - Untag(ptrs[i]);
+    if (abs(distance) < 1000 && abs(distance) > 1) {
+      return ptrs[i];
+    }
+  }
+  return NULL;
+}
+
+int main(int argc, char **argv) {
+  __hwasan_enable_allocator_tagging();
+  void *ptrs[ALLOC_ATTEMPTS] = {};
+  // Find two allocations that are close enough so that they would be
+  // candidates as buffer overflows for each other.
+  void *one;
+  void *other;
+  for (int i = 0; i < ALLOC_ATTEMPTS; ++i) {
+    one = malloc(16);
+    other = FindMatch(ptrs, one);
+    ptrs[i] = one;
+  }
+  if (!other) {
+    fprintf(stderr, "Could not find closeby allocations.\n");
+    abort();
+  }
+  __hwasan_tag_memory(Untag(one), 3, 16);
+  __hwasan_tag_memory(Untag(other), 3, 16);
+  void *retagged_one = __hwasan_tag_pointer(one, 3);
+  free(retagged_one);
+  volatile char *ptr = (char *)retagged_one;
+  *ptr = 1;
+}
+
+// CHECK-NOT: Cause: heap-buffer-overflow
+// CHECK: Cause: use-after-free
+// CHECK-NOT: Cause: heap-buffer-overflow

compiler-rt/test/hwasan/TestCases/use-after-free.c

int main() {
   fprintf(stderr, ISREAD ? "Going to do a READ\n" : "Going to do a WRITE\n");
   // CHECK: Going to do a [[TYPE:[A-Z]*]]
   int r = 0;
   if (ISREAD) r = x[5]; else x[5] = 42;  // should be on the same line.
   // CHECK: [[TYPE]] of size 1 at {{.*}} tags: [[PTR_TAG:[0-9a-f][0-9a-f]]]/[[MEM_TAG:[0-9a-f][0-9a-f]]] (ptr/mem)
   // CHECK: #{{[0-9]}} {{.*}} in main {{.*}}use-after-free.c:[[@LINE-2]]
   // Offset is 5 or 11 depending on left/right alignment.
   // CHECK: is a small unallocated heap chunk; size: 32 offset: {{5|11}}
+  // CHECK: Cause: use-after-free
   // CHECK: is located 5 bytes inside of 10-byte region
   //
   // CHECK: freed by thread {{.*}} here:
   // CHECK: #0 {{.*}} in {{.*}}free{{.*}} {{.*}}hwasan_allocation_functions.cpp
-  // CHECK: #1 {{.*}} in main {{.*}}use-after-free.c:[[@LINE-14]]
+  // CHECK: #1 {{.*}} in main {{.*}}use-after-free.c:[[@LINE-15]]
 
   // CHECK: previously allocated here:
   // CHECK: #0 {{.*}} in {{.*}}malloc{{.*}} {{.*}}hwasan_allocation_functions.cpp
-  // CHECK: #1 {{.*}} in main {{.*}}use-after-free.c:[[@LINE-19]]
+  // CHECK: #1 {{.*}} in main {{.*}}use-after-free.c:[[@LINE-20]]
   // CHECK: Memory tags around the buggy address (one tag corresponds to 16 bytes):
   // CHECK: =>{{.*}}[[MEM_TAG]]
   // CHECK: SUMMARY: HWAddressSanitizer: tag-mismatch
   return r;
 }