Make detect_invalid_pointer_pairs option to be tristate.

[marxin]

Summary

With the change, one can choose not to report comparison (or subtraction)
of a pointer with nullptr pointer.

[alekseyshl]

Is !nullptr expected to be special cased forever or it's transitional?

lib/asan/asan_flags.inc

141 ↗

(On Diff #127831)

Can we change it to match detect_odr_violation style?

int, detect_invalid_pointer_pairs, 0,
"If >= 2, detect operations like <, <=, >, >= and - on invalid pointer "
"pairs (e.g. when pointers belong to different objects); "
"If == 1, detect invalid operations only when both pointers are non-null.")

lib/asan/asan_report.cc

350 ↗

(On Diff #127831)

switch (flags()->detect_invalid_pointer_pairs) {
  case 0 : return;
  case 1 : if (p1 == nullptr || p2 == nullptr) return; break;
}

[marxin]

Update version where I addressed comments.
The tristate is not a transition, but it's design to have more fine control of the ASAN check.

[alekseyshl]

lib/asan/asan_report.cc

346 ↗

(On Diff #131206)

Well, now you don't need this check.

Also, move a1 and a2 declarations past switch statement.

[marxin]

Updated version that addresses the comment.

[marxin]

Fixed, can you please install the patch?

[alekseyshl]

lib/asan/asan_report.cc

346 ↗

(On Diff #131395)

Again, you do not need this check anymore, it's a part of switch statement now.

[marxin]

Comments fixed, please install it if you're fine with that.

lib/asan/asan_report.cc

346 ↗

(On Diff #131395)

No, sorry for overlooking that.

compiler-rt/trunk/lib/asan/asan_flags.inc

ASAN_FLAG(
     bool, start_deactivated, false,
     "If true, ASan tweaks a bunch of other flags (quarantine, redzone, heap "
     "poisoning) to reduce memory consumption as much as possible, and "
     "restores them to original values when the first instrumented module is "
     "loaded into the process. This is mainly intended to be used on "
     "Android. ")
 ASAN_FLAG(
     int, detect_invalid_pointer_pairs, 0,
-    "If non-zero, try to detect operations like <, <=, >, >= and - on "
-    "invalid pointer pairs (e.g. when pointers belong to different objects). "
-    "The bigger the value the harder we try.")
+    "If >= 2, detect operations like <, <=, >, >= and - on invalid pointer "
+    "pairs (e.g. when pointers belong to different objects); "
+    "If == 1, detect invalid operations only when both pointers are non-null.")
 ASAN_FLAG(
     bool, detect_container_overflow, true,
     "If true, honor the container overflow annotations. See "
     "https://github.com/google/sanitizers/wiki/AddressSanitizerContainerOverflow")
 ASAN_FLAG(int, detect_odr_violation, 2,
           "If >=2, detect violation of One-Definition-Rule (ODR); "
           "If ==1, detect ODR-violation only if the two variables "
           "have different sizes")

compiler-rt/trunk/lib/asan/asan_report.cc

if (t->GetStackVariableShadowStart(right) ||
       GetGlobalAddressInformation(right - 1, 0, &gdesc2))
     return true;
 
   // At this point we know nothing about both a1 and a2 addresses.
   return false;
 }
 
 static INLINE void CheckForInvalidPointerPair(void *p1, void *p2) {
-  if (!flags()->detect_invalid_pointer_pairs) return;
+  switch (flags()->detect_invalid_pointer_pairs) {
+    case 0 : return;
+    case 1 : if (p1 == nullptr || p2 == nullptr) return; break;
+  }
+
   uptr a1 = reinterpret_cast<uptr>(p1);
   uptr a2 = reinterpret_cast<uptr>(p2);
 
   if (IsInvalidPointerPair(a1, a2)) {
     GET_CALLER_PC_BP_SP;
     ReportInvalidPointerPair(pc, bp, sp, a1, a2);
   }
 }

compiler-rt/trunk/test/asan/TestCases/Posix/invalid-pointer-pairs-threads.cc

 // RUN: %clangxx_asan -O0 %s -pthread -o %t -mllvm -asan-detect-invalid-pointer-pair
 
-// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1 %run %t a 2>&1 | FileCheck %s -check-prefix=OK -allow-empty
-// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1 not %run %t b 2>&1 | FileCheck %s -check-prefix=B
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=2 %run %t a 2>&1 | FileCheck %s -check-prefix=OK -allow-empty
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=2 not %run %t b 2>&1 | FileCheck %s -check-prefix=B
 
 // pthread barriers are not available on OS X
 // UNSUPPORTED: darwin
 
 #include <assert.h>
 #include <pthread.h>
 #include <stdlib.h>
 #include <unistd.h>

compiler-rt/trunk/test/asan/TestCases/invalid-pointer-pairs-compare-errors.cc

 // RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
 
-// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1:halt_on_error=0 %run %t 2>&1 | FileCheck %s
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=2:halt_on_error=0 %run %t 2>&1 | FileCheck %s
 
 #include <assert.h>
 #include <stdlib.h>
 
 int foo(char *p, char *q) {
   return p > q;
 }
 

compiler-rt/trunk/test/asan/TestCases/invalid-pointer-pairs-compare-null.cc

+// RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
+
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1 %run %t
+
+#include <assert.h>
+#include <stdlib.h>
+
+int foo(char *p, char *q) {
+  return p <= q;
+}
+
+char global[8192] = {};
+char small_global[7] = {};
+
+int main() {
+  // Heap allocated memory.
+  char *p = (char *)malloc(42);
+  int r = foo(p, nullptr);
+  free(p);
+
+  p = (char *)malloc(1024);
+  foo(nullptr, p);
+  free(p);
+
+  p = (char *)malloc(4096);
+  foo(p, nullptr);
+  free(p);
+
+  // Global variable.
+  foo(&global[0], nullptr);
+  foo(&global[1000], nullptr);
+
+  p = &small_global[0];
+  foo(p, nullptr);
+
+  // Stack variable.
+  char stack[10000];
+  foo(&stack[0], nullptr);
+  foo(nullptr, &stack[9000]);
+
+  return 0;
+}

compiler-rt/trunk/test/asan/TestCases/invalid-pointer-pairs-compare-success.cc

 // RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
 
-// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1 %run %t
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=2 %run %t
 
 #include <assert.h>
 #include <stdlib.h>
 
 int foo(char *p) {
   char *p2 = p + 20;
   return p > p2;
 }

compiler-rt/trunk/test/asan/TestCases/invalid-pointer-pairs-subtract-errors.cc

 // RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
 
-// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1:halt_on_error=0 %run %t 2>&1 | FileCheck %s
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=2:halt_on_error=0 %run %t 2>&1 | FileCheck %s
 
 #include <assert.h>
 #include <stdlib.h>
 
 int foo(char *p, char *q) {
   return p - q;
 }
 

compiler-rt/trunk/test/asan/TestCases/invalid-pointer-pairs-subtract-success.cc

 // RUN: %clangxx_asan -O0 %s -o %t -mllvm -asan-detect-invalid-pointer-pair
 
-// RUN: %env_asan_opts=detect_invalid_pointer_pairs=1 %run %t
+// RUN: %env_asan_opts=detect_invalid_pointer_pairs=2 %run %t
 
 #include <assert.h>
 #include <stdlib.h>
 
 int bar(char *p, char *q) {
   return p - q;
 }