[ASan] Add use_madv_dontdump flag.

[ygribov]

Summary

This patch implements use_madv_dontdump flag suggested in https://code.google.com/p/address-sanitizer/issues/detail?id=345 . The flag disables dumping of shadow memory to corefile (which takes hours on 64-bit platforms). Tested on Linux x64.

I didn't add a testcase - generating a predictable corefile is hard given the zoo of modern crash reporters. E.g. see https://bugs.launchpad.net/ubuntu/+source/apport/+bug/1346497 (Apport from stock Ubuntu 14 multiples core limit by 1000).

[ygribov]

Perhaps also disable_core=0 on 64-bit in this patch?

[kcc]

lib/asan/asan_flags.inc

80

Please move the flag to sanitizer_common.
Even if you then use it only in asan, we may later want to use it in msan/tsan
(but you are welcome to implement that too, of course)

[ygribov]

Also implement for MSan and TSan.

[kcc]

LGTM
Thanks!

lib/asan/asan_flags.inc

ASAN_FLAG(
     "Number of seconds to sleep between printing an error report and "
     "terminating the program. Useful for debugging purposes (e.g. when one "
     "needs to attach gdb).")
 ASAN_FLAG(bool, check_malloc_usable_size, true,
           "Allows the users to work around the bug in Nvidia drivers prior to "
           "295.*.")
 ASAN_FLAG(bool, unmap_shadow_on_exit, false,
           "If set, explicitly unmaps the (huge) shadow at exit.")
+ASAN_FLAG(bool, use_madv_dontdump, true,
+          "If set, instructs kernel to not store the (huge) shadow "
+          "in core file.")
 ASAN_FLAG(
     bool, abort_on_error, false,
     "If set, the tool calls abort() instead of _exit() after printing the "
     "error report.")
 ASAN_FLAG(bool, print_stats, false,
           "Print various statistics after printing an error message or if "
           "atexit=1.")
 ASAN_FLAG(bool, print_legend, true, "Print the legend for the shadow bytes.")

lib/asan/asan_rtl.cc

void ReserveShadowMemoryRange(uptr beg, uptr end) {
   void *res = MmapFixedNoReserve(beg, size);
   if (res != (void*)beg) {
     Report("ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
            "Perhaps you're using ulimit -v\n", size);
     Abort();
   }
   if (common_flags()->no_huge_pages_for_shadow)
     NoHugePagesInRegion(beg, size);
+  if (flags()->use_madv_dontdump)
+    DontDumpShadowMemory(beg, size);
 }
 
 // --------------- LowLevelAllocateCallbac ---------- {{{1
 static void OnLowLevelAllocate(uptr ptr, uptr size) {
   PoisonShadow(ptr, size, kAsanInternalHeapMagic);
 }
 
 // -------------------------- Run-time entry ------------------- {{{1

lib/sanitizer_common/sanitizer_common.h

 void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type);
 // Used to check if we can map shadow memory to a fixed location.
 bool MemoryRangeIsAvailable(uptr range_start, uptr range_end);
 void FlushUnneededShadowMemory(uptr addr, uptr size);
 void IncreaseTotalMmap(uptr size);
 void DecreaseTotalMmap(uptr size);
 uptr GetRSS();
 void NoHugePagesInRegion(uptr addr, uptr length);
+void DontDumpShadowMemory(uptr addr, uptr length);
 
 // InternalScopedBuffer can be used instead of large stack arrays to
 // keep frame size low.
 // FIXME: use InternalAlloc instead of MmapOrDie once
 // InternalAlloc is made libc-free.
 template<typename T>
 class InternalScopedBuffer {
  public:

lib/sanitizer_common/sanitizer_posix_libcdep.cc

 }
 
 void NoHugePagesInRegion(uptr addr, uptr size) {
 #ifdef MADV_NOHUGEPAGE  // May not be defined on old systems.
   madvise((void *)addr, size, MADV_NOHUGEPAGE);
 #endif  // MADV_NOHUGEPAGE
 }
 
+void DontDumpShadowMemory(uptr addr, uptr length) {
+#ifdef MADV_DONTDUMP
+  madvise((void *)addr, length, MADV_DONTDUMP);
+#endif
+}
+
 static rlim_t getlim(int res) {
   rlimit rlim;
   CHECK_EQ(0, getrlimit(res, &rlim));
   return rlim.rlim_cur;
 }
 
 static void setlim(int res, rlim_t lim) {
   // The following magic is to prevent clang from replacing it with memset.

lib/sanitizer_common/sanitizer_win.cc

 
 void *Mprotect(uptr fixed_addr, uptr size) {
   return VirtualAlloc((LPVOID)fixed_addr, size,
                       MEM_RESERVE | MEM_COMMIT, PAGE_NOACCESS);
 }
 
 void FlushUnneededShadowMemory(uptr addr, uptr size) {
   // This is almost useless on 32-bits.
-  // FIXME: add madvice-analog when we move to 64-bits.
+  // FIXME: add madvise-analog when we move to 64-bits.
 }
 
 void NoHugePagesInRegion(uptr addr, uptr size) {
   // FIXME: probably similar to FlushUnneededShadowMemory.
 }
 
+void DontDumpShadowMemory(uptr addr, uptr length) {
+  // This is almost useless on 32-bits.
+  // FIXME: add madvise-analog when we move to 64-bits.
+}
+
 bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
   MEMORY_BASIC_INFORMATION mbi;
   CHECK(VirtualQuery((void *)range_start, &mbi, sizeof(mbi)));
   return mbi.Protect & PAGE_NOACCESS &&
          (uptr)mbi.BaseAddress + mbi.RegionSize >= range_end;
 }
 
 void *MapFileToMemory(const char *file_name, uptr *buff_size) {