[ASan] Add use_madv_dontdump flag.

[ygribov]

Summary

This patch implements use_madv_dontdump flag suggested in https://code.google.com/p/address-sanitizer/issues/detail?id=345 . The flag disables dumping of shadow memory to corefile (which takes hours on 64-bit platforms). Tested on Linux x64.

I didn't add a testcase - generating a predictable corefile is hard given the zoo of modern crash reporters. E.g. see https://bugs.launchpad.net/ubuntu/+source/apport/+bug/1346497 (Apport from stock Ubuntu 14 multiples core limit by 1000).

[ygribov]

Perhaps also disable_core=0 on 64-bit in this patch?

[kcc]

lib/asan/asan_flags.inc

80 ↗

(On Diff #19035)

Please move the flag to sanitizer_common.
Even if you then use it only in asan, we may later want to use it in msan/tsan
(but you are welcome to implement that too, of course)

[ygribov]

Also implement for MSan and TSan.

[kcc]

LGTM
Thanks!

compiler-rt/trunk/lib/asan/asan_rtl.cc

void ReserveShadowMemoryRange(uptr beg, uptr end) {
   void *res = MmapFixedNoReserve(beg, size);
   if (res != (void*)beg) {
     Report("ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
            "Perhaps you're using ulimit -v\n", size);
     Abort();
   }
   if (common_flags()->no_huge_pages_for_shadow)
     NoHugePagesInRegion(beg, size);
+  if (common_flags()->use_madv_dontdump)
+    DontDumpShadowMemory(beg, size);
 }
 
 // --------------- LowLevelAllocateCallbac ---------- {{{1
 static void OnLowLevelAllocate(uptr ptr, uptr size) {
   PoisonShadow(ptr, size, kAsanInternalHeapMagic);
 }
 
 // -------------------------- Run-time entry ------------------- {{{1

compiler-rt/trunk/lib/msan/msan_linux.cc

for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
     uptr start = kMemoryLayout[i].start;
     uptr end = kMemoryLayout[i].end;
     uptr size= end - start;
     MappingDesc::Type type = kMemoryLayout[i].type;
     if ((map_shadow && type == MappingDesc::SHADOW) ||
         (init_origins && type == MappingDesc::ORIGIN)) {
       if (!CheckMemoryRangeAvailability(start, size)) return false;
       if ((uptr)MmapFixedNoReserve(start, size) != start) return false;
+      if (common_flags()->use_madv_dontdump)
+        DontDumpShadowMemory(start, size);
     } else if (type == MappingDesc::INVALID) {
       if (!CheckMemoryRangeAvailability(start, size)) return false;
       if (!ProtectMemoryRange(start, size)) return false;
     }
   }
 
   return true;
 }

compiler-rt/trunk/lib/sanitizer_common/sanitizer_common.h

 void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type);
 // Used to check if we can map shadow memory to a fixed location.
 bool MemoryRangeIsAvailable(uptr range_start, uptr range_end);
 void FlushUnneededShadowMemory(uptr addr, uptr size);
 void IncreaseTotalMmap(uptr size);
 void DecreaseTotalMmap(uptr size);
 uptr GetRSS();
 void NoHugePagesInRegion(uptr addr, uptr length);
+void DontDumpShadowMemory(uptr addr, uptr length);
 
 // InternalScopedBuffer can be used instead of large stack arrays to
 // keep frame size low.
 // FIXME: use InternalAlloc instead of MmapOrDie once
 // InternalAlloc is made libc-free.
 template<typename T>
 class InternalScopedBuffer {
  public:

compiler-rt/trunk/lib/sanitizer_common/sanitizer_flags.inc

 COMMON_FLAG(const char *, suppressions, "", "Suppressions file name.")
 COMMON_FLAG(bool, print_suppressions, true,
             "Print matched suppressions at exit.")
 COMMON_FLAG(
     bool, disable_coredump, (SANITIZER_WORDSIZE == 64),
     "Disable core dumping. By default, disable_core=1 on 64-bit to avoid "
     "dumping a 16T+ core file. Ignored on OSes that don't dump core by"
     "default and for sanitizers that don't reserve lots of virtual memory.")
+COMMON_FLAG(bool, use_madv_dontdump, true,
+          "If set, instructs kernel to not store the (huge) shadow "
+          "in core file.")
 COMMON_FLAG(bool, symbolize_inline_frames, true,
             "Print inlined frames in stacktraces. Defaults to true.")
 COMMON_FLAG(const char *, stack_trace_format, "DEFAULT",
             "Format string used to render stack frames. "
             "See sanitizer_stacktrace_printer.h for the format description. "
             "Use DEFAULT to get default format.")
 COMMON_FLAG(bool, no_huge_pages_for_shadow, true,
             "If true, the shadow is not allowed to use huge pages. ")

compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix_libcdep.cc

 }
 
 void NoHugePagesInRegion(uptr addr, uptr size) {
 #ifdef MADV_NOHUGEPAGE  // May not be defined on old systems.
   madvise((void *)addr, size, MADV_NOHUGEPAGE);
 #endif  // MADV_NOHUGEPAGE
 }
 
+void DontDumpShadowMemory(uptr addr, uptr length) {
+#ifdef MADV_DONTDUMP
+  madvise((void *)addr, length, MADV_DONTDUMP);
+#endif
+}
+
 static rlim_t getlim(int res) {
   rlimit rlim;
   CHECK_EQ(0, getrlimit(res, &rlim));
   return rlim.rlim_cur;
 }
 
 static void setlim(int res, rlim_t lim) {
   // The following magic is to prevent clang from replacing it with memset.

compiler-rt/trunk/lib/sanitizer_common/sanitizer_win.cc

if (res == 0)
     Report("WARNING: %s failed to "
            "mprotect %p (%zd) bytes at %p (error code: %d)\n",
            SanitizerToolName, size, size, fixed_addr, GetLastError());
   return res;
 }
 
 void FlushUnneededShadowMemory(uptr addr, uptr size) {
   // This is almost useless on 32-bits.
-  // FIXME: add madvice-analog when we move to 64-bits.
+  // FIXME: add madvise-analog when we move to 64-bits.
 }
 
 void NoHugePagesInRegion(uptr addr, uptr size) {
   // FIXME: probably similar to FlushUnneededShadowMemory.
 }
 
+void DontDumpShadowMemory(uptr addr, uptr length) {
+  // This is almost useless on 32-bits.
+  // FIXME: add madvise-analog when we move to 64-bits.
+}
+
 bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
   MEMORY_BASIC_INFORMATION mbi;
   CHECK(VirtualQuery((void *)range_start, &mbi, sizeof(mbi)));
   return mbi.Protect == PAGE_NOACCESS &&
          (uptr)mbi.BaseAddress + mbi.RegionSize >= range_end;
 }
 
 void *MapFileToMemory(const char *file_name, uptr *buff_size) {

compiler-rt/trunk/lib/tsan/rtl/tsan_platform_linux.cc

void InitializeShadowMemory() {
   }
   // This memory range is used for thread stacks and large user mmaps.
   // Frequently a thread uses only a small part of stack and similarly
   // a program uses a small part of large mmap. On some programs
   // we see 20% memory usage reduction without huge pages for this range.
   // FIXME: don't use constants here.
   NoHugePagesInRegion(MemToShadow(0x7f0000000000ULL),
                       0x10000000000ULL * kShadowMultiplier);
+  if (common_flags()->use_madv_dontdump)
+    DontDumpShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg);
   DPrintf("memory shadow: %zx-%zx (%zuGB)\n",
       kShadowBeg, kShadowEnd,
       (kShadowEnd - kShadowBeg) >> 30);
 
   // Map meta shadow.
   uptr meta_size = kMetaShadowEnd - kMetaShadowBeg;
   uptr meta = (uptr)MmapFixedNoReserve(kMetaShadowBeg, meta_size);
   if (meta != kMetaShadowBeg) {
     Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
     Printf("FATAL: Make sure to compile with -fPIE and "
                "to link with -pie (%p, %p).\n", meta, kMetaShadowBeg);
     Die();
   }
+  if (common_flags()->use_madv_dontdump)
+    DontDumpShadowMemory(meta, meta_size);
   DPrintf("meta shadow: %zx-%zx (%zuGB)\n",
       meta, meta + meta_size, meta_size >> 30);
 
   MapRodata();
 }
 
 static void InitDataSeg() {
   MemoryMappingLayout proc_maps(true);

compiler-rt/trunk/lib/tsan/rtl/tsan_platform_mac.cc

void InitializeShadowMemory() {
   uptr shadow = (uptr)MmapFixedNoReserve(kShadowBeg,
     kShadowEnd - kShadowBeg);
   if (shadow != kShadowBeg) {
     Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
     Printf("FATAL: Make sure to compile with -fPIE and "
            "to link with -pie.\n");
     Die();
   }
+  if (common_flags()->use_madv_dontdump)
+    DontDumpShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg);
   DPrintf("kShadow %zx-%zx (%zuGB)\n",
       kShadowBeg, kShadowEnd,
       (kShadowEnd - kShadowBeg) >> 30);
   DPrintf("kAppMem %zx-%zx (%zuGB)\n",
       kAppMemBeg, kAppMemEnd,
       (kAppMemEnd - kAppMemBeg) >> 30);
 }
 #endif