Index: compiler-rt/trunk/lib/asan/asan_rtl.cc =================================================================== --- compiler-rt/trunk/lib/asan/asan_rtl.cc +++ compiler-rt/trunk/lib/asan/asan_rtl.cc @@ -100,6 +100,8 @@ } if (common_flags()->no_huge_pages_for_shadow) NoHugePagesInRegion(beg, size); + if (common_flags()->use_madv_dontdump) + DontDumpShadowMemory(beg, size); } // --------------- LowLevelAllocateCallbac ---------- {{{1 Index: compiler-rt/trunk/lib/msan/msan_linux.cc =================================================================== --- compiler-rt/trunk/lib/msan/msan_linux.cc +++ compiler-rt/trunk/lib/msan/msan_linux.cc @@ -119,6 +119,8 @@ (init_origins && type == MappingDesc::ORIGIN)) { if (!CheckMemoryRangeAvailability(start, size)) return false; if ((uptr)MmapFixedNoReserve(start, size) != start) return false; + if (common_flags()->use_madv_dontdump) + DontDumpShadowMemory(start, size); } else if (type == MappingDesc::INVALID) { if (!CheckMemoryRangeAvailability(start, size)) return false; if (!ProtectMemoryRange(start, size)) return false; Index: compiler-rt/trunk/lib/sanitizer_common/sanitizer_common.h =================================================================== --- compiler-rt/trunk/lib/sanitizer_common/sanitizer_common.h +++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_common.h @@ -77,6 +77,7 @@ void DecreaseTotalMmap(uptr size); uptr GetRSS(); void NoHugePagesInRegion(uptr addr, uptr length); +void DontDumpShadowMemory(uptr addr, uptr length); // InternalScopedBuffer can be used instead of large stack arrays to // keep frame size low. Index: compiler-rt/trunk/lib/sanitizer_common/sanitizer_flags.inc =================================================================== --- compiler-rt/trunk/lib/sanitizer_common/sanitizer_flags.inc +++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_flags.inc @@ -136,6 +136,9 @@ "Disable core dumping. By default, disable_core=1 on 64-bit to avoid " "dumping a 16T+ core file. Ignored on OSes that don't dump core by" "default and for sanitizers that don't reserve lots of virtual memory.") +COMMON_FLAG(bool, use_madv_dontdump, true, + "If set, instructs kernel to not store the (huge) shadow " + "in core file.") COMMON_FLAG(bool, symbolize_inline_frames, true, "Print inlined frames in stacktraces. Defaults to true.") COMMON_FLAG(const char *, stack_trace_format, "DEFAULT", Index: compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix_libcdep.cc =================================================================== --- compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix_libcdep.cc +++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix_libcdep.cc @@ -50,6 +50,12 @@ #endif // MADV_NOHUGEPAGE } +void DontDumpShadowMemory(uptr addr, uptr length) { +#ifdef MADV_DONTDUMP + madvise((void *)addr, length, MADV_DONTDUMP); +#endif +} + static rlim_t getlim(int res) { rlimit rlim; CHECK_EQ(0, getrlimit(res, &rlim)); Index: compiler-rt/trunk/lib/sanitizer_common/sanitizer_win.cc =================================================================== --- compiler-rt/trunk/lib/sanitizer_common/sanitizer_win.cc +++ compiler-rt/trunk/lib/sanitizer_common/sanitizer_win.cc @@ -134,13 +134,18 @@ void FlushUnneededShadowMemory(uptr addr, uptr size) { // This is almost useless on 32-bits. - // FIXME: add madvice-analog when we move to 64-bits. + // FIXME: add madvise-analog when we move to 64-bits. } void NoHugePagesInRegion(uptr addr, uptr size) { // FIXME: probably similar to FlushUnneededShadowMemory. } +void DontDumpShadowMemory(uptr addr, uptr length) { + // This is almost useless on 32-bits. + // FIXME: add madvise-analog when we move to 64-bits. +} + bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) { MEMORY_BASIC_INFORMATION mbi; CHECK(VirtualQuery((void *)range_start, &mbi, sizeof(mbi))); Index: compiler-rt/trunk/lib/tsan/rtl/tsan_platform_linux.cc =================================================================== --- compiler-rt/trunk/lib/tsan/rtl/tsan_platform_linux.cc +++ compiler-rt/trunk/lib/tsan/rtl/tsan_platform_linux.cc @@ -218,6 +218,8 @@ // FIXME: don't use constants here. NoHugePagesInRegion(MemToShadow(0x7f0000000000ULL), 0x10000000000ULL * kShadowMultiplier); + if (common_flags()->use_madv_dontdump) + DontDumpShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg); DPrintf("memory shadow: %zx-%zx (%zuGB)\n", kShadowBeg, kShadowEnd, (kShadowEnd - kShadowBeg) >> 30); @@ -231,6 +233,8 @@ "to link with -pie (%p, %p).\n", meta, kMetaShadowBeg); Die(); } + if (common_flags()->use_madv_dontdump) + DontDumpShadowMemory(meta, meta_size); DPrintf("meta shadow: %zx-%zx (%zuGB)\n", meta, meta + meta_size, meta_size >> 30); Index: compiler-rt/trunk/lib/tsan/rtl/tsan_platform_mac.cc =================================================================== --- compiler-rt/trunk/lib/tsan/rtl/tsan_platform_mac.cc +++ compiler-rt/trunk/lib/tsan/rtl/tsan_platform_mac.cc @@ -60,6 +60,8 @@ "to link with -pie.\n"); Die(); } + if (common_flags()->use_madv_dontdump) + DontDumpShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg); DPrintf("kShadow %zx-%zx (%zuGB)\n", kShadowBeg, kShadowEnd, (kShadowEnd - kShadowBeg) >> 30);