[TSan] Fix madvise(MADV_NOHUGEPAGE) for meta shadow memory

[alekseyshl]

Summary

Move madvise(MADV_NOHUGEPAGE) for the meta shadow memory after the meta
shadow memory is mapped (currently it silently fails with ENOMEM).

Add a diagnostic message to detect similar problems in the future.

[dvyukov]

lib/tsan/rtl/tsan_platform_posix.cc

32 ↗

(On Diff #151032)

If we want to notice this in future we need to Die.
We never print anything unless there is a bug in program, or we terminate the program. Altered output breaks programs in surprising ways (consider altering output of protoc). Also if this happened in chrome or in our server code base all the time, we will never notice a warning line in output and nobody will notify us.

[alekseyshl]

• Die on madvise-ing shadow memory failure.

[alekseyshl]

Thanks!

lib/tsan/rtl/tsan_platform_posix.cc

33 ↗

(On Diff #151168)

By the way, do you think we should hint on the flag name controlling this feature here (like "set no_huge_pages_for_shadow=0 if it does not work for you") or we'd rather hear about the problem? I think the latter is preferred. The same for MADV_DONTDUMP below.

compiler-rt/trunk/lib/sanitizer_common/sanitizer_common.h

 // Used to check if we can map shadow memory to a fixed location.
 bool MemoryRangeIsAvailable(uptr range_start, uptr range_end);
 // Releases memory pages entirely within the [beg, end] address range. Noop if
 // the provided range does not contain at least one entire page.
 void ReleaseMemoryPagesToOS(uptr beg, uptr end);
 void IncreaseTotalMmap(uptr size);
 void DecreaseTotalMmap(uptr size);
 uptr GetRSS();
-void NoHugePagesInRegion(uptr addr, uptr length);
-void DontDumpShadowMemory(uptr addr, uptr length);
+bool NoHugePagesInRegion(uptr addr, uptr length);
+bool DontDumpShadowMemory(uptr addr, uptr length);
 // Check if the built VMA size matches the runtime one.
 void CheckVMASize();
 void RunMallocHooks(const void *ptr, uptr size);
 void RunFreeHooks(const void *ptr);
 
 class ReservedAddressRange {
  public:
   uptr Init(uptr size, const char *name = nullptr, uptr fixed_addr = 0);

compiler-rt/trunk/lib/sanitizer_common/sanitizer_posix_libcdep.cc

void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
   if (beg_aligned < end_aligned)
     // In the default Solaris compilation environment, madvise() is declared
     // to take a caddr_t arg; casting it to void * results in an invalid
     // conversion error, so use char * instead.
     madvise((char *)beg_aligned, end_aligned - beg_aligned,
             SANITIZER_MADVISE_DONTNEED);
 }
 
-void NoHugePagesInRegion(uptr addr, uptr size) {
+bool NoHugePagesInRegion(uptr addr, uptr size) {
 #ifdef MADV_NOHUGEPAGE  // May not be defined on old systems.
-  madvise((void *)addr, size, MADV_NOHUGEPAGE);
+  return madvise((void *)addr, size, MADV_NOHUGEPAGE) == 0;
+#else
+  return true;
 #endif  // MADV_NOHUGEPAGE
 }
 
-void DontDumpShadowMemory(uptr addr, uptr length) {
+bool DontDumpShadowMemory(uptr addr, uptr length) {
 #ifdef MADV_DONTDUMP
-  madvise((void *)addr, length, MADV_DONTDUMP);
-#endif
+  return madvise((void *)addr, length, MADV_DONTDUMP) == 0;
+#else
+  return true;
+#endif  // MADV_DONTDUMP
 }
 
 static rlim_t getlim(int res) {
   rlimit rlim;
   CHECK_EQ(0, getrlimit(res, &rlim));
   return rlim.rlim_cur;
 }
 

compiler-rt/trunk/lib/sanitizer_common/sanitizer_win.cc

bool MprotectNoAccess(uptr addr, uptr size) {
   return VirtualProtect((LPVOID)addr, size, PAGE_NOACCESS, &old_protection);
 }
 
 void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
   // This is almost useless on 32-bits.
   // FIXME: add madvise-analog when we move to 64-bits.
 }
 
-void NoHugePagesInRegion(uptr addr, uptr size) {
+bool NoHugePagesInRegion(uptr addr, uptr size) {
   // FIXME: probably similar to ReleaseMemoryToOS.
+  return true;
 }
 
-void DontDumpShadowMemory(uptr addr, uptr length) {
+bool DontDumpShadowMemory(uptr addr, uptr length) {
   // This is almost useless on 32-bits.
   // FIXME: add madvise-analog when we move to 64-bits.
+  return true;
 }
 
 uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
                               uptr *largest_gap_found,
                               uptr *max_occupied_addr) {
   uptr address = 0;
   while (true) {
     MEMORY_BASIC_INFORMATION info;

compiler-rt/trunk/lib/tsan/rtl/tsan_platform_posix.cc

 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_procmaps.h"
 #include "tsan_platform.h"
 #include "tsan_rtl.h"
 
 namespace __tsan {
 
+static const char kShadowMemoryMappingWarning[] =
+    "FATAL: %s can not madvise shadow region [%zx, %zx] with %s\n";
+
+static void NoHugePagesInShadow(uptr addr, uptr size) {
+  if (common_flags()->no_huge_pages_for_shadow)
+    if (!NoHugePagesInRegion(addr, size)) {
+      Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size,
+             "MADV_NOHUGEPAGE");
+      Die();
+    }
+}
+
+static void DontDumpShadow(uptr addr, uptr size) {
+  if (common_flags()->use_madv_dontdump)
+    if (!DontDumpShadowMemory(addr, size)) {
+      Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size,
+             "MADV_DONTDUMP");
+      Die();
+    }
+}
+
 #if !SANITIZER_GO
 void InitializeShadowMemory() {
   // Map memory shadow.
   uptr shadow =
       (uptr)MmapFixedNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(),
                                "shadow");
   if (shadow != ShadowBeg()) {
     Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
if (vmaSize == 44) {
     kMadviseRangeSize = 0x0010000000ull;
   } else if (vmaSize == 46) {
     kMadviseRangeBeg  = 0x3f0000000000ull;
     kMadviseRangeSize = 0x010000000000ull;
   } else {
     DCHECK(0);
   }
 #endif
-  NoHugePagesInRegion(MemToShadow(kMadviseRangeBeg),
+  NoHugePagesInShadow(MemToShadow(kMadviseRangeBeg),
                       kMadviseRangeSize * kShadowMultiplier);
-  // Meta shadow is compressing and we don't flush it,
-  // so it makes sense to mark it as NOHUGEPAGE to not over-allocate memory.
-  // On one program it reduces memory consumption from 5GB to 2.5GB.
-  NoHugePagesInRegion(MetaShadowBeg(), MetaShadowEnd() - MetaShadowBeg());
-  if (common_flags()->use_madv_dontdump)
-    DontDumpShadowMemory(ShadowBeg(), ShadowEnd() - ShadowBeg());
+  DontDumpShadow(ShadowBeg(), ShadowEnd() - ShadowBeg());
   DPrintf("memory shadow: %zx-%zx (%zuGB)\n",
       ShadowBeg(), ShadowEnd(),
       (ShadowEnd() - ShadowBeg()) >> 30);
 
   // Map meta shadow.
   uptr meta_size = MetaShadowEnd() - MetaShadowBeg();
   uptr meta =
       (uptr)MmapFixedNoReserve(MetaShadowBeg(), meta_size, "meta shadow");
   if (meta != MetaShadowBeg()) {
     Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
     Printf("FATAL: Make sure to compile with -fPIE and "
                "to link with -pie (%p, %p).\n", meta, MetaShadowBeg());
     Die();
   }
-  if (common_flags()->use_madv_dontdump)
-    DontDumpShadowMemory(meta, meta_size);
+  NoHugePagesInShadow(meta, meta_size);
+  DontDumpShadow(meta, meta_size);
   DPrintf("meta shadow: %zx-%zx (%zuGB)\n",
       meta, meta + meta_size, meta_size >> 30);
 
   InitializeShadowMemoryPlatform();
 }
 
 static void ProtectRange(uptr beg, uptr end) {
   CHECK_LE(beg, end);