[tsan] Respect no_huge_pages_for_shadow.

[ckennelly]

Summary

Disable huge pages in the TSan shadow regions when no_huge_pages_for_shadow == true (default).

[eugenis]

I wonder if it would pay to selectively apply this for the parts of the shadow that correspond to shared library mappings.
Otherwise we are losing the benefits of huge pages on the heap, which should have a pretty dense shadow.
@dvyukov

compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp

93–94

This call can be removed now.

106–107

same

[dvyukov]

In D85841#2213859, @eugenis wrote:

I wonder if it would pay to selectively apply this for the parts of the shadow that correspond to shared library mappings.
Otherwise we are losing the benefits of huge pages on the heap, which should have a pretty dense shadow.
@dvyukov

Hard to say without any data.
I know we are close to RAM limits in lots of our use cases, so if we trade a bit memory for performance, we sure will get lots of new failures without any simple ways to resolve them.
Tsan may flush shadow pretty aggressively. Consider: heap may be "dense", but now if we have just 1K region that is not used between 2 flushes, we won't page shadow for it in. With huge pages we may page in way more.

[ckennelly]

Applied edits, and lint errors

[eugenis]

OK, this seems to be the safe choice then. LGTM.

[hubert.reinterpretcast]

compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp

93

@ckennelly:
Removing these calls are causing build failures because the internal-linkage function is no longer referenced anywhere: http://lab.llvm.org:8011/builders/clang-ppc64le-rhel/builds/6480/steps/build%20stage%201/logs/stdio

[ckennelly]

Fix in https://reviews.llvm.org/D85944

compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp

if (!DontDumpShadowMemory(addr, size)) {
       Printf(kShadowMemoryMappingHint, "MADV_DONTDUMP", "use_madv_dontdump");
       Die();
     }
 }
 
 #if !SANITIZER_GO
 void InitializeShadowMemory() {
   // Map memory shadow.
-  if (!MmapFixedNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(), "shadow")) {
+  if (!MmapFixedSuperNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(),
+                               "shadow")) {
     Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
     Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
     Die();
   }
   // This memory range is used for thread stacks and large user mmaps.
   // Frequently a thread uses only a small part of stack and similarly
   // a program uses a small part of large mmap. On some programs
   // we see 20% memory usage reduction without huge pages for this range.
if (vmaSize == 44) {
     kMadviseRangeBeg  = 0x0f60000000ull;
     kMadviseRangeSize = 0x0010000000ull;
   } else if (vmaSize == 46) {
     kMadviseRangeBeg  = 0x3f0000000000ull;
     kMadviseRangeSize = 0x010000000000ull;
   } else {
     DCHECK(0);
   }
 #endif
-  NoHugePagesInShadow(MemToShadow(kMadviseRangeBeg),
-                      kMadviseRangeSize * kShadowMultiplier);
   DontDumpShadow(ShadowBeg(), ShadowEnd() - ShadowBeg());
   DPrintf("memory shadow: %zx-%zx (%zuGB)\n",
       ShadowBeg(), ShadowEnd(),
       (ShadowEnd() - ShadowBeg()) >> 30);
 
   // Map meta shadow.
   const uptr meta = MetaShadowBeg();
   const uptr meta_size = MetaShadowEnd() - meta;
-  if (!MmapFixedNoReserve(meta, meta_size, "meta shadow")) {
+  if (!MmapFixedSuperNoReserve(meta, meta_size, "meta shadow")) {
     Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
     Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
     Die();
   }
-  NoHugePagesInShadow(meta, meta_size);
   DontDumpShadow(meta, meta_size);
   DPrintf("meta shadow: %zx-%zx (%zuGB)\n",
       meta, meta + meta_size, meta_size >> 30);
 
   InitializeShadowMemoryPlatform();
 }
 
 static void ProtectRange(uptr beg, uptr end) {
   CHECK_LE(beg, end);

compiler-rt/lib/tsan/rtl/tsan_rtl.cpp

 
 void MapShadow(uptr addr, uptr size) {
   // Global data is not 64K aligned, but there are no adjacent mappings,
   // so we can get away with unaligned mapping.
   // CHECK_EQ(addr, addr & ~((64 << 10) - 1));  // windows wants 64K alignment
   const uptr kPageSize = GetPageSizeCached();
   uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize);
   uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize);
-  if (!MmapFixedNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"))
+  if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin,
+                               "shadow"))
     Die();
 
   // Meta shadow is 2:1, so tread carefully.
   static bool data_mapped = false;
   static uptr mapped_meta_end = 0;
   uptr meta_begin = (uptr)MemToMeta(addr);
   uptr meta_end = (uptr)MemToMeta(addr + size);
   meta_begin = RoundDownTo(meta_begin, 64 << 10);
   meta_end = RoundUpTo(meta_end, 64 << 10);
   if (!data_mapped) {
     // First call maps data+bss.
     data_mapped = true;
-    if (!MmapFixedNoReserve(meta_begin, meta_end - meta_begin, "meta shadow"))
+    if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin,
+                                 "meta shadow"))
       Die();
   } else {
     // Mapping continous heap.
     // Windows wants 64K alignment.
     meta_begin = RoundDownTo(meta_begin, 64 << 10);
     meta_end = RoundUpTo(meta_end, 64 << 10);
     if (meta_end <= mapped_meta_end)
       return;
     if (meta_begin < mapped_meta_end)
       meta_begin = mapped_meta_end;
-    if (!MmapFixedNoReserve(meta_begin, meta_end - meta_begin, "meta shadow"))
+    if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin,
+                                 "meta shadow"))
       Die();
     mapped_meta_end = meta_end;
   }
   VPrintf(2, "mapped meta shadow for (%p-%p) at (%p-%p)\n",
       addr, addr+size, meta_begin, meta_end);
 }
 
 void MapThreadTrace(uptr addr, uptr size, const char *name) {
   DPrintf("#0: Mapping trace at %p-%p(0x%zx)\n", addr, addr + size, size);
   CHECK_GE(addr, TraceMemBeg());
   CHECK_LE(addr + size, TraceMemEnd());
   CHECK_EQ(addr, addr & ~((64 << 10) - 1));  // windows wants 64K alignment
-  if (!MmapFixedNoReserve(addr, size, name)) {
+  if (!MmapFixedSuperNoReserve(addr, size, name)) {
     Printf("FATAL: ThreadSanitizer can not mmap thread trace (%p/%p)\n",
         addr, size);
     Die();
   }
 }
 
 static void CheckShadowMapping() {
   uptr beg, end;
while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) {
       *p++ = val;
       for (uptr j = 1; j < kShadowCnt; j++)
         *p++ = 0;
     }
     // Reset middle part.
     u64 *p1 = p;
     p = RoundDown(end, kPageSize);
     UnmapOrDie((void*)p1, (uptr)p - (uptr)p1);
-    if (!MmapFixedNoReserve((uptr)p1, (uptr)p - (uptr)p1))
+    if (!MmapFixedSuperNoReserve((uptr)p1, (uptr)p - (uptr)p1))
       Die();
     // Set the ending.
     while (p < end) {
       *p++ = val;
       for (uptr j = 1; j < kShadowCnt; j++)
         *p++ = 0;
     }
   }

compiler-rt/lib/tsan/rtl/tsan_sync.cpp

void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
   }
   // Finally, page out the whole range (including the parts that we've just
   // freed). Note: we can't simply madvise, because we need to leave a zeroed
   // range (otherwise __tsan_java_move can crash if it encounters a left-over
   // meta objects in java heap).
   uptr metap = (uptr)MemToMeta(p0);
   uptr metasz = sz0 / kMetaRatio;
   UnmapOrDie((void*)metap, metasz);
-  if (!MmapFixedNoReserve(metap, metasz))
+  if (!MmapFixedSuperNoReserve(metap, metasz))
     Die();
 }
 
 MBlock* MetaMap::GetBlock(uptr p) {
   u32 *meta = MemToMeta(p);
   u32 idx = *meta;
   for (;;) {
     if (idx == 0)