[Tsan] Add support for FreeBSD memory layout

[kutuzov.viktor.84]

Fixed comments, indentation and TSAN_GO dependencies.

[kutuzov.viktor.84]

Fixed placement of curly braces.

[emaste]

lib/tsan/rtl/tsan_platform_linux.cc

12

Should be Linux- and FreeBSD-specific code. I think - and would be useful to explain why the FreeBSD code is in the _linux.cc file.

267

It's rather confusing for these constants to be named kLinux... for the FreeBSD-specific code.

Would it be OK for these to be kLowShadowBeg etc.?

323

Why is this one a bare constant here rather than some k constant?

[dvyukov]

samsonov added a reviewer: dvyukov.

Alexey, this did not work. I did not receive any email.

[dvyukov]

Do you essentially want to support non-pie binaries where something is mapped in the first 64GB? Is it the only difference with the current linux mapping?
If it's the only difference, then it's time for me to implement non-pie support finally.

Add me to tsan reviews, I am the author of tsan.

lib/tsan/rtl/tsan_platform.h

50

please use the same format as entries above -- the second address is exclusive, that is:
0000 0000 0000 - 0000 0100 0000

58

here is something wrong, the previous line already covers this range

60

what's at 5000 0000 0000 - 6000 0000 0000 ?

[kutuzov.viktor.84]

Fixed to reflect the notes and suggestions from Dmitry and Ed.

[kutuzov.viktor.84]

It's rather confusing for these constants to be named kLinux... for the FreeBSD-specific code.

Dmitry, do you think we should generalize these names?

Do you essentially want to support non-pie binaries where something is mapped in the first 64GB? Is it the only difference with the current linux mapping?
If it's the only difference, then it's time for me to implement non-pie support finally.

I didn't dig it up yet, but given it is supported on Linux, we certainly will consider support it on FreeBSD as well.

[dvyukov]

I didn't dig it up yet, but given it is supported on Linux, we certainly will consider support it on FreeBSD as well.

I am asking about what this change does. Does it just allow user data in the first 64GBs of memory?

[kutuzov.viktor.84]

I am asking about what this change does. Does it just allow user data in the first 64GBs of memory?

Yes, the only difference is that we have user data on both ends of the memory map. If you need similar changes to support non-PIEs on Linux, then indeed it would be good to implement some generic mechanism to address both the tasks.

[kutuzov.viktor.84]

Abandoned in favor of rL220571 and D5990.

lib/tsan/rtl/tsan_platform.h

 0000 0000 0000 - 0000 1000 0000: executable
 0000 1000 0000 - 00f8 0000 0000: -
 00c0 0000 0000 - 00e0 0000 0000: heap
 00e0 0000 0000 - 0100 0000 0000: -
 0100 0000 0000 - 0560 0000 0000: shadow
 0560 0000 0000 - 0760 0000 0000: traces
 0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects)
 07d0 0000 0000 - 07ff ffff ffff: -
+
+C++ FreeBSD memory layout:
+0000 0000 0000 - 0000 0100 0000: -
+0000 0100 0000 - 0010 0000 0000: executable, modules and service heap
+0010 0000 0000 - 1000 0000 0000: -
[phone] - 1000 0400 0000: protected
[phone] - 1040 0000 0000: low mem shadow
[phone] - 2400 0000 0000: protected
[phone] - 3000 0000 0000: high mem shadow
[phone] - 4000 0000 0000: low metainfo (memory blocks and sync objects)
[phone] - 5000 0000 0000: high metainfo (memory blocks and sync objects)
[phone] - 6000 0000 0000: -
[phone] - 6000 0010 0000: trace
[phone] - 7d00 0000 0000: -
+7d00 0000 0000 - 7e00 0000 0000: heap
+7e00 0000 0000 - 7fff ffff ffff: stacks & system
 */
 
 #ifndef TSAN_PLATFORM_H
 #define TSAN_PLATFORM_H
 
 #include "tsan_defs.h"
 #include "tsan_trace.h"
 
 #if defined(__LP64__) || defined(_WIN64)
 namespace __tsan {
 
 #if defined(TSAN_GO)
 static const uptr kLinuxAppMemBeg = 0x000000000000ULL;
 static const uptr kLinuxAppMemEnd = 0x04dfffffffffULL;
 # if SANITIZER_WINDOWS
 static const uptr kLinuxShadowMsk = 0x010000000000ULL;
 static const uptr kMetaShadow     = 0x076000000000ULL;
 static const uptr kMetaSize       = 0x007000000000ULL;
 # else  // if SANITIZER_WINDOWS
 static const uptr kLinuxShadowMsk = 0x200000000000ULL;
 static const uptr kMetaShadow     = 0x300000000000ULL;
 static const uptr kMetaSize       = 0x100000000000ULL;
 # endif  // if SANITIZER_WINDOWS
 #else  // defined(TSAN_GO)
+# if SANITIZER_FREEBSD
+static const uptr kLowMetaShadow       = 0x300000000000ULL;
+static const uptr kLowMetaSize         = 0x100000000000ULL;
+static const uptr kHighMetaShadow      = 0x400000000000ULL;
+static const uptr kHighMetaSize        = 0x100000000000ULL;
+static const uptr kLinuxLowAppMemBeg   = 0x000001000000ULL;
+static const uptr kLinuxLowAppMemEnd   = 0x000fffffffffULL;
+static const uptr kLinuxHighAppMemBeg  = 0x7d0000000000ULL;
+static const uptr kLinuxHighAppMemEnd  = 0x7fffffffffffULL;
+static const uptr kLinuxLowShadowMsk   = 0x100000000000ULL;
+static const uptr kLinuxHighShadowMsk  = 0x240000000000ULL;
+# else
 static const uptr kMetaShadow     = 0x300000000000ULL;
 static const uptr kMetaSize       = 0x100000000000ULL;
 static const uptr kLinuxAppMemBeg = 0x7cf000000000ULL;
 static const uptr kLinuxAppMemEnd = 0x7fffffffffffULL;
-#endif
+# endif
+#endif
 
+#if !defined(TSAN_GO) && SANITIZER_FREEBSD
+static const uptr kLinuxLowAppMemMsk  = 0x000000000000ULL;
+static const uptr kLinuxHighAppMemMsk = 0x7c0000000000ULL;
+#else
 static const uptr kLinuxAppMemMsk = 0x7c0000000000ULL;
+#endif
 
 #if SANITIZER_WINDOWS
 const uptr kTraceMemBegin = 0x056000000000ULL;
 #else
 const uptr kTraceMemBegin = 0x600000000000ULL;
 #endif
 const uptr kTraceMemSize = 0x020000000000ULL;
 
 // This has to be a macro to allow constant initialization of constants below.
 #ifndef TSAN_GO
-#define MemToShadow(addr) \
-    ((((uptr)addr) & ~(kLinuxAppMemMsk | (kShadowCell - 1))) * kShadowCnt)
-#define MemToMeta(addr) \
-    (u32*)(((((uptr)addr) & ~(kLinuxAppMemMsk | (kMetaShadowCell - 1))) \
+# if SANITIZER_FREEBSD
+#  define MemToLowShadow(addr) \
+     (((((uptr)(addr)) & ~(kShadowCell - 1)) * kShadowCnt) \
+       | kLinuxLowShadowMsk)
+#  define MemToHighShadow(addr) \
+     (((((uptr)(addr)) & ~(kLinuxHighAppMemMsk | (kShadowCell - 1))) \
+       * kShadowCnt) | kLinuxHighShadowMsk)
+
+#  define MemToLowMeta(addr) \
+     (u32*)(((((uptr)(addr)) & ~(kLinuxLowAppMemMsk | (kMetaShadowCell - 1))) \
+       / kMetaShadowCell * kMetaShadowSize) | kLowMetaShadow)
+#  define MemToHighMeta(addr) \
+     (u32*)(((((uptr)(addr)) & ~(kLinuxHighAppMemMsk | (kMetaShadowCell - 1))) \
+       / kMetaShadowCell * kMetaShadowSize) | kHighMetaShadow)
+
+#  define IsLowAppMem(mem) \
+     ((mem) >= kLinuxLowAppMemBeg && (mem) <= kLinuxLowAppMemEnd)
+#  define IsHighAppMem(mem) \
+     ((mem) >= kLinuxHighAppMemBeg && (mem) <= kLinuxHighAppMemEnd)
+
+#  define MemToShadow(mem) \
+     (IsLowAppMem((mem)) ? MemToLowShadow((mem)) : MemToHighShadow((mem)))
+#  define MemToMeta(mem) \
+     (IsLowAppMem((mem)) ? MemToLowMeta((mem)) : MemToHighMeta((mem)))
+# else  // SANITIZER_FREEBSD
+#  define MemToShadow(addr) \
+     ((((uptr)(addr)) & ~(kLinuxAppMemMsk | (kShadowCell - 1))) * kShadowCnt)
+#  define MemToMeta(addr) \
+     (u32*)(((((uptr)(addr)) & ~(kLinuxAppMemMsk | (kMetaShadowCell - 1))) \
        / kMetaShadowCell * kMetaShadowSize) | kMetaShadow)
-#else
-#define MemToShadow(addr) \
+# endif  // !SANITIZER_FREEBSD
+#else  // TSAN_GO
+# define MemToShadow(addr) \
     (((((uptr)addr) & ~(kShadowCell - 1)) * kShadowCnt) | kLinuxShadowMsk)
-#define MemToMeta(addr) \
+# define MemToMeta(addr) \
     (u32*)(((((uptr)addr) & ~(kMetaShadowCell - 1)) \
       / kMetaShadowCell * kMetaShadowSize) | kMetaShadow)
-#endif
+#endif  // !TSAN_GO
+
+#if !defined(TSAN_GO) && SANITIZER_FREEBSD
+static const uptr kLinuxLowShadowBeg = MemToShadow(kLinuxLowAppMemBeg);
+static const uptr kLinuxLowShadowEnd =
+    MemToShadow(kLinuxLowAppMemEnd) | 0xff;
+static const uptr kLinuxHighShadowBeg = MemToShadow(kLinuxHighAppMemBeg);
+static const uptr kLinuxHighShadowEnd =
+    MemToShadow(kLinuxHighAppMemEnd) | 0xff;
+
+static inline bool IsAppMem(uptr mem) {
+  return IsLowAppMem(mem) || IsHighAppMem(mem);
+}
+
+static inline bool IsLowShadowMem(uptr mem) {
+  return mem >= kLinuxLowShadowBeg && mem <= kLinuxLowShadowEnd;
+}
+
+static inline bool IsHighShadowMem(uptr mem) {
+  return mem >= kLinuxHighShadowBeg && mem <= kLinuxHighShadowEnd;
+}
+
+static inline bool IsShadowMem(uptr mem) {
+  return IsLowShadowMem(mem) || IsHighShadowMem(mem);
+}
+
+static inline uptr LowShadowToMem(uptr shadow) {
+  CHECK(IsLowShadowMem(shadow));
+  return ((shadow & ~kLinuxLowShadowMsk) / kShadowCnt) \
+    | kLinuxLowAppMemMsk;
+}
+
+static inline uptr HighShadowToMem(uptr shadow) {
+  CHECK(IsHighShadowMem(shadow));
+  return (shadow / kShadowCnt) | kLinuxHighAppMemMsk;
+}
 
+static inline uptr ShadowToMem(uptr shadow) {
+  CHECK(IsShadowMem(shadow));
+  return IsLowShadowMem(shadow) ?
+    LowShadowToMem(shadow) : HighShadowToMem(shadow);
+}
+#else  // !TSAN_GO && SANITIZER_FREEBSD
 static const uptr kLinuxShadowBeg = MemToShadow(kLinuxAppMemBeg);
 static const uptr kLinuxShadowEnd =
     MemToShadow(kLinuxAppMemEnd) | 0xff;
 
 static inline bool IsAppMem(uptr mem) {
 #if defined(TSAN_GO)
   return mem <= kLinuxAppMemEnd;
 #else
   return mem >= kLinuxAppMemBeg && mem <= kLinuxAppMemEnd;
 #endif
 }
 
 static inline bool IsShadowMem(uptr mem) {
   return mem >= kLinuxShadowBeg && mem <= kLinuxShadowEnd;
 }
 
 static inline uptr ShadowToMem(uptr shadow) {
   CHECK(IsShadowMem(shadow));
 #ifdef TSAN_GO
   return (shadow & ~kLinuxShadowMsk) / kShadowCnt;
 #else
   return (shadow / kShadowCnt) | kLinuxAppMemMsk;
 #endif
 }
+#endif  // TSAN_GO || !SANITIZER_FREEBSD
+
+#ifndef TSAN_GO
+# if SANITIZER_FREEBSD
+const uptr kModulesBeg = 0x000001000000;
+# else
+const uptr kModulesBeg = kLinuxShadowBeg;
+# endif
+#endif  // !TSAN_GO
 
 void FlushShadowMemory();
 void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive);
 uptr GetRSS();
 
 void InitializePlatform();
 void FinalizePlatform();
 

lib/tsan/rtl/tsan_platform_linux.cc

 //===-- tsan_platform_linux.cc --------------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of ThreadSanitizer (TSan), a race detector.
 //
-// Linux-specific code.
+// Linux- and FreeBSD-specific code.
 //===----------------------------------------------------------------------===//
 
 
 #include "sanitizer_common/sanitizer_platform.h"
 #if SANITIZER_LINUX || SANITIZER_FREEBSD
 
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_libc.h"
if (name[0] != 0 && name[0] != '['
         internal_mmap(p, Min<uptr>(marker.size(), shadow_end - p),
                       PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0);
       }
     }
   }
   internal_close(fd);
 }
 
-void InitializeShadowMemory() {
+static void MmapShadow(uptr shadowBeg, uptr shadowEnd,
+                       uptr appMemBeg, uptr appMemEnd,
+                       uptr metaShadow, uptr metaSize) {
+  CHECK_LT(shadowBeg, shadowEnd);
+  CHECK_LT(appMemBeg, appMemEnd);
+
   // Map memory shadow.
-  uptr shadow = (uptr)MmapFixedNoReserve(kLinuxShadowBeg,
-    kLinuxShadowEnd - kLinuxShadowBeg);
-  if (shadow != kLinuxShadowBeg) {
+  uptr shadow = (uptr)MmapFixedNoReserve(shadowBeg, shadowEnd - shadowBeg);
+  if (shadow != shadowBeg) {
     Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
     Printf("FATAL: Make sure to compile with -fPIE and "
-               "to link with -pie (%p, %p).\n", shadow, kLinuxShadowBeg);
+             "to link with -pie (%p, %p).\n", shadow, shadowBeg);
     Die();
   }
-  // This memory range is used for thread stacks and large user mmaps.
-  // Frequently a thread uses only a small part of stack and similarly
-  // a program uses a small part of large mmap. On some programs
-  // we see 20% memory usage reduction without huge pages for this range.
-#ifdef MADV_NOHUGEPAGE
-  madvise((void*)MemToShadow(0x7f0000000000ULL),
-      0x10000000000ULL * kShadowMultiplier, MADV_NOHUGEPAGE);
-#endif
   DPrintf("memory shadow: %zx-%zx (%zuGB)\n",
-      kLinuxShadowBeg, kLinuxShadowEnd,
-      (kLinuxShadowEnd - kLinuxShadowBeg) >> 30);
+          shadowBeg, shadowEnd, (shadowEnd - shadowBeg) >> 30);
 
   // Map meta shadow.
-  if (MemToMeta(kLinuxAppMemBeg) < (u32*)kMetaShadow) {
+  if (MemToMeta(appMemBeg) < (u32*)metaShadow) {
     Printf("ThreadSanitizer: bad meta shadow (%p -> %p < %p)\n",
-        kLinuxAppMemBeg, MemToMeta(kLinuxAppMemBeg), kMetaShadow);
+           appMemBeg, MemToMeta(appMemBeg), metaShadow);
     Die();
   }
-  if (MemToMeta(kLinuxAppMemEnd) >= (u32*)(kMetaShadow + kMetaSize)) {
+  if (MemToMeta(appMemEnd) >= (u32*)(metaShadow + metaSize)) {
     Printf("ThreadSanitizer: bad meta shadow (%p -> %p >= %p)\n",
-        kLinuxAppMemEnd, MemToMeta(kLinuxAppMemEnd), kMetaShadow + kMetaSize);
+           appMemEnd, MemToMeta(appMemEnd), metaShadow + metaSize);
     Die();
   }
-  uptr meta = (uptr)MmapFixedNoReserve(kMetaShadow, kMetaSize);
-  if (meta != kMetaShadow) {
+  uptr meta = (uptr)MmapFixedNoReserve(metaShadow, metaSize);
+  if (meta != metaShadow) {
     Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
     Printf("FATAL: Make sure to compile with -fPIE and "
-               "to link with -pie (%p, %p).\n", meta, kMetaShadow);
+             "to link with -pie (%p, %p).\n", meta, metaShadow);
     Die();
   }
   DPrintf("meta shadow: %zx-%zx (%zuGB)\n",
-      kMetaShadow, kMetaShadow + kMetaSize, kMetaSize >> 30);
+          metaShadow, metaShadow + metaSize, metaSize >> 30);
+  VPrintf(2, "app mem: %zx-%zx (%zuGB)\n",
+          appMemBeg, appMemEnd, (appMemEnd - appMemBeg) >> 30);
+}
+
+void InitializeShadowMemory() {
+#if SANITIZER_FREEBSD
+  MmapShadow(kLinuxLowShadowBeg, kLinuxLowShadowEnd,
+             kLinuxLowAppMemBeg, kLinuxLowAppMemEnd,
+             kLowMetaShadow, kLowMetaSize);
+  MmapShadow(kLinuxHighShadowBeg, kLinuxHighShadowEnd,
+             kLinuxHighAppMemBeg, kLinuxHighAppMemEnd,
+             kHighMetaShadow, kHighMetaSize);
+#else
+  MmapShadow(kLinuxShadowBeg, kLinuxShadowEnd,
+             kLinuxAppMemBeg, kLinuxAppMemEnd,
+             kMetaShadow, kMetaSize);
+#endif
+  // This memory range is used for thread stacks and large user mmaps.
+  // Frequently a thread uses only a small part of stack and similarly
+  // a program uses a small part of large mmap. On some programs
+  // we see 20% memory usage reduction without huge pages for this range.
+#ifdef MADV_NOHUGEPAGE
+  madvise((void*)MemToShadow(0x7f0000000000ULL),
+      0x10000000000ULL * kShadowMultiplier, MADV_NOHUGEPAGE);
+#endif
 
   // Protect gaps.
+#if SANITIZER_FREEBSD
+  const uptr kClosedLowBeg  = 0x100000000000;
+  const uptr kClosedLowEnd  = kLinuxLowShadowBeg - 1;
+  const uptr kClosedMidBeg = kLinuxLowShadowEnd + 1;
+  const uptr kClosedMidEnd = kLinuxHighShadowBeg - 1;
+#else
   const uptr kClosedLowBeg  = 0x200000;
   const uptr kClosedLowEnd  = kLinuxShadowBeg - 1;
   const uptr kClosedMidBeg = kLinuxShadowEnd + 1;
   const uptr kClosedMidEnd = min(min(kLinuxAppMemBeg, kTraceMemBegin),
       kMetaShadow);
-
+#endif
   ProtectRange(kClosedLowBeg, kClosedLowEnd);
   ProtectRange(kClosedMidBeg, kClosedMidEnd);
   VPrintf(2, "kClosedLow   %zx-%zx (%zuGB)\n",
       kClosedLowBeg, kClosedLowEnd, (kClosedLowEnd - kClosedLowBeg) >> 30);
   VPrintf(2, "kClosedMid   %zx-%zx (%zuGB)\n",
       kClosedMidBeg, kClosedMidEnd, (kClosedMidEnd - kClosedMidBeg) >> 30);
-  VPrintf(2, "app mem: %zx-%zx (%zuGB)\n",
-      kLinuxAppMemBeg, kLinuxAppMemEnd,
-      (kLinuxAppMemEnd - kLinuxAppMemBeg) >> 30);
-  VPrintf(2, "stack: %zx\n", (uptr)&shadow);
+
+  uptr stack = (uptr) &stack;
+  VPrintf(2, "stack: %zx\n", stack);
 
   MapRodata();
 }
-#endif
+#endif  // !TSAN_GO
 
 static uptr g_data_start;
 static uptr g_data_end;
 
 #ifndef TSAN_GO
 static void CheckPIE() {
   // Ensure that the binary is indeed compiled with -pie.
   MemoryMappingLayout proc_maps(true);
   uptr start, end;
   if (proc_maps.Next(&start, &end,
                      /*offset*/0, /*filename*/0, /*filename_size*/0,
                      /*protection*/0)) {
-    if ((u64)start < kLinuxAppMemBeg) {
+    if ((u64)start < kModulesBeg) {
       Printf("FATAL: ThreadSanitizer can not mmap the shadow memory ("
              "something is mapped at 0x%zx < 0x%zx)\n",
-             start, kLinuxAppMemBeg);
+             start, kModulesBeg);
       Printf("FATAL: Make sure to compile with -fPIE"
              " and to link with -pie.\n");
       Die();
     }
   }
 }
 
 static void InitDataSeg() {
   MemoryMappingLayout proc_maps(true);
   uptr start, end, offset;
   char name[128];
+#if SANITIZER_FREEBSD
+  // On FreeBSD BSS is usually the last block allocated within the
+  // low range and heap is the last block allocated within the range
+  // 0x800000000-0x8ffffffff.
+  while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name),
+                        /*protection*/ 0)) {
+    DPrintf("%p-%p %p %s\n", start, end, offset, name);
+    if ((start & 0xffff00000000ULL) == 0 && (end & 0xffff00000000ULL) == 0 &&
+        name[0] == '\0') {
+      g_data_start = start;
+      g_data_end = end;
+    }
+  }
+#else
   bool prev_is_data = false;
   while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name),
                         /*protection*/ 0)) {
     DPrintf("%p-%p %p %s\n", start, end, offset, name);
     bool is_data = offset != 0 && name[0] != 0;
     // BSS may get merged with [heap] in /proc/self/maps. This is not very
     // reliable.
     bool is_bss = offset == 0 &&
       (name[0] == 0 || internal_strcmp(name, "[heap]") == 0) && prev_is_data;
     if (g_data_start == 0 && is_data)
       g_data_start = start;
     if (is_bss)
       g_data_end = end;
     prev_is_data = is_data;
   }
+#endif
   DPrintf("guessed data_start=%p data_end=%p\n",  g_data_start, g_data_end);
   CHECK_LT(g_data_start, g_data_end);
   CHECK_GE((uptr)&g_data_start, g_data_start);
   CHECK_LT((uptr)&g_data_start, g_data_end);
 }
-
 #endif  // #ifndef TSAN_GO
 
 void InitializePlatform() {
   DisableCoreDumperIfNecessary();
 
   // Go maps shadow memory lazily and works fine with limited address space.
   // Unlimited stack is not a problem as well, because the executable
   // is not compiled with -pie.
int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m,
   // pthread_cleanup_push/pop are hardcore macros mess.
   // We can't intercept nor call them w/o including pthread.h.
   int res;
   pthread_cleanup_push(cleanup, arg);
   res = fn(c, m, abstime);
   pthread_cleanup_pop(0);
   return res;
 }
-#endif
+#endif  // #ifndef TSAN_GO
 
 }  // namespace __tsan
 
-#endif  // SANITIZER_LINUX
+#endif  // SANITIZER_LINUX || SANITIZER_FREEBSD