[tsan] Enable 48-bit VMA support on aarch64

[zatrazz]

Summary

This patch adds 48-bits VMA support for tsan on aarch64. As current
mappings for aarch64, 48-bit VMA also supports PIE executable. This
limits the mapping mechanism because the PIE address bits
(usually 0aaaaXXXXXXXX) makes it harder to create a mask/xor value
to include all memory regions. I think it is possible to create a
large application VAM range by either dropping PIE support or tune
current range.

The patch also remove the heap accounting and checking if the architecture
does not uses 64-bit size allocator. This is due the fact heap VAM
range is used only for the allocator on TSAN, so this allows use the
range for application usage.

It also changes slight the way addresses are packed in SyncVar structure:
previously it assumes x86_64 as the maximum VMA range. Since ID is 14 bits
wide, shifting 48 bits should be ok.

Tested on x86_64, ppc64le and aarch64 (39 and 48 bits VMA).

[zatrazz]

Ping.

[dvyukov]

lib/tsan/rtl/tsan_platform.h

197

I see that kHeapMemBeg/End situation is quite inconsistent across mappings. E.g. aarch64/42vma reserves a non-empty range for heap:

static const uptr kHeapMemBeg    = 0x3e000000000ull;
static const uptr kHeapMemEnd    = 0x3f000000000ull;

But it is actually unused.

I see ways to get situation under control:

1. Either guard the remaining uses of these constants in this file by TSAN_USE_ALLOCATOR64, and then remove definitions of the constants for configurations that don't use 64-bit allocator.
2. Or define kHeapMemBeg==kHeapMemEnd for such arches and do not other changes. Why did you guard all uses of these constants by SANITIZER_CAN_USE_ALLOCATOR64? I think ProtectRange should handle an empty range, CheckUserRegions should handle them as well. Can we remove all these ifdefs?

485

drop "on tsan"
everything in tsan/* is about tsan

486

s/used/use/

486

s/an user/a user/

lib/tsan/rtl/tsan_platform_posix.cc

145 ↗

(On Diff #64764)

#if SANITIZER_CAN_USE_ALLOCATOR64 is a wrong condition for tsan, it uses a different condition. Please create a new define TSAN_USE_ALLOCATOR64 for tsan.

lib/tsan/rtl/tsan_sync.h

52–53

Please also update this comment.

[zatrazz]

I simplified the patch a bit by removing the allocator64 define usage. I also updated the sync comments.

[dvyukov]

Much cleaner now. Thanks. LGTM

lib/tsan/rtl/tsan_platform.h

struct Mapping42 {
   static const uptr kHeapMemEnd    = 0x3f000000000ull;
   static const uptr kHiAppMemBeg   = 0x3f000000000ull;
   static const uptr kHiAppMemEnd   = 0x3ffffffffffull;
   static const uptr kAppMemMsk     = 0x3c000000000ull;
   static const uptr kAppMemXor     = 0x04000000000ull;
   static const uptr kVdsoBeg       = 0x37f00000000ull;
 };
 
+struct Mapping48 {
+  static const uptr kLoAppMemBeg   = 0x0000000001000ull;
+  static const uptr kLoAppMemEnd   = 0x0000200000000ull;
+  static const uptr kShadowBeg     = 0x0002000000000ull;
+  static const uptr kShadowEnd     = 0x0004000000000ull;
+  static const uptr kMetaShadowBeg = 0x0005000000000ull;
+  static const uptr kMetaShadowEnd = 0x0006000000000ull;
+  static const uptr kMidAppMemBeg  = 0x0aaaa00000000ull;
+  static const uptr kMidAppMemEnd  = 0x0aaaf00000000ull;
+  static const uptr kMidShadowOff  = 0x0aaa800000000ull;
+  static const uptr kTraceMemBeg   = 0x0f06000000000ull;
+  static const uptr kTraceMemEnd   = 0x0f06200000000ull;
+  static const uptr kHeapMemBeg    = 0x0ffff00000000ull;
+  static const uptr kHeapMemEnd    = 0x0ffff00000000ull;
+  static const uptr kHiAppMemBeg   = 0x0ffff00000000ull;
+  static const uptr kHiAppMemEnd   = 0x1000000000000ull;
+  static const uptr kAppMemMsk     = 0x0fff800000000ull;
+  static const uptr kAppMemXor     = 0x0000800000000ull;
+  static const uptr kVdsoBeg       = 0xffff000000000ull;
+};
+
 // Indicates the runtime will define the memory regions at runtime.
 #define TSAN_RUNTIME_VMA 1
 // Indicates that mapping defines a mid range memory segment.
 #define TSAN_MID_APP_RANGE 1
 #elif defined(__powerpc64__)
 // PPC64 supports multiple VMA which leads to multiple address transformation
 // functions.  To support these multiple VMAS transformations and mappings TSAN
 // runtime for PPC64 uses an external memory read (vmaSize) to select which
 // mapping to use.  Although slower, it make a same instrumented binary run on
 // multiple kernels.
 
 /*
 C/C++ on linux/powerpc64 (44-bit VMA)
#endif
     case MAPPING_TRACE_BEG: return Mapping::kTraceMemBeg;
     case MAPPING_TRACE_END: return Mapping::kTraceMemEnd;
   }
 }
 
 template<int Type>
 uptr MappingArchImpl(void) {
 #ifdef __aarch64__
-  if (vmaSize == 39)
-    return MappingImpl<Mapping39, Type>();
-  else
-    return MappingImpl<Mapping42, Type>();
+  switch (vmaSize) {
+    case 39: return MappingImpl<Mapping39, Type>();
+    case 42: return MappingImpl<Mapping42, Type>();
+    case 48: return MappingImpl<Mapping48, Type>();
+  }
   DCHECK(0);
+  return 0;
 #elif defined(__powerpc64__)
   if (vmaSize == 44)
     return MappingImpl<Mapping44, Type>();
   else
     return MappingImpl<Mapping46, Type>();
   DCHECK(0);
 #else
   return MappingImpl<Mapping, Type>();
case 2:
     *end = HeapMemEnd();
     return true;
 # ifdef TSAN_MID_APP_RANGE
   case 3:
     *start = MidAppMemBeg();
     *end = MidAppMemEnd();
     return true;
 # endif
 #else
   case 0:
     *start = AppMemBeg();
     *end = AppMemEnd();
     return true;
 #endif
   }
 }
 
 ALWAYS_INLINE

 #else
   return mem >= Mapping::kAppMemBeg && mem < Mapping::kAppMemEnd;
 #endif
 }
 
 ALWAYS_INLINE
 bool IsAppMem(uptr mem) {
 #ifdef __aarch64__
-  if (vmaSize == 39)
-    return IsAppMemImpl<Mapping39>(mem);
-  else
-    return IsAppMemImpl<Mapping42>(mem);
+  switch (vmaSize) {
+    case 39: return IsAppMemImpl<Mapping39>(mem);
+    case 42: return IsAppMemImpl<Mapping42>(mem);
+    case 48: return IsAppMemImpl<Mapping48>(mem);
+  }
   DCHECK(0);
+  return false;
 #elif defined(__powerpc64__)
   if (vmaSize == 44)
     return IsAppMemImpl<Mapping44>(mem);
   else
     return IsAppMemImpl<Mapping46>(mem);
   DCHECK(0);
 #else
   return IsAppMemImpl<Mapping>(mem);
 #endif
 }
 
 
 template<typename Mapping>
 bool IsShadowMemImpl(uptr mem) {
   return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
 }
 
 ALWAYS_INLINE
 bool IsShadowMem(uptr mem) {
 #ifdef __aarch64__
-  if (vmaSize == 39)
-    return IsShadowMemImpl<Mapping39>(mem);
-  else
-    return IsShadowMemImpl<Mapping42>(mem);
+  switch (vmaSize) {
+    case 39: return IsShadowMemImpl<Mapping39>(mem);
+    case 42: return IsShadowMemImpl<Mapping42>(mem);
+    case 48: return IsShadowMemImpl<Mapping48>(mem);
+  }
   DCHECK(0);
+  return false;
 #elif defined(__powerpc64__)
   if (vmaSize == 44)
     return IsShadowMemImpl<Mapping44>(mem);
   else
     return IsShadowMemImpl<Mapping46>(mem);
   DCHECK(0);
 #else
   return IsShadowMemImpl<Mapping>(mem);
 #endif
 }
 
 
 template<typename Mapping>
 bool IsMetaMemImpl(uptr mem) {
   return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
 }
 
 ALWAYS_INLINE
 bool IsMetaMem(uptr mem) {
 #ifdef __aarch64__
-  if (vmaSize == 39)
-    return IsMetaMemImpl<Mapping39>(mem);
-  else
-    return IsMetaMemImpl<Mapping42>(mem);
+  switch (vmaSize) {
+    case 39: return IsMetaMemImpl<Mapping39>(mem);
+    case 42: return IsMetaMemImpl<Mapping42>(mem);
+    case 48: return IsMetaMemImpl<Mapping48>(mem);
+  }
   DCHECK(0);
+  return false;
 #elif defined(__powerpc64__)
   if (vmaSize == 44)
     return IsMetaMemImpl<Mapping44>(mem);
   else
     return IsMetaMemImpl<Mapping46>(mem);
   DCHECK(0);
 #else
   return IsMetaMemImpl<Mapping>(mem);
# else
   return ((x & ~(kShadowCell - 1)) * kShadowCnt) + Mapping::kShadowBeg;
 # endif
 #endif
 }
 
 ALWAYS_INLINE
 uptr MemToShadow(uptr x) {
 #ifdef __aarch64__
-  if (vmaSize == 39)
-    return MemToShadowImpl<Mapping39>(x);
-  else
-    return MemToShadowImpl<Mapping42>(x);
+  switch (vmaSize) {
+    case 39: return MemToShadowImpl<Mapping39>(x);
+    case 42: return MemToShadowImpl<Mapping42>(x);
+    case 48: return MemToShadowImpl<Mapping48>(x);
+  }
   DCHECK(0);
+  return 0;
 #elif defined(__powerpc64__)
   if (vmaSize == 44)
     return MemToShadowImpl<Mapping44>(x);
   else
     return MemToShadowImpl<Mapping46>(x);
   DCHECK(0);
 #else
   return MemToShadowImpl<Mapping>(x);
#else
   return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
       kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg);
 #endif
 }
 
 ALWAYS_INLINE
 u32 *MemToMeta(uptr x) {
 #ifdef __aarch64__
-  if (vmaSize == 39)
-    return MemToMetaImpl<Mapping39>(x);
-  else
-    return MemToMetaImpl<Mapping42>(x);
+  switch (vmaSize) {
+    case 39: return MemToMetaImpl<Mapping39>(x);
+    case 42: return MemToMetaImpl<Mapping42>(x);
+    case 48: return MemToMetaImpl<Mapping48>(x);
+  }
   DCHECK(0);
+  return 0;
 #elif defined(__powerpc64__)
   if (vmaSize == 44)
     return MemToMetaImpl<Mapping44>(x);
   else
     return MemToMetaImpl<Mapping46>(x);
   DCHECK(0);
 #else
   return MemToMetaImpl<Mapping>(x);
# else
   return (s - Mapping::kShadowBeg) / kShadowCnt;
 # endif // SANITIZER_WINDOWS
 #endif
 }
 
 ALWAYS_INLINE
 uptr ShadowToMem(uptr s) {
 #ifdef __aarch64__
-  if (vmaSize == 39)
-    return ShadowToMemImpl<Mapping39>(s);
-  else
-    return ShadowToMemImpl<Mapping42>(s);
+  switch (vmaSize) {
+    case 39: return ShadowToMemImpl<Mapping39>(s);
+    case 42: return ShadowToMemImpl<Mapping42>(s);
+    case 48: return ShadowToMemImpl<Mapping48>(s);
+  }
   DCHECK(0);
+  return 0;
 #elif defined(__powerpc64__)
   if (vmaSize == 44)
     return ShadowToMemImpl<Mapping44>(s);
   else
     return ShadowToMemImpl<Mapping46>(s);
   DCHECK(0);
 #else
   return ShadowToMemImpl<Mapping>(s);
uptr GetThreadTraceImpl(int tid) {
   uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize;
   DCHECK_LT(p, Mapping::kTraceMemEnd);
   return p;
 }
 
 ALWAYS_INLINE
 uptr GetThreadTrace(int tid) {
 #ifdef __aarch64__
-  if (vmaSize == 39)
-    return GetThreadTraceImpl<Mapping39>(tid);
-  else
-    return GetThreadTraceImpl<Mapping42>(tid);
+  switch (vmaSize) {
+    case 39: return GetThreadTraceImpl<Mapping39>(tid);
+    case 42: return GetThreadTraceImpl<Mapping42>(tid);
+    case 48: return GetThreadTraceImpl<Mapping48>(tid);
+  }
   DCHECK(0);
+  return 0;
 #elif defined(__powerpc64__)
   if (vmaSize == 44)
     return GetThreadTraceImpl<Mapping44>(tid);
   else
     return GetThreadTraceImpl<Mapping46>(tid);
   DCHECK(0);
 #else
   return GetThreadTraceImpl<Mapping>(tid);
 #endif
 }
 
 
 template<typename Mapping>
 uptr GetThreadTraceHeaderImpl(int tid) {
   uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize
       + kTraceSize * sizeof(Event);
   DCHECK_LT(p, Mapping::kTraceMemEnd);
   return p;
 }
 
 ALWAYS_INLINE
 uptr GetThreadTraceHeader(int tid) {
 #ifdef __aarch64__
-  if (vmaSize == 39)
-    return GetThreadTraceHeaderImpl<Mapping39>(tid);
-  else
-    return GetThreadTraceHeaderImpl<Mapping42>(tid);
+  switch (vmaSize) {
+    case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid);
+    case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid);
+    case 48: return GetThreadTraceHeaderImpl<Mapping48>(tid);
+  }
   DCHECK(0);
+  return 0;
 #elif defined(__powerpc64__)
   if (vmaSize == 44)
     return GetThreadTraceHeaderImpl<Mapping44>(tid);
   else
     return GetThreadTraceHeaderImpl<Mapping46>(tid);
   DCHECK(0);
 #else
   return GetThreadTraceHeaderImpl<Mapping>(tid);

lib/tsan/rtl/tsan_platform_linux.cc

 
 #endif  // #ifndef SANITIZER_GO
 
 void InitializePlatformEarly() {
 #ifdef TSAN_RUNTIME_VMA
   vmaSize =
     (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
 #if defined(__aarch64__)
-  if (vmaSize != 39 && vmaSize != 42) {
+  if (vmaSize != 39 && vmaSize != 42 && vmaSize != 48) {
     Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
-    Printf("FATAL: Found %d - Supported 39 and 42\n", vmaSize);
+    Printf("FATAL: Found %d - Supported 39, 42 and 48\n", vmaSize);
     Die();
   }
 #elif defined(__powerpc64__)
   if (vmaSize != 44 && vmaSize != 46) {
     Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
     Printf("FATAL: Found %d - Supported 44 and 46\n", vmaSize);
     Die();
   }

lib/tsan/rtl/tsan_rtl.cc

Printf("FATAL: ThreadSanitizer can not mmap thread trace (%p/%p->%p)\n",
         addr, size, addr1);
     Die();
   }
 }
 
 static void CheckShadowMapping() {
   uptr beg, end;
   for (int i = 0; GetUserRegion(i, &beg, &end); i++) {
+    // Skip cases for empty regions (heap definition for architectures that
+    // do not use 64-bit allocator).
+    if (beg ==end)
+      continue;
     VPrintf(3, "checking shadow region %p-%p\n", beg, end);
     for (uptr p0 = beg; p0 <= end; p0 += (end - beg) / 4) {
       for (int x = -1; x <= 1; x++) {
         const uptr p = p0 + x;
         if (p < beg || p >= end)
           continue;
         const uptr s = MemToShadow(p);
         const uptr m = (uptr)MemToMeta(p);

lib/tsan/rtl/tsan_sync.h

struct SyncVar {
   SyncClock read_clock;  // Used for rw mutexes only.
   // The clock is placed last, so that it is situated on a different cache line
   // with the mtx. This reduces contention for hot sync objects.
   SyncClock clock;
 
   void Init(ThreadState *thr, uptr pc, uptr addr, u64 uid);
   void Reset(Processor *proc);
 
   u64 GetId() const {
-    // 47 lsb is addr, then 14 bits is low part of uid, then 3 zero bits.
-    return GetLsb((u64)addr | (uid << 47), 61);
+    // 48 lsb is addr, then 14 bits is low part of uid, then 2 zero bits.
+    return GetLsb((u64)addr | (uid << 48), 60);
   }
   bool CheckId(u64 uid) const {
     CHECK_EQ(uid, GetLsb(uid, 14));
     return GetLsb(this->uid, 14) == uid;
   }
   static uptr SplitId(u64 id, u64 *uid) {
-    *uid = id >> 47;
-    return (uptr)GetLsb(id, 47);
+    *uid = id >> 48;
+    return (uptr)GetLsb(id, 48);
   }
 };
 
 /* MetaMap allows to map arbitrary user pointers onto various descriptors.
    Currently it maps pointers to heap block descriptors and sync var descs.
    It uses 1/2 direct shadow, see tsan_platform.h.
 */
 class MetaMap {