tsan: introduce RawShadow type

[dvyukov]

Summary

Currently we hardcode u64 type for shadow everywhere
and do lots of uptr<->u64* casts. It makes it hard to
change u64 to another type (e.g. u32) and makes it easy
to introduce bugs.
Introduce RawShadow type and use it in MemToShadow, ShadowToMem,
IsShadowMem and throughout the code base as u64 replacement.
This makes it possible to change u64 to something else in future
and generally improves static typing.

Depends on D107481.

[vitalybuka]

compiler-rt/lib/tsan/tests/unit/tsan_shadow_test.cpp

67

IsAligned?

[dvyukov]

use IsAligned

[dvyukov]

fix build (s/CHECK_EQ/CHECK/)

compiler-rt/lib/tsan/rtl/tsan_debugging.cpp

const char *__tsan_locate_address(uptr addr, char *name, uptr name_size,
                                   uptr *region_size_ptr) {
   uptr region_address = 0;
   uptr region_size = 0;
   const char *region_kind = nullptr;
   if (name && name_size > 0) name[0] = 0;
 
   if (IsMetaMem(reinterpret_cast<u32 *>(addr))) {
     region_kind = "meta shadow";
-  } else if (IsShadowMem(addr)) {
+  } else if (IsShadowMem(reinterpret_cast<RawShadow *>(addr))) {
     region_kind = "shadow";
   } else {
     bool is_stack = false;
     MBlock *b = 0;
     Allocator *a = allocator();
     if (a->PointerIsMine((void *)addr)) {
       void *block_begin = a->GetBlockBegin((void *)addr);
       if (block_begin) b = ctx->metamap.GetBlock((uptr)block_begin);

compiler-rt/lib/tsan/rtl/tsan_defs.h

 const uptr kShadowStackSize = 64 * 1024;
 
 // Count of shadow values in a shadow cell.
 const uptr kShadowCnt = 4;
 
 // That many user bytes are mapped onto a single shadow cell.
 const uptr kShadowCell = 8;
 
-// Size of a single shadow value (u64).
-const uptr kShadowSize = 8;
+// Single shadow value.
+typedef u64 RawShadow;
+const uptr kShadowSize = sizeof(RawShadow);
 
 // Shadow memory is kShadowMultiplier times larger than user memory.
 const uptr kShadowMultiplier = kShadowSize * kShadowCnt / kShadowCell;
 
 // That many user bytes are mapped onto a single meta shadow cell.
 // Must be less or equal to minimal memory allocator alignment.
 const uptr kMetaShadowCell = 8;
 

compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp

 struct dl_iterate_phdr_data {
   ThreadState *thr;
   uptr pc;
   dl_iterate_phdr_cb_t cb;
   void *data;
 };
 
 static bool IsAppNotRodata(uptr addr) {
-  return IsAppMem(addr) && *(u64*)MemToShadow(addr) != kShadowRodata;
+  return IsAppMem(addr) && *MemToShadow(addr) != kShadowRodata;
 }
 
 static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size,
                               void *data) {
   dl_iterate_phdr_data *cbdata = (dl_iterate_phdr_data *)data;
   // dlopen/dlclose allocate/free dynamic-linker-internal memory, which is later
   // accessible in dl_iterate_phdr callback. But we don't see synchronization
   // inside of dynamic linker, so we "unpoison" it here in order to not

compiler-rt/lib/tsan/rtl/tsan_interface_java.cpp

void __tsan_java_move(jptr src, jptr dst, jptr size) {
   DCHECK_NE(dst, src);
   DCHECK_NE(size, 0);
 
   // Assuming it's not running concurrently with threads that do
   // memory accesses and mutex operations (stop-the-world phase).
   ctx->metamap.MoveMemory(src, dst, size);
 
   // Move shadow.
-  u64 *s = (u64*)MemToShadow(src);
-  u64 *d = (u64*)MemToShadow(dst);
-  u64 *send = (u64*)MemToShadow(src + size);
+  RawShadow *s = MemToShadow(src);
+  RawShadow *d = MemToShadow(dst);
+  RawShadow *send = MemToShadow(src + size);
   uptr inc = 1;
   if (dst > src) {
-    s = (u64*)MemToShadow(src + size) - 1;
-    d = (u64*)MemToShadow(dst + size) - 1;
-    send = (u64*)MemToShadow(src) - 1;
+    s = MemToShadow(src + size) - 1;
+    d = MemToShadow(dst + size) - 1;
+    send = MemToShadow(src) - 1;
     inc = -1;
   }
   for (; s != send; s += inc, d += inc) {
     *d = *s;
     *s = 0;
   }
 }
 

compiler-rt/lib/tsan/rtl/tsan_platform.h

 
 
 template<typename Mapping>
 bool IsShadowMemImpl(uptr mem) {
   return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
 }
 
 ALWAYS_INLINE
-bool IsShadowMem(uptr mem) {
+bool IsShadowMem(RawShadow *p) {
+  uptr mem = reinterpret_cast<uptr>(p);
 #if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
   switch (vmaSize) {
     case 39: return IsShadowMemImpl<Mapping39>(mem);
     case 42: return IsShadowMemImpl<Mapping42>(mem);
     case 48: return IsShadowMemImpl<Mapping48>(mem);
   }
   DCHECK(0);
   return false;
#endif
   }
   DCHECK(0);
   return false;
 #else
   return IsShadowMemImpl<Mapping>(mem);
 #endif
 }
 
-
 template<typename Mapping>
 bool IsMetaMemImpl(uptr mem) {
   return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
 }
 
 ALWAYS_INLINE
 bool IsMetaMem(const u32 *p) {
   uptr mem = reinterpret_cast<uptr>(p);
#endif
   }
   DCHECK(0);
   return false;
 #else
   return IsMetaMemImpl<Mapping>(mem);
 #endif
 }
 
-template<typename Mapping>
-uptr MemToShadowImpl(uptr x) {
+template <typename Mapping>
+uptr MemToShadowRaw(uptr x) {
   DCHECK(IsAppMem(x));
 #if !SANITIZER_GO
   return (((x) & ~(Mapping::kAppMemMsk | (kShadowCell - 1)))
       ^ Mapping::kAppMemXor) * kShadowCnt;
 #else
 # ifndef SANITIZER_WINDOWS
   return ((x & ~(kShadowCell - 1)) * kShadowCnt) | Mapping::kShadowBeg;
 # else
   return ((x & ~(kShadowCell - 1)) * kShadowCnt) + Mapping::kShadowBeg;
 # endif
 #endif
 }
 
+template <typename Mapping>
+RawShadow *MemToShadowImpl(uptr x) {
+  return reinterpret_cast<RawShadow *>(MemToShadowRaw<Mapping>(x));
+}
+
 ALWAYS_INLINE
-uptr MemToShadow(uptr x) {
+RawShadow *MemToShadow(uptr x) {
 #if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
   switch (vmaSize) {
     case 39: return MemToShadowImpl<Mapping39>(x);
     case 42: return MemToShadowImpl<Mapping42>(x);
     case 48: return MemToShadowImpl<Mapping48>(x);
   }
   DCHECK(0);
-  return 0;
+  return nullptr;
 #elif defined(__powerpc64__)
   switch (vmaSize) {
 #if !SANITIZER_GO
     case 44: return MemToShadowImpl<Mapping44>(x);
 #endif
     case 46: return MemToShadowImpl<Mapping46>(x);
     case 47: return MemToShadowImpl<Mapping47>(x);
   }
   DCHECK(0);
-  return 0;
+  return nullptr;
 #elif defined(__mips64)
   switch (vmaSize) {
 #if !SANITIZER_GO
     case 40: return MemToShadowImpl<Mapping40>(x);
 #else
     case 47: return MemToShadowImpl<Mapping47>(x);
 #endif
   }
   DCHECK(0);
-  return 0;
+  return nullptr;
 #else
   return MemToShadowImpl<Mapping>(x);
 #endif
 }
 
-
 template<typename Mapping>
 u32 *MemToMetaImpl(uptr x) {
   DCHECK(IsAppMem(x));
 #if !SANITIZER_GO
   return (u32*)(((((x) & ~(Mapping::kAppMemMsk | (kMetaShadowCell - 1)))) /
       kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg);
 #else
 # ifndef SANITIZER_WINDOWS
#endif
   }
   DCHECK(0);
   return 0;
 #else
   return MemToMetaImpl<Mapping>(x);
 #endif
 }
 
-
-template<typename Mapping>
-uptr ShadowToMemImpl(uptr s) {
+template <typename Mapping>
+uptr ShadowToMemImpl(RawShadow *s) {
   DCHECK(IsShadowMem(s));
+  uptr sp = reinterpret_cast<uptr>(s);
 #if !SANITIZER_GO
   // The shadow mapping is non-linear and we've lost some bits, so we don't have
   // an easy way to restore the original app address. But the mapping is a
   // bijection, so we try to restore the address as belonging to low/mid/high
   // range consecutively and see if shadow->app->shadow mapping gives us the
   // same address.
-  uptr p = (s / kShadowCnt) ^ Mapping::kAppMemXor;
+  uptr p = (sp / kShadowCnt) ^ Mapping::kAppMemXor;
   if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
       MemToShadow(p) == s)
     return p;
 # ifdef TSAN_MID_APP_RANGE
-  p = ((s / kShadowCnt) ^ Mapping::kAppMemXor) +
+  p = ((sp / kShadowCnt) ^ Mapping::kAppMemXor) +
       (Mapping::kMidAppMemBeg & Mapping::kAppMemMsk);
   if (p >= Mapping::kMidAppMemBeg && p < Mapping::kMidAppMemEnd &&
       MemToShadow(p) == s)
     return p;
 # endif
-  return ((s / kShadowCnt) ^ Mapping::kAppMemXor) | Mapping::kAppMemMsk;
+  return ((sp / kShadowCnt) ^ Mapping::kAppMemXor) | Mapping::kAppMemMsk;
 #else  // #if !SANITIZER_GO
 # ifndef SANITIZER_WINDOWS
-  return (s & ~Mapping::kShadowBeg) / kShadowCnt;
+  return (sp & ~Mapping::kShadowBeg) / kShadowCnt;
 # else
-  return (s - Mapping::kShadowBeg) / kShadowCnt;
+  return (sp - Mapping::kShadowBeg) / kShadowCnt;
 # endif // SANITIZER_WINDOWS
 #endif
 }
 
 ALWAYS_INLINE
-uptr ShadowToMem(uptr s) {
+uptr ShadowToMem(RawShadow *s) {
 #if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
   switch (vmaSize) {
     case 39: return ShadowToMemImpl<Mapping39>(s);
     case 42: return ShadowToMemImpl<Mapping42>(s);
     case 48: return ShadowToMemImpl<Mapping48>(s);
   }
   DCHECK(0);
   return 0;
#endif
   }
   DCHECK(0);
   return 0;
 #else
   return ShadowToMemImpl<Mapping>(s);
 #endif
 }
 
-
-
 // The additional page is to catch shadow stack overflow as paging fault.
 // Windows wants 64K alignment for mmaps.
 const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace)
     + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1);
 
 template<typename Mapping>
 uptr GetThreadTraceImpl(int tid) {
   uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize;

compiler-rt/lib/tsan/rtl/tsan_rtl.h

 };
 typedef SizeClassAllocator64<AP64> PrimaryAllocator;
 #endif
 typedef CombinedAllocator<PrimaryAllocator> Allocator;
 typedef Allocator::AllocatorCache AllocatorCache;
 Allocator *allocator();
 #endif
 
-const u64 kShadowRodata = (u64)-1;  // .rodata shadow marker
+const RawShadow kShadowRodata = (RawShadow)-1;  // .rodata shadow marker
 
 // FastState (from most significant bit):
 //   ignore          : 1
 //   tid             : kTidBits
 //   unused          : -
 //   history_size    : 3
 //   epoch           : kClkBits
 class FastState {
#if !SANITIZER_GO
   IgnoreSet mop_ignore_set;
   IgnoreSet sync_ignore_set;
 #endif
   // C/C++ uses fixed size shadow stack embed into Trace.
   // Go uses malloc-allocated shadow stack with dynamic size.
   uptr *shadow_stack;
   uptr *shadow_stack_end;
   uptr *shadow_stack_pos;
-  u64 *racy_shadow_addr;
-  u64 racy_state[2];
+  RawShadow *racy_shadow_addr;
+  RawShadow racy_state[2];
   MutexSet mset;
   ThreadClock clock;
 #if !SANITIZER_GO
   Vector<JmpBuf> jmp_bufs;
   int ignore_interceptors;
 #endif
   const Tid tid;
   const int unique_id;

compiler-rt/lib/tsan/rtl/tsan_rtl.cpp

static void StopBackgroundThread() {
   atomic_store(&ctx->stop_background_thread, 1, memory_order_relaxed);
   internal_join_thread(ctx->background_thread);
   ctx->background_thread = 0;
 }
 #endif
 #endif
 
 void DontNeedShadowFor(uptr addr, uptr size) {
-  ReleaseMemoryPagesToOS(MemToShadow(addr), MemToShadow(addr + size));
+  ReleaseMemoryPagesToOS(reinterpret_cast<uptr>(MemToShadow(addr)),
+                         reinterpret_cast<uptr>(MemToShadow(addr + size)));
 }
 
 #if !SANITIZER_GO
 void UnmapShadow(ThreadState *thr, uptr addr, uptr size) {
   if (size == 0) return;
   DontNeedShadowFor(addr, size);
   ScopedGlobalProcessor sgp;
   ctx->metamap.ResetRange(thr->proc(), addr, size);
if (beg == end)
       continue;
     VPrintf(3, "checking shadow region %p-%p\n", beg, end);
     uptr prev = 0;
     for (uptr p0 = beg; p0 <= end; p0 += (end - beg) / 4) {
       for (int x = -(int)kShadowCell; x <= (int)kShadowCell; x += kShadowCell) {
         const uptr p = RoundDown(p0 + x, kShadowCell);
         if (p < beg || p >= end)
           continue;
-        const uptr s = MemToShadow(p);
+        RawShadow *const s = MemToShadow(p);
         u32 *const m = MemToMeta(p);
         VPrintf(3, "  checking pointer %p: shadow=%p meta=%p\n", p, s, m);
         CHECK(IsAppMem(p));
         CHECK(IsShadowMem(s));
         CHECK_EQ(p, ShadowToMem(s));
         CHECK(IsMetaMem(m));
         if (prev) {
           // Ensure that shadow and meta mappings are linear within a single
           // user range. Lots of code that processes memory ranges assumes it.
-          const uptr prev_s = MemToShadow(prev);
+          RawShadow *const prev_s = MemToShadow(prev);
           u32 *const prev_m = MemToMeta(prev);
-          CHECK_EQ(s - prev_s, (p - prev) * kShadowMultiplier);
+          CHECK_EQ((s - prev_s) * kShadowSize, (p - prev) * kShadowMultiplier);
           CHECK_EQ(m - prev_m, (p - prev) / kMetaShadowCell);
         }
         prev = p;
       }
     }
   }
 }
 

 #else
   return ContainsSameAccessSlow(s, a, sync_epoch, is_write);
 #endif
 }
 
 ALWAYS_INLINE USED
 void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
     int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic) {
-  u64 *shadow_mem = (u64*)MemToShadow(addr);
+  RawShadow *shadow_mem = MemToShadow(addr);
   DPrintf2("#%d: MemoryAccess: @%p %p size=%d"
       " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n",
       (int)thr->fast_state.tid(), (void*)pc, (void*)addr,
       (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem,
       (uptr)shadow_mem[0], (uptr)shadow_mem[1],
       (uptr)shadow_mem[2], (uptr)shadow_mem[3]);
 #if SANITIZER_DEBUG
   if (!IsAppMem(addr)) {
     Printf("Access to non app mem %zx\n", addr);
     DCHECK(IsAppMem(addr));
   }
-  if (!IsShadowMem((uptr)shadow_mem)) {
+  if (!IsShadowMem(shadow_mem)) {
     Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
-    DCHECK(IsShadowMem((uptr)shadow_mem));
+    DCHECK(IsShadowMem(shadow_mem));
   }
 #endif
 
   if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) {
     // Access to .rodata section, no races here.
     // Measurements show that it can be 10-20% of all memory accesses.
     return;
   }
static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
   // let it just crash as usual.
   if (!IsAppMem(addr) || !IsAppMem(addr + size - 1))
     return;
   // Don't want to touch lots of shadow memory.
   // If a program maps 10MB stack, there is no need reset the whole range.
   size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
   // UnmapOrDie/MmapFixedNoReserve does not work on Windows.
   if (SANITIZER_WINDOWS || size < common_flags()->clear_shadow_mmap_threshold) {
-    u64 *p = (u64*)MemToShadow(addr);
-    CHECK(IsShadowMem((uptr)p));
-    CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1)));
+    RawShadow *p = MemToShadow(addr);
+    CHECK(IsShadowMem(p));
+    CHECK(IsShadowMem(p + size * kShadowCnt / kShadowCell - 1));
     // FIXME: may overwrite a part outside the region
     for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) {
       p[i++] = val;
       for (uptr j = 1; j < kShadowCnt; j++)
         p[i++] = 0;
     }
   } else {
     // The region is big, reset only beginning and end.
     const uptr kPageSize = GetPageSizeCached();
-    u64 *begin = (u64*)MemToShadow(addr);
-    u64 *end = begin + size / kShadowCell * kShadowCnt;
-    u64 *p = begin;
+    RawShadow *begin = MemToShadow(addr);
+    RawShadow *end = begin + size / kShadowCell * kShadowCnt;
+    RawShadow *p = begin;
     // Set at least first kPageSize/2 to page boundary.
     while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) {
       *p++ = val;
       for (uptr j = 1; j < kShadowCnt; j++)
         *p++ = 0;
     }
     // Reset middle part.
-    u64 *p1 = p;
+    RawShadow *p1 = p;
     p = RoundDown(end, kPageSize);
     if (!MmapFixedSuperNoReserve((uptr)p1, (uptr)p - (uptr)p1))
       Die();
     // Set the ending.
     while (p < end) {
       *p++ = val;
       for (uptr j = 1; j < kShadowCnt; j++)
         *p++ = 0;

compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp

void ReportRace(ThreadState *thr) {
 
   bool freed = false;
   {
     Shadow s(thr->racy_state[1]);
     freed = s.GetFreedAndReset();
     thr->racy_state[1] = s.raw();
   }
 
-  uptr addr = ShadowToMem((uptr)thr->racy_shadow_addr);
+  uptr addr = ShadowToMem(thr->racy_shadow_addr);
   uptr addr_min = 0;
   uptr addr_max = 0;
   {
     uptr a0 = addr + Shadow(thr->racy_state[0]).addr0();
     uptr a1 = addr + Shadow(thr->racy_state[1]).addr0();
     uptr e0 = a0 + Shadow(thr->racy_state[0]).size();
     uptr e1 = a1 + Shadow(thr->racy_state[1]).size();
     addr_min = min(a0, a1);

compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp

void ThreadSetName(ThreadState *thr, const char *name) {
   ctx->thread_registry.SetThreadName(thr->tid, name);
 }
 
 void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
                        uptr size, bool is_write) {
   if (size == 0)
     return;
 
-  u64 *shadow_mem = (u64*)MemToShadow(addr);
+  RawShadow *shadow_mem = MemToShadow(addr);
   DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_write=%d\n",
       thr->tid, (void*)pc, (void*)addr,
       (int)size, is_write);
 
 #if SANITIZER_DEBUG
   if (!IsAppMem(addr)) {
     Printf("Access to non app mem %zx\n", addr);
     DCHECK(IsAppMem(addr));
   }
   if (!IsAppMem(addr + size - 1)) {
     Printf("Access to non app mem %zx\n", addr + size - 1);
     DCHECK(IsAppMem(addr + size - 1));
   }
-  if (!IsShadowMem((uptr)shadow_mem)) {
+  if (!IsShadowMem(shadow_mem)) {
     Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
-    DCHECK(IsShadowMem((uptr)shadow_mem));
+    DCHECK(IsShadowMem(shadow_mem));
   }
-  if (!IsShadowMem((uptr)(shadow_mem + size * kShadowCnt / 8 - 1))) {
+  if (!IsShadowMem(shadow_mem + size * kShadowCnt / 8 - 1)) {
     Printf("Bad shadow addr %p (%zx)\n",
                shadow_mem + size * kShadowCnt / 8 - 1, addr + size - 1);
-    DCHECK(IsShadowMem((uptr)(shadow_mem + size * kShadowCnt / 8 - 1)));
+    DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt / 8 - 1));
   }
 #endif
 
   if (*shadow_mem == kShadowRodata) {
     DCHECK(!is_write);
     // Access to .rodata section, no races here.
     // Measurements show that it can be 10-20% of all memory accesses.
     return;

compiler-rt/lib/tsan/tests/unit/tsan_shadow_test.cpp

TEST(Shadow, Mapping) {
   CHECK(IsShadowMem(MemToShadow((uptr)&global)));
   CHECK(IsShadowMem(MemToShadow((uptr)&stack)));
   CHECK(IsShadowMem(MemToShadow((uptr)heap)));
 }
 
 TEST(Shadow, Celling) {
   u64 aligned_data[4];
   char *data = (char*)aligned_data;
-  CHECK_EQ((uptr)data % kShadowSize, 0);
-  uptr s0 = MemToShadow((uptr)&data[0]);
-  CHECK_EQ(s0 % kShadowSize, 0);
+  CHECK(IsAligned(reinterpret_cast<uptr>(data), kShadowSize));
+  RawShadow *s0 = MemToShadow((uptr)&data[0]);
+  CHECK(IsAligned(reinterpret_cast<uptr>(s0), kShadowSize));
   for (unsigned i = 1; i < kShadowCell; i++)
     CHECK_EQ(s0, MemToShadow((uptr)&data[i]));
   for (unsigned i = kShadowCell; i < 2*kShadowCell; i++)
-    CHECK_EQ(s0 + kShadowSize*kShadowCnt, MemToShadow((uptr)&data[i]));
+    CHECK_EQ(s0 + kShadowCnt, MemToShadow((uptr)&data[i]));
   for (unsigned i = 2*kShadowCell; i < 3*kShadowCell; i++)
-    CHECK_EQ(s0 + 2*kShadowSize*kShadowCnt, MemToShadow((uptr)&data[i]));
+    CHECK_EQ(s0 + 2 * kShadowCnt, MemToShadow((uptr)&data[i]));
 }
 
 }  // namespace __tsan