tsan: don't save creation stack for some sync objects

[dvyukov]

Summary

Currently we save the creation stack for sync objects always.
But it's not needed to some sync objects, most notably atomics.
We simply don't use atomic creation stack anywhere.
Allow callers to control saving of the creation stack
and don't save it for atomics.

Depends on D107257.

compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp

static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
   // Assume the access is atomic.
   // Strictly saying even relaxed store cuts off release sequence,
   // so must reset the clock.
   if (!IsReleaseOrder(mo)) {
     NoTsanAtomicStore(a, v, mo);
     return;
   }
   __sync_synchronize();
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a);
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
   Lock l(&s->mtx);
   thr->fast_state.IncrementEpoch();
   // Can't increment epoch w/o writing to the trace as well.
   TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
   ReleaseStoreImpl(thr, pc, &s->clock);
   NoTsanAtomicStore(a, v, mo);
 }
 
 template <typename T, T (*F)(volatile T *v, T op)>
 static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
   if (LIKELY(mo == mo_relaxed))
     return F(a, v);
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a);
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
   Lock l(&s->mtx);
   thr->fast_state.IncrementEpoch();
   // Can't increment epoch w/o writing to the trace as well.
   TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
   if (IsAcqRelOrder(mo))
     AcquireReleaseImpl(thr, pc, &s->clock);
   else if (IsReleaseOrder(mo))
     ReleaseImpl(thr, pc, &s->clock);
if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) {
     T pr = func_cas(a, cc, v);
     if (pr == cc)
       return true;
     *c = pr;
     return false;
   }
 
   bool release = IsReleaseOrder(mo);
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a);
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
   RWLock l(&s->mtx, release);
   T cc = *c;
   T pr = func_cas(a, cc, v);
   bool success = pr == cc;
   if (!success) {
     *c = pr;
     mo = fmo;
   }

compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp

 void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz);
   if (!(flagz & MutexFlagLinkerInit) && IsAppMem(addr)) {
     CHECK(!thr->is_freeing);
     thr->is_freeing = true;
     MemoryWrite(thr, pc, addr, kSizeLog1);
     thr->is_freeing = false;
   }
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr);
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
   Lock l(&s->mtx);
   s->SetFlags(flagz & MutexCreationFlagMask);
+  // Save stack in the case the sync object was created before as atomic.
   if (!SANITIZER_GO && s->creation_stack_id == 0)
     s->creation_stack_id = CurrentStackId(thr, pc);
 }
 
 void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr);
   bool unlock_locked = false;
   u64 mid = 0;
if (IsAppMem(addr)) {
     thr->is_freeing = false;
   }
   // s will be destroyed and freed in MetaMap::FreeBlock.
 }
 
 void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   DPrintf("#%d: MutexPreLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
   if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
-    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr);
+    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
     {
       ReadLock l(&s->mtx);
       s->UpdateFlags(flagz);
       if (s->owner_tid != thr->tid) {
         Callback cb(thr, pc);
         ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
       }
     }
else
     rec = 1;
   if (IsAppMem(addr))
     MemoryReadAtomic(thr, pc, addr, kSizeLog1);
   u64 mid = 0;
   bool pre_lock = false;
   bool first = false;
   bool report_double_lock = false;
   {
-    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr);
+    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
     Lock l(&s->mtx);
     s->UpdateFlags(flagz);
     thr->fast_state.IncrementEpoch();
     TraceAddEvent(thr, thr->fast_state, EventTypeLock, s->GetId());
     if (s->owner_tid == kInvalidTid) {
       CHECK_EQ(s->recursion, 0);
       s->owner_tid = thr->tid;
       s->last_lock = thr->fast_state.raw();

 int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   DPrintf("#%d: MutexUnlock %zx flagz=0x%x\n", thr->tid, addr, flagz);
   if (IsAppMem(addr))
     MemoryReadAtomic(thr, pc, addr, kSizeLog1);
   u64 mid = 0;
   bool report_bad_unlock = false;
   int rec = 0;
   {
-    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr);
+    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
     Lock l(&s->mtx);
     thr->fast_state.IncrementEpoch();
     TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId());
     if (!SANITIZER_GO && (s->recursion == 0 || s->owner_tid != thr->tid)) {
       if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
         s->SetFlags(MutexFlagBroken);
         report_bad_unlock = true;
       }
int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   }
   return rec;
 }
 
 void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   DPrintf("#%d: MutexPreReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
   if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
     {
-      SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr);
+      SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
       ReadLock l(&s->mtx);
       s->UpdateFlags(flagz);
       Callback cb(thr, pc);
       ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
     }
     Callback cb(thr, pc);
     ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
   }
 }
 
 void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
   DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
   if (IsAppMem(addr))
     MemoryReadAtomic(thr, pc, addr, kSizeLog1);
   u64 mid = 0;
   bool report_bad_lock = false;
   bool pre_lock = false;
   {
-    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr);
+    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
     ReadLock l(&s->mtx);
     s->UpdateFlags(flagz);
     thr->fast_state.IncrementEpoch();
     TraceAddEvent(thr, thr->fast_state, EventTypeRLock, s->GetId());
     if (s->owner_tid != kInvalidTid) {
       if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
         s->SetFlags(MutexFlagBroken);
         report_bad_lock = true;

 
 void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr);
   if (IsAppMem(addr))
     MemoryReadAtomic(thr, pc, addr, kSizeLog1);
   u64 mid = 0;
   bool report_bad_unlock = false;
   {
-    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr);
+    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
     Lock l(&s->mtx);
     thr->fast_state.IncrementEpoch();
     TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
     if (s->owner_tid != kInvalidTid) {
       if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
         s->SetFlags(MutexFlagBroken);
         report_bad_unlock = true;
       }

 
 void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: MutexReadOrWriteUnlock %zx\n", thr->tid, addr);
   if (IsAppMem(addr))
     MemoryReadAtomic(thr, pc, addr, kSizeLog1);
   u64 mid = 0;
   bool report_bad_unlock = false;
   {
-    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr);
+    SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
     Lock l(&s->mtx);
     bool write = true;
     if (s->owner_tid == kInvalidTid) {
       // Seems to be read unlock.
       write = false;
       thr->fast_state.IncrementEpoch();
       TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
       ReleaseImpl(thr, pc, &s->read_clock);
void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) {
   if (common_flags()->detect_deadlocks) {
     Callback cb(thr, pc);
     ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
   }
 }
 
 void MutexRepair(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: MutexRepair %zx\n", thr->tid, addr);
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr);
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
   Lock l(&s->mtx);
   s->owner_tid = kInvalidTid;
   s->recursion = 0;
 }
 
 void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: MutexInvalidAccess %zx\n", thr->tid, addr);
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr);
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
   ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr, s->GetId());
 }
 
 void Acquire(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: Acquire %zx\n", thr->tid, addr);
   if (thr->ignore_sync)
     return;
   SyncVar *s = ctx->metamap.GetSyncIfExists(addr);
void AcquireGlobal(ThreadState *thr) {
   ThreadRegistryLock l(&ctx->thread_registry);
   ctx->thread_registry.RunCallbackForEachThreadLocked(UpdateClockCallback, thr);
 }
 
 void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr);
   if (thr->ignore_sync)
     return;
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr);
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
   Lock l(&s->mtx);
   thr->fast_state.IncrementEpoch();
   // Can't increment epoch w/o writing to the trace as well.
   TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
   ReleaseStoreAcquireImpl(thr, pc, &s->clock);
 }
 
 void Release(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: Release %zx\n", thr->tid, addr);
   if (thr->ignore_sync)
     return;
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr);
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
   Lock l(&s->mtx);
   thr->fast_state.IncrementEpoch();
   // Can't increment epoch w/o writing to the trace as well.
   TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
   ReleaseImpl(thr, pc, &s->clock);
 }
 
 void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) {
   DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr);
   if (thr->ignore_sync)
     return;
-  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr);
+  SyncVar *s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
   Lock l(&s->mtx);
   thr->fast_state.IncrementEpoch();
   // Can't increment epoch w/o writing to the trace as well.
   TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
   ReleaseStoreImpl(thr, pc, &s->clock);
 }
 
 #if !SANITIZER_GO

compiler-rt/lib/tsan/rtl/tsan_sync.h

struct SyncVar {
   atomic_uint32_t flags;
   u32 next;  // in MetaMap
   DDMutex dd;
   SyncClock read_clock;  // Used for rw mutexes only.
   // The clock is placed last, so that it is situated on a different cache line
   // with the mtx. This reduces contention for hot sync objects.
   SyncClock clock;
 
-  void Init(ThreadState *thr, uptr pc, uptr addr, u64 uid);
+  void Init(ThreadState *thr, uptr pc, uptr addr, u64 uid, bool save_stack);
   void Reset(Processor *proc);
 
   u64 GetId() const {
     // 48 lsb is addr, then 14 bits is low part of uid, then 2 zero bits.
     return GetLsb((u64)addr | (uid << 48), 60);
   }
   bool CheckId(u64 uid) const {
     CHECK_EQ(uid, GetLsb(uid, 14));
public:
   MetaMap();
 
   void AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz);
   uptr FreeBlock(Processor *proc, uptr p);
   bool FreeRange(Processor *proc, uptr p, uptr sz);
   void ResetRange(Processor *proc, uptr p, uptr sz);
   MBlock* GetBlock(uptr p);
 
-  SyncVar *GetSyncOrCreate(ThreadState *thr, uptr pc, uptr addr) {
-    return GetSync(thr, pc, addr, true);
+  SyncVar *GetSyncOrCreate(ThreadState *thr, uptr pc, uptr addr,
+                           bool save_stack) {
+    return GetSync(thr, pc, addr, true, save_stack);
   }
   SyncVar *GetSyncIfExists(uptr addr) {
-    return GetSync(nullptr, 0, addr, false);
+    return GetSync(nullptr, 0, addr, false, false);
   }
 
   void MoveMemory(uptr src, uptr dst, uptr sz);
 
   void OnProcIdle(Processor *proc);
 
  private:
   static const u32 kFlagMask  = 3u << 30;
   static const u32 kFlagBlock = 1u << 30;
   static const u32 kFlagSync  = 2u << 30;
   typedef DenseSlabAlloc<MBlock, 1 << 18, 1 << 12, kFlagMask> BlockAlloc;
   typedef DenseSlabAlloc<SyncVar, 1 << 20, 1 << 10, kFlagMask> SyncAlloc;
   BlockAlloc block_alloc_;
   SyncAlloc sync_alloc_;
   atomic_uint64_t uid_gen_;
 
-  SyncVar *GetSync(ThreadState *thr, uptr pc, uptr addr, bool create);
+  SyncVar *GetSync(ThreadState *thr, uptr pc, uptr addr, bool create,
+                   bool save_stack);
 };
 
 }  // namespace __tsan
 
 #endif  // TSAN_SYNC_H

compiler-rt/lib/tsan/rtl/tsan_sync.cpp

 #include "tsan_mman.h"
 
 namespace __tsan {
 
 void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s);
 
 SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(0); }
 
-void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid) {
+void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid,
+                   bool save_stack) {
   this->addr = addr;
   this->uid = uid;
   this->next = 0;
 
   creation_stack_id = 0;
-  if (!SANITIZER_GO)  // Go does not use them
+  if (save_stack && !SANITIZER_GO)  // Go does not use them
     creation_stack_id = CurrentStackId(thr, pc);
   if (common_flags()->detect_deadlocks)
     DDMutexInit(thr, pc, this);
 }
 
 void SyncVar::Reset(Processor *proc) {
   uid = 0;
   creation_stack_id = 0;
for (;;) {
     if (idx & kFlagBlock)
       return block_alloc_.Map(idx & ~kFlagMask);
     DCHECK(idx & kFlagSync);
     SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);
     idx = s->next;
   }
 }
 
-SyncVar *MetaMap::GetSync(ThreadState *thr, uptr pc, uptr addr, bool create) {
+SyncVar *MetaMap::GetSync(ThreadState *thr, uptr pc, uptr addr, bool create,
+                          bool save_stack) {
   u32 *meta = MemToMeta(addr);
   u32 idx0 = *meta;
   u32 myidx = 0;
   SyncVar *mys = nullptr;
   for (;;) {
     u32 idx = idx0;
     for (;;) {
       if (UNLIKELY((idx == 0) || (idx & kFlagBlock)))
if (*meta != idx0) {
       idx0 = *meta;
       continue;
     }
 
     if (myidx == 0) {
       const u64 uid = atomic_fetch_add(&uid_gen_, 1, memory_order_relaxed);
       myidx = sync_alloc_.Alloc(&thr->proc()->sync_cache);
       mys = sync_alloc_.Map(myidx);
-      mys->Init(thr, pc, addr, uid);
+      mys->Init(thr, pc, addr, uid, save_stack);
     }
     mys->next = idx0;
     if (atomic_compare_exchange_strong((atomic_uint32_t*)meta, &idx0,
         myidx | kFlagSync, memory_order_release)) {
       return mys;
     }
   }
 }

compiler-rt/lib/tsan/tests/unit/tsan_sync_test.cpp

 //===-- tsan_sync_test.cpp ------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is a part of ThreadSanitizer (TSan), a race detector.
 //
 //===----------------------------------------------------------------------===//
 #include "tsan_sync.h"
 #include "tsan_rtl.h"
 #include "gtest/gtest.h"
 
 namespace __tsan {
 
 TEST(MetaMap, Basic) {
   ThreadState *thr = cur_thread();
   MetaMap *m = &ctx->metamap;
TEST(MetaMap, Sync) {
   // them from detecting that we exit runtime with mutexes held.
   ScopedIgnoreInterceptors ignore;
   ThreadState *thr = cur_thread();
   MetaMap *m = &ctx->metamap;
   u64 block[4] = {};  // fake malloc block
   m->AllocBlock(thr, 0, (uptr)&block[0], 4 * sizeof(u64));
   SyncVar *s1 = m->GetSyncIfExists((uptr)&block[0]);
   EXPECT_EQ(s1, (SyncVar*)0);
-  s1 = m->GetSyncOrCreate(thr, 0, (uptr)&block[0]);
+  s1 = m->GetSyncOrCreate(thr, 0, (uptr)&block[0], false);
   EXPECT_NE(s1, (SyncVar*)0);
   EXPECT_EQ(s1->addr, (uptr)&block[0]);
-  SyncVar *s2 = m->GetSyncOrCreate(thr, 0, (uptr)&block[1]);
+  SyncVar *s2 = m->GetSyncOrCreate(thr, 0, (uptr)&block[1], false);
   EXPECT_NE(s2, (SyncVar*)0);
   EXPECT_EQ(s2->addr, (uptr)&block[1]);
   m->FreeBlock(thr->proc(), (uptr)&block[0]);
   s1 = m->GetSyncIfExists((uptr)&block[0]);
   EXPECT_EQ(s1, (SyncVar*)0);
   s2 = m->GetSyncIfExists((uptr)&block[1]);
   EXPECT_EQ(s2, (SyncVar*)0);
   m->OnProcIdle(thr->proc());
 }
 
 TEST(MetaMap, MoveMemory) {
   ScopedIgnoreInterceptors ignore;
   ThreadState *thr = cur_thread();
   MetaMap *m = &ctx->metamap;
   u64 block1[4] = {};  // fake malloc block
   u64 block2[4] = {};  // fake malloc block
   m->AllocBlock(thr, 0, (uptr)&block1[0], 3 * sizeof(u64));
   m->AllocBlock(thr, 0, (uptr)&block1[3], 1 * sizeof(u64));
-  SyncVar *s1 = m->GetSyncOrCreate(thr, 0, (uptr)&block1[0]);
-  SyncVar *s2 = m->GetSyncOrCreate(thr, 0, (uptr)&block1[1]);
+  SyncVar *s1 = m->GetSyncOrCreate(thr, 0, (uptr)&block1[0], false);
+  SyncVar *s2 = m->GetSyncOrCreate(thr, 0, (uptr)&block1[1], false);
   m->MoveMemory((uptr)&block1[0], (uptr)&block2[0], 4 * sizeof(u64));
   MBlock *mb1 = m->GetBlock((uptr)&block1[0]);
   EXPECT_EQ(mb1, (MBlock*)0);
   MBlock *mb2 = m->GetBlock((uptr)&block1[3]);
   EXPECT_EQ(mb2, (MBlock*)0);
   mb1 = m->GetBlock((uptr)&block2[0]);
   EXPECT_NE(mb1, (MBlock*)0);
   EXPECT_EQ(mb1->siz, 3 * sizeof(u64));

 }
 
 TEST(MetaMap, ResetSync) {
   ScopedIgnoreInterceptors ignore;
   ThreadState *thr = cur_thread();
   MetaMap *m = &ctx->metamap;
   u64 block[1] = {};  // fake malloc block
   m->AllocBlock(thr, 0, (uptr)&block[0], 1 * sizeof(u64));
-  SyncVar *s = m->GetSyncOrCreate(thr, 0, (uptr)&block[0]);
+  SyncVar *s = m->GetSyncOrCreate(thr, 0, (uptr)&block[0], false);
   s->Reset(thr->proc());
   uptr sz = m->FreeBlock(thr->proc(), (uptr)&block[0]);
   EXPECT_EQ(sz, 1 * sizeof(u64));
 }
 
 }  // namespace __tsan