Index: include/sanitizer/tsan_interface.h =================================================================== --- include/sanitizer/tsan_interface.h +++ include/sanitizer/tsan_interface.h @@ -137,6 +137,24 @@ void __tsan_external_read(void *addr, void *caller_pc, void *tag); void __tsan_external_write(void *addr, void *caller_pc, void *tag); +// Fiber switching API. +// - TSAN context for fiber can be created by __tsan_create_fiber +// and freed by __tsan_destroy_fiber. +// - TSAN context of current fiber or thread can be obtained +// by calling __tsan_get_current_fiber. +// - __tsan_switch_to_fiber should be called immediatly before switch +// to fiber, such as call of swapcontext. +// - Fiber name can be set by __tsan_set_fiber_name. +void *__tsan_get_current_fiber(void); +void *__tsan_create_fiber(unsigned flags); +void __tsan_destroy_fiber(void *fiber); +void __tsan_switch_to_fiber(void *fiber, unsigned flags); +void __tsan_set_fiber_name(void *fiber, const char *name); + +// Flags for __tsan_switch_to_fiber: +// Do not establish a happens-before relation between fibers +const unsigned __tsan_switch_to_fiber_no_sync = 1 << 0; + #ifdef __cplusplus } // extern "C" #endif Index: lib/asan/asan_fuchsia.cc =================================================================== --- lib/asan/asan_fuchsia.cc +++ lib/asan/asan_fuchsia.cc @@ -179,7 +179,7 @@ SetCurrentThread(thread); // In lieu of AsanThread::ThreadStart. - asanThreadRegistry().StartThread(thread->tid(), os_id, /*workerthread*/ false, + asanThreadRegistry().StartThread(thread->tid(), os_id, ThreadType::Regular, nullptr); } Index: lib/asan/asan_mac.cc =================================================================== --- lib/asan/asan_mac.cc +++ lib/asan/asan_mac.cc @@ -182,8 +182,8 @@ t = AsanThread::Create(/* start_routine */ nullptr, /* arg */ nullptr, parent_tid, stack, /* detached */ true); t->Init(); - asanThreadRegistry().StartThread(t->tid(), GetTid(), - /* workerthread */ true, 0); + asanThreadRegistry().StartThread(t->tid(), GetTid(), ThreadType::Worker, + nullptr); SetCurrentThread(t); } } Index: lib/asan/asan_rtems.cc =================================================================== --- lib/asan/asan_rtems.cc +++ lib/asan/asan_rtems.cc @@ -184,8 +184,8 @@ // Determine whether we are starting or restarting the thread. if (status == ThreadStatusCreated) // In lieu of AsanThread::ThreadStart. - asanThreadRegistry().StartThread(thread->tid(), os_id, - /*workerthread*/ false, nullptr); + asanThreadRegistry().StartThread(thread->tid(), os_id, ThreadType::Regular, + nullptr); else { // In a thread restart, a thread may resume execution at an // arbitrary function entry point, with its stack and TLS state Index: lib/asan/asan_thread.cc =================================================================== --- lib/asan/asan_thread.cc +++ lib/asan/asan_thread.cc @@ -244,8 +244,7 @@ thread_return_t AsanThread::ThreadStart( tid_t os_id, atomic_uintptr_t *signal_thread_is_registered) { Init(); - asanThreadRegistry().StartThread(tid(), os_id, /*workerthread*/ false, - nullptr); + asanThreadRegistry().StartThread(tid(), os_id, ThreadType::Regular, nullptr); if (signal_thread_is_registered) atomic_store(signal_thread_is_registered, 1, memory_order_release); Index: lib/lsan/lsan_thread.h =================================================================== --- lib/lsan/lsan_thread.h +++ lib/lsan/lsan_thread.h @@ -45,7 +45,8 @@ void InitializeThreadRegistry(); -void ThreadStart(u32 tid, tid_t os_id, bool workerthread = false); +void ThreadStart(u32 tid, tid_t os_id, + ThreadType thread_type = ThreadType::Regular); void ThreadFinish(); u32 ThreadCreate(u32 tid, uptr uid, bool detached); void ThreadJoin(u32 tid); Index: lib/lsan/lsan_thread.cc =================================================================== --- lib/lsan/lsan_thread.cc +++ lib/lsan/lsan_thread.cc @@ -77,7 +77,7 @@ /* arg */ nullptr); } -void ThreadStart(u32 tid, tid_t os_id, bool workerthread) { +void ThreadStart(u32 tid, tid_t os_id, ThreadType thread_type) { OnStartedArgs args; uptr stack_size = 0; uptr tls_size = 0; @@ -87,7 +87,7 @@ args.tls_end = args.tls_begin + tls_size; GetAllocatorCacheRange(&args.cache_begin, &args.cache_end); args.dtls = DTLS_Get(); - thread_registry->StartThread(tid, os_id, workerthread, &args); + thread_registry->StartThread(tid, os_id, thread_type, &args); } void ThreadFinish() { Index: lib/sanitizer_common/sanitizer_thread_registry.h =================================================================== --- lib/sanitizer_common/sanitizer_thread_registry.h +++ lib/sanitizer_common/sanitizer_thread_registry.h @@ -29,6 +29,12 @@ ThreadStatusDead // Joined, but some info is still available. }; +enum class ThreadType { + Regular, // Normal thread + Worker, // GCD worker thread + Fiber, // Fiber +}; + // Generic thread context. Specific sanitizer tools may inherit from it. // If thread is dead, context may optionally be reused for a new thread. class ThreadContextBase { @@ -45,7 +51,7 @@ ThreadStatus status; bool detached; - bool workerthread; + ThreadType thread_type; u32 parent_tid; ThreadContextBase *next; // For storing thread contexts in a list. @@ -57,7 +63,7 @@ void SetDead(); void SetJoined(void *arg); void SetFinished(); - void SetStarted(tid_t _os_id, bool _workerthread, void *arg); + void SetStarted(tid_t _os_id, ThreadType _thread_type, void *arg); void SetCreated(uptr _user_id, u64 _unique_id, bool _detached, u32 _parent_tid, void *arg); void Reset(); @@ -121,7 +127,7 @@ void DetachThread(u32 tid, void *arg); void JoinThread(u32 tid, void *arg); void FinishThread(u32 tid); - void StartThread(u32 tid, tid_t os_id, bool workerthread, void *arg); + void StartThread(u32 tid, tid_t os_id, ThreadType thread_type, void *arg); void SetThreadUserId(u32 tid, uptr user_id); private: Index: lib/sanitizer_common/sanitizer_thread_registry.cc =================================================================== --- lib/sanitizer_common/sanitizer_thread_registry.cc +++ lib/sanitizer_common/sanitizer_thread_registry.cc @@ -18,8 +18,8 @@ ThreadContextBase::ThreadContextBase(u32 tid) : tid(tid), unique_id(0), reuse_count(), os_id(0), user_id(0), - status(ThreadStatusInvalid), - detached(false), workerthread(false), parent_tid(0), next(0) { + status(ThreadStatusInvalid), detached(false), + thread_type(ThreadType::Regular), parent_tid(0), next(0) { name[0] = '\0'; atomic_store(&thread_destroyed, 0, memory_order_release); } @@ -71,11 +71,11 @@ OnFinished(); } -void ThreadContextBase::SetStarted(tid_t _os_id, bool _workerthread, +void ThreadContextBase::SetStarted(tid_t _os_id, ThreadType _thread_type, void *arg) { status = ThreadStatusRunning; os_id = _os_id; - workerthread = _workerthread; + thread_type = _thread_type; OnStarted(arg); } @@ -303,7 +303,7 @@ tctx->SetDestroyed(); } -void ThreadRegistry::StartThread(u32 tid, tid_t os_id, bool workerthread, +void ThreadRegistry::StartThread(u32 tid, tid_t os_id, ThreadType thread_type, void *arg) { BlockingMutexLock l(&mtx_); running_threads_++; @@ -311,7 +311,7 @@ ThreadContextBase *tctx = threads_[tid]; CHECK_NE(tctx, 0); CHECK_EQ(ThreadStatusCreated, tctx->status); - tctx->SetStarted(os_id, workerthread, arg); + tctx->SetStarted(os_id, thread_type, arg); } void ThreadRegistry::QuarantinePush(ThreadContextBase *tctx) { Index: lib/tsan/go/tsan_go.cc =================================================================== --- lib/tsan/go/tsan_go.cc +++ lib/tsan/go/tsan_go.cc @@ -214,7 +214,7 @@ ThreadState *thr = AllocGoroutine(); *pthr = thr; int goid = ThreadCreate(parent, (uptr)pc, 0, true); - ThreadStart(thr, goid, 0, /*workerthread*/ false); + ThreadStart(thr, goid, 0, ThreadType::Regular); } void __tsan_go_end(ThreadState *thr) { Index: lib/tsan/rtl/tsan_interceptors.cc =================================================================== --- lib/tsan/rtl/tsan_interceptors.cc +++ lib/tsan/rtl/tsan_interceptors.cc @@ -964,7 +964,7 @@ internal_sched_yield(); Processor *proc = ProcCreate(); ProcWire(proc, thr); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ false); + ThreadStart(thr, tid, GetTid(), ThreadType::Regular); atomic_store(&p->tid, 0, memory_order_release); } void *res = callback(param); @@ -1056,6 +1056,14 @@ return res; } +TSAN_INTERCEPTOR(void, pthread_exit, void *retval) { + SCOPED_TSAN_INTERCEPTOR(pthread_exit, retval); +#if !SANITIZER_MAC && !SANITIZER_ANDROID + CHECK_EQ(cur_thread(), &cur_thread_placeholder); +#endif + REAL(pthread_exit)(retval); +} + #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) { SCOPED_TSAN_INTERCEPTOR(pthread_tryjoin_np, th, ret); @@ -2682,6 +2690,7 @@ TSAN_INTERCEPT(pthread_create); TSAN_INTERCEPT(pthread_join); TSAN_INTERCEPT(pthread_detach); + TSAN_INTERCEPT(pthread_exit); #if SANITIZER_LINUX TSAN_INTERCEPT(pthread_tryjoin_np); TSAN_INTERCEPT(pthread_timedjoin_np); Index: lib/tsan/rtl/tsan_interface.cc =================================================================== --- lib/tsan/rtl/tsan_interface.cc +++ lib/tsan/rtl/tsan_interface.cc @@ -124,6 +124,33 @@ __tsan_unaligned_write8(addr); *addr = v; } + +#if !SANITIZER_MAC && !SANITIZER_ANDROID +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_get_current_fiber() { + return cur_thread(); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_create_fiber(unsigned flags) { + return FiberCreate(cur_thread(), CALLERPC, flags); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_destroy_fiber(void *fiber) { + FiberDestroy(cur_thread(), CALLERPC, static_cast(fiber)); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_switch_to_fiber(void *fiber, unsigned flags) { + FiberSwitch(cur_thread(), CALLERPC, static_cast(fiber), flags); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_set_fiber_name(void *fiber, const char *name) { + ThreadSetName(static_cast(fiber), name); +} +#endif // !SANITIZER_MAC && !SANITIZER_ANDROID } // extern "C" void __tsan_acquire(void *addr) { Index: lib/tsan/rtl/tsan_interface_atomic.cc =================================================================== --- lib/tsan/rtl/tsan_interface_atomic.cc +++ lib/tsan/rtl/tsan_interface_atomic.cc @@ -474,8 +474,8 @@ } #define SCOPED_ATOMIC(func, ...) \ - ThreadState *const thr = cur_thread(); \ - if (thr->ignore_sync || thr->ignore_interceptors) { \ + ThreadState *const thr = cur_thread_fast(); \ + if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) { \ ProcessPendingSignals(thr); \ return NoTsanAtomic##func(__VA_ARGS__); \ } \ Index: lib/tsan/rtl/tsan_interface_inl.h =================================================================== --- lib/tsan/rtl/tsan_interface_inl.h +++ lib/tsan/rtl/tsan_interface_inl.h @@ -19,73 +19,73 @@ using namespace __tsan; // NOLINT void __tsan_read1(void *addr) { - MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog1); + MemoryRead(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog1); } void __tsan_read2(void *addr) { - MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog2); + MemoryRead(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog2); } void __tsan_read4(void *addr) { - MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog4); + MemoryRead(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog4); } void __tsan_read8(void *addr) { - MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8); + MemoryRead(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog8); } void __tsan_write1(void *addr) { - MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog1); + MemoryWrite(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog1); } void __tsan_write2(void *addr) { - MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog2); + MemoryWrite(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog2); } void __tsan_write4(void *addr) { - MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog4); + MemoryWrite(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog4); } void __tsan_write8(void *addr) { - MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8); + MemoryWrite(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog8); } void __tsan_read1_pc(void *addr, void *pc) { - MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog1); + MemoryRead(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog1); } void __tsan_read2_pc(void *addr, void *pc) { - MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog2); + MemoryRead(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog2); } void __tsan_read4_pc(void *addr, void *pc) { - MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog4); + MemoryRead(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog4); } void __tsan_read8_pc(void *addr, void *pc) { - MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog8); + MemoryRead(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog8); } void __tsan_write1_pc(void *addr, void *pc) { - MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog1); + MemoryWrite(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog1); } void __tsan_write2_pc(void *addr, void *pc) { - MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog2); + MemoryWrite(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog2); } void __tsan_write4_pc(void *addr, void *pc) { - MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog4); + MemoryWrite(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog4); } void __tsan_write8_pc(void *addr, void *pc) { - MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog8); + MemoryWrite(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog8); } void __tsan_vptr_update(void **vptr_p, void *new_val) { CHECK_EQ(sizeof(vptr_p), 8); if (*vptr_p != new_val) { - ThreadState *thr = cur_thread(); + ThreadState *thr = cur_thread_fast(); thr->is_vptr_access = true; MemoryWrite(thr, CALLERPC, (uptr)vptr_p, kSizeLog8); thr->is_vptr_access = false; @@ -94,7 +94,7 @@ void __tsan_vptr_read(void **vptr_p) { CHECK_EQ(sizeof(vptr_p), 8); - ThreadState *thr = cur_thread(); + ThreadState *thr = cur_thread_fast(); thr->is_vptr_access = true; MemoryRead(thr, CALLERPC, (uptr)vptr_p, kSizeLog8); thr->is_vptr_access = false; @@ -105,21 +105,21 @@ } void __tsan_func_exit() { - FuncExit(cur_thread()); + FuncExit(cur_thread_fast()); } void __tsan_ignore_thread_begin() { - ThreadIgnoreBegin(cur_thread(), CALLERPC); + ThreadIgnoreBegin(cur_thread_fast(), CALLERPC); } void __tsan_ignore_thread_end() { - ThreadIgnoreEnd(cur_thread(), CALLERPC); + ThreadIgnoreEnd(cur_thread_fast(), CALLERPC); } void __tsan_read_range(void *addr, uptr size) { - MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, false); + MemoryAccessRange(cur_thread_fast(), CALLERPC, (uptr)addr, size, false); } void __tsan_write_range(void *addr, uptr size) { - MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, true); + MemoryAccessRange(cur_thread_fast(), CALLERPC, (uptr)addr, size, true); } Index: lib/tsan/rtl/tsan_platform_mac.cc =================================================================== --- lib/tsan/rtl/tsan_platform_mac.cc +++ lib/tsan/rtl/tsan_platform_mac.cc @@ -213,7 +213,7 @@ ThreadState *parent_thread_state = nullptr; // No parent. int tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true); CHECK_NE(tid, 0); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ true); + ThreadStart(thr, tid, GetTid(), ThreadType::Worker); } } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) { if (thread == pthread_self()) { Index: lib/tsan/rtl/tsan_report.h =================================================================== --- lib/tsan/rtl/tsan_report.h +++ lib/tsan/rtl/tsan_report.h @@ -14,6 +14,7 @@ #define TSAN_REPORT_H #include "sanitizer_common/sanitizer_symbolizer.h" +#include "sanitizer_common/sanitizer_thread_registry.h" #include "sanitizer_common/sanitizer_vector.h" #include "tsan_defs.h" @@ -92,7 +93,7 @@ int id; tid_t os_id; bool running; - bool workerthread; + ThreadType thread_type; char *name; u32 parent_tid; ReportStack *stack; Index: lib/tsan/rtl/tsan_report.cc =================================================================== --- lib/tsan/rtl/tsan_report.cc +++ lib/tsan/rtl/tsan_report.cc @@ -255,7 +255,7 @@ Printf(" '%s'", rt->name); char thrbuf[kThreadBufSize]; const char *thread_status = rt->running ? "running" : "finished"; - if (rt->workerthread) { + if (rt->thread_type == ThreadType::Worker) { Printf(" (tid=%zu, %s) is a GCD worker thread\n", rt->os_id, thread_status); Printf("\n"); Printf("%s", d.Default()); Index: lib/tsan/rtl/tsan_rtl.h =================================================================== --- lib/tsan/rtl/tsan_rtl.h +++ lib/tsan/rtl/tsan_rtl.h @@ -463,11 +463,24 @@ #if SANITIZER_MAC || SANITIZER_ANDROID ThreadState *cur_thread(); void cur_thread_finalize(); +INLINE ThreadState *cur_thread_fast() { + return cur_thread(); +} #else __attribute__((tls_model("initial-exec"))) +extern THREADLOCAL ThreadState *cur_thread1; +__attribute__((tls_model("initial-exec"))) extern THREADLOCAL char cur_thread_placeholder[]; +INLINE ThreadState *cur_thread_fast() { + return cur_thread1; +} INLINE ThreadState *cur_thread() { - return reinterpret_cast(&cur_thread_placeholder); + if (UNLIKELY(!cur_thread1)) + cur_thread1 = reinterpret_cast(cur_thread_placeholder); + return cur_thread1; +} +INLINE void set_cur_thread(ThreadState *thr) { + cur_thread1 = thr; } INLINE void cur_thread_finalize() { } #endif // SANITIZER_MAC || SANITIZER_ANDROID @@ -765,7 +778,8 @@ void FuncExit(ThreadState *thr); int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached); -void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread); +void ThreadStart(ThreadState *thr, int tid, tid_t os_id, + ThreadType thread_type); void ThreadFinish(ThreadState *thr); int ThreadTid(ThreadState *thr, uptr pc, uptr uid); void ThreadJoin(ThreadState *thr, uptr pc, int tid); @@ -868,6 +882,16 @@ } #endif +ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags); +void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber); +void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags); + +// These need to match __tsan_switch_to_fiber_* flags defined in +// tsan_interface.h. See documentation there as well. +enum FiberSwitchFlags { + FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync +}; + } // namespace __tsan #endif // TSAN_RTL_H Index: lib/tsan/rtl/tsan_rtl.cc =================================================================== --- lib/tsan/rtl/tsan_rtl.cc +++ lib/tsan/rtl/tsan_rtl.cc @@ -47,6 +47,8 @@ #if !SANITIZER_GO && !SANITIZER_MAC __attribute__((tls_model("initial-exec"))) +THREADLOCAL ThreadState *cur_thread1; +__attribute__((tls_model("initial-exec"))) THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64); #endif static char ctx_placeholder[sizeof(Context)] ALIGNED(64); @@ -397,7 +399,7 @@ // Initialize thread 0. int tid = ThreadCreate(thr, 0, 0, true); CHECK_EQ(tid, 0); - ThreadStart(thr, tid, GetTid(), /*workerthread*/ false); + ThreadStart(thr, tid, GetTid(), ThreadType::Regular); #if TSAN_CONTAINS_UBSAN __ubsan::InitAsPlugin(); #endif @@ -815,7 +817,7 @@ } #endif - if (!SANITIZER_GO && *shadow_mem == kShadowRodata) { + if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) { // Access to .rodata section, no races here. // Measurements show that it can be 10-20% of all memory accesses. StatInc(thr, StatMop); @@ -826,7 +828,7 @@ } FastState fast_state = thr->fast_state; - if (fast_state.GetIgnoreBit()) { + if (UNLIKELY(fast_state.GetIgnoreBit())) { StatInc(thr, StatMop); StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); Index: lib/tsan/rtl/tsan_rtl_report.cc =================================================================== --- lib/tsan/rtl/tsan_rtl_report.cc +++ lib/tsan/rtl/tsan_rtl_report.cc @@ -202,7 +202,7 @@ rt->running = (tctx->status == ThreadStatusRunning); rt->name = internal_strdup(tctx->name); rt->parent_tid = tctx->parent_tid; - rt->workerthread = tctx->workerthread; + rt->thread_type = tctx->thread_type; rt->stack = 0; rt->stack = SymbolizeStackId(tctx->creation_stack_id); if (rt->stack) Index: lib/tsan/rtl/tsan_rtl_thread.cc =================================================================== --- lib/tsan/rtl/tsan_rtl_thread.cc +++ lib/tsan/rtl/tsan_rtl_thread.cc @@ -240,13 +240,15 @@ return tid; } -void ThreadStart(ThreadState *thr, int tid, tid_t os_id, bool workerthread) { +void ThreadStart(ThreadState *thr, int tid, tid_t os_id, + ThreadType thread_type) { uptr stk_addr = 0; uptr stk_size = 0; uptr tls_addr = 0; uptr tls_size = 0; #if !SANITIZER_GO - GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); + if (thread_type != ThreadType::Fiber) + GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); if (tid) { if (stk_addr && stk_size) @@ -258,7 +260,7 @@ ThreadRegistry *tr = ctx->thread_registry; OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size }; - tr->StartThread(tid, os_id, workerthread, &args); + tr->StartThread(tid, os_id, thread_type, &args); tr->Lock(); thr->tctx = (ThreadContext*)tr->GetThreadLocked(tid); @@ -404,4 +406,40 @@ } } +#if !SANITIZER_MAC && !SANITIZER_ANDROID && !SANITIZER_GO +void FiberSwitchImpl(ThreadState *from, ThreadState *to) { + Processor *proc = from->proc(); + ProcUnwire(proc, from); + ProcWire(proc, to); + set_cur_thread(to); +} + +ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags) { + void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadState)); + ThreadState *fiber = static_cast(mem); + internal_memset(fiber, 0, sizeof(*fiber)); + int tid = ThreadCreate(thr, pc, 0, true); + FiberSwitchImpl(thr, fiber); + ThreadStart(fiber, tid, 0, ThreadType::Fiber); + FiberSwitchImpl(fiber, thr); + return fiber; +} + +void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber) { + FiberSwitchImpl(thr, fiber); + ThreadFinish(fiber); + FiberSwitchImpl(fiber, thr); + internal_free(fiber); +} + +void FiberSwitch(ThreadState *thr, uptr pc, + ThreadState *fiber, unsigned flags) { + if (!(flags & FiberSwitchFlagNoSync)) + Release(thr, pc, (uptr)fiber); + FiberSwitchImpl(thr, fiber); + if (!(flags & FiberSwitchFlagNoSync)) + Acquire(fiber, pc, (uptr)fiber); +} +#endif + } // namespace __tsan Index: lib/tsan/rtl/tsan_update_shadow_word_inl.h =================================================================== --- lib/tsan/rtl/tsan_update_shadow_word_inl.h +++ lib/tsan/rtl/tsan_update_shadow_word_inl.h @@ -18,7 +18,7 @@ const unsigned kAccessSize = 1 << kAccessSizeLog; u64 *sp = &shadow_mem[idx]; old = LoadShadow(sp); - if (old.IsZero()) { + if (LIKELY(old.IsZero())) { StatInc(thr, StatShadowZero); if (store_word) StoreIfNotYetStored(sp, &store_word); @@ -27,12 +27,12 @@ break; } // is the memory access equal to the previous? - if (Shadow::Addr0AndSizeAreEqual(cur, old)) { + if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) { StatInc(thr, StatShadowSameSize); // same thread? - if (Shadow::TidsAreEqual(old, cur)) { + if (LIKELY(Shadow::TidsAreEqual(old, cur))) { StatInc(thr, StatShadowSameThread); - if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) + if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) StoreIfNotYetStored(sp, &store_word); break; } @@ -42,7 +42,7 @@ StoreIfNotYetStored(sp, &store_word); break; } - if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) + if (LIKELY(old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))) break; goto RACE; } @@ -56,7 +56,7 @@ StatInc(thr, StatShadowAnotherThread); if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) break; - if (HappensBefore(old, thr)) + if (LIKELY(HappensBefore(old, thr))) break; goto RACE; } Index: test/tsan/fiber_asm.cc =================================================================== --- /dev/null +++ test/tsan/fiber_asm.cc @@ -0,0 +1,85 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" + +struct ucontext { + void *sp; + void *fiber; +}; + +extern "C" { + void ucontext_do_switch(void **save, void **load); + void ucontext_trampoline(); +} + +__asm__(".global ucontext_do_switch\n" + "ucontext_do_switch:\n\t" + "pushq %rbp\n\t" + "pushq %r15\n\t" + "pushq %r14\n\t" + "pushq %r13\n\t" + "pushq %r12\n\t" + "pushq %rbx\n\t" + "movq %rsp, (%rdi)\n\t" + "movq (%rsi), %rsp\n\t" + "popq %rbx\n\t" + "popq %r12\n\t" + "popq %r13\n\t" + "popq %r14\n\t" + "popq %r15\n\t" + "popq %rbp\n\t" + "retq"); + +__asm__(".global ucontext_trampoline\n" + "ucontext_trampoline:\n\t" + ".cfi_startproc\n\t" + ".cfi_undefined rip\n\t" + "movq %r12, %rdi\n\t" + "jmpq *%rbx\n\t" + ".cfi_endproc"); + +void ucontext_init(ucontext *context, void *stack, unsigned stack_sz, + void (*func)(void*), void *arg) { + void **sp = reinterpret_cast(static_cast(stack) + stack_sz); + *(--sp) = 0; + *(--sp) = reinterpret_cast(ucontext_trampoline); + *(--sp) = 0; // rbp + *(--sp) = 0; // r15 + *(--sp) = 0; // r14 + *(--sp) = 0; // r13 + *(--sp) = arg; // r12 + *(--sp) = reinterpret_cast(func); // rbx + context->sp = sp; + context->fiber = __tsan_create_fiber(0); +} + +void ucontext_free(ucontext *context) { + __tsan_destroy_fiber(context->fiber); +} + +__attribute__((no_sanitize_thread)) +void ucontext_switch(ucontext *save, ucontext *load) { + save->fiber = __tsan_get_current_fiber(); + __tsan_switch_to_fiber(load->fiber, 0); + ucontext_do_switch(&save->sp, &load->sp); +} + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext uc, orig_uc; + +void func(void *arg) { + __asm__ __volatile__(".cfi_undefined rip"); + ucontext_switch(&uc, &orig_uc); +} + +int main() { + ucontext_init(&uc, stack, sizeof(stack), func, 0); + ucontext_switch(&orig_uc, &uc); + ucontext_free(&uc); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS Index: test/tsan/fiber_from_thread.cc =================================================================== --- /dev/null +++ test/tsan/fiber_from_thread.cc @@ -0,0 +1,48 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext_t uc, orig_uc1, orig_uc2; +void *fiber, *orig_fiber1, *orig_fiber2; + +int var; + +void *Thread(void *x) { + orig_fiber2 = __tsan_get_current_fiber(); + swapcontext(&orig_uc2, &orig_uc1); + return 0; +} + +void func() { + pthread_t t; + pthread_create(&t, 0, Thread, 0); + pthread_join(t, 0); + __tsan_switch_to_fiber(orig_fiber1, 0); + swapcontext(&uc, &orig_uc1); +} + +int main() { + orig_fiber1 = __tsan_get_current_fiber(); + fiber = __tsan_create_fiber(0); + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = 0; + makecontext(&uc, func, 0); + var = 1; + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc1, &uc); + var = 2; + __tsan_switch_to_fiber(orig_fiber2, 0); + swapcontext(&orig_uc1, &orig_uc2); + var = 3; + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS Index: test/tsan/fiber_longjmp.c =================================================================== --- /dev/null +++ test/tsan/fiber_longjmp.c @@ -0,0 +1,80 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +sigjmp_buf jmpbuf, orig_jmpbuf[2]; +void *fiber, *orig_fiber[2]; + +const unsigned N = 1000; + +__attribute__((noinline)) +void switch0() { + if (!sigsetjmp(jmpbuf, 0)) { + __tsan_switch_to_fiber(orig_fiber[0], 0); + siglongjmp(orig_jmpbuf[0], 1); + } +} + +void func() { + if (!sigsetjmp(jmpbuf, 0)) { + __tsan_switch_to_fiber(orig_fiber[0], 0); + siglongjmp(orig_jmpbuf[0], 1); + } + for (;;) { + switch0(); + if (!sigsetjmp(jmpbuf, 0)) { + __tsan_switch_to_fiber(orig_fiber[1], 0); + siglongjmp(orig_jmpbuf[1], 1); + } + } +} + +void *Thread(void *x) { + orig_fiber[1] = __tsan_get_current_fiber(); + for (unsigned i = 0; i < N; i++) { + barrier_wait(&barrier); + if (!sigsetjmp(orig_jmpbuf[1], 0)) { + __tsan_switch_to_fiber(fiber, 0); + siglongjmp(jmpbuf, 1); + } + barrier_wait(&barrier); + } + return 0; +} + +int main() { + fiber = __tsan_create_fiber(0); + barrier_init(&barrier, 2); + pthread_t t; + pthread_create(&t, 0, Thread, 0); + orig_fiber[0] = __tsan_get_current_fiber(); + ucontext_t uc, orig_uc; + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = 0; + makecontext(&uc, func, 0); + if (!sigsetjmp(orig_jmpbuf[0], 0)) { + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc, &uc); + } + for (unsigned i = 0; i < N; i++) { + if (!sigsetjmp(orig_jmpbuf[0], 0)) { + __tsan_switch_to_fiber(fiber, 0); + siglongjmp(jmpbuf, 1); + } + barrier_wait(&barrier); + barrier_wait(&barrier); + } + pthread_join(t, 0); + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS Index: test/tsan/fiber_race.cc =================================================================== --- /dev/null +++ test/tsan/fiber_race.cc @@ -0,0 +1,36 @@ +// RUN: %clang_tsan -O1 %s -o %t && %deflake %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext_t uc, orig_uc; +void *fiber, *orig_fiber; + +int var; + +void func() { + var = 1; + __tsan_switch_to_fiber(orig_fiber, __tsan_switch_to_fiber_no_sync); + swapcontext(&uc, &orig_uc); +} + +int main() { + orig_fiber = __tsan_get_current_fiber(); + fiber = __tsan_create_fiber(0); + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = 0; + makecontext(&uc, func, 0); + var = 2; + __tsan_switch_to_fiber(fiber, __tsan_switch_to_fiber_no_sync); + swapcontext(&orig_uc, &uc); + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK: WARNING: ThreadSanitizer: data race +// CHECK: PASS Index: test/tsan/fiber_simple.cc =================================================================== --- /dev/null +++ test/tsan/fiber_simple.cc @@ -0,0 +1,36 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext_t uc, orig_uc; +void *fiber, *orig_fiber; + +int var; + +void func() { + var = 1; + __tsan_switch_to_fiber(orig_fiber, 0); + swapcontext(&uc, &orig_uc); +} + +int main() { + orig_fiber = __tsan_get_current_fiber(); + fiber = __tsan_create_fiber(0); + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = 0; + makecontext(&uc, func, 0); + var = 2; + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc, &uc); + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS Index: test/tsan/fiber_two_threads.cc =================================================================== --- /dev/null +++ test/tsan/fiber_two_threads.cc @@ -0,0 +1,62 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext_t uc, orig_uc[2]; +void *fiber, *orig_fiber[2]; + +const unsigned N = 1000; + +__attribute__((noinline)) +void switch0() { + __tsan_switch_to_fiber(orig_fiber[0], 0); + swapcontext(&uc, &orig_uc[0]); +} + +void func() { + for (;;) { + switch0(); + __tsan_switch_to_fiber(orig_fiber[1], 0); + swapcontext(&uc, &orig_uc[1]); + } +} + +void *Thread(void *x) { + orig_fiber[1] = __tsan_get_current_fiber(); + for (unsigned i = 0; i < N; i++) { + barrier_wait(&barrier); + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc[1], &uc); + barrier_wait(&barrier); + } + return 0; +} + +int main() { + fiber = __tsan_create_fiber(0); + barrier_init(&barrier, 2); + pthread_t t; + pthread_create(&t, 0, Thread, 0); + orig_fiber[0] = __tsan_get_current_fiber(); + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = 0; + makecontext(&uc, func, 0); + for (unsigned i = 0; i < N; i++) { + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc[0], &uc); + barrier_wait(&barrier); + barrier_wait(&barrier); + } + pthread_join(t, 0); + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS Index: test/tsan/thread_exit.c =================================================================== --- /dev/null +++ test/tsan/thread_exit.c @@ -0,0 +1,25 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +#include "test.h" + +int var; + +void *Thread(void *x) { + pthread_exit(&var); + return 0; +} + +int main() { + pthread_t t; + pthread_create(&t, 0, Thread, 0); + void *retval = 0; + pthread_join(t, &retval); + if (retval != &var) { + fprintf(stderr, "Unexpected return value\n"); + exit(1); + } + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS