Index: include/sanitizer/tsan_interface.h =================================================================== --- include/sanitizer/tsan_interface.h +++ include/sanitizer/tsan_interface.h @@ -137,6 +137,24 @@ void __tsan_external_read(void *addr, void *caller_pc, void *tag); void __tsan_external_write(void *addr, void *caller_pc, void *tag); +// Fiber switching API. +// - TSAN context for fiber can be created by __tsan_create_fiber +// and freed by __tsan_destroy_fiber. +// - TSAN context of current fiber or thread can be obtained +// by calling __tsan_get_current_fiber. +// - __tsan_switch_to_fiber should be called immediatly before switch +// to fiber, such as call of swapcontext. +// - Fiber name can be set by __tsan_set_fiber_name. +void *__tsan_get_current_fiber(void); +void *__tsan_create_fiber(unsigned flags); +void __tsan_destroy_fiber(void *fiber); +void __tsan_switch_to_fiber(void *fiber, unsigned flags); +void __tsan_set_fiber_name(void *fiber, const char *name); + +// Flags for __tsan_switch_to_fiber: +// Do not establish a happens-before relation between fibers +const unsigned __tsan_switch_to_fiber_no_sync = 1 << 0; + #ifdef __cplusplus } // extern "C" #endif Index: lib/tsan/rtl/tsan_interface.cc =================================================================== --- lib/tsan/rtl/tsan_interface.cc +++ lib/tsan/rtl/tsan_interface.cc @@ -124,6 +124,33 @@ __tsan_unaligned_write8(addr); *addr = v; } + +#if !SANITIZER_MAC && !SANITIZER_ANDROID +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_get_current_fiber() { + return cur_thread(); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_create_fiber(unsigned flags) { + return FiberCreate(flags, CALLERPC); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_destroy_fiber(void *fiber) { + FiberDestroy(static_cast(fiber), CALLERPC); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_switch_to_fiber(void *fiber, unsigned flags) { + FiberSwitch(static_cast(fiber), flags, CALLERPC); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_set_fiber_name(void *fiber, const char *name) { + ThreadSetName(static_cast(fiber), name); +} +#endif // !SANITIZER_MAC && !SANITIZER_ANDROID } // extern "C" void __tsan_acquire(void *addr) { Index: lib/tsan/rtl/tsan_interface_atomic.cc =================================================================== --- lib/tsan/rtl/tsan_interface_atomic.cc +++ lib/tsan/rtl/tsan_interface_atomic.cc @@ -474,8 +474,8 @@ } #define SCOPED_ATOMIC(func, ...) \ - ThreadState *const thr = cur_thread(); \ - if (thr->ignore_sync || thr->ignore_interceptors) { \ + ThreadState *const thr = cur_thread_fast(); \ + if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) { \ ProcessPendingSignals(thr); \ return NoTsanAtomic##func(__VA_ARGS__); \ } \ Index: lib/tsan/rtl/tsan_interface_inl.h =================================================================== --- lib/tsan/rtl/tsan_interface_inl.h +++ lib/tsan/rtl/tsan_interface_inl.h @@ -19,73 +19,73 @@ using namespace __tsan; // NOLINT void __tsan_read1(void *addr) { - MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog1); + MemoryRead(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog1); } void __tsan_read2(void *addr) { - MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog2); + MemoryRead(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog2); } void __tsan_read4(void *addr) { - MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog4); + MemoryRead(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog4); } void __tsan_read8(void *addr) { - MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8); + MemoryRead(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog8); } void __tsan_write1(void *addr) { - MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog1); + MemoryWrite(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog1); } void __tsan_write2(void *addr) { - MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog2); + MemoryWrite(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog2); } void __tsan_write4(void *addr) { - MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog4); + MemoryWrite(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog4); } void __tsan_write8(void *addr) { - MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8); + MemoryWrite(cur_thread_fast(), CALLERPC, (uptr)addr, kSizeLog8); } void __tsan_read1_pc(void *addr, void *pc) { - MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog1); + MemoryRead(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog1); } void __tsan_read2_pc(void *addr, void *pc) { - MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog2); + MemoryRead(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog2); } void __tsan_read4_pc(void *addr, void *pc) { - MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog4); + MemoryRead(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog4); } void __tsan_read8_pc(void *addr, void *pc) { - MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog8); + MemoryRead(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog8); } void __tsan_write1_pc(void *addr, void *pc) { - MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog1); + MemoryWrite(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog1); } void __tsan_write2_pc(void *addr, void *pc) { - MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog2); + MemoryWrite(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog2); } void __tsan_write4_pc(void *addr, void *pc) { - MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog4); + MemoryWrite(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog4); } void __tsan_write8_pc(void *addr, void *pc) { - MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog8); + MemoryWrite(cur_thread_fast(), (uptr)pc, (uptr)addr, kSizeLog8); } void __tsan_vptr_update(void **vptr_p, void *new_val) { CHECK_EQ(sizeof(vptr_p), 8); if (*vptr_p != new_val) { - ThreadState *thr = cur_thread(); + ThreadState *thr = cur_thread_fast(); thr->is_vptr_access = true; MemoryWrite(thr, CALLERPC, (uptr)vptr_p, kSizeLog8); thr->is_vptr_access = false; @@ -94,7 +94,7 @@ void __tsan_vptr_read(void **vptr_p) { CHECK_EQ(sizeof(vptr_p), 8); - ThreadState *thr = cur_thread(); + ThreadState *thr = cur_thread_fast(); thr->is_vptr_access = true; MemoryRead(thr, CALLERPC, (uptr)vptr_p, kSizeLog8); thr->is_vptr_access = false; @@ -105,21 +105,21 @@ } void __tsan_func_exit() { - FuncExit(cur_thread()); + FuncExit(cur_thread_fast()); } void __tsan_ignore_thread_begin() { - ThreadIgnoreBegin(cur_thread(), CALLERPC); + ThreadIgnoreBegin(cur_thread_fast(), CALLERPC); } void __tsan_ignore_thread_end() { - ThreadIgnoreEnd(cur_thread(), CALLERPC); + ThreadIgnoreEnd(cur_thread_fast(), CALLERPC); } void __tsan_read_range(void *addr, uptr size) { - MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, false); + MemoryAccessRange(cur_thread_fast(), CALLERPC, (uptr)addr, size, false); } void __tsan_write_range(void *addr, uptr size) { - MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, true); + MemoryAccessRange(cur_thread_fast(), CALLERPC, (uptr)addr, size, true); } Index: lib/tsan/rtl/tsan_rtl.h =================================================================== --- lib/tsan/rtl/tsan_rtl.h +++ lib/tsan/rtl/tsan_rtl.h @@ -462,11 +462,24 @@ #if SANITIZER_MAC || SANITIZER_ANDROID ThreadState *cur_thread(); void cur_thread_finalize(); +INLINE ThreadState *cur_thread_fast() { + return cur_thread(); +} #else __attribute__((tls_model("initial-exec"))) +extern THREADLOCAL ThreadState *cur_thread1; +__attribute__((tls_model("initial-exec"))) extern THREADLOCAL char cur_thread_placeholder[]; +INLINE ThreadState *cur_thread_fast() { + return cur_thread1; +} INLINE ThreadState *cur_thread() { - return reinterpret_cast(&cur_thread_placeholder); + if (UNLIKELY(!cur_thread1)) + cur_thread1 = reinterpret_cast(cur_thread_placeholder); + return cur_thread1; +} +INLINE void set_cur_thread(ThreadState *thr) { + cur_thread1 = thr; } INLINE void cur_thread_finalize() { } #endif // SANITIZER_MAC || SANITIZER_ANDROID @@ -867,6 +880,16 @@ } #endif +ThreadState *FiberCreate(unsigned flags, uptr pc); +void FiberDestroy(ThreadState *fiber, uptr pc); +void FiberSwitch(ThreadState *fiber, unsigned flags, uptr pc); + +// These need to match __tsan_swtich_to_fiber_* flags defined in +// tsan_interface.h. See documentation there as well. +enum SwtchFiberFlags { + FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync +}; + } // namespace __tsan #endif // TSAN_RTL_H Index: lib/tsan/rtl/tsan_rtl.cc =================================================================== --- lib/tsan/rtl/tsan_rtl.cc +++ lib/tsan/rtl/tsan_rtl.cc @@ -47,6 +47,8 @@ #if !SANITIZER_GO && !SANITIZER_MAC __attribute__((tls_model("initial-exec"))) +THREADLOCAL ThreadState *cur_thread1; +__attribute__((tls_model("initial-exec"))) THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64); #endif static char ctx_placeholder[sizeof(Context)] ALIGNED(64); @@ -826,7 +828,7 @@ } FastState fast_state = thr->fast_state; - if (fast_state.GetIgnoreBit()) { + if (UNLIKELY(fast_state.GetIgnoreBit())) { StatInc(thr, StatMop); StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); Index: lib/tsan/rtl/tsan_rtl_thread.cc =================================================================== --- lib/tsan/rtl/tsan_rtl_thread.cc +++ lib/tsan/rtl/tsan_rtl_thread.cc @@ -246,7 +246,8 @@ uptr tls_addr = 0; uptr tls_size = 0; #if !SANITIZER_GO - GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); + if (os_id) + GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); if (tid) { if (stk_addr && stk_size) @@ -404,4 +405,42 @@ } } +#if !SANITIZER_MAC && !SANITIZER_ANDROID && !SANITIZER_GO +void FiberSwitchImpl(ThreadState *from, ThreadState *to) { + Processor *proc = from->proc(); + ProcUnwire(proc, from); + ProcWire(proc, to); + set_cur_thread(to); +} + +ThreadState *FiberCreate(unsigned flags, uptr pc) { + ThreadState *thr = cur_thread(); + void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadState)); + ThreadState *fiber = static_cast(mem); + internal_memset(fiber, 0, sizeof(*fiber)); + int tid = ThreadCreate(thr, pc, 0, true); + FiberSwitchImpl(thr, fiber); + ThreadStart(fiber, tid, 0, false); + FiberSwitchImpl(fiber, thr); + return fiber; +} + +void FiberDestroy(ThreadState *fiber, uptr pc) { + ThreadState *thr = cur_thread(); + FiberSwitchImpl(thr, fiber); + ThreadFinish(fiber); + FiberSwitchImpl(fiber, thr); + internal_free(fiber); +} + +void FiberSwitch(ThreadState *fiber, unsigned flags, uptr pc) { + ThreadState *thr = cur_thread(); + if (!(flags & FiberSwitchFlagNoSync)) + Release(thr, pc, (uptr)fiber); + FiberSwitchImpl(thr, fiber); + if (!(flags & FiberSwitchFlagNoSync)) + Acquire(fiber, pc, (uptr)fiber); +} +#endif + } // namespace __tsan Index: lib/tsan/rtl/tsan_update_shadow_word_inl.h =================================================================== --- lib/tsan/rtl/tsan_update_shadow_word_inl.h +++ lib/tsan/rtl/tsan_update_shadow_word_inl.h @@ -42,7 +42,7 @@ StoreIfNotYetStored(sp, &store_word); break; } - if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) + if (LIKELY(old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))) break; goto RACE; } @@ -56,7 +56,7 @@ StatInc(thr, StatShadowAnotherThread); if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) break; - if (HappensBefore(old, thr)) + if (LIKELY(HappensBefore(old, thr))) break; goto RACE; } Index: test/tsan/fiber_asm.cc =================================================================== --- /dev/null +++ test/tsan/fiber_asm.cc @@ -0,0 +1,85 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" + +struct ucontext { + void *sp; + void *fiber; +}; + +extern "C" { + void ucontext_do_switch(void **save, void **load); + void ucontext_trampoline(); +} + +__asm__(".global ucontext_do_switch\n" + "ucontext_do_switch:\n\t" + "pushq %rbp\n\t" + "pushq %r15\n\t" + "pushq %r14\n\t" + "pushq %r13\n\t" + "pushq %r12\n\t" + "pushq %rbx\n\t" + "movq %rsp, (%rdi)\n\t" + "movq (%rsi), %rsp\n\t" + "popq %rbx\n\t" + "popq %r12\n\t" + "popq %r13\n\t" + "popq %r14\n\t" + "popq %r15\n\t" + "popq %rbp\n\t" + "retq"); + +__asm__(".global ucontext_trampoline\n" + "ucontext_trampoline:\n\t" + ".cfi_startproc\n\t" + ".cfi_undefined rip\n\t" + "movq %r12, %rdi\n\t" + "jmpq *%rbx\n\t" + ".cfi_endproc"); + +void ucontext_init(ucontext *context, void *stack, unsigned stack_sz, + void (*func)(void*), void *arg) { + void **sp = reinterpret_cast(static_cast(stack) + stack_sz); + *(--sp) = 0; + *(--sp) = reinterpret_cast(ucontext_trampoline); + *(--sp) = 0; // rbp + *(--sp) = 0; // r15 + *(--sp) = 0; // r14 + *(--sp) = 0; // r13 + *(--sp) = arg; // r12 + *(--sp) = reinterpret_cast(func); // rbx + context->sp = sp; + context->fiber = __tsan_create_fiber(0); +} + +void ucontext_free(ucontext *context) { + __tsan_destroy_fiber(context->fiber); +} + +__attribute__((no_sanitize_thread)) +void ucontext_switch(ucontext *save, ucontext *load) { + save->fiber = __tsan_get_current_fiber(); + __tsan_switch_to_fiber(load->fiber, 0); + ucontext_do_switch(&save->sp, &load->sp); +} + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext uc, orig_uc; + +void func(void *arg) { + __asm__ __volatile__(".cfi_undefined rip"); + ucontext_switch(&uc, &orig_uc); +} + +int main() { + ucontext_init(&uc, stack, sizeof(stack), func, 0); + ucontext_switch(&orig_uc, &uc); + ucontext_free(&uc); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS Index: test/tsan/fiber_race.cc =================================================================== --- /dev/null +++ test/tsan/fiber_race.cc @@ -0,0 +1,36 @@ +// RUN: %clang_tsan -O1 %s -o %t && %deflake %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext_t uc, orig_uc; +void *fiber, *orig_fiber; + +int var; + +void func() { + var = 1; + __tsan_switch_to_fiber(orig_fiber, __tsan_switch_to_fiber_no_sync); + swapcontext(&uc, &orig_uc); +} + +int main() { + orig_fiber = __tsan_get_current_fiber(); + fiber = __tsan_create_fiber(0); + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = &orig_uc; + makecontext(&uc, func, 0); + var = 2; + __tsan_switch_to_fiber(fiber, __tsan_switch_to_fiber_no_sync); + swapcontext(&orig_uc, &uc); + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK: WARNING: ThreadSanitizer: data race +// CHECK: PASS Index: test/tsan/fiber_simple.cc =================================================================== --- /dev/null +++ test/tsan/fiber_simple.cc @@ -0,0 +1,36 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext_t uc, orig_uc; +void *fiber, *orig_fiber; + +int var; + +void func() { + var = 1; + __tsan_switch_to_fiber(orig_fiber, 0); + swapcontext(&uc, &orig_uc); +} + +int main() { + orig_fiber = __tsan_get_current_fiber(); + fiber = __tsan_create_fiber(0); + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = &orig_uc; + makecontext(&uc, func, 0); + var = 2; + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc, &uc); + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS Index: test/tsan/fiber_two_threads.cc =================================================================== --- /dev/null +++ test/tsan/fiber_two_threads.cc @@ -0,0 +1,62 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext_t uc, orig_uc[2]; +void *fiber, *orig_fiber[2]; + +const unsigned N = 1000; + +__attribute__((noinline)) +void switch0() { + __tsan_switch_to_fiber(orig_fiber[0], 0); + swapcontext(&uc, &orig_uc[0]); +} + +void func() { + for (;;) { + switch0(); + __tsan_switch_to_fiber(orig_fiber[1], 0); + swapcontext(&uc, &orig_uc[1]); + } +} + +void *Thread(void *x) { + orig_fiber[1] = __tsan_get_current_fiber(); + for (unsigned i = 0; i < N; i++) { + barrier_wait(&barrier); + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc[1], &uc); + barrier_wait(&barrier); + } + return 0; +} + +int main() { + fiber = __tsan_create_fiber(0); + barrier_init(&barrier, 2); + pthread_t t; + pthread_create(&t, 0, Thread, 0); + orig_fiber[0] = __tsan_get_current_fiber(); + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = &orig_uc[0]; + makecontext(&uc, func, 0); + for (unsigned i = 0; i < N; i++) { + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc[0], &uc); + barrier_wait(&barrier); + barrier_wait(&barrier); + } + pthread_join(t, 0); + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS