Index: include/sanitizer/tsan_interface.h =================================================================== --- include/sanitizer/tsan_interface.h +++ include/sanitizer/tsan_interface.h @@ -136,6 +136,24 @@ void __tsan_external_read(void *addr, void *caller_pc, void *tag); void __tsan_external_write(void *addr, void *caller_pc, void *tag); +// Fiber switching API. +// - TSAN context for fiber can be created by __tsan_create_fiber +// and freed by __tsan_destroy_fiber. +// - TSAN context of current fiber or thread can be obtained +// by calling __tsan_get_current_fiber. +// - __tsan_switch_to_fiber should be called immediatly before switch +// to fiber, such as call of swapcontext. +// - Fiber name can be set by __tsan_set_fiber_name. +void *__tsan_get_current_fiber(void); +void *__tsan_create_fiber(unsigned flags); +void __tsan_destroy_fiber(void *fiber); +void __tsan_switch_to_fiber(void *fiber, unsigned flags); +void __tsan_set_fiber_name(void *fiber, const char *name); + +// Flags for __tsan_switch_to_fiber: +// Do not establish a happens-before relation between fibers +const unsigned __tsan_switch_to_fiber_no_sync = 1 << 0; + #ifdef __cplusplus } // extern "C" #endif Index: lib/sanitizer_common/sanitizer_thread_registry.h =================================================================== --- lib/sanitizer_common/sanitizer_thread_registry.h +++ lib/sanitizer_common/sanitizer_thread_registry.h @@ -31,6 +31,7 @@ enum class ThreadType { Regular, // Normal thread Worker, // macOS Grand Central Dispatch (GCD) worker thread + Fiber, // Fiber }; // Generic thread context. Specific sanitizer tools may inherit from it. Index: lib/tsan/rtl/tsan_interceptors.h =================================================================== --- lib/tsan/rtl/tsan_interceptors.h +++ lib/tsan/rtl/tsan_interceptors.h @@ -23,6 +23,7 @@ #if !SANITIZER_GO INLINE bool in_symbolizer() { + cur_thread_init(); return UNLIKELY(cur_thread()->in_symbolizer); } #endif @@ -30,6 +31,7 @@ } // namespace __tsan #define SCOPED_INTERCEPTOR_RAW(func, ...) \ + cur_thread_init(); \ ThreadState *thr = cur_thread(); \ const uptr caller_pc = GET_CALLER_PC(); \ ScopedInterceptor si(thr, #func, caller_pc); \ Index: lib/tsan/rtl/tsan_interceptors.cc =================================================================== --- lib/tsan/rtl/tsan_interceptors.cc +++ lib/tsan/rtl/tsan_interceptors.cc @@ -554,6 +554,7 @@ // FIXME: put everything below into a common extern "C" block? extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) { + cur_thread_init(); SetJmp(cur_thread(), sp, mangled_sp); } @@ -942,6 +943,7 @@ void *param = p->param; int tid = 0; { + cur_thread_init(); ThreadState *thr = cur_thread(); // Thread-local state is not initialized yet. ScopedIgnoreInterceptors ignore; @@ -1053,6 +1055,9 @@ TSAN_INTERCEPTOR(void, pthread_exit, void *retval) { { SCOPED_INTERCEPTOR_RAW(pthread_exit, retval); +#if !SANITIZER_MAC && !SANITIZER_ANDROID + CHECK_EQ(thr, &cur_thread_placeholder); +#endif } REAL(pthread_exit)(retval); } @@ -1981,6 +1986,7 @@ void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, __sanitizer_siginfo *info, void *ctx) { + cur_thread_init(); ThreadState *thr = cur_thread(); ThreadSignalContext *sctx = SigCtx(thr); if (sig < 0 || sig >= kSigCount) { Index: lib/tsan/rtl/tsan_interface.cc =================================================================== --- lib/tsan/rtl/tsan_interface.cc +++ lib/tsan/rtl/tsan_interface.cc @@ -24,6 +24,7 @@ typedef u64 uint64_t; void __tsan_init() { + cur_thread_init(); Initialize(cur_thread()); } @@ -123,6 +124,33 @@ __tsan_unaligned_write8(addr); *addr = v; } + +#if !SANITIZER_MAC && !SANITIZER_ANDROID +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_get_current_fiber() { + return cur_thread(); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_create_fiber(unsigned flags) { + return FiberCreate(cur_thread(), CALLERPC, flags); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_destroy_fiber(void *fiber) { + FiberDestroy(cur_thread(), CALLERPC, static_cast(fiber)); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_switch_to_fiber(void *fiber, unsigned flags) { + FiberSwitch(cur_thread(), CALLERPC, static_cast(fiber), flags); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_set_fiber_name(void *fiber, const char *name) { + ThreadSetName(static_cast(fiber), name); +} +#endif // !SANITIZER_MAC && !SANITIZER_ANDROID } // extern "C" void __tsan_acquire(void *addr) { Index: lib/tsan/rtl/tsan_rtl.h =================================================================== --- lib/tsan/rtl/tsan_rtl.h +++ lib/tsan/rtl/tsan_rtl.h @@ -384,6 +384,9 @@ // taken by epoch between synchs. // This way we can save one load from tls. u64 fast_synch_epoch; + // Technically `current` should be a separate THREADLOCAL variable; + // but it is placed here in order to share cache line with previous fields. + ThreadState* current; // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read. // We do not distinguish beteween ignoring reads and writes // for better performance. @@ -462,11 +465,20 @@ #if SANITIZER_MAC || SANITIZER_ANDROID ThreadState *cur_thread(); void cur_thread_finalize(); +INLINE void cur_thread_init() { } #else __attribute__((tls_model("initial-exec"))) extern THREADLOCAL char cur_thread_placeholder[]; INLINE ThreadState *cur_thread() { - return reinterpret_cast(&cur_thread_placeholder); + return reinterpret_cast(cur_thread_placeholder)->current; +} +INLINE void cur_thread_init() { + ThreadState *thr = reinterpret_cast(cur_thread_placeholder); + if (UNLIKELY(!thr->current)) + thr->current = thr; +} +INLINE void set_cur_thread(ThreadState *thr) { + reinterpret_cast(cur_thread_placeholder)->current = thr; } INLINE void cur_thread_finalize() { } #endif // SANITIZER_MAC || SANITIZER_ANDROID @@ -868,6 +880,16 @@ } #endif +ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags); +void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber); +void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags); + +// These need to match __tsan_switch_to_fiber_* flags defined in +// tsan_interface.h. See documentation there as well. +enum FiberSwitchFlags { + FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync +}; + } // namespace __tsan #endif // TSAN_RTL_H Index: lib/tsan/rtl/tsan_rtl_thread.cc =================================================================== --- lib/tsan/rtl/tsan_rtl_thread.cc +++ lib/tsan/rtl/tsan_rtl_thread.cc @@ -246,7 +246,8 @@ uptr tls_addr = 0; uptr tls_size = 0; #if !SANITIZER_GO - GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); + if (thread_type != ThreadType::Fiber) + GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); if (tid) { if (stk_addr && stk_size) @@ -404,4 +405,40 @@ } } +#if !SANITIZER_MAC && !SANITIZER_ANDROID && !SANITIZER_GO +void FiberSwitchImpl(ThreadState *from, ThreadState *to) { + Processor *proc = from->proc(); + ProcUnwire(proc, from); + ProcWire(proc, to); + set_cur_thread(to); +} + +ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags) { + void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadState)); + ThreadState *fiber = static_cast(mem); + internal_memset(fiber, 0, sizeof(*fiber)); + int tid = ThreadCreate(thr, pc, 0, true); + FiberSwitchImpl(thr, fiber); + ThreadStart(fiber, tid, 0, ThreadType::Fiber); + FiberSwitchImpl(fiber, thr); + return fiber; +} + +void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber) { + FiberSwitchImpl(thr, fiber); + ThreadFinish(fiber); + FiberSwitchImpl(fiber, thr); + internal_free(fiber); +} + +void FiberSwitch(ThreadState *thr, uptr pc, + ThreadState *fiber, unsigned flags) { + if (!(flags & FiberSwitchFlagNoSync)) + Release(thr, pc, (uptr)fiber); + FiberSwitchImpl(thr, fiber); + if (!(flags & FiberSwitchFlagNoSync)) + Acquire(fiber, pc, (uptr)fiber); +} +#endif + } // namespace __tsan Index: test/tsan/fiber_asm.cc =================================================================== --- /dev/null +++ test/tsan/fiber_asm.cc @@ -0,0 +1,86 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// REQUIRES: x86_64-target-arch +// UNSUPPORTED: darwin +#include "test.h" + +struct ucontext { + void *sp; + void *fiber; +}; + +extern "C" { + void ucontext_do_switch(void **save, void **load); + void ucontext_trampoline(); +} + +__asm__(".global ucontext_do_switch\n" + "ucontext_do_switch:\n\t" + "pushq %rbp\n\t" + "pushq %r15\n\t" + "pushq %r14\n\t" + "pushq %r13\n\t" + "pushq %r12\n\t" + "pushq %rbx\n\t" + "movq %rsp, (%rdi)\n\t" + "movq (%rsi), %rsp\n\t" + "popq %rbx\n\t" + "popq %r12\n\t" + "popq %r13\n\t" + "popq %r14\n\t" + "popq %r15\n\t" + "popq %rbp\n\t" + "retq"); + +__asm__(".global ucontext_trampoline\n" + "ucontext_trampoline:\n\t" + ".cfi_startproc\n\t" + ".cfi_undefined rip\n\t" + "movq %r12, %rdi\n\t" + "jmpq *%rbx\n\t" + ".cfi_endproc"); + +void ucontext_init(ucontext *context, void *stack, unsigned stack_sz, + void (*func)(void*), void *arg) { + void **sp = reinterpret_cast(static_cast(stack) + stack_sz); + *(--sp) = 0; + *(--sp) = reinterpret_cast(ucontext_trampoline); + *(--sp) = 0; // rbp + *(--sp) = 0; // r15 + *(--sp) = 0; // r14 + *(--sp) = 0; // r13 + *(--sp) = arg; // r12 + *(--sp) = reinterpret_cast(func); // rbx + context->sp = sp; + context->fiber = __tsan_create_fiber(0); +} + +void ucontext_free(ucontext *context) { + __tsan_destroy_fiber(context->fiber); +} + +__attribute__((no_sanitize_thread)) +void ucontext_switch(ucontext *save, ucontext *load) { + save->fiber = __tsan_get_current_fiber(); + __tsan_switch_to_fiber(load->fiber, 0); + ucontext_do_switch(&save->sp, &load->sp); +} + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext uc, orig_uc; + +void func(void *arg) { + __asm__ __volatile__(".cfi_undefined rip"); + ucontext_switch(&uc, &orig_uc); +} + +int main() { + ucontext_init(&uc, stack, sizeof(stack), func, 0); + ucontext_switch(&orig_uc, &uc); + ucontext_free(&uc); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS Index: test/tsan/fiber_from_thread.cc =================================================================== --- /dev/null +++ test/tsan/fiber_from_thread.cc @@ -0,0 +1,48 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext_t uc, orig_uc1, orig_uc2; +void *fiber, *orig_fiber1, *orig_fiber2; + +int var; + +void *Thread(void *x) { + orig_fiber2 = __tsan_get_current_fiber(); + swapcontext(&orig_uc2, &orig_uc1); + return 0; +} + +void func() { + pthread_t t; + pthread_create(&t, 0, Thread, 0); + pthread_join(t, 0); + __tsan_switch_to_fiber(orig_fiber1, 0); + swapcontext(&uc, &orig_uc1); +} + +int main() { + orig_fiber1 = __tsan_get_current_fiber(); + fiber = __tsan_create_fiber(0); + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = 0; + makecontext(&uc, func, 0); + var = 1; + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc1, &uc); + var = 2; + __tsan_switch_to_fiber(orig_fiber2, 0); + swapcontext(&orig_uc1, &orig_uc2); + var = 3; + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS Index: test/tsan/fiber_longjmp.cc =================================================================== --- /dev/null +++ test/tsan/fiber_longjmp.cc @@ -0,0 +1,80 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +sigjmp_buf jmpbuf, orig_jmpbuf[2]; +void *fiber, *orig_fiber[2]; + +const unsigned N = 1000; + +__attribute__((noinline)) +void switch0() { + if (!sigsetjmp(jmpbuf, 0)) { + __tsan_switch_to_fiber(orig_fiber[0], 0); + siglongjmp(orig_jmpbuf[0], 1); + } +} + +void func() { + if (!sigsetjmp(jmpbuf, 0)) { + __tsan_switch_to_fiber(orig_fiber[0], 0); + siglongjmp(orig_jmpbuf[0], 1); + } + for (;;) { + switch0(); + if (!sigsetjmp(jmpbuf, 0)) { + __tsan_switch_to_fiber(orig_fiber[1], 0); + siglongjmp(orig_jmpbuf[1], 1); + } + } +} + +void *Thread(void *x) { + orig_fiber[1] = __tsan_get_current_fiber(); + for (unsigned i = 0; i < N; i++) { + barrier_wait(&barrier); + if (!sigsetjmp(orig_jmpbuf[1], 0)) { + __tsan_switch_to_fiber(fiber, 0); + siglongjmp(jmpbuf, 1); + } + barrier_wait(&barrier); + } + return 0; +} + +int main() { + fiber = __tsan_create_fiber(0); + barrier_init(&barrier, 2); + pthread_t t; + pthread_create(&t, 0, Thread, 0); + orig_fiber[0] = __tsan_get_current_fiber(); + ucontext_t uc, orig_uc; + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = 0; + makecontext(&uc, func, 0); + if (!sigsetjmp(orig_jmpbuf[0], 0)) { + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc, &uc); + } + for (unsigned i = 0; i < N; i++) { + if (!sigsetjmp(orig_jmpbuf[0], 0)) { + __tsan_switch_to_fiber(fiber, 0); + siglongjmp(jmpbuf, 1); + } + barrier_wait(&barrier); + barrier_wait(&barrier); + } + pthread_join(t, 0); + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS Index: test/tsan/fiber_race.cc =================================================================== --- /dev/null +++ test/tsan/fiber_race.cc @@ -0,0 +1,36 @@ +// RUN: %clang_tsan -O1 %s -o %t && %deflake %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext_t uc, orig_uc; +void *fiber, *orig_fiber; + +int var; + +void func() { + var = 1; + __tsan_switch_to_fiber(orig_fiber, __tsan_switch_to_fiber_no_sync); + swapcontext(&uc, &orig_uc); +} + +int main() { + orig_fiber = __tsan_get_current_fiber(); + fiber = __tsan_create_fiber(0); + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = 0; + makecontext(&uc, func, 0); + var = 2; + __tsan_switch_to_fiber(fiber, __tsan_switch_to_fiber_no_sync); + swapcontext(&orig_uc, &uc); + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK: WARNING: ThreadSanitizer: data race +// CHECK: PASS Index: test/tsan/fiber_simple.cc =================================================================== --- /dev/null +++ test/tsan/fiber_simple.cc @@ -0,0 +1,36 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext_t uc, orig_uc; +void *fiber, *orig_fiber; + +int var; + +void func() { + var = 1; + __tsan_switch_to_fiber(orig_fiber, 0); + swapcontext(&uc, &orig_uc); +} + +int main() { + orig_fiber = __tsan_get_current_fiber(); + fiber = __tsan_create_fiber(0); + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = 0; + makecontext(&uc, func, 0); + var = 2; + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc, &uc); + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS Index: test/tsan/fiber_two_threads.cc =================================================================== --- /dev/null +++ test/tsan/fiber_two_threads.cc @@ -0,0 +1,62 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext_t uc, orig_uc[2]; +void *fiber, *orig_fiber[2]; + +const unsigned N = 1000; + +__attribute__((noinline)) +void switch0() { + __tsan_switch_to_fiber(orig_fiber[0], 0); + swapcontext(&uc, &orig_uc[0]); +} + +void func() { + for (;;) { + switch0(); + __tsan_switch_to_fiber(orig_fiber[1], 0); + swapcontext(&uc, &orig_uc[1]); + } +} + +void *Thread(void *x) { + orig_fiber[1] = __tsan_get_current_fiber(); + for (unsigned i = 0; i < N; i++) { + barrier_wait(&barrier); + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc[1], &uc); + barrier_wait(&barrier); + } + return 0; +} + +int main() { + fiber = __tsan_create_fiber(0); + barrier_init(&barrier, 2); + pthread_t t; + pthread_create(&t, 0, Thread, 0); + orig_fiber[0] = __tsan_get_current_fiber(); + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = 0; + makecontext(&uc, func, 0); + for (unsigned i = 0; i < N; i++) { + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc[0], &uc); + barrier_wait(&barrier); + barrier_wait(&barrier); + } + pthread_join(t, 0); + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS