Index: include/sanitizer/tsan_interface.h =================================================================== --- include/sanitizer/tsan_interface.h +++ include/sanitizer/tsan_interface.h @@ -137,6 +137,20 @@ void __tsan_external_read(void *addr, void *caller_pc, void *tag); void __tsan_external_write(void *addr, void *caller_pc, void *tag); +// Fiber switching API. +// - TSAN context for fiber can be created by __tsan_create_fiber +// and freed by __tsan_destroy_fiber. +// - TSAN context of current fiber or thread can be obtained +// by calling __tsan_get_current_fiber. +// - __tsan_switch_to_fiber should be called immediatly before switch +// to fiber, such as call of swapcontext. +// - Fiber name can be set by __tsan_set_fiber_name. +void *__tsan_get_current_fiber(void); +void *__tsan_create_fiber(unsigned flags); +void __tsan_destroy_fiber(void *fiber); +void __tsan_switch_to_fiber(void *fiber, unsigned flags); +void __tsan_set_fiber_name(void *fiber, const char *name); + #ifdef __cplusplus } // extern "C" #endif Index: lib/tsan/rtl/tsan_interface.cc =================================================================== --- lib/tsan/rtl/tsan_interface.cc +++ lib/tsan/rtl/tsan_interface.cc @@ -124,6 +124,63 @@ __tsan_unaligned_write8(addr); *addr = v; } + +#if !SANITIZER_MAC && !SANITIZER_ANDROID +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_get_current_fiber() { + return cur_thread(); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void *__tsan_create_fiber(unsigned flags) { + const uptr pc = GET_CALLER_PC(); + ThreadState *thr = cur_thread(); + Processor *proc = thr->proc(); + void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadState)); + ThreadState *fiber = static_cast(mem); + internal_memset(fiber, 0, sizeof(*fiber)); + int tid = ThreadCreate(thr, pc, 0, true); + ProcUnwire(proc, thr); + ProcWire(proc, fiber); + set_cur_thread(fiber); + ThreadStart(fiber, tid, 0, false); + ProcUnwire(proc, fiber); + ProcWire(proc, thr); + set_cur_thread(thr); + return fiber; +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_destroy_fiber(void *fiber_arg) { + ThreadState *thr = cur_thread(); + Processor *proc = thr->proc(); + ThreadState *fiber = static_cast(fiber_arg); + ProcUnwire(proc, thr); + ProcWire(proc, fiber); + set_cur_thread(fiber); + ThreadFinish(fiber); + ProcUnwire(proc, fiber); + ProcWire(proc, thr); + set_cur_thread(thr); + internal_free(fiber); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_switch_to_fiber(void *fiber_arg, unsigned flags) { + ThreadState *thr = cur_thread(); + Processor *proc = thr->proc(); + ThreadState *fiber = static_cast(fiber_arg); + ProcUnwire(proc, thr); + ProcWire(proc, fiber); + set_cur_thread(fiber); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_set_fiber_name(void *fiber_arg, const char *name) { + ThreadState *fiber = static_cast(fiber_arg); + ThreadSetName(fiber, name); +} +#endif // !SANITIZER_MAC && !SANITIZER_ANDROID } // extern "C" void __tsan_acquire(void *addr) { Index: lib/tsan/rtl/tsan_rtl.h =================================================================== --- lib/tsan/rtl/tsan_rtl.h +++ lib/tsan/rtl/tsan_rtl.h @@ -462,10 +462,19 @@ ThreadState *cur_thread(); void cur_thread_finalize(); #else +struct ThreadStatePlaceholder { + uptr cur; + char buf[sizeof(ThreadState)]; +}; __attribute__((tls_model("initial-exec"))) -extern THREADLOCAL char cur_thread_placeholder[]; +extern THREADLOCAL ThreadStatePlaceholder cur_thread_placeholder; INLINE ThreadState *cur_thread() { - return reinterpret_cast(&cur_thread_placeholder); + return reinterpret_cast(cur_thread_placeholder.buf + + cur_thread_placeholder.cur); +} +INLINE void set_cur_thread(ThreadState *thr) { + cur_thread_placeholder.cur = reinterpret_cast(thr) - + cur_thread_placeholder.buf; } INLINE void cur_thread_finalize() { } #endif // SANITIZER_MAC || SANITIZER_ANDROID Index: lib/tsan/rtl/tsan_rtl.cc =================================================================== --- lib/tsan/rtl/tsan_rtl.cc +++ lib/tsan/rtl/tsan_rtl.cc @@ -47,7 +47,7 @@ #if !SANITIZER_GO && !SANITIZER_MAC __attribute__((tls_model("initial-exec"))) -THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64); +THREADLOCAL ThreadStatePlaceholder cur_thread_placeholder ALIGNED(64); #endif static char ctx_placeholder[sizeof(Context)] ALIGNED(64); Context *ctx; Index: lib/tsan/rtl/tsan_rtl_thread.cc =================================================================== --- lib/tsan/rtl/tsan_rtl_thread.cc +++ lib/tsan/rtl/tsan_rtl_thread.cc @@ -246,7 +246,8 @@ uptr tls_addr = 0; uptr tls_size = 0; #if !SANITIZER_GO - GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); + if (os_id) + GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size); if (tid) { if (stk_addr && stk_size) Index: test/tsan/fiber_asm.cc =================================================================== --- /dev/null +++ test/tsan/fiber_asm.cc @@ -0,0 +1,85 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" + +struct ucontext { + void *sp; + void *fiber; +}; + +extern "C" { + void ucontext_do_switch(void **save, void **load); + void ucontext_trampoline(); +} + +__asm__(".global ucontext_do_switch\n" + "ucontext_do_switch:\n\t" + "pushq %rbp\n\t" + "pushq %r15\n\t" + "pushq %r14\n\t" + "pushq %r13\n\t" + "pushq %r12\n\t" + "pushq %rbx\n\t" + "movq %rsp, (%rdi)\n\t" + "movq (%rsi), %rsp\n\t" + "popq %rbx\n\t" + "popq %r12\n\t" + "popq %r13\n\t" + "popq %r14\n\t" + "popq %r15\n\t" + "popq %rbp\n\t" + "retq"); + +__asm__(".global ucontext_trampoline\n" + "ucontext_trampoline:\n\t" + ".cfi_startproc\n\t" + ".cfi_undefined rip\n\t" + "movq %r12, %rdi\n\t" + "jmpq *%rbx\n\t" + ".cfi_endproc"); + +void ucontext_init(ucontext *context, void *stack, unsigned stack_sz, + void (*func)(void*), void *arg) { + void **sp = reinterpret_cast(static_cast(stack) + stack_sz); + *(--sp) = 0; + *(--sp) = reinterpret_cast(ucontext_trampoline); + *(--sp) = 0; // rbp + *(--sp) = 0; // r15 + *(--sp) = 0; // r14 + *(--sp) = 0; // r13 + *(--sp) = arg; // r12 + *(--sp) = reinterpret_cast(func); // rbx + context->sp = sp; + context->fiber = __tsan_create_fiber(0); +} + +void ucontext_free(ucontext *context) { + __tsan_destroy_fiber(context->fiber); +} + +__attribute__((no_sanitize_thread)) +void ucontext_switch(ucontext *save, ucontext *load) { + save->fiber = __tsan_get_current_fiber(); + __tsan_switch_to_fiber(load->fiber, 0); + ucontext_do_switch(&save->sp, &load->sp); +} + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext uc, orig_uc; + +void func(void *arg) { + __asm__ __volatile__(".cfi_undefined rip"); + ucontext_switch(&uc, &orig_uc); +} + +int main() { + ucontext_init(&uc, stack, sizeof(stack), func, 0); + ucontext_switch(&orig_uc, &uc); + ucontext_free(&uc); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS Index: test/tsan/fiber_race.cc =================================================================== --- /dev/null +++ test/tsan/fiber_race.cc @@ -0,0 +1,36 @@ +// RUN: %clang_tsan -O1 %s -o %t && %deflake %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext_t uc, orig_uc; +void *fiber, *orig_fiber; + +int var; + +void func() { + var = 1; + __tsan_switch_to_fiber(orig_fiber, 0); + swapcontext(&uc, &orig_uc); +} + +int main() { + orig_fiber = __tsan_get_current_fiber(); + fiber = __tsan_create_fiber(0); + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = &orig_uc; + makecontext(&uc, func, 0); + var = 2; + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc, &uc); + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK: WARNING: ThreadSanitizer: data race +// CHECK: PASS Index: test/tsan/fiber_simple.cc =================================================================== --- /dev/null +++ test/tsan/fiber_simple.cc @@ -0,0 +1,32 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext_t uc, orig_uc; +void *fiber, *orig_fiber; + +void func() { + __tsan_switch_to_fiber(orig_fiber, 0); + swapcontext(&uc, &orig_uc); +} + +int main() { + orig_fiber = __tsan_get_current_fiber(); + fiber = __tsan_create_fiber(0); + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = &orig_uc; + makecontext(&uc, func, 0); + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc, &uc); + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS Index: test/tsan/fiber_two_threads.cc =================================================================== --- /dev/null +++ test/tsan/fiber_two_threads.cc @@ -0,0 +1,66 @@ +// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s +// UNSUPPORTED: darwin +#include "test.h" +#include + +char stack[64 * 1024] __attribute__((aligned(16))); + +ucontext_t uc, orig_uc[2]; +void *fiber, *orig_fiber[2]; + +const unsigned N = 1000; + +__attribute__((noinline)) +void switch0() { + __tsan_acquire(&orig_fiber[0]); + __tsan_switch_to_fiber(orig_fiber[0], 0); + swapcontext(&uc, &orig_uc[0]); +} + +void func() { + for (;;) { + switch0(); + __tsan_acquire(&orig_fiber[1]); + __tsan_switch_to_fiber(orig_fiber[1], 0); + swapcontext(&uc, &orig_uc[1]); + } +} + +void *Thread(void *x) { + orig_fiber[1] = __tsan_get_current_fiber(); + for (unsigned i = 0; i < N; i++) { + barrier_wait(&barrier); + __tsan_release(&orig_fiber[1]); + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc[1], &uc); + barrier_wait(&barrier); + } + return 0; +} + +int main() { + fiber = __tsan_create_fiber(0); + barrier_init(&barrier, 2); + pthread_t t; + pthread_create(&t, 0, Thread, 0); + orig_fiber[0] = __tsan_get_current_fiber(); + getcontext(&uc); + uc.uc_stack.ss_sp = stack; + uc.uc_stack.ss_size = sizeof(stack); + uc.uc_link = &orig_uc[0]; + makecontext(&uc, func, 0); + for (unsigned i = 0; i < N; i++) { + __tsan_release(&orig_fiber[0]); + __tsan_switch_to_fiber(fiber, 0); + swapcontext(&orig_uc[0], &uc); + barrier_wait(&barrier); + barrier_wait(&barrier); + } + pthread_join(t, 0); + __tsan_destroy_fiber(fiber); + fprintf(stderr, "PASS\n"); + return 0; +} + +// CHECK-NOT: WARNING: ThreadSanitizer: +// CHECK: PASS