diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.h b/compiler-rt/lib/tsan/rtl/tsan_rtl.h --- a/compiler-rt/lib/tsan/rtl/tsan_rtl.h +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.h @@ -571,27 +571,6 @@ void AfterSleep(ThreadState *thr, uptr pc); void IncrementEpoch(ThreadState *thr); -// The hacky call uses custom calling convention and an assembly thunk. -// It is considerably faster that a normal call for the caller -// if it is not executed (it is intended for slow paths from hot functions). -// The trick is that the call preserves all registers and the compiler -// does not treat it as a call. -// If it does not work for you, use normal call. -#if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC -// The caller may not create the stack frame for itself at all, -// so we create a reserve stack frame for it (1024b must be enough). -#define HACKY_CALL(f) \ - __asm__ __volatile__("sub $1024, %%rsp;" \ - CFI_INL_ADJUST_CFA_OFFSET(1024) \ - ".hidden " #f "_thunk;" \ - "call " #f "_thunk;" \ - "add $1024, %%rsp;" \ - CFI_INL_ADJUST_CFA_OFFSET(-1024) \ - ::: "memory", "cc"); -#else -#define HACKY_CALL(f) f() -#endif - #if !SANITIZER_GO uptr ALWAYS_INLINE HeapEnd() { return HeapMemEnd() + PrimaryAllocator::AdditionalSize(); diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp --- a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp @@ -910,12 +910,6 @@ atomic_load_relaxed(&thr->trace_pos)); } -#if !SANITIZER_GO -extern "C" void __tsan_trace_switch() {} - -extern "C" void __tsan_report_race() {} -#endif - void ThreadIgnoreBegin(ThreadState* thr, uptr pc) { DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid); thr->ignore_reads_and_writes++; diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S b/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S deleted file mode 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_amd64.S +++ /dev/null @@ -1,446 +0,0 @@ -// The content of this file is x86_64-only: -#if defined(__x86_64__) - -#include "sanitizer_common/sanitizer_asm.h" - -#if !defined(__APPLE__) -.section .text -#else -.section __TEXT,__text -#endif - -ASM_HIDDEN(__tsan_trace_switch) -.globl ASM_SYMBOL(__tsan_trace_switch_thunk) -ASM_SYMBOL(__tsan_trace_switch_thunk): - CFI_STARTPROC - _CET_ENDBR - # Save scratch registers. - push %rax - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rax, 0) - push %rcx - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rcx, 0) - push %rdx - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rdx, 0) - push %rsi - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rsi, 0) - push %rdi - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rdi, 0) - push %r8 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r8, 0) - push %r9 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r9, 0) - push %r10 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r10, 0) - push %r11 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r11, 0) - # All XMM registers are caller-saved. - sub $0x100, %rsp - CFI_ADJUST_CFA_OFFSET(0x100) - vmovdqu %xmm0, 0x0(%rsp) - vmovdqu %xmm1, 0x10(%rsp) - vmovdqu %xmm2, 0x20(%rsp) - vmovdqu %xmm3, 0x30(%rsp) - vmovdqu %xmm4, 0x40(%rsp) - vmovdqu %xmm5, 0x50(%rsp) - vmovdqu %xmm6, 0x60(%rsp) - vmovdqu %xmm7, 0x70(%rsp) - vmovdqu %xmm8, 0x80(%rsp) - vmovdqu %xmm9, 0x90(%rsp) - vmovdqu %xmm10, 0xa0(%rsp) - vmovdqu %xmm11, 0xb0(%rsp) - vmovdqu %xmm12, 0xc0(%rsp) - vmovdqu %xmm13, 0xd0(%rsp) - vmovdqu %xmm14, 0xe0(%rsp) - vmovdqu %xmm15, 0xf0(%rsp) - # Align stack frame. - push %rbx # non-scratch - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rbx, 0) - mov %rsp, %rbx # save current rsp - CFI_DEF_CFA_REGISTER(%rbx) - shr $4, %rsp # clear 4 lsb, align to 16 - shl $4, %rsp - - call ASM_SYMBOL(__tsan_trace_switch) - - # Unalign stack frame back. - mov %rbx, %rsp # restore the original rsp - CFI_DEF_CFA_REGISTER(%rsp) - pop %rbx - CFI_ADJUST_CFA_OFFSET(-8) - # Restore scratch registers. - vmovdqu 0x0(%rsp), %xmm0 - vmovdqu 0x10(%rsp), %xmm1 - vmovdqu 0x20(%rsp), %xmm2 - vmovdqu 0x30(%rsp), %xmm3 - vmovdqu 0x40(%rsp), %xmm4 - vmovdqu 0x50(%rsp), %xmm5 - vmovdqu 0x60(%rsp), %xmm6 - vmovdqu 0x70(%rsp), %xmm7 - vmovdqu 0x80(%rsp), %xmm8 - vmovdqu 0x90(%rsp), %xmm9 - vmovdqu 0xa0(%rsp), %xmm10 - vmovdqu 0xb0(%rsp), %xmm11 - vmovdqu 0xc0(%rsp), %xmm12 - vmovdqu 0xd0(%rsp), %xmm13 - vmovdqu 0xe0(%rsp), %xmm14 - vmovdqu 0xf0(%rsp), %xmm15 - add $0x100, %rsp - CFI_ADJUST_CFA_OFFSET(-0x100) - pop %r11 - CFI_ADJUST_CFA_OFFSET(-8) - pop %r10 - CFI_ADJUST_CFA_OFFSET(-8) - pop %r9 - CFI_ADJUST_CFA_OFFSET(-8) - pop %r8 - CFI_ADJUST_CFA_OFFSET(-8) - pop %rdi - CFI_ADJUST_CFA_OFFSET(-8) - pop %rsi - CFI_ADJUST_CFA_OFFSET(-8) - pop %rdx - CFI_ADJUST_CFA_OFFSET(-8) - pop %rcx - CFI_ADJUST_CFA_OFFSET(-8) - pop %rax - CFI_ADJUST_CFA_OFFSET(-8) - CFI_RESTORE(%rax) - CFI_RESTORE(%rbx) - CFI_RESTORE(%rcx) - CFI_RESTORE(%rdx) - CFI_RESTORE(%rsi) - CFI_RESTORE(%rdi) - CFI_RESTORE(%r8) - CFI_RESTORE(%r9) - CFI_RESTORE(%r10) - CFI_RESTORE(%r11) - ret - CFI_ENDPROC - -ASM_HIDDEN(__tsan_report_race) -.globl ASM_SYMBOL(__tsan_report_race_thunk) -ASM_SYMBOL(__tsan_report_race_thunk): - CFI_STARTPROC - _CET_ENDBR - # Save scratch registers. - push %rax - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rax, 0) - push %rcx - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rcx, 0) - push %rdx - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rdx, 0) - push %rsi - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rsi, 0) - push %rdi - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rdi, 0) - push %r8 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r8, 0) - push %r9 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r9, 0) - push %r10 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r10, 0) - push %r11 - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%r11, 0) - # All XMM registers are caller-saved. - sub $0x100, %rsp - CFI_ADJUST_CFA_OFFSET(0x100) - vmovdqu %xmm0, 0x0(%rsp) - vmovdqu %xmm1, 0x10(%rsp) - vmovdqu %xmm2, 0x20(%rsp) - vmovdqu %xmm3, 0x30(%rsp) - vmovdqu %xmm4, 0x40(%rsp) - vmovdqu %xmm5, 0x50(%rsp) - vmovdqu %xmm6, 0x60(%rsp) - vmovdqu %xmm7, 0x70(%rsp) - vmovdqu %xmm8, 0x80(%rsp) - vmovdqu %xmm9, 0x90(%rsp) - vmovdqu %xmm10, 0xa0(%rsp) - vmovdqu %xmm11, 0xb0(%rsp) - vmovdqu %xmm12, 0xc0(%rsp) - vmovdqu %xmm13, 0xd0(%rsp) - vmovdqu %xmm14, 0xe0(%rsp) - vmovdqu %xmm15, 0xf0(%rsp) - # Align stack frame. - push %rbx # non-scratch - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rbx, 0) - mov %rsp, %rbx # save current rsp - CFI_DEF_CFA_REGISTER(%rbx) - shr $4, %rsp # clear 4 lsb, align to 16 - shl $4, %rsp - - call ASM_SYMBOL(__tsan_report_race) - - # Unalign stack frame back. - mov %rbx, %rsp # restore the original rsp - CFI_DEF_CFA_REGISTER(%rsp) - pop %rbx - CFI_ADJUST_CFA_OFFSET(-8) - # Restore scratch registers. - vmovdqu 0x0(%rsp), %xmm0 - vmovdqu 0x10(%rsp), %xmm1 - vmovdqu 0x20(%rsp), %xmm2 - vmovdqu 0x30(%rsp), %xmm3 - vmovdqu 0x40(%rsp), %xmm4 - vmovdqu 0x50(%rsp), %xmm5 - vmovdqu 0x60(%rsp), %xmm6 - vmovdqu 0x70(%rsp), %xmm7 - vmovdqu 0x80(%rsp), %xmm8 - vmovdqu 0x90(%rsp), %xmm9 - vmovdqu 0xa0(%rsp), %xmm10 - vmovdqu 0xb0(%rsp), %xmm11 - vmovdqu 0xc0(%rsp), %xmm12 - vmovdqu 0xd0(%rsp), %xmm13 - vmovdqu 0xe0(%rsp), %xmm14 - vmovdqu 0xf0(%rsp), %xmm15 - add $0x100, %rsp - CFI_ADJUST_CFA_OFFSET(-0x100) - pop %r11 - CFI_ADJUST_CFA_OFFSET(-8) - pop %r10 - CFI_ADJUST_CFA_OFFSET(-8) - pop %r9 - CFI_ADJUST_CFA_OFFSET(-8) - pop %r8 - CFI_ADJUST_CFA_OFFSET(-8) - pop %rdi - CFI_ADJUST_CFA_OFFSET(-8) - pop %rsi - CFI_ADJUST_CFA_OFFSET(-8) - pop %rdx - CFI_ADJUST_CFA_OFFSET(-8) - pop %rcx - CFI_ADJUST_CFA_OFFSET(-8) - pop %rax - CFI_ADJUST_CFA_OFFSET(-8) - CFI_RESTORE(%rax) - CFI_RESTORE(%rbx) - CFI_RESTORE(%rcx) - CFI_RESTORE(%rdx) - CFI_RESTORE(%rsi) - CFI_RESTORE(%rdi) - CFI_RESTORE(%r8) - CFI_RESTORE(%r9) - CFI_RESTORE(%r10) - CFI_RESTORE(%r11) - ret - CFI_ENDPROC - -ASM_HIDDEN(__tsan_setjmp) -#if defined(__NetBSD__) -.comm _ZN14__interception15real___setjmp14E,8,8 -#elif !defined(__APPLE__) -.comm _ZN14__interception11real_setjmpE,8,8 -#endif -#if defined(__NetBSD__) -.globl ASM_SYMBOL_INTERCEPTOR(__setjmp14) -ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__setjmp14)) -ASM_SYMBOL_INTERCEPTOR(__setjmp14): -#else -.globl ASM_SYMBOL_INTERCEPTOR(setjmp) -ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp)) -ASM_SYMBOL_INTERCEPTOR(setjmp): -#endif - CFI_STARTPROC - _CET_ENDBR - // save env parameter - push %rdi - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rdi, 0) - // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` -#if defined(__FreeBSD__) || defined(__NetBSD__) - lea 8(%rsp), %rdi -#elif defined(__linux__) || defined(__APPLE__) - lea 16(%rsp), %rdi -#else -# error "Unknown platform" -#endif - // call tsan interceptor - call ASM_SYMBOL(__tsan_setjmp) - // restore env parameter - pop %rdi - CFI_ADJUST_CFA_OFFSET(-8) - CFI_RESTORE(%rdi) - // tail jump to libc setjmp - movl $0, %eax -#if defined(__NetBSD__) - movq _ZN14__interception15real___setjmp14E@GOTPCREL(%rip), %rdx - jmp *(%rdx) -#elif !defined(__APPLE__) - movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx - jmp *(%rdx) -#else - jmp ASM_SYMBOL(setjmp) -#endif - CFI_ENDPROC -#if defined(__NetBSD__) -ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__setjmp14)) -#else -ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp)) -#endif - -.comm _ZN14__interception12real__setjmpE,8,8 -.globl ASM_SYMBOL_INTERCEPTOR(_setjmp) -ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp)) -ASM_SYMBOL_INTERCEPTOR(_setjmp): - CFI_STARTPROC - _CET_ENDBR - // save env parameter - push %rdi - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rdi, 0) - // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` -#if defined(__FreeBSD__) || defined(__NetBSD__) - lea 8(%rsp), %rdi -#elif defined(__linux__) || defined(__APPLE__) - lea 16(%rsp), %rdi -#else -# error "Unknown platform" -#endif - // call tsan interceptor - call ASM_SYMBOL(__tsan_setjmp) - // restore env parameter - pop %rdi - CFI_ADJUST_CFA_OFFSET(-8) - CFI_RESTORE(%rdi) - // tail jump to libc setjmp - movl $0, %eax -#if !defined(__APPLE__) - movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx - jmp *(%rdx) -#else - jmp ASM_SYMBOL(_setjmp) -#endif - CFI_ENDPROC -ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp)) - -#if defined(__NetBSD__) -.comm _ZN14__interception18real___sigsetjmp14E,8,8 -.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14) -ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)) -ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14): -#else -.comm _ZN14__interception14real_sigsetjmpE,8,8 -.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp) -ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) -ASM_SYMBOL_INTERCEPTOR(sigsetjmp): -#endif - CFI_STARTPROC - _CET_ENDBR - // save env parameter - push %rdi - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rdi, 0) - // save savesigs parameter - push %rsi - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rsi, 0) - // align stack frame - sub $8, %rsp - CFI_ADJUST_CFA_OFFSET(8) - // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` -#if defined(__FreeBSD__) || defined(__NetBSD__) - lea 24(%rsp), %rdi -#elif defined(__linux__) || defined(__APPLE__) - lea 32(%rsp), %rdi -#else -# error "Unknown platform" -#endif - // call tsan interceptor - call ASM_SYMBOL(__tsan_setjmp) - // unalign stack frame - add $8, %rsp - CFI_ADJUST_CFA_OFFSET(-8) - // restore savesigs parameter - pop %rsi - CFI_ADJUST_CFA_OFFSET(-8) - CFI_RESTORE(%rsi) - // restore env parameter - pop %rdi - CFI_ADJUST_CFA_OFFSET(-8) - CFI_RESTORE(%rdi) - // tail jump to libc sigsetjmp - movl $0, %eax -#if defined(__NetBSD__) - movq _ZN14__interception18real___sigsetjmp14E@GOTPCREL(%rip), %rdx - jmp *(%rdx) -#elif !defined(__APPLE__) - movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx - jmp *(%rdx) -#else - jmp ASM_SYMBOL(sigsetjmp) -#endif - CFI_ENDPROC -#if defined(__NetBSD__) -ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)) -#else -ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) -#endif - -#if !defined(__APPLE__) && !defined(__NetBSD__) -.comm _ZN14__interception16real___sigsetjmpE,8,8 -.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp) -ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) -ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): - CFI_STARTPROC - _CET_ENDBR - // save env parameter - push %rdi - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rdi, 0) - // save savesigs parameter - push %rsi - CFI_ADJUST_CFA_OFFSET(8) - CFI_REL_OFFSET(%rsi, 0) - // align stack frame - sub $8, %rsp - CFI_ADJUST_CFA_OFFSET(8) - // obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)` -#if defined(__FreeBSD__) - lea 24(%rsp), %rdi -#else - lea 32(%rsp), %rdi -#endif - // call tsan interceptor - call ASM_SYMBOL(__tsan_setjmp) - // unalign stack frame - add $8, %rsp - CFI_ADJUST_CFA_OFFSET(-8) - // restore savesigs parameter - pop %rsi - CFI_ADJUST_CFA_OFFSET(-8) - CFI_RESTORE(%rsi) - // restore env parameter - pop %rdi - CFI_ADJUST_CFA_OFFSET(-8) - CFI_RESTORE(%rdi) - // tail jump to libc sigsetjmp - movl $0, %eax - movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx - jmp *(%rdx) - CFI_ENDPROC -ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) -#endif // !defined(__APPLE__) && !defined(__NetBSD__) - -NO_EXEC_STACK_DIRECTIVE - -#endif