Index: lib/tsan/CMakeLists.txt =================================================================== --- lib/tsan/CMakeLists.txt +++ lib/tsan/CMakeLists.txt @@ -54,6 +54,7 @@ rtl/tsan_suppressions.cpp rtl/tsan_symbolize.cpp rtl/tsan_sync.cpp + rtl/fuzzing_scheduler/fuzzing_scheduler.cpp ) set(TSAN_CXX_SOURCES @@ -107,7 +108,8 @@ rtl/tsan_symbolize.h rtl/tsan_sync.h rtl/tsan_trace.h - rtl/tsan_update_shadow_word_inl.h) + rtl/tsan_update_shadow_word_inl.h + rtl/fuzzing_scheduler/fuzzing_scheduler.h) set(TSAN_RUNTIME_LIBRARIES) add_compiler_rt_component(tsan) Index: lib/tsan/rtl/fuzzing_scheduler/fuzzing_scheduler.h =================================================================== --- /dev/null +++ lib/tsan/rtl/fuzzing_scheduler/fuzzing_scheduler.h @@ -0,0 +1,12 @@ +#pragma once + +namespace __tsan { + +struct IFuzzingScheduler +{ + virtual void SynchronizationPoint() = 0; +}; + +IFuzzingScheduler& GetFuzzingScheduler(); + +} // namespace __tsan Index: lib/tsan/rtl/fuzzing_scheduler/fuzzing_scheduler.cpp =================================================================== --- /dev/null +++ lib/tsan/rtl/fuzzing_scheduler/fuzzing_scheduler.cpp @@ -0,0 +1,138 @@ +#include "fuzzing_scheduler.h" +#include +#include +#include +#include +#include +#include +#include + +namespace __interception { + extern int (*real_pthread_create)(void*, void*, void *(*)(void*), void*); + extern int (*real_pthread_detach)(void*); +} + +namespace __tsan { + +namespace { + +struct NullFuzzingScheduler : IFuzzingScheduler { + void SynchronizationPoint() override { + } +}; + +thread_local u64 tid = 0; +u64 max_tid = 0; + +struct RandomFuzzingScheduler : IFuzzingScheduler { + RandomFuzzingScheduler() { + srand(NanoTime()); + pthread_t t; + REAL(pthread_create)(&t, NULL, reinterpret_cast(&RandomFuzzingScheduler::WatchDog), this); + REAL(pthread_detach)(&t); + } + + +private: + enum class ThreadState { + UNKNOWN, + RUNNING, + WAIT, + OUT_TIME + }; + + struct ThreadContext { + ThreadState state = ThreadState::UNKNOWN; + u64 start_time = 0; + }; + + ThreadContext contexts[65536] = {}; + Stats stats; + + u64 GetTid() { + if (tid == 0) { + tid = __atomic_add_fetch(&max_tid, 1, __ATOMIC_RELAXED); + } + if (tid > 65535) { + Printf("FATAL: ThreadSanitizer The maximum number of threads created during the program should not exceed 65535"); + Die(); + } + return tid; + } + + void SynchronizationPoint() override { + auto tid = GetTid(); + auto old_state = __atomic_load_n(&contexts[tid].state, __ATOMIC_SEQ_CST); + __atomic_store_n(&contexts[tid].state, ThreadState::WAIT, __ATOMIC_SEQ_CST); + if (old_state == ThreadState::RUNNING) { + auto next_tid = GetNextTid(); + __atomic_store_n(&contexts[next_tid].start_time, NanoTime(), __ATOMIC_SEQ_CST); + __atomic_store_n(&contexts[next_tid].state, ThreadState::RUNNING, __ATOMIC_SEQ_CST); + } + //PrintStates(); + while (__atomic_load_n(&contexts[tid].state, __ATOMIC_SEQ_CST) == ThreadState::WAIT) { + internal_sched_yield(); + } + } + + u64 GetNextTid() { + const u64 local_max_tid = __atomic_load_n(&max_tid, __ATOMIC_SEQ_CST); + const u64 next_tid = rand() % local_max_tid + 1; + for (u64 i = 0; i < local_max_tid; i++) { + if (__atomic_load_n(&contexts[(next_tid + i) % max_tid + 1].state, __ATOMIC_SEQ_CST) == ThreadState::WAIT) { + return (next_tid + i) % max_tid + 1; + } + } + return next_tid; + } + + void* WatchDog() { + while (true) { + usleep(200 * 1000); + u64 local_max_tid = __atomic_load_n(&max_tid, __ATOMIC_SEQ_CST); + for (u64 i = 1; i <= local_max_tid; i++) { + if (__atomic_load_n(&contexts[i].state, __ATOMIC_SEQ_CST) == ThreadState::RUNNING && __atomic_load_n(&contexts[i].start_time, __ATOMIC_SEQ_CST) + 200 * 1000 * 1000ULL <= NanoTime()) { + __atomic_store_n(&contexts[i].state, ThreadState::OUT_TIME, __ATOMIC_SEQ_CST); + } + } + bool exists_running = false; + for (u64 i = 1; i <= local_max_tid; i++) { + if (__atomic_load_n(&contexts[i].state, __ATOMIC_SEQ_CST) == ThreadState::RUNNING) { + exists_running = true; + } + } + if (!exists_running) { + auto next_tid = GetNextTid(); + __atomic_store_n(&contexts[next_tid].start_time, NanoTime(), __ATOMIC_SEQ_CST); + __atomic_store_n(&contexts[next_tid].state, ThreadState::RUNNING, __ATOMIC_SEQ_CST); + } + } + return nullptr; + } + +}; + +IFuzzingScheduler& FuzzingSchedulerDispatcher() { + if (!strcmp(flags()->fuzzing_scheduler, "")) { + auto* scheduler = static_cast(InternalCalloc(1, sizeof(NullFuzzingScheduler))); + new (scheduler) NullFuzzingScheduler; + return *scheduler; + } else if (!strcmp(flags()->fuzzing_scheduler, "random")) { + auto* scheduler = static_cast(InternalCalloc(1, sizeof(RandomFuzzingScheduler))); + new (scheduler) RandomFuzzingScheduler; + Printf("WARNING! ThreadSanitizer lunched under the management of a random fuzzing scheduler new\n"); + return *scheduler; + } else { + Printf("FATAL: ThreadSanitizer invalid fuzzing scheduler. Please check TSAN_OPTIONS!\n"); + Die(); + } +} + +} + +IFuzzingScheduler& GetFuzzingScheduler() { + static IFuzzingScheduler& scheduler = FuzzingSchedulerDispatcher(); + return scheduler; +} + +} // namespace __tsan Index: lib/tsan/rtl/tsan_flags.inc =================================================================== --- lib/tsan/rtl/tsan_flags.inc +++ lib/tsan/rtl/tsan_flags.inc @@ -81,3 +81,5 @@ "modules.") TSAN_FLAG(bool, shared_ptr_interceptor, true, "Track atomic reference counting in libc++ shared_ptr and weak_ptr.") +TSAN_FLAG(const char *, fuzzing_scheduler, "", + "Choosing fuzzing scheduler") Index: lib/tsan/rtl/tsan_interceptors.cpp =================================================================== --- lib/tsan/rtl/tsan_interceptors.cpp +++ lib/tsan/rtl/tsan_interceptors.cpp @@ -30,6 +30,7 @@ #include "tsan_rtl.h" #include "tsan_mman.h" #include "tsan_fd.h" +#include "fuzzing_scheduler/fuzzing_scheduler.h" using namespace __tsan; // NOLINT @@ -953,6 +954,7 @@ ProcWire(proc, thr); ThreadStart(thr, tid, GetTid(), ThreadType::Regular); atomic_store(&p->tid, 0, memory_order_release); + GetFuzzingScheduler().SynchronizationPoint(); } void *res = callback(param); // Prevent the callback from being tail called, @@ -985,6 +987,7 @@ attr = &myattr; } int detached = 0; + GetFuzzingScheduler().SynchronizationPoint(); REAL(pthread_attr_getdetachstate)(attr, &detached); AdjustStackSize(attr); @@ -997,6 +1000,7 @@ // Otherwise we see false positives in pthread stack manipulation. ScopedIgnoreInterceptors ignore; ThreadIgnoreBegin(thr, pc); + GetFuzzingScheduler().SynchronizationPoint(); res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p); ThreadIgnoreEnd(thr, pc); } @@ -1016,6 +1020,7 @@ } if (attr == &myattr) pthread_attr_destroy(&myattr); + GetFuzzingScheduler().SynchronizationPoint(); return res; } @@ -1023,6 +1028,7 @@ SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret); int tid = ThreadTid(thr, pc, (uptr)th); ThreadIgnoreBegin(thr, pc); + GetFuzzingScheduler().SynchronizationPoint(); int res = BLOCK_REAL(pthread_join)(th, ret); ThreadIgnoreEnd(thr, pc); if (res == 0) { @@ -1036,6 +1042,7 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) { SCOPED_TSAN_INTERCEPTOR(pthread_detach, th); int tid = ThreadTid(thr, pc, (uptr)th); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_detach)(th); if (res == 0) { ThreadDetach(thr, pc, tid); @@ -1050,6 +1057,7 @@ CHECK_EQ(thr, &cur_thread_placeholder); #endif } + GetFuzzingScheduler().SynchronizationPoint(); REAL(pthread_exit)(retval); } @@ -1058,6 +1066,7 @@ SCOPED_TSAN_INTERCEPTOR(pthread_tryjoin_np, th, ret); int tid = ThreadTid(thr, pc, (uptr)th); ThreadIgnoreBegin(thr, pc); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_tryjoin_np)(th, ret); ThreadIgnoreEnd(thr, pc); if (res == 0) @@ -1072,6 +1081,7 @@ SCOPED_TSAN_INTERCEPTOR(pthread_timedjoin_np, th, ret, abstime); int tid = ThreadTid(thr, pc, (uptr)th); ThreadIgnoreBegin(thr, pc); + GetFuzzingScheduler().SynchronizationPoint(); int res = BLOCK_REAL(pthread_timedjoin_np)(th, ret, abstime); ThreadIgnoreEnd(thr, pc); if (res == 0) @@ -1146,6 +1156,7 @@ void *cond = init_cond(c, true); SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, cond, a); MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); + GetFuzzingScheduler().SynchronizationPoint(); return REAL(pthread_cond_init)(cond, a); } @@ -1172,6 +1183,7 @@ INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m); + GetFuzzingScheduler().SynchronizationPoint(); return cond_wait(thr, pc, &si, (int (*)(void *c, void *m, void *abstime))REAL( pthread_cond_wait), cond, m, 0); @@ -1180,6 +1192,7 @@ INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime); + GetFuzzingScheduler().SynchronizationPoint(); return cond_wait(thr, pc, &si, REAL(pthread_cond_timedwait), cond, m, abstime); } @@ -1189,6 +1202,7 @@ void *reltime) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait_relative_np, cond, m, reltime); + GetFuzzingScheduler().SynchronizationPoint(); return cond_wait(thr, pc, &si, REAL(pthread_cond_timedwait_relative_np), cond, m, reltime); } @@ -1198,6 +1212,7 @@ void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond); MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); + GetFuzzingScheduler().SynchronizationPoint(); return REAL(pthread_cond_signal)(cond); } @@ -1205,6 +1220,7 @@ void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, cond); MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); + GetFuzzingScheduler().SynchronizationPoint(); return REAL(pthread_cond_broadcast)(cond); } @@ -1212,6 +1228,7 @@ void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, cond); MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_cond_destroy)(cond); if (common_flags()->legacy_pthread_cond) { // Free our aux cond and zero the pointer to not leave dangling pointers. @@ -1228,6 +1245,7 @@ u32 flagz = 0; if (a) { int type = 0; + GetFuzzingScheduler().SynchronizationPoint(); if (REAL(pthread_mutexattr_gettype)(a, &type) == 0) if (type == PTHREAD_MUTEX_RECURSIVE || type == PTHREAD_MUTEX_RECURSIVE_NP) @@ -1240,6 +1258,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_mutex_destroy)(m); if (res == 0 || res == errno_EBUSY) { MutexDestroy(thr, pc, (uptr)m); @@ -1249,6 +1268,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_mutex_trylock)(m); if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); @@ -1260,6 +1280,7 @@ #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_mutex_timedlock)(m, abstime); if (res == 0) { MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); @@ -1271,6 +1292,7 @@ #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_spin_init)(m, pshared); if (res == 0) { MutexCreate(thr, pc, (uptr)m); @@ -1280,6 +1302,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_spin_destroy)(m); if (res == 0) { MutexDestroy(thr, pc, (uptr)m); @@ -1290,6 +1313,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m); MutexPreLock(thr, pc, (uptr)m); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_spin_lock)(m); if (res == 0) { MutexPostLock(thr, pc, (uptr)m); @@ -1299,6 +1323,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_spin_trylock)(m); if (res == 0) { MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); @@ -1308,6 +1333,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m); + GetFuzzingScheduler().SynchronizationPoint(); MutexUnlock(thr, pc, (uptr)m); int res = REAL(pthread_spin_unlock)(m); return res; @@ -1316,6 +1342,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_rwlock_init)(m, a); if (res == 0) { MutexCreate(thr, pc, (uptr)m); @@ -1325,6 +1352,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_rwlock_destroy)(m); if (res == 0) { MutexDestroy(thr, pc, (uptr)m); @@ -1335,6 +1363,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m); MutexPreReadLock(thr, pc, (uptr)m); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_rwlock_rdlock)(m); if (res == 0) { MutexPostReadLock(thr, pc, (uptr)m); @@ -1344,6 +1373,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_rwlock_tryrdlock)(m); if (res == 0) { MutexPostReadLock(thr, pc, (uptr)m, MutexFlagTryLock); @@ -1354,6 +1384,7 @@ #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_rwlock_timedrdlock)(m, abstime); if (res == 0) { MutexPostReadLock(thr, pc, (uptr)m); @@ -1365,6 +1396,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m); MutexPreLock(thr, pc, (uptr)m); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_rwlock_wrlock)(m); if (res == 0) { MutexPostLock(thr, pc, (uptr)m); @@ -1374,6 +1406,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_rwlock_trywrlock)(m); if (res == 0) { MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); @@ -1384,6 +1417,7 @@ #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_rwlock_timedwrlock)(m, abstime); if (res == 0) { MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); @@ -1395,6 +1429,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m); MutexReadOrWriteUnlock(thr, pc, (uptr)m); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_rwlock_unlock)(m); return res; } @@ -1403,6 +1438,7 @@ TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) { SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count); MemoryWrite(thr, pc, (uptr)b, kSizeLog1); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_barrier_init)(b, a, count); return res; } @@ -1410,6 +1446,7 @@ TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) { SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b); MemoryWrite(thr, pc, (uptr)b, kSizeLog1); + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_barrier_destroy)(b); return res; } @@ -1875,6 +1912,7 @@ TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { SCOPED_TSAN_INTERCEPTOR(pthread_sigmask, how, set, oldset); + GetFuzzingScheduler().SynchronizationPoint(); return REAL(pthread_sigmask)(how, set, oldset); } @@ -1960,6 +1998,7 @@ &signal->siginfo, &signal->ctx); } } + GetFuzzingScheduler().SynchronizationPoint(); res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->oldset, 0); CHECK_EQ(res, 0); atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); @@ -2067,6 +2106,7 @@ if (tid == pthread_self()) { sctx->int_signal_send = sig; } + GetFuzzingScheduler().SynchronizationPoint(); int res = REAL(pthread_kill)(tid, sig); if (tid == pthread_self()) { CHECK_EQ(sctx->int_signal_send, sig); Index: lib/tsan/rtl/tsan_interface_atomic.cpp =================================================================== --- lib/tsan/rtl/tsan_interface_atomic.cpp +++ lib/tsan/rtl/tsan_interface_atomic.cpp @@ -24,6 +24,7 @@ #include "tsan_flags.h" #include "tsan_interface.h" #include "tsan_rtl.h" +#include "fuzzing_scheduler/fuzzing_scheduler.h" using namespace __tsan; // NOLINT @@ -57,11 +58,13 @@ template T func_xchg(volatile T *v, T op) { T res = __sync_lock_test_and_set(v, op); // __sync_lock_test_and_set does not contain full barrier. + GetFuzzingScheduler().SynchronizationPoint(); __sync_synchronize(); return res; } template T func_add(volatile T *v, T op) { + GetFuzzingScheduler().SynchronizationPoint(); return __sync_fetch_and_add(v, op); } @@ -521,200 +524,237 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Load, a, mo); } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Load, a, mo); } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Load, a, mo); } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Load, a, mo); } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Load, a, mo); } #endif SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Store, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Store, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Store, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Store, a, v, mo); } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Store, a, v, mo); } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Exchange, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Exchange, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Exchange, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Exchange, a, v, mo); } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(Exchange, a, v, mo); } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchAdd, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchAdd, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchAdd, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchAdd, a, v, mo); } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchAdd, a, v, mo); } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchSub, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchSub, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchSub, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchSub, a, v, mo); } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchSub, a, v, mo); } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchAnd, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchAnd, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchAnd, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchAnd, a, v, mo); } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchAnd, a, v, mo); } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchOr, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchOr, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchOr, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchOr, a, v, mo); } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchOr, a, v, mo); } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchXor, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchXor, a, v, mo); } @@ -725,39 +765,46 @@ SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchXor, a, v, mo); } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchXor, a, v, mo); } #endif SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchNand, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchNand, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchNand, a, v, mo); } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchNand, a, v, mo); } #if __TSAN_HAS_INT128 SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(FetchNand, a, v, mo); } #endif @@ -765,24 +812,28 @@ SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } @@ -790,6 +841,7 @@ SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } #endif @@ -797,24 +849,28 @@ SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } @@ -822,6 +878,7 @@ SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } #endif @@ -829,24 +886,28 @@ SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } @@ -854,12 +915,14 @@ SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v, morder mo, morder fmo) { + GetFuzzingScheduler().SynchronizationPoint(); SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } #endif SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic_thread_fence(morder mo) { + GetFuzzingScheduler().SynchronizationPoint(); char* a = 0; SCOPED_ATOMIC(Fence, mo); } @@ -896,47 +959,56 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + GetFuzzingScheduler().SynchronizationPoint(); ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + GetFuzzingScheduler().SynchronizationPoint(); ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + GetFuzzingScheduler().SynchronizationPoint(); ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + GetFuzzingScheduler().SynchronizationPoint(); ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + GetFuzzingScheduler().SynchronizationPoint(); ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + GetFuzzingScheduler().SynchronizationPoint(); ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + GetFuzzingScheduler().SynchronizationPoint(); ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + GetFuzzingScheduler().SynchronizationPoint(); ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_compare_exchange( ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + GetFuzzingScheduler().SynchronizationPoint(); a32 cur = 0; a32 cmp = *(a32*)(a+8); ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire); @@ -946,6 +1018,7 @@ SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_compare_exchange( ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + GetFuzzingScheduler().SynchronizationPoint(); a64 cur = 0; a64 cmp = *(a64*)(a+8); ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire);