diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface.cpp b/compiler-rt/lib/tsan/rtl/tsan_interface.cpp --- a/compiler-rt/lib/tsan/rtl/tsan_interface.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interface.cpp @@ -26,20 +26,6 @@ FlushShadowMemory(); } -void __tsan_read16(void *addr) { - uptr pc = CALLERPC; - ThreadState *thr = cur_thread(); - MemoryAccess(thr, pc, (uptr)addr, 8, kAccessRead); - MemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessRead); -} - -void __tsan_write16(void *addr) { - uptr pc = CALLERPC; - ThreadState *thr = cur_thread(); - MemoryAccess(thr, pc, (uptr)addr, 8, kAccessWrite); - MemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessWrite); -} - void __tsan_read16_pc(void *addr, void *pc) { uptr pc_no_pac = STRIP_PAC_PC(pc); ThreadState *thr = cur_thread(); diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface.inc b/compiler-rt/lib/tsan/rtl/tsan_interface.inc --- a/compiler-rt/lib/tsan/rtl/tsan_interface.inc +++ b/compiler-rt/lib/tsan/rtl/tsan_interface.inc @@ -34,6 +34,10 @@ MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead); } +void __tsan_read16(void *addr) { + MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessRead); +} + void __tsan_write1(void *addr) { MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessWrite); } @@ -50,6 +54,10 @@ MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite); } +void __tsan_write16(void *addr) { + MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessWrite); +} + void __tsan_read1_pc(void *addr, void *pc) { MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessRead | kAccessExternalPC); } diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp --- a/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp @@ -451,6 +451,45 @@ CheckRaces(thr, shadow_mem, cur, shadow, access, typ); } +void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr, AccessType typ); + +NOINLINE +void RestartMemoryAccess16(ThreadState* thr, uptr pc, uptr addr, + AccessType typ) { + TraceSwitchPart(thr); + MemoryAccess16(thr, pc, addr, typ); +} + +ALWAYS_INLINE USED void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr, + AccessType typ) { + const uptr size = 16; + FastState fast_state = thr->fast_state; + if (UNLIKELY(fast_state.GetIgnoreBit())) + return; + RawShadow* shadow_mem = MemToShadow(addr); + bool traced = false; + { + Shadow cur(fast_state, 0, 8, typ); + LOAD_CURRENT_SHADOW(cur, shadow_mem); + if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) + goto SECOND; + if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ)) + return RestartMemoryAccess16(thr, pc, addr, typ); + traced = true; + if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ))) + return; + } +SECOND: + shadow_mem += kShadowCnt; + Shadow cur(fast_state, 0, 8, typ); + LOAD_CURRENT_SHADOW(cur, shadow_mem); + if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ))) + return; + if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ)) + return RestartMemoryAccess16(thr, pc, addr, typ); + CheckRaces(thr, shadow_mem, cur, shadow, access, typ); +} + NOINLINE void RestartUnalignedMemoryAccess(ThreadState* thr, uptr pc, uptr addr, uptr size, AccessType typ) {