tsan: optimize __tsan_read/write16

[dvyukov]

Summary

These callbacks are used for SSE vector accesses.
In some computational programs these accesses dominate.
Currently we do 2 uninlined 8-byte accesses to handle them.
Inline and optimize them similarly to unaligned accesses.
This reduces the vector access benchmark time from 8 to 3 seconds.

Depends on D112603.

[melver]

compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp

482

This code mostly duplicates the above. What if you wrote it as a 2-iteration for-loop? Will it generate worse or better code?

[dvyukov]

hoist and dedup declaration of the cur variable

[dvyukov]

compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp

482

I've tried this code:

ALWAYS_INLINE USED void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr,
                                       AccessType typ) {
  const uptr size = 16;
  FastState fast_state = thr->fast_state;
  if (UNLIKELY(fast_state.GetIgnoreBit()))
    return;
  RawShadow* shadow_mem = MemToShadow(addr);
  bool traced = false;
  Shadow cur(fast_state, 0, 8, typ);
  for (uptr i = 0; i < 2; i++, shadow_mem += kShadowCnt) {
    LOAD_CURRENT_SHADOW(cur, shadow_mem);
    if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
      continue;
    if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
      return RestartMemoryAccess16(thr, pc, addr, typ);
    traced = true;
    if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
      return;
  }
}

and it produces worse code:

before:
$ TIME="%e" time perf record ./bench_memory_access 1 1000000000 12
2.70
2.67
2.64

after:
$ TIME="%e" time perf record ./bench_memory_access 1 1000000000 12
3.22
3.19
3.17

Disasm before:
https://gist.githubusercontent.com/dvyukov/d898e8abaffe5809d9a3ec517ec81ae6/raw/4ce636ce23b3215ef6fc762444b29778b9ab27ff/gistfile1.txt

after:
https://gist.githubusercontent.com/dvyukov/69bf08dc5382c4512fea08ab95f87c61/raw/c357cff208df982428b9854cdeeea01f3ac6afb2/gistfile1.txt

Compiler fails to registrize everything and spills some values onto stack.

compiler-rt/lib/tsan/rtl/tsan_interface.cpp

 using namespace __tsan;
 
 void __tsan_init() { Initialize(cur_thread_init()); }
 
 void __tsan_flush_memory() {
   FlushShadowMemory();
 }
 
-void __tsan_read16(void *addr) {
-  uptr pc = CALLERPC;
-  ThreadState *thr = cur_thread();
-  MemoryAccess(thr, pc, (uptr)addr, 8, kAccessRead);
-  MemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessRead);
-}
-
-void __tsan_write16(void *addr) {
-  uptr pc = CALLERPC;
-  ThreadState *thr = cur_thread();
-  MemoryAccess(thr, pc, (uptr)addr, 8, kAccessWrite);
-  MemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessWrite);
-}
-
 void __tsan_read16_pc(void *addr, void *pc) {
   uptr pc_no_pac = STRIP_PAC_PC(pc);
   ThreadState *thr = cur_thread();
   MemoryAccess(thr, pc_no_pac, (uptr)addr, 8, kAccessRead);
   MemoryAccess(thr, pc_no_pac, (uptr)addr + 8, 8, kAccessRead);
 }
 
 void __tsan_write16_pc(void *addr, void *pc) {

compiler-rt/lib/tsan/rtl/tsan_interface.inc

 void __tsan_read4(void *addr) {
   MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessRead);
 }
 
 void __tsan_read8(void *addr) {
   MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead);
 }
 
+void __tsan_read16(void *addr) {
+  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessRead);
+}
+
 void __tsan_write1(void *addr) {
   MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessWrite);
 }
 
 void __tsan_write2(void *addr) {
   MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessWrite);
 }
 
 void __tsan_write4(void *addr) {
   MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessWrite);
 }
 
 void __tsan_write8(void *addr) {
   MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite);
 }
 
+void __tsan_write16(void *addr) {
+  MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessWrite);
+}
+
 void __tsan_read1_pc(void *addr, void *pc) {
   MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessRead | kAccessExternalPC);
 }
 
 void __tsan_read2_pc(void *addr, void *pc) {
   MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 2, kAccessRead | kAccessExternalPC);
 }
 

compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp

if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
     return;
   if (UNLIKELY(fast_state.GetIgnoreBit()))
     return;
   if (!TryTraceMemoryAccess(thr, pc, addr, size, typ))
     return TraceRestartMemoryAccess(thr, pc, addr, size, typ);
   CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
 }
 
+void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr, AccessType typ);
+
+NOINLINE
+void RestartMemoryAccess16(ThreadState* thr, uptr pc, uptr addr,
+                           AccessType typ) {
+  TraceSwitchPart(thr);
+  MemoryAccess16(thr, pc, addr, typ);
+}
+
+ALWAYS_INLINE USED void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr,
+                                       AccessType typ) {
+  const uptr size = 16;
+  FastState fast_state = thr->fast_state;
+  if (UNLIKELY(fast_state.GetIgnoreBit()))
+    return;
+  Shadow cur(fast_state, 0, 8, typ);
+  RawShadow* shadow_mem = MemToShadow(addr);
+  bool traced = false;
+  {
+    LOAD_CURRENT_SHADOW(cur, shadow_mem);
+    if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
+      goto SECOND;
+    if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
+      return RestartMemoryAccess16(thr, pc, addr, typ);
+    traced = true;
+    if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
+      return;
+  }
+SECOND:
+  shadow_mem += kShadowCnt;
+  LOAD_CURRENT_SHADOW(cur, shadow_mem);
+  if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
+    return;
+  if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
+    return RestartMemoryAccess16(thr, pc, addr, typ);
+  CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
+}
+
 NOINLINE
 void RestartUnalignedMemoryAccess(ThreadState* thr, uptr pc, uptr addr,
                                   uptr size, AccessType typ) {
   TraceSwitchPart(thr);
   UnalignedMemoryAccess(thr, pc, addr, size, typ);
 }
 
 ALWAYS_INLINE USED void UnalignedMemoryAccess(ThreadState* thr, uptr pc,