Optimize performance of Thread sanitizer memory access functions

[yuri]

Summary

• Manually unwind code in MemoryAccessImpl1() because clang do not optimize it
• Check for .rodata section only in read operations
• Place LIKELY/UNLIKELY on fast paths

[dvyukov]

Committed in 353401
http://llvm.org/viewvc/llvm-project?view=revision&revision=353401

lib/tsan/rtl/tsan_interface_atomic.cc

static morder convert_morder(morder mo) {
   // since we use __sync_ atomics for actual atomic operations,
   // we can safely ignore it as well. It also subtly affects semantics,
   // but we don't model the difference.
   return (morder)(mo & 0x7fff);
 }
 
 #define SCOPED_ATOMIC(func, ...) \
     ThreadState *const thr = cur_thread(); \
-    if (thr->ignore_sync || thr->ignore_interceptors) { \
+    if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) { \
       ProcessPendingSignals(thr); \
       return NoTsanAtomic##func(__VA_ARGS__); \
     } \
     const uptr callpc = (uptr)__builtin_return_address(0); \
     uptr pc = StackTrace::GetCurrentPc(); \
     mo = convert_morder(mo); \
     AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \
     ScopedAtomic sa(thr, callpc, a, mo, __func__); \

lib/tsan/rtl/tsan_rtl.cc

void MemoryAccessImpl1(ThreadState *thr, uptr addr,
   StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
   StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
 
   // This potentially can live in an MMX/SSE scratch register.
   // The required intrinsics are:
   // __m128i _mm_move_epi64(__m128i*);
   // _mm_storel_epi64(u64*, __m128i);
   u64 store_word = cur.raw();
+  bool stored = false;
 
   // scan all the shadow values and dispatch to 4 categories:
   // same, replace, candidate and race (see comments below).
   // we consider only 3 cases regarding access sizes:
   // equal, intersect and not intersect. initially I considered
   // larger and smaller as well, it allowed to replace some
   // 'candidates' with 'same' or 'replace', but I think
   // it's just not worth it (performance- and complexity-wise).
 
   Shadow old(0);
 
   // It release mode we manually unroll the loop,
   // because empirically gcc generates better code this way.
   // However, we can't afford unrolling in debug mode, because the function
   // consumes almost 4K of stack. Gtest gives only 4K of stack to death test
   // threads, which is not enough for the unrolled loop.
 #if SANITIZER_DEBUG
   for (int idx = 0; idx < 4; idx++) {
 #include "tsan_update_shadow_word_inl.h"
   }
 #else
   int idx = 0;
 #include "tsan_update_shadow_word_inl.h"
   idx = 1;
+  if (stored) {
 #include "tsan_update_shadow_word_inl.h"
+  } else {
+#include "tsan_update_shadow_word_inl.h"
+  }
   idx = 2;
+  if (stored) {
+#include "tsan_update_shadow_word_inl.h"
+  } else {
 #include "tsan_update_shadow_word_inl.h"
+  }
   idx = 3;
+  if (stored) {
+#include "tsan_update_shadow_word_inl.h"
+  } else {
 #include "tsan_update_shadow_word_inl.h"
+  }
 #endif
 
   // we did not find any races and had already stored
   // the current access info, so we are done
-  if (LIKELY(store_word == 0))
+  if (LIKELY(stored))
     return;
   // choose a random candidate slot and replace it
   StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
   StatInc(thr, StatShadowReplace);
   return;
  RACE:
   HandleRace(thr, shadow_mem, cur, old);
   return;
if (!IsAppMem(addr)) {
     DCHECK(IsAppMem(addr));
   }
   if (!IsShadowMem((uptr)shadow_mem)) {
     Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
     DCHECK(IsShadowMem((uptr)shadow_mem));
   }
 #endif
 
-  if (!SANITIZER_GO && *shadow_mem == kShadowRodata) {
+  if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) {
     // Access to .rodata section, no races here.
     // Measurements show that it can be 10-20% of all memory accesses.
     StatInc(thr, StatMop);
     StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
     StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
     StatInc(thr, StatMopRodata);
     return;
   }
 
   FastState fast_state = thr->fast_state;
-  if (fast_state.GetIgnoreBit()) {
+  if (UNLIKELY(fast_state.GetIgnoreBit())) {
     StatInc(thr, StatMop);
     StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
     StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
     StatInc(thr, StatMopIgnored);
     return;
   }
 
   Shadow cur(fast_state);

lib/tsan/rtl/tsan_update_shadow_word_inl.h

 // If we wrap this body into a function, compilers (both gcc and clang)
 // produce sligtly less efficient code.
 //===----------------------------------------------------------------------===//
 do {
   StatInc(thr, StatShadowProcessed);
   const unsigned kAccessSize = 1 << kAccessSizeLog;
   u64 *sp = &shadow_mem[idx];
   old = LoadShadow(sp);
-  if (old.IsZero()) {
+  if (LIKELY(old.IsZero())) {
     StatInc(thr, StatShadowZero);
-    if (store_word)
+    if (!stored) {
       StoreIfNotYetStored(sp, &store_word);
-    // The above StoreIfNotYetStored could be done unconditionally
-    // and it even shows 4% gain on synthetic benchmarks (r4307).
+      stored = true;
+    }
     break;
   }
   // is the memory access equal to the previous?
-  if (Shadow::Addr0AndSizeAreEqual(cur, old)) {
+  if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) {
     StatInc(thr, StatShadowSameSize);
     // same thread?
-    if (Shadow::TidsAreEqual(old, cur)) {
+    if (LIKELY(Shadow::TidsAreEqual(old, cur))) {
       StatInc(thr, StatShadowSameThread);
-      if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))
+      if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) {
         StoreIfNotYetStored(sp, &store_word);
+        stored = true;
+      }
       break;
     }
     StatInc(thr, StatShadowAnotherThread);
     if (HappensBefore(old, thr)) {
-      if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))
+      if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) {
         StoreIfNotYetStored(sp, &store_word);
+        stored = true;
+      }
       break;
     }
-    if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
+    if (LIKELY(old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)))
       break;
     goto RACE;
   }
   // Do the memory access intersect?
   if (Shadow::TwoRangesIntersect(old, cur, kAccessSize)) {
     StatInc(thr, StatShadowIntersect);
     if (Shadow::TidsAreEqual(old, cur)) {
       StatInc(thr, StatShadowSameThread);
       break;
     }
     StatInc(thr, StatShadowAnotherThread);
     if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
       break;
-    if (HappensBefore(old, thr))
+    if (LIKELY(HappensBefore(old, thr)))
       break;
     goto RACE;
   }
   // The accesses do not intersect.
   StatInc(thr, StatShadowNotIntersect);
   break;
 } while (0);