diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp --- a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp @@ -32,17 +32,15 @@ // transitively includes , // and it's prohibited to include std headers into tsan runtime. // So we do this dirty trick. -#define _MM_MALLOC_H_INCLUDED -#define __MM_MALLOC_H -#include +# define _MM_MALLOC_H_INCLUDED +# define __MM_MALLOC_H +# include typedef __m128i m128; #endif volatile int __tsan_resumed = 0; -extern "C" void __tsan_resume() { - __tsan_resumed = 1; -} +extern "C" void __tsan_resume() { __tsan_resumed = 1; } namespace __tsan { @@ -58,22 +56,22 @@ bool OnFinalize(bool failed); void OnInitialize(); #else -#include +# include SANITIZER_WEAK_CXX_DEFAULT_IMPL bool OnFinalize(bool failed) { -#if !SANITIZER_GO +# if !SANITIZER_GO if (auto *ptr = dlsym(RTLD_DEFAULT, "__tsan_on_finalize")) return reinterpret_cast(ptr)(failed); -#endif +# endif return failed; } SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnInitialize() { -#if !SANITIZER_GO +# if !SANITIZER_GO if (auto *ptr = dlsym(RTLD_DEFAULT, "__tsan_on_initialize")) { return reinterpret_cast(ptr)(); } -#endif +# endif } #endif @@ -87,7 +85,7 @@ const uptr hdr = GetThreadTraceHeader(tid); internal_snprintf(name, sizeof(name), "trace header %u", tid); MapThreadTrace(hdr, sizeof(Trace), name); - new((void*)hdr) Trace(); + new ((void *)hdr) Trace(); // We are going to use only a small part of the trace with the default // value of history_size. However, the constructor writes to the whole trace. // Release the unused part. @@ -98,13 +96,12 @@ ReleaseMemoryPagesToOS(hdr_end, hdr + sizeof(Trace)); uptr unused = hdr + sizeof(Trace) - hdr_end; if (hdr_end != (uptr)MmapFixedNoAccess(hdr_end, unused)) { - Report("ThreadSanitizer: failed to mprotect(%p, %p)\n", - hdr_end, unused); + Report("ThreadSanitizer: failed to mprotect(%p, %p)\n", hdr_end, unused); CHECK("unable to mprotect" && 0); } } void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadContext)); - return new(mem) ThreadContext(tid); + return new (mem) ThreadContext(tid); } #if !SANITIZER_GO @@ -129,9 +126,8 @@ } // The objects are allocated in TLS, so one may rely on zero-initialization. -ThreadState::ThreadState(Context *ctx, u32 tid, int unique_id, u64 epoch, - unsigned reuse_count, uptr stk_addr, uptr stk_size, - uptr tls_addr, uptr tls_size) +ThreadState::ThreadState(Context *ctx, u32 tid, int unique_id, u64 epoch, unsigned reuse_count, + uptr stk_addr, uptr stk_size, uptr tls_addr, uptr tls_size) : fast_state(tid, epoch) // Do not touch these, rely on zero initialization, // they may be accessed before the ctor. @@ -187,8 +183,7 @@ filename.append("%s.%d", flags()->profile_memory, (int)internal_getpid()); fd_t fd = OpenFile(filename.data(), WrOnly); if (fd == kInvalidFd) { - Printf("ThreadSanitizer: failed to open memory profile file '%s'\n", - filename.data()); + Printf("ThreadSanitizer: failed to open memory profile file '%s'\n", filename.data()); } else { mprof_fd = fd; } @@ -197,9 +192,7 @@ u64 last_flush = NanoTime(); uptr last_rss = 0; - for (int i = 0; - atomic_load(&ctx->stop_background_thread, memory_order_relaxed) == 0; - i++) { + for (int i = 0; atomic_load(&ctx->stop_background_thread, memory_order_relaxed) == 0; i++) { SleepForMillis(100); u64 now = NanoTime(); @@ -215,14 +208,15 @@ if (flags()->memory_limit_mb > 0) { uptr rss = GetRSS(); uptr limit = uptr(flags()->memory_limit_mb) << 20; - VPrintf(1, "ThreadSanitizer: memory flush check" - " RSS=%llu LAST=%llu LIMIT=%llu\n", + VPrintf(1, + "ThreadSanitizer: memory flush check" + " RSS=%llu LAST=%llu LIMIT=%llu\n", (u64)rss >> 20, (u64)last_rss >> 20, (u64)limit >> 20); if (2 * rss > limit + last_rss) { VPrintf(1, "ThreadSanitizer: flushing memory due to RSS\n"); FlushShadowMemory(); rss = GetRSS(); - VPrintf(1, "ThreadSanitizer: memory flushed RSS=%llu\n", (u64)rss>>20); + VPrintf(1, "ThreadSanitizer: memory flushed RSS=%llu\n", (u64)rss >> 20); } last_rss = rss; } @@ -233,8 +227,7 @@ // Flush symbolizer cache if requested. if (flags()->flush_symbolizer_ms > 0) { - u64 last = atomic_load(&ctx->last_symbolize_time_ns, - memory_order_relaxed); + u64 last = atomic_load(&ctx->last_symbolize_time_ns, memory_order_relaxed); if (last != 0 && last + flags()->flush_symbolizer_ms * kMs2Ns < now) { Lock l(&ctx->report_mtx); ScopedErrorReportLock l2; @@ -250,13 +243,13 @@ ctx->background_thread = internal_start_thread(&BackgroundThread, 0); } -#ifndef __mips__ +# ifndef __mips__ static void StopBackgroundThread() { atomic_store(&ctx->stop_background_thread, 1, memory_order_relaxed); internal_join_thread(ctx->background_thread); ctx->background_thread = 0; } -#endif +# endif #endif void DontNeedShadowFor(uptr addr, uptr size) { @@ -265,7 +258,8 @@ #if !SANITIZER_GO void UnmapShadow(ThreadState *thr, uptr addr, uptr size) { - if (size == 0) return; + if (size == 0) + return; DontNeedShadowFor(addr, size); ScopedGlobalProcessor sgp; ctx->metamap.ResetRange(thr->proc(), addr, size); @@ -279,8 +273,7 @@ const uptr kPageSize = GetPageSizeCached(); uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize); uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize); - if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin, - "shadow")) + if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow")) Die(); // Meta shadow is 2:1, so tread carefully. @@ -293,8 +286,7 @@ if (!data_mapped) { // First call maps data+bss. data_mapped = true; - if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin, - "meta shadow")) + if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin, "meta shadow")) Die(); } else { // Mapping continous heap. @@ -305,13 +297,12 @@ return; if (meta_begin < mapped_meta_end) meta_begin = mapped_meta_end; - if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin, - "meta shadow")) + if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin, "meta shadow")) Die(); mapped_meta_end = meta_end; } - VPrintf(2, "mapped meta shadow for (%p-%p) at (%p-%p)\n", - addr, addr+size, meta_begin, meta_end); + VPrintf(2, "mapped meta shadow for (%p-%p) at (%p-%p)\n", addr, addr + size, meta_begin, + meta_end); } void MapThreadTrace(uptr addr, uptr size, const char *name) { @@ -320,8 +311,7 @@ CHECK_LE(addr + size, TraceMemEnd()); CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment if (!MmapFixedSuperNoReserve(addr, size, name)) { - Printf("FATAL: ThreadSanitizer can not mmap thread trace (%p/%p)\n", - addr, size); + Printf("FATAL: ThreadSanitizer can not mmap thread trace (%p/%p)\n", addr, size); Die(); } } @@ -353,8 +343,7 @@ const uptr prev_s = MemToShadow(prev); const uptr prev_m = (uptr)MemToMeta(prev); CHECK_EQ(s - prev_s, (p - prev) * kShadowMultiplier); - CHECK_EQ((m - prev_m) / kMetaShadowSize, - (p - prev) / kMetaShadowCell); + CHECK_EQ((m - prev_m) / kMetaShadowSize, (p - prev) / kMetaShadowCell); } prev = p; } @@ -363,8 +352,7 @@ } #if !SANITIZER_GO -static void OnStackUnwind(const SignalContext &sig, const void *, - BufferedStackTrace *stack) { +static void OnStackUnwind(const SignalContext &sig, const void *, BufferedStackTrace *stack) { stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context, common_flags()->fast_unwind_on_fatal); } @@ -398,7 +386,7 @@ // Install tool-specific callbacks in sanitizer_common. SetCheckUnwindCallback(CheckUnwind); - ctx = new(ctx_placeholder) Context; + ctx = new (ctx_placeholder) Context; const char *env_name = SANITIZER_GO ? "GORACE" : "TSAN_OPTIONS"; const char *options = GetEnv(env_name); CacheBinaryName(); @@ -437,8 +425,7 @@ Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer); #endif - VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n", - (int)internal_getpid()); + VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n", (int)internal_getpid()); // Initialize thread 0. int tid = ThreadCreate(thr, 0, 0, true); @@ -454,10 +441,12 @@ #endif if (flags()->stop_on_start) { - Printf("ThreadSanitizer is suspended at startup (pid %d)." - " Call __tsan_resume().\n", - (int)internal_getpid()); - while (__tsan_resumed == 0) {} + Printf( + "ThreadSanitizer is suspended at startup (pid %d)." + " Call __tsan_resume().\n", + (int)internal_getpid()); + while (__tsan_resumed == 0) { + } } OnInitialize(); @@ -477,7 +466,6 @@ #endif } - int Finalize(ThreadState *thr) { bool failed = false; @@ -493,7 +481,8 @@ ctx->report_mtx.Unlock(); #if !SANITIZER_GO - if (Verbosity()) AllocatorPrintStats(); + if (Verbosity()) + AllocatorPrintStats(); #endif ThreadFinalize(thr); @@ -509,8 +498,7 @@ if (ctx->nmissed_expected) { failed = true; - Printf("ThreadSanitizer: missed %d expected races\n", - ctx->nmissed_expected); + Printf("ThreadSanitizer: missed %d expected races\n", ctx->nmissed_expected); } if (common_flags()->print_suppressions) @@ -558,8 +546,10 @@ uptr nthread = 0; ctx->thread_registry->GetNumberOfThreads(0, 0, &nthread /* alive threads */); - VPrintf(1, "ThreadSanitizer: forked new process with pid %d," - " parent had %d threads\n", (int)internal_getpid(), (int)nthread); + VPrintf(1, + "ThreadSanitizer: forked new process with pid %d," + " parent had %d threads\n", + (int)internal_getpid(), (int)nthread); if (nthread == 1) { StartBackgroundThread(); } else { @@ -579,8 +569,7 @@ void GrowShadowStack(ThreadState *thr) { const int sz = thr->shadow_stack_end - thr->shadow_stack; const int newsz = 2 * sz; - uptr *newstack = (uptr*)internal_alloc(MBlockShadowStack, - newsz * sizeof(uptr)); + uptr *newstack = (uptr *)internal_alloc(MBlockShadowStack, newsz * sizeof(uptr)); internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr)); internal_free(thr->shadow_stack); thr->shadow_stack = newstack; @@ -602,8 +591,7 @@ thr->shadow_stack_pos[0] = pc; thr->shadow_stack_pos++; } - u32 id = StackDepotPut( - StackTrace(thr->shadow_stack, thr->shadow_stack_pos - thr->shadow_stack)); + u32 id = StackDepotPut(StackTrace(thr->shadow_stack, thr->shadow_stack_pos - thr->shadow_stack)); if (pc != 0) thr->shadow_stack_pos--; return id; @@ -625,44 +613,32 @@ thr->nomalloc--; } -Trace *ThreadTrace(int tid) { - return (Trace*)GetThreadTraceHeader(tid); -} +Trace *ThreadTrace(int tid) { return (Trace *)GetThreadTraceHeader(tid); } uptr TraceTopPC(ThreadState *thr) { - Event *events = (Event*)GetThreadTrace(thr->tid); + Event *events = (Event *)GetThreadTrace(thr->tid); uptr pc = events[thr->fast_state.GetTracePos()]; return pc; } -uptr TraceSize() { - return (uptr)(1ull << (kTracePartSizeBits + flags()->history_size + 1)); -} +uptr TraceSize() { return (uptr)(1ull << (kTracePartSizeBits + flags()->history_size + 1)); } -uptr TraceParts() { - return TraceSize() / kTracePartSize; -} +uptr TraceParts() { return TraceSize() / kTracePartSize; } #if !SANITIZER_GO -extern "C" void __tsan_trace_switch() { - TraceSwitch(cur_thread()); -} +extern "C" void __tsan_trace_switch() { TraceSwitch(cur_thread()); } -extern "C" void __tsan_report_race() { - ReportRace(cur_thread()); -} +extern "C" void __tsan_report_race() { ReportRace(cur_thread()); } #endif ALWAYS_INLINE Shadow LoadShadow(u64 *p) { - u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed); + u64 raw = atomic_load((atomic_uint64_t *)p, memory_order_relaxed); return Shadow(raw); } ALWAYS_INLINE -void StoreShadow(u64 *sp, u64 s) { - atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed); -} +void StoreShadow(u64 *sp, u64 s) { atomic_store((atomic_uint64_t *)sp, s, memory_order_relaxed); } ALWAYS_INLINE void StoreIfNotYetStored(u64 *sp, u64 *s) { @@ -671,8 +647,7 @@ } ALWAYS_INLINE -void HandleRace(ThreadState *thr, u64 *shadow_mem, - Shadow cur, Shadow old) { +void HandleRace(ThreadState *thr, u64 *shadow_mem, Shadow cur, Shadow old) { thr->racy_state[0] = cur.raw(); thr->racy_state[1] = old.raw(); thr->racy_shadow_addr = shadow_mem; @@ -688,10 +663,8 @@ } ALWAYS_INLINE -void MemoryAccessImpl1(ThreadState *thr, uptr addr, - int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic, - u64 *shadow_mem, Shadow cur) { - +void MemoryAccessImpl1(ThreadState *thr, uptr addr, int kAccessSizeLog, bool kAccessIsWrite, + bool kIsAtomic, u64 *shadow_mem, Shadow cur) { // This potentially can live in an MMX/SSE scratch register. // The required intrinsics are: // __m128i _mm_move_epi64(__m128i*); @@ -716,28 +689,28 @@ // threads, which is not enough for the unrolled loop. #if SANITIZER_DEBUG for (int idx = 0; idx < 4; idx++) { -#include "tsan_update_shadow_word_inl.h" +# include "tsan_update_shadow_word_inl.h" } #else int idx = 0; -#include "tsan_update_shadow_word_inl.h" +# include "tsan_update_shadow_word_inl.h" idx = 1; if (stored) { -#include "tsan_update_shadow_word_inl.h" +# include "tsan_update_shadow_word_inl.h" } else { -#include "tsan_update_shadow_word_inl.h" +# include "tsan_update_shadow_word_inl.h" } idx = 2; if (stored) { -#include "tsan_update_shadow_word_inl.h" +# include "tsan_update_shadow_word_inl.h" } else { -#include "tsan_update_shadow_word_inl.h" +# include "tsan_update_shadow_word_inl.h" } idx = 3; if (stored) { -#include "tsan_update_shadow_word_inl.h" +# include "tsan_update_shadow_word_inl.h" } else { -#include "tsan_update_shadow_word_inl.h" +# include "tsan_update_shadow_word_inl.h" } #endif @@ -748,13 +721,13 @@ // choose a random candidate slot and replace it StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word); return; - RACE: +RACE: HandleRace(thr, shadow_mem, cur, old); return; } -void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, - int size, bool kAccessIsWrite, bool kIsAtomic) { +void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, int size, bool kAccessIsWrite, + bool kIsAtomic) { while (size) { int size1 = 1; int kAccessSizeLog = kSizeLog1; @@ -779,10 +752,8 @@ Shadow cur(a); for (uptr i = 0; i < kShadowCnt; i++) { Shadow old(LoadShadow(&s[i])); - if (Shadow::Addr0AndSizeAreEqual(cur, old) && - old.TidWithIgnore() == cur.TidWithIgnore() && - old.epoch() > sync_epoch && - old.IsAtomic() == cur.IsAtomic() && + if (Shadow::Addr0AndSizeAreEqual(cur, old) && old.TidWithIgnore() == cur.TidWithIgnore() && + old.epoch() > sync_epoch && old.IsAtomic() == cur.IsAtomic() && old.IsRead() <= cur.IsRead()) return true; } @@ -790,44 +761,44 @@ } #if defined(__SSE3__) -#define SHUF(v0, v1, i0, i1, i2, i3) _mm_castps_si128(_mm_shuffle_ps( \ - _mm_castsi128_ps(v0), _mm_castsi128_ps(v1), \ - (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64)) +# define SHUF(v0, v1, i0, i1, i2, i3) \ + _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(v0), _mm_castsi128_ps(v1), \ + (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64)) ALWAYS_INLINE bool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) { // This is an optimized version of ContainsSameAccessSlow. // load current access into access[0:63] - const m128 access = _mm_cvtsi64_si128(a); + const m128 access = _mm_cvtsi64_si128(a); // duplicate high part of access in addr0: // addr0[0:31] = access[32:63] // addr0[32:63] = access[32:63] // addr0[64:95] = access[32:63] // addr0[96:127] = access[32:63] - const m128 addr0 = SHUF(access, access, 1, 1, 1, 1); + const m128 addr0 = SHUF(access, access, 1, 1, 1, 1); // load 4 shadow slots - const m128 shadow0 = _mm_load_si128((__m128i*)s); - const m128 shadow1 = _mm_load_si128((__m128i*)s + 1); + const m128 shadow0 = _mm_load_si128((__m128i *)s); + const m128 shadow1 = _mm_load_si128((__m128i *)s + 1); // load high parts of 4 shadow slots into addr_vect: // addr_vect[0:31] = shadow0[32:63] // addr_vect[32:63] = shadow0[96:127] // addr_vect[64:95] = shadow1[32:63] // addr_vect[96:127] = shadow1[96:127] - m128 addr_vect = SHUF(shadow0, shadow1, 1, 3, 1, 3); + m128 addr_vect = SHUF(shadow0, shadow1, 1, 3, 1, 3); if (!is_write) { // set IsRead bit in addr_vect - const m128 rw_mask1 = _mm_cvtsi64_si128(1<<15); - const m128 rw_mask = SHUF(rw_mask1, rw_mask1, 0, 0, 0, 0); - addr_vect = _mm_or_si128(addr_vect, rw_mask); + const m128 rw_mask1 = _mm_cvtsi64_si128(1 << 15); + const m128 rw_mask = SHUF(rw_mask1, rw_mask1, 0, 0, 0, 0); + addr_vect = _mm_or_si128(addr_vect, rw_mask); } // addr0 == addr_vect? - const m128 addr_res = _mm_cmpeq_epi32(addr0, addr_vect); + const m128 addr_res = _mm_cmpeq_epi32(addr0, addr_vect); // epoch1[0:63] = sync_epoch - const m128 epoch1 = _mm_cvtsi64_si128(sync_epoch); + const m128 epoch1 = _mm_cvtsi64_si128(sync_epoch); // epoch[0:31] = sync_epoch[0:31] // epoch[32:63] = sync_epoch[0:31] // epoch[64:95] = sync_epoch[0:31] // epoch[96:127] = sync_epoch[0:31] - const m128 epoch = SHUF(epoch1, epoch1, 0, 0, 0, 0); + const m128 epoch = SHUF(epoch1, epoch1, 0, 0, 0, 0); // load low parts of shadow cell epochs into epoch_vect: // epoch_vect[0:31] = shadow0[0:31] // epoch_vect[32:63] = shadow0[64:95] @@ -835,14 +806,14 @@ // epoch_vect[96:127] = shadow1[64:95] const m128 epoch_vect = SHUF(shadow0, shadow1, 0, 2, 0, 2); // epoch_vect >= sync_epoch? - const m128 epoch_res = _mm_cmpgt_epi32(epoch_vect, epoch); + const m128 epoch_res = _mm_cmpgt_epi32(epoch_vect, epoch); // addr_res & epoch_res - const m128 res = _mm_and_si128(addr_res, epoch_res); + const m128 res = _mm_and_si128(addr_res, epoch_res); // mask[0] = res[7] // mask[1] = res[15] // ... // mask[15] = res[127] - const int mask = _mm_movemask_epi8(res); + const int mask = _mm_movemask_epi8(res); return mask != 0; } #endif @@ -861,16 +832,15 @@ #endif } -ALWAYS_INLINE USED -void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, - int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic) { - u64 *shadow_mem = (u64*)MemToShadow(addr); - DPrintf2("#%d: MemoryAccess: @%p %p size=%d" +ALWAYS_INLINE USED void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, int kAccessSizeLog, + bool kAccessIsWrite, bool kIsAtomic) { + u64 *shadow_mem = (u64 *)MemToShadow(addr); + DPrintf2( + "#%d: MemoryAccess: @%p %p size=%d" " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n", - (int)thr->fast_state.tid(), (void*)pc, (void*)addr, - (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem, - (uptr)shadow_mem[0], (uptr)shadow_mem[1], - (uptr)shadow_mem[2], (uptr)shadow_mem[3]); + (int)thr->fast_state.tid(), (void *)pc, (void *)addr, (int)(1 << kAccessSizeLog), + kAccessIsWrite, shadow_mem, (uptr)shadow_mem[0], (uptr)shadow_mem[1], (uptr)shadow_mem[2], + (uptr)shadow_mem[3]); #if SANITIZER_DEBUG if (!IsAppMem(addr)) { Printf("Access to non app mem %zx\n", addr); @@ -898,8 +868,7 @@ cur.SetWrite(kAccessIsWrite); cur.SetAtomic(kIsAtomic); - if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), - thr->fast_synch_epoch, kAccessIsWrite))) { + if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), thr->fast_synch_epoch, kAccessIsWrite))) { return; } @@ -910,26 +879,21 @@ cur.IncrementEpoch(); } - MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic, - shadow_mem, cur); + MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic, shadow_mem, cur); } // Called by MemoryAccessRange in tsan_rtl_thread.cpp -ALWAYS_INLINE USED -void MemoryAccessImpl(ThreadState *thr, uptr addr, - int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic, - u64 *shadow_mem, Shadow cur) { - if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), - thr->fast_synch_epoch, kAccessIsWrite))) { +ALWAYS_INLINE USED void MemoryAccessImpl(ThreadState *thr, uptr addr, int kAccessSizeLog, + bool kAccessIsWrite, bool kIsAtomic, u64 *shadow_mem, + Shadow cur) { + if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), thr->fast_synch_epoch, kAccessIsWrite))) { return; } - MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic, - shadow_mem, cur); + MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic, shadow_mem, cur); } -static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, - u64 val) { +static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, u64 val) { (void)thr; (void)pc; if (size == 0) @@ -953,26 +917,24 @@ size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1); // UnmapOrDie/MmapFixedNoReserve does not work on Windows. if (SANITIZER_WINDOWS || size < common_flags()->clear_shadow_mmap_threshold) { - u64 *p = (u64*)MemToShadow(addr); + u64 *p = (u64 *)MemToShadow(addr); CHECK(IsShadowMem((uptr)p)); CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1))); // FIXME: may overwrite a part outside the region for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) { p[i++] = val; - for (uptr j = 1; j < kShadowCnt; j++) - p[i++] = 0; + for (uptr j = 1; j < kShadowCnt; j++) p[i++] = 0; } } else { // The region is big, reset only beginning and end. const uptr kPageSize = GetPageSizeCached(); - u64 *begin = (u64*)MemToShadow(addr); + u64 *begin = (u64 *)MemToShadow(addr); u64 *end = begin + size / kShadowCell * kShadowCnt; u64 *p = begin; // Set at least first kPageSize/2 to page boundary. while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) { *p++ = val; - for (uptr j = 1; j < kShadowCnt; j++) - *p++ = 0; + for (uptr j = 1; j < kShadowCnt; j++) *p++ = 0; } // Reset middle part. u64 *p1 = p; @@ -982,8 +944,7 @@ // Set the ending. while (p < end) { *p++ = val; - for (uptr j = 1; j < kShadowCnt; j++) - *p++ = 0; + for (uptr j = 1; j < kShadowCnt; j++) *p++ = 0; } } } @@ -1026,17 +987,15 @@ MemoryRangeSet(thr, pc, addr, size, s.raw()); } -void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr, - uptr size) { +void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) { if (thr->ignore_reads_and_writes == 0) MemoryRangeImitateWrite(thr, pc, addr, size); else MemoryResetRange(thr, pc, addr, size); } -ALWAYS_INLINE USED -void FuncEntry(ThreadState *thr, uptr pc) { - DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc); +ALWAYS_INLINE USED void FuncEntry(ThreadState *thr, uptr pc) { + DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void *)pc); if (kCollectHistory) { thr->fast_state.IncrementEpoch(); TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc); @@ -1055,8 +1014,7 @@ thr->shadow_stack_pos++; } -ALWAYS_INLINE USED -void FuncExit(ThreadState *thr) { +ALWAYS_INLINE USED void FuncExit(ThreadState *thr) { DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid()); if (kCollectHistory) { thr->fast_state.IncrementEpoch(); @@ -1094,8 +1052,7 @@ } #if !SANITIZER_GO -extern "C" SANITIZER_INTERFACE_ATTRIBUTE -uptr __tsan_testonly_shadow_stack_current_size() { +extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr __tsan_testonly_shadow_stack_current_size() { ThreadState *thr = cur_thread(); return thr->shadow_stack_pos - thr->shadow_stack; } @@ -1135,5 +1092,5 @@ #if !SANITIZER_GO // Must be included in this file to make sure everything is inlined. -#include "tsan_interface_inl.h" +# include "tsan_interface_inl.h" #endif