diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h --- a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h @@ -28,6 +28,7 @@ #ifndef SANITIZER_TLS_GET_ADDR_H #define SANITIZER_TLS_GET_ADDR_H +#include "sanitizer_atomic.h" #include "sanitizer_common.h" namespace __sanitizer { @@ -38,9 +39,14 @@ struct DTV { uptr beg, size; }; + struct DTVBlock { + atomic_uintptr_t next; + DTV dtvs[(4096UL - sizeof(next)) / sizeof(DTLS::DTV)]; + }; + + static_assert(sizeof(DTVBlock) <= 4096UL, "Unexpected block size"); - uptr dtv_size; - DTV *dtv; // dtv_size elements, allocated by MmapOrDie. + atomic_uintptr_t dtv_block; // Auxiliary fields, don't access them outside sanitizer_tls_get_addr.cpp uptr last_memalign_size; @@ -49,7 +55,13 @@ template void ForEachDVT(DTLS *dtls, const Fn &fn) { - for (uptr j = 0; j < dtls->dtv_size; ++j) fn(dtls->dtv[j], j); + DTLS::DTVBlock *block = + (DTLS::DTVBlock *)atomic_load(&dtls->dtv_block, memory_order_acquire); + while (block) { + int id = 0; + for (auto &d : block->dtvs) fn(d, id++); + block = (DTLS::DTVBlock *)atomic_load(&block->next, memory_order_acquire); + } } // Returns pointer and size of a linker-allocated TLS block. diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp --- a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp @@ -12,6 +12,7 @@ #include "sanitizer_tls_get_addr.h" +#include "sanitizer_atomic.h" #include "sanitizer_flags.h" #include "sanitizer_platform_interceptors.h" @@ -42,39 +43,54 @@ static const uptr kDestroyedThread = -1; -static inline void DTLS_Deallocate(DTLS::DTV *dtv, uptr size) { - if (!size) return; - VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", dtv, size); - UnmapOrDie(dtv, size * sizeof(DTLS::DTV)); +static void DTLS_Deallocate(DTLS::DTVBlock *block) { + VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", block); + UnmapOrDie(block, sizeof(DTLS::DTVBlock)); atomic_fetch_sub(&number_of_live_dtls, 1, memory_order_relaxed); } -static inline void DTLS_Resize(uptr new_size) { - if (dtls.dtv_size >= new_size) return; - new_size = RoundUpToPowerOfTwo(new_size); - new_size = Max(new_size, 4096UL / sizeof(DTLS::DTV)); - DTLS::DTV *new_dtv = - (DTLS::DTV *)MmapOrDie(new_size * sizeof(DTLS::DTV), "DTLS_Resize"); +static DTLS::DTVBlock *DTLS_NextBlock(atomic_uintptr_t *cur) { + uptr v = atomic_load(cur, memory_order_acquire); + if (v == kDestroyedThread) + return nullptr; + DTLS::DTVBlock *next = (DTLS::DTVBlock *)v; + if (next) + return next; + DTLS::DTVBlock *new_dtv = + (DTLS::DTVBlock *)MmapOrDie(sizeof(DTLS::DTVBlock), "DTLS_NextBlock"); + uptr prev = 0; + if (!atomic_compare_exchange_strong(cur, &prev, (uptr)new_dtv, + memory_order_seq_cst)) { + UnmapOrDie(new_dtv, sizeof(DTLS::DTVBlock)); + return (DTLS::DTVBlock *)prev; + } uptr num_live_dtls = atomic_fetch_add(&number_of_live_dtls, 1, memory_order_relaxed); - VReport(2, "__tls_get_addr: DTLS_Resize %p %zd\n", &dtls, num_live_dtls); - CHECK_LT(num_live_dtls, 1 << 20); - uptr old_dtv_size = dtls.dtv_size; - DTLS::DTV *old_dtv = dtls.dtv; - if (old_dtv_size) - internal_memcpy(new_dtv, dtls.dtv, dtls.dtv_size * sizeof(DTLS::DTV)); - dtls.dtv = new_dtv; - dtls.dtv_size = new_size; - if (old_dtv_size) - DTLS_Deallocate(old_dtv, old_dtv_size); + VReport(2, "__tls_get_addr: DTLS_NextBlock %p %zd\n", &dtls, num_live_dtls); + return new_dtv; +} + +static DTLS::DTV *DTLS_Find(uptr id) { + VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", &dtls, id); + static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs); + DTLS::DTVBlock *cur = DTLS_NextBlock(&dtls.dtv_block); + if (!cur) + return nullptr; + for (; id >= kPerBlock; id -= kPerBlock) cur = DTLS_NextBlock(&cur->next); + return cur->dtvs + id; } void DTLS_Destroy() { if (!common_flags()->intercept_tls_get_addr) return; - VReport(2, "__tls_get_addr: DTLS_Destroy %p %zd\n", &dtls, dtls.dtv_size); - uptr s = dtls.dtv_size; - dtls.dtv_size = kDestroyedThread; // Do this before unmap for AS-safety. - DTLS_Deallocate(dtls.dtv, s); + VReport(2, "__tls_get_addr: DTLS_Destroy %p\n", &dtls); + DTLS::DTVBlock *block = (DTLS::DTVBlock *)atomic_exchange( + &dtls.dtv_block, kDestroyedThread, memory_order_release); + while (block) { + DTLS::DTVBlock *next = + (DTLS::DTVBlock *)atomic_load(&block->next, memory_order_acquire); + DTLS_Deallocate(block); + block = next; + } } #if defined(__powerpc64__) || defined(__mips__) @@ -96,9 +112,9 @@ if (!common_flags()->intercept_tls_get_addr) return 0; TlsGetAddrParam *arg = reinterpret_cast(arg_void); uptr dso_id = arg->dso_id; - if (dtls.dtv_size == kDestroyedThread) return 0; - DTLS_Resize(dso_id + 1); - if (dtls.dtv[dso_id].beg) return 0; + DTLS::DTV *dtv = DTLS_Find(dso_id); + if (!dtv || dtv->beg) + return 0; uptr tls_size = 0; uptr tls_beg = reinterpret_cast(res) - arg->offset - kDtvOffset; VReport(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p " @@ -126,9 +142,9 @@ // This may happen inside the DTOR of main thread, so just ignore it. tls_size = 0; } - dtls.dtv[dso_id].beg = tls_beg; - dtls.dtv[dso_id].size = tls_size; - return dtls.dtv + dso_id; + dtv->beg = tls_beg; + dtv->size = tls_size; + return dtv; } void DTLS_on_libc_memalign(void *ptr, uptr size) { @@ -141,7 +157,8 @@ DTLS *DTLS_Get() { return &dtls; } bool DTLSInDestruction(DTLS *dtls) { - return dtls->dtv_size == kDestroyedThread; + return atomic_load(&dtls->dtv_block, memory_order_relaxed) == + kDestroyedThread; } #else diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp @@ -0,0 +1,55 @@ +// RUN: %clangxx %s -DBUILD_DSO -fPIC -shared -o %t.so +// RUN: %clangxx --std=c++11 %s -o %t +// RUN: %env_tool_opts=verbosity=2 %run %t 2>&1 | FileCheck %s + +// Does not call __tls_get_addr +// UNSUPPORTED: i386-linux + +// Do not intercept __tls_get_addr +// UNSUPPORTED: lsan, ubsan, android + +#include + +#ifndef BUILD_DSO + +#include +#include +#include +#include + +char buff[10000]; + +int main(int argc, char *argv[]) { + sprintf(buff, "rm -f %s.so.*", argv[0]); + system(buff); + + void *prev_handle = 0; + for (int i = 0; i < 300; ++i) { + sprintf(buff, "cp %s.so %s.so.%d", argv[0], argv[0], i); + system(buff); + + sprintf(buff, "%s.so.%d", argv[0], i); + void *handle = dlopen(buff, RTLD_LAZY); + assert(handle != 0); + assert(handle != prev_handle); + prev_handle = handle; + + typedef void (*FnType)(char c); + ((FnType)dlsym(handle, "StoreToTLS"))(i); + } + return 0; +} + +#else // BUILD_DSO +__thread char huge_thread_local_array[1 << 12]; + +extern "C" void StoreToTLS(char c) { + memset(huge_thread_local_array, c, sizeof(huge_thread_local_array)); +} +#endif // BUILD_DSO + +// CHECK: DTLS_Find [[DTLS:0x[a-f0-9]+]] {{[0-9]+}} +// CHECK-NEXT: DTLS_NextBlock [[DTLS]] 0 +// CHECK: DTLS_Find [[DTLS:0x[a-f0-9]+]] 255 +// CHECK-NEXT: DTLS_NextBlock [[DTLS]] 1 +// CHECK-NOT: DTLS_NextBlock \ No newline at end of file