diff --git a/lld/CMakeLists.txt b/lld/CMakeLists.txt --- a/lld/CMakeLists.txt +++ b/lld/CMakeLists.txt @@ -224,6 +224,12 @@ add_definitions("-DLLD_DEFAULT_LD_LLD_IS_MINGW=1") endif() +option(LLD_THREAD_SAFE_MEMORY + "Make the bump pointer allocator and StringSaver thread safe." OFF) +if (LLD_THREAD_SAFE_MEMORY) + add_definitions("-DLLD_THREAD_SAFE_MEMORY=1") +endif() + if (MSVC) add_definitions(-wd4530) # Suppress 'warning C4530: C++ exception handler used, but unwind semantics are not enabled.' add_definitions(-wd4062) # Suppress 'warning C4062: enumerator X in switch of enum Y is not handled' from system header. diff --git a/lld/Common/CommonLinkerContext.cpp b/lld/Common/CommonLinkerContext.cpp --- a/lld/Common/CommonLinkerContext.cpp +++ b/lld/Common/CommonLinkerContext.cpp @@ -32,8 +32,15 @@ assert(lctx); // Explicitly call the destructors since we created the objects with placement // new in SpecificAlloc::create(). - for (auto &it : instances) + for (auto &it : allocContext.instances) it.second->~SpecificAllocBase(); + + llvm::sys::ScopedWriter lock(contextMutex); + for (AllocContext *context : perThreadContexts) { + for (auto &instance : context->instances) + instance.second->~SpecificAllocBase(); + delete context; + } lctx = nullptr; } @@ -48,4 +55,5 @@ if (lctx == nullptr) return; delete lctx; + lctx = nullptr; } diff --git a/lld/Common/Memory.cpp b/lld/Common/Memory.cpp --- a/lld/Common/Memory.cpp +++ b/lld/Common/Memory.cpp @@ -15,10 +15,34 @@ SpecificAllocBase * lld::SpecificAllocBase::getOrCreate(void *tag, size_t size, size_t align, SpecificAllocBase *(&creator)(void *)) { - auto &instances = context().instances; + AllocContext &allocContext = context().allocContext; + auto &instances = allocContext.instances; auto &instance = instances[tag]; if (instance == nullptr) { - void *storage = context().bAlloc.Allocate(size, align); + void *storage = allocContext.bAlloc.Allocate(size, align); + instance = creator(storage); + } + return instance; +} + +AllocContext *CommonLinkerContext::perThreadAllocContext() { + AllocContext *perThreadContext = threadContext.get(); + if (!perThreadContext) { + // Context didn't exist yet for this thread, so create a new one. + threadContext.set(perThreadContext = new AllocContext); + llvm::sys::ScopedWriter lock(contextMutex); + perThreadContexts.push_back(perThreadContext); + } + return perThreadContext; +} + +SpecificAllocBase *lld::SpecificAllocBase::getOrCreatePerThread( + void *tag, size_t size, size_t align, + SpecificAllocBase *(&creator)(void *)) { + AllocContext *threadContext = context().perThreadAllocContext(); + auto &instance = threadContext->instances[tag]; + if (!instance) { + void *storage = threadContext->bAlloc.Allocate(size, align); instance = creator(storage); } return instance; diff --git a/lld/include/lld/Common/CommonLinkerContext.h b/lld/include/lld/Common/CommonLinkerContext.h --- a/lld/include/lld/Common/CommonLinkerContext.h +++ b/lld/include/lld/Common/CommonLinkerContext.h @@ -21,7 +21,9 @@ #include "lld/Common/ErrorHandler.h" #include "lld/Common/Memory.h" +#include "llvm/Support/RWMutex.h" #include "llvm/Support/StringSaver.h" +#include "llvm/Support/ThreadLocal.h" namespace llvm { class raw_ostream; @@ -29,6 +31,15 @@ namespace lld { struct SpecificAllocBase; + +// TODO Maybe just use the CommonLinkerContext class - but I dont want +// unnecessary nesting of the mutexes +struct AllocContext { + llvm::BumpPtrAllocator bAlloc; + llvm::StringSaver saver{bAlloc}; + llvm::DenseMap instances; +}; + class CommonLinkerContext { public: CommonLinkerContext(); @@ -36,11 +47,21 @@ static void destroy(); - llvm::BumpPtrAllocator bAlloc; - llvm::StringSaver saver{bAlloc}; - llvm::DenseMap instances; + // The AllocContext shared by all threads. + AllocContext allocContext; ErrorHandler e; + + // Returns the thread-local AllocContext. + AllocContext *perThreadAllocContext(); + +private: + // Pointer to this thread's context. + llvm::sys::ThreadLocal threadContext; + + // Vector of all TLS contexts; + std::vector perThreadContexts; + llvm::sys::RWMutex contextMutex; }; // Retrieve the global state. Currently only one state can exist per process, @@ -51,11 +72,38 @@ template T &context() { return static_cast(commonContext()); } - bool hasContext(); -inline llvm::StringSaver &saver() { return context().saver; } -inline llvm::BumpPtrAllocator &bAlloc() { return context().bAlloc; } +inline llvm::StringSaver &perThreadSaver() { + return context().perThreadAllocContext()->saver; +} + +inline llvm::BumpPtrAllocator &perThreadBAlloc() { + return context().perThreadAllocContext()->bAlloc; +} + +inline llvm::StringSaver &saverUnsafe() { return context().allocContext.saver; } + +inline llvm::BumpPtrAllocator &bAllocUnsafe() { + return context().allocContext.bAlloc; +} + +inline llvm::StringSaver &saver() { +#if LLD_THREAD_SAFE_MEMORY + return context().perThreadAllocContext()->saver; +#else + return context().allocContext.saver; +#endif +} + +inline llvm::BumpPtrAllocator &bAlloc() { +#if LLD_THREAD_SAFE_MEMORY + return context().perThreadAllocContext()->bAlloc; +#else + return context().allocContext.bAlloc; +#endif +} + } // namespace lld #endif diff --git a/lld/include/lld/Common/Memory.h b/lld/include/lld/Common/Memory.h --- a/lld/include/lld/Common/Memory.h +++ b/lld/include/lld/Common/Memory.h @@ -22,14 +22,20 @@ #define LLD_COMMON_MEMORY_H #include "llvm/Support/Allocator.h" +#include "llvm/Support/RWMutex.h" namespace lld { + // A base class only used by the CommonLinkerContext to keep track of the // SpecificAlloc<> instances. struct SpecificAllocBase { virtual ~SpecificAllocBase() = default; static SpecificAllocBase *getOrCreate(void *tag, size_t size, size_t align, SpecificAllocBase *(&creator)(void *)); + + static SpecificAllocBase * + getOrCreatePerThread(void *tag, size_t size, size_t align, + SpecificAllocBase *(&creator)(void *)); }; // An arena of specific types T, created on-demand. @@ -55,13 +61,45 @@ return ((SpecificAlloc *)instance)->alloc; } +template +inline llvm::SpecificBumpPtrAllocator &getSpecificAllocSingletonPerThread() { + SpecificAllocBase *instance = SpecificAllocBase::getOrCreatePerThread( + &SpecificAlloc::tag, sizeof(SpecificAlloc), + alignof(SpecificAlloc), SpecificAlloc::create); + return ((SpecificAlloc *)instance)->alloc; +} + // Creates new instances of T off a (almost) contiguous arena/object pool. The // instances are destroyed whenever lldMain() goes out of scope. -template T *make(U &&... args) { +// This is NOT thread safe. +template T *makeUnsafe(U &&...args) { return new (getSpecificAllocSingleton().Allocate()) T(std::forward(args)...); } +// Creates new instances of T off a (almost) contiguous arena/object pool +// specific for this thread. The instances are destroyed whenever lldMain() goes +// out of scope. +// This is thread safe. +template T *makePerThread(U &&...args) { + return new (getSpecificAllocSingletonPerThread().Allocate()) + T(std::forward(args)...); +} + +// Creates new instances of T off a (almost) contiguous arena/object pool. The +// instances are destroyed whenever lldMain() goes out of scope. +// This may or may not be thread-safe, depending on whether THREAD_SAFE_MEMORY +// is set. +template T *make(U &&...args) { +#if LLD_THREAD_SAFE_MEMORY + return new (getSpecificAllocSingletonPerThread().Allocate()) + T(std::forward(args)...); +#else + return new (getSpecificAllocSingleton().Allocate()) + T(std::forward(args)...); +#endif +} + } // namespace lld #endif