diff --git a/compiler-rt/lib/orc/CMakeLists.txt b/compiler-rt/lib/orc/CMakeLists.txt --- a/compiler-rt/lib/orc/CMakeLists.txt +++ b/compiler-rt/lib/orc/CMakeLists.txt @@ -13,6 +13,7 @@ set(x86_64_SOURCES # x86-64 specific assembly files will go here. macho_tlv.x86-64.S + elfnix_tls.x86-64.S ) set(ORC_IMPL_HEADERS diff --git a/compiler-rt/lib/orc/elfnix_platform.cpp b/compiler-rt/lib/orc/elfnix_platform.cpp --- a/compiler-rt/lib/orc/elfnix_platform.cpp +++ b/compiler-rt/lib/orc/elfnix_platform.cpp @@ -62,6 +62,10 @@ return Error::success(); } +struct TLSDescriptor { + unsigned long Key = 0; + unsigned long DataAddress = 0; +}; class ELFNixPlatformRuntimeState { private: @@ -104,12 +108,18 @@ int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle); void runAtExits(void *DSOHandle); + /// Returns the base address of the section containing ThreadData. + Expected> + getTHreadDataSectionFor(const char *ThreadData); + private: PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle); PerJITDylibState *getJITDylibStateByName(string_view Path); PerJITDylibState & getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs); + Error registerThreadDataSection(span ThreadDataSection); + Expected lookupSymbolInJITDylib(void *DSOHandle, string_view Symbol); @@ -132,6 +142,9 @@ std::recursive_mutex JDStatesMutex; std::unordered_map JDStates; std::unordered_map JDNameToHeader; + + std::mutex ThreadDataSectionsMutex; + std::map ThreadDataSections; }; ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr; @@ -156,7 +169,10 @@ if (POSR.EHFrameSection.StartAddress) __register_frame(POSR.EHFrameSection.StartAddress.toPtr()); - // TODO: Register thread data sections. + if (POSR.ThreadDataSection.StartAddress) { + if (auto Err = registerThreadDataSection(POSR.ThreadDataSection.toSpan())) + return Err; + } return Error::success(); } @@ -235,6 +251,19 @@ } } +Expected> +ELFNixPlatformRuntimeState::getTHreadDataSectionFor(const char *ThreadData) { + std::lock_guard Lock(ThreadDataSectionsMutex); + auto I = ThreadDataSections.upper_bound(ThreadData); + // Check that we have a valid entry conovering this address. + if (I == ThreadDataSections.begin()) + return make_error("No thread local data section for key"); + I = std::prev(I); + if (ThreadData >= I->first + I->second) + return make_error("No thread local data section for key"); + return *I; +} + ELFNixPlatformRuntimeState::PerJITDylibState * ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) { auto I = JDStates.find(DSOHandle); @@ -274,6 +303,18 @@ return JDS; } +Error ELFNixPlatformRuntimeState::registerThreadDataSection(span ThreadDataSection) { + std::lock_guard Lock(ThreadDataSectionsMutex); + auto I = ThreadDataSections.upper_bound(ThreadDataSection.data()); + if (I != ThreadDataSections.begin()) { + auto J = std::prev(I); + if (J->first + J->second > ThreadDataSection.data()) + return make_error("Overlapping .tdata sections"); + } + ThreadDataSections.insert(I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size())); + return Error::success(); +} + Expected ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle, string_view Sym) { @@ -344,6 +385,41 @@ return Error::success(); } +class ELFNixPlatformRuntimeTLVManager { +public: + void *getInstance(const char *ThreadData); + +private: + std::unordered_map Instances; + std::unordered_map> AllocatedSections; +}; + +void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) { + auto I = Instances.find(ThreadData); + if (I != Instances.end()) + return I->second; + auto TDS = ELFNixPlatformRuntimeState::get().getTHreadDataSectionFor(ThreadData); + if (!TDS) { + __orc_rt_log_error(toString(TDS.takeError()).c_str()); + return nullptr; + } + + auto &Allocated = AllocatedSections[TDS->first]; + if (!Allocated) { + Allocated = std::make_unique(TDS->second); + memcpy(Allocated.get(), TDS->first, TDS->second); + } + size_t ThreadDataDelta = ThreadData - TDS->first; + assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds"); + + char *Instance = Allocated.get() + ThreadDataDelta; + Instances[ThreadData] = Instance; + return Instance; +} + +void destroyELFNixTLVMgr(void *ELFNixTLVMgr) { + delete static_cast(ELFNixTLVMgr); +} } // end anonymous namespace @@ -387,6 +463,39 @@ .release(); } +//------------------------------------------------------------------------------ +// TLV support +//------------------------------------------------------------------------------ + +ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSDescriptor *D) { + auto *TLVMgr = static_cast( + pthread_getspecific(D->Key)); + if (!TLVMgr) + TLVMgr = new ELFNixPlatformRuntimeTLVManager(); + if (pthread_setspecific(D->Key, TLVMgr)) { + __orc_rt_log_error("Call to pthread_setspecific failed"); + return nullptr; + } + + return TLVMgr->getInstance( + reinterpret_cast(static_cast(D->DataAddress))); +} + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) { + return WrapperFunction(void)>::handle( + ArgData, ArgSize, + []() -> Expected { + pthread_key_t Key; + if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) { + __orc_rt_log_error("Call to pthread_key_create failed"); + return make_error(strerror(Err)); + } + return static_cast(Key); + }) + .release(); +} + //------------------------------------------------------------------------------ // cxa_atexit support //------------------------------------------------------------------------------ diff --git a/compiler-rt/lib/orc/elfnix_tls.x86-64.S b/compiler-rt/lib/orc/elfnix_tls.x86-64.S new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/orc/elfnix_tls.x86-64.S @@ -0,0 +1,69 @@ + +//===-- orc_rt_elfnix_tls_x86-64.s -------------------------------*- ASM -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#define REGISTER_SAVE_SPACE_SIZE 512 + + .text + + // returns address of TLV in %rax, all other registers preserved + .globl ___orc_rt_elfnix_tls_get_addr +___orc_rt_elfnix_tls_get_addr: + pushq %rbp + movq %rsp, %rbp + subq $REGISTER_SAVE_SPACE_SIZE, %rsp + movq %rbx, -8(%rbp) + movq %rcx, -16(%rbp) + movq %rdx, -24(%rbp) + movq %rsi, -32(%rbp) + movq %rdi, -40(%rbp) + movq %r8, -48(%rbp) + movq %r9, -56(%rbp) + movq %r10, -64(%rbp) + movq %r11, -72(%rbp) + movq %r12, -80(%rbp) + movq %r13, -88(%rbp) + movq %r14, -96(%rbp) + movq %r15, -104(%rbp) + movdqa %xmm0, -128(%rbp) + movdqa %xmm1, -144(%rbp) + movdqa %xmm2, -160(%rbp) + movdqa %xmm3, -176(%rbp) + movdqa %xmm4, -192(%rbp) + movdqa %xmm5, -208(%rbp) + movdqa %xmm6, -224(%rbp) + movdqa %xmm7, -240(%rbp) + call __orc_rt_elfnix_tls_get_addr_impl + movq -8(%rbp), %rbx + movq -16(%rbp), %rcx + movq -24(%rbp), %rdx + movq -32(%rbp), %rsi + movq -40(%rbp), %rdi + movq -48(%rbp), %r8 + movq -56(%rbp), %r9 + movq -64(%rbp), %r10 + movq -72(%rbp), %r11 + movq -80(%rbp), %r12 + movq -88(%rbp), %r13 + movq -96(%rbp), %r14 + movq -104(%rbp), %r15 + movdqa -128(%rbp), %xmm0 + movdqa -144(%rbp), %xmm1 + movdqa -160(%rbp), %xmm2 + movdqa -176(%rbp), %xmm3 + movdqa -192(%rbp), %xmm4 + movdqa -208(%rbp), %xmm5 + movdqa -224(%rbp), %xmm6 + movdqa -240(%rbp), %xmm7 + addq $REGISTER_SAVE_SPACE_SIZE, %rsp + popq %rbp + ret diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h --- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h @@ -27,6 +27,7 @@ PCRel32GOTLoad, PCRel32GOTLoadRelaxable, PCRel32REXGOTLoadRelaxable, + PCRel32TLV, PCRel64GOT, GOTOFF64, GOT64, diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h --- a/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h @@ -324,6 +324,8 @@ /// PCRel32TLVPLoadREXRelaxable, + RequestTLSDescInGOTAndTransformToDelta32, + /// A TLVP entry getter/constructor, transformed to /// Delta32ToTLVPLoadREXRelaxable. /// diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h --- a/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h @@ -216,6 +216,8 @@ Error registerPerObjectSections(const ELFPerObjectSectionsToRegister &POSR); + Expected createPThreadKey(); + ExecutionSession &ES; ObjectLinkingLayer &ObjLinkingLayer; diff --git a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h --- a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h @@ -376,10 +376,11 @@ else return LSOrErr.takeError(); - if (Sym.isDefined() && - (Sym.getType() == ELF::STT_NOTYPE || Sym.getType() == ELF::STT_FUNC || - Sym.getType() == ELF::STT_OBJECT || - Sym.getType() == ELF::STT_SECTION)) { + if (Sym.isDefined() && (Sym.getType() == ELF::STT_NOTYPE || + Sym.getType() == ELF::STT_FUNC || + Sym.getType() == ELF::STT_OBJECT || + Sym.getType() == ELF::STT_SECTION || + Sym.getType() == ELF::STT_TLS)) { // FIXME: Handle extended tables. if (auto *GraphSec = getGraphSection(Sym.st_shndx)) { diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -39,6 +39,7 @@ public: static const uint8_t NullGOTEntryContent[8]; static const uint8_t StubContent[6]; + static const uint8_t TLSDescContent[16]; using PerGraphGOTAndPLTStubsBuilder< PerGraphGOTAndPLTStubsBuilder_ELF_x86_64>::PerGraphGOTAndPLTStubsBuilder; @@ -56,14 +57,21 @@ x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable || E.getKind() == x86_64::RequestGOTAndTransformToDelta64FromGOT || E.getKind() == - x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable; + x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable || + E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32; } - Symbol &createGOTEntry(Symbol &Target) { - auto &GOTEntryBlock = G.createContentBlock( - getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); - GOTEntryBlock.addEdge(x86_64::Pointer64, 0, Target, 0); - return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); + Symbol &createGOTEntry(Symbol &Target, Edge::Kind K) { + if (K == x86_64::RequestTLSDescInGOTAndTransformToDelta32) { + auto &TLSDescBlock = G.createContentBlock(getGOTSection(), getTLSDescBlockContent(), 0, 8, 0); + TLSDescBlock.addEdge(x86_64::Pointer64, 8, Target, 0); + return G.addAnonymousSymbol(TLSDescBlock, 0, 16, false, false); + } else { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); + GOTEntryBlock.addEdge(x86_64::Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); + } } void fixGOTEdge(Edge &E, Symbol &GOTEntry) { @@ -86,6 +94,7 @@ E.setKind(x86_64::Delta64FromGOT); break; case x86_64::RequestGOTAndTransformToDelta32: + case x86_64::RequestTLSDescInGOTAndTransformToDelta32: E.setKind(x86_64::Delta32); break; default: @@ -100,11 +109,11 @@ return E.getKind() == x86_64::BranchPCRel32 && !E.getTarget().isDefined(); } - Symbol &createPLTStub(Symbol &Target) { + Symbol &createPLTStub(Symbol &Target, Edge::Kind K) { auto &StubContentBlock = G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); // Re-use GOT entries for stub targets. - auto &GOTEntrySymbol = getGOTEntry(Target); + auto &GOTEntrySymbol = getGOTEntry(Target, K); StubContentBlock.addEdge(x86_64::Delta32, 2, GOTEntrySymbol, -4); return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false); } @@ -143,6 +152,11 @@ return {reinterpret_cast(StubContent), sizeof(StubContent)}; } + ArrayRef getTLSDescBlockContent() { + return {reinterpret_cast(TLSDescContent), + sizeof(TLSDescContent)}; + } + mutable Section *GOTSection = nullptr; mutable Section *StubsSection = nullptr; }; @@ -154,6 +168,13 @@ const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent[6] = { 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; +// TLS descriptor has 2 words. the first word is the platform pthread key, the second +// word is data address. +const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::TLSDescContent[16] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/ +}; + static const char *getELFX86_64RelocName(uint32_t Type) { switch (Type) { #define ELF_RELOC(Name, Number) \ @@ -199,6 +220,8 @@ return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64; case ELF::R_X86_64_PLT32: return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32; + case ELF::R_X86_64_TLSGD: + return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32TLV; } return make_error("Unsupported x86-64 relocation type " + formatv("{0:d}: ", Type) + @@ -315,6 +338,10 @@ Addend = 0; break; } + case PCRel32TLV: { + Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32; + break; + } case PCRel32GOTLoadRelaxable: { Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable; Addend = 0; diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -417,7 +417,7 @@ E.getKind() == PointerToGOT; } - Symbol &createGOTEntry(Symbol &Target) { + Symbol &createGOTEntry(Symbol &Target, Edge::Kind K) { auto &GOTEntryBlock = G.createContentBlock( getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); @@ -439,11 +439,11 @@ return E.getKind() == Branch26 && !E.getTarget().isDefined(); } - Symbol &createPLTStub(Symbol &Target) { + Symbol &createPLTStub(Symbol &Target, Edge::Kind K) { auto &StubContentBlock = G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); // Re-use GOT entries for stub targets. - auto &GOTEntrySymbol = getGOTEntry(Target); + auto &GOTEntrySymbol = getGOTEntry(Target, K); StubContentBlock.addEdge(LDRLiteral19, 0, GOTEntrySymbol, 0); return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); } diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -435,7 +435,7 @@ x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable; } - Symbol &createGOTEntry(Symbol &Target) { + Symbol &createGOTEntry(Symbol &Target, Edge::Kind K) { return x86_64::createAnonymousPointer(G, getGOTSection(), &Target); } @@ -459,9 +459,9 @@ return E.getKind() == x86_64::BranchPCRel32 && E.getTarget().isExternal(); } - Symbol &createPLTStub(Symbol &Target) { + Symbol &createPLTStub(Symbol &Target, Edge::Kind K) { return x86_64::createAnonymousPointerJumpStub(G, getStubsSection(), - getGOTEntry(Target)); + getGOTEntry(Target, K)); } void fixPLTEdge(Edge &E, Symbol &Stub) { diff --git a/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h b/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h --- a/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h @@ -51,7 +51,7 @@ << " (" << formatv("{0:x}", B->getAddress()) << " + " << formatv("{0:x}", E.getOffset()) << ")\n"; }); - impl().fixGOTEdge(E, getGOTEntry(E.getTarget())); + impl().fixGOTEdge(E, getGOTEntry(E.getTarget(), E.getKind())); } else if (impl().isExternalBranchEdge(E)) { LLVM_DEBUG({ dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) @@ -59,7 +59,7 @@ << " (" << formatv("{0:x}", B->getAddress()) << " + " << formatv("{0:x}", E.getOffset()) << ")\n"; }); - impl().fixPLTEdge(E, getPLTStub(E.getTarget())); + impl().fixPLTEdge(E, getPLTStub(E.getTarget(), E.getKind())); } } @@ -67,14 +67,14 @@ } protected: - Symbol &getGOTEntry(Symbol &Target) { + Symbol &getGOTEntry(Symbol &Target, Edge::Kind K) { assert(Target.hasName() && "GOT edge cannot point to anonymous target"); auto GOTEntryI = GOTEntries.find(Target.getName()); // Build the entry if it doesn't exist. if (GOTEntryI == GOTEntries.end()) { - auto &GOTEntry = impl().createGOTEntry(Target); + auto &GOTEntry = impl().createGOTEntry(Target, K); LLVM_DEBUG({ dbgs() << " Created GOT entry for " << Target.getName() << ": " << GOTEntry << "\n"; @@ -89,13 +89,13 @@ return *GOTEntryI->second; } - Symbol &getPLTStub(Symbol &Target) { + Symbol &getPLTStub(Symbol &Target, Edge::Kind K) { assert(Target.hasName() && "External branch edge can not point to an anonymous target"); auto StubI = PLTStubs.find(Target.getName()); if (StubI == PLTStubs.end()) { - auto &StubSymbol = impl().createPLTStub(Target); + auto &StubSymbol = impl().createPLTStub(Target, K); LLVM_DEBUG({ dbgs() << " Created PLT stub for " << Target.getName() << ": " << StubSymbol << "\n"; diff --git a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp --- a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp @@ -451,7 +451,8 @@ {"__orc_rt_elfnix_platform_bootstrap", &orc_rt_elfnix_platform_bootstrap}, {"__orc_rt_elfnix_platform_shutdown", &orc_rt_elfnix_platform_shutdown}, {"__orc_rt_elfnix_register_object_sections", - &orc_rt_elfnix_register_object_sections}}; + &orc_rt_elfnix_register_object_sections}, + {"__orc_rt_elfnix_create_pthread_key", &orc_rt_elfnix_create_pthread_key}}; SymbolLookupSet RuntimeSymbols; std::vector> AddrsToRecord; @@ -546,6 +547,17 @@ return ErrResult; } +Expected ELFNixPlatform::createPThreadKey() { + if (!orc_rt_elfnix_create_pthread_key) + return make_error( + "Attempting to create pthread key in target, but runtime support has not been loaded yet", inconvertibleErrorCode()); + + Expected Result(0); + if (auto Err = ES.callSPSWrapper(void)>(orc_rt_elfnix_create_pthread_key.getValue(), Result)) + return std::move(Err); + return Result; +} + void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig( MaterializationResponsibility &MR, jitlink::LinkGraph &LG, jitlink::PassConfiguration &Config) { @@ -624,8 +636,8 @@ // Insert TLV lowering at the start of the PostPrunePasses, since we want // it to run before GOT/PLT lowering. - Config.PostPrunePasses.insert( - Config.PostPrunePasses.begin(), + // auto BuildGOTPLTPass = llvm::find(Config.PostPrunePasses, ) + Config.PostPrunePasses.push_back( [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { return fixTLVSectionsAndEdges(G, JD); }); @@ -754,6 +766,61 @@ jitlink::LinkGraph &G, JITDylib &JD) { // TODO implement TLV support + for (auto *Sym : G.external_symbols()) + if (Sym->getName() == "__tls_get_addr") { + Sym->setName("___orc_rt_elfnix_tls_get_addr"); + } + + auto *ThreadDataSection = G.findSectionByName(ThreadDataSectionName); + auto *ThreadBssSection = G.findSectionByName(ThreadBSSSectionName); + + auto isTargetSymbolInThreadSections = [](const jitlink::Edge &E) -> bool { + // FIXME: If the tls symbol is an external symbol, getBlock method + // will fail. + if (E.getTarget().isExternal()) + return false; + const jitlink::Section &Sec = E.getTarget().getBlock().getSection(); + return Sec.getName() == ThreadDataSectionName || Sec.getName() == + ThreadBSSSectionName; + }; + + if (ThreadDataSection || ThreadBssSection) { + Optional Key; + { + std::lock_guard Lock(MP.PlatformMutex); + auto I = MP.JITDylibToPThreadKey.find(&JD); + if (I != MP.JITDylibToPThreadKey.end()) + Key = I->second; + } + if (!Key) { + if (auto KeyOrErr = MP.createPThreadKey()) + Key = *KeyOrErr; + else + return KeyOrErr.takeError(); + } + + uint64_t PlatformKeyBits = + support::endian::byte_swap(*Key, G.getEndianness()); + + // TLS descriptor was define in GOT, iterate the GOT section to find the TLS + // descriptor + auto *GOTSection = G.findSectionByName("$__GOT"); + for (auto *B : GOTSection->blocks()) { + // Blocks defined in GOT and has an edge points to the TLS variable symbol + // is TLS descriptor. + if (!llvm::any_of(B->edges(), isTargetSymbolInThreadSections)) + continue; + + // FIXME: The TLS descriptor byte length may different with different + // ISA + assert(B->getSize() == (G.getPointerSize() * 2) && + "TLS descriptor must be 2 words length"); + auto NewBlockContent = G.allocateBuffer(B->getSize()); + llvm::copy(B->getContent(), NewBlockContent.data()); + memcpy(NewBlockContent.data(), &PlatformKeyBits, G.getPointerSize()); + B->setContent(NewBlockContent); + } + } return Error::success(); } diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp --- a/llvm/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp @@ -84,7 +84,7 @@ if (!SymbolType) return SymbolType.takeError(); - if (*SymbolType & object::SymbolRef::ST_Function) + if (*SymbolType == object::SymbolRef::ST_Function) Flags |= JITSymbolFlags::Callable; return Flags;