Index: include/llvm/ExecutionEngine/ExecutionEngine.h =================================================================== --- include/llvm/ExecutionEngine/ExecutionEngine.h +++ include/llvm/ExecutionEngine/ExecutionEngine.h @@ -133,6 +133,7 @@ std::string *ErrorStr, std::shared_ptr MM, std::shared_ptr SR, + std::shared_ptr TLSSR, std::unique_ptr TM); static ExecutionEngine *(*OrcMCJITReplacementCtor)( @@ -502,6 +503,7 @@ CodeGenOpt::Level OptLevel; std::shared_ptr MemMgr; std::shared_ptr Resolver; + std::shared_ptr TLSResolver; TargetOptions Options; Reloc::Model RelocModel; CodeModel::Model CMModel; @@ -542,6 +544,9 @@ EngineBuilder& setSymbolResolver(std::unique_ptr SR); + EngineBuilder& + setTLSSymbolResolver(std::unique_ptr SR); + /// setErrorStr - Set the error string to write to on error. This option /// defaults to NULL. EngineBuilder &setErrorStr(std::string *e) { Index: include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h =================================================================== --- include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h +++ include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h @@ -40,7 +40,7 @@ public: LinkedObjectSet(RuntimeDyld::MemoryManager &MemMgr, RuntimeDyld::SymbolResolver &Resolver) - : RTDyld(llvm::make_unique(MemMgr, Resolver)), + : RTDyld(llvm::make_unique(MemMgr, Resolver, nullptr)), State(Raw) {} virtual ~LinkedObjectSet() {} Index: include/llvm/ExecutionEngine/RTDyldMemoryManager.h =================================================================== --- include/llvm/ExecutionEngine/RTDyldMemoryManager.h +++ include/llvm/ExecutionEngine/RTDyldMemoryManager.h @@ -23,10 +23,11 @@ namespace llvm { class ExecutionEngine; +class RuntimeDyld; - namespace object { - class ObjectFile; - } +namespace object { + class ObjectFile; +} class MCJITMemoryManager : public RuntimeDyld::MemoryManager { public: Index: include/llvm/ExecutionEngine/RuntimeDyld.h =================================================================== --- include/llvm/ExecutionEngine/RuntimeDyld.h +++ include/llvm/ExecutionEngine/RuntimeDyld.h @@ -53,6 +53,27 @@ uint64_t Address; }; + /// \brief Information about a thread-local named symbol. + class TLSSymbolInfo : public JITSymbolBase { + public: + TLSSymbolInfo() : JITSymbolBase(JITSymbolFlags::None), Data1(0), Data2(0), Data3(0) {} + TLSSymbolInfo(JITSymbolFlags Flags) : JITSymbolBase(Flags), Data1(0), Data2(0), Data3(0) {} + TLSSymbolInfo(uint64_t Data1, uint64_t Data2, uint64_t Data3, + JITSymbolFlags Flags = JITSymbolFlags::None) : + JITSymbolBase(Flags), Data1(Data1), Data2(Data2), Data3(Data3) {} + + uint64_t getFirst() { return Data1; }; + uint64_t getSecond() { return Data2; }; + uint64_t getThird() { return Data3; }; + protected: + // Opaque protected fields for use by the actual RuntimeDyld/SymbolInfo + // subclass to interpret. We add them here to allow us to define a uniform + // interface in the TLSSymbolResolver. + uint64_t Data1; + uint64_t Data2; + uint64_t Data3; + }; + /// \brief Information about the loaded object. class LoadedObjectInfo { friend class RuntimeDyldImpl; @@ -168,8 +189,24 @@ virtual void anchor(); }; + /// \brief TLS Symbol resolution. + /// + /// This is split out from the general symbol resolution mechanism since, in + /// general, TLS symbol resolution is a much harder problem that can depend + /// e.g. on implementation details of the C libraries used by the client. + /// Thus, it makes sense to be able to set the TLS symbol resolution mechanism + /// separately from the regular symbol resolution mechanism. + class TLSSymbolResolver { + public: + virtual ~TLSSymbolResolver() {}; + + /// This method returns the an opaque object describing the variables location + /// in thread-local (e.g. module ID and offset) + virtual TLSSymbolInfo findTLSSymbol(const std::string &Name) = 0; + }; + /// \brief Construct a RuntimeDyld instance. - RuntimeDyld(MemoryManager &MemMgr, SymbolResolver &Resolver); + RuntimeDyld(MemoryManager &MemMgr, SymbolResolver &Resolver, TLSSymbolResolver *TLSResolver); ~RuntimeDyld(); /// Add the referenced object file to the list of objects to be loaded and @@ -225,6 +262,7 @@ std::unique_ptr Dyld; MemoryManager &MemMgr; SymbolResolver &Resolver; + TLSSymbolResolver *TLSResolver; bool ProcessAllSections; RuntimeDyldCheckerImpl *Checker; }; Index: lib/ExecutionEngine/ExecutionEngine.cpp =================================================================== --- lib/ExecutionEngine/ExecutionEngine.cpp +++ lib/ExecutionEngine/ExecutionEngine.cpp @@ -49,6 +49,7 @@ std::unique_ptr M, std::string *ErrorStr, std::shared_ptr MemMgr, std::shared_ptr Resolver, + std::shared_ptr TLSResolver, std::unique_ptr TM) = nullptr; ExecutionEngine *(*ExecutionEngine::OrcMCJITReplacementCtor)( @@ -486,6 +487,12 @@ return *this; } +EngineBuilder& +EngineBuilder::setTLSSymbolResolver(std::unique_ptr SR) { + TLSResolver = std::shared_ptr(std::move(SR)); + return *this; +} + ExecutionEngine *EngineBuilder::create(TargetMachine *TM) { std::unique_ptr TheTM(TM); // Take ownership. @@ -525,7 +532,7 @@ EE->addModule(std::move(M)); } else if (ExecutionEngine::MCJITCtor) EE = ExecutionEngine::MCJITCtor(std::move(M), ErrorStr, std::move(MemMgr), - std::move(Resolver), std::move(TheTM)); + std::move(Resolver), std::move(TLSResolver), std::move(TheTM)); if (EE) { EE->setVerifyModules(VerifyModules); Index: lib/ExecutionEngine/MCJIT/MCJIT.h =================================================================== --- lib/ExecutionEngine/MCJIT/MCJIT.h +++ lib/ExecutionEngine/MCJIT/MCJIT.h @@ -69,7 +69,8 @@ class MCJIT : public ExecutionEngine { MCJIT(std::unique_ptr M, std::unique_ptr tm, std::shared_ptr MemMgr, - std::shared_ptr Resolver); + std::shared_ptr Resolver, + std::shared_ptr TLSResolver); typedef llvm::SmallPtrSet ModulePtrSet; @@ -182,6 +183,7 @@ MCContext *Ctx; std::shared_ptr MemMgr; LinkingSymbolResolver Resolver; + std::shared_ptr TLSResolver; RuntimeDyld Dyld; std::vector EventListeners; @@ -296,6 +298,7 @@ std::string *ErrorStr, std::shared_ptr MemMgr, std::shared_ptr Resolver, + std::shared_ptr TLSResolver, std::unique_ptr TM); // @} Index: lib/ExecutionEngine/MCJIT/MCJIT.cpp =================================================================== --- lib/ExecutionEngine/MCJIT/MCJIT.cpp +++ lib/ExecutionEngine/MCJIT/MCJIT.cpp @@ -47,6 +47,7 @@ std::string *ErrorStr, std::shared_ptr MemMgr, std::shared_ptr Resolver, + std::shared_ptr TLSResolver, std::unique_ptr TM) { // Try to register the program as a source of symbols to resolve against. // @@ -62,15 +63,17 @@ } return new MCJIT(std::move(M), std::move(TM), std::move(MemMgr), - std::move(Resolver)); + std::move(Resolver), std::move(TLSResolver)); } MCJIT::MCJIT(std::unique_ptr M, std::unique_ptr tm, std::shared_ptr MemMgr, - std::shared_ptr Resolver) + std::shared_ptr Resolver, + std::shared_ptr TLSResolver) : ExecutionEngine(std::move(M)), TM(std::move(tm)), Ctx(nullptr), MemMgr(std::move(MemMgr)), Resolver(*this, std::move(Resolver)), - Dyld(*this->MemMgr, this->Resolver), ObjCache(nullptr) { + TLSResolver(std::move(TLSResolver)), + Dyld(*this->MemMgr, this->Resolver, this->TLSResolver.get()), ObjCache(nullptr) { // FIXME: We are managing our modules, so we do not want the base class // ExecutionEngine to manage them as well. To avoid double destruction // of the first (and only) module added in ExecutionEngine constructor Index: lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt =================================================================== --- lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt +++ lib/ExecutionEngine/RuntimeDyld/CMakeLists.txt @@ -5,4 +5,5 @@ RuntimeDyldCOFF.cpp RuntimeDyldELF.cpp RuntimeDyldMachO.cpp + TLSSymbolResolverELF.cpp ) Index: lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp =================================================================== --- lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -83,6 +83,8 @@ MutexGuard locked(lock); // First, resolve relocations associated with external symbols. + if (TLSResolver) + resolveExternalTLSSymbols(); resolveExternalSymbols(); // Just iterate over the sections we have and resolve all the relocations @@ -649,14 +651,21 @@ } void RuntimeDyldImpl::addRelocationForSymbol(const RelocationEntry &RE, - StringRef SymbolName) { + StringRef SymbolName, + bool isTLS) { // Relocation by symbol. If the symbol is found in the global symbol table, // create an appropriate section relocation. Otherwise, add it to // ExternalSymbolRelocations. RTDyldSymbolTable::const_iterator Loc = GlobalSymbolTable.find(SymbolName); if (Loc == GlobalSymbolTable.end()) { - ExternalSymbolRelocations[SymbolName].push_back(RE); + if (isTLS) { + ExternalTLSRelocations[SymbolName].push_back(RE); + } else { + ExternalSymbolRelocations[SymbolName].push_back(RE); + } } else { + assert(!isTLS && "Declaring thread local variables in loaded objects " + "is not yet supported."); // Copy the RE since we want to modify its addend. RelocationEntry RECopy = RE; const auto &SymInfo = Loc->second; @@ -774,6 +783,11 @@ } void RuntimeDyldImpl::resolveExternalSymbols() { + // First give the subclass a chance to resolve any TLS symbols. + // This may require pulling in additional modules, creating new + // external relocations. + if (TLSResolver) + resolveExternalTLSSymbols(); while (!ExternalSymbolRelocations.empty()) { StringMap::iterator i = ExternalSymbolRelocations.begin(); @@ -820,6 +834,11 @@ } ExternalSymbolRelocations.erase(i); + + // Resolving this symbol may have loaded additional modules whose TLS + // Relocations need to be resolved. + if (TLSResolver) + resolveExternalTLSSymbols(); } } @@ -839,8 +858,9 @@ void RuntimeDyld::SymbolResolver::anchor() {} RuntimeDyld::RuntimeDyld(RuntimeDyld::MemoryManager &MemMgr, - RuntimeDyld::SymbolResolver &Resolver) - : MemMgr(MemMgr), Resolver(Resolver) { + RuntimeDyld::SymbolResolver &Resolver, + RuntimeDyld::TLSSymbolResolver *TLSResolver) + : MemMgr(MemMgr), Resolver(Resolver), TLSResolver(TLSResolver) { // FIXME: There's a potential issue lurking here if a single instance of // RuntimeDyld is used to load multiple objects. The current implementation // associates a single memory manager with a RuntimeDyld instance. Even @@ -869,7 +889,13 @@ createRuntimeDyldELF(RuntimeDyld::MemoryManager &MM, RuntimeDyld::SymbolResolver &Resolver, bool ProcessAllSections, RuntimeDyldCheckerImpl *Checker) { - std::unique_ptr Dyld(new RuntimeDyldELF(MM, Resolver)); + RuntimeDyld::TLSSymbolResolver *TLSResolver; + #ifdef __GLIBC__ + TLSResolver = new TLSSymbolResolverGLibCELF(&Resolver); + #elif defined(__APPLE__) + TLSResolver = new TLSSymbolResolverDarwinELF(&Resolver); + #endif + std::unique_ptr Dyld(new RuntimeDyldELF(MM, Resolver, TLSResolver)); Dyld->setProcessAllSections(ProcessAllSections); Dyld->setRuntimeDyldChecker(Checker); return Dyld; Index: lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h =================================================================== --- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h +++ lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h @@ -39,7 +39,7 @@ protected: RuntimeDyldCOFF(RuntimeDyld::MemoryManager &MemMgr, RuntimeDyld::SymbolResolver &Resolver) - : RuntimeDyldImpl(MemMgr, Resolver) {} + : RuntimeDyldImpl(MemMgr, Resolver, nullptr) {} uint64_t getSymbolOffset(const SymbolRef &Sym); }; Index: lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp =================================================================== --- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp +++ lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp @@ -34,6 +34,10 @@ getObjectForDebug(const ObjectFile &Obj) const override { return OwningBinary(); } + + RuntimeDyld::LoadedObjectInfo *clone() const + { return new LoadedCOFFObjectInfo(*this); } + }; } Index: lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h =================================================================== --- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h +++ lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h @@ -22,15 +22,51 @@ namespace llvm { class RuntimeDyldELF : public RuntimeDyldImpl { - +public: + // Wraps a RuntimeDyld::TLSSymbolInfo with accessors to easily access the ELF specific interpretation + class TLSSymbolInfoELF { + public: + struct ModuleInfo { + uint64_t ModuleID; + int64_t tpoff; + ModuleInfo(uint64_t ModuleID, int64_t tpoff) : + ModuleID(ModuleID), tpoff(tpoff) {} + }; + + TLSSymbolInfoELF(ModuleInfo Mod, uint64_t Offset, JITSymbolFlags Flags = JITSymbolFlags::None) : + Data(Mod.ModuleID,Offset,Mod.tpoff,Flags) {}; + TLSSymbolInfoELF(RuntimeDyld::TLSSymbolInfo Data) : Data(Data) {}; + + ModuleInfo getModInfo() { return ModuleInfo(Data.getFirst(), Data.getThird()); } + uint64_t getOffset() { return Data.getSecond(); } + RuntimeDyld::TLSSymbolInfo getOpaque() { return Data; } + + private: + RuntimeDyld::TLSSymbolInfo Data; + }; + + class TLSSymbolResolverELF : public RuntimeDyld::TLSSymbolResolver { + public: + virtual int64_t ExtraGOTAddend() { return 0; }; + virtual uint64_t GetAddrOverride() { return 0; }; + TLSSymbolInfoELF findTLSSymbolELF(const std::string &Name) { return TLSSymbolInfoELF(findTLSSymbol(Name)); } + }; + +private: void resolveRelocation(const SectionEntry &Section, uint64_t Offset, uint64_t Value, uint32_t Type, int64_t Addend, uint64_t SymOffset = 0); + void resolveRelocationTLS(const RelocationEntry &RE, + TLSSymbolInfoELF Value); + void resolveX86_64Relocation(const SectionEntry &Section, uint64_t Offset, uint64_t Value, uint32_t Type, int64_t Addend, uint64_t SymOffset); + void resolveX86_64RelocationTLS(const SectionEntry &Section, uint64_t Offset, + TLSSymbolInfoELF Value, uint32_t Type); + void resolveX86Relocation(const SectionEntry &Section, uint64_t Offset, uint32_t Value, uint32_t Type, int32_t Addend); @@ -126,12 +162,15 @@ public: RuntimeDyldELF(RuntimeDyld::MemoryManager &MemMgr, - RuntimeDyld::SymbolResolver &Resolver); + RuntimeDyld::SymbolResolver &Resolver, + RuntimeDyld::TLSSymbolResolver *TLSResolver); virtual ~RuntimeDyldELF(); std::unique_ptr loadObject(const object::ObjectFile &O) override; + void resolveExternalTLSSymbols() override; + void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override; relocation_iterator processRelocationRef(unsigned SectionID, relocation_iterator RelI, @@ -145,6 +184,32 @@ ObjSectionToIDMap &SectionMap) override; }; +// Useful out-of-the box TLS symbol resolver implementations for +// libc (those using the glibc ABI) and ELF-on-Darwin + +// TLS as implemented by glibc +class TLSSymbolResolverGLibCELF : public RuntimeDyldELF::TLSSymbolResolverELF { +private: + RuntimeDyld::SymbolResolver *SR; +public: + TLSSymbolResolverGLibCELF(RuntimeDyld::SymbolResolver *SR) : TLSSymbolResolverELF(), SR(SR) {}; + RuntimeDyld::TLSSymbolInfo findTLSSymbol(const std::string &Name) override; +}; + +// Support ELF-on-Darwin by matching ELF's TLS model onto that used by OS X +class TLSSymbolResolverDarwinELF : public RuntimeDyldELF::TLSSymbolResolverELF { +private: + RuntimeDyld::SymbolResolver *SR; + uint64_t tlv_get_addr_addr; +public: + TLSSymbolResolverDarwinELF(RuntimeDyld::SymbolResolver *SR) : TLSSymbolResolverELF(), SR(SR), + tlv_get_addr_addr(0) {}; + + int64_t ExtraGOTAddend() override { return -8; }; + uint64_t GetAddrOverride() override { return tlv_get_addr_addr; }; + RuntimeDyld::TLSSymbolInfo findTLSSymbol(const std::string &Name) override; +}; + } // end namespace llvm #endif Index: lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp =================================================================== --- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -20,7 +20,6 @@ #include "llvm/MC/MCStreamer.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Object/ObjectFile.h" -#include "llvm/Support/ELF.h" #include "llvm/Support/Endian.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/TargetRegistry.h" @@ -185,8 +184,9 @@ namespace llvm { RuntimeDyldELF::RuntimeDyldELF(RuntimeDyld::MemoryManager &MemMgr, - RuntimeDyld::SymbolResolver &Resolver) - : RuntimeDyldImpl(MemMgr, Resolver), GOTSectionID(0), CurrentGOTIndex(0) {} + RuntimeDyld::SymbolResolver &Resolver, + RuntimeDyld::TLSSymbolResolver *TLSResolver) + : RuntimeDyldImpl(MemMgr, Resolver, TLSResolver), GOTSectionID(0), CurrentGOTIndex(0) {} RuntimeDyldELF::~RuntimeDyldELF() {} void RuntimeDyldELF::registerEHFrames() { @@ -276,6 +276,26 @@ } } +void RuntimeDyldELF::resolveX86_64RelocationTLS(const SectionEntry &Section, uint64_t Offset, + RuntimeDyldELF::TLSSymbolInfoELF Value, + uint32_t Type) +{ + switch (Type) { + default: + llvm_unreachable("TLS Relocation type not implemented yet!"); + break; + case ELF::R_X86_64_DTPMOD64: + support::ulittle64_t::ref(Section.Address + Offset) = Value.getModInfo().ModuleID; + break; + case ELF::R_X86_64_TPOFF64: + support::ulittle64_t::ref(Section.Address + Offset) = Value.getModInfo().tpoff + Value.getOffset(); + break; + case ELF::R_X86_64_DTPOFF64: + support::ulittle64_t::ref(Section.Address + Offset) = Value.getOffset(); + break; + } +} + void RuntimeDyldELF::resolveX86Relocation(const SectionEntry &Section, uint64_t Offset, uint32_t Value, uint32_t Type, int32_t Addend) { @@ -888,6 +908,54 @@ } } +void RuntimeDyldELF::resolveRelocationTLS(const RelocationEntry &RE, + RuntimeDyldELF::TLSSymbolInfoELF Value) +{ + const SectionEntry &Section = Sections[RE.SectionID]; + switch (Arch) { + case Triple::x86_64: + resolveX86_64RelocationTLS(Section,RE.Offset,Value,RE.RelType); + break; + default: + llvm_unreachable("TLS Support not implemented for this CPU type"); + break; + } +} + +void RuntimeDyldELF::resolveExternalTLSSymbols() +{ + TLSSymbolResolverELF *SR = (TLSSymbolResolverELF *)TLSResolver; + + while (!ExternalTLSRelocations.empty()) { + StringMap::iterator i = ExternalTLSRelocations.begin(); + StringRef Name = i->first(); + RelocationList &Relocs = i->second; + assert(Name.size() != 0 && "TLS relocations should not be absolute."); + + TLSSymbolInfoELF Value = SR->findTLSSymbolELF(Name); + for (unsigned idx = 0, e = Relocs.size(); idx != e; ++idx) { + const RelocationEntry &RE = Relocs[idx]; + // Ignore relocations for sections that were not loaded + if (Sections[RE.SectionID].Address == nullptr) + continue; + resolveRelocationTLS(RE, Value); + } + + ExternalTLSRelocations.erase(i); + } + + // Some implementations may want to provide a custom get_addr function. + // Do the override here. + uint64_t AddrOverride = SR->GetAddrOverride(); + if (AddrOverride != 0) { + auto i = ExternalSymbolRelocations.find("__tls_get_addr"); + if (i != ExternalSymbolRelocations.end()) { + resolveRelocationList(i->second, AddrOverride); + ExternalSymbolRelocations.erase(i); + } + } +} + relocation_iterator RuntimeDyldELF::processRelocationRef( unsigned SectionID, relocation_iterator RelI, const ObjectFile &Obj, @@ -957,6 +1025,7 @@ uint64_t Offset; Check(RelI->getOffset(Offset)); + TLSSymbolResolverELF *TLSSR = (TLSSymbolResolverELF *)TLSResolver; DEBUG(dbgs() << "\t\tSectionID: " << SectionID << " Offset: " << Offset << "\n"); @@ -1277,76 +1346,109 @@ Addend); else resolveRelocation(Section, Offset, StubAddress, RelType, Addend); - } else if (Arch == Triple::x86_64 && RelType == ELF::R_X86_64_PLT32) { - // The way the PLT relocations normally work is that the linker allocates - // the - // PLT and this relocation makes a PC-relative call into the PLT. The PLT - // entry will then jump to an address provided by the GOT. On first call, - // the - // GOT address will point back into PLT code that resolves the symbol. After - // the first call, the GOT entry points to the actual function. - // - // For local functions we're ignoring all of that here and just replacing - // the PLT32 relocation type with PC32, which will translate the relocation - // into a PC-relative call directly to the function. For external symbols we - // can't be sure the function will be within 2^32 bytes of the call site, so - // we need to create a stub, which calls into the GOT. This case is - // equivalent to the usual PLT implementation except that we use the stub - // mechanism in RuntimeDyld (which puts stubs at the end of the section) - // rather than allocating a PLT section. - if (Value.SymbolName) { - // This is a call to an external function. - // Look for an existing stub. - SectionEntry &Section = Sections[SectionID]; - StubMap::const_iterator i = Stubs.find(Value); - uintptr_t StubAddress; - if (i != Stubs.end()) { - StubAddress = uintptr_t(Section.Address) + i->second; - DEBUG(dbgs() << " Stub function found\n"); - } else { - // Create a new stub function (equivalent to a PLT entry). - DEBUG(dbgs() << " Create a new stub function\n"); + } else if (Arch == Triple::x86_64) { + if (RelType == ELF::R_X86_64_PLT32) { + // The way the PLT relocations normally work is that the linker allocates + // the + // PLT and this relocation makes a PC-relative call into the PLT. The PLT + // entry will then jump to an address provided by the GOT. On first call, + // the + // GOT address will point back into PLT code that resolves the symbol. After + // the first call, the GOT entry points to the actual function. + // + // For local functions we're ignoring all of that here and just replacing + // the PLT32 relocation type with PC32, which will translate the relocation + // into a PC-relative call directly to the function. For external symbols we + // can't be sure the function will be within 2^32 bytes of the call site, so + // we need to create a stub, which calls into the GOT. This case is + // equivalent to the usual PLT implementation except that we use the stub + // mechanism in RuntimeDyld (which puts stubs at the end of the section) + // rather than allocating a PLT section. + if (Value.SymbolName) { + // This is a call to an external function. + // Look for an existing stub. + SectionEntry &Section = Sections[SectionID]; + StubMap::const_iterator i = Stubs.find(Value); + uintptr_t StubAddress; + if (i != Stubs.end()) { + StubAddress = uintptr_t(Section.Address) + i->second; + DEBUG(dbgs() << " Stub function found\n"); + } else { + // Create a new stub function (equivalent to a PLT entry). + DEBUG(dbgs() << " Create a new stub function\n"); - uintptr_t BaseAddress = uintptr_t(Section.Address); - uintptr_t StubAlignment = getStubAlignment(); - StubAddress = (BaseAddress + Section.StubOffset + StubAlignment - 1) & - -StubAlignment; - unsigned StubOffset = StubAddress - BaseAddress; - Stubs[Value] = StubOffset; - createStubFunction((uint8_t *)StubAddress); + uintptr_t BaseAddress = uintptr_t(Section.Address); + uintptr_t StubAlignment = getStubAlignment(); + StubAddress = (BaseAddress + Section.StubOffset + StubAlignment - 1) & + -StubAlignment; + unsigned StubOffset = StubAddress - BaseAddress; + Stubs[Value] = StubOffset; + createStubFunction((uint8_t *)StubAddress); - // Bump our stub offset counter - Section.StubOffset = StubOffset + getMaxStubFunctionSize(); + // Bump our stub offset counter + Section.StubOffset = StubOffset + getMaxStubFunctionSize(); - // Allocate a GOT Entry - uint64_t GOTOffset = allocateGOTEntries(SectionID, 1); + uint64_t GOTOffset = allocateGOTEntries(SectionID, 1); - // The load of the GOT address has an addend of -4 - resolveGOTOffsetRelocation(SectionID, StubOffset + 2, GOTOffset - 4); + resolveGOTOffsetRelocation(SectionID, StubOffset + 2, GOTOffset - 4); - // Fill in the value of the symbol we're targeting into the GOT - addRelocationForSymbol(computeGOTOffsetRE(SectionID,GOTOffset,0,ELF::R_X86_64_64), + // Fill in the value of the symbol we're targeting into the GOT + addRelocationForSymbol(computeGOTOffsetRE(SectionID,GOTOffset,0,ELF::R_X86_64_64), Value.SymbolName); + } + + // Make the target call a call into the stub table. + resolveRelocation(Section, Offset, StubAddress, ELF::R_X86_64_PC32, + Addend); + } else { + RelocationEntry RE(SectionID, Offset, ELF::R_X86_64_PC32, Value.Addend, + Value.Offset); + addRelocationForSection(RE, Value.SectionID); } + } else if (RelType == ELF::R_X86_64_GOTPCREL) { + uint64_t GOTOffset = allocateGOTEntries(SectionID, 1); + resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset + Addend); - // Make the target call a call into the stub table. - resolveRelocation(Section, Offset, StubAddress, ELF::R_X86_64_PC32, - Addend); + // Fill in the value of the symbol we're targeting into the GOT + RelocationEntry RE = computeGOTOffsetRE(SectionID, GOTOffset, Value.Offset, ELF::R_X86_64_64); + if (Value.SymbolName) + addRelocationForSymbol(RE, Value.SymbolName); + else + addRelocationForSection(RE, Value.SectionID); + } else if (RelType == ELF::R_X86_64_TLSGD) { + uint64_t GOTOffset = allocateGOTEntries(SectionID, 2); + resolveGOTOffsetRelocation(SectionID, Offset, + GOTOffset + Addend + TLSSR->ExtraGOTAddend()); + + RelocationEntry REMOD = computeGOTOffsetRE(SectionID, GOTOffset, Value.Offset, ELF::R_X86_64_DTPMOD64); + RelocationEntry REOFF = computeGOTOffsetRE(SectionID, GOTOffset + 8, Value.Offset, ELF::R_X86_64_DTPOFF64); + if (Value.SymbolName) { + addRelocationForSymbol(REMOD, Value.SymbolName, true); + addRelocationForSymbol(REOFF, Value.SymbolName, true); + } + else { + addRelocationForSection(REMOD, Value.SectionID); + addRelocationForSection(REOFF, Value.SectionID); + } + } else if (RelType == ELF::R_X86_64_GOTTPOFF) { + uint64_t GOTOffset = allocateGOTEntries(SectionID, 1); + resolveGOTOffsetRelocation(SectionID, Offset, + GOTOffset + Addend + TLSSR->ExtraGOTAddend()); + + RelocationEntry REOFF = computeGOTOffsetRE(SectionID, GOTOffset, Value.Offset, ELF::R_X86_64_TPOFF64); + if (Value.SymbolName) { + addRelocationForSymbol(REOFF, Value.SymbolName, true); + } + else { + addRelocationForSection(REOFF, Value.SectionID); + } } else { - RelocationEntry RE(SectionID, Offset, ELF::R_X86_64_PC32, Value.Addend, - Value.Offset); - addRelocationForSection(RE, Value.SectionID); + RelocationEntry RE(SectionID, Offset, RelType, Value.Addend, Value.Offset); + if (Value.SymbolName) + addRelocationForSymbol(RE, Value.SymbolName); + else + addRelocationForSection(RE, Value.SectionID); } - } else if (Arch == Triple::x86_64 && RelType == ELF::R_X86_64_GOTPCREL) { - uint64_t GOTOffset = allocateGOTEntries(SectionID, 1); - resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset + Addend); - - // Fill in the value of the symbol we're targeting into the GOT - RelocationEntry RE = computeGOTOffsetRE(SectionID, GOTOffset, Value.Offset, ELF::R_X86_64_64); - if (Value.SymbolName) - addRelocationForSymbol(RE, Value.SymbolName); - else - addRelocationForSection(RE, Value.SectionID); } else { RelocationEntry RE(SectionID, Offset, RelType, Value.Addend, Value.Offset); if (Value.SymbolName) Index: lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h =================================================================== --- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -194,6 +194,9 @@ // The symbol resolver to use for external symbols. RuntimeDyld::SymbolResolver &Resolver; + // The symbol resolver to use for external TLS symbols. + RuntimeDyld::TLSSymbolResolver *TLSResolver; + // Attached RuntimeDyldChecker instance. Null if no instance attached. RuntimeDyldCheckerImpl *Checker; @@ -231,6 +234,10 @@ // modules. This map is indexed by symbol name. StringMap ExternalSymbolRelocations; + // We separate out external relocations to TLS symbols so we can process + // them all in one go. + StringMap ExternalTLSRelocations; + typedef std::map StubMap; @@ -335,7 +342,10 @@ // \brief Add a relocation entry that uses the given symbol. This symbol may // be found in the global symbol table, or it may be external. - void addRelocationForSymbol(const RelocationEntry &RE, StringRef SymbolName); + // Also indicate whether processing this relocation will require dealing + // with thread local storage. + void addRelocationForSymbol(const RelocationEntry &RE, StringRef SymbolName, + bool isTLS = false); /// \brief Emits long jump instruction to Addr. /// \return Pointer to the memory area for emitting target address. @@ -361,6 +371,13 @@ /// \brief Resolve relocations to external symbols. void resolveExternalSymbols(); + /// \brief Resolve TLS relocations to external symbols + /// + /// This is split out from resolveExternalSymbols, because unlike regular symbols + /// TLS symbols do not require just an address but have an object-file specific + /// representation. + virtual void resolveExternalTLSSymbols() {}; + // \brief Compute an upper bound of the memory that is required to load all // sections void computeTotalAllocSize(const ObjectFile &Obj, uint64_t &CodeSize, @@ -375,8 +392,9 @@ public: RuntimeDyldImpl(RuntimeDyld::MemoryManager &MemMgr, - RuntimeDyld::SymbolResolver &Resolver) - : MemMgr(MemMgr), Resolver(Resolver), Checker(nullptr), + RuntimeDyld::SymbolResolver &Resolver, + RuntimeDyld::TLSSymbolResolver *TLSResolver) + : MemMgr(MemMgr), Resolver(Resolver), TLSResolver(TLSResolver), Checker(nullptr), ProcessAllSections(false), HasError(false) { } Index: lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h =================================================================== --- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h +++ lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h @@ -51,7 +51,7 @@ RuntimeDyldMachO(RuntimeDyld::MemoryManager &MemMgr, RuntimeDyld::SymbolResolver &Resolver) - : RuntimeDyldImpl(MemMgr, Resolver) {} + : RuntimeDyldImpl(MemMgr, Resolver, nullptr) {} /// This convenience method uses memcpy to extract a contiguous addend (the /// addend size and offset are taken from the corresponding fields of the RE). Index: lib/ExecutionEngine/RuntimeDyld/TLSSymbolResolverELF.cpp =================================================================== --- /dev/null +++ lib/ExecutionEngine/RuntimeDyld/TLSSymbolResolverELF.cpp @@ -0,0 +1,111 @@ +//===-- TLSSymbolResolverELF.cpp ----------------------------=---*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "RuntimeDyldELF.h" + +namespace llvm { + +// The purpose of these classes is to support the implementation +// specific details of Thread Local Storage, which can differ +// between operating systems platforms and even C libraries. + +#ifdef __GLIBC__ + +#include "pthread.h" + +// The DTV as specified in the ELF ABI +typedef union dtv { + size_t counter; + struct { + void *val; + bool _; + } pointer; +} dtv_t; + +typedef struct tcb_t { + struct tcb *self; + dtv_t *dtv; +} tcb_t; + +RuntimeDyld::TLSSymbolInfo +TLSSymbolResolverGLibCELF::findTLSSymbol(const std::string &Name) { + // It would be lovely to have an API for this. If we + // wanted to, we might be able to actually look at the + // internal libc datastructures, but that seems risky if + // they were to change between versions. This implementation + // is sketchy because it just searches through all the modules + // and sees which one is closest, but at least it only relies on + // the libc ABI, which should be stable. + const void *UnallocatedDTVSlot = (void *)-1l; + + // First ask the MemoryManager to dlsym this value for us + RuntimeDyld::SymbolInfo SI = SR->findSymbol(Name); + uint64_t Value = SI.getAddress(); + + // This is glibc specifc but followed by a number of other C libraries + tcb_t *tcb = (tcb_t *)pthread_self(); + dtv_t *dtv = tcb->dtv; + + // The number of allocated entries in the DTV is specified as the value + // of dtv[-1] + size_t cnt = dtv[-1].counter; + + // Find the module whose start block for the current thread is closest + // to the dlsym'd address. Ugly, but works. + uint64_t min_distance = (uint64_t)-1; + uint64_t found_i = 0, found_offset = 0; + for (size_t i = 1; i < cnt; ++i) { + uint64_t distance = (Value - (uint64_t)dtv[i].pointer.val); + if (dtv[i].pointer.val == UnallocatedDTVSlot) { + continue; + } else if ((uint64_t)dtv[i].pointer.val > Value) { + continue; + } else if (distance < min_distance) { + min_distance = distance; + found_i = i; + found_offset = distance; + } + } + assert(found_i != 0 && "Value could not be found in thread local storage"); + + RuntimeDyldELF::TLSSymbolInfoELF::ModuleInfo mod(found_i, + (uint64_t)dtv[found_i].pointer.val - (uint64_t)tcb); + return RuntimeDyldELF::TLSSymbolInfoELF(mod, found_offset, SI.getFlags()).getOpaque(); +} + +#endif + +#ifdef __APPLE__ + +typedef struct +{ + void* tlv_get_addr; + unsigned long key; + unsigned long offset; +} TLVDescriptor; + +// Support ELF-on-Darwin by matching ELF's TLS model onto that used by OS X +RuntimeDyld::TLSSymbolInfo +TLSSymbolResolverDarwinELF::findTLSSymbol(const std::string &Name) { + // Ask the MemoryManager to dlsym this value for us. + // The result will be a pointer to the TLVDescriptor. + RuntimeDyld::SymbolInfo SI = SR->findSymbol(Name); + TLVDescriptor *Descriptor = (TLVDescriptor *)SI.getAddress(); + if (tlv_get_addr_addr == 0) { + tlv_get_addr_addr = (uint64_t)Descriptor->tlv_get_addr; + } else { + assert(tlv_get_addr_addr == (uint64_t)Descriptor->tlv_get_addr && + "Multiple values for __tlv_get_addr not supported"); + } + return RuntimeDyldELF::TLSSymbolInfoELF(Descriptor->key, Descriptor->offset, SI.getFlags()).getOpaque(); +}; + +#endif + +} Index: lib/MC/MCObjectFileInfo.cpp =================================================================== --- lib/MC/MCObjectFileInfo.cpp +++ lib/MC/MCObjectFileInfo.cpp @@ -753,7 +753,7 @@ Arch == Triple::aarch64 || Arch == Triple::ppc || Arch == Triple::ppc64 || Arch == Triple::UnknownArch) && - (TT.isOSDarwin() || TT.isOSBinFormatMachO())) { + TT.isOSBinFormatMachO()) { Env = IsMachO; InitMachOMCObjectFileInfo(TT); } else if ((Arch == Triple::x86 || Arch == Triple::x86_64 || Index: tools/llvm-rtdyld/llvm-rtdyld.cpp =================================================================== --- tools/llvm-rtdyld/llvm-rtdyld.cpp +++ tools/llvm-rtdyld/llvm-rtdyld.cpp @@ -197,7 +197,7 @@ for(unsigned i = 0, e = InputFileList.size(); i != e; ++i) { // Instantiate a dynamic linker. TrivialMemoryManager MemMgr; - RuntimeDyld Dyld(MemMgr, MemMgr); + RuntimeDyld Dyld(MemMgr, MemMgr, nullptr); // Load the input memory buffer. @@ -265,7 +265,7 @@ // Instantiate a dynamic linker. TrivialMemoryManager MemMgr; - RuntimeDyld Dyld(MemMgr, MemMgr); + RuntimeDyld Dyld(MemMgr, MemMgr, nullptr); // If we don't have any input files, read from stdin. if (!InputFileList.size()) @@ -514,7 +514,7 @@ // Instantiate a dynamic linker. TrivialMemoryManager MemMgr; - RuntimeDyld Dyld(MemMgr, MemMgr); + RuntimeDyld Dyld(MemMgr, MemMgr, nullptr); Dyld.setProcessAllSections(true); RuntimeDyldChecker Checker(Dyld, Disassembler.get(), InstPrinter.get(), llvm::dbgs());