diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h --- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h +++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h @@ -49,7 +49,7 @@ std::make_unique(std::move(JTMB))), DL(std::move(DL)), Mangle(ES, this->DL), Ctx(std::make_unique()), - MainJD(ES.createJITDylib("
")) { + MainJD(ES.createBareJITDylib("
")) { MainJD.addGenerator( cantFail(DynamicLibrarySearchGenerator::GetForCurrentProcess( DL.getGlobalPrefix()))); diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h --- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h +++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h @@ -55,7 +55,7 @@ std::make_unique(std::move(JTMB))), OptimizeLayer(ES, CompileLayer, optimizeModule), DL(std::move(DL)), Mangle(ES, this->DL), Ctx(std::make_unique()), - MainJD(ES.createJITDylib("
")) { + MainJD(ES.createBareJITDylib("
")) { MainJD.addGenerator( cantFail(DynamicLibrarySearchGenerator::GetForCurrentProcess( DL.getGlobalPrefix()))); diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -488,6 +488,8 @@ /// Set the visibility for this Symbol. void setScope(Scope S) { + assert((!Name.empty() || S == Scope::Local) && + "Can not set anonymous symbol to non-local scope"); assert((S == Scope::Default || Base->isDefined() || Base->isAbsolute()) && "Invalid visibility for symbol type"); this->S = static_cast(S); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h --- a/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -94,6 +94,7 @@ /// Sets the ImplSymbolMap void setImplMap(ImplSymbolMap *Imp); + /// Emits the given module. This should not be called by clients: it will be /// called by the JIT when a definition added via the add method is requested. void emit(MaterializationResponsibility R, ThreadSafeModule TSM) override; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h --- a/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/CompileUtils.h @@ -30,7 +30,7 @@ class JITTargetMachineBuilder; -IRMaterializationUnit::ManglingOptions +IRSymbolMapper::ManglingOptions irManglingOptionsFromTargetOptions(const TargetOptions &Opts); /// Simple compile functor: Takes a single IR module and returns an ObjectFile. @@ -52,7 +52,7 @@ Expected operator()(Module &M) override; private: - IRMaterializationUnit::ManglingOptions + IRSymbolMapper::ManglingOptions manglingOptionsForTargetMachine(const TargetMachine &TM); CompileResult tryToLoadFromObjectCache(const Module &M); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h --- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h @@ -14,11 +14,11 @@ #define LLVM_EXECUTIONENGINE_ORC_CORE_H #include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/FunctionExtras.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h" #include "llvm/ExecutionEngine/OrcV1Deprecation.h" -#include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include @@ -456,6 +456,11 @@ /// before using. const SymbolFlagsMap &getSymbols() const { return SymbolFlags; } + /// Returns the initialization pseudo-symbol, if any. This symbol will also + /// be present in the SymbolFlagsMap for this MaterializationResponsibility + /// object. + const SymbolStringPtr &getInitializerSymbol() const { return InitSymbol; } + /// Returns the names of any symbols covered by this /// MaterializationResponsibility object that have queries pending. This /// information can be used to return responsibility for unrequested symbols @@ -532,10 +537,15 @@ /// Create a MaterializationResponsibility for the given JITDylib and /// initial symbols. MaterializationResponsibility(JITDylib &JD, SymbolFlagsMap SymbolFlags, - VModuleKey K); + SymbolStringPtr InitSymbol, VModuleKey K) + : JD(JD), SymbolFlags(std::move(SymbolFlags)), + InitSymbol(std::move(InitSymbol)), K(std::move(K)) { + assert(!this->SymbolFlags.empty() && "Materializing nothing?"); + } JITDylib &JD; SymbolFlagsMap SymbolFlags; + SymbolStringPtr InitSymbol; VModuleKey K; }; @@ -549,8 +559,13 @@ /// stronger definition is added or already present. class MaterializationUnit { public: - MaterializationUnit(SymbolFlagsMap InitalSymbolFlags, VModuleKey K) - : SymbolFlags(std::move(InitalSymbolFlags)), K(std::move(K)) {} + MaterializationUnit(SymbolFlagsMap InitalSymbolFlags, + SymbolStringPtr InitSymbol, VModuleKey K) + : SymbolFlags(std::move(InitalSymbolFlags)), + InitSymbol(std::move(InitSymbol)), K(std::move(K)) { + assert((!this->InitSymbol || this->SymbolFlags.count(this->InitSymbol)) && + "If set, InitSymbol should appear in InitialSymbolFlags map"); + } virtual ~MaterializationUnit() {} @@ -561,12 +576,15 @@ /// Return the set of symbols that this source provides. const SymbolFlagsMap &getSymbols() const { return SymbolFlags; } + /// Returns the initialization symbol for this MaterializationUnit (if any). + const SymbolStringPtr &getInitializerSymbol() const { return InitSymbol; } + /// Called by materialization dispatchers (see /// ExecutionSession::DispatchMaterializationFunction) to trigger /// materialization of this MaterializationUnit. void doMaterialize(JITDylib &JD) { - materialize(MaterializationResponsibility(JD, std::move(SymbolFlags), - std::move(K))); + materialize(MaterializationResponsibility( + JD, std::move(SymbolFlags), std::move(InitSymbol), std::move(K))); } /// Called by JITDylibs to notify MaterializationUnits that the given symbol @@ -578,6 +596,7 @@ protected: SymbolFlagsMap SymbolFlags; + SymbolStringPtr InitSymbol; VModuleKey K; private: @@ -697,12 +716,12 @@ /// Represents the state that a symbol has reached during materialization. enum class SymbolState : uint8_t { - Invalid, /// No symbol should be in this state. - NeverSearched, /// Added to the symbol table, never queried. - Materializing, /// Queried, materialization begun. - Resolved, /// Assigned address, still materializing. - Emitted, /// Emitted to memory, but waiting on transitive dependencies. - Ready = 0x3f /// Ready and safe for clients to access. + Invalid, /// No symbol should be in this state. + NeverSearched, /// Added to the symbol table, never queried. + Materializing, /// Queried, materialization begun. + Resolved, /// Assigned address, still materializing. + Emitted, /// Emitted to memory, but waiting on transitive deps. + Ready /// Symbol and transitive dependencies emitted. }; /// A symbol query that returns results via a callback when results are @@ -774,6 +793,7 @@ class JITDylib { friend class AsynchronousSymbolQuery; friend class ExecutionSession; + friend class Platform; friend class MaterializationResponsibility; public: /// Definition generators can be attached to JITDylibs to generate new @@ -1054,6 +1074,36 @@ JITDylibSearchOrder SearchOrder; }; +/// Platforms set up standard symbols and mediate interactions between dynamic +/// initializers (e.g. C++ static constructors) and ExecutionSession state. +/// Note that Platforms do not automatically run initializers: clients are still +/// responsible for doing this. +class Platform { +public: + virtual ~Platform(); + + /// This method will be called outside the session lock each time a JITDylib + /// is created (unless it is created with EmptyJITDylib set) to allow the + /// Platform to install any JITDylib specific standard symbols (e.g + /// __dso_handle). + virtual Error setupJITDylib(JITDylib &JD) = 0; + + /// This method will be called under the ExecutionSession lock each time a + /// MaterializationUnit is added to a JITDylib. + virtual Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) = 0; + + /// This method will be called under the ExecutionSession lock when a + /// VModuleKey is removed. + virtual Error notifyRemoving(JITDylib &JD, VModuleKey K) = 0; + + /// A utility function for looking up initializer symbols. Performs a blocking + /// lookup for the given symbols in each of the given JITDylibs. + static Expected> + lookupInitSymbols(ExecutionSession &ES, + const DenseMap &InitSyms); + +}; + /// An ExecutionSession represents a running JIT program. class ExecutionSession { // FIXME: Remove this when we remove the old ORC layers. @@ -1078,6 +1128,13 @@ /// Returns a shared_ptr to the SymbolStringPool for this ExecutionSession. std::shared_ptr getSymbolStringPool() const { return SSP; } + /// Set the Platform for this ExecutionSession. + void setPlatform(std::unique_ptr P) { this->P = std::move(P); } + + /// Get the Platform for this session. + /// Will return null if no Platform has been set for this ExecutionSession. + Platform *getPlatform() { return P.get(); } + /// Run the given lambda with the session mutex locked. template decltype(auto) runSessionLocked(Func &&F) { std::lock_guard Lock(SessionMutex); @@ -1088,12 +1145,26 @@ /// Ownership of JITDylib remains within Execution Session JITDylib *getJITDylibByName(StringRef Name); + /// Add a new bare JITDylib to this ExecutionSession. + /// + /// The JITDylib Name is required to be unique. Clients should verify that + /// names are not being re-used (E.g. by calling getJITDylibByName) if names + /// are based on user input. + /// + /// This call does not install any library code or symbols into the newly + /// created JITDylib. The client is responsible for all configuration. + JITDylib &createBareJITDylib(std::string Name); + /// Add a new JITDylib to this ExecutionSession. /// /// The JITDylib Name is required to be unique. Clients should verify that /// names are not being re-used (e.g. by calling getJITDylibByName) if names /// are based on user input. - JITDylib &createJITDylib(std::string Name); + /// + /// If a Platform is attached then Platform::setupJITDylib will be called to + /// install standard platform symbols (e.g. standard library interposes). + /// If no Platform is attached this call is equivalent to createBareJITDylib. + Expected createJITDylib(std::string Name); /// Allocate a module key for a new module to add to the JIT. VModuleKey allocateVModule() { @@ -1167,30 +1238,33 @@ /// or an error occurs. If WaitUntilReady is false and an error occurs /// after resolution, the function will return a success value, but the /// error will be reported via reportErrors. - Expected lookup(const JITDylibSearchOrder &SearchOrder, - const SymbolLookupSet &Symbols, - LookupKind K = LookupKind::Static, - SymbolState RequiredState = SymbolState::Ready, - RegisterDependenciesFunction RegisterDependencies = - NoDependenciesToRegister); + Expected + lookup(const JITDylibSearchOrder &SearchOrder, const SymbolLookupSet &Symbols, + LookupKind K = LookupKind::Static, + SymbolState RequiredState = SymbolState::Ready, + RegisterDependenciesFunction RegisterDependencies = + NoDependenciesToRegister); /// Convenience version of blocking lookup. /// Searches each of the JITDylibs in the search order in turn for the given /// symbol. - Expected lookup(const JITDylibSearchOrder &SearchOrder, - SymbolStringPtr Symbol); + Expected + lookup(const JITDylibSearchOrder &SearchOrder, SymbolStringPtr Symbol, + SymbolState RequiredState = SymbolState::Ready); /// Convenience version of blocking lookup. /// Searches each of the JITDylibs in the search order in turn for the given /// symbol. The search will not find non-exported symbols. - Expected lookup(ArrayRef SearchOrder, - SymbolStringPtr Symbol); + Expected + lookup(ArrayRef SearchOrder, SymbolStringPtr Symbol, + SymbolState RequiredState = SymbolState::Ready); /// Convenience version of blocking lookup. /// Searches each of the JITDylibs in the search order in turn for the given /// symbol. The search will not find non-exported symbols. - Expected lookup(ArrayRef SearchOrder, - StringRef Symbol); + Expected + lookup(ArrayRef SearchOrder, StringRef Symbol, + SymbolState RequiredState = SymbolState::Ready); /// Materialize the given unit. void dispatchMaterialization(JITDylib &JD, @@ -1221,6 +1295,7 @@ mutable std::recursive_mutex SessionMutex; std::shared_ptr SSP; + std::unique_ptr P; VModuleKey LastKey = 0; ErrorReporter ReportError = logErrorsToStdErr; DispatchMaterializationFunction DispatchMaterialization = @@ -1256,6 +1331,11 @@ if (auto Err = defineImpl(*MU)) return Err; + if (auto *P = ES.getPlatform()) { + if (auto Err = P->notifyAdding(*this, *MU)) + return Err; + } + /// defineImpl succeeded. auto UMI = std::make_shared(std::move(MU)); for (auto &KV : UMI->MU->getSymbols()) @@ -1273,6 +1353,11 @@ if (auto Err = defineImpl(*MU)) return Err; + if (auto *P = ES.getPlatform()) { + if (auto Err = P->notifyAdding(*this, *MU)) + return Err; + } + /// defineImpl succeeded. auto UMI = std::make_shared(std::move(MU)); for (auto &KV : UMI->MU->getSymbols()) @@ -1305,18 +1390,6 @@ SymbolPredicate Allow; }; -/// Mangles symbol names then uniques them in the context of an -/// ExecutionSession. -class MangleAndInterner { -public: - MangleAndInterner(ExecutionSession &ES, const DataLayout &DL); - SymbolStringPtr operator()(StringRef Name); - -private: - ExecutionSession &ES; - const DataLayout &DL; -}; - } // End namespace orc } // End namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h --- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -17,6 +17,7 @@ #include "llvm/ADT/iterator_range.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Mangling.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/Object/Archive.h" @@ -104,6 +105,55 @@ /// array. iterator_range getDestructors(const Module &M); +/// This iterator provides a convenient way to iterate over GlobalValues that +/// have initialization effects. +class StaticInitGVIterator { +public: + + StaticInitGVIterator() = default; + + StaticInitGVIterator(Module &M) + : I(M.global_values().begin()), E(M.global_values().end()), + ObjFmt(Triple(M.getTargetTriple()).getObjectFormat()) { + if (I != E) { + if (!isStaticInitGlobal(*I)) + moveToNextStaticInitGlobal(); + } else + I = E = Module::global_value_iterator(); + } + + bool operator==(const StaticInitGVIterator &O) const { return I == O.I; } + bool operator!=(const StaticInitGVIterator &O) const { return I != O.I; } + + StaticInitGVIterator &operator++() { + assert(I != E && "Increment past end of range"); + moveToNextStaticInitGlobal(); + return *this; + } + + GlobalValue &operator*() { return *I; } + +private: + + bool isStaticInitGlobal(GlobalValue &GV); + void moveToNextStaticInitGlobal() { + ++I; + while (I != E && !isStaticInitGlobal(*I)) + ++I; + if (I == E) + I = E = Module::global_value_iterator(); + } + + Module::global_value_iterator I, E; + Triple::ObjectFormatType ObjFmt; +}; + +/// Create an iterator range over the GlobalValues that contribute to static +/// initialization. +inline iterator_range getStaticInitGVs(Module &M) { + return make_range(StaticInitGVIterator(M), StaticInitGVIterator()); +} + /// Convenience class for recording constructor/destructor names for /// later execution. template @@ -246,6 +296,22 @@ Error enable(JITDylib &JD, MangleAndInterner &Mangler); }; +/// An interface for Itanium __cxa_atexit interposer implementations. +class ItaniumCXAAtExitSupport { +public: + struct AtExitRecord { + void (*F)(void *); + void *Ctx; + }; + + void registerAtExit(void (*F)(void *), void *Ctx, void *DSOHandle); + void runAtExits(void *DSOHandle); + +private: + std::mutex AtExitsMutex; + DenseMap> AtExitRecords; +}; + /// A utility class to expose symbols found via dlsym to the JIT. /// /// If an instance of this class is attached to a JITDylib as a fallback diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h --- a/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h @@ -31,18 +31,18 @@ public: class IRCompiler { public: - IRCompiler(IRMaterializationUnit::ManglingOptions MO) : MO(std::move(MO)) {} + IRCompiler(IRSymbolMapper::ManglingOptions MO) : MO(std::move(MO)) {} virtual ~IRCompiler(); - const IRMaterializationUnit::ManglingOptions &getManglingOptions() const { + const IRSymbolMapper::ManglingOptions &getManglingOptions() const { return MO; } virtual Expected> operator()(Module &M) = 0; protected: - IRMaterializationUnit::ManglingOptions &manglingOptions() { return MO; } + IRSymbolMapper::ManglingOptions &manglingOptions() { return MO; } private: - IRMaterializationUnit::ManglingOptions MO; + IRSymbolMapper::ManglingOptions MO; }; using NotifyCompiledFunction = @@ -61,7 +61,7 @@ mutable std::mutex IRLayerMutex; ObjectLayer &BaseLayer; std::unique_ptr Compile; - const IRMaterializationUnit::ManglingOptions *ManglingOpts; + const IRSymbolMapper::ManglingOptions *ManglingOpts; NotifyCompiledFunction NotifyCompiled = NotifyCompiledFunction(); }; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h --- a/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h @@ -28,7 +28,7 @@ class IRTransformLayer : public IRLayer { public: using TransformFunction = std::function( - ThreadSafeModule, const MaterializationResponsibility &R)>; + ThreadSafeModule, MaterializationResponsibility &R)>; IRTransformLayer(ExecutionSession &ES, IRLayer &BaseLayer, TransformFunction Transform = identityTransform); @@ -39,9 +39,8 @@ void emit(MaterializationResponsibility R, ThreadSafeModule TSM) override; - static ThreadSafeModule - identityTransform(ThreadSafeModule TSM, - const MaterializationResponsibility &R) { + static ThreadSafeModule identityTransform(ThreadSafeModule TSM, + MaterializationResponsibility &R) { return TSM; } diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h --- a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -201,7 +201,7 @@ ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) : TP(std::move(TP)), ES(ES), - CallbacksJD(ES.createJITDylib("")), + CallbacksJD(ES.createBareJITDylib("")), ErrorHandlerAddress(ErrorHandlerAddress) {} void setTrampolinePool(std::unique_ptr TP) { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h --- a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h @@ -35,7 +35,23 @@ class LLJIT { template friend class LLJITBuilderSetters; + friend void setUpGenericLLVMIRPlatform(LLJIT &J); + public: + /// Initializer support for LLJIT. + class PlatformSupport { + public: + virtual ~PlatformSupport(); + + virtual Error initialize(JITDylib &JD) = 0; + + virtual Error deinitialize(JITDylib &JD) = 0; + + protected: + static void setInitTransform(LLJIT &J, + IRTransformLayer::TransformFunction T); + }; + static Expected> Create(LLJITBuilderState &S); /// Destruct this instance. If a multi-threaded instance, waits for all @@ -52,7 +68,7 @@ const DataLayout &getDataLayout() const { return DL; } /// Returns a reference to the JITDylib representing the JIT'd main program. - JITDylib &getMainJITDylib() { return Main; } + JITDylib &getMainJITDylib() { return *Main; } /// Returns the JITDylib with the given name, or nullptr if no JITDylib with /// that name exists. @@ -66,7 +82,7 @@ /// input or elsewhere in the environment then the client should check /// (e.g. by calling getJITDylibByName) that the given name is not already in /// use. - JITDylib &createJITDylib(std::string Name) { + Expected createJITDylib(std::string Name) { return ES->createJITDylib(std::move(Name)); } @@ -78,7 +94,7 @@ /// Adds an IR module to the Main JITDylib. Error addIRModule(ThreadSafeModule TSM) { - return addIRModule(Main, std::move(TSM)); + return addIRModule(*Main, std::move(TSM)); } /// Adds an object file to the given JITDylib. @@ -86,7 +102,7 @@ /// Adds an object file to the given JITDylib. Error addObjectFile(std::unique_ptr Obj) { - return addObjectFile(Main, std::move(Obj)); + return addObjectFile(*Main, std::move(Obj)); } /// Look up a symbol in JITDylib JD by the symbol's linker-mangled name (to @@ -98,7 +114,7 @@ /// (to look up symbols based on their IR name use the lookup function /// instead). Expected lookupLinkerMangled(StringRef Name) { - return lookupLinkerMangled(Main, Name); + return lookupLinkerMangled(*Main, Name); } /// Look up a symbol in JITDylib JD based on its IR symbol name. @@ -108,14 +124,28 @@ /// Look up a symbol in the main JITDylib based on its IR symbol name. Expected lookup(StringRef UnmangledName) { - return lookup(Main, UnmangledName); + return lookup(*Main, UnmangledName); } - /// Runs all not-yet-run static constructors. - Error runConstructors() { return CtorRunner.run(); } + /// Set the PlatformSupport instance. + void setPlatformSupport(std::unique_ptr PS) { + this->PS = std::move(PS); + } - /// Runs all not-yet-run static destructors. - Error runDestructors() { return DtorRunner.run(); } + /// Get the PlatformSupport instance. + PlatformSupport *getPlatformSupport() { return PS.get(); } + + /// Run the initializers for the given JITDylib. + Error initialize(JITDylib &JD) { + assert(PS && "PlatformSupport must be set to run initializers."); + return PS->initialize(JD); + } + + /// Run the deinitializers for the given JITDylib. + Error deinitialize(JITDylib &JD) { + assert(PS && "PlatformSupport must be set to run initializers."); + return PS->deinitialize(JD); + } /// Returns a reference to the ObjLinkingLayer ObjectLayer &getObjLinkingLayer() { return *ObjLinkingLayer; } @@ -126,6 +156,9 @@ /// Returns a reference to the IR transform layer. IRTransformLayer &getIRTransformLayer() { return *TransformLayer; } + /// Returns a reference to the IR compile layer. + IRCompileLayer &getIRCompileLayer() { return *CompileLayer; } + protected: static std::unique_ptr createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES); @@ -143,7 +176,9 @@ void recordCtorDtors(Module &M); std::unique_ptr ES; - JITDylib &Main; + std::unique_ptr PS; + + JITDylib *Main = nullptr; DataLayout DL; Triple TT; @@ -153,8 +188,7 @@ ObjectTransformLayer ObjTransformLayer; std::unique_ptr CompileLayer; std::unique_ptr TransformLayer; - - CtorDtorRunner CtorRunner, DtorRunner; + std::unique_ptr InitHelperTransformLayer; }; /// An extended version of LLJIT that supports lazy function-at-a-time @@ -175,7 +209,7 @@ /// Add a module to be lazily compiled to the main JITDylib. Error addLazyIRModule(ThreadSafeModule M) { - return addLazyIRModule(Main, std::move(M)); + return addLazyIRModule(*Main, std::move(M)); } private: @@ -196,10 +230,14 @@ std::function>( JITTargetMachineBuilder JTMB)>; + using PlatformSetupFunction = std::function; + std::unique_ptr ES; Optional JTMB; + Optional DL; ObjectLinkingLayerCreator CreateObjectLinkingLayer; CompileFunctionCreator CreateCompileFunction; + PlatformSetupFunction SetUpPlatform; unsigned NumCompileThreads = 0; /// Called prior to JIT class construcion to fix up defaults. @@ -224,6 +262,13 @@ return impl().JTMB; } + /// Set a DataLayout for this instance. If no data layout is specified then + /// the target's default data layout will be used. + SetterImpl &setDataLayout(Optional DL) { + impl().DL = std::move(DL); + return impl(); + } + /// Set an ObjectLinkingLayer creation function. /// /// If this method is not called, a default creation function will be used @@ -246,6 +291,16 @@ return impl(); } + /// Set up an PlatformSetupFunction. + /// + /// If this method is not called then setUpGenericLLVMIRPlatform + /// will be used to configure the JIT's platform support. + SetterImpl & + setPlatformSetUp(LLJITBuilderState::PlatformSetupFunction SetUpPlatform) { + impl().SetUpPlatform = std::move(SetUpPlatform); + return impl(); + } + /// Set the number of compile threads to use. /// /// If set to zero, compilation will be performed on the execution thread when @@ -334,6 +389,26 @@ public LLLazyJITBuilderSetters {}; +/// Configure the LLJIT instance to scrape modules for llvm.global_ctors and +/// llvm.global_dtors variables and (if present) build initialization and +/// deinitialization functions. Platform specific initialization configurations +/// should be preferred where available. +void setUpGenericLLVMIRPlatform(LLJIT &J); + +/// Configure the LLJIT instance to use MachOPlatform support. +/// +/// Warning: MachOPlatform *requires* that LLJIT be configured to use +/// ObjectLinkingLayer (default on platforms supported by JITLink). If +/// MachOPlatform is used with RTDyldObjectLinkingLayer it will result in +/// undefined behavior). +/// +/// MachOPlatform installs an ObjectLinkingLayer plugin to scrape initializers +/// from the __mod_inits section. It also provides interposes for the dlfcn +/// functions (dlopen, dlclose, dlsym, dlerror) that work for JITDylibs as +/// well as regular libraries (JITDylibs will be preferenced, so make sure +/// your JITDylib names do not shadow any real library paths). +Error setUpMachOPlatform(LLJIT &J); + } // End namespace orc } // End namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Layer.h b/llvm/include/llvm/ExecutionEngine/Orc/Layer.h --- a/llvm/include/llvm/ExecutionEngine/Orc/Layer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Layer.h @@ -14,6 +14,7 @@ #define LLVM_EXECUTIONENGINE_ORC_LAYER_H #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Mangling.h" #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" #include "llvm/IR/Module.h" #include "llvm/Support/MemoryBuffer.h" @@ -27,15 +28,12 @@ /// their linkage is changed to available-externally. class IRMaterializationUnit : public MaterializationUnit { public: - struct ManglingOptions { - bool EmulatedTLS = false; - }; - using SymbolNameToDefinitionMap = std::map; /// Create an IRMaterializationLayer. Scans the module to build the /// SymbolFlags and SymbolToDefinition maps. - IRMaterializationUnit(ExecutionSession &ES, const ManglingOptions &MO, + IRMaterializationUnit(ExecutionSession &ES, + const IRSymbolMapper::ManglingOptions &MO, ThreadSafeModule TSM, VModuleKey K); /// Create an IRMaterializationLayer from a module, and pre-existing @@ -44,12 +42,13 @@ /// This constructor is useful for delegating work from one /// IRMaterializationUnit to another. IRMaterializationUnit(ThreadSafeModule TSM, VModuleKey K, - SymbolFlagsMap SymbolFlags, + SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol, SymbolNameToDefinitionMap SymbolToDefinition); /// Return the ModuleIdentifier as the name for this MaterializationUnit. StringRef getName() const override; + /// Return a reference to the contained ThreadSafeModule. const ThreadSafeModule &getModule() const { return TSM; } protected: @@ -57,14 +56,16 @@ SymbolNameToDefinitionMap SymbolToDefinition; private: + static SymbolStringPtr getInitSymbol(ExecutionSession &ES, + const ThreadSafeModule &TSM); + void discard(const JITDylib &JD, const SymbolStringPtr &Name) override; }; /// Interface for layers that accept LLVM IR. class IRLayer { public: - IRLayer(ExecutionSession &ES, - const IRMaterializationUnit::ManglingOptions *&MO) + IRLayer(ExecutionSession &ES, const IRSymbolMapper::ManglingOptions *&MO) : ES(ES), MO(MO) {} virtual ~IRLayer(); @@ -73,7 +74,7 @@ ExecutionSession &getExecutionSession() { return ES; } /// Get the mangling options for this layer. - const IRMaterializationUnit::ManglingOptions *&getManglingOptions() const { + const IRSymbolMapper::ManglingOptions *&getManglingOptions() const { return MO; } @@ -104,14 +105,15 @@ private: bool CloneToNewContextOnEmit = false; ExecutionSession &ES; - const IRMaterializationUnit::ManglingOptions *&MO; + const IRSymbolMapper::ManglingOptions *&MO; }; /// MaterializationUnit that materializes modules by calling the 'emit' method /// on the given IRLayer. class BasicIRLayerMaterializationUnit : public IRMaterializationUnit { public: - BasicIRLayerMaterializationUnit(IRLayer &L, const ManglingOptions &MO, + BasicIRLayerMaterializationUnit(IRLayer &L, + const IRSymbolMapper::ManglingOptions &MO, ThreadSafeModule TSM, VModuleKey K); private: @@ -153,7 +155,8 @@ BasicObjectLayerMaterializationUnit(ObjectLayer &L, VModuleKey K, std::unique_ptr O, - SymbolFlagsMap SymbolFlags); + SymbolFlagsMap SymbolFlags, + SymbolStringPtr InitSymbol); /// Return the buffer's identifier as the name for this MaterializationUnit. StringRef getName() const override; @@ -167,12 +170,6 @@ std::unique_ptr O; }; -/// Returns a SymbolFlagsMap for the object file represented by the given -/// buffer, or an error if the buffer does not contain a valid object file. -// FIXME: Maybe move to Core.h? -Expected getObjectSymbolFlags(ExecutionSession &ES, - MemoryBufferRef ObjBuffer); - } // End namespace orc } // End namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -0,0 +1,151 @@ +//===-- MachOPlatform.h - Utilities for executing MachO in Orc --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities for executing JIT'd MachO in Orc. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_MACHOPLATFORM_H +#define LLVM_EXECUTIONENGINE_ORC_MACHOPLATFORM_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" + +#include +#include +#include + +namespace llvm { +namespace orc { + +/// Enable registration of JIT'd ObjC classes and selectors. +Error enableObjCRegistration(const char *PathToLibObjC); +bool objCRegistrationEnabled(); + +class MachOJITDylibInitializers { +public: + struct SectionExtent { + SectionExtent() = default; + SectionExtent(JITTargetAddress Address, uint64_t NumPtrs) + : Address(Address), NumPtrs(NumPtrs) {} + JITTargetAddress Address = 0; + uint64_t NumPtrs = 0; + }; + + void setObjCImageInfoAddr(JITTargetAddress ObjCImageInfoAddr) { + this->ObjCImageInfoAddr = ObjCImageInfoAddr; + } + + void addModInitsSection(SectionExtent ModInit) { + ModInitSections.push_back(std::move(ModInit)); + } + + void addObjCSelRefsSection(SectionExtent ObjCSelRefs) { + ObjCSelRefsSections.push_back(std::move(ObjCSelRefs)); + } + + void addObjCClassListSection(SectionExtent ObjCClassList) { + ObjCClassListSections.push_back(std::move(ObjCClassList)); + } + + void runModInits() const; + void registerObjCSelectors() const; + void registerObjCClasses() const; + + void dump() const; + +private: + + using RawPointerSectionList = std::vector; + + JITTargetAddress ObjCImageInfoAddr; + RawPointerSectionList ModInitSections; + RawPointerSectionList ObjCSelRefsSections; + RawPointerSectionList ObjCClassListSections; +}; + +class MachOJITDylibDeinitializers {}; + +/// Mediates between MachO initialization and ExecutionSession state. +class MachOPlatform : public Platform { +public: + using InitializerSequence = + std::vector>; + + using DeinitializerSequence = + std::vector>; + + MachOPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + std::unique_ptr StandardSymbolsObject); + + ExecutionSession &getExecutionSession() const { return ES; } + + Error setupJITDylib(JITDylib &JD) override; + Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) override; + Error notifyRemoving(JITDylib &JD, VModuleKey K) override; + + Expected getInitializerSequence(JITDylib &JD); + + Expected getDeinitializerSequence(JITDylib &JD); + +private: + // This ObjectLinkingLayer plugin scans JITLink graphs for __mod_init_func, + // __objc_classlist and __sel_ref sections and records their extents so that + // they can be run in the target process. + class InitScraperPlugin : public ObjectLinkingLayer::Plugin { + public: + InitScraperPlugin(MachOPlatform &MP) : MP(MP) {} + + void modifyPassConfig(MaterializationResponsibility &MR, const Triple &TT, + jitlink::PassConfiguration &Config) override; + + LocalDependenciesMap + getSyntheticSymbolLocalDependencies( + MaterializationResponsibility &MR) override; + + private: + + using InitSymbolDepMap = + DenseMap; + + void preserveInitSectionIfPresent(JITLinkSymbolVector &Syms, + jitlink::LinkGraph &G, + StringRef SectionName); + + Error processObjCImageInfo(jitlink::LinkGraph &G, + MaterializationResponsibility &MR); + + std::mutex InitScraperMutex; + MachOPlatform &MP; + DenseMap> ObjCImageInfos; + InitSymbolDepMap InitSymbolDeps; + }; + + static std::vector getDFSLinkOrder(JITDylib &JD); + + void registerInitInfo(JITDylib &JD, + JITTargetAddress ObjCImageInfoAddr, + MachOJITDylibInitializers::SectionExtent ModInits, + MachOJITDylibInitializers::SectionExtent ObjCSelRefs, + MachOJITDylibInitializers::SectionExtent ObjCClassList); + + std::mutex PlatformMutex; + ExecutionSession &ES; + ObjectLinkingLayer &ObjLinkingLayer; + std::unique_ptr StandardSymbolsObject; + + DenseMap RegisteredInitSymbols; + DenseMap InitSeqs; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_MACHOPLATFORM_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Mangling.h b/llvm/include/llvm/ExecutionEngine/Orc/Mangling.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/Mangling.h @@ -0,0 +1,66 @@ +//===------ Mangling.h -- Name Mangling Utilities for ORC -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Name mangling utilities for ORC. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_MANGLING_H +#define LLVM_EXECUTIONENGINE_ORC_MANGLING_H + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +/// Mangles symbol names then uniques them in the context of an +/// ExecutionSession. +class MangleAndInterner { +public: + MangleAndInterner(ExecutionSession &ES, const DataLayout &DL); + SymbolStringPtr operator()(StringRef Name); + +private: + ExecutionSession &ES; + const DataLayout &DL; +}; + +/// Maps IR global values to their linker symbol names / flags. +/// +/// This utility can be used when adding new IR globals in the JIT. +class IRSymbolMapper { +public: + struct ManglingOptions { + bool EmulatedTLS = false; + }; + + using SymbolNameToDefinitionMap = std::map; + + /// Add mangled symbols for the given GlobalValues to SymbolFlags. + /// If a SymbolToDefinitionMap pointer is supplied then it will be populated + /// with Name-to-GlobalValue* mappings. Note that this mapping is not + /// necessarily one-to-one: thread-local GlobalValues, for example, may + /// produce more than one symbol, in which case the map will contain duplicate + /// values. + static void add(ExecutionSession &ES, const ManglingOptions &MO, + ArrayRef GVs, SymbolFlagsMap &SymbolFlags, + SymbolNameToDefinitionMap *SymbolToDefinition = nullptr); +}; + +/// Returns a SymbolFlagsMap for the object file represented by the given +/// buffer, or an error if the buffer does not contain a valid object file. +Expected> +getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer); + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_MANGLING_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h --- a/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h @@ -35,6 +35,7 @@ namespace jitlink { class EHFrameRegistrar; +class Symbol; } // namespace jitlink namespace object { @@ -59,10 +60,16 @@ /// configured. class Plugin { public: + + using JITLinkSymbolVector = std::vector; + using LocalDependenciesMap = + DenseMap; + virtual ~Plugin(); virtual void modifyPassConfig(MaterializationResponsibility &MR, const Triple &TT, jitlink::PassConfiguration &Config) {} + virtual void notifyLoaded(MaterializationResponsibility &MR) {} virtual Error notifyEmitted(MaterializationResponsibility &MR) { return Error::success(); @@ -71,6 +78,15 @@ return Error::success(); } virtual Error notifyRemovingAllModules() { return Error::success(); } + + /// Return any dependencies that synthetic symbols (e.g. init symbols) + /// have on locally scoped jitlink::Symbols. This is used by the + /// ObjectLinkingLayer to update the dependencies for the synthetic + /// symbols. + virtual LocalDependenciesMap + getSyntheticSymbolLocalDependencies(MaterializationResponsibility &MR) { + return LocalDependenciesMap(); + } }; using ReturnObjectBufferFunction = diff --git a/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h b/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h --- a/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h @@ -53,6 +53,7 @@ public: SymbolStringPtr() = default; + SymbolStringPtr(nullptr_t) {} SymbolStringPtr(const SymbolStringPtr &Other) : S(Other.S) { if (isRealPoolEntry(S)) @@ -85,6 +86,8 @@ --S->getValue(); } + explicit operator bool() const { return S; } + StringRef operator*() const { return S->first(); } friend bool operator==(const SymbolStringPtr &LHS, diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt --- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -12,6 +12,8 @@ Legacy.cpp Layer.cpp LLJIT.cpp + MachOPlatform.cpp + Mangling.cpp NullResolver.cpp ObjectLinkingLayer.cpp ObjectTransformLayer.cpp diff --git a/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp --- a/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/IR/Mangler.h" #include "llvm/IR/Module.h" @@ -68,18 +69,18 @@ class PartitioningIRMaterializationUnit : public IRMaterializationUnit { public: PartitioningIRMaterializationUnit(ExecutionSession &ES, - const ManglingOptions &MO, + const IRSymbolMapper::ManglingOptions &MO, ThreadSafeModule TSM, VModuleKey K, CompileOnDemandLayer &Parent) : IRMaterializationUnit(ES, MO, std::move(TSM), std::move(K)), Parent(Parent) {} PartitioningIRMaterializationUnit( - ThreadSafeModule TSM, SymbolFlagsMap SymbolFlags, - SymbolNameToDefinitionMap SymbolToDefinition, + ThreadSafeModule TSM, VModuleKey K, SymbolFlagsMap SymbolFlags, + SymbolStringPtr InitSymbol, SymbolNameToDefinitionMap SymbolToDefinition, CompileOnDemandLayer &Parent) : IRMaterializationUnit(std::move(TSM), std::move(K), - std::move(SymbolFlags), + std::move(SymbolFlags), std::move(InitSymbol), std::move(SymbolToDefinition)), Parent(Parent) {} @@ -172,21 +173,23 @@ auto I = DylibResources.find(&TargetD); if (I == DylibResources.end()) { auto &ImplD = - getExecutionSession().createJITDylib(TargetD.getName() + ".impl"); + getExecutionSession().createBareJITDylib(TargetD.getName() + ".impl"); + JITDylibSearchOrder NewSearchOrder; TargetD.withSearchOrderDo( [&](const JITDylibSearchOrder &TargetSearchOrder) { - auto NewSearchOrder = TargetSearchOrder; - assert( - !NewSearchOrder.empty() && - NewSearchOrder.front().first == &TargetD && - NewSearchOrder.front().second == - JITDylibLookupFlags::MatchAllSymbols && - "TargetD must be at the front of its own search order and match " - "non-exported symbol"); - NewSearchOrder.insert(std::next(NewSearchOrder.begin()), - {&ImplD, JITDylibLookupFlags::MatchAllSymbols}); - ImplD.setSearchOrder(std::move(NewSearchOrder), false); + NewSearchOrder = TargetSearchOrder; }); + + assert( + !NewSearchOrder.empty() && NewSearchOrder.front().first == &TargetD && + NewSearchOrder.front().second == JITDylibLookupFlags::MatchAllSymbols && + "TargetD must be at the front of its own search order and match " + "non-exported symbol"); + NewSearchOrder.insert(std::next(NewSearchOrder.begin()), + {&ImplD, JITDylibLookupFlags::MatchAllSymbols}); + ImplD.setSearchOrder(NewSearchOrder, false); + TargetD.setSearchOrder(std::move(NewSearchOrder), false); + PerDylibResources PDR(ImplD, BuildIndirectStubsManager()); I = DylibResources.insert(std::make_pair(&TargetD, std::move(PDR))).first; } @@ -251,8 +254,15 @@ auto &ES = getExecutionSession(); GlobalValueSet RequestedGVs; for (auto &Name : R.getRequestedSymbols()) { - assert(Defs.count(Name) && "No definition for symbol"); - RequestedGVs.insert(Defs[Name]); + if (Name == R.getInitializerSymbol()) + TSM.withModuleDo([&](Module &M) { + for (auto &GV : getStaticInitGVs(M)) + RequestedGVs.insert(&GV); + }); + else { + assert(Defs.count(Name) && "No definition for symbol"); + RequestedGVs.insert(Defs[Name]); + } } /// Perform partitioning with the context lock held, since the partition @@ -272,7 +282,8 @@ // If the partition is empty, return the whole module to the symbol table. if (GVsToExtract->empty()) { R.replace(std::make_unique( - std::move(TSM), R.getSymbols(), std::move(Defs), *this)); + std::move(TSM), R.getVModuleKey(), R.getSymbols(), + R.getInitializerSymbol(), std::move(Defs), *this)); return; } diff --git a/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp b/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp --- a/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp @@ -24,9 +24,9 @@ namespace llvm { namespace orc { -IRMaterializationUnit::ManglingOptions +IRSymbolMapper::ManglingOptions irManglingOptionsFromTargetOptions(const TargetOptions &Opts) { - IRMaterializationUnit::ManglingOptions MO; + IRSymbolMapper::ManglingOptions MO; MO.EmulatedTLS = Opts.EmulatedTLS; diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp --- a/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -11,7 +11,6 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/Config/llvm-config.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" -#include "llvm/IR/Mangler.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Format.h" @@ -421,12 +420,6 @@ QueryRegistrations.clear(); } -MaterializationResponsibility::MaterializationResponsibility( - JITDylib &JD, SymbolFlagsMap SymbolFlags, VModuleKey K) - : JD(JD), SymbolFlags(std::move(SymbolFlags)), K(std::move(K)) { - assert(!this->SymbolFlags.empty() && "Materializing nothing?"); -} - MaterializationResponsibility::~MaterializationResponsibility() { assert(SymbolFlags.empty() && "All symbols should have been explicitly materialized or failed"); @@ -501,9 +494,13 @@ void MaterializationResponsibility::replace( std::unique_ptr MU) { + for (auto &KV : MU->getSymbols()) SymbolFlags.erase(KV.first); + if (MU->getInitializerSymbol() == InitSymbol) + InitSymbol = nullptr; + LLVM_DEBUG(JD.getExecutionSession().runSessionLocked([&]() { dbgs() << "In " << JD.getName() << " replacing symbols with " << *MU << "\n"; @@ -519,6 +516,7 @@ if (NewKey == VModuleKey()) NewKey = K; + SymbolStringPtr DelegatedInitSymbol; SymbolFlagsMap DelegatedFlags; for (auto &Name : Symbols) { @@ -528,10 +526,14 @@ "instance"); DelegatedFlags[Name] = std::move(I->second); + if (Name == InitSymbol) + std::swap(InitSymbol, DelegatedInitSymbol); + SymbolFlags.erase(I); } return MaterializationResponsibility(JD, std::move(DelegatedFlags), + std::move(DelegatedInitSymbol), std::move(NewKey)); } @@ -550,7 +552,7 @@ AbsoluteSymbolsMaterializationUnit::AbsoluteSymbolsMaterializationUnit( SymbolMap Symbols, VModuleKey K) - : MaterializationUnit(extractFlags(Symbols), std::move(K)), + : MaterializationUnit(extractFlags(Symbols), nullptr, std::move(K)), Symbols(std::move(Symbols)) {} StringRef AbsoluteSymbolsMaterializationUnit::getName() const { @@ -581,7 +583,7 @@ ReExportsMaterializationUnit::ReExportsMaterializationUnit( JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags, SymbolAliasMap Aliases, VModuleKey K) - : MaterializationUnit(extractFlags(Aliases), std::move(K)), + : MaterializationUnit(extractFlags(Aliases), nullptr, std::move(K)), SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {} @@ -972,7 +974,7 @@ // Assert that this symbol exists and has not reached the ready state // already. assert(OtherSymI != OtherJITDylib.Symbols.end() && - (OtherSymI->second.getState() != SymbolState::Ready && + (OtherSymI->second.getState() < SymbolState::Ready && "Dependency on emitted/ready symbol")); #endif @@ -1101,6 +1103,7 @@ Error JITDylib::emit(const SymbolFlagsMap &Emitted) { AsynchronousSymbolQuerySet CompletedQueries; SymbolNameSet SymbolsInErrorState; + DenseMap ReadySymbols; ES.runSessionLocked([&, this]() { std::vector Worklist; @@ -1145,6 +1148,7 @@ // dependencies) then notify any pending queries. for (auto &KV : MI.Dependants) { auto &DependantJD = *KV.first; + auto &DependantJDReadySymbols = ReadySymbols[&DependantJD]; for (auto &DependantName : KV.second) { auto DependantMII = DependantJD.MaterializingInfos.find(DependantName); @@ -1184,6 +1188,7 @@ // Since this dependant is now ready, we erase its MaterializingInfo // and update its materializing state. DependantSymEntry.setState(SymbolState::Ready); + DependantJDReadySymbols.push_back(DependantName); for (auto &Q : DependantMI.takeQueriesMeeting(SymbolState::Ready)) { Q->notifySymbolMetRequiredState( @@ -1192,22 +1197,21 @@ CompletedQueries.insert(Q); Q->removeQueryDependence(DependantJD, DependantName); } - - DependantJD.MaterializingInfos.erase(DependantMII); } } } + auto &ThisJDReadySymbols = ReadySymbols[this]; MI.Dependants.clear(); if (MI.UnemittedDependencies.empty()) { SymI->second.setState(SymbolState::Ready); + ThisJDReadySymbols.push_back(Name); for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) { Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); if (Q->isComplete()) CompletedQueries.insert(Q); Q->removeQueryDependence(*this, Name); } - MaterializingInfos.erase(MII); } } }); @@ -1882,6 +1886,57 @@ } } +Platform::~Platform() {} + +Expected> Platform::lookupInitSymbols( + ExecutionSession &ES, + const DenseMap &InitSyms) { + + DenseMap CompoundResult; + Error CompoundErr = Error::success(); + std::mutex LookupMutex; + std::condition_variable CV; + uint64_t Count = InitSyms.size(); + + LLVM_DEBUG({ + dbgs() << "Issuing init-symbol lookup:\n"; + for (auto &KV : InitSyms) + dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n"; + }); + + for (auto &KV : InitSyms) { + auto *JD = KV.first; + auto Names = std::move(KV.second); + ES.lookup( + LookupKind::Static, + JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}), + std::move(Names), SymbolState::Ready, + [&, JD](Expected Result) { + { + std::lock_guard Lock(LookupMutex); + --Count; + if (Result) { + assert(!CompoundResult.count(JD) && + "Duplicate JITDylib in lookup?"); + CompoundResult[JD] = std::move(*Result); + } else + CompoundErr = + joinErrors(std::move(CompoundErr), Result.takeError()); + } + CV.notify_one(); + }, + NoDependenciesToRegister); + } + + std::unique_lock Lock(LookupMutex); + CV.wait(Lock, [&] { return Count == 0 || CompoundErr; }); + + if (CompoundErr) + return std::move(CompoundErr); + + return std::move(CompoundResult); +} + ExecutionSession::ExecutionSession(std::shared_ptr SSP) : SSP(SSP ? std::move(SSP) : std::make_shared()) { } @@ -1895,7 +1950,7 @@ }); } -JITDylib &ExecutionSession::createJITDylib(std::string Name) { +JITDylib &ExecutionSession::createBareJITDylib(std::string Name) { assert(!getJITDylibByName(Name) && "JITDylib with that name already exists"); return runSessionLocked([&, this]() -> JITDylib & { JDs.push_back( @@ -1904,6 +1959,14 @@ }); } +Expected ExecutionSession::createJITDylib(std::string Name) { + auto &JD = createBareJITDylib(Name); + if (P) + if (auto Err = P->setupJITDylib(JD)) + return std::move(Err); + return JD; +} + void ExecutionSession::legacyFailQuery(AsynchronousSymbolQuery &Q, Error Err) { assert(!!Err && "Error should be in failure state"); @@ -2144,11 +2207,11 @@ Expected ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, - SymbolStringPtr Name) { + SymbolStringPtr Name, SymbolState RequiredState) { SymbolLookupSet Names({Name}); if (auto ResultMap = lookup(SearchOrder, std::move(Names), LookupKind::Static, - SymbolState::Ready, NoDependenciesToRegister)) { + RequiredState, NoDependenciesToRegister)) { assert(ResultMap->size() == 1 && "Unexpected number of results"); assert(ResultMap->count(Name) && "Missing result for symbol"); return std::move(ResultMap->begin()->second); @@ -2157,14 +2220,15 @@ } Expected -ExecutionSession::lookup(ArrayRef SearchOrder, - SymbolStringPtr Name) { - return lookup(makeJITDylibSearchOrder(SearchOrder), Name); +ExecutionSession::lookup(ArrayRef SearchOrder, SymbolStringPtr Name, + SymbolState RequiredState) { + return lookup(makeJITDylibSearchOrder(SearchOrder), Name, RequiredState); } Expected -ExecutionSession::lookup(ArrayRef SearchOrder, StringRef Name) { - return lookup(SearchOrder, intern(Name)); +ExecutionSession::lookup(ArrayRef SearchOrder, StringRef Name, + SymbolState RequiredState) { + return lookup(SearchOrder, intern(Name), RequiredState); } void ExecutionSession::dump(raw_ostream &OS) { @@ -2195,17 +2259,5 @@ } } -MangleAndInterner::MangleAndInterner(ExecutionSession &ES, const DataLayout &DL) - : ES(ES), DL(DL) {} - -SymbolStringPtr MangleAndInterner::operator()(StringRef Name) { - std::string MangledName; - { - raw_string_ostream MangledNameStream(MangledName); - Mangler::getNameWithPrefix(MangledNameStream, Name, DL); - } - return ES.intern(MangledName); -} - } // End namespace orc. } // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp --- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -113,6 +113,27 @@ CtorDtorIterator(DtorsList, true)); } +bool StaticInitGVIterator::isStaticInitGlobal(GlobalValue &GV) { + if (GV.isDeclaration()) + return false; + + if (GV.hasName() && + (GV.getName() == "llvm.global_ctors" || + GV.getName() == "llvm.global_dtors")) + return true; + + if (ObjFmt == Triple::MachO) { + // FIXME: These section checks are too strict: We should match first and + // second word split by comma. + if (GV.hasSection() && + (GV.getSection().startswith("__DATA,__objc_classlist") || + GV.getSection().startswith("__DATA,__objc_selrefs"))) + return true; + } + + return false; +} + void CtorDtorRunner::add(iterator_range CtorDtors) { if (CtorDtors.empty()) return; @@ -198,6 +219,30 @@ return JD.define(absoluteSymbols(std::move(RuntimeInterposes))); } +void ItaniumCXAAtExitSupport::registerAtExit(void (*F)(void *), void *Ctx, + void *DSOHandle) { + std::lock_guard Lock(AtExitsMutex); + AtExitRecords[DSOHandle].push_back({F, Ctx}); +} + +void ItaniumCXAAtExitSupport::runAtExits(void *DSOHandle) { + std::vector AtExitsToRun; + + { + std::lock_guard Lock(AtExitsMutex); + auto I = AtExitRecords.find(DSOHandle); + if (I != AtExitRecords.end()) { + AtExitsToRun = std::move(I->second); + AtExitRecords.erase(I); + } + } + + while (!AtExitsToRun.empty()) { + AtExitsToRun.back().F(AtExitsToRun.back().Ctx); + AtExitsToRun.pop_back(); + } +} + DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator( sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow) : Dylib(std::move(Dylib)), Allow(std::move(Allow)), diff --git a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp --- a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -28,7 +28,7 @@ CompileCallbackMaterializationUnit(SymbolStringPtr Name, CompileFunction Compile, VModuleKey K) : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), - std::move(K)), + nullptr, std::move(K)), Name(std::move(Name)), Compile(std::move(Compile)) {} StringRef getName() const override { return ""; } diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp --- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -8,15 +8,824 @@ #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/Mangler.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/DynamicLibrary.h" + +#include + +using namespace llvm; +using namespace llvm::orc; + +namespace { + +/// Add a reference to the __dso_handle global to the given module. +/// Returns a reference to the __dso_handle IR decl. +GlobalVariable *addDSOHandleDecl(Module &M) { + auto DSOHandleTy = StructType::create(M.getContext(), "lljit.dso_handle"); + return new GlobalVariable(M, DSOHandleTy, true, GlobalValue::ExternalLinkage, + nullptr, "__dso_handle"); +} + +/// Adds helper function decls and wrapper functions that call the helper with +/// some additional prefix arguments. +/// +/// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix +/// args i32 4 and i16 12345, this function will add: +/// +/// declare i8 @bar(i32, i16, i8, i64) +/// +/// define i8 @foo(i8, i64) { +/// entry: +/// %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1) +/// ret i8 %2 +/// } +/// +Function *addHelperAndWrapper(Module &M, StringRef WrapperName, + FunctionType *WrapperFnType, + GlobalValue::VisibilityTypes WrapperVisibility, + StringRef HelperName, + ArrayRef HelperPrefixArgs) { + std::vector HelperArgTypes; + for (auto *Arg : HelperPrefixArgs) + HelperArgTypes.push_back(Arg->getType()); + for (auto *T : WrapperFnType->params()) + HelperArgTypes.push_back(T); + auto *HelperFnType = + FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false); + auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage, + HelperName, M); + + auto *WrapperFn = Function::Create( + WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M); + WrapperFn->setVisibility(WrapperVisibility); + + auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn); + IRBuilder<> IB(EntryBlock); + + std::vector HelperArgs; + for (auto *Arg : HelperPrefixArgs) + HelperArgs.push_back(Arg); + for (auto &Arg : WrapperFn->args()) + HelperArgs.push_back(&Arg); + auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs); + if (HelperFn->getReturnType()->isVoidTy()) + IB.CreateRetVoid(); + else + IB.CreateRet(HelperResult); + + return WrapperFn; +} + +class GenericLLVMIRPlatformSupport; + +/// orc::Platform component of Generic LLVM IR Platform support. +/// Just forwards calls to the GenericLLVMIRPlatformSupport class below. +class GenericLLVMIRPlatform : public Platform { +public: + GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {} + Error setupJITDylib(JITDylib &JD) override; + Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) override; + Error notifyRemoving(JITDylib &JD, VModuleKey K) override { + // Noop -- Nothing to do (yet). + return Error::success(); + } + +private: + GenericLLVMIRPlatformSupport &S; +}; + +/// This transform parses llvm.global_ctors to produce a single initialization +/// function for the module, records the function, then deletes +/// llvm.global_ctors. +class GlobalCtorDtorScraper { +public: + GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS) : PS(PS) {} + Expected operator()(ThreadSafeModule TSM, + MaterializationResponsibility &R); + +private: + GenericLLVMIRPlatformSupport &PS; +}; + +/// Generic IR Platform Support +/// +/// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with +/// specially named 'init' and 'deinit'. Injects definitions / interposes for +/// some runtime API, including __cxa_atexit, dlopen, and dlclose. +class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport { +public: + // GenericLLVMIRPlatform &P) : P(P) { + GenericLLVMIRPlatformSupport(LLJIT &J) : J(J) { + + getExecutionSession().setPlatform( + std::make_unique(*this)); + + setInitTransform(J, GlobalCtorDtorScraper(*this)); + + MangleAndInterner Mangle(getExecutionSession(), J.getDataLayout()); + SymbolMap StdInterposes; + + StdInterposes[Mangle("__lljit.platform_support_instance")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(this), JITSymbolFlags()); + StdInterposes[Mangle("__lljit.cxa_atexit_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(registerAtExitHelper), JITSymbolFlags()); + StdInterposes[Mangle("__lljit.run_atexits_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(runAtExitsHelper), JITSymbolFlags()); + + cantFail( + J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes)))); + cantFail(setupJITDylib(J.getMainJITDylib())); + cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule())); + } + + ExecutionSession &getExecutionSession() { return J.getExecutionSession(); } + + /// Adds a module that defines the __dso_handle global. + Error setupJITDylib(JITDylib &JD) { + auto Ctx = std::make_unique(); + auto M = std::make_unique("__standard_lib", *Ctx); + M->setDataLayout(J.getDataLayout()); + + auto *Int64Ty = Type::getInt64Ty(*Ctx); + auto *DSOHandle = new GlobalVariable( + *M, Int64Ty, true, GlobalValue::ExternalLinkage, + ConstantInt::get(Int64Ty, reinterpret_cast(&JD)), + "__dso_handle"); + DSOHandle->setVisibility(GlobalValue::HiddenVisibility); + DSOHandle->setInitializer( + ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD))); + return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx))); + } + + Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) { + std::lock_guard Lock(PlatformSupportMutex); + if (auto &InitSym = MU.getInitializerSymbol()) + InitSymbols[&JD].add(InitSym); + return Error::success(); + } + + Error initialize(JITDylib &JD) override { + if (auto Initializers = getInitializers(JD)) { + for (auto InitFnAddr : *Initializers) { + auto *InitFn = jitTargetAddressToFunction(InitFnAddr); + InitFn(); + } + } else + return Initializers.takeError(); + return Error::success(); + } + + Error deinitialize(JITDylib &JD) override { + if (auto Deinitializers = getDeinitializers(JD)) { + for (auto DeinitFnAddr : *Deinitializers) { + auto *DeinitFn = jitTargetAddressToFunction(DeinitFnAddr); + DeinitFn(); + } + } else + return Deinitializers.takeError(); + + return Error::success(); + } + + void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) { + std::lock_guard Lock(PlatformSupportMutex); + InitFunctions[&JD].add(InitName); + } + +private: + Expected> getInitializers(JITDylib &JD) { + if (auto Err = issueInitLookups(JD)) + return std::move(Err); + + DenseMap LookupSymbols; + std::vector DFSLinkOrder; + + { + std::lock_guard Lock(PlatformSupportMutex); + DFSLinkOrder = getDFSLinkOrder(JD); + + for (auto *NextJD : DFSLinkOrder) { + auto IFItr = InitFunctions.find(NextJD); + if (IFItr != InitFunctions.end()) { + LookupSymbols[NextJD] = std::move(IFItr->second); + InitFunctions.erase(IFItr); + } + } + } + + auto &ES = getExecutionSession(); + auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols); + + if (!LookupResult) + return LookupResult.takeError(); + + std::vector Initializers; + while (!DFSLinkOrder.empty()) { + auto &NextJD = *DFSLinkOrder.back(); + DFSLinkOrder.pop_back(); + auto InitsItr = LookupResult->find(&NextJD); + if (InitsItr == LookupResult->end()) + continue; + for (auto &KV : InitsItr->second) + Initializers.push_back(KV.second.getAddress()); + } + + return Initializers; + } + + Expected> getDeinitializers(JITDylib &JD) { + auto &ES = getExecutionSession(); + + MangleAndInterner Mangle(getExecutionSession(), J.getDataLayout()); + auto LLJITRunAtExits = Mangle("__lljit_run_atexits"); + + DenseMap LookupSymbols; + std::vector DFSLinkOrder; + + { + std::lock_guard Lock(PlatformSupportMutex); + DFSLinkOrder = getDFSLinkOrder(JD); + + for (auto *NextJD : DFSLinkOrder) { + auto &JDLookupSymbols = LookupSymbols[NextJD]; + auto DIFItr = DeInitFunctions.find(NextJD); + if (DIFItr != DeInitFunctions.end()) { + LookupSymbols[NextJD] = std::move(DIFItr->second); + DeInitFunctions.erase(DIFItr); + } + JDLookupSymbols.add(LLJITRunAtExits, + SymbolLookupFlags::WeaklyReferencedSymbol); + } + } + + auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols); + + if (!LookupResult) + return LookupResult.takeError(); + + std::vector DeInitializers; + for (auto *NextJD : DFSLinkOrder) { + auto DeInitsItr = LookupResult->find(NextJD); + assert(DeInitsItr != LookupResult->end() && + "Every JD should have at least __lljit_run_atexits"); + + auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits); + if (RunAtExitsItr != DeInitsItr->second.end()) + DeInitializers.push_back(RunAtExitsItr->second.getAddress()); + + for (auto &KV : DeInitsItr->second) + if (KV.first != LLJITRunAtExits) + DeInitializers.push_back(KV.second.getAddress()); + } + + return DeInitializers; + } + + // Returns a DFS traversal order of the JITDylibs reachable (via + // links-against edges) from JD, starting with JD itself. + static std::vector getDFSLinkOrder(JITDylib &JD) { + std::vector DFSLinkOrder; + std::vector WorkStack({&JD}); + DenseSet Visited; + + while (!WorkStack.empty()) { + auto &NextJD = *WorkStack.back(); + WorkStack.pop_back(); + if (Visited.count(&NextJD)) + continue; + Visited.insert(&NextJD); + DFSLinkOrder.push_back(&NextJD); + NextJD.withSearchOrderDo([&](const JITDylibSearchOrder &SearchOrder) { + for (auto &KV : SearchOrder) + WorkStack.push_back(KV.first); + }); + } + + return DFSLinkOrder; + } + + /// Issue lookups for all init symbols required to initialize JD (and any + /// JITDylibs that it depends on). + Error issueInitLookups(JITDylib &JD) { + DenseMap RequiredInitSymbols; + + { + std::lock_guard Lock(PlatformSupportMutex); + + auto DFSLinkOrder = getDFSLinkOrder(JD); + + for (auto *NextJD : DFSLinkOrder) { + auto ISItr = InitSymbols.find(NextJD); + if (ISItr != InitSymbols.end()) { + RequiredInitSymbols[NextJD] = std::move(ISItr->second); + InitSymbols.erase(ISItr); + } + } + } + + return Platform::lookupInitSymbols(getExecutionSession(), + RequiredInitSymbols) + .takeError(); + } + + static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx, + void *DSOHandle) { + static_cast(Self)->AtExitMgr.registerAtExit( + F, Ctx, DSOHandle); + } + + static void runAtExitsHelper(void *Self, void *DSOHandle) { + static_cast(Self)->AtExitMgr.runAtExits( + DSOHandle); + } + + // Constructs an LLVM IR module containing platform runtime globals, + // functions, and interposes. + ThreadSafeModule createPlatformRuntimeModule() { + auto Ctx = std::make_unique(); + auto M = std::make_unique("__standard_lib", *Ctx); + M->setDataLayout(J.getDataLayout()); + + auto *GenericIRPlatformSupportTy = + StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport"); + + auto *PlatformInstanceDecl = new GlobalVariable( + *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage, + nullptr, "__lljit.platform_support_instance"); + + auto *DSOHandleDecl = addDSOHandleDecl(*M); + + auto *Int8Ty = Type::getInt8Ty(*Ctx); + auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT); + auto *VoidTy = Type::getVoidTy(*Ctx); + auto *BytePtrTy = PointerType::getUnqual(Int8Ty); + auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false); + auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy); + + addHelperAndWrapper( + *M, "__cxa_atexit", + FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy}, + false), + GlobalValue::HiddenVisibility, "__lljit.cxa_atexit_helper", + {PlatformInstanceDecl}); + + addHelperAndWrapper( + *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false), + GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper", + {PlatformInstanceDecl, DSOHandleDecl}); + + return ThreadSafeModule(std::move(M), std::move(Ctx)); + } + + std::mutex PlatformSupportMutex; + LLJIT &J; + DenseMap InitSymbols; + DenseMap InitFunctions; + DenseMap DeInitFunctions; + ItaniumCXAAtExitSupport AtExitMgr; +}; + +Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) { + return S.setupJITDylib(JD); +} + +Error GenericLLVMIRPlatform::notifyAdding(JITDylib &JD, + const MaterializationUnit &MU) { + return S.notifyAdding(JD, MU); +} + +Expected +GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM, + MaterializationResponsibility &R) { + auto Err = TSM.withModuleDo([&](Module &M) -> Error { + auto &Ctx = M.getContext(); + auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors"); + + // If there's no llvm.global_ctors or it's just a decl then skip. + if (!GlobalCtors || GlobalCtors->isDeclaration()) + return Error::success(); + + std::string InitFunctionName; + raw_string_ostream(InitFunctionName) + << "__orc_init." << M.getModuleIdentifier(); + + MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout()); + auto InternedName = Mangle(InitFunctionName); + if (auto Err = + R.defineMaterializing({{InternedName, JITSymbolFlags::Callable}})) + return Err; + + auto *InitFunc = + Function::Create(FunctionType::get(Type::getVoidTy(Ctx), {}, false), + GlobalValue::ExternalLinkage, InitFunctionName, &M); + InitFunc->setVisibility(GlobalValue::HiddenVisibility); + std::vector> Inits; + for (auto E : getConstructors(M)) + Inits.push_back(std::make_pair(E.Func, E.Priority)); + llvm::sort(Inits, [](const std::pair &LHS, + const std::pair &RHS) { + return LHS.first < RHS.first; + }); + auto *EntryBlock = BasicBlock::Create(Ctx, "entry", InitFunc); + IRBuilder<> IB(EntryBlock); + for (auto &KV : Inits) + IB.CreateCall(KV.first); + IB.CreateRetVoid(); + + PS.registerInitFunc(R.getTargetJITDylib(), InternedName); + GlobalCtors->eraseFromParent(); + return Error::success(); + }); + + if (Err) + return std::move(Err); + + return TSM; +} + +class MachOPlatformSupport : public LLJIT::PlatformSupport { +public: + using DLOpenType = void *(*)(const char *Name, int Mode); + using DLCloseType = int (*)(void *Handle); + using DLSymType = void *(*)(void *Handle, const char *Name); + using DLErrorType = const char *(*)(); + + struct DlFcnValues { + Optional RTLDDefault; + DLOpenType dlopen = nullptr; + DLCloseType dlclose = nullptr; + DLSymType dlsym = nullptr; + DLErrorType dlerror = nullptr; + }; + + static Expected> + Create(LLJIT &J, JITDylib &PlatformJITDylib) { + + // Make process symbols visible. + { + std::string ErrMsg; + auto Lib = sys::DynamicLibrary::getPermanentLibrary(nullptr, &ErrMsg); + if (!Lib.isValid()) + return make_error(std::move(ErrMsg), + inconvertibleErrorCode()); + } + + DlFcnValues DlFcn; + + // Add support for RTLDDefault on known platforms. +#ifdef __APPLE__ + DlFcn.RTLDDefault = reinterpret_cast(-2); +#endif // __APPLE__ + + if (auto Err = hookUpFunction(DlFcn.dlopen, "dlopen")) + return std::move(Err); + if (auto Err = hookUpFunction(DlFcn.dlclose, "dlclose")) + return std::move(Err); + if (auto Err = hookUpFunction(DlFcn.dlsym, "dlsym")) + return std::move(Err); + if (auto Err = hookUpFunction(DlFcn.dlerror, "dlerror")) + return std::move(Err); + + std::unique_ptr MP( + new MachOPlatformSupport(J, PlatformJITDylib, DlFcn)); + return std::move(MP); + } + + Error initialize(JITDylib &JD) override { + if (auto InitSeq = MP.getInitializerSequence(JD)) { + for (auto &KV : *InitSeq) { + KV.second.registerObjCSelectors(); + KV.second.registerObjCClasses(); + } + for (auto &KV : *InitSeq) + KV.second.runModInits(); + } else + return InitSeq.takeError(); + return Error::success(); + } + + Error deinitialize(JITDylib &JD) override { + auto &ES = J.getExecutionSession(); + if (auto DeinitSeq = MP.getDeinitializerSequence(JD)) { + for (auto &KV : *DeinitSeq) { + auto DSOHandleName = ES.intern("___dso_handle"); + + // FIXME: Run DeInits here. + auto Result = ES.lookup( + {{KV.first, JITDylibLookupFlags::MatchAllSymbols}}, + SymbolLookupSet(DSOHandleName, + SymbolLookupFlags::WeaklyReferencedSymbol)); + if (!Result) + return Result.takeError(); + if (Result->empty()) + continue; + assert(Result->count(DSOHandleName) && + "Result does not contain __dso_handle"); + auto *DSOHandle = jitTargetAddressToPointer( + Result->begin()->second.getAddress()); + AtExitMgr.runAtExits(DSOHandle); + } + } else + return DeinitSeq.takeError(); + return Error::success(); + } + +private: + template + static Error hookUpFunction(FunctionPtrTy &Fn, const char *Name) { + if (auto *FnAddr = sys::DynamicLibrary::SearchForAddressOfSymbol(Name)) { + Fn = reinterpret_cast(Fn); + return Error::success(); + } + + return make_error((Twine("Can not enable MachO JIT Platform: " + "missing function: ") + + Name) + .str(), + inconvertibleErrorCode()); + } + + MachOPlatformSupport(LLJIT &J, JITDylib &PlatformJITDylib, DlFcnValues DlFcn) + : J(J), MP(setupPlatform(J)), DlFcn(std::move(DlFcn)) { + + MangleAndInterner Mangle(J.getExecutionSession(), J.getDataLayout()); + SymbolMap HelperSymbols; + + // platform and atexit helpers. + HelperSymbols[Mangle("__lljit.platform_support_instance")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(this), JITSymbolFlags()); + HelperSymbols[Mangle("__lljit.cxa_atexit_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(registerAtExitHelper), JITSymbolFlags()); + HelperSymbols[Mangle("__lljit.run_atexits_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(runAtExitsHelper), JITSymbolFlags()); + + // dlfcn helpers. + HelperSymbols[Mangle("__lljit.dlopen_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(dlopenHelper), JITSymbolFlags()); + HelperSymbols[Mangle("__lljit.dlclose_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(dlcloseHelper), JITSymbolFlags()); + HelperSymbols[Mangle("__lljit.dlsym_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(dlsymHelper), JITSymbolFlags()); + HelperSymbols[Mangle("__lljit.dlerror_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(dlerrorHelper), JITSymbolFlags()); + + cantFail( + PlatformJITDylib.define(absoluteSymbols(std::move(HelperSymbols)))); + cantFail(MP.setupJITDylib(J.getMainJITDylib())); + cantFail(J.addIRModule(PlatformJITDylib, createPlatformRuntimeModule())); + } + + static MachOPlatform &setupPlatform(LLJIT &J) { + auto Tmp = std::make_unique( + J.getExecutionSession(), + static_cast(J.getObjLinkingLayer()), + createStandardSymbolsObject(J)); + auto &MP = *Tmp; + J.getExecutionSession().setPlatform(std::move(Tmp)); + return MP; + } + + static std::unique_ptr createStandardSymbolsObject(LLJIT &J) { + LLVMContext Ctx; + Module M("__standard_symbols", Ctx); + M.setDataLayout(J.getDataLayout()); + + auto *Int64Ty = Type::getInt64Ty(Ctx); + + auto *DSOHandle = + new GlobalVariable(M, Int64Ty, true, GlobalValue::ExternalLinkage, + ConstantInt::get(Int64Ty, 0), "__dso_handle"); + DSOHandle->setVisibility(GlobalValue::HiddenVisibility); + + return cantFail(J.getIRCompileLayer().getCompiler()(M)); + } + + ThreadSafeModule createPlatformRuntimeModule() { + auto Ctx = std::make_unique(); + auto M = std::make_unique("__standard_lib", *Ctx); + M->setDataLayout(J.getDataLayout()); + + auto *MachOPlatformSupportTy = + StructType::create(*Ctx, "lljit.MachOPlatformSupport"); + + auto *PlatformInstanceDecl = new GlobalVariable( + *M, MachOPlatformSupportTy, true, GlobalValue::ExternalLinkage, nullptr, + "__lljit.platform_support_instance"); + + auto *Int8Ty = Type::getInt8Ty(*Ctx); + auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT); + auto *VoidTy = Type::getVoidTy(*Ctx); + auto *BytePtrTy = PointerType::getUnqual(Int8Ty); + auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false); + auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy); + + addHelperAndWrapper( + *M, "__cxa_atexit", + FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy}, + false), + GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper", + {PlatformInstanceDecl}); + + addHelperAndWrapper(*M, "dlopen", + FunctionType::get(BytePtrTy, {BytePtrTy, IntTy}, false), + GlobalValue::DefaultVisibility, "__lljit.dlopen_helper", + {PlatformInstanceDecl}); + + addHelperAndWrapper(*M, "dlclose", + FunctionType::get(IntTy, {BytePtrTy}, false), + GlobalValue::DefaultVisibility, + "__lljit.dlclose_helper", {PlatformInstanceDecl}); + + addHelperAndWrapper( + *M, "dlsym", + FunctionType::get(BytePtrTy, {BytePtrTy, BytePtrTy}, false), + GlobalValue::DefaultVisibility, "__lljit.dlsym_helper", + {PlatformInstanceDecl}); + + addHelperAndWrapper(*M, "dlerror", FunctionType::get(BytePtrTy, {}, false), + GlobalValue::DefaultVisibility, + "__lljit.dlerror_helper", {PlatformInstanceDecl}); + + return ThreadSafeModule(std::move(M), std::move(Ctx)); + } + + static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx, + void *DSOHandle) { + static_cast(Self)->AtExitMgr.registerAtExit( + F, Ctx, DSOHandle); + } + + static void runAtExitsHelper(void *Self, void *DSOHandle) { + static_cast(Self)->AtExitMgr.runAtExits(DSOHandle); + } + + void *jit_dlopen(const char *Path, int Mode) { + JITDylib *JDToOpen = nullptr; + // FIXME: Do the right thing with Mode flags. + { + std::lock_guard Lock(PlatformSupportMutex); + + // Clear any existing error messages. + dlErrorMsgs.erase(std::this_thread::get_id()); + + if (auto *JD = J.getExecutionSession().getJITDylibByName(Path)) { + auto I = JDRefCounts.find(JD); + if (I != JDRefCounts.end()) { + ++I->second; + return JD; + } + + JDRefCounts[JD] = 1; + JDToOpen = JD; + } + } + + if (JDToOpen) { + if (auto Err = initialize(*JDToOpen)) { + recordError(std::move(Err)); + return 0; + } + } + + // Fall through to dlopen if no JITDylib found for Path. + return DlFcn.dlopen(Path, Mode); + } + + static void *dlopenHelper(void *Self, const char *Path, int Mode) { + return static_cast(Self)->jit_dlopen(Path, Mode); + } + + int jit_dlclose(void *Handle) { + JITDylib *JDToClose = nullptr; + + { + std::lock_guard Lock(PlatformSupportMutex); + + // Clear any existing error messages. + dlErrorMsgs.erase(std::this_thread::get_id()); + + auto I = JDRefCounts.find(Handle); + if (I != JDRefCounts.end()) { + --I->second; + if (I->second == 0) { + JDRefCounts.erase(I); + JDToClose = static_cast(Handle); + } else + return 0; + } + } + + if (JDToClose) { + if (auto Err = deinitialize(*JDToClose)) { + recordError(std::move(Err)); + return -1; + } + return 0; + } + + // Fall through to dlclose if no JITDylib found for Path. + return DlFcn.dlclose(Handle); + } + + static int dlcloseHelper(void *Self, void *Handle) { + return static_cast(Self)->jit_dlclose(Handle); + } + + void *jit_dlsym(void *Handle, const char *Name) { + JITDylibSearchOrder JITSymSearchOrder; + + // FIXME: RTLD_NEXT, RTLD_SELF not supported. + { + std::lock_guard Lock(PlatformSupportMutex); + + // Clear any existing error messages. + dlErrorMsgs.erase(std::this_thread::get_id()); + + if (JDRefCounts.count(Handle)) { + JITSymSearchOrder.push_back( + {static_cast(Handle), + JITDylibLookupFlags::MatchExportedSymbolsOnly}); + } else if (Handle == DlFcn.RTLDDefault) { + for (auto &KV : JDRefCounts) + JITSymSearchOrder.push_back( + {static_cast(KV.first), + JITDylibLookupFlags::MatchExportedSymbolsOnly}); + } + } + + if (!JITSymSearchOrder.empty()) { + MangleAndInterner Mangle(J.getExecutionSession(), J.getDataLayout()); + auto MangledName = Mangle(Name); + SymbolLookupSet Syms(MangledName, + SymbolLookupFlags::WeaklyReferencedSymbol); + if (auto Result = J.getExecutionSession().lookup(JITSymSearchOrder, Syms, + LookupKind::DLSym)) { + auto I = Result->find(MangledName); + if (I != Result->end()) + return jitTargetAddressToPointer(I->second.getAddress()); + } else { + recordError(Result.takeError()); + return 0; + } + } + + // Fall through to dlsym. + return DlFcn.dlsym(Handle, Name); + } + + static void *dlsymHelper(void *Self, void *Handle, const char *Name) { + return static_cast(Self)->jit_dlsym(Handle, Name); + } + + const char *jit_dlerror() { + { + std::lock_guard Lock(PlatformSupportMutex); + auto I = dlErrorMsgs.find(std::this_thread::get_id()); + if (I != dlErrorMsgs.end()) + return I->second->c_str(); + } + return DlFcn.dlerror(); + } + + static const char *dlerrorHelper(void *Self) { + return static_cast(Self)->jit_dlerror(); + } + + void recordError(Error Err) { + std::lock_guard Lock(PlatformSupportMutex); + dlErrorMsgs[std::this_thread::get_id()] = + std::make_unique(toString(std::move(Err))); + } + + std::mutex PlatformSupportMutex; + LLJIT &J; + MachOPlatform &MP; + DlFcnValues DlFcn; + ItaniumCXAAtExitSupport AtExitMgr; + DenseMap JDRefCounts; + std::map> dlErrorMsgs; +}; + +} // end anonymous namespace namespace llvm { namespace orc { +void LLJIT::PlatformSupport::setInitTransform( + LLJIT &J, IRTransformLayer::TransformFunction T) { + J.InitHelperTransformLayer->setTransform(std::move(T)); +} + +LLJIT::PlatformSupport::~PlatformSupport() {} + Error LLJITBuilderState::prepareForConstruction() { if (!JTMB) { @@ -57,7 +866,7 @@ Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) { auto InternedName = ES->intern(Name); SymbolMap Symbols({{InternedName, Sym}}); - return Main.define(absoluteSymbols(std::move(Symbols))); + return Main->define(absoluteSymbols(std::move(Symbols))); } Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { @@ -67,7 +876,8 @@ TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); })) return Err; - return TransformLayer->add(JD, std::move(TSM), ES->allocateVModule()); + return InitHelperTransformLayer->add(JD, std::move(TSM), + ES->allocateVModule()); } Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr Obj) { @@ -128,16 +938,23 @@ } LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) - : ES(S.ES ? std::move(S.ES) : std::make_unique()), - Main(this->ES->createJITDylib("
")), DL(""), - TT(S.JTMB->getTargetTriple()), + : ES(S.ES ? std::move(S.ES) : std::make_unique()), Main(), + DL(""), TT(S.JTMB->getTargetTriple()), ObjLinkingLayer(createObjectLinkingLayer(S, *ES)), - ObjTransformLayer(*this->ES, *ObjLinkingLayer), CtorRunner(Main), - DtorRunner(Main) { + ObjTransformLayer(*this->ES, *ObjLinkingLayer) { ErrorAsOutParameter _(&Err); - if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) + if (auto MainOrErr = this->ES->createJITDylib("main")) + Main = &*MainOrErr; + else { + Err = MainOrErr.takeError(); + return; + } + + if (S.DL) + DL = std::move(*S.DL); + else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) DL = std::move(*DLOrErr); else { Err = DLOrErr.takeError(); @@ -153,10 +970,12 @@ CompileLayer = std::make_unique( *ES, ObjTransformLayer, std::move(*CompileFunction)); TransformLayer = std::make_unique(*ES, *CompileLayer); + InitHelperTransformLayer = + std::make_unique(*ES, *TransformLayer); } if (S.NumCompileThreads > 0) { - TransformLayer->setCloneToNewContextOnEmit(true); + InitHelperTransformLayer->setCloneToNewContextOnEmit(true); CompileThreads = std::make_unique(hardware_concurrency(S.NumCompileThreads)); ES->setDispatchMaterialization( @@ -167,6 +986,11 @@ CompileThreads->async(std::move(Work)); }); } + + if (S.SetUpPlatform) + Err = S.SetUpPlatform(*this); + else + setUpGenericLLVMIRPlatform(*this); } std::string LLJIT::mangle(StringRef UnmangledName) { @@ -184,15 +1008,24 @@ if (M.getDataLayout() != DL) return make_error( - "Added modules have incompatible data layouts", + "Added modules have incompatible data layouts: " + + M.getDataLayout().getStringRepresentation() + " (module) vs " + + DL.getStringRepresentation() + " (jit)", inconvertibleErrorCode()); return Error::success(); } -void LLJIT::recordCtorDtors(Module &M) { - CtorRunner.add(getConstructors(M)); - DtorRunner.add(getDestructors(M)); +void setUpGenericLLVMIRPlatform(LLJIT &J) { + J.setPlatformSupport(std::make_unique(J)); +} + +Error setUpMachOPlatform(LLJIT &J) { + auto MP = MachOPlatformSupport::Create(J, J.getMainJITDylib()); + if (!MP) + return MP.takeError(); + J.setPlatformSupport(std::move(*MP)); + return Error::success(); } Error LLLazyJITBuilderState::prepareForConstruction() { @@ -205,13 +1038,8 @@ Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { assert(TSM && "Can not add null module"); - if (auto Err = TSM.withModuleDo([&](Module &M) -> Error { - if (auto Err = applyDataLayout(M)) - return Err; - - recordCtorDtors(M); - return Error::success(); - })) + if (auto Err = TSM.withModuleDo( + [&](Module &M) -> Error { return applyDataLayout(M); })) return Err; return CODLayer->add(JD, std::move(TSM), ES->allocateVModule()); @@ -256,7 +1084,7 @@ // Create the COD layer. CODLayer = std::make_unique( - *ES, *TransformLayer, *LCTMgr, std::move(ISMBuilder)); + *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder)); if (S.NumCompileThreads > 0) CODLayer->setCloneToNewContextOnEmit(true); diff --git a/llvm/lib/ExecutionEngine/Orc/Layer.cpp b/llvm/lib/ExecutionEngine/Orc/Layer.cpp --- a/llvm/lib/ExecutionEngine/Orc/Layer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Layer.cpp @@ -7,7 +7,10 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/Layer.h" + +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/IR/Constants.h" +#include "llvm/Object/MachO.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Debug.h" @@ -23,17 +26,20 @@ *this, *getManglingOptions(), std::move(TSM), std::move(K))); } -IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES, - const ManglingOptions &MO, - ThreadSafeModule TSM, VModuleKey K) - : MaterializationUnit(SymbolFlagsMap(), std::move(K)), TSM(std::move(TSM)) { +IRMaterializationUnit::IRMaterializationUnit( + ExecutionSession &ES, const IRSymbolMapper::ManglingOptions &MO, + ThreadSafeModule TSM, VModuleKey K) + : MaterializationUnit(SymbolFlagsMap(), nullptr, std::move(K)), + TSM(std::move(TSM)) { assert(this->TSM && "Module must not be null"); MangleAndInterner Mangle(ES, this->TSM.getModuleUnlocked()->getDataLayout()); this->TSM.withModuleDo([&](Module &M) { + for (auto &G : M.global_values()) { // Skip globals that don't generate symbols. + if (!G.hasName() || G.isDeclaration() || G.hasLocalLinkage() || G.hasAvailableExternallyLinkage() || G.hasAppendingLinkage()) continue; @@ -72,13 +78,23 @@ SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G); SymbolToDefinition[MangledName] = &G; } + + // If we need an init symbol for this module then create one. + if (!llvm::empty(getStaticInitGVs(M))) { + std::string InitSymbolName; + raw_string_ostream(InitSymbolName) + << "$." << M.getModuleIdentifier() << ".__inits"; + InitSymbol = ES.intern(InitSymbolName); + SymbolFlags[InitSymbol] = JITSymbolFlags(); + } }); } IRMaterializationUnit::IRMaterializationUnit( ThreadSafeModule TSM, VModuleKey K, SymbolFlagsMap SymbolFlags, - SymbolNameToDefinitionMap SymbolToDefinition) - : MaterializationUnit(std::move(SymbolFlags), std::move(K)), + SymbolStringPtr InitSymbol, SymbolNameToDefinitionMap SymbolToDefinition) + : MaterializationUnit(std::move(SymbolFlags), std::move(InitSymbol), + std::move(K)), TSM(std::move(TSM)), SymbolToDefinition(std::move(SymbolToDefinition)) {} StringRef IRMaterializationUnit::getName() const { @@ -105,7 +121,8 @@ } BasicIRLayerMaterializationUnit::BasicIRLayerMaterializationUnit( - IRLayer &L, const ManglingOptions &MO, ThreadSafeModule TSM, VModuleKey K) + IRLayer &L, const IRSymbolMapper::ManglingOptions &MO, ThreadSafeModule TSM, + VModuleKey K) : IRMaterializationUnit(L.getExecutionSession(), MO, std::move(TSM), std::move(K)), L(L), K(std::move(K)) {} @@ -150,22 +167,26 @@ Expected> BasicObjectLayerMaterializationUnit::Create(ObjectLayer &L, VModuleKey K, std::unique_ptr O) { - auto SymbolFlags = - getObjectSymbolFlags(L.getExecutionSession(), O->getMemBufferRef()); + auto ObjSymInfo = + getObjectSymbolInfo(L.getExecutionSession(), O->getMemBufferRef()); + + if (!ObjSymInfo) + return ObjSymInfo.takeError(); - if (!SymbolFlags) - return SymbolFlags.takeError(); + auto &SymbolFlags = ObjSymInfo->first; + auto &InitSymbol = ObjSymInfo->second; return std::unique_ptr( - new BasicObjectLayerMaterializationUnit(L, K, std::move(O), - std::move(*SymbolFlags))); + new BasicObjectLayerMaterializationUnit( + L, K, std::move(O), std::move(SymbolFlags), std::move(InitSymbol))); } BasicObjectLayerMaterializationUnit::BasicObjectLayerMaterializationUnit( ObjectLayer &L, VModuleKey K, std::unique_ptr O, - SymbolFlagsMap SymbolFlags) - : MaterializationUnit(std::move(SymbolFlags), std::move(K)), L(L), - O(std::move(O)) {} + SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) + : MaterializationUnit(std::move(SymbolFlags), std::move(InitSymbol), + std::move(K)), + L(L), O(std::move(O)) {} StringRef BasicObjectLayerMaterializationUnit::getName() const { if (O) @@ -184,35 +205,5 @@ // filter to pass to the object layer along with the object itself. } -Expected getObjectSymbolFlags(ExecutionSession &ES, - MemoryBufferRef ObjBuffer) { - auto Obj = object::ObjectFile::createObjectFile(ObjBuffer); - - if (!Obj) - return Obj.takeError(); - - SymbolFlagsMap SymbolFlags; - for (auto &Sym : (*Obj)->symbols()) { - // Skip symbols not defined in this object file. - if (Sym.getFlags() & object::BasicSymbolRef::SF_Undefined) - continue; - - // Skip symbols that are not global. - if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Global)) - continue; - - auto Name = Sym.getName(); - if (!Name) - return Name.takeError(); - auto InternedName = ES.intern(*Name); - auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym); - if (!SymFlags) - return SymFlags.takeError(); - SymbolFlags[InternedName] = std::move(*SymFlags); - } - - return SymbolFlags; -} - } // End namespace orc. } // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp --- a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp +++ b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -51,7 +51,7 @@ auto LookupResult = ES.lookup( makeJITDylibSearchOrder(SourceJD, JITDylibLookupFlags::MatchAllSymbols), - SymbolName); + SymbolName, SymbolState::Ready); if (!LookupResult) { ES.reportError(LookupResult.takeError()); @@ -123,10 +123,9 @@ LazyCallThroughManager &LCTManager, IndirectStubsManager &ISManager, JITDylib &SourceJD, SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc, VModuleKey K) - : MaterializationUnit(extractFlags(CallableAliases), std::move(K)), + : MaterializationUnit(extractFlags(CallableAliases), nullptr, std::move(K)), LCTManager(LCTManager), ISManager(ISManager), SourceJD(SourceJD), - CallableAliases(std::move(CallableAliases)), - AliaseeTable(SrcJDLoc) {} + CallableAliases(std::move(CallableAliases)), AliaseeTable(SrcJDLoc) {} StringRef LazyReexportsMaterializationUnit::getName() const { return ""; diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -0,0 +1,466 @@ +//===------ MachOPlatform.cpp - Utilities for executing MachO in Orc ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/MachOPlatform.h" + +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/Support/BinaryByteStream.h" + +namespace { + +struct objc_class; +struct objc_image_info; +struct objc_object; +struct objc_selector; + +using Class = objc_class*; +using id = objc_object*; +using SEL = objc_selector*; + +using ObjCMsgSendTy = id(*)(id, SEL, ...); +using ObjCReadClassPairTy = Class (*)(Class, const objc_image_info *); +using SelRegisterNameTy = SEL (*)(const char *); + +enum class ObjCRegistrationAPI { + Uninitialized, + Unavailable, + Initialized +}; + +ObjCRegistrationAPI ObjCRegistrationAPIState = ObjCRegistrationAPI::Uninitialized; +ObjCMsgSendTy objc_msgSend = nullptr; +ObjCReadClassPairTy objc_readClassPair = nullptr; +SelRegisterNameTy sel_registerName = nullptr; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +template +static Error setUpObjCRegAPIFunc(FnTy &Target, sys::DynamicLibrary &LibObjC, + const char *Name) { + if (void *Addr = LibObjC.getAddressOfSymbol(Name)) + Target = reinterpret_cast(Addr); + else + return make_error((Twine("Could not find address for ") + Name).str(), + inconvertibleErrorCode()); + return Error::success(); +} + +Error enableObjCRegistration(const char *PathToLibObjC) { + // If we've already tried to initialize then just bail out. + if (ObjCRegistrationAPIState != ObjCRegistrationAPI::Uninitialized) + return Error::success(); + + ObjCRegistrationAPIState = ObjCRegistrationAPI::Unavailable; + + std::string ErrMsg; + auto LibObjC = + sys::DynamicLibrary::getPermanentLibrary(PathToLibObjC, &ErrMsg); + + if (!LibObjC.isValid()) + return make_error(std::move(ErrMsg), + inconvertibleErrorCode()); + + if (auto Err = + setUpObjCRegAPIFunc(objc_msgSend, LibObjC, "objc_msgSend")) + return Err; + if (auto Err = + setUpObjCRegAPIFunc(objc_readClassPair, LibObjC, "objc_readClassPair")) + return Err; + if (auto Err = + setUpObjCRegAPIFunc(sel_registerName, LibObjC, "sel_registerName")) + return Err; + + ObjCRegistrationAPIState = ObjCRegistrationAPI::Initialized; + return Error::success(); +} + +bool objcRegistrationEnabled() { + return ObjCRegistrationAPIState == ObjCRegistrationAPI::Initialized; +} + +void MachOJITDylibInitializers::runModInits() const { + for (const auto &ModInit : ModInitSections) { + for (uint64_t I = 0; I != ModInit.NumPtrs; ++I) { + auto *InitializerAddr = jitTargetAddressToPointer( + ModInit.Address + (I * sizeof(uintptr_t))); + auto *Initializer = + jitTargetAddressToFunction(*InitializerAddr); + Initializer(); + } + } +} + +void MachOJITDylibInitializers::registerObjCSelectors() const { + assert(objcRegistrationEnabled() && "ObjC registration not enabled."); + + for (const auto &ObjCSelRefs : ObjCSelRefsSections) { + for (uint64_t I = 0; I != ObjCSelRefs.NumPtrs; ++I) { + auto SelEntryAddr = ObjCSelRefs.Address + (I * sizeof(uintptr_t)); + const auto *SelName = + *jitTargetAddressToPointer(SelEntryAddr); + dbgs() << "Registering selector " << SelName << "...\n"; + auto Sel = sel_registerName(SelName); + *jitTargetAddressToPointer(SelEntryAddr) = Sel; + } + } +} + +void MachOJITDylibInitializers::registerObjCClasses() const { + assert(objcRegistrationEnabled() && "ObjC registration not enabled."); + + struct ObjCClassCompiled { + void *Metaclass; + void *Parent; + void *Cache1; + void *Cache2; + void *Data; + }; + + auto *ImageInfo = + jitTargetAddressToPointer(ObjCImageInfoAddr); + auto ClassSelector = sel_registerName("class"); + + for (const auto &ObjCClassList : ObjCClassListSections) { + for (uint64_t I = 0; I != ObjCClassList.NumPtrs; ++I) { + auto ClassPtrAddr = ObjCClassList.Address + (I * sizeof(uintptr_t)); + auto Cls = *jitTargetAddressToPointer(ClassPtrAddr); + auto *ClassCompiled = + *jitTargetAddressToPointer(ClassPtrAddr); + objc_msgSend(reinterpret_cast(ClassCompiled->Parent), ClassSelector); + auto Registered = objc_readClassPair(Cls, ImageInfo); + if (Registered != Cls) { + dbgs() << "Failed to register class!\n"; + exit(1); + } + } + } +} + +void MachOJITDylibInitializers::dump() const { + for (auto &Extent : ModInitSections) + dbgs() << formatv("{0:x16}", Extent.Address) << " -- " + << formatv("{0:x16}", Extent.Address + 8 * Extent.NumPtrs) << "\n"; +} + +MachOPlatform::MachOPlatform( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + std::unique_ptr StandardSymbolsObject) + : ES(ES), ObjLinkingLayer(ObjLinkingLayer), + StandardSymbolsObject(std::move(StandardSymbolsObject)) { + ObjLinkingLayer.addPlugin(std::make_unique(*this)); +} + +Error MachOPlatform::setupJITDylib(JITDylib &JD) { + auto ObjBuffer = MemoryBuffer::getMemBuffer( + StandardSymbolsObject->getMemBufferRef(), false); + return ObjLinkingLayer.add(JD, std::move(ObjBuffer)); +} + +Error MachOPlatform::notifyAdding(JITDylib &JD, const MaterializationUnit &MU) { + const auto &InitSym = MU.getInitializerSymbol(); + if (!InitSym) + return Error::success(); + + std::lock_guard Lock(PlatformMutex); + RegisteredInitSymbols[&JD].add(InitSym); + return Error::success(); +} + +Error MachOPlatform::notifyRemoving(JITDylib &JD, VModuleKey K) { + llvm_unreachable("Not supported yet"); +} + +Expected +MachOPlatform::getInitializerSequence(JITDylib &JD) { + + std::vector DFSLinkOrder; + + while (true) { + // Lock the platform while we search for any initializer symbols to + // look up. + DenseMap NewInitSymbols; + { + std::lock_guard Lock(PlatformMutex); + DFSLinkOrder = getDFSLinkOrder(JD); + + for (auto *InitJD : DFSLinkOrder) { + auto RISItr = RegisteredInitSymbols.find(InitJD); + if (RISItr != RegisteredInitSymbols.end()) { + NewInitSymbols[InitJD] = std::move(RISItr->second); + RegisteredInitSymbols.erase(RISItr); + } + } + } + + if (NewInitSymbols.empty()) + break; + + // Outside the lock, issue the lookup. + if (auto R = lookupInitSymbols(JD.getExecutionSession(), NewInitSymbols)) + ; // Nothing to do in the success case. + else + return R.takeError(); + } + + // Lock again to collect the initializers. + InitializerSequence FullInitSeq; + { + std::lock_guard Lock(PlatformMutex); + for (auto *InitJD : reverse(DFSLinkOrder)) { + auto ISItr = InitSeqs.find(InitJD); + if (ISItr != InitSeqs.end()) { + FullInitSeq.emplace_back(InitJD, std::move(ISItr->second)); + InitSeqs.erase(ISItr); + } + } + } + + return FullInitSeq; +} + +Expected +MachOPlatform::getDeinitializerSequence(JITDylib &JD) { + std::vector DFSLinkOrder = getDFSLinkOrder(JD); + + DeinitializerSequence FullDeinitSeq; + { + std::lock_guard Lock(PlatformMutex); + for (auto *DeinitJD : DFSLinkOrder) { + FullDeinitSeq.emplace_back(DeinitJD, MachOJITDylibDeinitializers()); + } + } + + return FullDeinitSeq; +} + +std::vector MachOPlatform::getDFSLinkOrder(JITDylib &JD) { + std::vector Result, WorkStack({&JD}); + DenseSet Visited; + + while (!WorkStack.empty()) { + auto *NextJD = WorkStack.back(); + WorkStack.pop_back(); + if (Visited.count(NextJD)) + continue; + Visited.insert(NextJD); + Result.push_back(NextJD); + NextJD->withSearchOrderDo([&](const JITDylibSearchOrder &SO) { + for (auto &KV : SO) + WorkStack.push_back(KV.first); + }); + } + + return Result; +} + +void MachOPlatform::registerInitInfo( + JITDylib &JD, + JITTargetAddress ObjCImageInfoAddr, + MachOJITDylibInitializers::SectionExtent ModInits, + MachOJITDylibInitializers::SectionExtent ObjCSelRefs, + MachOJITDylibInitializers::SectionExtent ObjCClassList) { + std::lock_guard Lock(PlatformMutex); + + auto &InitSeq = InitSeqs[&JD]; + + InitSeq.setObjCImageInfoAddr(ObjCImageInfoAddr); + + if (ModInits.Address) + InitSeq.addModInitsSection(std::move(ModInits)); + + if (ObjCSelRefs.Address) + InitSeq.addObjCSelRefsSection(std::move(ObjCSelRefs)); + + if (ObjCClassList.Address) + InitSeq.addObjCClassListSection(std::move(ObjCClassList)); +} + + +static Expected +getSectionExtent(jitlink::LinkGraph &G, StringRef SectionName) { + auto *Sec = G.findSectionByName(SectionName); + if (!Sec) + return MachOJITDylibInitializers::SectionExtent(); + jitlink::SectionRange R(*Sec); + if (R.getSize() % G.getPointerSize() != 0) + return make_error(SectionName + " section size is not a " + "multiple of the pointer size", + inconvertibleErrorCode()); + return MachOJITDylibInitializers::SectionExtent( + R.getStart(), R.getSize() / G.getPointerSize()); +} + +void MachOPlatform::InitScraperPlugin::modifyPassConfig( + MaterializationResponsibility &MR, const Triple &TT, + jitlink::PassConfiguration &Config) { + + Config.PrePrunePasses.push_back( + [this,&MR](jitlink::LinkGraph &G) -> Error { + + JITLinkSymbolVector InitSectionSymbols; + preserveInitSectionIfPresent(InitSectionSymbols, G, "__mod_init_func"); + preserveInitSectionIfPresent(InitSectionSymbols, G, "__objc_selrefs"); + preserveInitSectionIfPresent(InitSectionSymbols, G, "__objc_classlist"); + + if (!InitSymbolDeps.empty()) { + std::lock_guard Lock(InitScraperMutex); + InitSymbolDeps[&MR] = std::move(InitSectionSymbols); + } + + if (auto Err = processObjCImageInfo(G, MR)) + return Err; + + return Error::success(); + }); + + Config.PostFixupPasses.push_back( + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) -> Error { + MachOJITDylibInitializers::SectionExtent ModInits, ObjCSelRefs, + ObjCClassList; + + JITTargetAddress ObjCImageInfoAddr = 0; + if (auto *ObjCImageInfoSec = G.findSectionByName("__objc_image_info")) { + if (auto Addr = jitlink::SectionRange(*ObjCImageInfoSec).getStart()) { + ObjCImageInfoAddr = Addr; + dbgs() << "Recorded __objc_imageinfo @ " << formatv("{0:x16}", Addr); + } + } + + // Record __mod_init_func. + if (auto ModInitsOrErr = getSectionExtent(G, "__mod_init_func")) + ModInits = std::move(*ModInitsOrErr); + else + return ModInitsOrErr.takeError(); + + // Record __objc_selrefs. + if (auto ObjCSelRefsOrErr = getSectionExtent(G, "__objc_selrefs")) + ObjCSelRefs = std::move(*ObjCSelRefsOrErr); + else + return ObjCSelRefsOrErr.takeError(); + + // Record __objc_classlist. + if (auto ObjCClassListOrErr = getSectionExtent(G, "__objc_classlist")) + ObjCClassList = std::move(*ObjCClassListOrErr); + else + return ObjCClassListOrErr.takeError(); + + MP.registerInitInfo(JD, ObjCImageInfoAddr, std::move(ModInits), + std::move(ObjCSelRefs), std::move(ObjCClassList)); + + return Error::success(); + }); +} + +ObjectLinkingLayer::Plugin::LocalDependenciesMap +MachOPlatform::InitScraperPlugin::getSyntheticSymbolLocalDependencies( + MaterializationResponsibility &MR) { + std::lock_guard Lock(InitScraperMutex); + auto I = InitSymbolDeps.find(&MR); + if (I != InitSymbolDeps.end()) { + LocalDependenciesMap Result; + Result[MR.getInitializerSymbol()] = std::move(I->second); + InitSymbolDeps.erase(&MR); + return Result; + } + return LocalDependenciesMap(); +} + +void MachOPlatform::InitScraperPlugin::preserveInitSectionIfPresent( + JITLinkSymbolVector &Symbols, + jitlink::LinkGraph &G, + StringRef SectionName) { + if (auto *Sec = G.findSectionByName(SectionName)) { + auto SecBlocks = Sec->blocks(); + if (!llvm::empty(SecBlocks)) + Symbols.push_back( + &G.addAnonymousSymbol(**SecBlocks.begin(), 0, 0, false, true)); + } +} + +Error MachOPlatform::InitScraperPlugin::processObjCImageInfo(jitlink::LinkGraph &G, + MaterializationResponsibility &MR) { + + // If there's an ObjC imagine info then either + // (1) It's the first __objc_imageinfo we've seen in this JITDylib. In + // this case we name and record it. + // OR + // (2) We already have a recorded __objc_imageinfo for this JITDylib, + // in which case we just verify it. + auto *ObjCImageInfo = G.findSectionByName("__objc_imageinfo"); + if (!ObjCImageInfo) + return Error::success(); + + auto ObjCImageInfoBlocks = ObjCImageInfo->blocks(); + + // Check that the section is not empty if present. + if (llvm::empty(ObjCImageInfoBlocks)) + return make_error("Empty __objc_imageinfo section in " + + G.getName(), + inconvertibleErrorCode()); + + // Check that there's only one block in the section. + if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end()) + return make_error("Multiple blocks in __objc_imageinfo " + "section in " + G.getName(), + inconvertibleErrorCode()); + + // Check that the __objc_imageinfo section is unreferenced. + // FIXME: We could optimize this check if Symbols had a ref-count. + for (auto &Sec : G.sections()) { + if (&Sec != ObjCImageInfo) + for (auto *B : Sec.blocks()) + for (auto &E : B->edges()) + if (E.getTarget().isDefined() && + &E.getTarget().getBlock().getSection() == ObjCImageInfo) + return make_error("__objc_imageinfo is referenced " + "within file " + G.getName(), + inconvertibleErrorCode()); + } + + auto &ObjCImageInfoBlock = **ObjCImageInfoBlocks.begin(); + auto *ObjCImageInfoData = ObjCImageInfoBlock.getContent().data(); + auto Version = + support::endian::read32(ObjCImageInfoData, G.getEndianness()); + auto Flags = + support::endian::read32(ObjCImageInfoData + 4, G.getEndianness()); + + // Lock the mutex while we verify / update the ObjCImageInfos map. + std::lock_guard Lock(InitScraperMutex); + + auto ObjCImageInfoItr = ObjCImageInfos.find(&MR.getTargetJITDylib()); + if (ObjCImageInfoItr != ObjCImageInfos.end()) { + // We've already registered an __objc_imageinfo section. Verify the + // content of this new section matches, then delete it. + if (ObjCImageInfoItr->second.first != Version) + return make_error("ObjC version in " + G.getName() + + " does not match first registered version", + inconvertibleErrorCode()); + if (ObjCImageInfoItr->second.second != Flags) + return make_error("ObjC flags in " + G.getName() + + " do not match first registered flags", + inconvertibleErrorCode()); + + // __objc_imageinfo is valid. Delete the block. + for (auto *S : ObjCImageInfo->symbols()) + G.removeDefinedSymbol(*S); + G.removeBlock(ObjCImageInfoBlock); + } else { + // We haven't registered an __objc_imageinfo section yet. Register and + // move on. The section should already be marked no-dead-strip. + ObjCImageInfos[&MR.getTargetJITDylib()] = + std::make_pair(Version, Flags); + } + + return Error::success(); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/Mangling.cpp b/llvm/lib/ExecutionEngine/Orc/Mangling.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/Mangling.cpp @@ -0,0 +1,131 @@ +//===----------- Mangling.cpp -- Name Mangling Utilities for ORC ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Mangling.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Mangler.h" +#include "llvm/Object/MachO.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { + +MangleAndInterner::MangleAndInterner(ExecutionSession &ES, const DataLayout &DL) + : ES(ES), DL(DL) {} + +SymbolStringPtr MangleAndInterner::operator()(StringRef Name) { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, Name, DL); + } + return ES.intern(MangledName); +} + +void IRSymbolMapper::add(ExecutionSession &ES, const ManglingOptions &MO, + ArrayRef GVs, + SymbolFlagsMap &SymbolFlags, + SymbolNameToDefinitionMap *SymbolToDefinition) { + if (GVs.empty()) + return; + + MangleAndInterner Mangle(ES, GVs[0]->getParent()->getDataLayout()); + for (auto *G : GVs) { + assert(G && "GVs cannot contain null elements"); + if (!G->hasName() || G->isDeclaration() || G->hasLocalLinkage() || + G->hasAvailableExternallyLinkage() || G->hasAppendingLinkage()) + continue; + + if (G->isThreadLocal() && MO.EmulatedTLS) { + auto *GV = cast(G); + + auto Flags = JITSymbolFlags::fromGlobalValue(*GV); + + auto EmuTLSV = Mangle(("__emutls_v." + GV->getName()).str()); + SymbolFlags[EmuTLSV] = Flags; + if (SymbolToDefinition) + (*SymbolToDefinition)[EmuTLSV] = GV; + + // If this GV has a non-zero initializer we'll need to emit an + // __emutls.t symbol too. + if (GV->hasInitializer()) { + const auto *InitVal = GV->getInitializer(); + + // Skip zero-initializers. + if (isa(InitVal)) + continue; + const auto *InitIntValue = dyn_cast(InitVal); + if (InitIntValue && InitIntValue->isZero()) + continue; + + auto EmuTLST = Mangle(("__emutls_t." + GV->getName()).str()); + SymbolFlags[EmuTLST] = Flags; + if (SymbolToDefinition) + (*SymbolToDefinition)[EmuTLST] = GV; + } + continue; + } + + // Otherwise we just need a normal linker mangling. + auto MangledName = Mangle(G->getName()); + SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(*G); + if (SymbolToDefinition) + (*SymbolToDefinition)[MangledName] = G; + } +} + +Expected> +getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer) { + auto Obj = object::ObjectFile::createObjectFile(ObjBuffer); + + if (!Obj) + return Obj.takeError(); + + SymbolFlagsMap SymbolFlags; + for (auto &Sym : (*Obj)->symbols()) { + // Skip symbols not defined in this object file. + if (Sym.getFlags() & object::BasicSymbolRef::SF_Undefined) + continue; + + // Skip symbols that are not global. + if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Global)) + continue; + + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + auto InternedName = ES.intern(*Name); + auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym); + if (!SymFlags) + return SymFlags.takeError(); + SymbolFlags[InternedName] = std::move(*SymFlags); + } + + SymbolStringPtr InitSymbol; + + if (auto *MachOObj = dyn_cast(Obj->get())) { + for (auto &Sec : MachOObj->sections()) { + auto SecType = MachOObj->getSectionType(Sec); + if ((SecType & MachO::SECTION_TYPE) == MachO::S_MOD_INIT_FUNC_POINTERS) { + std::string InitSymString; + raw_string_ostream(InitSymString) + << "$." << ObjBuffer.getBufferIdentifier() << ".__inits"; + InitSymbol = ES.intern(InitSymString); + break; + } + } + } + + return std::make_pair(std::move(SymbolFlags), std::move(InitSymbol)); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp --- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -144,6 +144,10 @@ if (!ExtraSymbolsToClaim.empty()) if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim)) return notifyFailed(std::move(Err)); + + if (const auto &InitSym = MR.getInitializerSymbol()) + InternedResult[InitSym] = JITEvaluatedSymbol(); + if (auto Err = MR.notifyResolved(InternedResult)) { Layer.getExecutionSession().reportError(std::move(Err)); MR.failMaterialization(); @@ -184,8 +188,13 @@ } private: - using JITLinkSymbolSet = DenseSet; - using LocalToNamedDependenciesMap = DenseMap; + + struct LocalSymbolNamedDependencies { + SymbolNameSet Internal, External; + }; + + using LocalSymbolNamedDependenciesMap = + DenseMap; Error externalizeWeakAndCommonSymbols(LinkGraph &G) { auto &ES = Layer.getExecutionSession(); @@ -216,6 +225,7 @@ auto &ES = MR.getTargetJITDylib().getExecutionSession(); auto LocalDeps = computeLocalDeps(G); + // Compute dependencies for symbols defined in the JITLink graph. for (auto *Sym : G.defined_symbols()) { // Skip local symbols: we do not track dependencies for these. @@ -239,15 +249,12 @@ assert(TargetSym.isDefined() && "local symbols must be defined"); auto I = LocalDeps.find(&TargetSym); - if (I != LocalDeps.end()) - for (auto &S : I->second) { - assert(S->hasName() && - "LocalDeps should only contain named values"); - if (S->isExternal()) - ExternalSymDeps.insert(ES.intern(S->getName())); - else if (S != Sym) - InternalSymDeps.insert(ES.intern(S->getName())); - } + if (I != LocalDeps.end()) { + for (auto &S : I->second.External) + ExternalSymDeps.insert(S); + for (auto &S : I->second.Internal) + InternalSymDeps.insert(S); + } } } @@ -261,11 +268,33 @@ InternalNamedSymbolDeps[SymName] = std::move(InternalSymDeps); } + for (auto &P : Layer.Plugins) { + auto SyntheticLocalDeps = P->getSyntheticSymbolLocalDependencies(MR); + if (SyntheticLocalDeps.empty()) + continue; + + for (auto &KV : SyntheticLocalDeps) { + auto &Name = KV.first; + auto &LocalDepsForName = KV.second; + for (auto *Local : LocalDepsForName) { + assert(Local->getScope() == Scope::Local && + "Dependence on non-local symbol"); + auto LocalNamedDepsItr = LocalDeps.find(Local); + if (LocalNamedDepsItr == LocalDeps.end()) + continue; + for (auto &S : LocalNamedDepsItr->second.Internal) + InternalNamedSymbolDeps[Name].insert(S); + for (auto &S : LocalNamedDepsItr->second.External) + ExternalNamedSymbolDeps[Name].insert(S); + } + } + } + return Error::success(); } - LocalToNamedDependenciesMap computeLocalDeps(LinkGraph &G) { - LocalToNamedDependenciesMap DepMap; + LocalSymbolNamedDependenciesMap computeLocalDeps(LinkGraph &G) { + DenseMap> DepMap; // For all local symbols: // (1) Add their named dependencies. @@ -319,7 +348,26 @@ } } while (Changed); - return DepMap; + // Intern the results to produce a mapping of jitlink::Symbol* to internal + // and external symbol names. + auto &ES = Layer.getExecutionSession(); + LocalSymbolNamedDependenciesMap Result; + for (auto &KV : DepMap) { + auto *Local = KV.first; + assert(Local->getScope() == Scope::Local && + "DepMap keys should all be local symbols"); + auto &LocalNamedDeps = Result[Local]; + for (auto *Named : KV.second) { + assert(Named->getScope() != Scope::Local && + "DepMap values should all be non-local symbol sets"); + if (Named->isExternal()) + LocalNamedDeps.External.insert(ES.intern(Named->getName())); + else + LocalNamedDeps.Internal.insert(ES.intern(Named->getName())); + } + } + + return Result; } void registerDependencies(const SymbolDependenceMap &QueryDeps) { diff --git a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp --- a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -88,7 +88,8 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, std::unique_ptr O) { assert(O && "Object must not be null"); - + dbgs() << "Emitting via RTDyldObjectLinkingLayer:\n" + << R.getSymbols() << "\n"; // This method launches an asynchronous link step that will fulfill our // materialization responsibility. We need to switch R to be heap // allocated before that happens so it can live as long as the asynchronous @@ -229,6 +230,9 @@ Symbols.erase(KV.first); } + if (const auto &InitSym = R.getInitializerSymbol()) + Symbols[InitSym] = JITEvaluatedSymbol(); + if (auto Err = R.notifyResolved(Symbols)) { R.failMaterialization(); return Err; diff --git a/llvm/tools/lli/lli.cpp b/llvm/tools/lli/lli.cpp --- a/llvm/tools/lli/lli.cpp +++ b/llvm/tools/lli/lli.cpp @@ -24,9 +24,11 @@ #include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/ExecutionEngine/MCJIT.h" #include "llvm/ExecutionEngine/ObjectCache.h" +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h" #include "llvm/ExecutionEngine/OrcMCJITReplacement.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" @@ -206,6 +208,19 @@ cl::desc("Do not resolve lli process symbols in JIT'd code"), cl::init(false)); + enum class LLJITPlatform { DetectHost, GenericIR, MachO }; + + cl::opt + Platform("lljit-platform", cl::desc("Platform to use with LLJIT"), + cl::init(LLJITPlatform::DetectHost), + cl::values(clEnumValN(LLJITPlatform::DetectHost, "DetectHost", + "Select based on JIT target triple"), + clEnumValN(LLJITPlatform::GenericIR, "GenericIR", + "Use LLJITGenericIRPlatform"), + clEnumValN(LLJITPlatform::MachO, "MachO", + "Use LLJITMachOPlatform")), + cl::Hidden); + enum class DumpKind { NoDump, DumpFuncsToStdOut, @@ -772,16 +787,20 @@ if (!MainModule) reportError(Err, ProgName); - const auto &TT = MainModule->getTargetTriple(); + Triple TT(MainModule->getTargetTriple()); orc::LLLazyJITBuilder Builder; Builder.setJITTargetMachineBuilder( - TT.empty() ? ExitOnErr(orc::JITTargetMachineBuilder::detectHost()) - : orc::JITTargetMachineBuilder(Triple(TT))); + MainModule->getTargetTriple().empty() + ? ExitOnErr(orc::JITTargetMachineBuilder::detectHost()) + : orc::JITTargetMachineBuilder(TT)); if (!MArch.empty()) Builder.getJITTargetMachineBuilder()->getTargetTriple().setArchName(MArch); + if (!MainModule->getDataLayout().isDefault()) + Builder.setDataLayout(MainModule->getDataLayout()); + Builder.getJITTargetMachineBuilder() ->setCPU(getCPUStr()) .addFeatures(getFeatureList()) @@ -796,6 +815,29 @@ pointerToJITTargetAddress(exitOnLazyCallThroughFailure)); Builder.setNumCompileThreads(LazyJITCompileThreads); + // Set up LLJIT platform. + { + LLJITPlatform P = Platform; + if (P == LLJITPlatform::DetectHost) { + if (TT.isOSBinFormatMachO()) + P = LLJITPlatform::MachO; + else + P = LLJITPlatform::GenericIR; + } + + switch (P) { + case LLJITPlatform::GenericIR: + // Nothing to do: LLJITBuilder will use this by default. + break; + case LLJITPlatform::MachO: + Builder.setPlatformSetUp(orc::setUpMachOPlatform); + ExitOnErr(orc::enableObjCRegistration("libobjc.dylib")); + break; + default: + llvm_unreachable("Unrecognized platform value"); + } + } + auto J = ExitOnErr(Builder.create()); if (PerModuleLazy) @@ -828,9 +870,6 @@ return Name != MainName; }))); - orc::LocalCXXRuntimeOverrides CXXRuntimeOverrides; - ExitOnErr(CXXRuntimeOverrides.enable(J->getMainJITDylib(), Mangle)); - // Add the main module. ExitOnErr( J->addLazyIRModule(orc::ThreadSafeModule(std::move(MainModule), TSCtx))); @@ -845,8 +884,11 @@ for (auto JDItr = JITDylibs.begin(), JDEnd = JITDylibs.end(); JDItr != JDEnd; ++JDItr) { orc::JITDylib *JD = J->getJITDylibByName(*JDItr); - if (!JD) - JD = &J->createJITDylib(*JDItr); + if (!JD) { + JD = &ExitOnErr(J->createJITDylib(*JDItr)); + J->getMainJITDylib().addToSearchOrder(*JD); + JD->addToSearchOrder(J->getMainJITDylib()); + } IdxToDylib[JITDylibs.getPosition(JDItr - JITDylibs.begin())] = JD; } @@ -882,7 +924,7 @@ } // Run any static constructors. - ExitOnErr(J->runConstructors()); + ExitOnErr(J->initialize(J->getMainJITDylib())); // Run any -thread-entry points. std::vector AltEntryThreads; @@ -907,8 +949,7 @@ AltEntryThread.join(); // Run destructors. - ExitOnErr(J->runDestructors()); - CXXRuntimeOverrides.runDestructors(); + ExitOnErr(J->deinitialize(J->getMainJITDylib())); return Result; } diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.h b/llvm/tools/llvm-jitlink/llvm-jitlink.h --- a/llvm/tools/llvm-jitlink/llvm-jitlink.h +++ b/llvm/tools/llvm-jitlink/llvm-jitlink.h @@ -26,12 +26,13 @@ struct Session { orc::ExecutionSession ES; - orc::JITDylib &MainJD; + orc::JITDylib *MainJD; orc::ObjectLinkingLayer ObjLayer; std::vector JDSearchOrder; Triple TT; Session(Triple TT); + static Expected> Create(Triple TT); void dumpSessionInfo(raw_ostream &OS); void modifyPassConfig(const Triple &FTT, jitlink::PassConfiguration &PassConfig); @@ -63,6 +64,9 @@ FileInfoMap FileInfos; uint64_t SizeBeforePruning = 0; uint64_t SizeAfterFixups = 0; + +private: + Session(Triple TT, Error &Err); }; Error registerMachOStubsAndGOT(Session &S, jitlink::LinkGraph &G); diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp --- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp +++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp @@ -396,9 +396,18 @@ return std::make_unique(); } -Session::Session(Triple TT) - : MainJD(ES.createJITDylib("
")), ObjLayer(ES, createMemoryManager()), - TT(std::move(TT)) { +Expected> Session::Create(Triple TT) { + Error Err = Error::success(); + std::unique_ptr S(new Session(std::move(TT), Err)); + if (Err) + return std::move(Err); + return std::move(S); +} + +// FIXME: Move to createJITDylib if/when we start using Platform support in +// llvm-jitlink. +Session::Session(Triple TT, Error &Err) + : ObjLayer(ES, createMemoryManager()), TT(std::move(TT)) { /// Local ObjectLinkingLayer::Plugin class to forward modifyPassConfig to the /// Session. @@ -414,6 +423,15 @@ Session &S; }; + ErrorAsOutParameter _(&Err); + + if (auto MainJDOrErr = ES.createJITDylib("main")) + MainJD = &*MainJDOrErr; + else { + Err = MainJDOrErr.takeError(); + return; + } + if (!NoExec && !TT.isOSWindows()) ObjLayer.addPlugin(std::make_unique( InProcessEHFrameRegistrar::getInstance())); @@ -561,7 +579,7 @@ auto FilterMainEntryPoint = [InternedEntryPointName](SymbolStringPtr Name) { return Name != InternedEntryPointName; }; - S.MainJD.addGenerator( + S.MainJD->addGenerator( ExitOnErr(orc::DynamicLibrarySearchGenerator::GetForCurrentProcess( GlobalPrefix, FilterMainEntryPoint))); @@ -590,20 +608,22 @@ LLVM_DEBUG(dbgs() << "Creating JITDylibs...\n"); { // Create a "main" JITLinkDylib. - IdxToJLD[0] = &S.MainJD; - S.JDSearchOrder.push_back(&S.MainJD); - LLVM_DEBUG(dbgs() << " 0: " << S.MainJD.getName() << "\n"); + IdxToJLD[0] = S.MainJD; + S.JDSearchOrder.push_back(S.MainJD); + LLVM_DEBUG(dbgs() << " 0: " << S.MainJD->getName() << "\n"); // Add any extra JITLinkDylibs from the command line. std::string JDNamePrefix("lib"); for (auto JLDItr = JITLinkDylibs.begin(), JLDEnd = JITLinkDylibs.end(); JLDItr != JLDEnd; ++JLDItr) { - auto &JD = S.ES.createJITDylib(JDNamePrefix + *JLDItr); + auto JD = S.ES.createJITDylib(JDNamePrefix + *JLDItr); + if (!JD) + return JD.takeError(); unsigned JDIdx = JITLinkDylibs.getPosition(JLDItr - JITLinkDylibs.begin()); - IdxToJLD[JDIdx] = &JD; - S.JDSearchOrder.push_back(&JD); - LLVM_DEBUG(dbgs() << " " << JDIdx << ": " << JD.getName() << "\n"); + IdxToJLD[JDIdx] = &*JD; + S.JDSearchOrder.push_back(&*JD); + LLVM_DEBUG(dbgs() << " " << JDIdx << ": " << JD->getName() << "\n"); } // Set every dylib to link against every other, in command line order. @@ -790,32 +810,32 @@ std::unique_ptr Timers = ShowTimes ? std::make_unique() : nullptr; - Session S(getFirstFileTriple()); + auto S = ExitOnErr(Session::Create(getFirstFileTriple())); - ExitOnErr(sanitizeArguments(S)); + ExitOnErr(sanitizeArguments(*S)); if (!NoProcessSymbols) - ExitOnErr(loadProcessSymbols(S)); + ExitOnErr(loadProcessSymbols(*S)); ExitOnErr(loadDylibs()); { TimeRegion TR(Timers ? &Timers->LoadObjectsTimer : nullptr); - ExitOnErr(loadObjects(S)); + ExitOnErr(loadObjects(*S)); } JITEvaluatedSymbol EntryPoint = 0; { TimeRegion TR(Timers ? &Timers->LinkTimer : nullptr); - EntryPoint = ExitOnErr(getMainEntryPoint(S)); + EntryPoint = ExitOnErr(getMainEntryPoint(*S)); } if (ShowAddrs) - S.dumpSessionInfo(outs()); + S->dumpSessionInfo(outs()); - ExitOnErr(runChecks(S)); + ExitOnErr(runChecks(*S)); - dumpSessionStats(S); + dumpSessionStats(*S); if (NoExec) return 0; diff --git a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp --- a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp @@ -113,6 +113,7 @@ cantFail(R.notifyResolved({{Bar, BarSym}})); cantFail(R.notifyEmitted()); }, + nullptr, [&](const JITDylib &JD, const SymbolStringPtr &Name) { EXPECT_EQ(Name, Bar) << "Expected \"Bar\" to be discarded"; if (Name == Bar) @@ -126,6 +127,7 @@ cantFail(JD.define(std::make_unique( SymbolFlagsMap({{Baz, BazSym.getFlags()}}), [&](MaterializationResponsibility R) { BazR.emplace(std::move(R)); }, + nullptr, [](const JITDylib &JD, const SymbolStringPtr &Name) { ADD_FAILURE() << "\"Baz\" discarded unexpectedly"; }))); @@ -176,7 +178,7 @@ TEST_F(CoreAPIsStandardTest, ChainedJITDylibLookup) { cantFail(JD.define(absoluteSymbols({{Foo, FooSym}}))); - auto &JD2 = ES.createJITDylib("JD2"); + auto &JD2 = ES.createBareJITDylib("JD2"); bool OnCompletionRun = false; @@ -198,7 +200,7 @@ cantFail(JD.define(absoluteSymbols({{Foo, FooSym}, {Bar, BarHiddenSym}}))); - auto &JD2 = ES.createJITDylib("JD2"); + auto &JD2 = ES.createBareJITDylib("JD2"); cantFail(JD2.define(absoluteSymbols({{Bar, QuxSym}}))); /// Try a blocking lookup. @@ -307,7 +309,7 @@ // JITDylib works. cantFail(JD.define(absoluteSymbols({{Foo, FooSym}}))); - auto &JD2 = ES.createJITDylib("JD2"); + auto &JD2 = ES.createBareJITDylib("JD2"); cantFail(JD2.define(reexports(JD, {{Bar, {Foo, BarSym.getFlags()}}}))); @@ -332,7 +334,7 @@ cantFail(JD.define(BarMU)); - auto &JD2 = ES.createJITDylib("JD2"); + auto &JD2 = ES.createBareJITDylib("JD2"); cantFail(JD2.define(reexports( JD, {{Baz, {Foo, BazSym.getFlags()}}, {Qux, {Bar, QuxSym.getFlags()}}}))); @@ -347,7 +349,7 @@ TEST_F(CoreAPIsStandardTest, TestReexportsGenerator) { // Test that a re-exports generator can dynamically generate reexports. - auto &JD2 = ES.createJITDylib("JD2"); + auto &JD2 = ES.createBareJITDylib("JD2"); cantFail(JD2.define(absoluteSymbols({{Foo, FooSym}, {Bar, BarSym}}))); auto Filter = [this](SymbolStringPtr Name) { return Name != Bar; }; @@ -838,6 +840,7 @@ [](MaterializationResponsibility R) { llvm_unreachable("Unexpected call to materialize"); }, + nullptr, [&](const JITDylib &JD, SymbolStringPtr Name) { EXPECT_TRUE(Name == Foo || Name == Bar) << "Discard of unexpected symbol?"; @@ -872,6 +875,7 @@ cantFail(R.notifyEmitted()); FooMaterialized = true; }, + nullptr, [&](const JITDylib &JD, SymbolStringPtr Name) { EXPECT_EQ(Name, Bar) << "Expected Name to be Bar"; BarDiscarded = true; @@ -920,6 +924,7 @@ ADD_FAILURE() << "Attempt to materialize Bar from the wrong unit"; R.failMaterialization(); }, + nullptr, [&](const JITDylib &JD, SymbolStringPtr Name) { EXPECT_EQ(Name, Bar) << "Expected \"Bar\" to be discarded"; DuplicateBarDiscarded = true; diff --git a/llvm/unittests/ExecutionEngine/Orc/OrcTestCommon.h b/llvm/unittests/ExecutionEngine/Orc/OrcTestCommon.h --- a/llvm/unittests/ExecutionEngine/Orc/OrcTestCommon.h +++ b/llvm/unittests/ExecutionEngine/Orc/OrcTestCommon.h @@ -47,7 +47,7 @@ protected: std::shared_ptr SSP = std::make_shared(); ExecutionSession ES{SSP}; - JITDylib &JD = ES.createJITDylib("JD"); + JITDylib &JD = ES.createBareJITDylib("JD"); SymbolStringPtr Foo = ES.intern("foo"); SymbolStringPtr Bar = ES.intern("bar"); SymbolStringPtr Baz = ES.intern("baz"); @@ -93,9 +93,11 @@ SimpleMaterializationUnit( orc::SymbolFlagsMap SymbolFlags, MaterializeFunction Materialize, + orc::SymbolStringPtr InitSym = nullptr, DiscardFunction Discard = DiscardFunction(), DestructorFunction Destructor = DestructorFunction()) - : MaterializationUnit(std::move(SymbolFlags), orc::VModuleKey()), + : MaterializationUnit(std::move(SymbolFlags), std::move(InitSym), + orc::VModuleKey()), Materialize(std::move(Materialize)), Discard(std::move(Discard)), Destructor(std::move(Destructor)) {} diff --git a/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp --- a/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp @@ -50,7 +50,7 @@ bool DebugSectionSeen = false; ExecutionSession ES; - auto &JD = ES.createJITDylib("main"); + auto &JD = ES.createBareJITDylib("main"); auto Foo = ES.intern("foo"); RTDyldObjectLinkingLayer ObjLayer(ES, [&DebugSectionSeen]() { @@ -151,7 +151,7 @@ // Create a simple stack and set the override flags option. ExecutionSession ES; - auto &JD = ES.createJITDylib("main"); + auto &JD = ES.createBareJITDylib("main"); auto Foo = ES.intern("foo"); RTDyldObjectLinkingLayer ObjLayer( ES, []() { return std::make_unique(); }); @@ -218,7 +218,7 @@ // Create a simple stack and set the override flags option. ExecutionSession ES; - auto &JD = ES.createJITDylib("main"); + auto &JD = ES.createBareJITDylib("main"); auto Foo = ES.intern("foo"); RTDyldObjectLinkingLayer ObjLayer( ES, []() { return std::make_unique(); });